From 1cf0992b015336142c9027238246fbe86228f8b1 Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Fri, 14 Jul 2023 20:52:25 +0900 Subject: [PATCH 01/10] WIP --- micropip/__init__.py | 2 ++ micropip/_commands/index_urls.py | 24 ++++++++++++++++++ micropip/_commands/install.py | 9 +++++++ micropip/package_index.py | 42 ++++++++++++++++++++++++++++++++ micropip/transaction.py | 19 ++++----------- 5 files changed, 82 insertions(+), 14 deletions(-) create mode 100644 micropip/_commands/index_urls.py create mode 100644 micropip/package_index.py diff --git a/micropip/__init__.py b/micropip/__init__.py index 9047f964..3a890d39 100644 --- a/micropip/__init__.py +++ b/micropip/__init__.py @@ -1,4 +1,5 @@ from ._commands.freeze import freeze +from ._commands.index_urls import set_index_urls from ._commands.install import install from ._commands.list import _list as list from ._commands.mock_package import ( @@ -21,5 +22,6 @@ "list_mock_packages", "remove_mock_package", "uninstall", + "set_index_urls", "__version__", ] diff --git a/micropip/_commands/index_urls.py b/micropip/_commands/index_urls.py new file mode 100644 index 00000000..e308a2ae --- /dev/null +++ b/micropip/_commands/index_urls.py @@ -0,0 +1,24 @@ +from ..package_index import INDEX_URLS, _check_index_url + +def set_index_urls(urls: list[str] | str) -> None: + """ + Set the index URLs to use when looking up packages. + + The URLs should contain the placeholder {package_name} which will be + replaced with the package name when looking up a package. + + Parameters + ---------- + urls + A list of URLs or a single URL to use as the package index. + """ + + global INDEX_URLS + + if isinstance(urls, str): + urls = [urls] + + for url in urls: + _check_index_url(url) + + INDEX_URLS = urls \ No newline at end of file diff --git a/micropip/_commands/install.py b/micropip/_commands/install.py index a636491f..cb2a0a70 100644 --- a/micropip/_commands/install.py +++ b/micropip/_commands/install.py @@ -16,6 +16,7 @@ async def install( deps: bool = True, credentials: str | None = None, pre: bool = False, + index_urls: list[str] | str | None = None, *, verbose: bool | int = False, ) -> None: @@ -86,6 +87,13 @@ async def install( If ``True``, include pre-release and development versions. By default, micropip only finds stable versions. + index_urls : + + A list of URLs or a single URL to use as the package index. + By default, micropip uses the PyPI. The URLs should contain + the placeholder {package_name} which will be replaced with + the package name when looking up a package. + verbose : Print more information about the process. By default, micropip is silent. Setting ``verbose=True`` will print @@ -117,6 +125,7 @@ async def install( pre=pre, fetch_kwargs=fetch_kwargs, verbose=verbose, + index_urls=index_urls, ) await transaction.gather_requirements(requirements) diff --git a/micropip/package_index.py b/micropip/package_index.py new file mode 100644 index 00000000..c7a798f0 --- /dev/null +++ b/micropip/package_index.py @@ -0,0 +1,42 @@ +import json +from typing import Any + +from ._compat import fetch_string + +DEFAULT_INDEX_URLS = ["https://pypi.org/pypi/{package_name}/json"] +INDEX_URLS = DEFAULT_INDEX_URLS + +def _check_index_url(url: str) -> None: + try: + url.format(package_name=".") + except KeyError: + raise ValueError( + f"Invalid index URL: {url!r}. " + "Please make sure it contains the placeholder {package_name}." + ) + + +async def search_packages(pkgname: str, fetch_kwargs: dict[str, str], index_urls: list[str] | str | None = None) -> Any: + global INDEX_URLS + + if index_urls is None: + index_urls = INDEX_URLS + elif isinstance(index_urls, str): + index_urls = [index_urls] + + for url in index_urls: + _check_index_url(url) + + url = url.format(package_name=pkgname) + + try: + metadata = await fetch_string(url, fetch_kwargs) + except OSError: + continue + + return json.loads(metadata) + else: + raise ValueError( + f"Can't fetch metadata for '{pkgname}' from PyPI. " + "Please make sure you have entered a correct package name." + ) \ No newline at end of file diff --git a/micropip/transaction.py b/micropip/transaction.py index 2fba6e66..67020ebe 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -17,10 +17,10 @@ from packaging.utils import canonicalize_name, parse_wheel_filename from packaging.version import InvalidVersion, Version +from . import package_index from ._compat import ( REPODATA_PACKAGES, fetch_bytes, - fetch_string, get_dynlibs, loadDynlib, loadedPackages, @@ -234,6 +234,7 @@ class Transaction: deps: bool pre: bool fetch_kwargs: dict[str, str] + index_urls: list[str] | str | None locked: dict[str, PackageMetadata] = field(default_factory=dict) wheels: list[WheelInfo] = field(default_factory=list) @@ -352,7 +353,9 @@ def eval_marker(e: dict[str, str]) -> bool: ) return - metadata = await _get_pypi_json(req.name, self.fetch_kwargs) + metadata = await package_index.search_packages( + req.name, self.fetch_kwargs, index_urls=self.index_urls + ) try: wheel = find_wheel(metadata, req) @@ -479,18 +482,6 @@ def find_wheel(metadata: dict[str, Any], req: Requirement) -> WheelInfo: ) -async def _get_pypi_json(pkgname: str, fetch_kwargs: dict[str, str]) -> Any: - url = f"https://pypi.org/pypi/{pkgname}/json" - try: - metadata = await fetch_string(url, fetch_kwargs) - except OSError as e: - raise ValueError( - f"Can't fetch metadata for '{pkgname}' from PyPI. " - "Please make sure you have entered a correct package name." - ) from e - return json.loads(metadata) - - def _generate_package_hash(data: IO[bytes]) -> str: sha256_hash = hashlib.sha256() data.seek(0) From 948d09a59cd3f831b157ae101876efcb447165ac Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Sun, 16 Jul 2023 23:03:20 +0900 Subject: [PATCH 02/10] Add pytest-httpserver to a test dependency --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 20f8f5fc..6deaefbd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ dynamic = ["version"] dependencies = ["packaging>=23.0"] [project.optional-dependencies] test = [ + "pytest-httpserver", "pytest-pyodide", "pytest-cov", "build", From 1e042ff7e6e72057260a40b8a60554891f50d61b Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Sun, 16 Jul 2023 23:03:57 +0900 Subject: [PATCH 03/10] Implement alternative index url support --- micropip/_commands/index_urls.py | 9 ++++---- micropip/package_index.py | 39 ++++++++++++++++++++++++++------ 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/micropip/_commands/index_urls.py b/micropip/_commands/index_urls.py index e308a2ae..d89a401e 100644 --- a/micropip/_commands/index_urls.py +++ b/micropip/_commands/index_urls.py @@ -1,4 +1,5 @@ -from ..package_index import INDEX_URLS, _check_index_url +from .. import package_index + def set_index_urls(urls: list[str] | str) -> None: """ @@ -12,13 +13,11 @@ def set_index_urls(urls: list[str] | str) -> None: urls A list of URLs or a single URL to use as the package index. """ - - global INDEX_URLS if isinstance(urls, str): urls = [urls] for url in urls: - _check_index_url(url) + package_index._check_index_url(url) - INDEX_URLS = urls \ No newline at end of file + package_index.INDEX_URLS = urls diff --git a/micropip/package_index.py b/micropip/package_index.py index a5919e7e..a2e55658 100644 --- a/micropip/package_index.py +++ b/micropip/package_index.py @@ -1,4 +1,5 @@ import json +import string import sys from collections import defaultdict from collections.abc import Generator @@ -14,6 +15,9 @@ DEFAULT_INDEX_URLS = ["https://pypi.org/pypi/{package_name}/json"] INDEX_URLS = DEFAULT_INDEX_URLS +_formatter = string.Formatter() + + # TODO: Merge this class with WheelInfo @dataclass class ProjectInfoFile: @@ -187,28 +191,49 @@ def _fast_check_incompatibility(filename: str) -> bool: return True + def _check_index_url(url: str) -> None: - try: - url.format(package_name=".") - except KeyError: + fields = [parsed[1] for parsed in _formatter.parse(url)] + + if "package_name" not in fields: raise ValueError( f"Invalid index URL: {url!r}. " "Please make sure it contains the placeholder {package_name}." ) -async def search_packages(pkgname: str, fetch_kwargs: dict[str, str], index_urls: list[str] | str | None = None) -> Any: +async def search_packages( + pkg: str, + fetch_kwargs: dict[str, str] | None = None, + index_urls: list[str] | str | None = None, +) -> ProjectInfo: + """ + Search for packages from given index URLs. + + Parameters + ---------- + pkg + Name of the package to search for. + fetch_kwargs + Keyword arguments to pass to the fetch function. + index_urls + A list of URLs or a single URL to use as the package index. + If None, the default index URLs are used. + """ global INDEX_URLS + if not fetch_kwargs: + fetch_kwargs = {} + if index_urls is None: index_urls = INDEX_URLS elif isinstance(index_urls, str): index_urls = [index_urls] - + for url in index_urls: _check_index_url(url) - url = url.format(package_name=pkgname) + url = url.format(package_name=pkg) try: metadata = await fetch_string(url, fetch_kwargs) @@ -218,6 +243,6 @@ async def search_packages(pkgname: str, fetch_kwargs: dict[str, str], index_urls return ProjectInfo.from_json_api(json.loads(metadata)) else: raise ValueError( - f"Can't fetch metadata for '{pkgname}' from PyPI. " + f"Can't fetch metadata for '{pkg}'." "Please make sure you have entered a correct package name." ) From 464336dba43c869570429aeee65b40a4010e8e81 Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Sun, 16 Jul 2023 23:04:07 +0900 Subject: [PATCH 04/10] Write some tests --- tests/conftest.py | 51 ++++++++++++++++++++-- tests/test_install.py | 4 ++ tests/test_package_index.py | 85 +++++++++++++++++++++++++++++-------- tests/test_transaction.py | 1 + 4 files changed, 121 insertions(+), 20 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 088193f0..79982c72 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,5 @@ +import functools +import gzip import io import sys import zipfile @@ -14,6 +16,12 @@ PLATFORM = f"emscripten_{EMSCRIPTEN_VER.replace('.', '_')}_wasm32" CPVER = f"cp{sys.version_info.major}{sys.version_info.minor}" +TEST_PYPI_RESPONSE_DIR = Path(__file__).parent / "test_data" / "pypi_response" + + +def _read_pypi_response(file: Path) -> bytes: + return gzip.decompress(file.read_bytes()) + def _build(build_dir, dist_dir): import build @@ -186,7 +194,7 @@ def add_pkg_version( self.metadata_map[filename] = metadata self.top_level_map[filename] = top_level - async def _get_pypi_json(self, pkgname, kwargs): + async def search_packages(self, pkgname, kwargs, index_urls=None): from micropip.package_index import ProjectInfo try: @@ -229,9 +237,46 @@ def write_file(filename, contents): @pytest.fixture def mock_fetch(monkeypatch, mock_importlib): pytest.importorskip("packaging") - from micropip import transaction + from micropip import package_index, transaction result = mock_fetch_cls() - monkeypatch.setattr(transaction, "_get_pypi_json", result._get_pypi_json) + monkeypatch.setattr(package_index, "search_packages", result.search_packages) monkeypatch.setattr(transaction, "fetch_bytes", result._fetch_bytes) return result + + +def _mock_package_index_gen( + httpserver, + pkgs=("black", "pytest", "numpy", "pytz", "snowballstemmer"), + content_type="application/json", + suffix="_json.json.gz", +): + # pytest-httpserver is not very good at handling multiple servers + # so we run a single server with different endpoints to simulate + # multiple package indexes + import secrets + + base = secrets.token_hex(16) + + for pkg in pkgs: + data = _read_pypi_response(TEST_PYPI_RESPONSE_DIR / f"{pkg}{suffix}") + httpserver.expect_request(f"/{base}/{pkg}/").respond_with_data( + data, + content_type=content_type, + headers={"Access-Control-Allow-Origin": "*"}, + ) + + base_url = httpserver.url_for(base) + index_url = base_url + "/{package_name}/" + + return index_url + + +@pytest.fixture +def mock_package_index_json_api(httpserver): + return functools.partial( + _mock_package_index_gen, + httpserver=httpserver, + suffix="_json.json.gz", + content_type="application/json", + ) diff --git a/tests/test_install.py b/tests/test_install.py index f86ec8cc..682010d4 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -368,3 +368,7 @@ async def run_test(selenium, url, name, version): assert f"Successfully installed {name}-{version}" in captured run_test(selenium_standalone_micropip, wheel_url, name, version) + + +def test_custom_index_urls(): + pass diff --git a/tests/test_package_index.py b/tests/test_package_index.py index 982feffa..26cf7374 100644 --- a/tests/test_package_index.py +++ b/tests/test_package_index.py @@ -1,25 +1,18 @@ -import gzip import json -from pathlib import Path -from typing import Any import pytest +from conftest import TEST_PYPI_RESPONSE_DIR, _read_pypi_response +import micropip._commands.index_urls as index_urls import micropip.package_index as package_index -TEST_TEMPLATES_DIR = Path(__file__).parent / "test_data" / "pypi_response" - - -def _read_test_data(file: Path) -> dict[str, Any]: - return json.loads(gzip.decompress(file.read_bytes())) - @pytest.mark.parametrize( "name", ["numpy", "black", "pytest", "snowballstemmer", "pytz"] ) def test_project_info_from_json(name): - test_file = TEST_TEMPLATES_DIR / f"{name}_json.json.gz" - test_data = _read_test_data(test_file) + test_file = TEST_PYPI_RESPONSE_DIR / f"{name}_json.json.gz" + test_data = json.loads(_read_pypi_response(test_file)) index = package_index.ProjectInfo.from_json_api(test_data) assert index.name == name @@ -39,8 +32,8 @@ def test_project_info_from_json(name): "name", ["numpy", "black", "pytest", "snowballstemmer", "pytz"] ) def test_project_info_from_simple_json(name): - test_file = TEST_TEMPLATES_DIR / f"{name}_simple.json.gz" - test_data = _read_test_data(test_file) + test_file = TEST_PYPI_RESPONSE_DIR / f"{name}_simple.json.gz" + test_data = json.loads(_read_pypi_response(test_file)) index = package_index.ProjectInfo.from_simple_api(test_data) assert index.name == name @@ -61,11 +54,11 @@ def test_project_info_from_simple_json(name): ) def test_project_info_equal(name): # The different ways of parsing the same data should result in the same - test_file_json = TEST_TEMPLATES_DIR / f"{name}_json.json.gz" - test_file_simple_json = TEST_TEMPLATES_DIR / f"{name}_simple.json.gz" + test_file_json = TEST_PYPI_RESPONSE_DIR / f"{name}_json.json.gz" + test_file_simple_json = TEST_PYPI_RESPONSE_DIR / f"{name}_simple.json.gz" - test_data_json = _read_test_data(test_file_json) - test_data_simple_json = _read_test_data(test_file_simple_json) + test_data_json = json.loads(_read_pypi_response(test_file_json)) + test_data_simple_json = json.loads(_read_pypi_response(test_file_simple_json)) index_json = package_index.ProjectInfo.from_json_api(test_data_json) index_simple_json = package_index.ProjectInfo.from_simple_api(test_data_simple_json) @@ -85,3 +78,61 @@ def test_project_info_equal(name): assert f_json.url == f_simple_json.url assert f_json.version == f_simple_json.version assert f_json.sha256 == f_simple_json.sha256 + + +def test_set_index_urls(): + default_index_urls = package_index.DEFAULT_INDEX_URLS + assert package_index.INDEX_URLS == default_index_urls + + valid_url1 = "https://pkg-index.com/{package_name}/json/" + valid_url2 = "https://another-pkg-index.com/{package_name}" + invalid_url = "https://invalid-pkg-index.com/json" + try: + index_urls.set_index_urls(valid_url1) + assert package_index.INDEX_URLS == [valid_url1] + + index_urls.set_index_urls([valid_url1, valid_url2]) + assert package_index.INDEX_URLS == [valid_url1, valid_url2] + + with pytest.raises(ValueError, match="Invalid index URL"): + index_urls.set_index_urls([invalid_url]) + finally: + index_urls.set_index_urls(default_index_urls) + assert package_index.INDEX_URLS == default_index_urls + + +@pytest.mark.asyncio +async def test_search_packages(mock_package_index_json_api): + mock_server_snowballstemmer = mock_package_index_json_api(pkgs=["snowballstemmer"]) + mock_server_pytest = mock_package_index_json_api(pkgs=["pytest"]) + + project_info = await package_index.search_packages( + "snowballstemmer", index_urls=[mock_server_snowballstemmer] + ) + + assert project_info.name == "snowballstemmer" + assert project_info.releases + + project_info = await package_index.search_packages( + "snowballstemmer", index_urls=mock_server_snowballstemmer + ) + + assert project_info.name == "snowballstemmer" + assert project_info.releases + + project_info = await package_index.search_packages( + "snowballstemmer", index_urls=[mock_server_pytest, mock_server_snowballstemmer] + ) + + assert project_info.name == "snowballstemmer" + assert project_info.releases + + with pytest.raises(ValueError, match="Can't fetch metadata"): + await package_index.search_packages( + "snowballstemmer", index_urls=[mock_server_pytest] + ) + + with pytest.raises(ValueError, match="Invalid index URL"): + await package_index.search_packages( + "snowballstemmer", index_urls=["http://without-placeholder.com"] + ) diff --git a/tests/test_transaction.py b/tests/test_transaction.py index 3513e564..b22dddf4 100644 --- a/tests/test_transaction.py +++ b/tests/test_transaction.py @@ -62,6 +62,7 @@ def create_transaction(Transaction): ctx={}, ctx_extras=[], fetch_kwargs={}, + index_urls=None, ) From 9cb49fbccc28ce1bf5e221b7ee347f2fa554ecdc Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Mon, 17 Jul 2023 22:23:16 +0900 Subject: [PATCH 05/10] Add another index url test --- tests/conftest.py | 17 ++++++++++++++--- tests/test_install.py | 30 ++++++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 79982c72..dcf026a9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,6 +11,7 @@ from pytest_pyodide import spawn_web_server SNOWBALL_WHEEL = "snowballstemmer-2.0.0-py2.py3-none-any.whl" +DUMMY_WHEEL = "dummy-1.0.0-py3-none-any.whl" EMSCRIPTEN_VER = "3.1.14" PLATFORM = f"emscripten_{EMSCRIPTEN_VER.replace('.', '_')}_wasm32" @@ -251,9 +252,7 @@ def _mock_package_index_gen( content_type="application/json", suffix="_json.json.gz", ): - # pytest-httpserver is not very good at handling multiple servers - # so we run a single server with different endpoints to simulate - # multiple package indexes + # Run a mock server that serves as a package index import secrets base = secrets.token_hex(16) @@ -280,3 +279,15 @@ def mock_package_index_json_api(httpserver): suffix="_json.json.gz", content_type="application/json", ) + + +@pytest.fixture(scope="module") +def mock_pythonhosted_org(): + from pytest_httpserver import HTTPServer + + try: + server = HTTPServer(host="https://files.pythonhosted.org", port=443) + yield server + finally: + server.clear() + server.stop() diff --git a/tests/test_install.py b/tests/test_install.py index 682010d4..1cbc44f1 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -370,5 +370,31 @@ async def run_test(selenium, url, name, version): run_test(selenium_standalone_micropip, wheel_url, name, version) -def test_custom_index_urls(): - pass +def test_custom_index_urls( + selenium_standalone_micropip, mock_package_index_json_api, mock_pythonhosted_org +): + from conftest import DUMMY_WHEEL + + mock_server_snowballstemmer = mock_package_index_json_api(pkgs=["snowballstemmer"]) + + # Install a dummy wheel from a custom index URL + wheel_data = Path(Path(__file__).parent, "dist", DUMMY_WHEEL).read_bytes() + mock_pythonhosted_org.expect_oneshot_request("*.whl").respond_with_data( + wheel_data, + content_type="application/octet-stream", + headers={"Access-Control-Allow-Origin": "*"}, + ) + + @run_in_pyodide(packages=["micropip"]) + async def run_test(selenium, index_url): + import micropip + + await micropip.install("snowballstemmer", index_urls=[index_url]) + + assert "dummy" in micropip.list() + + import dummy + + assert dummy.say_hello() == "hello" + + run_test(selenium_standalone_micropip, mock_server_snowballstemmer) From 0de73c3e8644415e93c9807b43af62aabc9ce768 Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Mon, 17 Jul 2023 22:52:12 +0900 Subject: [PATCH 06/10] Rewrite tests --- tests/conftest.py | 13 ------ .../fake-pkg-micropip-test_json.json.gz | Bin 0 -> 645 bytes .../fake-pkg-micropip-test_simple.json.gz | Bin 0 -> 393 bytes tests/test_install.py | 41 +++++++++--------- 4 files changed, 21 insertions(+), 33 deletions(-) create mode 100644 tests/test_data/pypi_response/fake-pkg-micropip-test_json.json.gz create mode 100644 tests/test_data/pypi_response/fake-pkg-micropip-test_simple.json.gz diff --git a/tests/conftest.py b/tests/conftest.py index dcf026a9..3ebffb89 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,7 +11,6 @@ from pytest_pyodide import spawn_web_server SNOWBALL_WHEEL = "snowballstemmer-2.0.0-py2.py3-none-any.whl" -DUMMY_WHEEL = "dummy-1.0.0-py3-none-any.whl" EMSCRIPTEN_VER = "3.1.14" PLATFORM = f"emscripten_{EMSCRIPTEN_VER.replace('.', '_')}_wasm32" @@ -279,15 +278,3 @@ def mock_package_index_json_api(httpserver): suffix="_json.json.gz", content_type="application/json", ) - - -@pytest.fixture(scope="module") -def mock_pythonhosted_org(): - from pytest_httpserver import HTTPServer - - try: - server = HTTPServer(host="https://files.pythonhosted.org", port=443) - yield server - finally: - server.clear() - server.stop() diff --git a/tests/test_data/pypi_response/fake-pkg-micropip-test_json.json.gz b/tests/test_data/pypi_response/fake-pkg-micropip-test_json.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..a71c2875cc8260b616d87cd4f7011a5933034b6d GIT binary patch literal 645 zcmV;00($))iwFonL$zc817=}sWi4=PXDw}MV{&hBX>cucWpi|2YIARHE^2dcZUC)S z%aWTg5WM>em$c~7X`LQ36A>JQ zye%nci%1YXdaxRKRd>+~z}bNVb{*BNWDP{o6U+&np}k1$_)OxQm zx85xukf#CPP@_R2uZkKtP~&Bz;ZMUf7*a4i!NL;*tXeeo!hWWyYpUCM_ytYen>E^m zn{bS^RGgtesHEJ$0w5Zhx~(0P)#xNQyBN@ncBhEq@F`5vqzC&WPUTt0t~!p>M6s!U zE-{yN?PdpOW_Fq(r(h@Sva4|Q!J`qGTiqp;jKkPL1RX@)9Yi_@1x~F%wed!^Ng$+v zfQoi=sko0&MGZ0RwA6CQsEbeJM!PGT7m#KVO~ZYe?ha{|?eiqvhf#8%$*xQ@Uya_# z?{`Yp*@Rpw057APOUS7I{aJj;OK(lhHzXSNw<~@~!Z7y0jdHh^WjAT#Fph!<-}Fbc zKPLD-`Kph05oRRWr(tww`(}5Dck$t;_cOPVs;pJV?N&&&K`U+lOMCB4$}GV%G4IcH fSUpcB>80U#$I*;&W*OZ6Z&ufT1ZUa}>jnS-HJCW? literal 0 HcmV?d00001 diff --git a/tests/test_data/pypi_response/fake-pkg-micropip-test_simple.json.gz b/tests/test_data/pypi_response/fake-pkg-micropip-test_simple.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..b7864d5962d7c2c5f14540216bef39aeb4327b5b GIT binary patch literal 393 zcmV;40e1c$iwFpPLbYT717=}sWi4=PXDw}MV{&hBX>cucWpi|2b7^gGY-KKLb8l_{ zrBcgo!ypj6=PM%4#R2??o$u(WRb*}OQlt1mjnm4F^6v%PNo!wvC_(}>%k0d+b+dJY zC|#i?hwbZl^yB*2jRmspkk;5k0X?`v1(n68WsQeup>0ofquRf!eai-FbTCnH#@w9W z@>8lT<_)$9n#xeOJj9nDq~E=ipeZNc>)e(L4I3l=@%?!*~^;r(pCOO+4ttK+cm8T zWLxk49q9ng8H@YlJ+RsJaaxFf-U0vsU3kQq literal 0 HcmV?d00001 diff --git a/tests/test_install.py b/tests/test_install.py index 1cbc44f1..de6e0680 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -370,31 +370,32 @@ async def run_test(selenium, url, name, version): run_test(selenium_standalone_micropip, wheel_url, name, version) -def test_custom_index_urls( - selenium_standalone_micropip, mock_package_index_json_api, mock_pythonhosted_org -): - from conftest import DUMMY_WHEEL - - mock_server_snowballstemmer = mock_package_index_json_api(pkgs=["snowballstemmer"]) +@pytest.mark.asyncio +async def test_custom_index_urls(mock_package_index_json_api, monkeypatch): + from io import BytesIO - # Install a dummy wheel from a custom index URL - wheel_data = Path(Path(__file__).parent, "dist", DUMMY_WHEEL).read_bytes() - mock_pythonhosted_org.expect_oneshot_request("*.whl").respond_with_data( - wheel_data, - content_type="application/octet-stream", - headers={"Access-Control-Allow-Origin": "*"}, + mock_server_fake_package = mock_package_index_json_api( + pkgs=["fake-pkg-micropip-test"] ) - @run_in_pyodide(packages=["micropip"]) - async def run_test(selenium, index_url): - import micropip + _wheel_url = "" - await micropip.install("snowballstemmer", index_urls=[index_url]) + async def _mock_fetch_bytes(url, *args): + nonlocal _wheel_url + _wheel_url = url + return BytesIO(b"fake wheel") - assert "dummy" in micropip.list() + from micropip import transaction - import dummy + monkeypatch.setattr(transaction, "fetch_bytes", _mock_fetch_bytes) - assert dummy.say_hello() == "hello" + try: + await micropip.install( + "fake-pkg-micropip-test", index_urls=[mock_server_fake_package] + ) + except Exception: + # We just check that the custom index url was used + # install will fail because the package is not real, but it doesn't matter. + pass - run_test(selenium_standalone_micropip, mock_server_snowballstemmer) + assert "fake_pkg_micropip_test-1.0.0-py2.py3-none-any.whl" in _wheel_url From 02edf5b347d5f4c8e958a79ff9c4da4ceb18b096 Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Tue, 18 Jul 2023 21:05:18 +0900 Subject: [PATCH 07/10] Address comments --- micropip/_commands/install.py | 16 ++++++++++++---- micropip/package_index.py | 20 ++++++++++++-------- micropip/transaction.py | 2 +- tests/conftest.py | 4 ++-- 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/micropip/_commands/install.py b/micropip/_commands/install.py index cb2a0a70..addbf2f7 100644 --- a/micropip/_commands/install.py +++ b/micropip/_commands/install.py @@ -89,10 +89,18 @@ async def install( index_urls : - A list of URLs or a single URL to use as the package index. - By default, micropip uses the PyPI. The URLs should contain - the placeholder {package_name} which will be replaced with - the package name when looking up a package. + A list of URLs or a single URL to use as the package index when looking + up packages. If None, `https://pypi.org/pypi/{package_name}/json` is used. + + The index URL may contain the placeholder {package_name} which will be + replaced with the package name when looking up a package. If it does not + contain the placeholder, the package name will be appended to the URL. + + The index URL should support the + [JSON API](https://warehouse.pypa.io/api-reference/json/). + + If a list of URLs is provided, micropip will try each URL in order until + it finds a package. If no package is found, an error will be raised. verbose : Print more information about the process. diff --git a/micropip/package_index.py b/micropip/package_index.py index a2e55658..e362d6c7 100644 --- a/micropip/package_index.py +++ b/micropip/package_index.py @@ -202,23 +202,26 @@ def _check_index_url(url: str) -> None: ) -async def search_packages( - pkg: str, +async def query_package( + name: str, fetch_kwargs: dict[str, str] | None = None, index_urls: list[str] | str | None = None, ) -> ProjectInfo: """ - Search for packages from given index URLs. + Query for a package from package indexes. Parameters ---------- - pkg + name Name of the package to search for. fetch_kwargs Keyword arguments to pass to the fetch function. index_urls A list of URLs or a single URL to use as the package index. - If None, the default index URLs are used. + If None, the default index URL is used. + + If a list of URLs is provided, it will be tried in order until + it finds a package. If no package is found, an error will be raised. """ global INDEX_URLS @@ -233,7 +236,7 @@ async def search_packages( for url in index_urls: _check_index_url(url) - url = url.format(package_name=pkg) + url = url.format(package_name=name) try: metadata = await fetch_string(url, fetch_kwargs) @@ -243,6 +246,7 @@ async def search_packages( return ProjectInfo.from_json_api(json.loads(metadata)) else: raise ValueError( - f"Can't fetch metadata for '{pkg}'." - "Please make sure you have entered a correct package name." + f"Can't fetch metadata for '{name}'." + "Please make sure you have entered a correct package name " + "and correctly specified index_urls (if you changed them)." ) diff --git a/micropip/transaction.py b/micropip/transaction.py index a94cedde..431612bc 100644 --- a/micropip/transaction.py +++ b/micropip/transaction.py @@ -296,7 +296,7 @@ def eval_marker(e: dict[str, str]) -> bool: ) return - metadata = await package_index.search_packages( + metadata = await package_index.query_package( req.name, self.fetch_kwargs, index_urls=self.index_urls ) diff --git a/tests/conftest.py b/tests/conftest.py index 3ebffb89..55c7bb90 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -194,7 +194,7 @@ def add_pkg_version( self.metadata_map[filename] = metadata self.top_level_map[filename] = top_level - async def search_packages(self, pkgname, kwargs, index_urls=None): + async def query_package(self, pkgname, kwargs, index_urls=None): from micropip.package_index import ProjectInfo try: @@ -240,7 +240,7 @@ def mock_fetch(monkeypatch, mock_importlib): from micropip import package_index, transaction result = mock_fetch_cls() - monkeypatch.setattr(package_index, "search_packages", result.search_packages) + monkeypatch.setattr(package_index, "query_package", result.query_package) monkeypatch.setattr(transaction, "fetch_bytes", result._fetch_bytes) return result From ccb8f705891b37d1a8edf7b42b98e3b3a655902a Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Tue, 18 Jul 2023 21:17:40 +0900 Subject: [PATCH 08/10] Support index URLs that does not have placeholder --- micropip/_commands/index_urls.py | 14 +++++++++----- micropip/package_index.py | 15 ++++++--------- tests/conftest.py | 3 +-- tests/test_package_index.py | 32 ++++++++++++++++---------------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/micropip/_commands/index_urls.py b/micropip/_commands/index_urls.py index d89a401e..48f25c57 100644 --- a/micropip/_commands/index_urls.py +++ b/micropip/_commands/index_urls.py @@ -5,8 +5,15 @@ def set_index_urls(urls: list[str] | str) -> None: """ Set the index URLs to use when looking up packages. - The URLs should contain the placeholder {package_name} which will be - replaced with the package name when looking up a package. + The index URL may contain the placeholder {package_name} which will be + replaced with the package name when looking up a package. If it does not + contain the placeholder, the package name will be appended to the URL. + + The index URL should support the + [JSON API](https://warehouse.pypa.io/api-reference/json/). + + If a list of URLs is provided, micropip will try each URL in order until + it finds a package. If no package is found, an error will be raised. Parameters ---------- @@ -17,7 +24,4 @@ def set_index_urls(urls: list[str] | str) -> None: if isinstance(urls, str): urls = [urls] - for url in urls: - package_index._check_index_url(url) - package_index.INDEX_URLS = urls diff --git a/micropip/package_index.py b/micropip/package_index.py index e362d6c7..c66cbad6 100644 --- a/micropip/package_index.py +++ b/micropip/package_index.py @@ -192,14 +192,10 @@ def _fast_check_incompatibility(filename: str) -> bool: return True -def _check_index_url(url: str) -> None: +def _contain_placeholder(url: str, placeholder: str = "package_name") -> bool: fields = [parsed[1] for parsed in _formatter.parse(url)] - if "package_name" not in fields: - raise ValueError( - f"Invalid index URL: {url!r}. " - "Please make sure it contains the placeholder {package_name}." - ) + return placeholder in fields async def query_package( @@ -234,9 +230,10 @@ async def query_package( index_urls = [index_urls] for url in index_urls: - _check_index_url(url) - - url = url.format(package_name=name) + if _contain_placeholder(url): + url = url.format(package_name=name) + else: + url = f"{url}/{name}/" try: metadata = await fetch_string(url, fetch_kwargs) diff --git a/tests/conftest.py b/tests/conftest.py index 55c7bb90..153cb39a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -264,8 +264,7 @@ def _mock_package_index_gen( headers={"Access-Control-Allow-Origin": "*"}, ) - base_url = httpserver.url_for(base) - index_url = base_url + "/{package_name}/" + index_url = httpserver.url_for(base) return index_url diff --git a/tests/test_package_index.py b/tests/test_package_index.py index 26cf7374..03df2341 100644 --- a/tests/test_package_index.py +++ b/tests/test_package_index.py @@ -86,41 +86,46 @@ def test_set_index_urls(): valid_url1 = "https://pkg-index.com/{package_name}/json/" valid_url2 = "https://another-pkg-index.com/{package_name}" - invalid_url = "https://invalid-pkg-index.com/json" + valid_url3 = "https://another-pkg-index.com/simple/" try: index_urls.set_index_urls(valid_url1) assert package_index.INDEX_URLS == [valid_url1] - index_urls.set_index_urls([valid_url1, valid_url2]) - assert package_index.INDEX_URLS == [valid_url1, valid_url2] - - with pytest.raises(ValueError, match="Invalid index URL"): - index_urls.set_index_urls([invalid_url]) + index_urls.set_index_urls([valid_url1, valid_url2, valid_url3]) + assert package_index.INDEX_URLS == [valid_url1, valid_url2, valid_url3] finally: index_urls.set_index_urls(default_index_urls) assert package_index.INDEX_URLS == default_index_urls +def test_contain_placeholder(): + assert package_index._contain_placeholder("https://pkg-index.com/{package_name}/") + assert package_index._contain_placeholder( + "https://pkg-index.com/{placeholder}/", placeholder="placeholder" + ) + assert not package_index._contain_placeholder("https://pkg-index.com/") + + @pytest.mark.asyncio -async def test_search_packages(mock_package_index_json_api): +async def test_query_package(mock_package_index_json_api): mock_server_snowballstemmer = mock_package_index_json_api(pkgs=["snowballstemmer"]) mock_server_pytest = mock_package_index_json_api(pkgs=["pytest"]) - project_info = await package_index.search_packages( + project_info = await package_index.query_package( "snowballstemmer", index_urls=[mock_server_snowballstemmer] ) assert project_info.name == "snowballstemmer" assert project_info.releases - project_info = await package_index.search_packages( + project_info = await package_index.query_package( "snowballstemmer", index_urls=mock_server_snowballstemmer ) assert project_info.name == "snowballstemmer" assert project_info.releases - project_info = await package_index.search_packages( + project_info = await package_index.query_package( "snowballstemmer", index_urls=[mock_server_pytest, mock_server_snowballstemmer] ) @@ -128,11 +133,6 @@ async def test_search_packages(mock_package_index_json_api): assert project_info.releases with pytest.raises(ValueError, match="Can't fetch metadata"): - await package_index.search_packages( + await package_index.query_package( "snowballstemmer", index_urls=[mock_server_pytest] ) - - with pytest.raises(ValueError, match="Invalid index URL"): - await package_index.search_packages( - "snowballstemmer", index_urls=["http://without-placeholder.com"] - ) From 6b50de67cd5470e81b936f5376e42b165d1ed431 Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Tue, 18 Jul 2023 21:20:15 +0900 Subject: [PATCH 09/10] Update changelog --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index efe35a4c..4bf4bb32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `verbose` parameter to micropip.install and micropip.uninstall [#60](https://github.com/pyodide/micropip/pull/60) - +- Added `index_urls` parameter to micropip.install to support installing + from custom package indexes. + [#74](https://github.com/pyodide/micropip/pull/74) +- Added `micropip.set_index_urls` to support installing from custom package + indexes. + [#74](https://github.com/pyodide/micropip/pull/74) ### Fixed - Fixed `micropip.add_mock_package` to work with Pyodide>=0.23.0 From e212ce220c61fdabaf1abae265f51484bd80e240 Mon Sep 17 00:00:00 2001 From: ryanking13 Date: Tue, 18 Jul 2023 21:30:17 +0900 Subject: [PATCH 10/10] Improve docstring --- micropip/_commands/index_urls.py | 14 +++++++------- micropip/_commands/install.py | 16 ++++++++-------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/micropip/_commands/index_urls.py b/micropip/_commands/index_urls.py index 48f25c57..60cc9b19 100644 --- a/micropip/_commands/index_urls.py +++ b/micropip/_commands/index_urls.py @@ -5,15 +5,15 @@ def set_index_urls(urls: list[str] | str) -> None: """ Set the index URLs to use when looking up packages. - The index URL may contain the placeholder {package_name} which will be - replaced with the package name when looking up a package. If it does not - contain the placeholder, the package name will be appended to the URL. + - The index URL should support the + `JSON API `__ . - The index URL should support the - [JSON API](https://warehouse.pypa.io/api-reference/json/). + - The index URL may contain the placeholder {package_name} which will be + replaced with the package name when looking up a package. If it does not + contain the placeholder, the package name will be appended to the URL. - If a list of URLs is provided, micropip will try each URL in order until - it finds a package. If no package is found, an error will be raised. + - If a list of URLs is provided, micropip will try each URL in order until + it finds a package. If no package is found, an error will be raised. Parameters ---------- diff --git a/micropip/_commands/install.py b/micropip/_commands/install.py index addbf2f7..dc856ec1 100644 --- a/micropip/_commands/install.py +++ b/micropip/_commands/install.py @@ -90,17 +90,17 @@ async def install( index_urls : A list of URLs or a single URL to use as the package index when looking - up packages. If None, `https://pypi.org/pypi/{package_name}/json` is used. + up packages. If None, *https://pypi.org/pypi/{package_name}/json* is used. - The index URL may contain the placeholder {package_name} which will be - replaced with the package name when looking up a package. If it does not - contain the placeholder, the package name will be appended to the URL. + - The index URL should support the + `JSON API `__ . - The index URL should support the - [JSON API](https://warehouse.pypa.io/api-reference/json/). + - The index URL may contain the placeholder {package_name} which will be + replaced with the package name when looking up a package. If it does not + contain the placeholder, the package name will be appended to the URL. - If a list of URLs is provided, micropip will try each URL in order until - it finds a package. If no package is found, an error will be raised. + - If a list of URLs is provided, micropip will try each URL in order until + it finds a package. If no package is found, an error will be raised. verbose : Print more information about the process.