diff --git a/.github/workflows/remote_package_index_test.yml b/.github/workflows/remote_package_index_test.yml new file mode 100644 index 00000000..84aefdad --- /dev/null +++ b/.github/workflows/remote_package_index_test.yml @@ -0,0 +1,56 @@ +name: run remote package index tests + +on: + workflow_dispatch: + +permissions: + contents: read + + +jobs: + test: + runs-on: ${{ matrix.os }} + env: + DISPLAY: :99 + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04] + pyodide-version: [0.23.4] + test-config: [ + {runner: selenium, runtime: chrome, runtime-version: latest }, + ] + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: 3.11.1 + + - uses: pyodide/pyodide-actions/download-pyodide@v1 + with: + version: ${{ matrix.pyodide-version }} + to: dist + + - uses: pyodide/pyodide-actions/install-browser@v1 + with: + runner: ${{ matrix.test-config.runner }} + browser: ${{ matrix.test-config.runtime }} + browser-version: ${{ matrix.test-config.runtime-version }} + + - name: Install requirements + shell: bash -l {0} + run: | + python3 -m pip install -e .[test] + python3 -m pip install requests + + - name: Run tests + shell: bash -l {0} + run: | + pytest -v \ + --dist-dir=./dist/ \ + --runner=${{ matrix.test-config.runner }} \ + --rt ${{ matrix.test-config.runtime }} \ + --run-remote-index-tests \ + tests/test_remote_indexes.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bf4bb32..3676824d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,11 +16,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `micropip.set_index_urls` to support installing from custom package indexes. [#74](https://github.com/pyodide/micropip/pull/74) +- Added support for Simple API (PEP 503 / PEP 691) + [#75](https://github.com/pyodide/micropip/pull/75) ### Fixed - Fixed `micropip.add_mock_package` to work with Pyodide>=0.23.0 [#66](https://github.com/pyodide/micropip/pull/66) +### Changed + +- The default index URL is changed to https://pypi.org/simple + [#75](https://github.com/pyodide/micropip/pull/75) + ## [0.3.0] - 2023/03/29 ### Added diff --git a/micropip/_compat.py b/micropip/_compat.py index 967ec609..b445f56a 100644 --- a/micropip/_compat.py +++ b/micropip/_compat.py @@ -7,7 +7,7 @@ REPODATA_INFO, REPODATA_PACKAGES, fetch_bytes, - fetch_string, + fetch_string_and_headers, get_dynlibs, loadDynlib, loadedPackages, @@ -20,7 +20,7 @@ REPODATA_INFO, REPODATA_PACKAGES, fetch_bytes, - fetch_string, + fetch_string_and_headers, get_dynlibs, loadDynlib, loadedPackages, @@ -33,7 +33,7 @@ "REPODATA_INFO", "REPODATA_PACKAGES", "fetch_bytes", - "fetch_string", + "fetch_string_and_headers", "loadedPackages", "loadDynlib", "loadPackage", diff --git a/micropip/_compat_in_pyodide.py b/micropip/_compat_in_pyodide.py index 673b0ca2..7c5b03af 100644 --- a/micropip/_compat_in_pyodide.py +++ b/micropip/_compat_in_pyodide.py @@ -8,6 +8,7 @@ try: import pyodide_js + from js import Object from pyodide_js import loadedPackages, loadPackage from pyodide_js._api import loadBinaryFile, loadDynlib # type: ignore[import] @@ -30,13 +31,23 @@ async def fetch_bytes(url: str, kwargs: dict[str, str]) -> IO[bytes]: return BytesIO(result_bytes) -async def fetch_string(url: str, kwargs: dict[str, str]) -> str: - return await (await pyfetch(url, **kwargs)).string() +async def fetch_string_and_headers( + url: str, kwargs: dict[str, str] +) -> tuple[str, dict[str, str]]: + response = await pyfetch(url, **kwargs) + + content = await response.string() + # TODO: replace with response.headers when pyodide>= 0.24 is released + headers: dict[str, str] = Object.fromEntries( + response.js_response.headers.entries() + ).to_py() + + return content, headers __all__ = [ "fetch_bytes", - "fetch_string", + "fetch_string_and_headers", "REPODATA_INFO", "REPODATA_PACKAGES", "loadedPackages", diff --git a/micropip/_compat_not_in_pyodide.py b/micropip/_compat_not_in_pyodide.py index 73f525a9..892fef13 100644 --- a/micropip/_compat_not_in_pyodide.py +++ b/micropip/_compat_not_in_pyodide.py @@ -14,14 +14,24 @@ def to_py(): from urllib.request import Request, urlopen +from urllib.response import addinfourl -async def fetch_bytes(url: str, kwargs: dict[str, str]) -> IO[bytes]: - return BytesIO(urlopen(Request(url, headers=kwargs)).read()) +def _fetch(url: str, kwargs: dict[str, Any]) -> addinfourl: + return urlopen(Request(url, **kwargs)) -async def fetch_string(url: str, kwargs: dict[str, str]) -> str: - return (await fetch_bytes(url, kwargs)).read().decode() +async def fetch_bytes(url: str, kwargs: dict[str, Any]) -> IO[bytes]: + response = _fetch(url, kwargs=kwargs) + return BytesIO(response.read()) + + +async def fetch_string_and_headers( + url: str, kwargs: dict[str, Any] +) -> tuple[str, dict[str, str]]: + response = _fetch(url, kwargs=kwargs) + headers = {k.lower(): v for k, v in response.headers.items()} + return response.read().decode(), headers async def loadDynlib(dynlib: str, is_shared_lib: bool) -> None: @@ -108,7 +118,7 @@ def loadPackage(packages: str | list[str]) -> None: __all__ = [ "loadDynlib", "fetch_bytes", - "fetch_string", + "fetch_string_and_headers", "REPODATA_INFO", "REPODATA_PACKAGES", "loadedPackages", diff --git a/micropip/externals/mousebender/simple.py b/micropip/externals/mousebender/simple.py new file mode 100644 index 00000000..d3adb3dd --- /dev/null +++ b/micropip/externals/mousebender/simple.py @@ -0,0 +1,226 @@ +# Adapted from: https://github.com/brettcannon/mousebender/blob/main/mousebender/simple.py +# Only relevant parts are included here. + +import html +import html.parser +import urllib.parse +import warnings +from typing import Any, Dict, List, Optional, Union, Literal, TypeAlias, TypedDict + +import packaging.utils + + +ACCEPT_JSON_V1 = "application/vnd.pypi.simple.v1+json" + + + +class UnsupportedAPIVersion(Exception): + """The major version of an API response is not supported.""" + + def __init__(self, version: str) -> None: + """Initialize the exception with a message based on the provided version.""" + super().__init__(f"Unsupported API major version: {version!r}") + + +class APIVersionWarning(Warning): + """The minor version of an API response is not supported.""" + + def __init__(self, version: str) -> None: + """Initialize the warning with a message based on the provided version.""" + super().__init__(f"Unsupported API minor version: {version!r}") + + +class UnsupportedMIMEType(Exception): + """An unsupported MIME type was provided in a ``Content-Type`` header.""" + + +_Meta_1_0 = TypedDict("_Meta_1_0", {"api-version": Literal["1.0"]}) +_Meta_1_1 = TypedDict("_Meta_1_1", {"api-version": Literal["1.1"]}) + + +_HashesDict: TypeAlias = Dict[str, str] + +_OptionalProjectFileDetails_1_0 = TypedDict( + "_OptionalProjectFileDetails_1_0", + { + "requires-python": str, + "dist-info-metadata": Union[bool, _HashesDict], + "gpg-sig": bool, + "yanked": Union[bool, str], + }, + total=False, +) + + +class ProjectFileDetails_1_0(_OptionalProjectFileDetails_1_0): + """A :class:`~typing.TypedDict` for the ``files`` key of :class:`ProjectDetails_1_0`.""" + + filename: str + url: str + hashes: _HashesDict + + +_OptionalProjectFileDetails_1_1 = TypedDict( + "_OptionalProjectFileDetails_1_1", + { + "requires-python": str, + "dist-info-metadata": Union[bool, _HashesDict], + "gpg-sig": bool, + "yanked": Union[bool, str], + # PEP 700 + "upload-time": str, + }, + total=False, +) + + +class ProjectFileDetails_1_1(_OptionalProjectFileDetails_1_1): + """A :class:`~typing.TypedDict` for the ``files`` key of :class:`ProjectDetails_1_1`.""" + + filename: str + url: str + hashes: _HashesDict + # PEP 700 + size: int + + +class ProjectDetails_1_0(TypedDict): + """A :class:`~typing.TypedDict` for a project details response (:pep:`691`).""" + + meta: _Meta_1_0 + name: packaging.utils.NormalizedName + files: list[ProjectFileDetails_1_0] + + +class ProjectDetails_1_1(TypedDict): + """A :class:`~typing.TypedDict` for a project details response (:pep:`700`).""" + + meta: _Meta_1_1 + name: packaging.utils.NormalizedName + files: list[ProjectFileDetails_1_1] + # PEP 700 + versions: List[str] + + +ProjectDetails: TypeAlias = Union[ProjectDetails_1_0, ProjectDetails_1_1] + + +def _check_version(tag: str, attrs: Dict[str, Optional[str]]) -> None: + if ( + tag == "meta" + and attrs.get("name") == "pypi:repository-version" + and "content" in attrs + and attrs["content"] + ): + version = attrs["content"] + major_version, minor_version = map(int, version.split(".")) + if major_version != 1: + raise UnsupportedAPIVersion(version) + elif minor_version > 1: + warnings.warn(APIVersionWarning(version), stacklevel=7) + + +class _ArchiveLinkHTMLParser(html.parser.HTMLParser): + def __init__(self) -> None: + self.archive_links: List[Dict[str, Any]] = [] + super().__init__() + + def handle_starttag( + self, tag: str, attrs_list: list[tuple[str, Optional[str]]] + ) -> None: + attrs = dict(attrs_list) + _check_version(tag, attrs) + if tag != "a": + return + # PEP 503: + # The href attribute MUST be a URL that links to the location of the + # file for download ... + if "href" not in attrs or not attrs["href"]: + return + full_url: str = attrs["href"] + parsed_url = urllib.parse.urlparse(full_url) + # PEP 503: + # ... the text of the anchor tag MUST match the final path component + # (the filename) of the URL. + _, _, raw_filename = parsed_url.path.rpartition("/") + filename = urllib.parse.unquote(raw_filename) + url = urllib.parse.urlunparse((*parsed_url[:5], "")) + args: Dict[str, Any] = {"filename": filename, "url": url} + # PEP 503: + # The URL SHOULD include a hash in the form of a URL fragment with the + # following syntax: #= ... + if parsed_url.fragment: + hash_algo, hash_value = parsed_url.fragment.split("=", 1) + args["hashes"] = hash_algo.lower(), hash_value + # PEP 503: + # A repository MAY include a data-requires-python attribute on a file + # link. This exposes the Requires-Python metadata field ... + # In the attribute value, < and > have to be HTML encoded as < and + # >, respectively. + if "data-requires-python" in attrs and attrs["data-requires-python"]: + requires_python_data = html.unescape(attrs["data-requires-python"]) + args["requires-python"] = requires_python_data + # PEP 503: + # A repository MAY include a data-gpg-sig attribute on a file link with + # a value of either true or false ... + if "data-gpg-sig" in attrs: + args["gpg-sig"] = attrs["data-gpg-sig"] == "true" + # PEP 592: + # Links in the simple repository MAY have a data-yanked attribute which + # may have no value, or may have an arbitrary string as a value. + if "data-yanked" in attrs: + args["yanked"] = attrs.get("data-yanked") or True + # PEP 658: + # ... each anchor tag pointing to a distribution MAY have a + # data-dist-info-metadata attribute. + if "data-dist-info-metadata" in attrs: + found_metadata = attrs.get("data-dist-info-metadata") + if found_metadata and found_metadata != "true": + # The repository SHOULD provide the hash of the Core Metadata + # file as the data-dist-info-metadata attribute's value using + # the syntax =, where is the + # lower cased name of the hash function used, and is + # the hex encoded digest. + algorithm, _, hash_ = found_metadata.partition("=") + metadata = (algorithm.lower(), hash_) + else: + # The repository MAY use true as the attribute's value if a hash + # is unavailable. + metadata = "", "" + args["metadata"] = metadata + + self.archive_links.append(args) + + +def from_project_details_html(html: str, name: str) -> ProjectDetails_1_0: + """Convert the HTML response for a project details page to a :pep:`691` response. + + Due to HTML project details pages lacking the name of the project, it must + be specified via the *name* parameter to fill in the JSON data. + """ + parser = _ArchiveLinkHTMLParser() + parser.feed(html) + files: List[ProjectFileDetails_1_0] = [] + for archive_link in parser.archive_links: + details: ProjectFileDetails_1_0 = { + "filename": archive_link["filename"], + "url": archive_link["url"], + "hashes": {}, + } + if "hashes" in archive_link: + details["hashes"] = dict([archive_link["hashes"]]) + if "metadata" in archive_link: + algorithm, value = archive_link["metadata"] + if algorithm: + details["dist-info-metadata"] = {algorithm: value} + else: + details["dist-info-metadata"] = True + for key in {"requires-python", "yanked", "gpg-sig"}: + if key in archive_link: + details[key] = archive_link[key] # type: ignore + files.append(details) + return { + "meta": {"api-version": "1.0"}, + "name": packaging.utils.canonicalize_name(name), + "files": files, + } \ No newline at end of file diff --git a/micropip/package_index.py b/micropip/package_index.py index c66cbad6..8b787265 100644 --- a/micropip/package_index.py +++ b/micropip/package_index.py @@ -2,17 +2,19 @@ import string import sys from collections import defaultdict -from collections.abc import Generator +from collections.abc import Callable, Generator from dataclasses import dataclass +from functools import partial from typing import Any from packaging.utils import InvalidWheelFilename from packaging.version import InvalidVersion, Version -from ._compat import fetch_string +from ._compat import fetch_string_and_headers from ._utils import is_package_compatible, parse_version +from .externals.mousebender.simple import from_project_details_html -DEFAULT_INDEX_URLS = ["https://pypi.org/pypi/{package_name}/json"] +DEFAULT_INDEX_URLS = ["https://pypi.org/simple"] INDEX_URLS = DEFAULT_INDEX_URLS _formatter = string.Formatter() @@ -47,15 +49,17 @@ class ProjectInfo: releases: dict[Version, Generator[ProjectInfoFile, None, None]] @staticmethod - def from_json_api(data: dict[str, Any]) -> "ProjectInfo": + def from_json_api(data: str | bytes | dict[str, Any]) -> "ProjectInfo": """ Parse JSON API response https://warehouse.pypa.io/api-reference/json.html """ - name: str = data.get("info", {}).get("name", "UNKNOWN") - releases_raw: dict[str, list[Any]] = data["releases"] + data_dict = json.loads(data) if isinstance(data, str | bytes) else data + + name: str = data_dict.get("info", {}).get("name", "UNKNOWN") + releases_raw: dict[str, list[Any]] = data_dict["releases"] # Filter out non PEP 440 compliant versions releases: dict[Version, list[Any]] = {} @@ -73,19 +77,39 @@ def from_json_api(data: dict[str, Any]) -> "ProjectInfo": return ProjectInfo._compatible_only(name, releases) @staticmethod - def from_simple_api(data: dict[str, Any]) -> "ProjectInfo": + def from_simple_json_api(data: str | bytes | dict[str, Any]) -> "ProjectInfo": """ - Parse Simple API response + Parse Simple JSON API response - https://peps.python.org/pep-0503/ https://peps.python.org/pep-0691/ """ - name = data["name"] + + data_dict = json.loads(data) if isinstance(data, str | bytes) else data + name, releases = ProjectInfo._parse_pep691_response(data_dict) + return ProjectInfo._compatible_only(name, releases) + + @staticmethod + def from_simple_html_api(data: str, pkgname: str) -> "ProjectInfo": + """ + Parse Simple HTML API response + + https://peps.python.org/pep-0503 + """ + project_detail = from_project_details_html(data, pkgname) + name, releases = ProjectInfo._parse_pep691_response(project_detail) # type: ignore[arg-type] + return ProjectInfo._compatible_only(name, releases) + + @staticmethod + def _parse_pep691_response( + resp: dict[str, Any] + ) -> tuple[str, dict[Version, list[Any]]]: + name = resp["name"] # List of versions (PEP 700), this key is not critical to find packages # but it is required to ensure that the same class instance is returned - # from JSON and Simple APIs. - versions = data.get("versions", []) + # from JSON and Simple JSON APIs. + # Note that Simple HTML API does not have this key. + versions = resp.get("versions", []) # Group files by version releases: dict[Version, list[Any]] = defaultdict(list) @@ -97,7 +121,7 @@ def from_simple_api(data: dict[str, Any]) -> "ProjectInfo": releases[version] = [] - for file in data["files"]: + for file in resp["files"]: filename = file["filename"] if not _fast_check_incompatibility(filename): @@ -111,7 +135,7 @@ def from_simple_api(data: dict[str, Any]) -> "ProjectInfo": releases[version].append(file) - return ProjectInfo._compatible_only(name, releases) + return name, releases @classmethod def _compatible_only( @@ -198,9 +222,24 @@ def _contain_placeholder(url: str, placeholder: str = "package_name") -> bool: return placeholder in fields +def _select_parser(content_type: str, pkgname: str) -> Callable[[str], ProjectInfo]: + """ + Select the function to parse the response based on the content type. + """ + match content_type: + case "application/vnd.pypi.simple.v1+json": + return ProjectInfo.from_simple_json_api + case "application/json": + return ProjectInfo.from_json_api + case "application/vnd.pypi.simple.v1+html" | "text/html": + return partial(ProjectInfo.from_simple_html_api, pkgname=pkgname) + case _: + raise ValueError(f"Unsupported content type: {content_type}") + + async def query_package( name: str, - fetch_kwargs: dict[str, str] | None = None, + fetch_kwargs: dict[str, Any] | None = None, index_urls: list[str] | str | None = None, ) -> ProjectInfo: """ @@ -221,8 +260,15 @@ async def query_package( """ global INDEX_URLS - if not fetch_kwargs: - fetch_kwargs = {} + _fetch_kwargs = fetch_kwargs.copy() if fetch_kwargs else {} + + if "headers" not in _fetch_kwargs: + _fetch_kwargs["headers"] = {} + + # If not specified, prefer Simple JSON API over Simple HTML API or JSON API + _fetch_kwargs["headers"].setdefault( + "accept", "application/vnd.pypi.simple.v1+json, */*;q=0.01" + ) if index_urls is None: index_urls = INDEX_URLS @@ -236,14 +282,16 @@ async def query_package( url = f"{url}/{name}/" try: - metadata = await fetch_string(url, fetch_kwargs) + metadata, headers = await fetch_string_and_headers(url, _fetch_kwargs) except OSError: continue - return ProjectInfo.from_json_api(json.loads(metadata)) + content_type = headers.get("content-type", "").lower() + parser = _select_parser(content_type, name) + return parser(metadata) else: raise ValueError( - f"Can't fetch metadata for '{name}'." + f"Can't fetch metadata for '{name}'. " "Please make sure you have entered a correct package name " "and correctly specified index_urls (if you changed them)." ) diff --git a/tests/conftest.py b/tests/conftest.py index 153cb39a..4e587d4f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,16 @@ import pytest from pytest_pyodide import spawn_web_server + +def pytest_addoption(parser): + parser.addoption( + "--run-remote-index-tests", + action="store_true", + default=None, + help="Run tests that query remote package indexes.", + ) + + SNOWBALL_WHEEL = "snowballstemmer-2.0.0-py2.py3-none-any.whl" EMSCRIPTEN_VER = "3.1.14" @@ -277,3 +287,23 @@ def mock_package_index_json_api(httpserver): suffix="_json.json.gz", content_type="application/json", ) + + +@pytest.fixture +def mock_package_index_simple_json_api(httpserver): + return functools.partial( + _mock_package_index_gen, + httpserver=httpserver, + suffix="_simple.json.gz", + content_type="application/vnd.pypi.simple.v1+json", + ) + + +@pytest.fixture +def mock_package_index_simple_html_api(httpserver): + return functools.partial( + _mock_package_index_gen, + httpserver=httpserver, + suffix="_simple.html.gz", + content_type="text/html", + ) diff --git a/tests/test_data/pypi_response/black_simple.html.gz b/tests/test_data/pypi_response/black_simple.html.gz new file mode 100644 index 00000000..bf62737e Binary files /dev/null and b/tests/test_data/pypi_response/black_simple.html.gz differ diff --git a/tests/test_data/pypi_response/gen_responses.sh b/tests/test_data/pypi_response/gen_responses.sh index 267f7238..4f6dbf9a 100644 --- a/tests/test_data/pypi_response/gen_responses.sh +++ b/tests/test_data/pypi_response/gen_responses.sh @@ -14,6 +14,6 @@ do echo "Generating response for ${package}" # Gzip the response so grepping the source code wouldn't produce all sorts of noise with text data curl -s "https://pypi.org/pypi/${package}/json" | gzip > "${package}_json.json.gz" - # curl -s "https://pypi.org/simple/${package}/" > "${package}_simple.html" + curl -s "https://pypi.org/simple/${package}/" -H "Aceept: text/html" | gzip > "${package}_simple.html.gz" curl -s "https://pypi.org/simple/${package}/" -H "Accept: application/vnd.pypi.simple.v1+json" | gzip > "${package}_simple.json.gz" done diff --git a/tests/test_data/pypi_response/numpy_simple.html.gz b/tests/test_data/pypi_response/numpy_simple.html.gz new file mode 100644 index 00000000..a8b646bf Binary files /dev/null and b/tests/test_data/pypi_response/numpy_simple.html.gz differ diff --git a/tests/test_data/pypi_response/pytest_simple.html.gz b/tests/test_data/pypi_response/pytest_simple.html.gz new file mode 100644 index 00000000..ef6868be Binary files /dev/null and b/tests/test_data/pypi_response/pytest_simple.html.gz differ diff --git a/tests/test_data/pypi_response/pytz_simple.html.gz b/tests/test_data/pypi_response/pytz_simple.html.gz new file mode 100644 index 00000000..b5fffd21 Binary files /dev/null and b/tests/test_data/pypi_response/pytz_simple.html.gz differ diff --git a/tests/test_data/pypi_response/snowballstemmer_simple.html.gz b/tests/test_data/pypi_response/snowballstemmer_simple.html.gz new file mode 100644 index 00000000..1c1920e2 Binary files /dev/null and b/tests/test_data/pypi_response/snowballstemmer_simple.html.gz differ diff --git a/tests/test_data/pypi_response/xo-gd_simple.html.gz b/tests/test_data/pypi_response/xo-gd_simple.html.gz new file mode 100644 index 00000000..f9c5ef50 Binary files /dev/null and b/tests/test_data/pypi_response/xo-gd_simple.html.gz differ diff --git a/tests/test_package_index.py b/tests/test_package_index.py index 03df2341..a0e6b0da 100644 --- a/tests/test_package_index.py +++ b/tests/test_package_index.py @@ -1,5 +1,3 @@ -import json - import pytest from conftest import TEST_PYPI_RESPONSE_DIR, _read_pypi_response @@ -7,25 +5,30 @@ import micropip.package_index as package_index -@pytest.mark.parametrize( - "name", ["numpy", "black", "pytest", "snowballstemmer", "pytz"] -) -def test_project_info_from_json(name): - test_file = TEST_PYPI_RESPONSE_DIR / f"{name}_json.json.gz" - test_data = json.loads(_read_pypi_response(test_file)) - - index = package_index.ProjectInfo.from_json_api(test_data) - assert index.name == name - assert index.releases +def _check_project_info(project_info: package_index.ProjectInfo): + assert project_info.name + assert project_info.releases - versions = list(index.releases.keys()) + versions = list(project_info.releases.keys()) assert versions assert versions == sorted(versions) - for files in index.releases.values(): + for files in project_info.releases.values(): for file in files: assert file.filename in file.url - assert len(file.sha256) == 64 + if file.sha256 is not None: + assert len(file.sha256) == 64 + + +@pytest.mark.parametrize( + "name", ["numpy", "black", "pytest", "snowballstemmer", "pytz"] +) +def test_project_info_from_json(name): + test_file = TEST_PYPI_RESPONSE_DIR / f"{name}_json.json.gz" + test_data = _read_pypi_response(test_file) + + info = package_index.ProjectInfo.from_json_api(test_data) + _check_project_info(info) @pytest.mark.parametrize( @@ -33,20 +36,23 @@ def test_project_info_from_json(name): ) def test_project_info_from_simple_json(name): test_file = TEST_PYPI_RESPONSE_DIR / f"{name}_simple.json.gz" - test_data = json.loads(_read_pypi_response(test_file)) + test_data = _read_pypi_response(test_file) - index = package_index.ProjectInfo.from_simple_api(test_data) - assert index.name == name - assert index.releases + info = package_index.ProjectInfo.from_simple_json_api(test_data) + _check_project_info(info) - versions = list(index.releases.keys()) - assert versions - assert versions == sorted(versions) - for files in index.releases.values(): - for file in files: - assert file.filename in file.url - assert len(file.sha256) == 64 +@pytest.mark.parametrize( + "name", ["numpy", "black", "pytest", "snowballstemmer", "pytz"] +) +def test_project_info_from_simple_html(name): + test_file = TEST_PYPI_RESPONSE_DIR / f"{name}_simple.html.gz" + test_data = _read_pypi_response(test_file) + + info = package_index.ProjectInfo.from_simple_html_api( + test_data.decode("utf-8"), name + ) + _check_project_info(info) @pytest.mark.parametrize( @@ -54,14 +60,17 @@ def test_project_info_from_simple_json(name): ) def test_project_info_equal(name): # The different ways of parsing the same data should result in the same + # Simple HTML API does not contain `versions` key, so it is not easy to compare... test_file_json = TEST_PYPI_RESPONSE_DIR / f"{name}_json.json.gz" test_file_simple_json = TEST_PYPI_RESPONSE_DIR / f"{name}_simple.json.gz" - test_data_json = json.loads(_read_pypi_response(test_file_json)) - test_data_simple_json = json.loads(_read_pypi_response(test_file_simple_json)) + test_data_json = _read_pypi_response(test_file_json) + test_data_simple_json = _read_pypi_response(test_file_simple_json) index_json = package_index.ProjectInfo.from_json_api(test_data_json) - index_simple_json = package_index.ProjectInfo.from_simple_api(test_data_simple_json) + index_simple_json = package_index.ProjectInfo.from_simple_json_api( + test_data_simple_json + ) assert index_json.name == index_simple_json.name @@ -106,33 +115,51 @@ def test_contain_placeholder(): assert not package_index._contain_placeholder("https://pkg-index.com/") -@pytest.mark.asyncio -async def test_query_package(mock_package_index_json_api): - mock_server_snowballstemmer = mock_package_index_json_api(pkgs=["snowballstemmer"]) - mock_server_pytest = mock_package_index_json_api(pkgs=["pytest"]) - - project_info = await package_index.query_package( - "snowballstemmer", index_urls=[mock_server_snowballstemmer] - ) +async def _test_query_package(pkg1, pkg1_index_url, pkg2, pkg2_index_url): + project_info = await package_index.query_package(pkg1, index_urls=[pkg1_index_url]) - assert project_info.name == "snowballstemmer" + assert project_info.name == pkg1 assert project_info.releases - project_info = await package_index.query_package( - "snowballstemmer", index_urls=mock_server_snowballstemmer - ) + project_info = await package_index.query_package(pkg1, index_urls=pkg1_index_url) - assert project_info.name == "snowballstemmer" + assert project_info.name == pkg1 assert project_info.releases project_info = await package_index.query_package( - "snowballstemmer", index_urls=[mock_server_pytest, mock_server_snowballstemmer] + pkg1, index_urls=[pkg2_index_url, pkg1_index_url] ) - assert project_info.name == "snowballstemmer" + assert project_info.name == pkg1 assert project_info.releases with pytest.raises(ValueError, match="Can't fetch metadata"): - await package_index.query_package( - "snowballstemmer", index_urls=[mock_server_pytest] - ) + await package_index.query_package(pkg1, index_urls=[pkg2_index_url]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "pkg1, pkg2", + [ + ("snowballstemmer", "pytest"), + ("pytest", "snowballstemmer"), + ("black", "pytest"), + ("numpy", "black"), + ], +) +async def test_query_package( + pkg1, + pkg2, + mock_package_index_json_api, + mock_package_index_simple_json_api, + mock_package_index_simple_html_api, +): + for gen_mock_server in ( + mock_package_index_json_api, + mock_package_index_simple_json_api, + mock_package_index_simple_html_api, + ): + mock_server_1 = gen_mock_server(pkgs=[pkg1]) + mock_server_2 = gen_mock_server(pkgs=[pkg2]) + + await _test_query_package(pkg1, mock_server_1, pkg2, mock_server_2) diff --git a/tests/test_remote_indexes.py b/tests/test_remote_indexes.py new file mode 100644 index 00000000..f3928be7 --- /dev/null +++ b/tests/test_remote_indexes.py @@ -0,0 +1,100 @@ +# This file contains tests that actually query remote package indexes, +# to ensure that micropip works with real-world package indexes. +# Since running these tests will send many requests to remote servers, +# these tests are disabled by default. +# +# To run these tests, add `--run-remote-index-tests` flag, or +# these tests can also be run in Github Actions manually. +import functools +import random + +from pytest_pyodide import run_in_pyodide + + +@run_in_pyodide +async def _query(selenium, index_url, header_accept, packages): + from micropip.package_index import query_package + + for package in packages: + await query_package( + package, + fetch_kwargs={"Accept": header_accept}, + index_urls=[index_url], + ) + + +@functools.cache +def _random_pypi_packages(k: int) -> list[str]: + # Select random K PyPI packages + import requests # type: ignore[import] + + top_pypi_packages = ( + "https://hugovk.github.io/top-pypi-packages/top-pypi-packages-30-days.min.json" + ) + packages = requests.get(top_pypi_packages).json() + rows = packages["rows"] + + packages = random.choices(rows, k=k) + names = [package["project"] for package in packages] + return names + + +# 1) PyPI + + +def test_pypi_json_api(selenium_standalone_micropip, pytestconfig): + pytestconfig.getoption("--run-remote-index-tests", skip=True) + PYPI_PACKAGES = _random_pypi_packages(k=10) + _query( + selenium_standalone_micropip, + index_url="https://pypi.org/pypi/{package_name}/json", + header_accept="application/json", + packages=PYPI_PACKAGES, + ) + + +def test_pypi_simple_json_api(selenium_standalone_micropip, pytestconfig): + pytestconfig.getoption("--run-remote-index-tests", skip=True) + PYPI_PACKAGES = _random_pypi_packages(k=10) + _query( + selenium_standalone_micropip, + index_url="https://pypi.org/simple", + header_accept="application/vnd.pypi.simple.v1+json", + packages=PYPI_PACKAGES, + ) + + +# As of 07/2023, some Simple HTML API responses from PyPI does not contain CORS headers + +# def test_pypi_simple_html_api(selenium_standalone_micropip, pytestconfig): +# pytestconfig.getoption("--run-remote-index-tests", skip=True) +# PYPI_PACKAGES = _random_pypi_packages(k=5) +# PYPI_PACKAGES=["inflection"] +# _query( +# selenium_standalone_micropip, +# index_url="https://pypi.org/simple", +# header_accept="text/html", +# packages=PYPI_PACKAGES, +# ) + +# 2) Anaconda.org +# As of 07/2023: +# - only support simple HTML API (PEP 503) +# - does not contain CORS headers in its response + +# def test_anaconda_simple_html_api(selenium_standalone_micropip, pytestconfig): +# pytestconfig.getoption("--run-remote-index-tests", skip=True) + +# # One of the indexes in anaconda.org +# ANACONDA_INDEX_URL = "https://pypi.anaconda.org/beeware/simple" +# ANACONDA_PACKAGES = [ +# "pynacl", +# "bitarray", +# ] + +# _query( +# selenium_standalone_micropip, +# index_url=ANACONDA_INDEX_URL, +# header_accept="text/html", +# packages=ANACONDA_PACKAGES, +# )