Add support for Simple Repository API (PEP 503 / PEP 691)

.github/workflows/remote_package_index_test.yml

@@ -0,0 +1,56 @@
+name: run remote package index tests
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    env:
+      DISPLAY: :99
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-20.04]
+        pyodide-version: [0.23.4]
+        test-config: [
+          {runner: selenium, runtime: chrome, runtime-version: latest },
+        ]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.11.1
+
+      - uses: pyodide/pyodide-actions/download-pyodide@v1
+        with:
+          version: ${{ matrix.pyodide-version }}
+          to: dist
+
+      - uses: pyodide/pyodide-actions/install-browser@v1
+        with:
+          runner: ${{ matrix.test-config.runner }}
+          browser: ${{ matrix.test-config.runtime }}
+          browser-version: ${{ matrix.test-config.runtime-version }}
+
+      - name: Install requirements
+        shell: bash -l {0}
+        run: |
+          python3 -m pip install -e .[test]
+          python3 -m pip install requests
+
+      - name: Run tests
+        shell: bash -l {0}
+        run: |
+          pytest -v \
+            --dist-dir=./dist/ \
+            --runner=${{ matrix.test-config.runner }} \
+            --rt ${{ matrix.test-config.runtime }} \
+            --run-remote-index-tests \
+            tests/test_remote_indexes.py

CHANGELOG.md

@@ -16,11 +16,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added `micropip.set_index_urls` to support installing from custom package
   indexes.
   [#74](https://github.com/pyodide/micropip/pull/74)
+- Added support for Simple API (PEP 503 / PEP 691)
+  [#75](https://github.com/pyodide/micropip/pull/75)
 ### Fixed
 
 - Fixed `micropip.add_mock_package` to work with Pyodide>=0.23.0
   [#66](https://github.com/pyodide/micropip/pull/66)
 
+### Changed
+
+- The default index URL is changed to https://pypi.org/simple
+  [#75](https://github.com/pyodide/micropip/pull/75)
+
 ## [0.3.0] - 2023/03/29
 
 ### Added

micropip/_compat.py

@@ -7,7 +7,7 @@
         REPODATA_INFO,
         REPODATA_PACKAGES,
         fetch_bytes,
-        fetch_string,
+        fetch_string_and_headers,
         get_dynlibs,
         loadDynlib,
         loadedPackages,
@@ -20,7 +20,7 @@
         REPODATA_INFO,
         REPODATA_PACKAGES,
         fetch_bytes,
-        fetch_string,
+        fetch_string_and_headers,
         get_dynlibs,
         loadDynlib,
         loadedPackages,
@@ -33,7 +33,7 @@
     "REPODATA_INFO",
     "REPODATA_PACKAGES",
     "fetch_bytes",
-    "fetch_string",
+    "fetch_string_and_headers",
     "loadedPackages",
     "loadDynlib",
     "loadPackage",

micropip/_compat_in_pyodide.py

@@ -8,6 +8,7 @@
 
 try:
     import pyodide_js
+    from js import Object
     from pyodide_js import loadedPackages, loadPackage
     from pyodide_js._api import loadBinaryFile, loadDynlib  # type: ignore[import]
 
@@ -30,13 +31,23 @@ async def fetch_bytes(url: str, kwargs: dict[str, str]) -> IO[bytes]:
     return BytesIO(result_bytes)
 
 
-async def fetch_string(url: str, kwargs: dict[str, str]) -> str:
-    return await (await pyfetch(url, **kwargs)).string()
+async def fetch_string_and_headers(
+    url: str, kwargs: dict[str, str]
+) -> tuple[str, dict[str, str]]:
+    response = await pyfetch(url, **kwargs)
+
+    content = await response.string()
+    # TODO: replace with response.headers when pyodide>= 0.24 is released
+    headers: dict[str, str] = Object.fromEntries(
+        response.js_response.headers.entries()
+    ).to_py()
+
+    return content, headers
 
 
 __all__ = [
     "fetch_bytes",
-    "fetch_string",
+    "fetch_string_and_headers",
     "REPODATA_INFO",
     "REPODATA_PACKAGES",
     "loadedPackages",

micropip/_compat_not_in_pyodide.py

@@ -14,14 +14,24 @@ def to_py():
 
 
 from urllib.request import Request, urlopen
+from urllib.response import addinfourl
 
 
-async def fetch_bytes(url: str, kwargs: dict[str, str]) -> IO[bytes]:
-    return BytesIO(urlopen(Request(url, headers=kwargs)).read())
+def _fetch(url: str, kwargs: dict[str, Any]) -> addinfourl:
+    return urlopen(Request(url, **kwargs))
 
 
-async def fetch_string(url: str, kwargs: dict[str, str]) -> str:
-    return (await fetch_bytes(url, kwargs)).read().decode()
+async def fetch_bytes(url: str, kwargs: dict[str, Any]) -> IO[bytes]:
+    response = _fetch(url, kwargs=kwargs)
+    return BytesIO(response.read())
+
+
+async def fetch_string_and_headers(
+    url: str, kwargs: dict[str, Any]
+) -> tuple[str, dict[str, str]]:
+    response = _fetch(url, kwargs=kwargs)
+    headers = {k.lower(): v for k, v in response.headers.items()}
+    return response.read().decode(), headers
 
 
 async def loadDynlib(dynlib: str, is_shared_lib: bool) -> None:
@@ -108,7 +118,7 @@ def loadPackage(packages: str | list[str]) -> None:
 __all__ = [
     "loadDynlib",
     "fetch_bytes",
-    "fetch_string",
+    "fetch_string_and_headers",
     "REPODATA_INFO",
     "REPODATA_PACKAGES",
     "loadedPackages",