fetch_string_and_headers compat: raise in and out of pyodide

Currently only the not_in_pyodide will raise on non-success, because this
is the default behavior of urllib, the in_pyodide will not, so I added a
raise_for_status.

It is better to raise, as otherwise the package parser will potentially
get proper URL and not manage to parse it, and decide there is no
wheels, while we actually just got an error (404, or maybe 500).

In addition wraps both case in a custom local HttpStatusError, so that
we can actually catch these errors in the right places when we encounter
them.

I think we could define HttpStatusError on the ABC and not make it an
ABC itself, but it small enough that I think duplication is ok.

Author: Carreau

micropip/_compat/__init__.py

@@ -34,6 +34,8 @@
 
 to_js = compatibility_layer.to_js
 
+HttpStatusError = compatibility_layer.HttpStatusError
+
 
 __all__ = [
     "REPODATA_INFO",
@@ -45,4 +47,5 @@
     "loadPackage",
     "get_dynlibs",
     "to_js",
+    "HttpStatusError",
 ]

micropip/_compat/_compat_in_pyodide.py

@@ -7,7 +7,7 @@
 
 from pyodide._package_loader import get_dynlibs
 from pyodide.ffi import IN_BROWSER, to_js
-from pyodide.http import pyfetch
+from pyodide.http import HttpStatusError, pyfetch
 
 from .compatibility_layer import CompatibilityLayer
 
@@ -28,6 +28,15 @@
 
 
 class CompatibilityInPyodide(CompatibilityLayer):
+    class HttpStatusError(Exception):
+        status_code: int
+        message: str
+
+        def __init__(self, status_code: int, message: str):
+            self.status_code = status_code
+            self.message = message
+            super().__init__(message)
+
     @staticmethod
     def repodata_info() -> dict[str, str]:
         return REPODATA_INFO
@@ -50,7 +59,11 @@ async def fetch_bytes(url: str, kwargs: dict[str, str]) -> bytes:
     async def fetch_string_and_headers(
         url: str, kwargs: dict[str, str]
     ) -> tuple[str, dict[str, str]]:
-        response = await pyfetch(url, **kwargs)
+        try:
+            response = await pyfetch(url, **kwargs)
+            response.raise_for_status()
+        except HttpStatusError as e:
+            raise CompatibilityInPyodide.HttpStatusError(e.status, str(e)) from e
 
         content = await response.string()
         headers: dict[str, str] = response.headers

micropip/_compat/_compat_not_in_pyodide.py

@@ -1,6 +1,7 @@
 import re
 from pathlib import Path
 from typing import IO, TYPE_CHECKING, Any
+from urllib.error import HTTPError
 from urllib.request import Request, urlopen
 from urllib.response import addinfourl
 
@@ -15,6 +16,15 @@ class CompatibilityNotInPyodide(CompatibilityLayer):
     # Vendored from packaging
     _canonicalize_regex = re.compile(r"[-_.]+")
 
+    class HttpStatusError(Exception):
+        status_code: int
+        message: str
+
+        def __init__(self, status_code: int, message: str):
+            self.status_code = status_code
+            self.message = message
+            super().__init__(message)
+
     class loadedPackages(CompatibilityLayer.loadedPackages):
         @staticmethod
         def to_py():
@@ -40,7 +50,11 @@ async def fetch_bytes(url: str, kwargs: dict[str, Any]) -> bytes:
     async def fetch_string_and_headers(
         url: str, kwargs: dict[str, Any]
     ) -> tuple[str, dict[str, str]]:
-        response = CompatibilityNotInPyodide._fetch(url, kwargs=kwargs)
+        try:
+            response = CompatibilityNotInPyodide._fetch(url, kwargs=kwargs)
+        except HTTPError as e:
+            raise CompatibilityNotInPyodide.HttpStatusError(e.code, str(e)) from e
+
         headers = {k.lower(): v for k, v in response.headers.items()}
         return response.read().decode(), headers
 

micropip/_compat/compatibility_layer.py

@@ -13,6 +13,14 @@ class CompatibilityLayer(ABC):
     All of the following methods / properties must be implemented for use both inside and outside of pyodide.
     """
 
+    class HttpStatusError(ABC, Exception):
+        status_code: int
+        message: str
+
+        @abstractmethod
+        def __init__(self, status_code: int, message: str):
+            pass
+
     class loadedPackages(ABC):
         @staticmethod
         @abstractmethod