Add support for pulling from http location.

.github/workflows/test.yml

@@ -19,12 +19,13 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          - {os: macos-latest,   py: '3.11'}
-          - {os: ubuntu-latest,  py: '3.8'}
           - {os: ubuntu-latest,  py: '3.9'}
           - {os: ubuntu-latest,  py: '3.10'}
           - {os: ubuntu-latest,  py: '3.11'}
-          - {os: windows-latest, py: '3.11'}
+          - {os: ubuntu-latest,  py: '3.12'}
+          - {os: ubuntu-latest,  py: '3.13'}
+          - {os: macos-latest,   py: '3.13'}
+          - {os: windows-latest, py: '3.13'}
 
     steps:
       - uses: actions/checkout@v3
@@ -36,6 +37,18 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install hatch
+
+      # Compiling outpack server from source takes a few minutes each time.
+      # This action will cache the result and re-use it on subsequent builds.
+      # The cache is keyed by Git revision, allowing us to pick up new versions
+      # of the server immediately.
+      - name: Install outpack server
+        uses: baptiste0928/cargo-install@v3
+        with:
+          crate: outpack
+          git: https://github.com/mrc-ide/outpack_server
+          features: git2/vendored-libgit2
+
       - name: Test
         run: |
           hatch run cov-ci

pyproject.toml

@@ -7,7 +7,7 @@ name = "pyorderly"
 dynamic = ["version"]
 description = "Reproducible and collaborative reporting"
 readme = "README.md"
-requires-python = ">=3.7"
+requires-python = ">=3.9"
 license = "MIT"
 keywords = []
 authors = [
@@ -16,11 +16,12 @@ authors = [
 classifiers = [
   "Development Status :: 4 - Beta",
   "Programming Language :: Python",
-  "Programming Language :: Python :: 3.7",
   "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
   "Programming Language :: Python :: Implementation :: CPython",
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
@@ -33,6 +34,8 @@ dependencies = [
   "humanize",
   "tblib",
   "paramiko",
+  "requests",
+  "typing-extensions",
 ]
 
 [project.urls]
@@ -46,15 +49,16 @@ path = "src/pyorderly/__about__.py"
 [tool.hatch.envs.default]
 dependencies = [
   "coverage[toml]>=6.5",
+  "myst-parser",
   "pytest",
-  "pytest_mock",
-  "pytest-unordered",
   "pytest-cov",
+  "pytest-unordered",
+  "pytest_mock",
+  "responses",
   "sphinx",
-  "sphinx-rtd-theme",
-  "myst-parser",
+  "sphinx-autoapi",
   "sphinx-copybutton",
-  "sphinx-autoapi"
+  "sphinx-rtd-theme",
 ]
 [tool.hatch.envs.default.scripts]
 test = "pytest {args:tests}"
@@ -75,7 +79,7 @@ generate-docs = [
 ]
 
 [[tool.hatch.envs.all.matrix]]
-python = ["3.7", "3.8", "3.9", "3.10", "3.11"]
+python = ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
 
 [tool.hatch.envs.lint]
 extra-dependencies = [

src/pyorderly/outpack/location.py

@@ -4,6 +4,8 @@
 
 from pyorderly.outpack.config import Location, update_config
 from pyorderly.outpack.location_driver import LocationDriver
+from pyorderly.outpack.location_http import OutpackLocationHTTP
+from pyorderly.outpack.location_packit import outpack_location_packit
 from pyorderly.outpack.location_path import OutpackLocationPath
 from pyorderly.outpack.location_ssh import OutpackLocationSSH, parse_ssh_url
 from pyorderly.outpack.root import OutpackRoot, root_open
@@ -34,7 +36,7 @@ def outpack_location_add(name, type, args, root=None, *, locate=True):
         root_open(loc.args["path"], locate=False)
     elif type == "ssh":
         parse_ssh_url(loc.args["url"])
-    elif type in ("http", "custom"):  # pragma: no cover
+    elif type in ("custom",):  # pragma: no cover
         msg = f"Cannot add a location with type '{type}' yet."
         raise Exception(msg)
 
@@ -156,8 +158,11 @@ def _location_driver(location_name, root) -> LocationDriver:
             location.args.get("password"),
         )
     elif location.type == "http":
-        msg = "Http remote not yet supported"
-        raise Exception(msg)
+        return OutpackLocationHTTP(location.args["url"])
+    elif location.type == "packit":
+        return outpack_location_packit(
+            location.args["url"], location.args.get("token")
+        )
     elif location.type == "custom":
         msg = "custom remote not yet supported"
         raise Exception(msg)

src/pyorderly/outpack/location_http.py

@@ -0,0 +1,80 @@
+import shutil
+from typing import Dict, List
+from urllib.parse import urljoin
+
+import requests
+from typing_extensions import override
+
+from pyorderly.outpack.location_driver import LocationDriver
+from pyorderly.outpack.metadata import MetadataCore, PacketFile, PacketLocation
+
+
+def raise_http_error(response: requests.Response):
+    if response.headers.get("Content-Type") == "application/json":
+        result = response.json()
+        # Unfortunately the schema is a bit inconsistent. Packit uses a
+        # singular `error` whereas outpack_server uses a list of
+        # `errors`.
+        if "error" in result:
+            detail = result["error"]["detail"]
+        else:
+            detail = result["errors"][0]["detail"]
+
+        msg = f"{response.status_code} Error: {detail}"
+        raise requests.HTTPError(msg)
+    else:
+        response.raise_for_status()
+
+
+class OutpackHTTPClient(requests.Session):
+    def __init__(self, url: str, authentication=None):
+        super().__init__()
+        self._base_url = url
+        self._authentication = authentication
+
+    @override
+    def request(self, method, path, *args, **kwargs):
+        if self._authentication is not None:
+            headers = kwargs.setdefault("headers", {})
+            headers.update(self._authentication())
+
+        url = urljoin(self._base_url, path)
+        response = super().request(method, url, *args, **kwargs)
+        if not response.ok:
+            raise_http_error(response)
+        return response
+
+
+class OutpackLocationHTTP(LocationDriver):
+    def __init__(self, url: str, authentication=None):
+        self._base_url = url
+        self._client = OutpackHTTPClient(url, authentication)
+
+    def __enter__(self):
+        self._client.__enter__()
+        return self
+
+    def __exit__(self, *args):
+        self._client.__exit__(*args)
+
+    @override
+    def list(self) -> Dict[str, PacketLocation]:
+        response = self._client.get("metadata/list").json()
+        data = response["data"]
+        return {
+            entry["packet"]: PacketLocation.from_dict(entry) for entry in data
+        }
+
+    @override
+    def metadata(self, ids: List[str]) -> Dict[str, str]:
+        result = {}
+        for i in ids:
+            result[i] = self._client.get(f"metadata/{i}/text").text
+
+        return result
+
+    @override
+    def fetch_file(self, packet: MetadataCore, file: PacketFile, dest: str):
+        response = self._client.get(f"file/{file.hash}", stream=True)
+        with open(dest, "wb") as f:
+            shutil.copyfileobj(response.raw, f)