api/rest: Handle patchwork pagination

If there are many patches (or other objects) to be listed, patchwork
only returns the first 30 by default. In such a case continue fetching
objects using the "next" link provided in the Link: header.

Note this changes REST._list() to return a generator instead of a list.
All callers only use it to generate a list, so this isn't a problem.

Author: ukleinek

pwclient/api.py

@@ -19,6 +19,8 @@
 from . import __version__
 from .xmlrpc import xmlrpclib
 
+from requests.utils import parse_header_links
+
 
 class API(metaclass=abc.ABCMeta):
     @abc.abstractmethod
@@ -594,8 +596,19 @@ def _list(
             url = f'{url}{resource_id}/{subresource_type}/'
         if params:
             url = f'{url}?{urllib.parse.urlencode(params)}'
-        data, _ = self._get(url)
-        return json.loads(data)
+
+        while url:
+            data, headers = self._get(url)
+            ret = json.loads(data)
+
+            yield from ret
+
+            try:
+                link = next(value for name, value in headers if name == 'Link')
+                url = next(a['url'] for a in parse_header_links(link) if a['rel'] == "next")
+
+            except StopIteration:
+                url = None
 
     # project
 

requirements.txt

@@ -0,0 +1 @@
+requests