Merge pull request #351 from EbbLabs/fix/issue-345

Feature: Fix playlist v2 endpoint usage. Add pagination workers from mopidy-tidal

Author: tehkillerbee

tests/test_user.py

@@ -68,10 +68,10 @@ def test_get_user_playlists(session):
 def test_get_playlist_folders(session):
     folder = session.user.create_folder(title="testfolder")
     assert folder
-    folder_ids = [folder.id for folder in session.user.playlist_folders()]
+    folder_ids = [folder.id for folder in session.user.favorites.playlist_folders()]
     assert folder.id in folder_ids
     folder.remove()
-    folder_ids = [folder.id for folder in session.user.playlist_folders()]
+    folder_ids = [folder.id for folder in session.user.favorites.playlist_folders()]
     assert folder.id not in folder_ids
 
 

tidalapi/playlist.py

@@ -88,12 +88,14 @@ def __init__(self, session: "Session", playlist_id: Optional[str]):
                 self._etag = request.headers["etag"]
                 self.parse(request.json())
 
-    def parse(self, json_obj: JsonObj) -> "Playlist":
+    def parse(self, obj: JsonObj) -> "Playlist":
         """Parses a playlist from tidal, replaces the current playlist object.
 
-        :param json_obj: Json data returned from api.tidal.com containing a playlist
+        :param obj: Json data returned from api.tidal.com containing a playlist
         :return: Returns a copy of the original :exc: 'Playlist': object
         """
+        json_obj = obj.get("data", obj)
+
         self.id = json_obj["uuid"]
         self.trn = f"trn:playlist:{self.id}"
         self.name = json_obj["title"]

tidalapi/session.py

@@ -361,7 +361,7 @@ def parse_v2_mix(self, obj: JsonObj) -> mix.Mix:
     def parse_playlist(self, obj: JsonObj) -> playlist.Playlist:
         """Parse a playlist from the given response."""
         # Note: When parsing playlists from v2 response, "data" field must be parsed
-        return self.playlist().parse(obj.get("data", obj))
+        return self.playlist().parse(obj)
 
     def parse_folder(self, obj: JsonObj) -> playlist.Folder:
         """Parse an album from the given response."""

tidalapi/user.py

@@ -37,6 +37,7 @@
     PlaylistOrder,
     VideoOrder,
 )
+from tidalapi.workers import get_items
 
 if TYPE_CHECKING:
     from tidalapi.album import Album
@@ -159,36 +160,6 @@ def playlists(self) -> List[Union["Playlist", "UserPlaylist"]]:
             ),
         )
 
-    def playlist_folders(
-        self, offset: int = 0, limit: int = 50, parent_folder_id: str = "root"
-    ) -> List["Folder"]:
-        """Get a list of folders created by the user.
-
-        :param offset: The amount of items you want returned.
-        :param limit: The index of the first item you want included.
-        :param parent_folder_id: Parent folder ID. Default: 'root' playlist folder
-        :return: Returns a list of :class:`~tidalapi.playlist.Folder` objects containing the Folders.
-        """
-        params = {
-            "folderId": parent_folder_id,
-            "offset": offset,
-            "limit": limit,
-            "order": "NAME",
-            "includeOnly": "FOLDER",
-        }
-        endpoint = "my-collection/playlists/folders"
-        return cast(
-            List["Folder"],
-            self.session.request.map_request(
-                url=urljoin(
-                    self.session.config.api_v2_location,
-                    endpoint,
-                ),
-                params=params,
-                parse=self.session.parse_folder,
-            ),
-        )
-
     def public_playlists(
         self, offset: int = 0, limit: int = 50
     ) -> List[Union["Playlist", "UserPlaylist"]]:
@@ -573,6 +544,19 @@ def remove_folders_playlists(
         )
         return response.ok
 
+    def artists_paginated(
+        self,
+        order: Optional[ArtistOrder] = None,
+        order_direction: Optional[OrderDirection] = None,
+    ) -> List["Artist"]:
+        """Get the users favorite artists, using pagination.
+
+        :param order: Optional; A :class:`ArtistOrder` describing the ordering type when returning the user favorite artists. eg.: "NAME, "DATE"
+        :param order_direction: Optional; A :class:`OrderDirection` describing the ordering direction when sorting by `order`. eg.: "ASC", "DESC"
+        :return: A :class:`list` :class:`~tidalapi.artist.Artist` objects containing the favorite artists.
+        """
+        return get_items(self.session.user.favorites.artists, order, order_direction)
+
     def artists(
         self,
         limit: Optional[int] = None,
@@ -603,6 +587,19 @@ def artists(
             ),
         )
 
+    def albums_paginated(
+        self,
+        order: Optional[AlbumOrder] = None,
+        order_direction: Optional[OrderDirection] = None,
+    ) -> List["Album"]:
+        """Get the users favorite albums, using pagination.
+
+        :param order: Optional; A :class:`AlbumOrder` describing the ordering type when returning the user favorite albums. eg.: "NAME, "DATE"
+        :param order_direction: Optional; A :class:`OrderDirection` describing the ordering direction when sorting by `order`. eg.: "ASC", "DESC"
+        :return: A :class:`list` :class:`~tidalapi.album.Album` objects containing the favorite albums.
+        """
+        return get_items(self.session.user.favorites.albums, order, order_direction)
+
     def albums(
         self,
         limit: Optional[int] = None,
@@ -631,17 +628,32 @@ def albums(
             ),
         )
 
+    def playlists_paginated(
+        self,
+        order: Optional[PlaylistOrder] = None,
+        order_direction: Optional[OrderDirection] = None,
+    ) -> List["Playlist"]:
+        """Get the users favorite playlists relative to the root folder, using
+        pagination.
+
+        :param order: Optional; A :class:`PlaylistOrder` describing the ordering type when returning the user favorite playlists. eg.: "NAME, "DATE"
+        :param order_direction: Optional; A :class:`OrderDirection` describing the ordering direction when sorting by `order`. eg.: "ASC", "DESC"
+        :return: A :class:`list` :class:`~tidalapi.playlist.Playlist` objects containing the favorite playlists.
+        """
+        return get_items(self.session.user.favorites.playlists, order, order_direction)
+
     def playlists(
         self,
         limit: Optional[int] = 50,
         offset: int = 0,
         order: Optional[PlaylistOrder] = None,
         order_direction: Optional[OrderDirection] = None,
     ) -> List["Playlist"]:
-        """Get the users favorite playlists (v2 endpoint)
+        """Get the users favorite playlists (v2 endpoint), relative to the root folder
+        This function is limited to 50 by TIDAL, requiring pagination.
 
-        :param limit: Optional; The amount of playlists you want returned.
-        :param offset: The index of the first playlist you want included.
+        :param limit: Optional; The number of playlists you want returned (Note: Cannot exceed 50)
+        :param offset: The index of the first playlist to fetch
         :param order: Optional; A :class:`PlaylistOrder` describing the ordering type when returning the user favorite playlists. eg.: "NAME, "DATE"
         :param order_direction: Optional; A :class:`OrderDirection` describing the ordering direction when sorting by `order`. eg.: "ASC", "DESC"
         :return: A :class:`list` :class:`~tidalapi.playlist.Playlist` objects containing the favorite playlists.
@@ -650,14 +662,14 @@ def playlists(
             "folderId": "root",
             "offset": offset,
             "limit": limit,
-            "includeOnly": "",
+            "includeOnly": "PLAYLIST",  # Include only PLAYLIST types, FOLDER will be ignored
         }
         if order:
             params["order"] = order.value
         if order_direction:
             params["orderDirection"] = order_direction.value
 
-        endpoint = "my-collection/playlists/folders"
+        endpoint = "my-collection/playlists"
         return cast(
             List["Playlist"],
             self.session.request.map_request(
@@ -670,6 +682,62 @@ def playlists(
             ),
         )
 
+    def playlist_folders(
+        self,
+        limit: Optional[int] = 50,
+        offset: int = 0,
+        order: Optional[PlaylistOrder] = None,
+        order_direction: Optional[OrderDirection] = None,
+        parent_folder_id: str = "root",
+    ) -> List["Folder"]:
+        """Get a list of folders created by the user.
+
+        :param limit: Optional; The number of playlists you want returned (Note: Cannot exceed 50)
+        :param offset: The index of the first playlist folder to fetch
+        :param order: Optional; A :class:`PlaylistOrder` describing the ordering type when returning the user favorite playlists. eg.: "NAME, "DATE"
+        :param order_direction: Optional; A :class:`OrderDirection` describing the ordering direction when sorting by `order`. eg.: "ASC", "DESC"
+        :param parent_folder_id: Parent folder ID. Default: 'root' playlist folder
+        :return: Returns a list of :class:`~tidalapi.playlist.Folder` objects containing the Folders.
+        """
+        params = {
+            "folderId": parent_folder_id,
+            "offset": offset,
+            "limit": limit,
+            "order": "NAME",
+            "includeOnly": "FOLDER",
+        }
+        if order:
+            params["order"] = order.value
+        if order_direction:
+            params["orderDirection"] = order_direction.value
+
+        endpoint = "my-collection/playlists/folders"
+        return cast(
+            List["Folder"],
+            self.session.request.map_request(
+                url=urljoin(
+                    self.session.config.api_v2_location,
+                    endpoint,
+                ),
+                params=params,
+                parse=self.session.parse_folder,
+            ),
+        )
+
+    def tracks_paginated(
+        self,
+        order: Optional[ItemOrder] = None,
+        order_direction: Optional[OrderDirection] = None,
+    ) -> List["Playlist"]:
+        """Get the users favorite playlists relative to the root folder, using
+        pagination.
+
+        :param order: Optional; A :class:`ItemOrder` describing the ordering type when returning the user favorite tracks. eg.: "NAME, "DATE"
+        :param order_direction: Optional; A :class:`OrderDirection` describing the ordering direction when sorting by `order`. eg.: "ASC", "DESC"
+        :return: A :class:`list` :class:`~tidalapi.playlist.Playlist` objects containing the favorite tracks.
+        """
+        return get_items(self.session.user.favorites.tracks, order, order_direction)
+
     def tracks(
         self,
         limit: Optional[int] = None,

tidalapi/workers.py

@@ -0,0 +1,64 @@
+import logging
+from concurrent.futures import ThreadPoolExecutor
+from typing import Callable
+
+log = logging.getLogger(__name__)
+
+
+def func_wrapper(args):
+    (f, offset, *args) = args
+    try:
+        items = f(*args)
+    except Exception as e:
+        log.error("Failed to run %s(offset=%d, args=%s)", f, offset, args)
+        log.exception(e)
+        items = []
+    return list((i + offset, item) for i, item in enumerate(items))
+
+
+def get_items(
+    func: Callable,
+    *args,
+    parse: Callable = lambda _: _,
+    chunk_size: int = 50,
+    processes: int = 2,
+):
+    """This function performs pagination on a function that supports `limit`/`offset`
+    parameters and it runs API requests in parallel to speed things up."""
+    items = []
+    offsets = [-chunk_size]
+    remaining = chunk_size * processes
+
+    with ThreadPoolExecutor(
+        processes, thread_name_prefix=f"mopidy-tidal-{func.__name__}-"
+    ) as pool:
+        while remaining == chunk_size * processes:
+            offsets = [offsets[-1] + chunk_size * (i + 1) for i in range(processes)]
+
+            pool_results = pool.map(
+                func_wrapper,
+                [
+                    (
+                        func,
+                        offset,
+                        chunk_size,  # limit
+                        offset,  # offset
+                        *args,  # extra args (e.g. order, order_direction)
+                    )
+                    for offset in offsets
+                ],
+            )
+
+            new_items = []
+            for results in pool_results:
+                new_items.extend(results)
+
+            remaining = len(new_items)
+            items.extend(new_items)
+
+    items = [_ for _ in items if _]
+    sorted_items = list(
+        map(lambda item: item[1], sorted(items, key=lambda item: item[0]))
+    )
+
+    return list(map(parse, sorted_items))