Skip to content

Commit 9e78b17

Browse files
Apply suggestions from code review
Improve robustness, reduce number of workers to 2. Co-authored-by: Fabio Manganiello <[email protected]>
1 parent ba20831 commit 9e78b17

File tree

1 file changed

+10
-2
lines changed

1 file changed

+10
-2
lines changed

tidalapi/workers.py

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,18 @@
11
from concurrent.futures import ThreadPoolExecutor
22
from typing import Callable
3+
import logging
4+
5+
log = logging.getLogger(__name__)
36

47

58
def func_wrapper(args):
69
(f, offset, *args) = args
7-
items = f(*args)
10+
try:
11+
items = f(*args)
12+
except Exception as e:
13+
log.error("Failed to run %s(offset=%d, args=%s)", f, offset, args)
14+
log.exception(e)
15+
items = []
816
return list((i + offset, item) for i, item in enumerate(items))
917

1018

@@ -13,7 +21,7 @@ def get_items(
1321
*args,
1422
parse: Callable = lambda _: _,
1523
chunk_size: int = 50,
16-
processes: int = 5,
24+
processes: int = 2,
1725
):
1826
"""This function performs pagination on a function that supports `limit`/`offset`
1927
parameters and it runs API requests in parallel to speed things up."""

0 commit comments

Comments
 (0)