Lazy-import requests and scrapinghub in utility functions

Author: jdemaeyer

shub/utils.py

@@ -19,17 +19,8 @@
 from six.moves.urllib.parse import urljoin
 
 import click
-import requests
 import yaml
 
-from scrapinghub import Connection, APIError
-
-try:
-    from scrapinghub import HubstorageClient
-except ImportError:
-    # scrapinghub < 1.9.0
-    from hubstorage import HubstorageClient
-
 import shub
 from shub.compat import to_native_str
 from shub.exceptions import (BadParameterException, InvalidAuthException,
@@ -77,6 +68,7 @@ def create_default_setup_py(**kwargs):
 
 
 def make_deploy_request(url, data, files, auth, verbose, keep_log):
+    import requests
     last_logs = deque(maxlen=LAST_N_LOGS)
     try:
         rsp = requests.post(url=url, auth=auth, data=data, files=files,
@@ -400,6 +392,11 @@ def get_job_specs(job):
 
 
 def get_job(job):
+    try:
+        from scrapinghub import HubstorageClient
+    except ImportError:
+        # scrapinghub < 1.9.0
+        from hubstorage import HubstorageClient
     jobid, apikey = get_job_specs(job)
     hsc = HubstorageClient(auth=apikey)
     job = hsc.get_job(jobid)
@@ -554,6 +551,7 @@ def latest_github_release(force_update=False, timeout=1., cache=None):
         # saved
         if release_data.get('_shub_last_update', 0) == today:
             return release_data
+    import requests
     release_data = requests.get(REQ_URL, timeout=timeout).json()
     release_data['_shub_last_update'] = today
     try:
@@ -646,6 +644,7 @@ def has_project_access(project, endpoint, apikey):
     """Check whether an API key has access to a given project. May raise
     InvalidAuthException if the API key is invalid (but not if it is valid but
     lacks access to the project)"""
+    from scrapinghub import Connection, APIError
     conn = Connection(apikey, url=endpoint)
     try:
         return project in conn.project_ids()

tests/test_utils.py

@@ -148,7 +148,7 @@ def test_get_job_specs_validates_jobid(self):
             with self.assertRaises(BadParameterException):
                 utils.get_job_specs(job_id)
 
-    @patch('shub.utils.HubstorageClient', autospec=True)
+    @patch('scrapinghub.HubstorageClient', autospec=True)
     def test_get_job(self, mock_HSC):
         class MockJob(object):
             metadata = {'some': 'val'}
@@ -271,7 +271,7 @@ def jri_result(follow, tail=None):
         job.resource.stats.return_value = {'totals': {'input_values': 1000}}
         self.assertEqual(jri_result(True, tail=3), [])
 
-    @patch('shub.utils.requests.get', autospec=True)
+    @patch('requests.get', autospec=True)
     def test_latest_github_release(self, mock_get):
         with self.runner.isolated_filesystem():
             mock_get.return_value.json.return_value = {'key': 'value'}
@@ -463,7 +463,7 @@ def call_update_yaml_dict():
             result = runner.invoke(call_update_yaml_dict)
         assert 'deprecated' in result.output
 
-    @patch('shub.utils.Connection')
+    @patch('scrapinghub.Connection')
     def test_has_project_access(self, mock_conn):
         mock_conn.return_value.project_ids.side_effect = APIError(
             'Authentication failed')