Merge pull request #3 from singular-labs/504_refactor

Pool connections and manage retries properly

Author: shoote

singular_api_client/etl_manager.py

@@ -103,6 +103,17 @@ def __init__(self, api_key,
         self.max_update_window_days = max_update_window_days
         self.state = self.load_state()
         self.should_stop = False
+        session = requests.Session()
+        retry = Retry(
+            connect=5,
+            backoff_factor=0.5,
+            status_forcelist=(500, 502, 504),
+            method_whitelist=('GET', 'POST')
+        )
+        adapter = HTTPAdapter(max_retries=retry)
+        session.mount('http://', adapter)
+        session.mount('https://', adapter)
+        self.session = session
 
     def handle_new_data(self, source, date, download_url, report_id):
         """
@@ -124,7 +135,7 @@ def handle_new_data(self, source, date, download_url, report_id):
         :param report_id: Singular Internal Report ID, can be used for auditing
         :type date: str
         """
-        r = requests.get(download_url)
+        r = self.session.get(download_url)
         if not r.ok:
             if r.status_code is None or (500 <= r.status_code < 600):
                 raise UnexpectedAPIException("unexpected error when downloading report_id=%s, url=%s" % (

singular_api_client/singular_client.py

@@ -23,6 +23,17 @@ class SingularClient(object):
 
     def __init__(self, api_key):
         self.api_key = api_key
+        session = requests.Session()
+        retry = Retry(
+            connect=5,
+            backoff_factor=0.5,
+            status_forcelist=(500, 502, 504),
+            method_whitelist=('GET', 'POST')
+        )
+        adapter = HTTPAdapter(max_retries=retry)
+        session.mount('http://', adapter)
+        session.mount('https://', adapter)
+        self.session = session
 
     def run_report(self, start_date, end_date,
                    format=Format.JSON,
@@ -340,7 +351,7 @@ def __api_request(self, method, endpoint, **kwargs):
         headers = {"Authorization": self.api_key,
                    'User-Agent': 'Singular API Client v%s' % __version__}
 
-        response = requests.request(method, url, headers=headers, **kwargs)
+        response = self.session.request(method, url, headers=headers, **kwargs)
 
         logger.info("%(method)s %(url)s, kwargs = %(kwargs)s --> code = %(code)s" %
                     dict(method=method, url=url, kwargs=repr(kwargs), code=response.status_code))

singular_api_client/version.py

@@ -1 +1 @@
-__version__ = "0.5.7"
+__version__ = "0.5.8"