enh: use HTTPAdapter with urlib3's Retry to handle connection issues

Author: paulmueller

CHANGELOG

@@ -1,4 +1,6 @@
 1.1.3
+ - enh: use `HTTPAdapter` with `urlib3`'s `Retry` to handle connection issues
+ - enh: use `requests.Session` when caching is disabled
  - enh: print UTC time in error messages to make debugging easier
 1.1.2
  - fix: archiving to BagIt format circle selection did not include all circles

dcoraid/api/ckan_api.py

@@ -9,10 +9,11 @@
 import random
 import traceback
 import urllib.parse
-
+from urllib3.util.retry import Retry
 
 from dclab.external.packaging import parse as parse_version
 import requests
+from requests.adapters import HTTPAdapter
 import requests_cache
 
 from .._version import version
@@ -34,6 +35,18 @@
 #: List of supported resource suffixes
 SERVER_RSUFFIX = {}
 
+# Configure retry strategy
+retry_strategy = Retry(
+    total=3,  # Total retries
+    backoff_factor=1.5,  # {backoff factor} * (2 ** ({number prev. retries}))
+    connect=3,  # Handle connection errors (e.g. because the server is busy)
+    read=0,  # Fail on read errors
+    redirect=False,  # implies raise_on_redirect=False
+    status=0,  # Fail on bad status codes
+    allowed_methods=["HEAD", "GET", "OPTIONS"]  # Retry only for safe methods
+)
+adapter = HTTPAdapter(max_retries=retry_strategy)
+
 
 class CKANAPI:
     def __init__(self,
@@ -107,7 +120,10 @@ def __init__(self,
                 },
                 **kwargs)
         else:
-            self.req_ses = requests
+            self.req_ses = requests.Session()
+
+        self.req_ses.mount("https://", adapter)
+        self.req_ses.mount("http://", adapter)
 
     def __repr__(self):
         return f"<CKANAPI {self.api_url} at {hex(id(self))}>"