improved error handling + backoff strategy

Author: janlukasschroeder

sec_api/index.py

@@ -12,6 +12,10 @@
 exec_comp_api_endpoint = "https://api.sec-api.io/compensation"
 
 
+def handle_api_error(response):
+    raise Exception("API error: {} - {}".format(response.status_code, response.text))
+
+
 class QueryApi:
     """
     Base class for Query API
@@ -22,6 +26,8 @@ def __init__(self, api_key):
         self.api_endpoint = query_api_endpoint + "?token=" + api_key
 
     def get_filings(self, query):
+        response = {}
+
         # use backoff strategy to handle "too many requests" error.
         for x in range(3):
             response = requests.post(self.api_endpoint, json=query)
@@ -31,12 +37,9 @@ def get_filings(self, query):
                 # wait 500 * (x + 1) milliseconds and try again
                 time.sleep(0.5 * (x + 1))
             else:
-                raise Exception(
-                    "API error: " + str(response.status_code) + " - " + response.text
-                )
+                handle_api_error(response)
         else:
-            # request failed
-            raise Exception("API error")
+            handle_api_error(response)
 
 
 class FullTextSearchApi:
@@ -49,6 +52,8 @@ def __init__(self, api_key):
         self.api_endpoint = full_text_search_api_endpoint + "?token=" + api_key
 
     def get_filings(self, query):
+        response = {}
+
         # use backoff strategy to handle "too many requests" error.
         for x in range(3):
             response = requests.post(self.api_endpoint, json=query)
@@ -58,12 +63,9 @@ def get_filings(self, query):
                 # wait 500 * (x + 1) milliseconds and try again
                 time.sleep(0.5 * (x + 1))
             else:
-                raise Exception(
-                    "API error: " + str(response.status_code) + " - " + response.text
-                )
+                handle_api_error(response)
         else:
-            # request failed
-            raise Exception("API error")
+            handle_api_error(response)
 
 
 class RenderApi:
@@ -76,6 +78,7 @@ def __init__(self, api_key):
         self.api_endpoint = render_api_endpoint
 
     def get_filing(self, url):
+        response = {}
         filename = re.sub(r"https://www.sec.gov/Archives/edgar/data", "", url)
         _url = self.api_endpoint + filename + "?token=" + self.api_key
 
@@ -88,12 +91,9 @@ def get_filing(self, url):
                 # wait 500 * (x + 1) milliseconds and try again
                 time.sleep(0.5 * (x + 1))
             else:
-                raise Exception(
-                    "API error: " + str(response.status_code) + " - " + response.text
-                )
+                handle_api_error(response)
         else:
-            # request failed
-            raise Exception("API error")
+            handle_api_error(response)
 
 
 class XbrlApi:
@@ -110,6 +110,7 @@ def xbrl_to_json(self, htm_url="", xbrl_url="", accession_no=""):
             raise ValueError("htm_url, xbrl_url or accession_no must be present")
 
         _url = ""
+        response = {}
 
         if len(htm_url):
             _url = self.api_endpoint + "&htm-url=" + htm_url
@@ -123,19 +124,17 @@ def xbrl_to_json(self, htm_url="", xbrl_url="", accession_no=""):
         # use backoff strategy to handle "too many requests" error.
         for x in range(3):
             response = requests.get(_url)
+
             if response.status_code == 200:
                 data = json.loads(response.text)
                 return data
             elif response.status_code == 429:
                 # wait 500 * (x + 1) milliseconds and try again
                 time.sleep(0.5 * (x + 1))
             else:
-                raise Exception(
-                    "API error: " + str(response.status_code) + " - " + response.text
-                )
+                handle_api_error(response)
         else:
-            # request failed
-            raise Exception("API error")
+            handle_api_error(response)
 
 
 class ExtractorApi:
@@ -151,6 +150,7 @@ def get_section(self, filing_url="", section="1A", return_type="text"):
         if len(filing_url) == 0:
             raise ValueError("filing_url must be present")
 
+        response = {}
         _url = (
             self.api_endpoint
             + "&url="
@@ -162,20 +162,18 @@ def get_section(self, filing_url="", section="1A", return_type="text"):
         )
 
         # use backoff strategy to handle "too many requests" error.
-        for x in range(3):
+        for x in range(5):
             response = requests.get(_url)
+
             if response.status_code == 200:
                 return response.text
             elif response.status_code == 429:
                 # wait 500 * (x + 1) milliseconds and try again
                 time.sleep(0.5 * (x + 1))
             else:
-                raise Exception(
-                    "API error: " + str(response.status_code) + " - " + response.text
-                )
+                handle_api_error(response)
         else:
-            # request failed
-            raise Exception("API error")
+            handle_api_error(response)
 
 
 class MappingApi:
@@ -203,6 +201,7 @@ def resolve(self, parameter="", value=""):
         if not parameter.lower() in self.supported_parameters:
             raise ValueError("Parameter not supported")
 
+        response = {}
         _url = (
             self.api_endpoint
             + "/"
@@ -216,18 +215,16 @@ def resolve(self, parameter="", value=""):
         # use backoff strategy to handle "too many requests" error.
         for x in range(3):
             response = requests.get(_url)
+
             if response.status_code == 200:
                 return response.json()
             elif response.status_code == 429:
                 # wait 500 * (x + 1) milliseconds and try again
                 time.sleep(0.5 * (x + 1))
             else:
-                raise Exception(
-                    "API error: " + str(response.status_code) + " - " + response.text
-                )
+                handle_api_error(response)
         else:
-            # request failed
-            raise Exception("API error")
+            handle_api_error(response)
 
 
 class ExecCompApi:
@@ -248,6 +245,8 @@ def get_data(self, parameter=""):
         else:
             raise Exception("Invalid parameter")
 
+        response = {}
+
         # use backoff strategy to handle "too many requests" error.
         for x in range(3):
             if http_method == "GET":
@@ -269,9 +268,6 @@ def get_data(self, parameter=""):
                 # wait 500 * (x + 1) milliseconds and try again
                 time.sleep(0.5 * (x + 1))
             else:
-                raise Exception(
-                    "API error: " + str(response.status_code) + " - " + response.text
-                )
+                handle_api_error(response)
         else:
-            # request failed
-            raise Exception("API error")
+            handle_api_error(response)

setup.py

@@ -5,7 +5,7 @@
 
 setup(
     name="sec-api",
-    version="1.0.11",
+    version="1.0.12",
     author="SEC API",
     author_email="support@sec-api.io",
     description="SEC EDGAR Filings API",