feat(proxy): implement proxy request handling and update site classes

CHANGES.md

@@ -15,7 +15,7 @@ Releases are also tagged in git, if that's helpful.
 The following changes are not yet released, but are code complete:
 
 Features:
--
+- Add residential proxies to refresh ip address when necessary #1616
 
 Changes:
 - Retrieve lower court information in `sc` #1569

juriscraper/AbstractSite.py

@@ -405,25 +405,40 @@ def _request_url_get(self, url):
         values
         """
         self.request["url"] = url
+
+        # check for proxy attribute and set proxies if present
+        proxies = None
+        if hasattr(self, "proxy") and self.proxy:
+            proxies = {"http": self.proxy, "https": self.proxy}
+
         self.request["response"] = self.request["session"].get(
             url,
             headers=self.request["headers"],
             verify=self.request["verify"],
             timeout=60,
+            proxies=proxies,
             **self.request["parameters"],
         )
+
         if self.save_response:
             self.save_response(self)
 
     def _request_url_post(self, url):
         """Execute POST request and assign appropriate request dictionary values"""
         self.request["url"] = url
+
+        # check for proxy attribute and set proxies if present
+        proxies = None
+        if hasattr(self, "proxy") and self.proxy:
+            proxies = {"http": self.proxy, "https": self.proxy}
+
         self.request["response"] = self.request["session"].post(
             url,
             headers=self.request["headers"],
             verify=self.request["verify"],
             data=self.parameters,
             timeout=60,
+            proxies=proxies,
             **self.request["parameters"],
         )
         if self.save_response:

juriscraper/OpinionSiteLinearProxyRequestHandler.py

@@ -0,0 +1,18 @@
+import os
+
+from juriscraper.OpinionSiteLinear import OpinionSiteLinear
+
+
+class OpinionSiteLinearProxyRequestHandler(OpinionSiteLinear):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def _download(self, request_dict=None):
+        USERNAME = os.environ.get("DECODO_USER")
+        PASSWORD = os.environ.get("DECODO_PASSWORD")
+        PROXY_ADDRESS = os.environ.get("DECODO_PROXY")
+        PORT = os.environ.get("DECODO_PORT")
+        proxy_values = [USERNAME, PASSWORD, PROXY_ADDRESS, PORT]
+        if not all(proxy_values) and not self.test_mode_enabled():
+            raise ValueError("Missing DECODO proxy environment variables")
+        return super()._download(request_dict)

juriscraper/opinions/united_states/state/okla.py

@@ -11,10 +11,12 @@
 
 from juriscraper.AbstractSite import logger
 from juriscraper.lib.html_utils import strip_bad_html_tags_insecure
-from juriscraper.OpinionSiteLinear import OpinionSiteLinear
+from juriscraper.OpinionSiteLinearProxyRequestHandler import (
+    OpinionSiteLinearProxyRequestHandler,
+)
 
 
-class Site(OpinionSiteLinear):
+class Site(OpinionSiteLinearProxyRequestHandler):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.court_id = self.__module__