Push up the connection limit to param

Lyetenth · Lyetenth · commit 9bc98bb16066 · 2020-06-11T12:37:30.000-04:00
diff --git a/firststreet/api/adaptation.py b/firststreet/api/adaptation.py
@@ -16,34 +16,36 @@ class Adaptation(Api):
             get_summary: Retrieves a list of Adaptation Summary for the given list of IDs
         """
 
-    def get_detail(self, fsids, csv=False):
+    def get_detail(self, fsids, csv=False, limit=100):
         """Retrieves adaptation detail product data from the First Street Foundation API given a list of FSIDs and
         returns a list of Adaptation Detail objects.
 
         Args:
             fsids (list): A First Street ID
             csv (bool): To output extracted data to a csv or not
+            limit (int): max number of connections to make
         Returns:
             A list of Adaptation Detail
         """
 
         # Get data from api and create objects
-        api_datas = self.call_api(fsids, "adaptation", "detail", None)
+        api_datas = self.call_api(fsids, "adaptation", "detail", None, limit=limit)
         product = [AdaptationDetail(api_data) for api_data in api_datas]
 
         if csv:
             csv_format.to_csv(product, "adaptation", "detail")
 
         return product
 
-    def get_summary(self, fsids, location_type, csv=False):
+    def get_summary(self, fsids, location_type, csv=False, limit=100):
         """Retrieves adaptation summary product data from the First Street Foundation API given a list of FSIDs and
         returns a list of Adaptation Summary objects.
 
         Args:
             fsids (list): A First Street ID
             location_type (str): The location lookup type
             csv (bool): To output extracted data to a csv or not
+            limit (int): max number of connections to make
         Returns:
             A list of Adaptation Summary
         """
@@ -54,7 +56,7 @@ def get_summary(self, fsids, location_type, csv=False):
             raise TypeError("location is not a string")
 
         # Get data from api and create objects
-        api_datas = self.call_api(fsids, "adaptation", "summary", location_type)
+        api_datas = self.call_api(fsids, "adaptation", "summary", location_type, limit=limit)
         product = [AdaptationSummary(api_data) for api_data in api_datas]
 
         if csv:
diff --git a/firststreet/api/api.py b/firststreet/api/api.py
@@ -20,7 +20,7 @@ class Api:
     def __init__(self, http):
         self._http = http
 
-    def call_api(self, fsids, product, product_subtype, location):
+    def call_api(self, fsids, product, product_subtype, location, limit=100):
         """Receives FSIDs, a product, a product subtype, and a location to create and call an endpoint to the First
         Street Foundation API.
 
@@ -29,6 +29,7 @@ def call_api(self, fsids, product, product_subtype, location):
             product (str): The overall product to call
             product_subtype (str): The product subtype (if suitable)
             location (str/None): The location lookup type (if suitable)
+            limit (int): max number of connections to make
         Returns:
             A list of JSON responses
         """
@@ -51,6 +52,6 @@ def call_api(self, fsids, product, product_subtype, location):
 
         # Asynchronously call the API for each endpoint
         loop = asyncio.get_event_loop()
-        response = loop.run_until_complete(self._http.endpoint_execute(endpoints))
+        response = loop.run_until_complete(self._http.endpoint_execute(endpoints, limit))
 
         return response
diff --git a/firststreet/api/environmental.py b/firststreet/api/environmental.py
@@ -14,19 +14,20 @@ class Environmental(Api):
             get_precipitation: Retrieves a list of Environmental Precipitation for the given list of IDs
         """
 
-    def get_precipitation(self, fsids, csv=False):
+    def get_precipitation(self, fsids, csv=False, limit=100):
         """Retrieves environmental precipitation product data from the First Street Foundation API given a list of FSIDs
         and returns a list of Environmental Precipitation objects.
 
         Args:
             fsids (list): A First Street ID
             csv (bool): To output extracted data to a csv or not
+            limit (int): max number of connections to make
         Returns:
             A list of Adaptation Detail
         """
 
         # Get data from api and create objects
-        api_datas = self.call_api(fsids, "environmental", "precipitation", "county")
+        api_datas = self.call_api(fsids, "environmental", "precipitation", "county", limit=limit)
         product = [EnvironmentalPrecipitation(api_data) for api_data in api_datas]
 
         if csv:
diff --git a/firststreet/api/fema.py b/firststreet/api/fema.py
@@ -15,14 +15,15 @@ class Fema(Api):
             get_nfip: Retrieves a list of Fema Nfip for the given list of IDs
         """
 
-    def get_nfip(self, fsids, location_type, csv=False):
+    def get_nfip(self, fsids, location_type, csv=False, limit=100):
         """Retrieves fema nfip product data from the First Street Foundation API given a list of FSIDs and
         returns a list of Fema Nfip objects.
 
         Args:
             fsids (list): A First Street ID
             location_type (str): The location lookup type
             csv (bool): To output extracted data to a csv or not
+            limit (int): max number of connections to make
         Returns:
             A list of Fema Nfip
         Raises:
@@ -36,7 +37,7 @@ def get_nfip(self, fsids, location_type, csv=False):
             raise TypeError("location is not a string")
 
         # Get data from api and create objects
-        api_datas = self.call_api(fsids, "fema", "nfip", location_type)
+        api_datas = self.call_api(fsids, "fema", "nfip", location_type, limit=limit)
         product = [FemaNfip(api_data) for api_data in api_datas]
 
         if csv:
diff --git a/firststreet/api/historic.py b/firststreet/api/historic.py
@@ -16,34 +16,36 @@ class Historic(Api):
             get_summary: Retrieves a list of Historic Summary for the given list of IDs
         """
 
-    def get_event(self, fsids, csv=False):
+    def get_event(self, fsids, csv=False, limit=100):
         """Retrieves historic event product data from the First Street Foundation API given a list of FSIDs and
         returns a list of Historic Event objects.
 
         Args:
             fsids (list): A First Street ID
             csv (bool): To output extracted data to a csv or not
+            limit (int): max number of connections to make
         Returns:
             A list of Historic Event
         """
 
         # Get data from api and create objects
-        api_datas = self.call_api(fsids, "historic", "event", None)
+        api_datas = self.call_api(fsids, "historic", "event", None, limit)
         product = [HistoricEvent(api_data) for api_data in api_datas]
 
         if csv:
             csv_format.to_csv(product, "historic", "event")
 
         return product
 
-    def get_summary(self, fsids, location_type, csv=False):
+    def get_summary(self, fsids, location_type, csv=False, limit=100):
         """Retrieves historic summary product data from the First Street Foundation API given a list of FSIDs and
         returns a list of Historic Summary objects.
 
         Args:
             fsids (list): A First Street ID
             location_type (str): The location lookup type
             csv (bool): To output extracted data to a csv or not
+            limit (int): max number of connections to make
         Returns:
             A list of Historic Summary
         """
@@ -54,7 +56,7 @@ def get_summary(self, fsids, location_type, csv=False):
             raise TypeError("location is not a string")
 
         # Get data from api and create objects
-        api_datas = self.call_api(fsids, "historic", "summary", location_type)
+        api_datas = self.call_api(fsids, "historic", "summary", location_type, limit)
         product = [HistoricSummary(api_data) for api_data in api_datas]
 
         if csv:
diff --git a/firststreet/api/location.py b/firststreet/api/location.py
@@ -18,14 +18,15 @@ class Location(Api):
             get_summary: Retrieves a list of Location Summary for the given list of IDs
         """
 
-    def get_detail(self, fsids, location_type, csv=False):
+    def get_detail(self, fsids, location_type, csv=False, limit=100):
         """Retrieves location detail product data from the First Street Foundation API given a list of FSIDs and
         returns a list of Location Detail objects.
 
         Args:
             fsids (list): A First Street ID
             location_type (str): The location lookup type
             csv (bool): To output extracted data to a csv or not
+            limit (int): max number of connections to make
         Returns:
             A list of Location Detail
         Raises:
@@ -39,7 +40,7 @@ def get_detail(self, fsids, location_type, csv=False):
             raise TypeError("location is not a string")
 
         # Get data from api and create objects
-        api_datas = self.call_api(fsids, "location", "detail", location_type)
+        api_datas = self.call_api(fsids, "location", "detail", location_type, limit=limit)
 
         if location_type == 'property':
             product = [LocationDetailProperty(api_data) for api_data in api_datas]
@@ -73,14 +74,15 @@ def get_detail(self, fsids, location_type, csv=False):
 
         return product
 
-    def get_summary(self, fsids, location_type, csv=False):
+    def get_summary(self, fsids, location_type, csv=False, limit=100):
         """Retrieves location summary product data from the First Street Foundation API given a list of FSIDs and
         returns a list of Location Summary objects.
 
         Args:
             fsids (list): A First Street ID
             location_type (str): The location lookup type
             csv (bool): To output extracted data to a csv or not
+            limit (int): max number of connections to make
         Returns:
             A list of Location Summary
         Raises:
@@ -94,7 +96,7 @@ def get_summary(self, fsids, location_type, csv=False):
             raise TypeError("location is not a string")
 
         # Get data from api and create objects
-        api_datas = self.call_api(fsids, "location", "summary", location_type)
+        api_datas = self.call_api(fsids, "location", "summary", location_type, limit=limit)
 
         if location_type == "property":
             product = [LocationSummaryProperty(api_data) for api_data in api_datas]
diff --git a/firststreet/api/probability.py b/firststreet/api/probability.py
@@ -20,54 +20,57 @@ class Probability(Api):
             get_cumulative: Retrieves a list of Probability Depth for the given list of IDs
         """
 
-    def get_depth(self, fsids, csv=False):
+    def get_depth(self, fsids, csv=False, limit=100):
         """Retrieves probability depth product data from the First Street Foundation API given a list of FSIDs and
         returns a list of Probability Depth objects.
 
         Args:
             fsids (list): A First Street ID
             csv (bool): To output extracted data to a csv or not
+            limit (int): max number of connections to make
         Returns:
             A list of Probability Depth
         """
 
         # Get data from api and create objects
-        api_datas = self.call_api(fsids, "probability", "depth", "property")
+        api_datas = self.call_api(fsids, "probability", "depth", "property", limit=limit)
         product = [ProbabilityDepth(api_data) for api_data in api_datas]
 
         if csv:
             csv_format.to_csv(product, "probability", "depth")
 
         return product
 
-    def get_chance(self, fsids, csv=False):
+    def get_chance(self, fsids, csv=False, limit=100):
         """Retrieves probability chance product data from the First Street Foundation API given a list of FSIDs and
         returns a list of Probability Chance objects.
 
         Args:
             fsids (list): A First Street ID
             csv (bool): To output extracted data to a csv or not
+            limit (int): max number of connections to make
         Returns:
             A list of Probability Chance
         """
 
         # Get data from api and create objects
-        api_datas = self.call_api(fsids, "probability", "chance", "property")
+        api_datas = self.call_api(fsids, "probability", "chance", "property", limit=limit)
         product = [ProbabilityChance(api_data) for api_data in api_datas]
 
         if csv:
             csv_format.to_csv(product, "probability", "chance")
 
         return product
 
-    def get_count(self, fsids, location_type, csv=False):
+    def get_count(self, fsids, location_type, csv=False, limit=100):
         """Retrieves probability count product data from the First Street Foundation API given a list of FSIDs and
         returns a list of Probability Count objects.
 
         Args:
             fsids (list): A First Street ID
             location_type (str): The location lookup type
             csv (bool): To output extracted data to a csv or not
+            limit (int): max number of connections to make
         Returns:
             A list of Probability Count
         Raises:
@@ -81,47 +84,49 @@ def get_count(self, fsids, location_type, csv=False):
             raise TypeError("location is not a string")
 
         # Get data from api and create objects
-        api_datas = self.call_api(fsids, "probability", "count", location_type)
+        api_datas = self.call_api(fsids, "probability", "count", location_type, limit=limit)
         product = [ProbabilityCount(api_data) for api_data in api_datas]
 
         if csv:
             csv_format.to_csv(product, "probability", "count", location_type)
 
         return product
 
-    def get_count_summary(self, fsids, csv=False):
+    def get_count_summary(self, fsids, csv=False, limit=100):
         """Retrieves probability Count-Summary product data from the First Street Foundation API given a list of FSIDs
         and returns a list of Probability Count-Summary object.
 
         Args:
             fsids (list): A First Street ID
             csv (bool): To output extracted data to a csv or not
+            limit (int): max number of connections to make
         Returns:
             A list of Probability Count-Summary
         """
 
         # Get data from api and create objects
-        api_datas = self.call_api(fsids, "probability", "count-summary", "property")
+        api_datas = self.call_api(fsids, "probability", "count-summary", "property", limit)
         product = [ProbabilityCountSummary(api_data) for api_data in api_datas]
 
         if csv:
             csv_format.to_csv(product, "probability", "count-summary")
 
         return product
 
-    def get_cumulative(self, fsids, csv=False):
+    def get_cumulative(self, fsids, csv=False, limit=100):
         """Retrieves probability cumulative product data from the First Street Foundation API given a list of FSIDs and
         returns a list of Probability Cumulative object.
 
         Args:
             fsids (list): A First Street ID
             csv (bool): To output extracted data to a csv or not
+            limit (int): max number of connections to make
         Returns:
             A list of Probability Cumulative
         """
 
         # Get data from api and create objects
-        api_datas = self.call_api(fsids, "probability", "cumulative", "property")
+        api_datas = self.call_api(fsids, "probability", "cumulative", "property", limit=limit)
         product = [ProbabilityCumulative(api_data) for api_data in api_datas]
 
         if csv:
diff --git a/firststreet/http_util.py b/firststreet/http_util.py
@@ -40,15 +40,16 @@ def __init__(self, api_key, options=None):
                         }}
         self.version = SUMMARY_VERSION
 
-    async def endpoint_execute(self, endpoints):
+    async def endpoint_execute(self, endpoints, limit=100):
         """Asynchronously calls each endpoint and returns the JSON responses
         Args:
             endpoints (list): List of endpoints to get
+            limit (int): max number of connections to make
         Returns:
             The list of JSON responses corresponding to each endpoint
         """
 
-        connector = aiohttp.TCPConnector()
+        connector = aiohttp.TCPConnector(limit_per_host=limit)
         session = aiohttp.ClientSession(connector=connector)
 
         try:
