Bug fixes, performance fixes, and documentation updates

Author: Lyetenth

firststreet/__init__.py

@@ -41,7 +41,7 @@ class FirstStreet:
             MissingAPIError: If the API is not provided
     """
 
-    def __init__(self, api_key=None, connection_limit=100, rate_limit=20000, rate_period=1, version=None, log=True):
+    def __init__(self, api_key=None, connection_limit=100, rate_limit=4990, rate_period=60, version=None, log=True):
 
         if not api_key:
             raise MissingAPIKeyError('Missing API Key.')

firststreet/__main__.py

@@ -24,12 +24,11 @@
         parser.add_argument("-p", "--product", help="Example: adaptation_detail", required=False)
         parser.add_argument("-api_key", "--api_key", required=False)
         parser.add_argument("-v", "--version", required=False)
-        # Convert log default to an actual boolean
         parser.add_argument("-log", "--log", help="Example: False", required=False, default="True")
         parser.add_argument("-connection_limit", "--connection_limit", help="Example: 100",
                             required=False, default="100")
-        parser.add_argument("-rate_limit", "--rate_limit", help="Example: 5000", required=False, default="20000")
-        parser.add_argument("-rate_period", "--rate_period", help="Example: 3600", required=False, default="1")
+        parser.add_argument("-rate_limit", "--rate_limit", help="Example: 4990", required=False, default="4990")
+        parser.add_argument("-rate_period", "--rate_period", help="Example: 60", required=False, default="60")
         parser.add_argument("-o", "--output_dir", help="Example: /output", required=False)
         parser.add_argument("-s", "--search_items", help="Example: 28,29", required=False,)
         parser.add_argument("-l", "--location_type", help="Example: property", required=False)
@@ -156,16 +155,23 @@
                 sys.exit()
 
         # Adjust the connection variables
-        connection_adjust = input("Adjust connection parameters (Y/N)? Defaults to 100 connections, with a rate limit "
-                                  "of 5000 calls per 60 seconds: ")
-        if connection_adjust.lower() == "y":
-            argument.connection_limit = input("Input new connection limit (default 100): ")
-            argument.rate_limit = input("Input new rate limit (default 5000): ")
-            argument.rate_period = input("Input new rate period in seconds (default 60): ")
-        else:
-            argument.connection_limit = 100
-            argument.rate_limit = 5000
-            argument.rate_period = 60
+        if not argument.connection_limit and not argument.rate_limit and not argument.rate_period:
+            connection_adjust = input("Adjust connection parameters (Y/N)? Defaults to 100 connections, with a rate "
+                                      "limit of 4990 calls per 60 seconds: ")
+            if connection_adjust.lower() == "y":
+                input_params = input("Input new connection limit (default 100): ")
+                if input_params != '':
+                    argument.connection_limit = "providerid:{}".format(input_params)
+                input_params = input("Input new rate limit (default 5000): ")
+                if input_params != '':
+                    argument.rate_limit = "providerid:{}".format(input_params)
+                input_params = input("Input new rate period in seconds (default 60): ")
+                if input_params != '':
+                    argument.rate_period = "providerid:{}".format(input_params)
+            else:
+                argument.connection_limit = 100
+                argument.rate_limit = 4990
+                argument.rate_period = 60
 
         # Adjust the output directory
         if not argument.output_dir:
@@ -182,7 +188,12 @@
             if os.path.isfile(argument.search_items):
                 search_items = read_search_items_from_file(argument.search_items)
             else:
-                for search_item in argument.search_items.strip().split(";"):
+                items = argument.search_items.strip().split(";")
+                if len(items) == 1:
+                    logging.warning("Could not find the file '{}'. Treating the input as a search_item instead. "
+                                    "If this is unexpected, check the spelling or path of the input"
+                                    .format(argument.search_items))
+                for search_item in items:
                     try:
                         search_items.append(ast.literal_eval(search_item))
                     except (SyntaxError, ValueError):

firststreet/api/csv_format.py

@@ -6,10 +6,10 @@
 import logging
 import os
 import pathlib
-import shapely.geometry
 
 # External Imports
 import pandas as pd
+import shapely.geometry
 
 
 def to_csv(data, product, product_subtype, location_type=None, output_dir=None):

firststreet/api/location.py

@@ -41,7 +41,7 @@ def get_detail(self, search_items, location_type, csv=False, output_dir=None, ex
         """
 
         if not location_type:
-            raise InvalidArgument("No loocation type provided: {}".format(location_type))
+            raise InvalidArgument("No location type provided: {}".format(location_type))
         elif not isinstance(location_type, str):
             raise TypeError("location is not a string")
 

firststreet/http_util.py

@@ -141,7 +141,7 @@ async def tile_response(self, response, endpoint):
 
         if response.status != 200 and response.status != 500:
             raise self._network_error(self.options, rate_limit,
-                                      status=response.reason, message=response.status)
+                                      status=response.reason, endpoint=endpoint, message=response.status)
 
         elif response.status == 500:
             logging.info(
@@ -162,7 +162,7 @@ async def product_response(self, response, endpoint):
 
         try:
             if response.status != 200 and response.status != 404 and response.status != 500:
-                raise self._network_error(self.options, rate_limit, error=body.get('error'))
+                raise self._network_error(self.options, rate_limit, endpoint, error=body.get('error'))
 
             error = body.get("error")
             if error:
@@ -213,11 +213,12 @@ def _parse_rate_limit(headers):
                 'reset': headers.get('x-ratelimit-reset'), 'requestId': headers.get('x-request-id')}
 
     @staticmethod
-    def _network_error(options, rate_limit, error=None, status=None, message=None):
+    def _network_error(options, rate_limit, endpoint, error=None, status=None, message=None):
         """Handles any network errors as a result of the First Street Foundation API
         Args:
             options (dict): The options used in the header of the response
             rate_limit (dict): The rate limit information
+            endpoint (str): The failing endpoint
             error (dict): The body returned from the request call
             status (str): The status error from the response
             message (str): The message error from the response
@@ -229,10 +230,11 @@ def _network_error(options, rate_limit, error=None, status=None, message=None):
             message = error.get('message')
 
         if not status == 429:
-            formatted = "Network Error {}: {}".format(status, message)
+            formatted = "Network Error {}: {}. {}".format(status, message, endpoint)
         else:
-            formatted = "Network Error {}: {}. Limit: {}. Remaining: {}. Reset: {}".format(status,
+            formatted = "Network Error {}: {}. {}. Limit: {}. Remaining: {}. Reset: {}".format(status,
                                                                                            message,
+                                                                                           endpoint,
                                                                                            rate_limit.get('limit'),
                                                                                            rate_limit.get('remaining'),
                                                                                            rate_limit.get('reset'))

firststreet/models/geometry.py

@@ -1,7 +1,7 @@
 # Author: Kelvin Lai <kelvin@firststreet.org>
 # Copyright: This module is owned by First Street Foundation
 
-# Internal Imports
+# External Imports
 from shapely.geometry import shape