Add lat/lng

Author: Lyetenth

firststreet/__main__.py

@@ -24,6 +24,7 @@
     parser.add_argument("-limit", "--limit", help="Example: 100", required=False, default="100")
     parser.add_argument("-log", "--log", help="Example: False", required=False, default="True")
     parser.add_argument("-f", "--file", help="Example: ./sample.txt", required=False)
+    parser.add_argument("-e", "--extra_param", required=False)
 
     argument = parser.parse_args()
 
@@ -58,49 +59,49 @@
         limit = int(argument.limit)
 
         if argument.product == 'adaptation.get_detail':
-            fs.adaptation.get_detail(search_items, csv=True, limit=limit)
+            fs.adaptation.get_detail(search_items, csv=True, limit=limit, extra_param=argument.extra_param)
 
         elif argument.product == 'adaptation.get_summary':
-            fs.adaptation.get_summary(search_items, argument.location, csv=True, limit=limit)
+            fs.adaptation.get_summary(search_items, argument.location, csv=True, limit=limit, extra_param=argument.extra_param)
 
         elif argument.product == 'adaptation.get_details_by_location':
-            fs.adaptation.get_details_by_location(search_items, argument.location, csv=True, limit=limit)
+            fs.adaptation.get_details_by_location(search_items, argument.location, csv=True, limit=limit, extra_param=argument.extra_param)
 
         elif argument.product == 'probability.get_depth':
-            fs.probability.get_depth(search_items, csv=True, limit=limit)
+            fs.probability.get_depth(search_items, csv=True, limit=limit, extra_param=argument.extra_param)
 
         elif argument.product == 'probability.get_chance':
-            fs.probability.get_chance(search_items, csv=True, limit=limit)
+            fs.probability.get_chance(search_items, csv=True, limit=limit, extra_param=argument.extra_param)
 
         elif argument.product == 'probability.get_count_summary':
-            fs.probability.get_count_summary(search_items, csv=True, limit=limit)
+            fs.probability.get_count_summary(search_items, csv=True, limit=limit, extra_param=argument.extra_param)
 
         elif argument.product == 'probability.get_cumulative':
-            fs.probability.get_cumulative(search_items, csv=True, limit=limit)
+            fs.probability.get_cumulative(search_items, csv=True, limit=limit, extra_param=argument.extra_param)
 
         elif argument.product == 'probability.get_count':
-            fs.probability.get_count(search_items, argument.location, csv=True, limit=limit)
+            fs.probability.get_count(search_items, argument.location, csv=True, limit=limit, extra_param=argument.extra_param)
 
         elif argument.product == 'historic.get_event':
-            fs.historic.get_event(search_items, csv=True, limit=limit)
+            fs.historic.get_event(search_items, csv=True, limit=limit, extra_param=argument.extra_param)
 
         elif argument.product == 'historic.get_summary':
-            fs.historic.get_summary(search_items, argument.location, csv=True, limit=limit)
+            fs.historic.get_summary(search_items, argument.location, csv=True, limit=limit, extra_param=argument.extra_param)
 
         elif argument.product == 'historic.get_events_by_location':
-            fs.historic.get_events_by_location(search_items, argument.location, csv=True, limit=limit)
+            fs.historic.get_events_by_location(search_items, argument.location, csv=True, limit=limit, extra_param=argument.extra_param)
 
         elif argument.product == 'location.get_detail':
-            fs.location.get_detail(search_items, argument.location, csv=True, limit=limit)
+            fs.location.get_detail(search_items, argument.location, csv=True, limit=limit, extra_param=argument.extra_param)
 
         elif argument.product == 'location.get_summary':
-            fs.location.get_summary(search_items, argument.location, csv=True, limit=limit)
+            fs.location.get_summary(search_items, argument.location, csv=True, limit=limit, extra_param=argument.extra_param)
 
         elif argument.product == 'fema.get_nfip':
-            fs.fema.get_nfip(search_items, argument.location, csv=True, limit=limit)
+            fs.fema.get_nfip(search_items, argument.location, csv=True, limit=limit, extra_param=argument.extra_param)
 
         elif argument.product == 'environmental.get_precipitation':
-            fs.environmental.get_precipitation(search_items, csv=True, limit=limit)
+            fs.environmental.get_precipitation(search_items, csv=True, limit=limit, extra_param=argument.extra_param)
 
         else:
             logging.error("Product not found. Please check that the argument"

firststreet/api/adaptation.py

@@ -19,7 +19,7 @@ class Adaptation(Api):
             get_summary: Retrieves a list of Adaptation Summary for the given list of IDs
         """
 
-    def get_detail(self, search_item, csv=False, limit=100, output_dir=None):
+    def get_detail(self, search_item, csv=False, limit=100, output_dir=None, extra_param=None):
         """Retrieves adaptation detail product data from the First Street Foundation API given a list of search_items
          and returns a list of Adaptation Detail objects.
 
@@ -29,11 +29,13 @@ def get_detail(self, search_item, csv=False, limit=100, output_dir=None):
             csv (bool): To output extracted data to a csv or not
             limit (int): max number of connections to make
             output_dir (str): The output directory to save the generated csvs
+            extra_param (str): Extra parameter to be added to the url
+
         Returns:
             A list of Adaptation Detail
         """
         # Get data from api and create objects
-        api_datas = self.call_api(search_item, "adaptation", "detail", None, limit=limit)
+        api_datas = self.call_api(search_item, "adaptation", "detail", None, limit=limit, extra_param=extra_param)
         product = [AdaptationDetail(api_data) for api_data in api_datas]
 
         if csv:
@@ -43,7 +45,8 @@ def get_detail(self, search_item, csv=False, limit=100, output_dir=None):
 
         return product
 
-    def get_details_by_location(self, search_item, location_type, csv=False, limit=100, output_dir=None):
+    def get_details_by_location(self, search_item, location_type, csv=False, limit=100, output_dir=None,
+                                extra_param=None):
         """Retrieves adaptation detail product data from the First Street Foundation API given a list of location
         search_items and returns a list of Adaptation Detail objects.
 
@@ -54,6 +57,8 @@ def get_details_by_location(self, search_item, location_type, csv=False, limit=1
             csv (bool): To output extracted data to a csv or not
             limit (int): max number of connections to make
             output_dir (str): The output directory to save the generated csvs
+            extra_param (str): Extra parameter to be added to the url
+
         Returns:
             A list of list of Adaptation Summary and Adaptation Detail
         Raises:
@@ -67,28 +72,31 @@ def get_details_by_location(self, search_item, location_type, csv=False, limit=1
             raise TypeError("location is not a string")
 
         # Get data from api and create objects
-        api_datas_summary = self.call_api(search_item, "adaptation", "summary", location_type, limit=limit)
+        api_datas_summary = self.call_api(search_item, "adaptation", "summary", location_type, limit=limit,
+                                          extra_param=extra_param)
         summary = [AdaptationSummary(api_data) for api_data in api_datas_summary]
 
         search_items = list(set([adaptation for sum_adap in summary if sum_adap.adaptation for
-                          adaptation in sum_adap.adaptation]))
+                                 adaptation in sum_adap.adaptation]))
 
         if search_items:
-            api_datas_detail = self.call_api(search_items, "adaptation", "detail", None, limit=limit)
+            api_datas_detail = self.call_api(search_items, "adaptation", "detail", None, limit=limit,
+                                             extra_param=extra_param)
 
         else:
             api_datas_detail = [{"adaptationId": None}]
 
         detail = [AdaptationDetail(api_data) for api_data in api_datas_detail]
 
         if csv:
-            csv_format.to_csv([summary, detail], "adaptation", "summary_detail", location_type, output_dir=output_dir)
+            csv_format.to_csv([summary, detail], "adaptation", "summary_detail", location_type,
+                              output_dir=output_dir)
 
         logging.info("Adaptation Summary Detail Data Ready.")
 
         return [summary, detail]
 
-    def get_summary(self, search_item, location_type, csv=False, limit=100, output_dir=None):
+    def get_summary(self, search_item, location_type, csv=False, limit=100, output_dir=None, extra_param=None):
         """Retrieves adaptation summary product data from the First Street Foundation API given a list of
         search_items and returns a list of Adaptation Summary objects.
 
@@ -99,6 +107,8 @@ def get_summary(self, search_item, location_type, csv=False, limit=100, output_d
             csv (bool): To output extracted data to a csv or not
             limit (int): max number of connections to make
             output_dir (str): The output directory to save the generated csvs
+            extra_param (str): Extra parameter to be added to the url
+
         Returns:
             A list of Adaptation Summary
         Raises:
@@ -112,7 +122,8 @@ def get_summary(self, search_item, location_type, csv=False, limit=100, output_d
             raise TypeError("location is not a string")
 
         # Get data from api and create objects
-        api_datas = self.call_api(search_item, "adaptation", "summary", location_type, limit=limit)
+        api_datas = self.call_api(search_item, "adaptation", "summary", location_type, limit=limit,
+                                  extra_param=extra_param)
         product = [AdaptationSummary(api_data) for api_data in api_datas]
 
         if csv:

firststreet/api/api.py

@@ -23,7 +23,7 @@ class Api:
     def __init__(self, http):
         self._http = http
 
-    def call_api(self, search_item, product, product_subtype, location, limit=100):
+    def call_api(self, search_item, product, product_subtype, location, limit=100, extra_param=None):
         """Receives an item, a product, a product subtype, and a location to create and call an endpoint to the First
         Street Foundation API.
 
@@ -34,6 +34,7 @@ def call_api(self, search_item, product, product_subtype, location, limit=100):
             product_subtype (str): The product subtype (if suitable)
             location (str/None): The location type (if suitable)
             limit (int): max number of connections to make
+            extra_param (str): Extra parameter to be added to the url
         Returns:
             A list of JSON responses
         """
@@ -71,15 +72,15 @@ def call_api(self, search_item, product, product_subtype, location, limit=100):
 
             # fsid
             if isinstance(item, int):
-                endpoint = endpoint + "/{}".format(item)
+                endpoint = endpoint + "/{}".format(item) + "?{}".format(extra_param)
 
             # lat/lng
             elif isinstance(item, tuple):
-                endpoint = endpoint + "?lat={}&lng={}".format(item[0], item[1])
+                endpoint = endpoint + "?lat={}&lng={}&{}".format(item[0], item[1], extra_param)
 
             # address
             elif isinstance(item, str):
-                endpoint = endpoint + "?address={}".format(item)
+                endpoint = endpoint + "?address={}&{}".format(item, extra_param)
 
             endpoints.append((endpoint, item, product, product_subtype))
 

firststreet/api/csv_format.py

@@ -5,6 +5,7 @@
 import datetime
 import logging
 import os
+import shapely.geometry
 
 # External Imports
 import pandas as pd
@@ -146,6 +147,25 @@ def to_csv(data, product, product_subtype, location_type=None, output_dir=None):
     logging.info("CSV generated to '{}'.".format(output_dir + '/' + file_name))
 
 
+def get_geom_center(geom):
+
+    if hasattr(geom, "center"):
+        if isinstance(geom.center, shapely.geometry.MultiPolygon):
+            return {"latitude": geom.center.centroid.y, "longitude": geom.center.centroid.x}
+
+        elif isinstance(geom.center, shapely.geometry.Point):
+            return {"latitude": geom.center.y, "longitude": geom.center.x}
+
+        else:
+            raise NotImplementedError
+
+    elif hasattr(geom, "y") and hasattr(geom, "x"):
+        return {"latitude": geom.y, "longitude": geom.x}
+
+    return {"latitude": None, "longitude": None}
+
+
+
 def format_adaptation_detail(data):
     """Reformat the list of data to Adaptation Detail format
 
@@ -155,10 +175,12 @@ def format_adaptation_detail(data):
         A pandas formatted DataFrame
     """
     df = pd.DataFrame([vars(o) for o in data]).explode('type').explode('scenario').reset_index(drop=True)
-    df.drop(["serving", "geometry"], axis=1, inplace=True)
     df['adaptationId'] = df['adaptationId'].astype('Int64').apply(str)
     df['returnPeriod'] = df['returnPeriod'].astype('Int64').apply(str)
-    return df[['adaptationId', 'name', 'type', 'scenario', 'conveyance', 'returnPeriod']]
+    df['geometry'] = df['geometry'].apply(get_geom_center)
+    df = pd.concat([df.drop(['geometry'], axis=1), df['geometry'].apply(pd.Series)], axis=1)
+
+    return df[['adaptationId', 'name', 'type', 'scenario', 'conveyance', 'returnPeriod', 'latitude', 'longitude']]
 
 
 def format_adaptation_summary(data):
@@ -410,7 +432,6 @@ def format_historic_event(data):
         A pandas formatted DataFrame
     """
     df = pd.DataFrame([vars(o) for o in data])
-    df.drop(["geometry"], axis=1, inplace=True)
     if not df['properties'].isna().values.all():
         df = pd.concat([df.drop(['properties'], axis=1), df['properties'].apply(pd.Series)], axis=1)
         df['eventId'] = df['eventId'].astype('Int64').apply(str)
@@ -425,8 +446,11 @@ def format_historic_event(data):
         df.drop(['properties'], axis=1, inplace=True)
         df['propertiesTotal'] = pd.NA
         df['propertiesAffected'] = pd.NA
+    df['geometry'] = df['geometry'].apply(get_geom_center)
+    df = pd.concat([df.drop(['geometry'], axis=1), df['geometry'].apply(pd.Series)], axis=1)
 
-    return df[['eventId', 'name', 'month', 'year', 'returnPeriod', 'type', 'propertiesTotal', 'propertiesAffected']]
+    return df[['eventId', 'name', 'month', 'year', 'returnPeriod', 'type', 'propertiesTotal',
+               'propertiesAffected', 'latitude', 'longitude']]
 
 
 def format_historic_summary_property(data):
@@ -567,9 +591,11 @@ def format_location_detail_property(data):
     df['state_fips'] = df['state_fips'].astype('Int64').apply(str).apply(lambda x: x.zfill(2))
     df['footprintId'] = df['footprintId'].astype('Int64').apply(str)
     df['elevation'] = df['city_fips'].apply(str)
+    df['geometry'] = df['geometry'].apply(get_geom_center)
+    df = pd.concat([df.drop(['geometry'], axis=1), df['geometry'].apply(pd.Series)], axis=1)
     return df[['fsid', 'streetNumber', 'route', 'city_fips', 'city_name', 'zipCode', 'neighborhood_fips',
                'neighborhood_name', 'tract_fips', 'county_fips', 'county_name', 'cd_fips',
-               'cd_name', 'state_fips', 'state_name', 'footprintId', 'elevation', 'fema']]
+               'cd_name', 'state_fips', 'state_name', 'footprintId', 'elevation', 'fema', 'latitude', 'longitude']]
 
 
 def format_location_detail_neighborhood(data):
@@ -612,8 +638,10 @@ def format_location_detail_neighborhood(data):
     df['city_fips'] = df['city_fips'].astype('Int64').apply(str)
     df['county_fips'] = df['county_fips'].astype('Int64').apply(str)
     df['state_fips'] = df['state_fips'].astype('Int64').apply(str).apply(lambda x: x.zfill(2))
+    df['geometry'] = df['geometry'].apply(get_geom_center)
+    df = pd.concat([df.drop(['geometry'], axis=1), df['geometry'].apply(pd.Series)], axis=1)
     return df[['fsid', 'name', 'city_fips', 'city_name', 'county_fips', 'county_name', 'subtype',
-               'state_fips', 'state_name']]
+               'state_fips', 'state_name', 'latitude', 'longitude']]
 
 
 def format_location_detail_city(data):
@@ -666,8 +694,10 @@ def format_location_detail_city(data):
     df['county_fips'] = df['county_fips'].astype('Int64').apply(str)
     df['neighborhood_fips'] = df['neighborhood_fips'].astype('Int64').apply(str)
     df['state_fips'] = df['state_fips'].astype('Int64').apply(str).apply(lambda x: x.zfill(2))
+    df['geometry'] = df['geometry'].apply(get_geom_center)
+    df = pd.concat([df.drop(['geometry'], axis=1), df['geometry'].apply(pd.Series)], axis=1)
     return df[['fsid', 'name', 'lsad', 'zipCode', 'neighborhood_fips', 'neighborhood_name',
-               'county_fips', 'county_name', 'state_fips', 'state_name']]
+               'county_fips', 'county_name', 'state_fips', 'state_name', 'latitude', 'longitude']]
 
 
 def format_location_detail_zcta(data):
@@ -710,7 +740,10 @@ def format_location_detail_zcta(data):
     df['city_fips'] = df['city_fips'].astype('Int64').apply(str)
     df['county_fips'] = df['county_fips'].astype('Int64').apply(str)
     df['state_fips'] = df['state_fips'].astype('Int64').apply(str).apply(lambda x: x.zfill(2))
-    return df[['fsid', 'name', 'city_fips', 'city_name', 'county_fips', 'county_name', 'state_fips', 'state_name']]
+    df['geometry'] = df['geometry'].apply(get_geom_center)
+    df = pd.concat([df.drop(['geometry'], axis=1), df['geometry'].apply(pd.Series)], axis=1)
+    return df[['fsid', 'name', 'city_fips', 'city_name', 'county_fips', 'county_name', 'state_fips',
+               'state_name', 'latitude', 'longitude']]
 
 
 def format_location_detail_tract(data):
@@ -744,7 +777,9 @@ def format_location_detail_tract(data):
     df['fsid'] = df['fsid'].apply(str)
     df['county_fips'] = df['county_fips'].astype('Int64').apply(str)
     df['state_fips'] = df['state_fips'].astype('Int64').apply(str).apply(lambda x: x.zfill(2))
-    return df[['fsid', 'fips', 'county_fips', 'county_name', 'state_fips', 'state_name']]
+    df['geometry'] = df['geometry'].apply(get_geom_center)
+    df = pd.concat([df.drop(['geometry'], axis=1), df['geometry'].apply(pd.Series)], axis=1)
+    return df[['fsid', 'fips', 'county_fips', 'county_name', 'state_fips', 'state_name', 'latitude', 'longitude']]
 
 
 def format_location_detail_county(data):
@@ -797,8 +832,10 @@ def format_location_detail_county(data):
     df['zipCode'] = df['zipCode'].astype('Int64').apply(str)
     df['cd_fips'] = df['cd_fips'].astype('Int64').apply(str)
     df['state_fips'] = df['state_fips'].astype('Int64').apply(str).apply(lambda x: x.zfill(2))
+    df['geometry'] = df['geometry'].apply(get_geom_center)
+    df = pd.concat([df.drop(['geometry'], axis=1), df['geometry'].apply(pd.Series)], axis=1)
     return df[['fsid', 'city_fips', 'city_name', 'zipCode', 'fips', 'isCoastal', 'cd_fips',
-               'cd_name', 'state_fips', 'state_name']]
+               'cd_name', 'state_fips', 'state_name', 'latitude', 'longitude']]
 
 
 def format_location_detail_cd(data):
@@ -832,7 +869,9 @@ def format_location_detail_cd(data):
     df['fsid'] = df['fsid'].apply(str)
     df['county_fips'] = df['county_fips'].astype('Int64').apply(str)
     df['state_fips'] = df['state_fips'].astype('Int64').apply(str).apply(lambda x: x.zfill(2))
-    return df[['fsid', 'district', 'county_fips', 'county_name', 'state_fips', 'state_name']]
+    df['geometry'] = df['geometry'].apply(get_geom_center)
+    df = pd.concat([df.drop(['geometry'], axis=1), df['geometry'].apply(pd.Series)], axis=1)
+    return df[['fsid', 'district', 'county_fips', 'county_name', 'state_fips', 'state_name', 'latitude', 'longitude']]
 
 
 def format_location_detail_state(data):
@@ -845,7 +884,9 @@ def format_location_detail_state(data):
     """
     df = pd.DataFrame([vars(o) for o in data])
     df['fsid'] = df['fsid'].apply(str)
-    return df[['fsid', 'name', 'fips']]
+    df['geometry'] = df['geometry'].apply(get_geom_center)
+    df = pd.concat([df.drop(['geometry'], axis=1), df['geometry'].apply(pd.Series)], axis=1)
+    return df[['fsid', 'name', 'fips', 'latitude', 'longitude']]
 
 
 def format_location_summary_property(data):