Test cleanup

Author: Lyetenth

.circleci/config.yml

@@ -0,0 +1,36 @@
+version: 1
+jobs:
+  build:
+    docker:
+      - image: circleci/python:3.6
+
+    working_directory: ~/fsf-api-access-python
+
+    steps:
+      - checkout
+
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "requirements.txt" }}
+
+      - run:
+          name: install dependencies
+          command: |
+            python3 -m venv venv
+            . venv/bin/activate
+            pip install -e .[testing]
+
+      - save_cache:
+          paths:
+            - ./venv
+          key: v1-dependencies-{{ checksum "requirements.txt" }}
+
+      - run:
+          name: run tests
+          command: |
+            . venv/bin/activate
+            pytest tests --basetemp=C:/Users/Lyetenth/Documents/FSF/flood_lab/tests/temp
+
+      - store_artifacts:
+          path: test-reports
+          destination: test-reports

README.md

@@ -1,6 +1,14 @@
 # First Street Foundation API Access Documentation (Python)
+[![CircleCI](https://circleci.com/gh/FirstStreet/firststreet-python.svg?style=svg&circle-token=9f4c22fd34a87690970add09e6454f17764812b4)](https://circleci.com/gh/FirstStreet/firststreet-python)
+
 The First Street Foundation API Access (Python) is a wrapper used to bulk extract flood data from the First Street Foundation API
 
+
+### Current Release:
+Current release: 0.1
+
+**Notice:** This API wrapper is subject to change.
+
 # Table of contents
 - **[Installation](#installation)**
   * [Running the Project - Method 1: Through the Command Line](#method1)
@@ -20,6 +28,7 @@ The First Street Foundation API Access (Python) is a wrapper used to bulk extrac
   - [CSV File Name:](#csv-name)
   - [CSV File Content](#csv-content)
 - **[Updating the Project to the Newest Version:](#updating)**
+- **[License](#license)**
 
 <a name="installation"></a>
 # Installation
@@ -434,6 +443,31 @@ If an update is made to this project, you will need to pull the changes from git
 
 3. The project should now be updated to the newest version
 
+<a name="license"></a>
+# License
+```  
+MIT License
+
+Copyright (c) 2020 First Street Foundation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+```
 
 
 [git]: <https://git-scm.com/downloads>

conftest.py

@@ -1 +1,24 @@
+# Author: Kelvin Lai <kelvin@firststreet.org>
+# Copyright: This module is owned by First Street Foundation
+
+# External Imports
+import pytest
+
 pytest_plugins = 'aiohttp.pytest_plugin'
+
+
+def pytest_addoption(parser):
+    """Sets up command line arguments for pytest to skip the stress test
+    """
+    parser.addoption("--runstress", action="store_true", default=False, help="runs the stress test")
+
+
+def pytest_collection_modifyitems(config, items):
+    """Skips the stress test if the argument is not set
+    """
+    if config.getoption("--runstress"):
+        return
+    skip_stress = pytest.mark.skip(reason="need --runstress option to run")
+    for item in items:
+        if "stress" in item.keywords:
+            item.add_marker(skip_stress)

firststreet/__main__.py

@@ -6,10 +6,9 @@
 import os
 import logging
 from distutils.util import strtobool
-
-# Internal Imports
 import sys
 
+# Internal Imports
 import firststreet
 from firststreet.errors import InvalidArgument
 from firststreet.util import read_fsid_file

firststreet/api/adaptation.py

@@ -19,14 +19,15 @@ class Adaptation(Api):
             get_summary: Retrieves a list of Adaptation Summary for the given list of IDs
         """
 
-    def get_detail(self, fsids, csv=False, limit=100):
+    def get_detail(self, fsids, csv=False, limit=100, output_dir=None):
         """Retrieves adaptation detail product data from the First Street Foundation API given a list of FSIDs and
         returns a list of Adaptation Detail objects.
 
         Args:
             fsids (list/file): A First Street Foundation IDs or a file of First Street Foundation IDs
             csv (bool): To output extracted data to a csv or not
             limit (int): max number of connections to make
+            output_dir (str): The output directory to save the generated csvs
         Returns:
             A list of Adaptation Detail
         """
@@ -35,13 +36,13 @@ def get_detail(self, fsids, csv=False, limit=100):
         product = [AdaptationDetail(api_data) for api_data in api_datas]
 
         if csv:
-            csv_format.to_csv(product, "adaptation", "detail")
+            csv_format.to_csv(product, "adaptation", "detail", output_dir=output_dir)
 
         logging.info("Adaptation Detail Data Ready.")
 
         return product
 
-    def get_summary(self, fsids, location_type, csv=False, limit=100):
+    def get_summary(self, fsids, location_type, csv=False, limit=100, output_dir=None):
         """Retrieves adaptation summary product data from the First Street Foundation API given a list of FSIDs and
         returns a list of Adaptation Summary objects.
 
@@ -50,6 +51,7 @@ def get_summary(self, fsids, location_type, csv=False, limit=100):
             location_type (str): The location lookup type
             csv (bool): To output extracted data to a csv or not
             limit (int): max number of connections to make
+            output_dir (str): The output directory to save the generated csvs
         Returns:
             A list of Adaptation Summary
         """
@@ -64,7 +66,7 @@ def get_summary(self, fsids, location_type, csv=False, limit=100):
         product = [AdaptationSummary(api_data) for api_data in api_datas]
 
         if csv:
-            csv_format.to_csv(product, "adaptation", "summary", location_type)
+            csv_format.to_csv(product, "adaptation", "summary", location_type, output_dir=output_dir)
 
         logging.info("Adaptation Summary Data Ready.")
 

firststreet/api/api.py

@@ -38,12 +38,14 @@ def call_api(self, fsids, product, product_subtype, location, limit=100):
         """
         if not isinstance(fsids, list):
             if isinstance(fsids, str):
-                if os.path.isfile(fsids):
-                    fsids = read_fsid_file(fsids)
+                if os.path.isfile(os.getcwd() + "\\" + fsids):
+                    fsids = read_fsid_file(os.getcwd() + "\\" + fsids)
                 else:
-                    raise InvalidArgument("File provided is not a list or a valid file. Please check the file name.")
+                    raise InvalidArgument("File provided is not a valid file. "
+                                          "Please check the file name. '{}'".format(os.path.curdir + str(fsids)))
             else:
-                raise InvalidArgument("File provided is not a list or a valid file. Please check the file name.")
+                raise InvalidArgument("File provided is not a list or a valid file. "
+                                      "Please check the file name. '{}'".format(os.path.curdir + str(fsids)))
 
         if not fsids:
             raise InvalidArgument(fsids)

firststreet/api/csv_format.py

@@ -9,14 +9,15 @@
 import pandas as pd
 
 
-def to_csv(data, product, product_subtype, location_type=None):
+def to_csv(data, product, product_subtype, location_type=None, output_dir=None):
     """Receives a list of data, a product, a product subtype, and a location to create a CSV
 
     Args:
         data (list): A list of FSF object
         product (str): The overall product to call
         product_subtype (str): The product subtype (if suitable)
         location_type (str): The location lookup type (if suitable)
+        output_dir (str): The output directory to save the generated csvs
     """
 
     date = datetime.datetime.today().strftime('%Y_%m_%d_%H_%M_%S')
@@ -27,7 +28,8 @@ def to_csv(data, product, product_subtype, location_type=None):
     else:
         file_name = "_".join([date, product, product_subtype]) + ".csv"
 
-    output_dir = "/".join([os.getcwd(), "data_csv"])
+    if not output_dir:
+        output_dir = os.getcwd() + "\\data_csv"
 
     if not os.path.exists(output_dir):
         os.makedirs(output_dir)
@@ -162,7 +164,7 @@ def format_adaptation_summary(data):
     """
     df = pd.DataFrame([vars(o) for o in data]).explode('adaptation').reset_index(drop=True)
     df['fsid'] = df['fsid'].apply(str)
-    return df[['fsid', 'adaptation']]
+    return df[['fsid', 'adaptation', 'properties']]
 
 
 def format_probability_chance(data):

firststreet/api/environmental.py

@@ -17,14 +17,15 @@ class Environmental(Api):
             get_precipitation: Retrieves a list of Environmental Precipitation for the given list of IDs
         """
 
-    def get_precipitation(self, fsids, csv=False, limit=100):
+    def get_precipitation(self, fsids, csv=False, limit=100, output_dir=None):
         """Retrieves environmental precipitation product data from the First Street Foundation API given a list of FSIDs
         and returns a list of Environmental Precipitation objects.
 
         Args:
             fsids (list/file): A First Street Foundation IDs or a file of First Street Foundation IDs
             csv (bool): To output extracted data to a csv or not
             limit (int): max number of connections to make
+            output_dir (str): The output directory to save the generated csvs
         Returns:
             A list of Adaptation Detail
         """
@@ -34,7 +35,7 @@ def get_precipitation(self, fsids, csv=False, limit=100):
         product = [EnvironmentalPrecipitation(api_data) for api_data in api_datas]
 
         if csv:
-            csv_format.to_csv(product, "environmental", "precipitation", "county")
+            csv_format.to_csv(product, "environmental", "precipitation", "county", output_dir=output_dir)
 
         logging.info("Environmental Precipitation Data Ready.")
 

firststreet/api/fema.py

@@ -18,7 +18,7 @@ class Fema(Api):
             get_nfip: Retrieves a list of Fema Nfip for the given list of IDs
         """
 
-    def get_nfip(self, fsids, location_type, csv=False, limit=100):
+    def get_nfip(self, fsids, location_type, csv=False, limit=100, output_dir=None):
         """Retrieves fema nfip product data from the First Street Foundation API given a list of FSIDs and
         returns a list of Fema Nfip objects.
 
@@ -27,6 +27,7 @@ def get_nfip(self, fsids, location_type, csv=False, limit=100):
             location_type (str): The location lookup type
             csv (bool): To output extracted data to a csv or not
             limit (int): max number of connections to make
+            output_dir (str): The output directory to save the generated csvs
         Returns:
             A list of Fema Nfip
         Raises:
@@ -44,7 +45,7 @@ def get_nfip(self, fsids, location_type, csv=False, limit=100):
         product = [FemaNfip(api_data) for api_data in api_datas]
 
         if csv:
-            csv_format.to_csv(product, "fema", "nfip", location_type)
+            csv_format.to_csv(product, "fema", "nfip", location_type, output_dir=output_dir)
 
         logging.info("Fema Nfip Data Ready.")
 

firststreet/api/historic.py

@@ -19,14 +19,15 @@ class Historic(Api):
             get_summary: Retrieves a list of Historic Summary for the given list of IDs
         """
 
-    def get_event(self, fsids, csv=False, limit=100):
+    def get_event(self, fsids, csv=False, limit=100, output_dir=None):
         """Retrieves historic event product data from the First Street Foundation API given a list of FSIDs and
         returns a list of Historic Event objects.
 
         Args:
             fsids (list/file): A First Street Foundation IDs or a file of First Street Foundation IDs
             csv (bool): To output extracted data to a csv or not
             limit (int): max number of connections to make
+            output_dir (str): The output directory to save the generated csvs
         Returns:
             A list of Historic Event
         """
@@ -36,13 +37,13 @@ def get_event(self, fsids, csv=False, limit=100):
         product = [HistoricEvent(api_data) for api_data in api_datas]
 
         if csv:
-            csv_format.to_csv(product, "historic", "event")
+            csv_format.to_csv(product, "historic", "event", output_dir=output_dir)
 
         logging.info("Historic Event Data Ready.")
 
         return product
 
-    def get_summary(self, fsids, location_type, csv=False, limit=100):
+    def get_summary(self, fsids, location_type, csv=False, limit=100, output_dir=None):
         """Retrieves historic summary product data from the First Street Foundation API given a list of FSIDs and
         returns a list of Historic Summary objects.
 
@@ -51,6 +52,7 @@ def get_summary(self, fsids, location_type, csv=False, limit=100):
             location_type (str): The location lookup type
             csv (bool): To output extracted data to a csv or not
             limit (int): max number of connections to make
+            output_dir (str): The output directory to save the generated csvs
         Returns:
             A list of Historic Summary
         """
@@ -65,7 +67,7 @@ def get_summary(self, fsids, location_type, csv=False, limit=100):
         product = [HistoricSummary(api_data) for api_data in api_datas]
 
         if csv:
-            csv_format.to_csv(product, "historic", "summary", location_type)
+            csv_format.to_csv(product, "historic", "summary", location_type, output_dir=output_dir)
 
         logging.info("Historic Summary Data Ready.")