RA Toolkit support for not printing default availability values of 1 (#1365)

Author: anamileva

data_toolkit/project/availability/outages/create_availability_iteration_input_csvs.py

@@ -158,6 +158,14 @@ def parse_arguments(args):
         help="The number of projects to simulate in parallel. Defaults to 1.",
     )
 
+    parser.add_argument(
+        "-print_ones",
+        "--print_ones",
+        default=False,
+        action="store_true",
+        help="Include rows where derate values equal 1. Defaults to False.",
+    )
+
     parser.add_argument("-q", "--quiet", default=False, action="store_true")
 
     parsed_arguments = parser.parse_known_args(args=args)[0]
@@ -268,6 +276,7 @@ def simulate_project_availability(
     stage_id,
     study_year,
     filepath,
+    print_ones,
 ):
 
     stage_tmp_dict = get_temporal_structure(study_year)
@@ -294,6 +303,31 @@ def simulate_project_availability(
         }
     )
 
+    # Filter out rows where derate values are 1, unless print_ones is True
+    if not print_ones:
+        # For non-hybrids, find rows with (!=1, None)
+        export_df_non_hyb = export_df[
+            (
+                (export_df["availability_derate_independent"] != 1)
+                & (export_df["hyb_stor_cap_availability_derate"].isna())
+            )
+        ]
+
+        # For hybrids, find rows where either column is not 1
+        # First, skip the rows where the storage derate is NA (so that we
+        # don't end up including the ones for non-hybrids)
+        export_df_hyb = export_df[
+            (
+                (export_df["hyb_stor_cap_availability_derate"].notna())
+                & (
+                    (export_df["availability_derate_independent"] != 1)
+                    | ((export_df["hyb_stor_cap_availability_derate"] != 1))
+                )
+            )
+        ]
+
+        export_df = pd.concat([export_df_non_hyb, export_df_hyb]).drop_duplicates()
+
     export_df.to_csv(
         filepath,
         mode="a",
@@ -401,6 +435,7 @@ def simulate_project_availability_pool(pool_datum):
         stage_id,
         study_year,
         filepath,
+        print_ones,
     ] = pool_datum
 
     simulate_project_availability(
@@ -414,6 +449,7 @@ def simulate_project_availability_pool(pool_datum):
         stage_id=stage_id,
         study_year=study_year,
         filepath=filepath,
+        print_ones=print_ones,
     )
 
 
@@ -520,6 +556,7 @@ def main(args=None):
                     parsed_args.stage_id,
                     int(parsed_args.study_year),
                     filepath,
+                    parsed_args.print_ones,
                 ]
             )
 

data_toolkit/project/availability/weather_derates/create_monte_carlo_gen_weather_derate_input_csvs.py

@@ -153,6 +153,14 @@ def parse_arguments(args):
         help="The number of projects to simulate in parallel. Defaults to 1.",
     )
 
+    parser.add_argument(
+        "-ones",
+        "--print_ones",
+        default=False,
+        action="store_true",
+        help="Include rows where derate values equal 1. " "Defaults to False.",
+    )
+
     parser.add_argument("-q", "--quiet", default=False, action="store_true")
 
     parsed_arguments = parser.parse_known_args(args=args)[0]
@@ -203,6 +211,8 @@ def main(args=None):
         param_name="availability_derate_weather",
         raw_data_table="raw_data_availability_profiles",
         study_year=int(parsed_args.study_year),
+        print_default_values=parsed_args.print_ones,
+        default_value=1,
         no_hydro_iteration=True,
     )
 

data_toolkit/project/availability/weather_derates/create_sync_gen_weather_derate_input_csvs.py

@@ -53,7 +53,6 @@
 from db.common_functions import connect_to_database
 from data_toolkit.load_raw_data import read_and_import_csv
 from data_toolkit.project.create_sync_gen_input_csvs_common import (
-    create_project_profile_csv,
     get_sync_project_pool_and_make_profile_csvs,
 )
 
@@ -125,6 +124,14 @@ def parse_arguments(args):
         help="The number of projects to simulate in parallel. Defaults to 1.",
     )
 
+    parser.add_argument(
+        "-ones",
+        "--print_ones",
+        default=False,
+        action="store_true",
+        help="Include rows where derate values equal 1. Defaults to False.",
+    )
+
     parser.add_argument("-q", "--quiet", default=False, action="store_true")
 
     parsed_arguments = parser.parse_known_args(args=args)[0]
@@ -175,6 +182,8 @@ def main(args=None):
         varies_by_hydro=0,
         include_hydro_iteration_column=False,
         n_parallel_projects=parsed_args.n_parallel_projects,
+        print_default_values=parsed_args.print_ones,
+        default_value=1,
     )
 
 

data_toolkit/project/create_monte_carlo_gen_input_csvs_common.py

@@ -43,6 +43,8 @@ def get_monte_carlo_timeseries_project_pool_and_make_profile_csvs(
     param_name,
     raw_data_table,
     study_year,
+    print_default_values,
+    default_value,
     no_hydro_iteration=False,
 ):
     conn = connect_to_database(db_path=db_path)
@@ -100,6 +102,8 @@ def get_monte_carlo_timeseries_project_pool_and_make_profile_csvs(
                 param_name,
                 raw_data_table,
                 study_year,
+                print_default_values,
+                default_value,
                 no_hydro_iteration,
             ]
             for timeseries_name in timeseries_project_unit_dict.keys()
@@ -130,6 +134,8 @@ def create_project_profile_csv(
     param_name,
     raw_data_table,
     study_year,
+    print_default_values,
+    default_value,
     no_hydro_iteration=False,
 ):
     # Connect to database
@@ -194,6 +200,11 @@ def create_project_profile_csv(
         # Put into a dataframe and add to file
         df = pd.read_sql(project_query, con=conn)
 
+        # Filter out rows where the value is the default, unless
+        # print_default_values is True
+        if not print_default_values:
+            df = df[df[param_name] != default_value]
+
         filename = os.path.join(
             output_directory,
             f"{project}-{profile_scenario_id}-" f"{profile_scenario_name}.csv",
@@ -244,6 +255,8 @@ def create_project_profile_csv_pool(pool_datum):
         param_name,
         raw_data_table,
         study_year,
+        print_default_values,
+        default_value,
         no_hydro_iteration,
     ] = pool_datum
 
@@ -261,5 +274,7 @@ def create_project_profile_csv_pool(pool_datum):
         param_name=param_name,
         raw_data_table=raw_data_table,
         study_year=study_year,
+        print_default_values=print_default_values,
+        default_value=default_value,
         no_hydro_iteration=no_hydro_iteration,
     )

data_toolkit/project/create_sync_gen_input_csvs_common.py

@@ -37,6 +37,8 @@ def get_sync_project_pool_and_make_profile_csvs(
     varies_by_hydro,
     include_hydro_iteration_column,
     n_parallel_projects,
+    print_default_values,
+    default_value,
 ):
 
     conn = connect_to_database(db_path=db_path)
@@ -65,6 +67,8 @@ def get_sync_project_pool_and_make_profile_csvs(
                 varies_by_weather,
                 varies_by_hydro,
                 include_hydro_iteration_column,
+                print_default_values,
+                default_value,
             ]
             for prj in projects
         ]
@@ -93,6 +97,8 @@ def create_project_profile_csv(
     varies_by_weather,
     varies_by_hydro,
     include_hydro_iteration_column,
+    print_default_values,
+    default_value,
 ):
     conn = connect_to_database(db_path=db_path)
 
@@ -133,6 +139,11 @@ def create_project_profile_csv(
     # Put into a dataframe and add to file
     df = pd.read_sql(query, con=conn)
 
+    # Filter out rows where the value is at the default, unless
+    # print_default_values is True
+    if not print_default_values:
+        df = df[df[param_name] != default_value]
+
     filename = os.path.join(
         output_directory,
         f"{project}-{profile_scenario_id}-" f"{profile_scenario_name}.csv",
@@ -183,6 +194,8 @@ def create_project_profile_csv_pool(pool_datum):
         varies_by_weather,
         varies_by_hydro,
         include_hydro_iteration_column,
+        print_default_values,
+        default_value,
     ] = pool_datum
 
     create_project_profile_csv(
@@ -199,4 +212,6 @@ def create_project_profile_csv_pool(pool_datum):
         varies_by_weather=varies_by_weather,
         varies_by_hydro=varies_by_hydro,
         include_hydro_iteration_column=include_hydro_iteration_column,
+        print_default_values=print_default_values,
+        default_value=default_value,
     )

data_toolkit/project/opchar/var_profiles/create_monte_carlo_var_gen_input_csvs.py

@@ -173,6 +173,8 @@ def main(args=None):
         param_name="cap_factor",
         raw_data_table="raw_data_var_profiles",
         study_year=parsed_args.study_year,
+        print_default_values=True,
+        default_value=None,
     )
 
     conn.close()

data_toolkit/project/opchar/var_profiles/create_sync_var_gen_input_csvs.py

@@ -173,6 +173,8 @@ def main(args=None):
         varies_by_hydro=0,
         include_hydro_iteration_column=True,
         n_parallel_projects=parsed_args.n_parallel_projects,
+        print_default_values=True,
+        default_value=None,
     )