Results import efficiencies (#1369)

* Remove unnecessary setup_results_import function and all calls
* Refactor db to conn for consistency, add commit_immediately flag to spin_on_db_lock
* Commit on closing DB, not on spin_on_db_lock
* Commit after each CSV structure row import

Author: anamileva

data_toolkit/load_raw_data.py

@@ -73,6 +73,7 @@ def main(args=None):
 
             read_and_import_csv(conn, f_path, table)
 
+    conn.commit()
     conn.close()
 
 

data_toolkit/manual_adjustments.py

@@ -229,6 +229,7 @@ def main(args=None):
         tech_dur_dict=tech_dur_dict,
     )
 
+    conn.commit()
     conn.close()
 
 

data_toolkit/project/availability/outages/create_availability_iteration_input_csvs.py

@@ -475,12 +475,12 @@ def main(args=None):
     if not parsed_args.quiet:
         print("Creating availability iteration CSVs...")
 
-    db = connect_to_database(parsed_args.database)
+    conn = connect_to_database(parsed_args.database)
 
     # ### Load data from CSV
     if parsed_args.input_csv is not None:
         read_and_import_csv(
-            conn=db,
+            conn=conn,
             f_path=parsed_args.input_csv,
             table="raw_data_unit_availability_params",
         )
@@ -490,7 +490,7 @@ def main(args=None):
         os.makedirs(parsed_args.output_directory)
 
     # Get projects
-    projects = [i[0] for i in db.execute("""
+    projects = [i[0] for i in conn.execute("""
         SELECT DISTINCT project FROM raw_data_unit_availability_params;
         """).fetchall()]
 
@@ -527,7 +527,7 @@ def main(args=None):
                     SELECT * FROM raw_data_unit_availability_params
                     WHERE project = '{project}'
                     ;""",
-                db,
+                conn,
             )
 
             # Pass user provided seed values if user_provide_seeding
@@ -584,7 +584,7 @@ def main(args=None):
                 ascending=[True, True, True],
             )
 
-    db.close()
+    conn.close()
 
 
 if __name__ == "__main__":

data_toolkit/project/availability/weather_derates/create_monte_carlo_gen_weather_derate_input_csvs.py

@@ -195,6 +195,7 @@ def main(args=None):
             table="raw_data_unit_availability_params",
         )
 
+    conn.commit()
     conn.close()
 
     get_monte_carlo_timeseries_project_pool_and_make_profile_csvs(

data_toolkit/project/availability/weather_derates/create_sync_gen_weather_derate_input_csvs.py

@@ -166,6 +166,7 @@ def main(args=None):
             table="raw_data_unit_availability_params",
         )
 
+    conn.commit()
     conn.close()
 
     get_sync_project_pool_and_make_profile_csvs(

data_toolkit/project/create_monte_carlo_gen_input_csvs_common.py

@@ -117,6 +117,7 @@ def get_monte_carlo_timeseries_project_pool_and_make_profile_csvs(
     pool.map(create_project_profile_csv_pool, pool_data)
     pool.close()
 
+    conn.commit()
     conn.close()
 
 
@@ -237,6 +238,7 @@ def create_project_profile_csv(
             overwrite=True,
         )
 
+    conn.commit()
     conn.close()
 
 

data_toolkit/project/create_sync_gen_input_csvs_common.py

@@ -176,6 +176,7 @@ def create_project_profile_csv(
         overwrite=overwrite,
     )
 
+    conn.commit()
     conn.close()
 
 

data_toolkit/project/opchar/hydro/create_hydro_iteration_input_csvs.py

@@ -267,8 +267,6 @@ def calculate_from_project_year_month_data(
 
             # TODO: add iterations CSVs
 
-    conn.close()
-
 
 def calculate_from_project_year_month_data_pool(pool_datum):
     (
@@ -317,31 +315,31 @@ def main(args=None):
 
     os.makedirs(parsed_args.output_directory, exist_ok=True)
 
-    db = connect_to_database(parsed_args.database)
+    conn = connect_to_database(parsed_args.database)
 
     # ### Load data from CSV
     if parsed_args.project_hydro_opchars_by_year_month_input_csv is not None:
         read_and_import_csv(
-            conn=db,
+            conn=conn,
             f_path=parsed_args.project_hydro_opchars_by_year_month_input_csv,
             table="raw_data_project_hydro_opchars_by_year_month",
         )
 
     if parsed_args.hydro_years_input_csv is not None:
         read_and_import_csv(
-            conn=db,
+            conn=conn,
             f_path=parsed_args.hydro_years_input_csv,
             table="raw_data_hydro_years",
         )
 
     if parsed_args.balancing_type_horizons_input_csv is not None:
         read_and_import_csv(
-            conn=db,
+            conn=conn,
             f_path=parsed_args.balancing_type_horizons_input_csv,
             table="user_defined_balancing_type_horizons",
         )
 
-    c = db.cursor()
+    c = conn.cursor()
     projects = [prj[0] for prj in c.execute("""
                 SELECT DISTINCT project
                 FROM raw_data_project_hydro_opchars_by_year_month;
@@ -368,7 +366,8 @@ def main(args=None):
     pool.map(calculate_from_project_year_month_data_pool, pool_data)
     pool.close()
 
-    db.close()
+    conn.commit()
+    conn.close()
 
 
 if __name__ == "__main__":

data_toolkit/project/opchar/var_profiles/create_monte_carlo_var_gen_input_csvs.py

@@ -177,6 +177,7 @@ def main(args=None):
         default_value=None,
     )
 
+    conn.commit()
     conn.close()
 
 

data_toolkit/project/opchar/var_profiles/create_sync_var_gen_input_csvs.py

@@ -157,6 +157,7 @@ def main(args=None):
             table="raw_data_var_project_units",
         )
 
+    conn.commit()
     conn.close()
 
     get_sync_project_pool_and_make_profile_csvs(