GridPath v2026.3.0

Merge pull request #1376 from blue-marble/develop

Author: anamileva

.github/workflows/test_gridpath.yml

@@ -23,13 +23,13 @@ jobs:
         python-version: [ "3.11", "3.12", "3.13" ]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install sqlite3 3.45.0 on Linux
       if: startsWith(runner.os, 'Linux')
       run: |
         bash ./.github/workflows/upgrade_sqlite_on_linux.sh
     - name: Set up Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install Cbc and check version (Linux)

README.md

@@ -3,7 +3,7 @@
 [![Coverage Status](https://coveralls.io/repos/github/blue-marble/gridpath/badge.svg?branch=main)](https://coveralls.io/github/blue-marble/gridpath?branch=main)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 [![Lint Black](https://github.com/blue-marble/gridpath/actions/workflows/black.yml/badge.svg?branch=main)](https://github.com/blue-marble/gridpath/actions/workflows/black.yml)
-[![DOI](https://zenodo.org/badge/65574330.svg)](https://zenodo.org/badge/latestdoi/65574330)
+[![DOI](https://zenodo.org/badge/65574330.svg)](https://doi.org/10.5281/zenodo.5822994)
 
 # Welcome to GridPath
 

data_toolkit/load_raw_data.py

@@ -23,6 +23,8 @@
 import sys
 from argparse import ArgumentParser
 import os.path
+from sqlite3 import Connection
+
 import pandas as pd
 
 from db.common_functions import spin_on_database_lock_generic, connect_to_database
@@ -69,23 +71,28 @@ def main(args=None):
                 print(f"... {f}...")
             f_path = str(os.path.join(parsed_args.csv_location, f))
 
-            # Set low_memory to False to avoid dtype warning
-            # TODO: actually specify dtypes instead
-            df = pd.read_csv(
-                f_path, delimiter=",", low_memory=False, on_bad_lines="warn"
-            )
-
-            spin_on_database_lock_generic(
-                command=df.to_sql(
-                    name=table,
-                    con=conn,
-                    if_exists="append",
-                    index=False,
-                )
-            )
+            read_and_import_csv(conn, f_path, table)
 
+    conn.commit()
     conn.close()
 
 
+def read_and_import_csv(conn: Connection, f_path: str, table):
+    # Set low_memory to False to avoid dtype warning
+    # TODO: actually specify dtypes instead
+    df = pd.read_csv(f_path, delimiter=",", low_memory=False, on_bad_lines="warn")
+
+    # print(f_path)
+    # print(df)
+    spin_on_database_lock_generic(
+        command=df.to_sql(
+            name=table,
+            con=conn,
+            if_exists="append",
+            index=False,
+        )
+    )
+
+
 if __name__ == "__main__":
     main()

data_toolkit/manual_adjustments.py

@@ -20,9 +20,11 @@
 import pandas as pd
 
 from db.common_functions import connect_to_database
+from data_toolkit.load_raw_data import read_and_import_csv
 from data_toolkit.project.project_data_filters_common import (
     get_eia860_sql_filter_string,
     DISAGG_PROJECT_NAME_STR,
+    AGG_PROJECT_NAME_STR,
 )
 
 # Storage durations
@@ -93,6 +95,14 @@ def parse_arguments(args):
         help="Overwrite existing CSV files.",
     )
 
+    parser.add_argument(
+        "-agg",
+        "--aggregate_projects",
+        default=False,
+        action="store_true",
+        help="Aggregate all projects to the BA-technology level.",
+    )
+
     parser.add_argument("-q", "--quiet", default=False, action="store_true")
 
     parsed_arguments = parser.parse_known_args(args=args)[0]
@@ -127,13 +137,14 @@ def make_copy_files(
 
 def add_battery_durations(
     conn,
-    disagg_project_name_str,
+    project_name_str,
     study_year,
     eia860_sql_filter_string,
     csv_location,
     subscenario_id,
     subscenario_name,
     tech_dur_dict,
+    aggregate_projects=False,
 ):
     duckdb_conn = duckdb.connect(database=":memory:")
     spec_cap_df = pd.read_csv(
@@ -145,17 +156,19 @@ def add_battery_durations(
     )
 
     for tech in tech_dur_dict.keys():
+        group_by = "GROUP BY project" if aggregate_projects else ""
         sql = f"""
-            SELECT {disagg_project_name_str} AS project, 
+            SELECT {project_name_str} AS project,
             {study_year} as period
             FROM raw_data_eia860_generators
             JOIN user_defined_eia_gridpath_key ON
-                    raw_data_eia860_generators.prime_mover_code = 
+                    raw_data_eia860_generators.prime_mover_code =
                     user_defined_eia_gridpath_key.prime_mover_code
                     AND energy_source_code_1 = energy_source_code
             WHERE 1 = 1
             AND {eia860_sql_filter_string}
             AND raw_data_eia860_generators.prime_mover_code = '{tech}'
+            {group_by}
             ;
         """
         relevant_projects_df = pd.read_sql(sql, conn)
@@ -215,9 +228,15 @@ def main(args=None):
         "PS": parsed_args.pumped_storage_duration,
     }
 
+    project_name_str = (
+        AGG_PROJECT_NAME_STR
+        if parsed_args.aggregate_projects
+        else DISAGG_PROJECT_NAME_STR
+    )
+
     add_battery_durations(
         conn=conn,
-        disagg_project_name_str=DISAGG_PROJECT_NAME_STR,
+        project_name_str=project_name_str,
         study_year=parsed_args.study_year,
         eia860_sql_filter_string=get_eia860_sql_filter_string(
             study_year=parsed_args.study_year, region=parsed_args.region
@@ -226,8 +245,10 @@ def main(args=None):
         subscenario_id=parsed_args.project_specified_capacity_scenario_id,
         subscenario_name=parsed_args.project_specified_capacity_scenario_name,
         tech_dur_dict=tech_dur_dict,
+        aggregate_projects=parsed_args.aggregate_projects,
     )
 
+    conn.commit()
     conn.close()
 
 

data_toolkit/open_data_toolkit_settings_sample.csv

@@ -87,6 +87,14 @@ def parse_arguments(args):
         "-avl_name", "--project_availability_scenario_name", default="no_derates"
     )
 
+    parser.add_argument(
+        "-agg",
+        "--aggregate_projects",
+        default=False,
+        action="store_true",
+        help="Aggregate all projects to the BA-technology level.",
+    )
+
     parser.add_argument("-q", "--quiet", default=False, action="store_true")
 
     parsed_arguments = parser.parse_known_args(args=args)[0]
@@ -104,14 +112,36 @@ def get_project_availability(
     csv_location,
     subscenario_id,
     subscenario_name,
+    aggregate_projects=False,
 ):
-    sql = f"""
+    if aggregate_projects:
+        sql = f"""
+        SELECT {agg_project_name_str} AS project,
+        'exogenous' AS availability_type,
+        NULL AS exogenous_availability_independent_scenario_id,
+        NULL AS exogenous_availability_weather_scenario_id,
+        NULL AS exogenous_availability_independent_bt_hrz_scenario_id,
+        NULL AS exogenous_availability_weather_bt_hrz_scenario_id,
+        NULL AS endogenous_availability_scenario_id
+        FROM raw_data_eia860_generators
+        JOIN user_defined_eia_gridpath_key ON
+                raw_data_eia860_generators.prime_mover_code =
+                user_defined_eia_gridpath_key.prime_mover_code
+                AND energy_source_code_1 = energy_source_code
+         WHERE 1 = 1
+         AND {eia860_sql_filter_string}
+         GROUP BY project
+        ;
+        """
+    else:
+        sql = f"""
     SELECT {disagg_project_name_str} AS project, 
     'exogenous' AS availability_type,
     NULL AS exogenous_availability_independent_scenario_id,
     NULL AS exogenous_availability_weather_scenario_id,
     NULL AS exogenous_availability_independent_bt_hrz_scenario_id,
     NULL AS exogenous_availability_weather_bt_hrz_scenario_id,
+    NULL AS exogenous_availability_monthly_scenario_id,
     NULL AS endogenous_availability_scenario_id
     FROM raw_data_eia860_generators
     JOIN user_defined_eia_gridpath_key ON
@@ -130,6 +160,7 @@ def get_project_availability(
     NULL AS exogenous_availability_weather_scenario_id,
     NULL AS exogenous_availability_independent_bt_hrz_scenario_id,
     NULL AS exogenous_availability_weather_bt_hrz_scenario_id,
+    NULL AS exogenous_availability_monthly_scenario_id,
     NULL AS endogenous_availability_scenario_id
     FROM raw_data_eia860_generators
     JOIN user_defined_eia_gridpath_key
@@ -171,6 +202,7 @@ def main(args=None):
         csv_location=parsed_args.output_directory,
         subscenario_id=parsed_args.project_availability_scenario_id,
         subscenario_name=parsed_args.project_availability_scenario_name,
+        aggregate_projects=parsed_args.aggregate_projects,
     )
 
     conn.close()