From 64c767c8156d8ab21b2fba8542c10ae69ee8d1b1 Mon Sep 17 00:00:00 2001 From: Ana Mileva Date: Thu, 30 Apr 2026 12:39:23 -0700 Subject: [PATCH 1/3] save work --- gridpath/auxiliary/scenario_chars.py | 777 +++++++++++++-------------- gridpath/get_scenario_inputs.py | 72 ++- gridpath/import_scenario_results.py | 21 +- gridpath/run_scenario.py | 65 ++- gridpath/validate_inputs.py | 67 +-- 5 files changed, 497 insertions(+), 505 deletions(-) diff --git a/gridpath/auxiliary/scenario_chars.py b/gridpath/auxiliary/scenario_chars.py index 68d2ee7c54..dcc712c3e0 100644 --- a/gridpath/auxiliary/scenario_chars.py +++ b/gridpath/auxiliary/scenario_chars.py @@ -1,4 +1,5 @@ -# Copyright 2016-2023 Blue Marble Analytics LLC. +# Copyright 2016-2025 Blue Marble Analytics LLC. +# Copyright 2026 Sylvan Energy Analytics LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,7 +17,10 @@ Scenario characteristics in database. """ +import copy import os.path +import pandas as pd +from pathlib import Path from gridpath.auxiliary.auxiliary import ( check_for_integer_subdirectories, @@ -100,44 +104,63 @@ def get_all_available_subscenarios(self): class ScenarioStructure(object): - def __init__(self, iteration_structure, subproblem_stage_structure): - # Weather, hydro, and availability iterations - # {weather_iteration: {hydro_iteration: [availability_iterations]}} + def __init__( + self, + weather_hydro_avail_subproblem_stage_dict, + weather_iteration_flag, + hydro_iteration_flag, + availability_iteration_flag, + subproblem_flag, + stage_flag, + ): + # Weather, hydro, and availability iterations are the first three + # levels, followed by subproblem and stage + # {weather_iteration: + # {hydro_iteration: + # {availability_iterations: + # {subproblem: [stage] + # } + # } + # } + # } # If no weather iterations, we're expecting a single key with 0 as value # If no hydro iterations, we're expecting a single key with 0 as value # If no availability itertions, we're expecting a list with 0 as member # [0] # No weather, no hydro, no availability: {0: {0: [0]}} - self.ITERATION_STRUCTURE = iteration_structure - # List of stages by subproblem in dict {subproblem: [stages]} + # List of stages by subproblem in dict within each iteration + # combination in a dictionary format {subproblem: [stages]} # This should have a single key, 1, if a single subproblem # This should be subproblem: [1] when a single stage in the subproblem - self.SUBPROBLEM_STAGES = subproblem_stage_structure - - # If any subproblem's stage list is non-empty, we have stages, so set - # the MULTI_STAGE flag to True to pass to determine_modules - # This tells the determine_modules function to include the - # stages-related modules - self.MULTI_STAGE = any( - [ - len(self.SUBPROBLEM_STAGES[subp]) > 1 - for subp in list(self.SUBPROBLEM_STAGES.keys()) - ] + self.WEATHER_HYDRO_AVAIL_SUBPROBLEM_STAGE_DICT = ( + weather_hydro_avail_subproblem_stage_dict ) + self.WEATHER_ITERATION_FLAG = weather_iteration_flag + self.HYDRO_ITERATION_FLAG = hydro_iteration_flag + self.AVAILABILITY_ITERATION_FLAG = availability_iteration_flag + self.SUBPROBLEM_FLAG = subproblem_flag + self.STAGE_FLAG = stage_flag + + # print(weather_hydro_avail_subproblem_stage_dict) + # print(weather_iteration_flag) + # print(hydro_iteration_flag) + # print(availability_iteration_flag) + # print(subproblem_flag) + # print(stage_flag) + # For determining whether we can parallelize self.N_SUBPROBLEMS = self.calculate_n_subproblems( - iteration_structure=iteration_structure, - subproblem_stage_structure=subproblem_stage_structure, + weather_hydro_avail_subproblem_stage_dict=weather_hydro_avail_subproblem_stage_dict, ) - def calculate_n_subproblems(self, iteration_structure, subproblem_stage_structure): + def calculate_n_subproblems(self, weather_hydro_avail_subproblem_stage_dict): # Total subproblems to determine parallelization n_subproblems = 0 - for w in iteration_structure.keys(): - for h in iteration_structure[w].keys(): - for a in iteration_structure[w][h]: - for s in subproblem_stage_structure.keys(): + for w in weather_hydro_avail_subproblem_stage_dict.keys(): + for h in weather_hydro_avail_subproblem_stage_dict[w].keys(): + for a in weather_hydro_avail_subproblem_stage_dict[w][h].keys(): + for s in weather_hydro_avail_subproblem_stage_dict[w][h][a].keys(): n_subproblems += 1 return n_subproblems @@ -145,193 +168,76 @@ def calculate_n_subproblems(self, iteration_structure, subproblem_stage_structur class ScenarioDirectoryStructure(object): def __init__(self, scenario_structure): - self.ITERATION_DIRECTORIES = ( - determine_iteration_directories_from_iteration_structure(scenario_structure) - ) - self.SUBPROBLEM_STAGE_DIRECTORIES = ( - determine_subproblem_stage_directory_structure(scenario_structure) + self.SCENARIO_DIRECTORY_STRUCTURE = ( + determine_directory_structure_from_scenario_structure(scenario_structure) ) -def determine_iteration_directories_from_iteration_structure(scenario_structure): +def determine_directory_structure_from_scenario_structure(scenario_structure): """ Determine whether we will have iteration (weather, hydro iteration), We write the subdirectories if we have multiple items at that level """ iteration_directory_strings_dict = {} - - # Check the top-level keys; these are the weather iterations - # If no weather iterations, something went wrong upstream, so raise an - # error (we're expecting at least one weather iteration) - if len(scenario_structure.ITERATION_STRUCTURE.keys()) == 0: - raise ValueError("Expecting at least one weather iteration.") - - # Don't make the directory for weather iterations if it's a single - # iteration with ID 0 - if len(scenario_structure.ITERATION_STRUCTURE.keys()) > 1: - make_weather_iteration_dirs = True - elif list(scenario_structure.ITERATION_STRUCTURE.keys())[0] != 0: - make_weather_iteration_dirs = True - else: - make_weather_iteration_dirs = False - - for weather_iteration in scenario_structure.ITERATION_STRUCTURE.keys(): - weather_iteration_str = ( - f"weather_iteration_{weather_iteration}" - if make_weather_iteration_dirs - else "empty_string" + for w in scenario_structure.WEATHER_HYDRO_AVAIL_SUBPROBLEM_STAGE_DICT.keys(): + w_string = ( + f"weather_iteration_{w}" + if scenario_structure.WEATHER_ITERATION_FLAG + else "" ) - - # Check the hydro iteration level - iteration_directory_strings_dict[weather_iteration_str] = {} - # Determine whether we will have hydro iterations - # Make the hydro directories if 1) there are multiple - # hydro iterations within a weather iteration or 2) there are - # multiple hydro iterations across weather iterations - # Case 2) captures case 1), but keeping separate for clarity - if len(scenario_structure.ITERATION_STRUCTURE[weather_iteration].keys()) > 1: - make_hydro_iteration_dirs = True - elif ( - len( - set( - [ - i - for sublist in [ - scenario_structure.ITERATION_STRUCTURE[w] - for w in scenario_structure.ITERATION_STRUCTURE.keys() - ] - for i in sublist - ] - ) + iteration_directory_strings_dict[w_string] = {} + for h in scenario_structure.WEATHER_HYDRO_AVAIL_SUBPROBLEM_STAGE_DICT[w].keys(): + h_string = ( + f"hydro_iteration_{h}" + if (scenario_structure.HYDRO_ITERATION_FLAG) + else "" ) - > 1 - ): - make_hydro_iteration_dirs = True - elif ( - list(scenario_structure.ITERATION_STRUCTURE[weather_iteration].keys())[0] - != 0 - ): - make_hydro_iteration_dirs = True - else: - make_hydro_iteration_dirs = False - - for hydro_iteration in scenario_structure.ITERATION_STRUCTURE[ - weather_iteration - ].keys(): - hydro_iteration_str = ( - f"hydro_iteration_{hydro_iteration}" - if make_hydro_iteration_dirs - else "empty_string" - ) - iteration_directory_strings_dict[weather_iteration_str][ - hydro_iteration_str - ] = [] - - # Check the availability level - # Make the availability directories if 1) there are multiple - # availability iterations within a hydro iteration or 2) there are - # multiple availability iterations across hydro iterations - # Case 2) captures case 1), but keeping separate for clarity - if ( - len( - scenario_structure.ITERATION_STRUCTURE[weather_iteration][ - hydro_iteration - ] + iteration_directory_strings_dict[w_string][h_string] = {} + for a in scenario_structure.WEATHER_HYDRO_AVAIL_SUBPROBLEM_STAGE_DICT[w][ + h + ].keys(): + a_string = ( + f"availability_iteration_{a}" + if (scenario_structure.AVAILABILITY_ITERATION_FLAG) + else "" ) - > 1 - ): - make_availability_iteration_dirs = True - elif ( - len( - set( - [ - i - for sublist in [ - scenario_structure.ITERATION_STRUCTURE[w][h] - for w in scenario_structure.ITERATION_STRUCTURE.keys() - for h in scenario_structure.ITERATION_STRUCTURE[ - w - ].keys() - ] - for i in sublist - ] + iteration_directory_strings_dict[w_string][h_string][a_string] = {} + # The subproblem structure is the same within each iteration + # If we only have a single subproblem AND it does not have stages, set the + # subproblem_string to an empty string (the subproblem directory should not + # have been created) + # If we have multiple subproblems or a single subproblems with stages, + # we're expecting a subproblem directory + for ( + subproblem + ) in scenario_structure.WEATHER_HYDRO_AVAIL_SUBPROBLEM_STAGE_DICT[w][h][ + a + ].keys(): + subproblem_string = ( + f"{subproblem}" + if ( + scenario_structure.SUBPROBLEM_FLAG + or scenario_structure.STAGE_FLAG + ) + else "" ) - ) - > 1 - ): - make_availability_iteration_dirs = True - elif ( - list( - scenario_structure.ITERATION_STRUCTURE[weather_iteration][ - hydro_iteration - ] - )[0] - != 0 - ): - make_availability_iteration_dirs = True - else: - make_availability_iteration_dirs = False - for availability_iteration in scenario_structure.ITERATION_STRUCTURE[ - weather_iteration - ][hydro_iteration]: - availability_iteration_str = ( - f"availability_iteration_{availability_iteration}" - if make_availability_iteration_dirs - else "empty_string" - ) - iteration_directory_strings_dict[weather_iteration_str][ - hydro_iteration_str - ].append(availability_iteration_str) + iteration_directory_strings_dict[w_string][h_string][a_string][ + subproblem_string + ] = [] + for stage in ( + scenario_structure.WEATHER_HYDRO_AVAIL_SUBPROBLEM_STAGE_DICT + )[w][h][a][subproblem]: + stage_string = ( + f"{stage}" if (scenario_structure.STAGE_FLAG) else "" + ) + iteration_directory_strings_dict[w_string][h_string][a_string][ + subproblem_string + ].append(stage_string) return iteration_directory_strings_dict -def determine_subproblem_stage_directory_structure(scenario_structure): - """ - The subproblem structure is the same within each iteration - If we only have a single subproblem AND it does not have stages, set the - subproblem_string to an empty string (the subproblem directory should not - have been created) - If we have multiple subproblems or a single subproblems with stages, - we're expecting a subproblem directory - """ - subproblem_stage_directory_strings = {} - - if ( - len(scenario_structure.SUBPROBLEM_STAGES) <= 1 - and scenario_structure.MULTI_STAGE is False - ): - make_subproblem_directories = False - else: - make_subproblem_directories = True - - for subproblem in scenario_structure.SUBPROBLEM_STAGES.keys(): - # If there are subproblems/stages, input directory will be nested - if make_subproblem_directories: - subproblem_str = str(subproblem) - else: - subproblem_str = "" - - subproblem_stage_directory_strings[subproblem_str] = [] - - stages = scenario_structure.SUBPROBLEM_STAGES[subproblem] - if len(stages) == 1: - make_stage_directories = False - else: - make_stage_directories = True - - for stage in stages: - if make_stage_directories: - stage_str = str(stage) - else: - stage_str = "" - - subproblem_stage_directory_strings[subproblem_str].append(stage_str) - - return subproblem_stage_directory_strings - - def get_scenario_structure_from_db(conn, scenario_id): """ @@ -340,58 +246,54 @@ def get_scenario_structure_from_db(conn, scenario_id): """ cursor = conn.cursor() - # Weather iterations - weather_iterations = [ - row[0] for row in cursor.execute("""SELECT DISTINCT weather_iteration + # Iterations + iterations_query = f"""SELECT weather_iteration, + hydro_iteration, availability_iteration FROM inputs_temporal_iterations INNER JOIN scenarios USING (temporal_scenario_id) - WHERE scenario_id = {};""".format(scenario_id)).fetchall() - ] + WHERE scenario_id = {scenario_id};""" - # Store weather iterations and hydro iterations in dict - # {weather_iteration: [hydro_iterations]} - # If we don't find any weather iterations, there were no iterations of - # any kind (we know there are no hydro and availability iterations since - # NULL values are not allowed) - if not weather_iterations: - iteration_structure_dict = {0: {0: [0]}} + iter_df = pd.read_sql(iterations_query, conn) + print(iter_df) + + if iter_df.empty: + weather_hydro_avail_subproblem_stage_dict = {0: {0: {0: None}}} + weather_iteration_flag = False + hydro_iteration_flag = False + availability_iteration_flag = False else: - iteration_structure_dict = {} - for weather_iteration in weather_iterations: - # Get the hydro iterations for this weather iteration - hydro_iterations = cursor.execute(f"""SELECT hydro_iteration - FROM inputs_temporal_iterations - INNER JOIN scenarios - USING (temporal_scenario_id) - WHERE scenario_id = {scenario_id} - AND weather_iteration = {weather_iteration} - ;""").fetchall() - hydro_iterations_dict = { - hydro_iteration[0]: [] for hydro_iteration in hydro_iterations - } # to list - iteration_structure_dict[weather_iteration] = hydro_iterations_dict - - for hydro_iteration in hydro_iterations_dict.keys(): - # Get the availability iterations for this weather/hydro - # iteration - availability_iterations = cursor.execute( - f"""SELECT availability_iteration - FROM inputs_temporal_iterations - INNER JOIN scenarios - USING (temporal_scenario_id) - WHERE scenario_id = {scenario_id} - AND weather_iteration = {weather_iteration} - AND hydro_iteration = {hydro_iteration} - ;""" - ).fetchall() - availability_iterations = [ - availability_iteration[0] - for availability_iteration in availability_iterations - ] - iteration_structure_dict[weather_iteration][ - hydro_iteration - ] = availability_iterations + weather_hydro_avail_subproblem_stage_dict = {} + for row in iter_df.itertuples(): + ix, weather_iteration, hydro_iteration, availability_iteration = row + weather_hydro_avail_subproblem_stage_dict[weather_iteration] = { + hydro_iteration: {availability_iteration: None} + } + + weather_iterations = get_distinct_iterations_from_db( + conn, scenario_id, "weather_iteration" + ) + weather_iteration_flag = ( + False + if (len(weather_iterations) == 1 and weather_iterations[0] == 0) + else True + ) + + hydro_iterations = get_distinct_iterations_from_db( + conn, scenario_id, "hydro_iteration" + ) + hydro_iteration_flag = ( + False if (len(hydro_iterations) == 1 and hydro_iterations[0] == 0) else True + ) + + availability_iterations = get_distinct_iterations_from_db( + conn, scenario_id, "availability_iteration" + ) + availability_iteration_flag = ( + False + if (len(availability_iterations) == 1 and availability_iterations[0] == 0) + else True + ) # TODO: make sure there is data integrity between subproblems_stages # and inputs_temporal_horizons and inputs_temporal @@ -404,6 +306,8 @@ def get_scenario_structure_from_db(conn, scenario_id): USING (temporal_scenario_id) WHERE scenario_id = {};""".format(scenario_id)).fetchall() ] + print("SUPROBLEMS", all_subproblems) + subproblem_flag = False if len(all_subproblems) == 1 else True # Store subproblems and stages in dict {subproblem: [stages]} stages_by_subproblem = {} @@ -418,163 +322,211 @@ def get_scenario_structure_from_db(conn, scenario_id): stages = [stage[0] for stage in stages] # convert to simple list stages_by_subproblem[subproblem] = stages - return ScenarioStructure( - iteration_structure=iteration_structure_dict, - subproblem_stage_structure=stages_by_subproblem, + # Assuming the same for all subproblems + stage_flag = ( + False + if len(stages_by_subproblem[next(iter(stages_by_subproblem))]) == 1 + else True ) + # Assumes same subproblems/stages for each iteration combo + for w_it in weather_hydro_avail_subproblem_stage_dict.keys(): + for h_it in weather_hydro_avail_subproblem_stage_dict[w_it].keys(): + for ave_it in weather_hydro_avail_subproblem_stage_dict[w_it][h_it].keys(): + weather_hydro_avail_subproblem_stage_dict[w_it][h_it][ + ave_it + ] = stages_by_subproblem -def get_scenario_structure_from_disk(scenario_directory): - iteration_structure_dict = {} + print("HERE: ", weather_hydro_avail_subproblem_stage_dict) - # Get the iteration structure - # Check if there are weather directories - weather_directories = check_for_starting_string_subdirectories( - main_directory=scenario_directory, starting_string="weather_iteration" + return ScenarioStructure( + weather_hydro_avail_subproblem_stage_dict=weather_hydro_avail_subproblem_stage_dict, + weather_iteration_flag=weather_iteration_flag, + hydro_iteration_flag=hydro_iteration_flag, + availability_iteration_flag=availability_iteration_flag, + subproblem_flag=subproblem_flag, + stage_flag=stage_flag, ) - iteration_structure_dict = {} - if not weather_directories: - # Check if there are hydro and availability iterations - hydro_and_availability_iterations = ( - check_hydro_and_availability_iteration_levels( - starting_directory=scenario_directory - ) - ) - iteration_structure_dict = {0: hydro_and_availability_iterations} - else: - for weather_directory in weather_directories: - weather_iteration = int(weather_directory.replace("weather_iteration_", "")) - w_d_full_path = os.path.join(scenario_directory, weather_directory) - # Check if there are hydro and availability iterations for this - # weather iteration - hydro_and_availability_iterations = ( - check_hydro_and_availability_iteration_levels( - starting_directory=w_d_full_path - ) - ) - iteration_structure_dict[weather_iteration] = ( - hydro_and_availability_iterations - ) - # Get the subproblem structure - subproblem_main_directory = get_directory_for_subproblem_structure( - scenario_directory=scenario_directory - ) - stages_by_subproblem = check_subproblem_structure( - subproblem_main_directory=subproblem_main_directory - ) +def get_distinct_iterations_from_db(conn, scenario_id, iteration_type): + c = conn.cursor() + iterations = [i[0] for i in c.execute(f"""SELECT DISTINCT + {iteration_type} + FROM inputs_temporal_iterations + INNER JOIN scenarios + USING (temporal_scenario_id) + WHERE scenario_id = {scenario_id};""").fetchall()] + return iterations - return ScenarioStructure( - iteration_structure=iteration_structure_dict, - subproblem_stage_structure=stages_by_subproblem, - ) +def get_scenario_structure_from_disk(scenario_directory): -def check_hydro_and_availability_iteration_levels(starting_directory): - """ - Check if there are hydro directories in the starting directory. - If not found, return a dictionary with 0 as key and a list of availability - iterations found in the starting directory, else return a list with - the hydro iteration numbers as keys and the availability iterations found - within each hydro iteration (as list for the respective key). - """ - # Check for hydro directories in the starting directory - hydro_directories = check_for_starting_string_subdirectories( - main_directory=starting_directory, starting_string="hydro_iteration" + # Iterate through items in the scenario_directory and create a nested + # dictionary of the directory structure + # Limit to five levels (weather, hydro, availability, subproblem, stage) + # We'll then parse this dictionary to understand which levels exist + level_start = 1 + max_level = 5 + dir_structure_from_disk = dir_to_nested_dict( + Path(scenario_directory), level_start, max_level ) - # If there are no hydro directories, check for availability - # directories in the main directory - if not hydro_directories: - availability_iterations = check_availability_iteration_level( - starting_directory=starting_directory - ) + print("DIR STRUCTURE FROM DISK", dir_structure_from_disk) - hydro_and_availability_iterations = {0: availability_iterations} + # First, reverse engineer the directory structure + # We need empty strings for layers that do not exist + # We'll start with what we got from disk + dir_structure_intermediate = copy.deepcopy(dir_structure_from_disk) + # Check if we have a weather iteration level; if we don't, add the level + # with a single key weather_iteration_0 + if not check_dict_key_for_string_recursive( + dir_structure_from_disk, "weather_iteration_" + ): + weather_iteration_flag = False + dir_structure_intermediate = {"weather_iteration_0": dir_structure_intermediate} else: - hydro_and_availability_iterations = {} - for hydro_directory in hydro_directories: - hydro_iteration = int(hydro_directory.replace("hydro_iteration_", "")) - availability_iterations = check_availability_iteration_level( - starting_directory=os.path.join(starting_directory, hydro_directory) - ) - - hydro_and_availability_iterations[hydro_iteration] = availability_iterations - - return hydro_and_availability_iterations + weather_iteration_flag = True + # print(dir_structure_intermediate) -def check_availability_iteration_level(starting_directory): - """ - Check if there are availability directories in the starting directory. - If not found, return a list with 0, else return a list with the - availability iteration numbers. - """ - availability_directories = check_for_starting_string_subdirectories( - main_directory=starting_directory, - starting_string="availability_iteration", - ) - - # If there are no availability directories, return the "empty" - # iteration structure - if not availability_directories: - availability_iterations = [0] - # subproblem_main_directory = scenario_directory - # If there are availability directory, add those iterations to - # the iteration structure + # Check if we have a hydro iteration level; if we don't, add the level + # with a single hydro_iteration_0 under each weather iteration key + if not check_dict_key_for_string_recursive( + dir_structure_from_disk, "hydro_iteration_" + ): + hydro_iteration_flag = False + for w in dir_structure_intermediate.keys(): + dir_structure_intermediate[w]["hydro_iteration_0"] = {} else: - availability_iterations = dir_strings_to_iteration_numbers_list( - dir_list=availability_directories, - starting_string="availability_iteration", - ) + hydro_iteration_flag = True - return availability_iterations + # print(dir_structure_intermediate) + # print(dir_structure_from_disk) - -def get_directory_for_subproblem_structure(scenario_directory): - weather_directories = check_for_starting_string_subdirectories( - main_directory=scenario_directory, starting_string="weather_iteration" - ) - if weather_directories: - weather_dir_for_subproblem_structure = weather_directories[0] + # Check if we have an availability iteration level; if we don't, add the level + # with a single availability_iteration_0 under each weather/hydro iteration + # key + if not check_dict_key_for_string_recursive( + dir_structure_from_disk, "availability_iteration_" + ): + availability_iteration_flag = False + for w in dir_structure_intermediate.keys(): + for h in dir_structure_intermediate[w].keys(): + dir_structure_intermediate[w][h]["availability_iteration_0"] = {} else: - weather_dir_for_subproblem_structure = "" + availability_iteration_flag = True + # print(dir_structure_intermediate) - hydro_directories = check_for_starting_string_subdirectories( - main_directory=os.path.join( - scenario_directory, weather_dir_for_subproblem_structure - ), - starting_string="hydro_iteration", + # Iteration layers + iteration_layers_n = sum( + [weather_iteration_flag, hydro_iteration_flag, availability_iteration_flag] ) - - if hydro_directories: - hydro_dir_for_subproblem_structure = hydro_directories[0] + print("ITERATION LAYERS", iteration_layers_n) + print("THIS IS THE DICT ", dir_structure_from_disk) + print("ORIGINAL DEPTH", get_dictionary_depth(dir_structure_from_disk)) + dir_structure_depth = get_dictionary_depth(dir_structure_from_disk) - 1 + print("DIR STRUCTURE DEPTH", dir_structure_depth) + non_iteration_layers_n = dir_structure_depth - iteration_layers_n + + # Finally, figure out the subproblem/stage structure from the non-iteration + # layers + # 0 - no subproblems exist, add the levels as {1:[1]} + # 1 - subproblems exist or single subproblem with stages if there + # is a passthrough directory + # 2 - subproblems and stages exist + if non_iteration_layers_n == 0: + subproblem_flag = False + stage_flag = False + for w in dir_structure_intermediate.keys(): + for h in dir_structure_intermediate[w].keys(): + for a in dir_structure_intermediate[w][h].keys(): + dir_structure_intermediate[w][h][a] = {1: [1]} + elif non_iteration_layers_n == 1: + # If stages, add the subproblem + if os.path.exists(os.path.join(scenario_directory, "pass_through_inputs")): + subproblem_flag = False + stage_flag = True + for w in dir_structure_intermediate.keys(): + for h in dir_structure_intermediate[w].keys(): + for a in dir_structure_intermediate[w][h].keys(): + dir_structure_intermediate[w][h][a] = { + 1: list(dir_structure_intermediate[w][h][a].keys()) + } + # If no stages, add them + else: + subproblem_flag = True + stage_flag = False + for w in dir_structure_intermediate.keys(): + for h in dir_structure_intermediate[w].keys(): + for a in dir_structure_intermediate[w][h].keys(): + for subproblem in dir_structure_intermediate[w][h][a].keys(): + dir_structure_intermediate[w][h][a][subproblem] = [1] + elif non_iteration_layers_n == 2: + subproblem_flag = True + stage_flag = True + for w in dir_structure_intermediate.keys(): + for h in dir_structure_intermediate[w].keys(): + for a in dir_structure_intermediate[w][h].keys(): + for subproblem in dir_structure_intermediate[w][h][a].keys(): + dir_structure_intermediate[w][h][a][subproblem] = list( + dir_structure_intermediate[w][h][a][subproblem].keys() + ) else: - hydro_dir_for_subproblem_structure = "" - - availability_directories = check_for_starting_string_subdirectories( - main_directory=os.path.join( - scenario_directory, - weather_dir_for_subproblem_structure, - hydro_dir_for_subproblem_structure, - ), - starting_string="availability_iteration", - ) + raise ValueError( + "ERROR: Scenario directory structure is not supported. " + "The number of non-iteration layers must be 0, 1, or 2." + ) - if availability_directories: - availability_dir_for_subproblem_structure = availability_directories[0] - else: - availability_dir_for_subproblem_structure = "" + # Remove the starting strings for the iteration levels and make the final + # dictionary + weather_hydro_avail_subproblem_stage_dict_final = {} + for w in dir_structure_intermediate.keys(): + w_int = int(w.replace(f"weather_iteration_", "")) + print(w, w_int) + weather_hydro_avail_subproblem_stage_dict_final[w_int] = {} + print(weather_hydro_avail_subproblem_stage_dict_final) + for h in dir_structure_intermediate[w].keys(): + h_int = int(h.replace(f"hydro_iteration_", "")) + print(h, h_int) + weather_hydro_avail_subproblem_stage_dict_final[w_int][h_int] = {} + print(weather_hydro_avail_subproblem_stage_dict_final) + for a in dir_structure_intermediate[w][h].keys(): + a_int = int(a.replace(f"availability_iteration_", "")) + print(a, a_int) + weather_hydro_avail_subproblem_stage_dict_final[w_int][h_int][ + a_int + ] = {} + for subproblem, stages in dir_structure_intermediate[w][h][a].items(): + weather_hydro_avail_subproblem_stage_dict_final[w_int][h_int][ + a_int + ][subproblem] = stages + + print("IS THIS THE PROBLEM", + weather_hydro_avail_subproblem_stage_dict_final) - directory_for_subproblem_structure = os.path.join( - scenario_directory, - weather_dir_for_subproblem_structure, - hydro_dir_for_subproblem_structure, - availability_dir_for_subproblem_structure, + return ScenarioStructure( + weather_hydro_avail_subproblem_stage_dict=weather_hydro_avail_subproblem_stage_dict_final, + weather_iteration_flag=weather_iteration_flag, + hydro_iteration_flag=hydro_iteration_flag, + availability_iteration_flag=availability_iteration_flag, + subproblem_flag=subproblem_flag, + stage_flag=stage_flag, ) - return directory_for_subproblem_structure + +def check_dict_key_for_string_recursive(d, starting_string): + # print(d, starting_string) + if not d: + return False + else: + for key, value in d.items(): + if key.startswith(starting_string): + return True + if isinstance(value, dict): + if check_dict_key_for_string_recursive(value, starting_string): + return True + return False def dir_strings_to_iteration_numbers_list(dir_list, starting_string): @@ -583,37 +535,9 @@ def dir_strings_to_iteration_numbers_list(dir_list, starting_string): return iteration_numbers_list -def check_subproblem_structure(subproblem_main_directory): - # Convert to integers - subproblem_directories = [ - str(i) for i in check_for_integer_subdirectories(subproblem_main_directory) - ] - - # Make dictionary for the stages by subproblem, starting with empty - # list for each subproblem - stages_by_subproblem = {subp: [] for subp in subproblem_directories} - - # If we have subproblems, check for stage subdirectories for each - # subproblem directory - if subproblem_directories: - for subproblem in subproblem_directories: - subproblem_dir = os.path.join(subproblem_main_directory, subproblem) - # Convert to integers - stages = [str(i) for i in check_for_integer_subdirectories(subproblem_dir)] - if stages: - stages_by_subproblem[subproblem] = stages - else: - # If we didn't find stage directories, we have a single - # stage - # Downstream, we need a list with just 1 as member - stages_by_subproblem[subproblem] = [1] - else: - # If we didn't find integer directories, we have a single subproblem - # with a single stage - # Downstream, we need {1: [1]} - stages_by_subproblem[1] = [1] - - return stages_by_subproblem +import os +from pathlib import Path +from typing import Dict, Union class SolverOptions(object): @@ -686,3 +610,52 @@ def get_scenario_table_columns(conn): column_names = [description[0] for description in scenario_query.description] return column_names + + +def dir_to_nested_dict( + path: Union[str, Path], current_level, max_level: int = 5 +) -> Dict[str, dict]: + """ + Recursively convert a directory structure to a nested dictionary. + + Args: + path: Path to the directory to convert + + Returns: + Nested dictionary where keys are directory names and values are + either empty dicts (for leaf directories) or nested dicts (for + directories containing subdirectories) + """ + if current_level > max_level: + return + + path = Path(path) + + if not path.is_dir(): + raise ValueError(f"{path} is not a directory") + + result = {} + + # Iterate through items in the directory + for item in sorted(path.iterdir()): + # Only process directories, skip files + # Only add directories that are integers or start with one of + # weather_iteration_, hydro_iteration_, or availability_iteration_ + if item.is_dir() and ( + str(item.name).isdigit() + or str(item.name).startswith("weather_iteration_") + or str(item.name).startswith("hydro_iteration_") + or str(item.name).startswith("availability_iteration_") + ): + # Recursively process subdirectories + subdirs = dir_to_nested_dict(item, current_level + 1, max_level) + # If subdirectory has no subdirs, use empty dict + result[item.name] = subdirs if subdirs else {} + + return result + + +def get_dictionary_depth(d): + if isinstance(d, dict): + return 1 + (max(map(get_dictionary_depth, d.values()), default=0)) + return 0 diff --git a/gridpath/get_scenario_inputs.py b/gridpath/get_scenario_inputs.py index 533438ee0b..99ac7b40ad 100644 --- a/gridpath/get_scenario_inputs.py +++ b/gridpath/get_scenario_inputs.py @@ -82,15 +82,10 @@ def write_model_inputs( # files delete_prior_aux_files(scenario_directory=scenario_directory) - # Determine whether we will have iteration (weather, hydro iteration), - # and subproblem and stage directories - # The subproblem structure is the same within each iteration - iteration_directory_strings = ScenarioDirectoryStructure( + # Get the directory structure from the scenario structure + scenario_directory_structure = ScenarioDirectoryStructure( scenario_structure - ).ITERATION_DIRECTORIES - subproblem_stage_directory_strings = ScenarioDirectoryStructure( - scenario_structure - ).SUBPROBLEM_STAGE_DIRECTORIES + ).SCENARIO_DIRECTORY_STRUCTURE # Do a few checks on parallelization request if n_parallel_subproblems < 1: @@ -111,13 +106,13 @@ def write_model_inputs( # If no parallelization requested, loop through the iterations # and subproblems if n_parallel_subproblems == 1: - for weather_iteration_str in iteration_directory_strings.keys(): - for hydro_iteration_str in iteration_directory_strings[ + for weather_iteration_str in scenario_directory_structure.keys(): + for hydro_iteration_str in scenario_directory_structure[ weather_iteration_str ].keys(): - for availability_iteration_str in iteration_directory_strings[ + for availability_iteration_str in scenario_directory_structure[ weather_iteration_str - ][hydro_iteration_str]: + ][hydro_iteration_str].keys(): # We may have passed "empty_string" to avoid actual empty # strings as dictionary keys; convert to actual empty # strings here to pass to the directory creation methods @@ -127,8 +122,12 @@ def write_model_inputs( availability_iteration_str ) - for subproblem_str in subproblem_stage_directory_strings.keys(): - for stage_str in subproblem_stage_directory_strings[ + for subproblem_str in scenario_directory_structure[ + weather_iteration_str + ][hydro_iteration_str][availability_iteration_str].keys(): + for stage_str in scenario_directory_structure[ + weather_iteration_str + ][hydro_iteration_str][availability_iteration_str][ subproblem_str ]: write_inputs( @@ -145,13 +144,13 @@ def write_model_inputs( ) else: pool_data = [] - for weather_iteration_str in iteration_directory_strings.keys(): - for hydro_iteration_str in iteration_directory_strings[ + for weather_iteration_str in scenario_directory_structure.keys(): + for hydro_iteration_str in scenario_directory_structure[ weather_iteration_str ].keys(): - for availability_iteration_str in iteration_directory_strings[ + for availability_iteration_str in scenario_directory_structure[ weather_iteration_str - ][hydro_iteration_str]: + ][hydro_iteration_str].keys(): # We may have passed "empty_string" to avoid actual empty # strings as dictionary keys; convert to actual empty # strings here to pass to the directory creation methods @@ -161,24 +160,23 @@ def write_model_inputs( availability_iteration_str ) - for subproblem_str in subproblem_stage_directory_strings.keys(): - for stage_str in subproblem_stage_directory_strings[ - subproblem_str - ]: - pool_data.append( - [ - scenario_directory, - weather_iteration_str, - hydro_iteration_str, - availability_iteration_str, - subproblem_str, - stage_str, - modules_to_use, - scenario_id, - subscenarios, - db_path, - ] - ) + for subproblem_str in scenario_directory_structure[ + weather_iteration_str + ][hydro_iteration_str][availability_iteration_str].keys(): + pool_data.append( + [ + scenario_directory, + weather_iteration_str, + hydro_iteration_str, + availability_iteration_str, + subproblem_str, + stage_str, + modules_to_use, + scenario_id, + subscenarios, + db_path, + ] + ) pool_data = tuple(pool_data) @@ -495,7 +493,7 @@ def main(args=None): # Figure out which modules to use and load the modules modules_to_use = determine_modules( - features=feature_list, multi_stage=scenario_structure.MULTI_STAGE + features=feature_list, multi_stage=scenario_structure.STAGE_FLAG ) # Get appropriate inputs from database and write the .tab file model inputs diff --git a/gridpath/import_scenario_results.py b/gridpath/import_scenario_results.py index 4dd5e6298e..2076fb2ca8 100644 --- a/gridpath/import_scenario_results.py +++ b/gridpath/import_scenario_results.py @@ -80,19 +80,16 @@ def import_scenario_results_into_database( :return: """ - iteration_directory_strings = ScenarioDirectoryStructure( + scenario_directory_structure = ScenarioDirectoryStructure( scenario_structure - ).ITERATION_DIRECTORIES - subproblem_stage_directory_strings = ScenarioDirectoryStructure( - scenario_structure - ).SUBPROBLEM_STAGE_DIRECTORIES + ).SCENARIO_DIRECTORY_STRUCTURE # Hydro years first - for weather_iteration_str in iteration_directory_strings.keys(): - for hydro_iteration_str in iteration_directory_strings[ + for weather_iteration_str in scenario_directory_structure.keys(): + for hydro_iteration_str in scenario_directory_structure[ weather_iteration_str ].keys(): - for availability_iteration_str in iteration_directory_strings[ + for availability_iteration_str in scenario_directory_structure[ weather_iteration_str ][hydro_iteration_str]: # We may have passed "empty_string" to avoid actual empty @@ -123,9 +120,13 @@ def import_scenario_results_into_database( ) ) ) - for subproblem_str in subproblem_stage_directory_strings.keys(): + for subproblem_str in scenario_directory_structure[ + weather_iteration_str + ][hydro_iteration_str][availability_iteration_str].keys(): subproblem = 0 if subproblem_str == "" else int(subproblem_str) - for stage_str in subproblem_stage_directory_strings[subproblem_str]: + for stage_str in scenario_directory_structure[ + weather_iteration_str + ][hydro_iteration_str][availability_iteration_str][subproblem_str]: stage = 0 if stage_str == "" else int(stage_str) results_directory = os.path.join( scenario_directory, diff --git a/gridpath/run_scenario.py b/gridpath/run_scenario.py index f55ec73c7c..c49b4882e6 100644 --- a/gridpath/run_scenario.py +++ b/gridpath/run_scenario.py @@ -211,6 +211,13 @@ def run_optimization_for_subproblem_stage( """ + print("DIRECTORY PASSED: ", os.path.join( + scenario_directory, + weather_iteration_directory, + hydro_iteration_directory, + availability_iteration_directory, + subproblem_directory, + stage_directory,)) # If directed to do so, log optimization run if parsed_arguments.log: logs_directory = create_logs_directory_if_not_exists( @@ -426,6 +433,7 @@ def run_optimization_for_subproblem( Check if there are stages in the subproblem; if not solve subproblem; if, yes, solve each stage sequentially """ + print("DO WE MAKE IT HERE") subproblem = 1 if subproblem_directory == "" else int(subproblem_directory) for stage_directory in stage_directories: @@ -480,8 +488,7 @@ def run_optimization_for_subproblem_pool(pool_datum): def solve_sequentially( - iteration_directory_strings, - subproblem_stage_directory_strings, + scenario_directory_structure, scenario_directory, scenario_structure, parsed_arguments, @@ -490,12 +497,13 @@ def solve_sequentially( # objective function values objective_values = {} + print("SCENARIO DIR STRUCTURE IN SOLVE ", scenario_directory_structure) # TODO: refactor this - for weather_iteration_str in iteration_directory_strings.keys(): - for hydro_iteration_str in iteration_directory_strings[ + for weather_iteration_str in scenario_directory_structure.keys(): + for hydro_iteration_str in scenario_directory_structure[ weather_iteration_str ].keys(): - for availability_iteration_str in iteration_directory_strings[ + for availability_iteration_str in scenario_directory_structure[ weather_iteration_str ][hydro_iteration_str]: # We may have passed "empty_string" to avoid actual empty @@ -507,7 +515,9 @@ def solve_sequentially( availability_iteration_str ) - for subproblem_str in subproblem_stage_directory_strings.keys(): + for subproblem_str in scenario_directory_structure[ + weather_iteration_str + ][hydro_iteration_str][availability_iteration_str].keys(): subproblem = 1 if subproblem_str == "" else int(subproblem_str) # Write pass through input file headers @@ -518,7 +528,7 @@ def solve_sequentially( # alternatively be created by the first stage that # exports pass through inputs, but this will require # changes to the formulation (for commitment) - if scenario_structure.MULTI_STAGE: + if scenario_structure.STAGE_FLAG: create_pass_through_inputs( scenario_directory, scenario_structure, @@ -542,10 +552,12 @@ def solve_sequentially( hydro_iteration_directory=hydro_iteration_str, availability_iteration_directory=availability_iteration_str, subproblem_directory=subproblem_str, - stage_directories=subproblem_stage_directory_strings[ + stage_directories=scenario_directory_structure[ + weather_iteration_str + ][hydro_iteration_str][availability_iteration_str][ subproblem_str ], - multi_stage=scenario_structure.MULTI_STAGE, + multi_stage=scenario_structure.STAGE_FLAG, parsed_arguments=parsed_arguments, objective_values=objective_values, ) @@ -572,12 +584,10 @@ def run_scenario( 'testing' mode. """ - iteration_directory_strings = ScenarioDirectoryStructure( - scenario_structure - ).ITERATION_DIRECTORIES - subproblem_stage_directory_strings = ScenarioDirectoryStructure( + print("RUN SCENARIO FUNCTION CALLED") + scenario_directory_structure = ScenarioDirectoryStructure( scenario_structure - ).SUBPROBLEM_STAGE_DIRECTORIES + ).SCENARIO_DIRECTORY_STRUCTURE # TODO: consolidate parallelization checks try: @@ -600,9 +610,12 @@ def run_scenario( # If parallelization is not requested, solve sequentially if n_parallel_subproblems == 1: + print("UNDER no parallelization") + print(scenario_directory_structure) + print(scenario_directory) + print(scenario_structure.WEATHER_HYDRO_AVAIL_SUBPROBLEM_STAGE_DICT) objective_values = solve_sequentially( - iteration_directory_strings=iteration_directory_strings, - subproblem_stage_directory_strings=subproblem_stage_directory_strings, + scenario_directory_structure=scenario_directory_structure, scenario_directory=scenario_directory, scenario_structure=scenario_structure, parsed_arguments=parsed_arguments, @@ -624,7 +637,7 @@ def run_scenario( "sequentially." ) objective_values = solve_sequentially( - iteration_directory_strings=iteration_directory_strings, + scenario_directory_structure=scenario_directory_structure, subproblem_stage_directory_strings=subproblem_stage_directory_strings, scenario_directory=scenario_directory, scenario_structure=scenario_structure, @@ -641,11 +654,11 @@ def run_scenario( manager = Manager() objective_values = manager.dict() - for weather_iteration_str in iteration_directory_strings.keys(): - for hydro_iteration_str in iteration_directory_strings[ + for weather_iteration_str in scenario_directory_structure.keys(): + for hydro_iteration_str in scenario_directory_structure[ weather_iteration_str ].keys(): - for availability_iteration_str in iteration_directory_strings[ + for availability_iteration_str in scenario_directory_structure[ weather_iteration_str ][hydro_iteration_str]: # We may have passed "empty_string" to avoid actual empty @@ -659,7 +672,7 @@ def run_scenario( availability_iteration_str ) for subproblem_str in subproblem_stage_directory_strings.keys(): - if scenario_structure.MULTI_STAGE: + if scenario_structure.STAGE_FLAG: create_pass_through_inputs( scenario_directory, scenario_structure, @@ -691,11 +704,11 @@ def run_scenario( pool = get_context("spawn").Pool(n_parallel_subproblems) pool_data = [] - for weather_iteration_str in iteration_directory_strings.keys(): - for hydro_iteration_str in iteration_directory_strings[ + for weather_iteration_str in scenario_directory_structure.keys(): + for hydro_iteration_str in scenario_directory_structure[ weather_iteration_str ].keys(): - for availability_iteration_str in iteration_directory_strings[ + for availability_iteration_str in scenario_directory_structure[ weather_iteration_str ][hydro_iteration_str]: # We may have passed "empty_string" to avoid actual empty @@ -717,7 +730,7 @@ def run_scenario( availability_iteration_str, subproblem_str, subproblem_stage_directory_strings[subproblem_str], - scenario_structure.MULTI_STAGE, + scenario_structure.STAGE_FLAG, parsed_arguments, objective_values, ] @@ -741,7 +754,7 @@ def create_pass_through_inputs( ): modules_to_use, loaded_modules = set_up_gridpath_modules( scenario_directory=scenario_directory, - multi_stage=scenario_structure.MULTI_STAGE, + multi_stage=scenario_structure.STAGE_FLAG, ) pass_through_directory = os.path.join( scenario_directory, diff --git a/gridpath/validate_inputs.py b/gridpath/validate_inputs.py index fe71c34609..856ca6be63 100644 --- a/gridpath/validate_inputs.py +++ b/gridpath/validate_inputs.py @@ -38,7 +38,7 @@ def validate_inputs( - subproblems, + scenario_structure, loaded_modules, scenario_id, weather_iteration, @@ -50,8 +50,8 @@ def validate_inputs( """ " For each module, load the inputs from the database and validate them - :param subproblems: SubProblems object with info on the subproblem/stage - structure + :param scenario_structure: ScenarioStructure object with info on the + iteration and subproblem/stage structure :param loaded_modules: list of imported modules (Python objects) :param subscenarios: SubScenarios object with all subscenario info @@ -67,23 +67,27 @@ def validate_inputs( # each table in the database? Problem is that you don't necessarily want # to check the full table but only the appropriate subscenario - subproblems_list = subproblems.SUBPROBLEM_STAGES.keys() - for subproblem in subproblems_list: - stages = subproblems.SUBPROBLEM_STAGES[subproblem] - for stage in stages: - # 1. input validation within each module - for m in loaded_modules: - if hasattr(m, "validate_inputs"): - m.validate_inputs( - scenario_id=scenario_id, - subscenarios=subscenarios, - weather_iteration=weather_iteration, - hydro_iteration=hydro_iteration, - availability_iteration=availability_iteration, - subproblem=subproblem, - stage=stage, - conn=conn, - ) + for w in scenario_structure.WEATHER_HYDRO_AVAIL_SUBPROBLEM_STAGE_DICT.keys(): + for h in scenario_structure.WEATHER_HYDRO_AVAIL_SUBPROBLEM_STAGE_DICT[w].keys(): + for a in scenario_structure.WEATHER_HYDRO_AVAIL_SUBPROBLEM_STAGE_DICT[w][h]: + for subproblem in ( + scenario_structure.WEATHER_HYDRO_AVAIL_SUBPROBLEM_STAGE_DICT + )[w][h][w].keys(): + for stage in ( + scenario_structure.WEATHER_HYDRO_AVAIL_SUBPROBLEM_STAGE_DICT + )[w][h][w][subproblem]: + for m in loaded_modules: + if hasattr(m, "validate_inputs"): + m.validate_inputs( + scenario_id=scenario_id, + subscenarios=subscenarios, + weather_iteration=w, + hydro_iteration=h, + availability_iteration=a, + subproblem=subproblem, + stage=stage, + conn=conn, + ) # 2. input validation across modules # make sure geography and projects are in line @@ -428,25 +432,28 @@ def main(args=None): # the stages_flag to True to pass to determine_modules below # This tells the determine_modules function to include the # stages-related modules - stages_flag = any( - [ - len(scenario_structure.SUBPROBLEM_STAGES[subp]) > 1 - for subp in list(scenario_structure.SUBPROBLEM_STAGES.keys()) - ] - ) + stages_flag = scenario_structure.STAGE_FLAG modules_to_use = determine_modules( features=feature_list, multi_stage=stages_flag ) loaded_modules = load_modules(modules_to_use=modules_to_use) # Read in inputs from db and validate inputs for loaded modules - for weather_iteration in scenario_structure.ITERATION_STRUCTURE.keys(): - for hydro_iteration in scenario_structure.ITERATION_STRUCTURE[ + for ( + weather_iteration + ) in scenario_structure.WEATHER_HYDRO_AVAIL_SUBPROBLEM_STAGE_DICT.keys(): + for ( + hydro_iteration + ) in scenario_structure.WEATHER_HYDRO_AVAIL_SUBPROBLEM_STAGE_DICT[ weather_iteration ].keys(): - for availability_iteration in scenario_structure.ITERATION_STRUCTURE[ + for ( + availability_iteration + ) in scenario_structure.WEATHER_HYDRO_AVAIL_SUBPROBLEM_STAGE_DICT[ weather_iteration - ][hydro_iteration]: + ][ + hydro_iteration + ]: validate_inputs( scenario_structure, loaded_modules, From 6f316db69e67fdee567dac78ac6e6efa386d8ec7 Mon Sep 17 00:00:00 2001 From: Ana Mileva Date: Thu, 30 Apr 2026 14:54:34 -0700 Subject: [PATCH 2/3] What seems to be a fully working version --- examples/hydro_system/features.csv | 2 - examples/hydro_system/inputs/fuel_prices.tab | 1 - examples/hydro_system/inputs/fuels.tab | 1 - .../inputs/horizon_timepoints.tab | 25 ---- examples/hydro_system/inputs/horizons.tab | 2 - examples/hydro_system/inputs/load_mw.tab | 25 ---- examples/hydro_system/inputs/load_zones.tab | 2 - examples/hydro_system/inputs/periods.tab | 2 - examples/hydro_system/inputs/powerhouses.tab | 4 - examples/hydro_system/inputs/projects.tab | 7 - .../inputs/reservoir_target_volumes.tab | 4 - .../reservoir_volume_to_elevation_curves.tab | 4 - .../inputs/spec_capacity_period_params.tab | 7 - examples/hydro_system/inputs/timepoints.tab | 25 ---- .../hydro_system/inputs/water_flow_bounds.tab | 97 ------------ .../hydro_system/inputs/water_inflows.tab | 121 --------------- .../hydro_system/inputs/water_network.tab | 5 - .../inputs/water_node_reservoirs.tab | 4 - .../inputs/water_system_params.tab | 2 - .../results/objective_function_value.txt | 1 - .../hydro_system/results/solver_status.txt | 1 - .../results/termination_condition.txt | 1 - .../hydro_system/scenario_description.csv | 138 ------------------ examples/hydro_system/solver_options.csv | 4 - examples/hydro_system/units.csv | 15 -- .../multi_stage_flag.txt | 1 + .../multi_stage_flag.txt | 1 + .../multi_stage_flag.txt | 1 + .../multi_stage_flag.txt | 1 + examples/open_data/inputs/carbon_cap.tab | 2 - .../open_data/inputs/carbon_cap_zones.tab | 2 - examples/open_data/inputs/fuel_prices.tab | 7 - examples/open_data/inputs/fuels.tab | 4 - .../open_data/inputs/heat_rate_curves.tab | 12 -- .../open_data/inputs/horizon_timepoints.tab | 3 - examples/open_data/inputs/horizons.tab | 2 - .../lf_reserves_down_tmp_requirement.tab | 3 - .../inputs/lf_reserves_up_tmp_requirement.tab | 3 - .../load_following_down_balancing_areas.tab | 2 - .../load_following_up_balancing_areas.tab | 2 - examples/open_data/inputs/load_mw.tab | 7 - examples/open_data/inputs/load_zones.tab | 4 - .../new_build_generator_vintage_costs.tab | 2 - .../open_data/inputs/new_build_potentials.tab | 2 - examples/open_data/inputs/periods.tab | 2 - .../inputs/project_carbon_cap_zones.tab | 4 - examples/open_data/inputs/project_fuels.tab | 7 - examples/open_data/inputs/projects.tab | 9 -- .../regulation_down_balancing_areas.tab | 2 - .../regulation_down_tmp_requirement.tab | 3 - .../inputs/regulation_up_balancing_areas.tab | 2 - .../inputs/regulation_up_tmp_requirement.tab | 3 - .../inputs/spec_capacity_period_params.tab | 8 - ...specified_transmission_line_capacities.tab | 4 - examples/open_data/inputs/timepoints.tab | 3 - .../inputs/transmission_flow_limits.tab | 5 - .../open_data/inputs/transmission_lines.tab | 4 - ...smission_simultaneous_flow_limit_lines.tab | 5 - .../transmission_simultaneous_flow_limits.tab | 3 - .../inputs/variable_generator_profiles.tab | 5 - gridpath/auxiliary/scenario_chars.py | 137 +++++++++-------- gridpath/get_scenario_inputs.py | 47 ++++-- gridpath/run_scenario.py | 27 ++-- gridpath/validate_inputs.py | 4 +- 64 files changed, 113 insertions(+), 732 deletions(-) delete mode 100644 examples/hydro_system/features.csv delete mode 100644 examples/hydro_system/inputs/fuel_prices.tab delete mode 100644 examples/hydro_system/inputs/fuels.tab delete mode 100644 examples/hydro_system/inputs/horizon_timepoints.tab delete mode 100644 examples/hydro_system/inputs/horizons.tab delete mode 100644 examples/hydro_system/inputs/load_mw.tab delete mode 100644 examples/hydro_system/inputs/load_zones.tab delete mode 100644 examples/hydro_system/inputs/periods.tab delete mode 100644 examples/hydro_system/inputs/powerhouses.tab delete mode 100644 examples/hydro_system/inputs/projects.tab delete mode 100644 examples/hydro_system/inputs/reservoir_target_volumes.tab delete mode 100644 examples/hydro_system/inputs/reservoir_volume_to_elevation_curves.tab delete mode 100644 examples/hydro_system/inputs/spec_capacity_period_params.tab delete mode 100644 examples/hydro_system/inputs/timepoints.tab delete mode 100644 examples/hydro_system/inputs/water_flow_bounds.tab delete mode 100644 examples/hydro_system/inputs/water_inflows.tab delete mode 100644 examples/hydro_system/inputs/water_network.tab delete mode 100644 examples/hydro_system/inputs/water_node_reservoirs.tab delete mode 100644 examples/hydro_system/inputs/water_system_params.tab delete mode 100644 examples/hydro_system/results/objective_function_value.txt delete mode 100644 examples/hydro_system/results/solver_status.txt delete mode 100644 examples/hydro_system/results/termination_condition.txt delete mode 100644 examples/hydro_system/scenario_description.csv delete mode 100644 examples/hydro_system/solver_options.csv delete mode 100644 examples/hydro_system/units.csv create mode 100644 examples/multi_stage_prod_cost/multi_stage_flag.txt create mode 100644 examples/multi_stage_prod_cost_linked_subproblems/multi_stage_flag.txt create mode 100644 examples/multi_stage_prod_cost_w_hydro/multi_stage_flag.txt create mode 100644 examples/multi_stage_prod_cost_w_markets/multi_stage_flag.txt delete mode 100644 examples/open_data/inputs/carbon_cap.tab delete mode 100644 examples/open_data/inputs/carbon_cap_zones.tab delete mode 100644 examples/open_data/inputs/fuel_prices.tab delete mode 100644 examples/open_data/inputs/fuels.tab delete mode 100644 examples/open_data/inputs/heat_rate_curves.tab delete mode 100644 examples/open_data/inputs/horizon_timepoints.tab delete mode 100644 examples/open_data/inputs/horizons.tab delete mode 100644 examples/open_data/inputs/lf_reserves_down_tmp_requirement.tab delete mode 100644 examples/open_data/inputs/lf_reserves_up_tmp_requirement.tab delete mode 100644 examples/open_data/inputs/load_following_down_balancing_areas.tab delete mode 100644 examples/open_data/inputs/load_following_up_balancing_areas.tab delete mode 100644 examples/open_data/inputs/load_mw.tab delete mode 100644 examples/open_data/inputs/load_zones.tab delete mode 100644 examples/open_data/inputs/new_build_generator_vintage_costs.tab delete mode 100644 examples/open_data/inputs/new_build_potentials.tab delete mode 100644 examples/open_data/inputs/periods.tab delete mode 100644 examples/open_data/inputs/project_carbon_cap_zones.tab delete mode 100644 examples/open_data/inputs/project_fuels.tab delete mode 100644 examples/open_data/inputs/projects.tab delete mode 100644 examples/open_data/inputs/regulation_down_balancing_areas.tab delete mode 100644 examples/open_data/inputs/regulation_down_tmp_requirement.tab delete mode 100644 examples/open_data/inputs/regulation_up_balancing_areas.tab delete mode 100644 examples/open_data/inputs/regulation_up_tmp_requirement.tab delete mode 100644 examples/open_data/inputs/spec_capacity_period_params.tab delete mode 100644 examples/open_data/inputs/specified_transmission_line_capacities.tab delete mode 100644 examples/open_data/inputs/timepoints.tab delete mode 100644 examples/open_data/inputs/transmission_flow_limits.tab delete mode 100644 examples/open_data/inputs/transmission_lines.tab delete mode 100644 examples/open_data/inputs/transmission_simultaneous_flow_limit_lines.tab delete mode 100644 examples/open_data/inputs/transmission_simultaneous_flow_limits.tab delete mode 100644 examples/open_data/inputs/variable_generator_profiles.tab diff --git a/examples/hydro_system/features.csv b/examples/hydro_system/features.csv deleted file mode 100644 index 25b31a379a..0000000000 --- a/examples/hydro_system/features.csv +++ /dev/null @@ -1,2 +0,0 @@ -features -water diff --git a/examples/hydro_system/inputs/fuel_prices.tab b/examples/hydro_system/inputs/fuel_prices.tab deleted file mode 100644 index 774319c793..0000000000 --- a/examples/hydro_system/inputs/fuel_prices.tab +++ /dev/null @@ -1 +0,0 @@ -fuel period month fuel_price_per_mmbtu diff --git a/examples/hydro_system/inputs/fuels.tab b/examples/hydro_system/inputs/fuels.tab deleted file mode 100644 index 9626eb5932..0000000000 --- a/examples/hydro_system/inputs/fuels.tab +++ /dev/null @@ -1 +0,0 @@ -fuel co2_intensity_tons_per_mmbtu fuel_group diff --git a/examples/hydro_system/inputs/horizon_timepoints.tab b/examples/hydro_system/inputs/horizon_timepoints.tab deleted file mode 100644 index 5b20f876de..0000000000 --- a/examples/hydro_system/inputs/horizon_timepoints.tab +++ /dev/null @@ -1,25 +0,0 @@ -horizon balancing_type_horizon timepoint -1 day 1 -1 day 2 -1 day 3 -1 day 4 -1 day 5 -1 day 6 -1 day 7 -1 day 8 -1 day 9 -1 day 10 -1 day 11 -1 day 12 -1 day 13 -1 day 14 -1 day 15 -1 day 16 -1 day 17 -1 day 18 -1 day 19 -1 day 20 -1 day 21 -1 day 22 -1 day 23 -1 day 24 diff --git a/examples/hydro_system/inputs/horizons.tab b/examples/hydro_system/inputs/horizons.tab deleted file mode 100644 index 43f159c7bf..0000000000 --- a/examples/hydro_system/inputs/horizons.tab +++ /dev/null @@ -1,2 +0,0 @@ -horizon balancing_type_horizon boundary -1 day circular diff --git a/examples/hydro_system/inputs/load_mw.tab b/examples/hydro_system/inputs/load_mw.tab deleted file mode 100644 index 35d81cf0cd..0000000000 --- a/examples/hydro_system/inputs/load_mw.tab +++ /dev/null @@ -1,25 +0,0 @@ -LOAD_ZONES timepoint load_mw -Zone1 1 77.84 -Zone1 2 75.76 -Zone1 3 73.52 -Zone1 4 74.16 -Zone1 5 74.16 -Zone1 6 75.84 -Zone1 7 78.96 -Zone1 8 84.744 -Zone1 9 88.264 -Zone1 10 90.464 -Zone1 11 93.984 -Zone1 12 102.528 -Zone1 13 103.872 -Zone1 14 104.256 -Zone1 15 106.272 -Zone1 16 106.368 -Zone1 17 111.456 -Zone1 18 109.248 -Zone1 19 107.904 -Zone1 20 98.384 -Zone1 21 96.096 -Zone1 22 93.104 -Zone1 23 82.656 -Zone1 24 77.868 diff --git a/examples/hydro_system/inputs/load_zones.tab b/examples/hydro_system/inputs/load_zones.tab deleted file mode 100644 index 76954fdd41..0000000000 --- a/examples/hydro_system/inputs/load_zones.tab +++ /dev/null @@ -1,2 +0,0 @@ -load_zone allow_overgeneration overgeneration_penalty_per_mw allow_unserved_energy unserved_energy_penalty_per_mwh unserved_energy_limit_mwh max_unserved_load_penalty_per_mw max_unserved_load_limit_mw export_penalty_cost_per_mwh -Zone1 1 99999999.0 1 99999999.0 . 0.0 . 0.0 diff --git a/examples/hydro_system/inputs/periods.tab b/examples/hydro_system/inputs/periods.tab deleted file mode 100644 index 932c171114..0000000000 --- a/examples/hydro_system/inputs/periods.tab +++ /dev/null @@ -1,2 +0,0 @@ -period discount_factor period_start_year period_end_year hours_in_period_timepoints -2026 1.0 2026.0 2027.0 8760.0 diff --git a/examples/hydro_system/inputs/powerhouses.tab b/examples/hydro_system/inputs/powerhouses.tab deleted file mode 100644 index dc54630821..0000000000 --- a/examples/hydro_system/inputs/powerhouses.tab +++ /dev/null @@ -1,4 +0,0 @@ -powerhouse powerhouse_water_node tailwater_elevation headloss_factor turbine_efficiency -Powerhouse1 Water_Node_1 800.0 0.05 0.9 -Powerhouse2 Water_Node_2 400.0 0.05 0.9 -Powerhouse3 Water_Node_3 100.0 0.05 0.9 diff --git a/examples/hydro_system/inputs/projects.tab b/examples/hydro_system/inputs/projects.tab deleted file mode 100644 index 46601eb863..0000000000 --- a/examples/hydro_system/inputs/projects.tab +++ /dev/null @@ -1,7 +0,0 @@ -project capacity_type availability_type operational_type balancing_type_project technology load_zone variable_om_cost_per_mwh min_stable_level_fraction unit_size_mw startup_cost_per_mw shutdown_cost_per_mw startup_fuel_mmbtu_per_mw startup_plus_ramp_up_rate shutdown_plus_ramp_down_rate ramp_up_when_on_rate ramp_down_when_on_rate ramp_up_violation_penalty ramp_down_violation_penalty min_up_time_hours min_up_time_violation_penalty min_down_time_hours min_down_time_violation_penalty allow_startup_shutdown_power storage_efficiency charging_efficiency discharging_efficiency charging_capacity_multiplier discharging_capacity_multiplier minimum_duration_hours maximum_duration_hours aux_consumption_frac_capacity aux_consumption_frac_power last_commitment_stage curtailment_cost_per_pwh powerunithour_per_fuelunit soc_penalty_cost_per_energyunit soc_last_tmp_penalty_cost_per_energyunit partial_availability_threshold nonfuel_carbon_emissions_per_mwh powerhouse generator_efficiency -Gen11 gen_spec exogenous gen_hydro_water day Hydro_Water Zone1 0.0 . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Powerhouse1 0.95 -Gen12 gen_spec exogenous gen_hydro_water day Hydro_Water Zone1 0.0 . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Powerhouse1 0.95 -Gen21 gen_spec exogenous gen_hydro_water day Hydro_Water Zone1 0.0 . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Powerhouse2 0.95 -Gen22 gen_spec exogenous gen_hydro_water day Hydro_Water Zone1 0.0 . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Powerhouse2 0.95 -Gen31 gen_spec exogenous gen_hydro_water day Hydro_Water Zone1 0.0 . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Powerhouse3 0.95 -Gen32 gen_spec exogenous gen_hydro_water day Hydro_Water Zone1 0.0 . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . Powerhouse3 0.95 diff --git a/examples/hydro_system/inputs/reservoir_target_volumes.tab b/examples/hydro_system/inputs/reservoir_target_volumes.tab deleted file mode 100644 index 2af68ba6fb..0000000000 --- a/examples/hydro_system/inputs/reservoir_target_volumes.tab +++ /dev/null @@ -1,4 +0,0 @@ -reservoir timepoint reservoir_target_starting_volume -Water_Node_1 1.0 23750 -Water_Node_2 1.0 1500 -Water_Node_3 1.0 62.5 diff --git a/examples/hydro_system/inputs/reservoir_volume_to_elevation_curves.tab b/examples/hydro_system/inputs/reservoir_volume_to_elevation_curves.tab deleted file mode 100644 index a04d515de0..0000000000 --- a/examples/hydro_system/inputs/reservoir_volume_to_elevation_curves.tab +++ /dev/null @@ -1,4 +0,0 @@ -reservoir segment volume_to_elevation_slope volume_to_elevation_intercept -Water_Node_1 1 0.04 0.0 -Water_Node_2 1 0.4 0.0 -Water_Node_3 1 4.0 0.0 diff --git a/examples/hydro_system/inputs/spec_capacity_period_params.tab b/examples/hydro_system/inputs/spec_capacity_period_params.tab deleted file mode 100644 index 638f837bb9..0000000000 --- a/examples/hydro_system/inputs/spec_capacity_period_params.tab +++ /dev/null @@ -1,7 +0,0 @@ -project period specified_capacity_mw hyb_gen_specified_capacity_mw hyb_stor_specified_capacity_mw specified_stor_capacity_mwh fuel_production_capacity_fuelunitperhour fuel_release_capacity_fuelunitperhour fuel_storage_capacity_fuelunit fixed_cost_per_mw_yr hyb_gen_fixed_cost_per_mw_yr hyb_stor_fixed_cost_per_mw_yr fixed_cost_per_stor_mwh_yr fuel_production_capacity_fixed_cost_per_fuelunitperhour_yr fuel_release_capacity_fixed_cost_per_fuelunitperhour_yr fuel_storage_capacity_fixed_cost_per_fuelunit_yr -Gen11 2026 225.0 0.0 -Gen12 2026 225.0 0.0 -Gen21 2026 80.0 0.0 -Gen22 2026 80.0 0.0 -Gen31 2026 85.0 0.0 -Gen32 2026 85.0 0.0 diff --git a/examples/hydro_system/inputs/timepoints.tab b/examples/hydro_system/inputs/timepoints.tab deleted file mode 100644 index 3f22e2346b..0000000000 --- a/examples/hydro_system/inputs/timepoints.tab +++ /dev/null @@ -1,25 +0,0 @@ -timepoint period timepoint_weight number_of_hours_in_timepoint previous_stage_timepoint_map month day_of_month hour_of_day -1 2026 365.0 1 . 1 1 1.0 -2 2026 365.0 1 . 1 1 2.0 -3 2026 365.0 1 . 1 1 3.0 -4 2026 365.0 1 . 1 1 4.0 -5 2026 365.0 1 . 1 1 5.0 -6 2026 365.0 1 . 1 1 6.0 -7 2026 365.0 1 . 1 1 7.0 -8 2026 365.0 1 . 1 1 8.0 -9 2026 365.0 1 . 1 1 9.0 -10 2026 365.0 1 . 1 1 10.0 -11 2026 365.0 1 . 1 1 11.0 -12 2026 365.0 1 . 1 1 12.0 -13 2026 365.0 1 . 1 1 13.0 -14 2026 365.0 1 . 1 1 14.0 -15 2026 365.0 1 . 1 1 15.0 -16 2026 365.0 1 . 1 1 16.0 -17 2026 365.0 1 . 1 1 17.0 -18 2026 365.0 1 . 1 1 18.0 -19 2026 365.0 1 . 1 1 19.0 -20 2026 365.0 1 . 1 1 20.0 -21 2026 365.0 1 . 1 1 21.0 -22 2026 365.0 1 . 1 1 22.0 -23 2026 365.0 1 . 1 1 23.0 -24 2026 365.0 1 . 1 1 24.0 diff --git a/examples/hydro_system/inputs/water_flow_bounds.tab b/examples/hydro_system/inputs/water_flow_bounds.tab deleted file mode 100644 index 3918abcda8..0000000000 --- a/examples/hydro_system/inputs/water_flow_bounds.tab +++ /dev/null @@ -1,97 +0,0 @@ -water_link timepoint min_tmp_flow_vol_per_second max_tmp_flow_vol_per_second -Water_Link_12 1.0 0.002 0.02 -Water_Link_12 2.0 0.002 0.02 -Water_Link_12 3.0 0.002 0.02 -Water_Link_12 4.0 0.002 0.02 -Water_Link_12 5.0 0.002 0.02 -Water_Link_12 6.0 0.002 0.02 -Water_Link_12 7.0 0.002 0.02 -Water_Link_12 8.0 0.002 0.02 -Water_Link_12 9.0 0.002 0.02 -Water_Link_12 10.0 0.002 0.02 -Water_Link_12 11.0 0.002 0.02 -Water_Link_12 12.0 0.002 0.02 -Water_Link_12 13.0 0.002 0.02 -Water_Link_12 14.0 0.002 0.02 -Water_Link_12 15.0 0.002 0.02 -Water_Link_12 16.0 0.002 0.02 -Water_Link_12 17.0 0.002 0.02 -Water_Link_12 18.0 0.002 0.02 -Water_Link_12 19.0 0.002 0.02 -Water_Link_12 20.0 0.002 0.02 -Water_Link_12 21.0 0.002 0.02 -Water_Link_12 22.0 0.002 0.02 -Water_Link_12 23.0 0.002 0.02 -Water_Link_12 24.0 0.002 0.02 -Water_Link_23 1.0 0.002 0.02 -Water_Link_23 2.0 0.002 0.02 -Water_Link_23 3.0 0.002 0.02 -Water_Link_23 4.0 0.002 0.02 -Water_Link_23 5.0 0.002 0.02 -Water_Link_23 6.0 0.002 0.02 -Water_Link_23 7.0 0.002 0.02 -Water_Link_23 8.0 0.002 0.02 -Water_Link_23 9.0 0.002 0.02 -Water_Link_23 10.0 0.002 0.02 -Water_Link_23 11.0 0.002 0.02 -Water_Link_23 12.0 0.002 0.02 -Water_Link_23 13.0 0.002 0.02 -Water_Link_23 14.0 0.002 0.02 -Water_Link_23 15.0 0.002 0.02 -Water_Link_23 16.0 0.002 0.02 -Water_Link_23 17.0 0.002 0.02 -Water_Link_23 18.0 0.002 0.02 -Water_Link_23 19.0 0.002 0.02 -Water_Link_23 20.0 0.002 0.02 -Water_Link_23 21.0 0.002 0.02 -Water_Link_23 22.0 0.002 0.02 -Water_Link_23 23.0 0.002 0.02 -Water_Link_23 24.0 0.002 0.02 -Water_Link_34 1.0 0.002 0.02 -Water_Link_34 2.0 0.002 0.02 -Water_Link_34 3.0 0.002 0.02 -Water_Link_34 4.0 0.002 0.02 -Water_Link_34 5.0 0.002 0.02 -Water_Link_34 6.0 0.002 0.02 -Water_Link_34 7.0 0.002 0.02 -Water_Link_34 8.0 0.002 0.02 -Water_Link_34 9.0 0.002 0.02 -Water_Link_34 10.0 0.002 0.02 -Water_Link_34 11.0 0.002 0.02 -Water_Link_34 12.0 0.002 0.02 -Water_Link_34 13.0 0.002 0.02 -Water_Link_34 14.0 0.002 0.02 -Water_Link_34 15.0 0.002 0.02 -Water_Link_34 16.0 0.002 0.02 -Water_Link_34 17.0 0.002 0.02 -Water_Link_34 18.0 0.002 0.02 -Water_Link_34 19.0 0.002 0.02 -Water_Link_34 20.0 0.002 0.02 -Water_Link_34 21.0 0.002 0.02 -Water_Link_34 22.0 0.002 0.02 -Water_Link_34 23.0 0.002 0.02 -Water_Link_34 24.0 0.002 0.02 -Water_Link_45 1.0 0.002 0.02 -Water_Link_45 2.0 0.002 0.02 -Water_Link_45 3.0 0.002 0.02 -Water_Link_45 4.0 0.002 0.02 -Water_Link_45 5.0 0.002 0.02 -Water_Link_45 6.0 0.002 0.02 -Water_Link_45 7.0 0.002 0.02 -Water_Link_45 8.0 0.002 0.02 -Water_Link_45 9.0 0.002 0.02 -Water_Link_45 10.0 0.002 0.02 -Water_Link_45 11.0 0.002 0.02 -Water_Link_45 12.0 0.002 0.02 -Water_Link_45 13.0 0.002 0.02 -Water_Link_45 14.0 0.002 0.02 -Water_Link_45 15.0 0.002 0.02 -Water_Link_45 16.0 0.002 0.02 -Water_Link_45 17.0 0.002 0.02 -Water_Link_45 18.0 0.002 0.02 -Water_Link_45 19.0 0.002 0.02 -Water_Link_45 20.0 0.002 0.02 -Water_Link_45 21.0 0.002 0.02 -Water_Link_45 22.0 0.002 0.02 -Water_Link_45 23.0 0.002 0.02 -Water_Link_45 24.0 0.002 0.02 diff --git a/examples/hydro_system/inputs/water_inflows.tab b/examples/hydro_system/inputs/water_inflows.tab deleted file mode 100644 index 4f3c9d981e..0000000000 --- a/examples/hydro_system/inputs/water_inflows.tab +++ /dev/null @@ -1,121 +0,0 @@ -water_node timepoint exogenous_water_inflow_rate_vol_per_sec -Water_Node_1 1.0 0.003 -Water_Node_1 2.0 0.003 -Water_Node_1 3.0 0.003 -Water_Node_1 4.0 0.003 -Water_Node_1 5.0 0.003 -Water_Node_1 6.0 0.003 -Water_Node_1 7.0 0.003 -Water_Node_1 8.0 0.003 -Water_Node_1 9.0 0.003 -Water_Node_1 10.0 0.003 -Water_Node_1 11.0 0.003 -Water_Node_1 12.0 0.003 -Water_Node_1 13.0 0.003 -Water_Node_1 14.0 0.003 -Water_Node_1 15.0 0.003 -Water_Node_1 16.0 0.003 -Water_Node_1 17.0 0.003 -Water_Node_1 18.0 0.003 -Water_Node_1 19.0 0.003 -Water_Node_1 20.0 0.003 -Water_Node_1 21.0 0.003 -Water_Node_1 22.0 0.003 -Water_Node_1 23.0 0.003 -Water_Node_1 24.0 0.003 -Water_Node_2 1.0 0.0001 -Water_Node_2 2.0 0.0001 -Water_Node_2 3.0 0.0001 -Water_Node_2 4.0 0.0001 -Water_Node_2 5.0 0.0001 -Water_Node_2 6.0 0.0001 -Water_Node_2 7.0 0.0001 -Water_Node_2 8.0 0.0001 -Water_Node_2 9.0 0.0001 -Water_Node_2 10.0 0.0001 -Water_Node_2 11.0 0.0001 -Water_Node_2 12.0 0.0001 -Water_Node_2 13.0 0.0001 -Water_Node_2 14.0 0.0001 -Water_Node_2 15.0 0.0001 -Water_Node_2 16.0 0.0001 -Water_Node_2 17.0 0.0001 -Water_Node_2 18.0 0.0001 -Water_Node_2 19.0 0.0001 -Water_Node_2 20.0 0.0001 -Water_Node_2 21.0 0.0001 -Water_Node_2 22.0 0.0001 -Water_Node_2 23.0 0.0001 -Water_Node_2 24.0 0.0001 -Water_Node_3 1.0 0.0001 -Water_Node_3 2.0 0.0001 -Water_Node_3 3.0 0.0001 -Water_Node_3 4.0 0.0001 -Water_Node_3 5.0 0.0001 -Water_Node_3 6.0 0.0001 -Water_Node_3 7.0 0.0001 -Water_Node_3 8.0 0.0001 -Water_Node_3 9.0 0.0001 -Water_Node_3 10.0 0.0001 -Water_Node_3 11.0 0.0001 -Water_Node_3 12.0 0.0001 -Water_Node_3 13.0 0.0001 -Water_Node_3 14.0 0.0001 -Water_Node_3 15.0 0.0001 -Water_Node_3 16.0 0.0001 -Water_Node_3 17.0 0.0001 -Water_Node_3 18.0 0.0001 -Water_Node_3 19.0 0.0001 -Water_Node_3 20.0 0.0001 -Water_Node_3 21.0 0.0001 -Water_Node_3 22.0 0.0001 -Water_Node_3 23.0 0.0001 -Water_Node_3 24.0 0.0001 -Water_Node_4 1.0 5.0e-05 -Water_Node_4 2.0 5.0e-05 -Water_Node_4 3.0 5.0e-05 -Water_Node_4 4.0 5.0e-05 -Water_Node_4 5.0 5.0e-05 -Water_Node_4 6.0 5.0e-05 -Water_Node_4 7.0 5.0e-05 -Water_Node_4 8.0 5.0e-05 -Water_Node_4 9.0 5.0e-05 -Water_Node_4 10.0 5.0e-05 -Water_Node_4 11.0 5.0e-05 -Water_Node_4 12.0 5.0e-05 -Water_Node_4 13.0 5.0e-05 -Water_Node_4 14.0 5.0e-05 -Water_Node_4 15.0 5.0e-05 -Water_Node_4 16.0 5.0e-05 -Water_Node_4 17.0 5.0e-05 -Water_Node_4 18.0 5.0e-05 -Water_Node_4 19.0 5.0e-05 -Water_Node_4 20.0 5.0e-05 -Water_Node_4 21.0 5.0e-05 -Water_Node_4 22.0 5.0e-05 -Water_Node_4 23.0 5.0e-05 -Water_Node_4 24.0 5.0e-05 -Water_Node_5 1.0 5.0e-05 -Water_Node_5 2.0 5.0e-05 -Water_Node_5 3.0 5.0e-05 -Water_Node_5 4.0 5.0e-05 -Water_Node_5 5.0 5.0e-05 -Water_Node_5 6.0 5.0e-05 -Water_Node_5 7.0 5.0e-05 -Water_Node_5 8.0 5.0e-05 -Water_Node_5 9.0 5.0e-05 -Water_Node_5 10.0 5.0e-05 -Water_Node_5 11.0 5.0e-05 -Water_Node_5 12.0 5.0e-05 -Water_Node_5 13.0 5.0e-05 -Water_Node_5 14.0 5.0e-05 -Water_Node_5 15.0 5.0e-05 -Water_Node_5 16.0 5.0e-05 -Water_Node_5 17.0 5.0e-05 -Water_Node_5 18.0 5.0e-05 -Water_Node_5 19.0 5.0e-05 -Water_Node_5 20.0 5.0e-05 -Water_Node_5 21.0 5.0e-05 -Water_Node_5 22.0 5.0e-05 -Water_Node_5 23.0 5.0e-05 -Water_Node_5 24.0 5.0e-05 diff --git a/examples/hydro_system/inputs/water_network.tab b/examples/hydro_system/inputs/water_network.tab deleted file mode 100644 index 8ecc1f9b13..0000000000 --- a/examples/hydro_system/inputs/water_network.tab +++ /dev/null @@ -1,5 +0,0 @@ -water_link water_node_from water_node_to water_link_flow_transport_time_hours -Water_Link_12 Water_Node_1 Water_Node_2 1.0 -Water_Link_23 Water_Node_2 Water_Node_3 2.0 -Water_Link_34 Water_Node_3 Water_Node_4 1.0 -Water_Link_45 Water_Node_4 Water_Node_5 2.0 diff --git a/examples/hydro_system/inputs/water_node_reservoirs.tab b/examples/hydro_system/inputs/water_node_reservoirs.tab deleted file mode 100644 index c04e905fab..0000000000 --- a/examples/hydro_system/inputs/water_node_reservoirs.tab +++ /dev/null @@ -1,4 +0,0 @@ -water_node minimum_volume_volumeunit maximum_volume_volumeunit max_spill evaporation_coefficient elevation_type -Water_Node_1 22500.0 25000.0 0.1 0.0 endogenous -Water_Node_2 1250.0 1750.0 0.1 0.0 endogenous -Water_Node_3 50.0 75.0 0.1 0.0 endogenous diff --git a/examples/hydro_system/inputs/water_system_params.tab b/examples/hydro_system/inputs/water_system_params.tab deleted file mode 100644 index 20d51e51c4..0000000000 --- a/examples/hydro_system/inputs/water_system_params.tab +++ /dev/null @@ -1,2 +0,0 @@ -water_system_balancing_type theoretical_power_coefficient -day 84.7 diff --git a/examples/hydro_system/results/objective_function_value.txt b/examples/hydro_system/results/objective_function_value.txt deleted file mode 100644 index 4acb2486a1..0000000000 --- a/examples/hydro_system/results/objective_function_value.txt +++ /dev/null @@ -1 +0,0 @@ -17520.0 \ No newline at end of file diff --git a/examples/hydro_system/results/solver_status.txt b/examples/hydro_system/results/solver_status.txt deleted file mode 100644 index b5754e2037..0000000000 --- a/examples/hydro_system/results/solver_status.txt +++ /dev/null @@ -1 +0,0 @@ -ok \ No newline at end of file diff --git a/examples/hydro_system/results/termination_condition.txt b/examples/hydro_system/results/termination_condition.txt deleted file mode 100644 index 644457f4c2..0000000000 --- a/examples/hydro_system/results/termination_condition.txt +++ /dev/null @@ -1 +0,0 @@ -optimal \ No newline at end of file diff --git a/examples/hydro_system/scenario_description.csv b/examples/hydro_system/scenario_description.csv deleted file mode 100644 index ea3b22088c..0000000000 --- a/examples/hydro_system/scenario_description.csv +++ /dev/null @@ -1,138 +0,0 @@ -scenario_id,119 -scenario_name,hydro_system -of_transmission, -of_transmission_hurdle_rates, -of_simultaneous_flow_limits, -of_lf_reserves_up, -of_lf_reserves_down, -of_regulation_up, -of_regulation_down, -of_frequency_response, -of_spinning_reserves, -of_period_energy_target, -of_horizon_energy_target, -of_transmission_target, -of_instantaneous_penetration, -of_carbon_cap, -of_track_carbon_imports, -of_carbon_tax, -of_performance_standard, -of_carbon_credits, -of_fuel_burn_limit, -of_subsidies, -of_prm, -of_capacity_transfers, -of_deliverability, -of_elcc_surface, -of_local_capacity, -of_markets, -of_water,1 -of_tuning, -temporal_scenario_id,20 -load_zone_scenario_id,1 -lf_reserves_up_ba_scenario_id,NULL -lf_reserves_down_ba_scenario_id,NULL -regulation_up_ba_scenario_id,NULL -regulation_down_ba_scenario_id,NULL -frequency_response_ba_scenario_id,NULL -spinning_reserves_ba_scenario_id,NULL -energy_target_zone_scenario_id,NULL -instantaneous_penetration_zone_scenario_id,NULL -transmission_target_zone_scenario_id,NULL -carbon_cap_zone_scenario_id,NULL -carbon_tax_zone_scenario_id,NULL -performance_standard_zone_scenario_id,NULL -carbon_credits_zone_scenario_id,NULL -carbon_cap_zones_carbon_credits_zones_scenario_id,NULL -performance_standard_zones_carbon_credits_zones_scenario_id,NULL -carbon_tax_zones_carbon_credits_zones_scenario_id,NULL -carbon_credits_params_scenario_id,NULL -fuel_burn_limit_ba_scenario_id,NULL -prm_zone_scenario_id,NULL -local_capacity_zone_scenario_id,NULL -market_scenario_id,NULL -water_system_params_scenario_id,1 -water_network_scenario_id,1 -project_portfolio_scenario_id,24 -project_operational_chars_scenario_id,1 -project_availability_scenario_id,1 -fuel_scenario_id,NULL -project_load_zone_scenario_id,1 -project_lf_reserves_up_ba_scenario_id,NULL -project_lf_reserves_down_ba_scenario_id,NULL -project_regulation_up_ba_scenario_id,NULL -project_regulation_down_ba_scenario_id,NULL -project_frequency_response_ba_scenario_id,NULL -project_spinning_reserves_ba_scenario_id,NULL -project_energy_target_zone_scenario_id,NULL -project_instantaneous_penetration_zone_scenario_id,NULL -tx_line_transmission_target_zone_scenario_id,NULL -project_carbon_cap_zone_scenario_id,NULL -project_carbon_tax_zone_scenario_id,NULL -project_carbon_tax_allowance_scenario_id,NULL -project_performance_standard_zone_scenario_id,NULL -project_carbon_credits_generation_zone_scenario_id,NULL -project_carbon_credits_purchase_zone_scenario_id,NULL -project_carbon_credits_scenario_id,NULL -project_fuel_burn_limit_ba_scenario_id,NULL -project_prm_zone_scenario_id,NULL -prm_capacity_transfer_scenario_id,NULL -prm_capacity_transfer_params_scenario_id,NULL -transmission_prm_zone_scenario_id,NULL -project_elcc_chars_scenario_id,NULL -prm_deliverability_cost_scenario_id,NULL -prm_deliverability_existing_scenario_id,NULL -prm_deliverability_potential_scenario_id,NULL -project_prm_deliverability_multipliers_scenario_id,NULL -project_local_capacity_zone_scenario_id,NULL -project_local_capacity_chars_scenario_id,NULL -load_zone_market_scenario_id,NULL -project_specified_capacity_scenario_id,1 -project_specified_fixed_cost_scenario_id,1 -fuel_price_scenario_id,NULL -project_new_cost_scenario_id,NULL -project_new_potential_scenario_id,NULL -project_new_binary_build_size_scenario_id,NULL -project_capacity_group_requirement_scenario_id,NULL -project_relative_capacity_requirement_scenario_id,NULL -project_capacity_group_scenario_id,NULL -transmission_portfolio_scenario_id,NULL -transmission_load_zone_scenario_id,NULL -transmission_specified_capacity_scenario_id,NULL -transmission_new_cost_scenario_id,NULL -transmission_availability_scenario_id,NULL -transmission_operational_chars_scenario_id,NULL -transmission_hurdle_rate_scenario_id,NULL -transmission_new_potential_scenario_id,NULL -transmission_flow_scenario_id,NULL -transmission_capacity_group_requirement_scenario_id,NULL -transmission_capacity_group_scenario_id,NULL -transmission_carbon_cap_zone_scenario_id,NULL -transmission_simultaneous_flow_limit_scenario_id,NULL -transmission_simultaneous_flow_limit_line_group_scenario_id,NULL -load_scenario_id,7 -lf_reserves_up_scenario_id,NULL -lf_reserves_down_scenario_id,NULL -regulation_up_scenario_id,NULL -regulation_down_scenario_id,NULL -frequency_response_scenario_id,NULL -spinning_reserves_scenario_id,NULL -period_energy_target_scenario_id,NULL -horizon_energy_target_scenario_id,NULL -instantaneous_penetration_scenario_id,NULL -transmission_target_scenario_id,NULL -carbon_cap_target_scenario_id,NULL -carbon_tax_scenario_id,NULL -performance_standard_scenario_id,NULL -fuel_burn_limit_scenario_id,NULL -subsidy_scenario_id,NULL -prm_requirement_scenario_id,NULL -local_capacity_requirement_scenario_id,NULL -elcc_surface_scenario_id,NULL -market_price_scenario_id,NULL -market_volume_scenario_id,NULL -water_node_reservoir_scenario_id,1 -water_flow_scenario_id,1 -water_inflow_scenario_id,1 -water_powerhouse_scenario_id,1 -tuning_scenario_id,NULL diff --git a/examples/hydro_system/solver_options.csv b/examples/hydro_system/solver_options.csv deleted file mode 100644 index 7ff746441e..0000000000 --- a/examples/hydro_system/solver_options.csv +++ /dev/null @@ -1,4 +0,0 @@ -solver_name,multistart -max_iter,3000.0 -solver,ipopt -tol,1e-08 diff --git a/examples/hydro_system/units.csv b/examples/hydro_system/units.csv deleted file mode 100644 index 50a4433558..0000000000 --- a/examples/hydro_system/units.csv +++ /dev/null @@ -1,15 +0,0 @@ -metric,unit -power,MW -energy,MWh -fuel_energy,MMBtu -cost,USD -carbon_emissions,tonnes CO2 -time_for_cost,yr. -heat_rate,MMBtu/MWh -fuel_emission_intensity,tonnes CO2/MMBtu -fuel_price,USD/MMBtu -variable_om_cost,USD/MWh -fixed_om_cost,USD/MW-yr. -fixed_om_cost_energy,USD/MW-yr. -capacity_cost,USD/MW-yr. -energy_capacity_cost,USD/MWh-yr. diff --git a/examples/multi_stage_prod_cost/multi_stage_flag.txt b/examples/multi_stage_prod_cost/multi_stage_flag.txt new file mode 100644 index 0000000000..4791ed5559 --- /dev/null +++ b/examples/multi_stage_prod_cost/multi_stage_flag.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/examples/multi_stage_prod_cost_linked_subproblems/multi_stage_flag.txt b/examples/multi_stage_prod_cost_linked_subproblems/multi_stage_flag.txt new file mode 100644 index 0000000000..4791ed5559 --- /dev/null +++ b/examples/multi_stage_prod_cost_linked_subproblems/multi_stage_flag.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/examples/multi_stage_prod_cost_w_hydro/multi_stage_flag.txt b/examples/multi_stage_prod_cost_w_hydro/multi_stage_flag.txt new file mode 100644 index 0000000000..4791ed5559 --- /dev/null +++ b/examples/multi_stage_prod_cost_w_hydro/multi_stage_flag.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/examples/multi_stage_prod_cost_w_markets/multi_stage_flag.txt b/examples/multi_stage_prod_cost_w_markets/multi_stage_flag.txt new file mode 100644 index 0000000000..4791ed5559 --- /dev/null +++ b/examples/multi_stage_prod_cost_w_markets/multi_stage_flag.txt @@ -0,0 +1 @@ +True \ No newline at end of file diff --git a/examples/open_data/inputs/carbon_cap.tab b/examples/open_data/inputs/carbon_cap.tab deleted file mode 100644 index 8cfa6e2690..0000000000 --- a/examples/open_data/inputs/carbon_cap.tab +++ /dev/null @@ -1,2 +0,0 @@ -carbon_cap_zone period carbon_cap_target -Zone1 2020 300.0 diff --git a/examples/open_data/inputs/carbon_cap_zones.tab b/examples/open_data/inputs/carbon_cap_zones.tab deleted file mode 100644 index 71fb3db371..0000000000 --- a/examples/open_data/inputs/carbon_cap_zones.tab +++ /dev/null @@ -1,2 +0,0 @@ -carbon_cap_zone allow_violation violation_penalty_per_emission -Zone1 0 0.0 diff --git a/examples/open_data/inputs/fuel_prices.tab b/examples/open_data/inputs/fuel_prices.tab deleted file mode 100644 index d3925717db..0000000000 --- a/examples/open_data/inputs/fuel_prices.tab +++ /dev/null @@ -1,7 +0,0 @@ -fuel period month fuel_price_per_mmbtu -Coal 2020 1 4.0 -Coal 2020 2 4.0 -Gas 2020 1 5.0 -Gas 2020 2 5.0 -Uranium 2020 1 2.0 -Uranium 2020 2 2.0 diff --git a/examples/open_data/inputs/fuels.tab b/examples/open_data/inputs/fuels.tab deleted file mode 100644 index 28fb3cd64a..0000000000 --- a/examples/open_data/inputs/fuels.tab +++ /dev/null @@ -1,4 +0,0 @@ -fuel co2_intensity_tons_per_mmbtu fuel_group -Coal 0.09552 Solid -Gas 0.05306 Gas -Uranium 0.0 Other diff --git a/examples/open_data/inputs/heat_rate_curves.tab b/examples/open_data/inputs/heat_rate_curves.tab deleted file mode 100644 index 55da0e3cfd..0000000000 --- a/examples/open_data/inputs/heat_rate_curves.tab +++ /dev/null @@ -1,12 +0,0 @@ -project period load_point_fraction average_heat_rate_mmbtu_per_mwh -Coal 0 0.4 1250.0 -Coal 0 1.0 506.0 -Gas_CCGT 0 0.5 506.0 -Gas_CCGT 0 1.0 256.0 -Gas_CCGT_z2 0 0.5 506.0 -Gas_CCGT_z2 0 1.0 256.0 -Gas_CCGT_z3 0 0.5 350.0 -Gas_CCGT_z3 0 1.0 200.0 -Gas_CT 0 0.4 208.33333 -Gas_CT 0 1.0 88.13333 -Nuclear 0 1.0 1666.67 diff --git a/examples/open_data/inputs/horizon_timepoints.tab b/examples/open_data/inputs/horizon_timepoints.tab deleted file mode 100644 index ba17413487..0000000000 --- a/examples/open_data/inputs/horizon_timepoints.tab +++ /dev/null @@ -1,3 +0,0 @@ -horizon balancing_type_horizon timepoint -202001 day 20200101 -202001 day 20200102 diff --git a/examples/open_data/inputs/horizons.tab b/examples/open_data/inputs/horizons.tab deleted file mode 100644 index 169518ded0..0000000000 --- a/examples/open_data/inputs/horizons.tab +++ /dev/null @@ -1,2 +0,0 @@ -horizon balancing_type_horizon boundary -202001 day circular diff --git a/examples/open_data/inputs/lf_reserves_down_tmp_requirement.tab b/examples/open_data/inputs/lf_reserves_down_tmp_requirement.tab deleted file mode 100644 index 891ff1859a..0000000000 --- a/examples/open_data/inputs/lf_reserves_down_tmp_requirement.tab +++ /dev/null @@ -1,3 +0,0 @@ -ba timepoint requirement -Zone1 20200101 1.0 -Zone1 20200102 2.0 diff --git a/examples/open_data/inputs/lf_reserves_up_tmp_requirement.tab b/examples/open_data/inputs/lf_reserves_up_tmp_requirement.tab deleted file mode 100644 index 891ff1859a..0000000000 --- a/examples/open_data/inputs/lf_reserves_up_tmp_requirement.tab +++ /dev/null @@ -1,3 +0,0 @@ -ba timepoint requirement -Zone1 20200101 1.0 -Zone1 20200102 2.0 diff --git a/examples/open_data/inputs/load_following_down_balancing_areas.tab b/examples/open_data/inputs/load_following_down_balancing_areas.tab deleted file mode 100644 index 7e4fdfbf29..0000000000 --- a/examples/open_data/inputs/load_following_down_balancing_areas.tab +++ /dev/null @@ -1,2 +0,0 @@ -balancing_area allow_violation violation_penalty_per_mw reserve_to_energy_adjustment -Zone1 1 999999999.0 . diff --git a/examples/open_data/inputs/load_following_up_balancing_areas.tab b/examples/open_data/inputs/load_following_up_balancing_areas.tab deleted file mode 100644 index 7e4fdfbf29..0000000000 --- a/examples/open_data/inputs/load_following_up_balancing_areas.tab +++ /dev/null @@ -1,2 +0,0 @@ -balancing_area allow_violation violation_penalty_per_mw reserve_to_energy_adjustment -Zone1 1 999999999.0 . diff --git a/examples/open_data/inputs/load_mw.tab b/examples/open_data/inputs/load_mw.tab deleted file mode 100644 index 0afc616217..0000000000 --- a/examples/open_data/inputs/load_mw.tab +++ /dev/null @@ -1,7 +0,0 @@ -LOAD_ZONES timepoint load_mw -Zone1 20200101 10.0 -Zone1 20200102 20.0 -Zone2 20200101 10.0 -Zone2 20200102 10.0 -Zone3 20200101 30.0 -Zone3 20200102 30.0 diff --git a/examples/open_data/inputs/load_zones.tab b/examples/open_data/inputs/load_zones.tab deleted file mode 100644 index 1f2c0f632f..0000000000 --- a/examples/open_data/inputs/load_zones.tab +++ /dev/null @@ -1,4 +0,0 @@ -load_zone allow_overgeneration overgeneration_penalty_per_mw allow_unserved_energy unserved_energy_penalty_per_mwh unserved_energy_limit_mwh max_unserved_load_penalty_per_mw max_unserved_load_limit_mw export_penalty_cost_per_mwh -Zone1 1 99999999.0 1 99999999.0 . 0.0 . 0.0 -Zone2 1 99999999.0 1 99999999.0 . 0.0 . 0.0 -Zone3 1 99999999.0 1 99999999.0 . 0.0 . 0.0 diff --git a/examples/open_data/inputs/new_build_generator_vintage_costs.tab b/examples/open_data/inputs/new_build_generator_vintage_costs.tab deleted file mode 100644 index 61847973fd..0000000000 --- a/examples/open_data/inputs/new_build_generator_vintage_costs.tab +++ /dev/null @@ -1,2 +0,0 @@ -project vintage operational_lifetime_yrs fixed_cost_per_mw_yr financial_lifetime_yrs annualized_real_cost_per_mw_yr -Solar 2020 30.0 0.0 30.0 100.0 diff --git a/examples/open_data/inputs/new_build_potentials.tab b/examples/open_data/inputs/new_build_potentials.tab deleted file mode 100644 index d3a2e850c0..0000000000 --- a/examples/open_data/inputs/new_build_potentials.tab +++ /dev/null @@ -1,2 +0,0 @@ -project period min_new_build_power max_new_build_power min_capacity_power max_capacity_power min_new_build_stor_energy max_new_build_stor_energy min_capacity_stor_energy max_capacity_stor_energy -Solar 2020 . . 0.0 2.0 . . . . diff --git a/examples/open_data/inputs/periods.tab b/examples/open_data/inputs/periods.tab deleted file mode 100644 index cd509b3ba2..0000000000 --- a/examples/open_data/inputs/periods.tab +++ /dev/null @@ -1,2 +0,0 @@ -period discount_factor period_start_year period_end_year hours_in_period_timepoints -2020 1.0 2020.0 2021.0 8760.0 diff --git a/examples/open_data/inputs/project_carbon_cap_zones.tab b/examples/open_data/inputs/project_carbon_cap_zones.tab deleted file mode 100644 index b46fd6a924..0000000000 --- a/examples/open_data/inputs/project_carbon_cap_zones.tab +++ /dev/null @@ -1,4 +0,0 @@ -project carbon_cap_zone -Coal Zone1 -Gas_CCGT Zone1 -Gas_CT Zone1 diff --git a/examples/open_data/inputs/project_fuels.tab b/examples/open_data/inputs/project_fuels.tab deleted file mode 100644 index adb3bad41d..0000000000 --- a/examples/open_data/inputs/project_fuels.tab +++ /dev/null @@ -1,7 +0,0 @@ -project fuel min_fraction_in_fuel_blend max_fraction_in_fuel_blend -Coal Coal . . -Gas_CCGT Gas . . -Gas_CCGT_z2 Gas . . -Gas_CCGT_z3 Gas . . -Gas_CT Gas . . -Nuclear Uranium . . diff --git a/examples/open_data/inputs/projects.tab b/examples/open_data/inputs/projects.tab deleted file mode 100644 index 28aa283e1a..0000000000 --- a/examples/open_data/inputs/projects.tab +++ /dev/null @@ -1,9 +0,0 @@ -project capacity_type availability_type operational_type balancing_type_project technology load_zone variable_om_cost_per_mwh min_stable_level_fraction unit_size_mw startup_cost_per_mw shutdown_cost_per_mw startup_fuel_mmbtu_per_mw startup_plus_ramp_up_rate shutdown_plus_ramp_down_rate ramp_up_when_on_rate ramp_down_when_on_rate ramp_up_violation_penalty ramp_down_violation_penalty min_up_time_hours min_up_time_violation_penalty min_down_time_hours min_down_time_violation_penalty allow_startup_shutdown_power storage_efficiency charging_efficiency discharging_efficiency charging_capacity_multiplier discharging_capacity_multiplier minimum_duration_hours maximum_duration_hours aux_consumption_frac_capacity aux_consumption_frac_power last_commitment_stage curtailment_cost_per_pwh powerunithour_per_fuelunit soc_penalty_cost_per_energyunit soc_last_tmp_penalty_cost_per_energyunit partial_availability_threshold nonfuel_carbon_emissions_per_mwh lf_reserves_up_ba lf_reserves_up_derate lf_reserves_down_ba lf_reserves_down_derate regulation_up_ba regulation_up_derate regulation_down_ba regulation_down_derate lf_reserves_up_ramp_rate lf_reserves_down_ramp_rate regulation_up_ramp_rate regulation_down_ramp_rate -Coal gen_spec exogenous gen_commit_cap day Coal Zone1 1.0 0.4 6.0 1.0 0.0 . . . . . . . . . . . . . . . . . . . . . 2 . . . . . . . . . . Zone1 . Zone1 . . . . . -Gas_CCGT gen_spec exogenous gen_commit_cap day Gas Zone1 2.0 0.4 6.0 1.0 2.0 . . . . . . . . . . . . . . . . . . . . . 1 . . . . . . Zone1 . Zone1 . Zone1 . Zone1 . . . . . -Gas_CCGT_z2 gen_spec exogenous gen_commit_cap day Gas Zone2 2.0 0.4 6.0 1.0 2.0 . . . . . . . . . . . . . . . . . . . . . 1 . . . . . . . . . . . . . . . . . . -Gas_CCGT_z3 gen_spec exogenous gen_commit_cap day Gas Zone3 2.0 0.4 6.0 1.0 2.0 . . . . . . . . . . . . . . . . . . . . . 1 . . . . . . . . . . . . . . . . . . -Gas_CT gen_spec exogenous gen_commit_cap day Gas Zone1 2.0 0.4 6.0 0.0 1.0 . . . . . . . . . . . . . . . . . . . . . 3 . . . . . . . . . . . . . . . . . . -Nuclear gen_spec exogenous gen_must_run day Nuclear Zone1 1.0 . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . -Solar gen_new_lin exogenous gen_var day Solar Zone1 0.0 . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . -Wind gen_spec exogenous gen_var day Wind Zone1 0.0 . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . diff --git a/examples/open_data/inputs/regulation_down_balancing_areas.tab b/examples/open_data/inputs/regulation_down_balancing_areas.tab deleted file mode 100644 index 7e4fdfbf29..0000000000 --- a/examples/open_data/inputs/regulation_down_balancing_areas.tab +++ /dev/null @@ -1,2 +0,0 @@ -balancing_area allow_violation violation_penalty_per_mw reserve_to_energy_adjustment -Zone1 1 999999999.0 . diff --git a/examples/open_data/inputs/regulation_down_tmp_requirement.tab b/examples/open_data/inputs/regulation_down_tmp_requirement.tab deleted file mode 100644 index 891ff1859a..0000000000 --- a/examples/open_data/inputs/regulation_down_tmp_requirement.tab +++ /dev/null @@ -1,3 +0,0 @@ -ba timepoint requirement -Zone1 20200101 1.0 -Zone1 20200102 2.0 diff --git a/examples/open_data/inputs/regulation_up_balancing_areas.tab b/examples/open_data/inputs/regulation_up_balancing_areas.tab deleted file mode 100644 index 7e4fdfbf29..0000000000 --- a/examples/open_data/inputs/regulation_up_balancing_areas.tab +++ /dev/null @@ -1,2 +0,0 @@ -balancing_area allow_violation violation_penalty_per_mw reserve_to_energy_adjustment -Zone1 1 999999999.0 . diff --git a/examples/open_data/inputs/regulation_up_tmp_requirement.tab b/examples/open_data/inputs/regulation_up_tmp_requirement.tab deleted file mode 100644 index 891ff1859a..0000000000 --- a/examples/open_data/inputs/regulation_up_tmp_requirement.tab +++ /dev/null @@ -1,3 +0,0 @@ -ba timepoint requirement -Zone1 20200101 1.0 -Zone1 20200102 2.0 diff --git a/examples/open_data/inputs/spec_capacity_period_params.tab b/examples/open_data/inputs/spec_capacity_period_params.tab deleted file mode 100644 index 4b3737d771..0000000000 --- a/examples/open_data/inputs/spec_capacity_period_params.tab +++ /dev/null @@ -1,8 +0,0 @@ -project period specified_capacity_mw hyb_gen_specified_capacity_mw hyb_stor_specified_capacity_mw specified_stor_capacity_mwh fuel_production_capacity_fuelunitperhour fuel_release_capacity_fuelunitperhour fuel_storage_capacity_fuelunit fixed_cost_per_mw_yr hyb_gen_fixed_cost_per_mw_yr hyb_stor_fixed_cost_per_mw_yr fixed_cost_per_stor_mwh_yr fuel_production_capacity_fixed_cost_per_fuelunitperhour_yr fuel_release_capacity_fixed_cost_per_fuelunitperhour_yr fuel_storage_capacity_fixed_cost_per_fuelunit_yr -Coal 2020 6.0 0.0 -Gas_CCGT 2020 6.0 0.0 -Gas_CCGT_z2 2020 12.0 0.0 -Gas_CCGT_z3 2020 35.0 0.0 -Gas_CT 2020 6.0 0.0 -Nuclear 2020 6.0 0.0 -Wind 2020 2.0 0.0 diff --git a/examples/open_data/inputs/specified_transmission_line_capacities.tab b/examples/open_data/inputs/specified_transmission_line_capacities.tab deleted file mode 100644 index a31c44fd80..0000000000 --- a/examples/open_data/inputs/specified_transmission_line_capacities.tab +++ /dev/null @@ -1,4 +0,0 @@ -transmission_line period specified_tx_min_mw specified_tx_max_mw fixed_cost_per_mw_yr -Tx1 2020 -10.0 10.0 . -Tx2 2020 -10.0 10.0 . -Tx3 2020 -10.0 10.0 . diff --git a/examples/open_data/inputs/timepoints.tab b/examples/open_data/inputs/timepoints.tab deleted file mode 100644 index 841f558371..0000000000 --- a/examples/open_data/inputs/timepoints.tab +++ /dev/null @@ -1,3 +0,0 @@ -timepoint period timepoint_weight number_of_hours_in_timepoint previous_stage_timepoint_map month day_of_month hour_of_day -20200101 2020 4380.0 1 . 1 . 1.0 -20200102 2020 4380.0 1 . 1 . 2.0 diff --git a/examples/open_data/inputs/transmission_flow_limits.tab b/examples/open_data/inputs/transmission_flow_limits.tab deleted file mode 100644 index d754803b90..0000000000 --- a/examples/open_data/inputs/transmission_flow_limits.tab +++ /dev/null @@ -1,5 +0,0 @@ -transmission_line timepoint min_flow_mw max_flow_mw -Tx1 20200101 -10.0 -0.1 -Tx1 20200102 2.0 . -Tx2 20200101 . -4.0 -Tx2 20200102 2.0 . diff --git a/examples/open_data/inputs/transmission_lines.tab b/examples/open_data/inputs/transmission_lines.tab deleted file mode 100644 index 499ce0b4e2..0000000000 --- a/examples/open_data/inputs/transmission_lines.tab +++ /dev/null @@ -1,4 +0,0 @@ -transmission_line tx_capacity_type tx_availability_type tx_operational_type load_zone_from load_zone_to tx_simple_loss_factor reactance_ohms carbon_cap_zone carbon_cap_zone_import_direction tx_co2_intensity_tons_per_mwh -Tx1 tx_spec exogenous tx_simple Zone1 Zone2 . . Zone1 negative 0.6 -Tx2 tx_spec exogenous tx_simple Zone2 Zone3 . . . . . -Tx3 tx_spec exogenous tx_simple Zone1 Zone3 . . Zone1 negative 0.6 diff --git a/examples/open_data/inputs/transmission_simultaneous_flow_limit_lines.tab b/examples/open_data/inputs/transmission_simultaneous_flow_limit_lines.tab deleted file mode 100644 index 1f06424452..0000000000 --- a/examples/open_data/inputs/transmission_simultaneous_flow_limit_lines.tab +++ /dev/null @@ -1,5 +0,0 @@ -simultaneous_flow_limit transmission_line simultaneous_flow_coefficient -From_Zone1 Tx1 1 -From_Zone1 Tx3 1 -To_Zone1 Tx1 -1 -To_Zone1 Tx3 -1 diff --git a/examples/open_data/inputs/transmission_simultaneous_flow_limits.tab b/examples/open_data/inputs/transmission_simultaneous_flow_limits.tab deleted file mode 100644 index 55ff1d69f7..0000000000 --- a/examples/open_data/inputs/transmission_simultaneous_flow_limits.tab +++ /dev/null @@ -1,3 +0,0 @@ -simultaneous_flow_limit period simultaneous_flow_limit_mw -From_Zone1 2020 2.0 -To_Zone1 2020 0.0 diff --git a/examples/open_data/inputs/variable_generator_profiles.tab b/examples/open_data/inputs/variable_generator_profiles.tab deleted file mode 100644 index d06d39e883..0000000000 --- a/examples/open_data/inputs/variable_generator_profiles.tab +++ /dev/null @@ -1,5 +0,0 @@ -project timepoint cap_factor -Solar 20200101 0.05 -Solar 20200102 1.0 -Wind 20200101 0.9 -Wind 20200102 0.5 diff --git a/gridpath/auxiliary/scenario_chars.py b/gridpath/auxiliary/scenario_chars.py index dcc712c3e0..f25643ec7c 100644 --- a/gridpath/auxiliary/scenario_chars.py +++ b/gridpath/auxiliary/scenario_chars.py @@ -21,6 +21,7 @@ import os.path import pandas as pd from pathlib import Path +from typing import Dict, Union from gridpath.auxiliary.auxiliary import ( check_for_integer_subdirectories, @@ -142,13 +143,6 @@ def __init__( self.SUBPROBLEM_FLAG = subproblem_flag self.STAGE_FLAG = stage_flag - # print(weather_hydro_avail_subproblem_stage_dict) - # print(weather_iteration_flag) - # print(hydro_iteration_flag) - # print(availability_iteration_flag) - # print(subproblem_flag) - # print(stage_flag) - # For determining whether we can parallelize self.N_SUBPROBLEMS = self.calculate_n_subproblems( weather_hydro_avail_subproblem_stage_dict=weather_hydro_avail_subproblem_stage_dict, @@ -255,7 +249,6 @@ def get_scenario_structure_from_db(conn, scenario_id): WHERE scenario_id = {scenario_id};""" iter_df = pd.read_sql(iterations_query, conn) - print(iter_df) if iter_df.empty: weather_hydro_avail_subproblem_stage_dict = {0: {0: {0: None}}} @@ -266,9 +259,18 @@ def get_scenario_structure_from_db(conn, scenario_id): weather_hydro_avail_subproblem_stage_dict = {} for row in iter_df.itertuples(): ix, weather_iteration, hydro_iteration, availability_iteration = row - weather_hydro_avail_subproblem_stage_dict[weather_iteration] = { - hydro_iteration: {availability_iteration: None} - } + if weather_iteration not in weather_hydro_avail_subproblem_stage_dict: + weather_hydro_avail_subproblem_stage_dict[weather_iteration] = {} + if ( + hydro_iteration + not in weather_hydro_avail_subproblem_stage_dict[weather_iteration] + ): + weather_hydro_avail_subproblem_stage_dict[weather_iteration][ + hydro_iteration + ] = {} + weather_hydro_avail_subproblem_stage_dict[weather_iteration][ + hydro_iteration + ][availability_iteration] = None weather_iterations = get_distinct_iterations_from_db( conn, scenario_id, "weather_iteration" @@ -306,7 +308,6 @@ def get_scenario_structure_from_db(conn, scenario_id): USING (temporal_scenario_id) WHERE scenario_id = {};""".format(scenario_id)).fetchall() ] - print("SUPROBLEMS", all_subproblems) subproblem_flag = False if len(all_subproblems) == 1 else True # Store subproblems and stages in dict {subproblem: [stages]} @@ -337,8 +338,6 @@ def get_scenario_structure_from_db(conn, scenario_id): ave_it ] = stages_by_subproblem - print("HERE: ", weather_hydro_avail_subproblem_stage_dict) - return ScenarioStructure( weather_hydro_avail_subproblem_stage_dict=weather_hydro_avail_subproblem_stage_dict, weather_iteration_flag=weather_iteration_flag, @@ -371,12 +370,12 @@ def get_scenario_structure_from_disk(scenario_directory): dir_structure_from_disk = dir_to_nested_dict( Path(scenario_directory), level_start, max_level ) - print("DIR STRUCTURE FROM DISK", dir_structure_from_disk) - # First, reverse engineer the directory structure + # First, reverse engineer the directory structure that was sent to the + # scenario # We need empty strings for layers that do not exist # We'll start with what we got from disk - dir_structure_intermediate = copy.deepcopy(dir_structure_from_disk) + dir_structure_start = copy.deepcopy(dir_structure_from_disk) # Check if we have a weather iteration level; if we don't, add the level # with a single key weather_iteration_0 @@ -384,11 +383,12 @@ def get_scenario_structure_from_disk(scenario_directory): dir_structure_from_disk, "weather_iteration_" ): weather_iteration_flag = False - dir_structure_intermediate = {"weather_iteration_0": dir_structure_intermediate} + dir_structure_w_weather = { + "weather_iteration_0": copy.deepcopy(dir_structure_start) + } else: weather_iteration_flag = True - - # print(dir_structure_intermediate) + dir_structure_w_weather = copy.deepcopy(dir_structure_start) # Check if we have a hydro iteration level; if we don't, add the level # with a single hydro_iteration_0 under each weather iteration key @@ -396,13 +396,15 @@ def get_scenario_structure_from_disk(scenario_directory): dir_structure_from_disk, "hydro_iteration_" ): hydro_iteration_flag = False - for w in dir_structure_intermediate.keys(): - dir_structure_intermediate[w]["hydro_iteration_0"] = {} + dir_structure_w_weather_hydro = {} + for w in dir_structure_w_weather.keys(): + dir_structure_w_weather_hydro[w] = {} + dir_structure_w_weather_hydro[w]["hydro_iteration_0"] = copy.deepcopy( + dir_structure_w_weather + )[w] else: hydro_iteration_flag = True - - # print(dir_structure_intermediate) - # print(dir_structure_from_disk) + dir_structure_w_weather_hydro = copy.deepcopy(dir_structure_w_weather) # Check if we have an availability iteration level; if we don't, add the level # with a single availability_iteration_0 under each weather/hydro iteration @@ -411,22 +413,23 @@ def get_scenario_structure_from_disk(scenario_directory): dir_structure_from_disk, "availability_iteration_" ): availability_iteration_flag = False - for w in dir_structure_intermediate.keys(): - for h in dir_structure_intermediate[w].keys(): - dir_structure_intermediate[w][h]["availability_iteration_0"] = {} + dir_structure_w_weather_hydro_av = {} + for w in dir_structure_w_weather_hydro.keys(): + dir_structure_w_weather_hydro_av[w] = {} + for h in dir_structure_w_weather_hydro[w].keys(): + dir_structure_w_weather_hydro_av[w][h] = {} + dir_structure_w_weather_hydro_av[w][h]["availability_iteration_0"] = ( + copy.deepcopy(dir_structure_w_weather_hydro[w][h]) + ) else: availability_iteration_flag = True - # print(dir_structure_intermediate) + dir_structure_w_weather_hydro_av = copy.deepcopy(dir_structure_w_weather_hydro) # Iteration layers iteration_layers_n = sum( [weather_iteration_flag, hydro_iteration_flag, availability_iteration_flag] ) - print("ITERATION LAYERS", iteration_layers_n) - print("THIS IS THE DICT ", dir_structure_from_disk) - print("ORIGINAL DEPTH", get_dictionary_depth(dir_structure_from_disk)) dir_structure_depth = get_dictionary_depth(dir_structure_from_disk) - 1 - print("DIR STRUCTURE DEPTH", dir_structure_depth) non_iteration_layers_n = dir_structure_depth - iteration_layers_n # Finally, figure out the subproblem/stage structure from the non-iteration @@ -438,39 +441,41 @@ def get_scenario_structure_from_disk(scenario_directory): if non_iteration_layers_n == 0: subproblem_flag = False stage_flag = False - for w in dir_structure_intermediate.keys(): - for h in dir_structure_intermediate[w].keys(): - for a in dir_structure_intermediate[w][h].keys(): - dir_structure_intermediate[w][h][a] = {1: [1]} + for w in dir_structure_w_weather_hydro_av.keys(): + for h in dir_structure_w_weather_hydro_av[w].keys(): + for a in dir_structure_w_weather_hydro_av[w][h].keys(): + dir_structure_w_weather_hydro_av[w][h][a] = {1: [1]} elif non_iteration_layers_n == 1: # If stages, add the subproblem - if os.path.exists(os.path.join(scenario_directory, "pass_through_inputs")): + if os.path.exists(os.path.join(scenario_directory, "multi_stage_flag.csv")): subproblem_flag = False stage_flag = True - for w in dir_structure_intermediate.keys(): - for h in dir_structure_intermediate[w].keys(): - for a in dir_structure_intermediate[w][h].keys(): - dir_structure_intermediate[w][h][a] = { - 1: list(dir_structure_intermediate[w][h][a].keys()) + for w in dir_structure_w_weather_hydro_av.keys(): + for h in dir_structure_w_weather_hydro_av[w].keys(): + for a in dir_structure_w_weather_hydro_av[w][h].keys(): + dir_structure_w_weather_hydro_av[w][h][a] = { + 1: list(dir_structure_w_weather_hydro_av[w][h][a].keys()) } # If no stages, add them else: subproblem_flag = True stage_flag = False - for w in dir_structure_intermediate.keys(): - for h in dir_structure_intermediate[w].keys(): - for a in dir_structure_intermediate[w][h].keys(): - for subproblem in dir_structure_intermediate[w][h][a].keys(): - dir_structure_intermediate[w][h][a][subproblem] = [1] + for w in dir_structure_w_weather_hydro_av.keys(): + for h in dir_structure_w_weather_hydro_av[w].keys(): + for a in dir_structure_w_weather_hydro_av[w][h].keys(): + for subproblem in dir_structure_w_weather_hydro_av[w][h][ + a + ].keys(): + dir_structure_w_weather_hydro_av[w][h][a][subproblem] = [1] elif non_iteration_layers_n == 2: subproblem_flag = True stage_flag = True - for w in dir_structure_intermediate.keys(): - for h in dir_structure_intermediate[w].keys(): - for a in dir_structure_intermediate[w][h].keys(): - for subproblem in dir_structure_intermediate[w][h][a].keys(): - dir_structure_intermediate[w][h][a][subproblem] = list( - dir_structure_intermediate[w][h][a][subproblem].keys() + for w in dir_structure_w_weather_hydro_av.keys(): + for h in dir_structure_w_weather_hydro_av[w].keys(): + for a in dir_structure_w_weather_hydro_av[w][h].keys(): + for subproblem in dir_structure_w_weather_hydro_av[w][h][a].keys(): + dir_structure_w_weather_hydro_av[w][h][a][subproblem] = list( + dir_structure_w_weather_hydro_av[w][h][a][subproblem].keys() ) else: raise ValueError( @@ -481,30 +486,24 @@ def get_scenario_structure_from_disk(scenario_directory): # Remove the starting strings for the iteration levels and make the final # dictionary weather_hydro_avail_subproblem_stage_dict_final = {} - for w in dir_structure_intermediate.keys(): + for w in dir_structure_w_weather_hydro_av.keys(): w_int = int(w.replace(f"weather_iteration_", "")) - print(w, w_int) weather_hydro_avail_subproblem_stage_dict_final[w_int] = {} - print(weather_hydro_avail_subproblem_stage_dict_final) - for h in dir_structure_intermediate[w].keys(): + for h in dir_structure_w_weather_hydro_av[w].keys(): h_int = int(h.replace(f"hydro_iteration_", "")) - print(h, h_int) weather_hydro_avail_subproblem_stage_dict_final[w_int][h_int] = {} - print(weather_hydro_avail_subproblem_stage_dict_final) - for a in dir_structure_intermediate[w][h].keys(): + for a in dir_structure_w_weather_hydro_av[w][h].keys(): a_int = int(a.replace(f"availability_iteration_", "")) - print(a, a_int) weather_hydro_avail_subproblem_stage_dict_final[w_int][h_int][ a_int ] = {} - for subproblem, stages in dir_structure_intermediate[w][h][a].items(): + for subproblem, stages in dir_structure_w_weather_hydro_av[w][h][ + a + ].items(): weather_hydro_avail_subproblem_stage_dict_final[w_int][h_int][ a_int ][subproblem] = stages - print("IS THIS THE PROBLEM", - weather_hydro_avail_subproblem_stage_dict_final) - return ScenarioStructure( weather_hydro_avail_subproblem_stage_dict=weather_hydro_avail_subproblem_stage_dict_final, weather_iteration_flag=weather_iteration_flag, @@ -516,7 +515,6 @@ def get_scenario_structure_from_disk(scenario_directory): def check_dict_key_for_string_recursive(d, starting_string): - # print(d, starting_string) if not d: return False else: @@ -535,11 +533,6 @@ def dir_strings_to_iteration_numbers_list(dir_list, starting_string): return iteration_numbers_list -import os -from pathlib import Path -from typing import Dict, Union - - class SolverOptions(object): def __init__(self, conn, scenario_id): """ diff --git a/gridpath/get_scenario_inputs.py b/gridpath/get_scenario_inputs.py index 99ac7b40ad..58beb3b71e 100644 --- a/gridpath/get_scenario_inputs.py +++ b/gridpath/get_scenario_inputs.py @@ -163,20 +163,25 @@ def write_model_inputs( for subproblem_str in scenario_directory_structure[ weather_iteration_str ][hydro_iteration_str][availability_iteration_str].keys(): - pool_data.append( - [ - scenario_directory, - weather_iteration_str, - hydro_iteration_str, - availability_iteration_str, - subproblem_str, - stage_str, - modules_to_use, - scenario_id, - subscenarios, - db_path, - ] - ) + for stage_str in scenario_directory_structure[ + weather_iteration_str + ][hydro_iteration_str][availability_iteration_str][ + subproblem_str + ]: + pool_data.append( + [ + scenario_directory, + weather_iteration_str, + hydro_iteration_str, + availability_iteration_str, + subproblem_str, + stage_str, + modules_to_use, + scenario_id, + subscenarios, + db_path, + ] + ) pool_data = tuple(pool_data) @@ -423,6 +428,19 @@ def write_solver_options(scenario_directory, solver_options): writer.writerow([opt, solver_options.SOLVER_OPTIONS[opt]]) +def write_multi_stage_flag(scenario_directory, stage_flag): + """ + Write the multi-stage flag to the scenario directory. + """ + if stage_flag: + with open( + os.path.join(scenario_directory, "multi_stage_flag.txt"), + "w", + newline="", + ) as stage_flag_file: + stage_flag_file.write(str(stage_flag)) + + def write_linked_subproblems_map(scenario_directory, conn, subscenarios): sql = """ SELECT subproblem_id as subproblem, stage_id as stage, timepoint, @@ -485,6 +503,7 @@ def main(args=None): scenario_structure = get_scenario_structure_from_db( conn=conn, scenario_id=scenario_id ) + write_multi_stage_flag(scenario_directory, scenario_structure.STAGE_FLAG) solver_options = SolverOptions(conn=conn, scenario_id=scenario_id) # Determine requested features and use this to determine what modules to diff --git a/gridpath/run_scenario.py b/gridpath/run_scenario.py index c49b4882e6..659cb76d18 100644 --- a/gridpath/run_scenario.py +++ b/gridpath/run_scenario.py @@ -210,14 +210,6 @@ def run_optimization_for_subproblem_stage( Return the objective function (Total_Cost) value; only used in testing mode """ - - print("DIRECTORY PASSED: ", os.path.join( - scenario_directory, - weather_iteration_directory, - hydro_iteration_directory, - availability_iteration_directory, - subproblem_directory, - stage_directory,)) # If directed to do so, log optimization run if parsed_arguments.log: logs_directory = create_logs_directory_if_not_exists( @@ -433,7 +425,6 @@ def run_optimization_for_subproblem( Check if there are stages in the subproblem; if not solve subproblem; if, yes, solve each stage sequentially """ - print("DO WE MAKE IT HERE") subproblem = 1 if subproblem_directory == "" else int(subproblem_directory) for stage_directory in stage_directories: @@ -497,7 +488,6 @@ def solve_sequentially( # objective function values objective_values = {} - print("SCENARIO DIR STRUCTURE IN SOLVE ", scenario_directory_structure) # TODO: refactor this for weather_iteration_str in scenario_directory_structure.keys(): for hydro_iteration_str in scenario_directory_structure[ @@ -584,7 +574,6 @@ def run_scenario( 'testing' mode. """ - print("RUN SCENARIO FUNCTION CALLED") scenario_directory_structure = ScenarioDirectoryStructure( scenario_structure ).SCENARIO_DIRECTORY_STRUCTURE @@ -610,10 +599,6 @@ def run_scenario( # If parallelization is not requested, solve sequentially if n_parallel_subproblems == 1: - print("UNDER no parallelization") - print(scenario_directory_structure) - print(scenario_directory) - print(scenario_structure.WEATHER_HYDRO_AVAIL_SUBPROBLEM_STAGE_DICT) objective_values = solve_sequentially( scenario_directory_structure=scenario_directory_structure, scenario_directory=scenario_directory, @@ -671,7 +656,9 @@ def run_scenario( availability_iteration_str = ensure_empty_string( availability_iteration_str ) - for subproblem_str in subproblem_stage_directory_strings.keys(): + for subproblem_str in scenario_directory_structure[ + weather_iteration_str + ][hydro_iteration_str][availability_iteration_str].keys(): if scenario_structure.STAGE_FLAG: create_pass_through_inputs( scenario_directory, @@ -721,7 +708,9 @@ def run_scenario( availability_iteration_str = ensure_empty_string( availability_iteration_str ) - for subproblem_str in subproblem_stage_directory_strings.keys(): + for subproblem_str in scenario_directory_structure[ + weather_iteration_str + ][hydro_iteration_str][availability_iteration_str].keys(): pool_data.append( [ scenario_directory, @@ -729,7 +718,9 @@ def run_scenario( hydro_iteration_str, availability_iteration_str, subproblem_str, - subproblem_stage_directory_strings[subproblem_str], + scenario_directory_structure[weather_iteration_str][ + hydro_iteration_str + ][availability_iteration_str][subproblem_str], scenario_structure.STAGE_FLAG, parsed_arguments, objective_values, diff --git a/gridpath/validate_inputs.py b/gridpath/validate_inputs.py index 856ca6be63..cfee448da5 100644 --- a/gridpath/validate_inputs.py +++ b/gridpath/validate_inputs.py @@ -72,10 +72,10 @@ def validate_inputs( for a in scenario_structure.WEATHER_HYDRO_AVAIL_SUBPROBLEM_STAGE_DICT[w][h]: for subproblem in ( scenario_structure.WEATHER_HYDRO_AVAIL_SUBPROBLEM_STAGE_DICT - )[w][h][w].keys(): + )[w][h][a].keys(): for stage in ( scenario_structure.WEATHER_HYDRO_AVAIL_SUBPROBLEM_STAGE_DICT - )[w][h][w][subproblem]: + )[w][h][a][subproblem]: for m in loaded_modules: if hasattr(m, "validate_inputs"): m.validate_inputs( From a018d3d3352a9ed785ffebab1736097e2dbb2695 Mon Sep 17 00:00:00 2001 From: Ana Mileva Date: Thu, 30 Apr 2026 15:18:14 -0700 Subject: [PATCH 3/3] a few tweaks --- gridpath/auxiliary/scenario_chars.py | 78 +++++++++++++++++++--------- 1 file changed, 54 insertions(+), 24 deletions(-) diff --git a/gridpath/auxiliary/scenario_chars.py b/gridpath/auxiliary/scenario_chars.py index f25643ec7c..4509dd3e88 100644 --- a/gridpath/auxiliary/scenario_chars.py +++ b/gridpath/auxiliary/scenario_chars.py @@ -447,7 +447,21 @@ def get_scenario_structure_from_disk(scenario_directory): dir_structure_w_weather_hydro_av[w][h][a] = {1: [1]} elif non_iteration_layers_n == 1: # If stages, add the subproblem - if os.path.exists(os.path.join(scenario_directory, "multi_stage_flag.csv")): + txt_file_for_stage_flag = os.path.join( + scenario_directory, "multi_stage_flag.txt" + ) + if os.path.exists(txt_file_for_stage_flag): + with open(txt_file_for_stage_flag) as f: + with open(txt_file_for_stage_flag) as f: + check_true = f.read() + if check_true != "True": + raise ValueError( + "ERROR: Scenario directory structure appears to have " + "stages but the multi_stage_flag.txt file is not set " + "to True, indicating an upstream error in handling " + "the scenario structure." + ) + subproblem_flag = False stage_flag = True for w in dir_structure_w_weather_hydro_av.keys(): @@ -468,6 +482,26 @@ def get_scenario_structure_from_disk(scenario_directory): ].keys(): dir_structure_w_weather_hydro_av[w][h][a][subproblem] = [1] elif non_iteration_layers_n == 2: + txt_file_for_stage_flag = os.path.join( + scenario_directory, "multi_stage_flag.txt" + ) + if os.path.exists(txt_file_for_stage_flag): + with open(txt_file_for_stage_flag) as f: + check_true = f.read() + if check_true != "True": + raise ValueError( + "ERROR: Scenario directory structure appears to have " + "stages but the multi_stage_flag.txt file is not set " + "to True, indicating an upstream error in handling " + "the scenario structure." + ) + else: + raise ValueError( + "ERROR: Scenario directory structure appears to have " + "stages but the multi_stage_flag.txt does not exist, " + "indicating an upstream error in handling the scenario " + "structure." + ) subproblem_flag = True stage_flag = True for w in dir_structure_w_weather_hydro_av.keys(): @@ -514,25 +548,6 @@ def get_scenario_structure_from_disk(scenario_directory): ) -def check_dict_key_for_string_recursive(d, starting_string): - if not d: - return False - else: - for key, value in d.items(): - if key.startswith(starting_string): - return True - if isinstance(value, dict): - if check_dict_key_for_string_recursive(value, starting_string): - return True - return False - - -def dir_strings_to_iteration_numbers_list(dir_list, starting_string): - iteration_numbers_list = [d.replace(f"{starting_string}_", "") for d in dir_list] - - return iteration_numbers_list - - class SolverOptions(object): def __init__(self, conn, scenario_id): """ @@ -610,14 +625,16 @@ def dir_to_nested_dict( ) -> Dict[str, dict]: """ Recursively convert a directory structure to a nested dictionary. + Only add directories that are integers or start with one of + weather_iteration_, hydro_iteration_, or availability_iteration_. Args: - path: Path to the directory to convert + path: Path to the directory to convert Returns: - Nested dictionary where keys are directory names and values are - either empty dicts (for leaf directories) or nested dicts (for - directories containing subdirectories) + Nested dictionary where keys are directory names and values are + either empty dicts (for leaf directories) or nested dicts (for + directories containing subdirectories). """ if current_level > max_level: return @@ -648,6 +665,19 @@ def dir_to_nested_dict( return result +def check_dict_key_for_string_recursive(d, starting_string): + if not d: + return False + else: + for key, value in d.items(): + if key.startswith(starting_string): + return True + if isinstance(value, dict): + if check_dict_key_for_string_recursive(value, starting_string): + return True + return False + + def get_dictionary_depth(d): if isinstance(d, dict): return 1 + (max(map(get_dictionary_depth, d.values()), default=0))