diff --git a/rate_design/ny/hp_rates/data/tariff_map/Justfile b/rate_design/ny/hp_rates/data/tariff_map/Justfile index e4353b94..ed8e40f7 100644 --- a/rate_design/ny/hp_rates/data/tariff_map/Justfile +++ b/rate_design/ny/hp_rates/data/tariff_map/Justfile @@ -1,10 +1,11 @@ # Resolve project root via git (works from any subdirectory in the repo) project_root := `git rev-parse --show-toplevel` path_s3_resstock_metadata := "s3://data.sb/nrel/resstock/res_2024_amy2018_2/metadata" +tariff_map_dir_ny := "{{project_root}}/rate_design/ny/hp_rates/data/tariff_map" -# Usage: just map-electric-tariff [electric_utility] [SB_scenario_type] [SB_scenario_year] [upgrade_id] -# Example: just map-electric-tariff Coned default 1 00 -map-electric-tariff electric_utility SB_scenario_type SB_scenario_year upgrade_id output_dir="": +# Usage: just map-electric-tariff [electric_utility] [SB_scenario_type] [SB_scenario_year] [upgrade_id] [output_dir] +# Example: just map-electric-tariff Coned default 1 00 ./rate_design/ny/hp_rates/data/tariff_map +map-electric-tariff electric_utility SB_scenario_type SB_scenario_year upgrade_id output_dir: uv run python {{project_root}}/utils/electric_tariff_mapper.py \ --metadata_path "{{path_s3_resstock_metadata}}" \ --state NY \ @@ -12,4 +13,74 @@ map-electric-tariff electric_utility SB_scenario_type SB_scenario_year upgrade_i --electric_utility "{{electric_utility}}" \ --SB_scenario_type "{{SB_scenario_type}}" \ --SB_scenario_year "{{SB_scenario_year}}" \ - {{ if output_dir != "" { "--output_dir \"" + output_dir + "\"" } else { "" } }} + --output_dir "{{output_dir}}" + +# Some useful enumerations: + +map-electric-coned-default: + just map-electric-tariff Coned default 1 00 {{tariff_map_dir_ny}} + +map-electric-coned-seasonal: + just map-electric-tariff Coned seasonal 1 00 {{tariff_map_dir_ny}} + +map-celectric-oned-class-specific-seasonal: + just map-electric-tariff Coned class_specific_seasonal 1 00 {{tariff_map_dir_ny}} + +map-electric-national-grid-default: + just map-electric-tariff "National Grid" default 1 00 {{tariff_map_dir_ny}} + +map-electric-national-grid-seasonal: + just map-electric-tariff "National Grid" seasonal 1 00 {{tariff_map_dir_ny}} + +map-electric-national-grid-class-specific-seasonal: + just map-electric-tariff "National Grid" class_specific_seasonal 1 00 {{tariff_map_dir_ny}} + +map-electric-nyseg-default: + just map-electric-tariff NYSEG default 1 00 {{tariff_map_dir_ny}} + +map-electric-nyseg-seasonal: + just map-electric-tariff NYSEG seasonal 1 00 {{tariff_map_dir_ny}} + +map-electric-nyseg-class-specific-seasonal: + just map-electric-tariff NYSEG class_specific_seasonal 1 00 {{tariff_map_dir_ny}} + +# Usage: just map-gas-tariff [electric_utility] [upgrade_id] [output_dir] +# Example: just map-gas-tariff NYSEG 00 ./rate_design/ny/hp_rates/data/tariff_map +map-gas-tariff electric_utility upgrade_id output_dir: + uv run python {{project_root}}/utils/gas_tariff_mapper.py \ + --metadata_path "{{path_s3_resstock_metadata}}" \ + --state NY \ + --upgrade_id "{{upgrade_id}}" \ + --electric_utility "{{electric_utility}}" \ + --output_dir "{{output_dir}}" + +# Some useful enumerations: + +map-gas-coned-default: + just map-gas-tariff Coned 00 {{tariff_map_dir_ny}} + +map-gas-coned-seasonal: + just map-gas-tariff Coned 00 {{tariff_map_dir_ny}} + +map-gas-coned-class-specific-seasonal: + just map-gas-tariff Coned 00 {{tariff_map_dir_ny}} + +map-gas-national-grid-default: + just map-gas-tariff "National Grid" 00 {{tariff_map_dir_ny}} + +map-gas-national-grid-seasonal: + just map-gas-tariff "National Grid" 00 {{tariff_map_dir_ny}} + +map-gas-national-grid-class-specific-seasonal: + just map-gas-tariff "National Grid" 00 {{tariff_map_dir_ny}} + +map-gas-nyseg-default: + just map-gas-tariff NYSEG 00 {{tariff_map_dir_ny}} + +map-gas-nyseg-seasonal: + just map-gas-tariff NYSEG 00 {{tariff_map_dir_ny}} + +map-gas-nyseg-class-specific-seasonal: + just map-gas-tariff NYSEG 00 {{tariff_map_dir_ny}} + + diff --git a/rate_design/ri/hp_rates/data/tariff_map/Justfile b/rate_design/ri/hp_rates/data/tariff_map/Justfile index 985b7c32..15147e88 100644 --- a/rate_design/ri/hp_rates/data/tariff_map/Justfile +++ b/rate_design/ri/hp_rates/data/tariff_map/Justfile @@ -1,15 +1,85 @@ # Resolve project root via git (works from any subdirectory in the repo) project_root := `git rev-parse --show-toplevel` path_s3_resstock_metadata := "s3://data.sb/nrel/resstock/res_2024_amy2018_2/metadata" +tariff_map_dir_ri := "{{project_root}}/rate_design/ri/hp_rates/data/tariff_map" -# Usage: just map-electric-tariff [electric_utility] [SB_scenario_type] [SB_scenario_year] [upgrade_id] -# Example: just map-electric-tariff Coned default 1 00 -map-electric-tariff electric_utility SB_scenario_type SB_scenario_year upgrade_id output_dir="": +# Usage: just map-electric-tariff [electric_utility] [SB_scenario_type] [SB_scenario_year] [upgrade_id] [output_dir] +# Example: just map-electric-tariff Coned default 1 00 ./rate_design/ri/hp_rates/data/tariff_map +map-electric-tariff electric_utility SB_scenario_type SB_scenario_year upgrade_id output_dir: uv run python {{project_root}}/utils/electric_tariff_mapper.py \ --metadata_path "{{path_s3_resstock_metadata}}" \ --state RI \ --upgrade_id "{{upgrade_id}}" \ --electric_utility "{{electric_utility}}" \ --SB_scenario_type "{{SB_scenario_type}}" \ - --SB_scenario_year "{{SB_scenario_year}}" - {{ if output_dir != "" { "--output_dir \"" + output_dir + "\"" } else { "" } }} + --SB_scenario_year "{{SB_scenario_year}}" \ + --output_dir "{{output_dir}}" + +# Some useful enumerations: + +map-electric-coned-default: + just map-electric-tariff Coned default 1 00 {{tariff_map_dir_ri}} + +map-electric-coned-seasonal: + just map-electric-tariff Coned seasonal 1 00 {{tariff_map_dir_ri}} + +map-celectric-oned-class-specific-seasonal: + just map-electric-tariff Coned class_specific_seasonal 1 00 {{tariff_map_dir_ri}} + +map-electric-national-grid-default: + just map-electric-tariff "National Grid" default 1 00 {{tariff_map_dir_ri}} + +map-electric-national-grid-seasonal: + just map-electric-tariff "National Grid" seasonal 1 00 {{tariff_map_dir_ri}} + +map-electric-national-grid-class-specific-seasonal: + just map-electric-tariff "National Grid" class_specific_seasonal 1 00 {{tariff_map_dir_ri}} + +map-electric-nyseg-default: + just map-electric-tariff NYSEG default 1 00 {{tariff_map_dir_ri}} + +map-electric-nyseg-seasonal: + just map-electric-tariff NYSEG seasonal 1 00 {{tariff_map_dir_ri}} + +map-electric-nyseg-class-specific-seasonal: + just map-electric-tariff NYSEG class_specific_seasonal 1 00 {{tariff_map_dir_ri}} + + +# Usage: just map-gas-tariff [electric_utility] [upgrade_id] [output_dir] +# Example: just map-gas-tariff "National Grid" 00 ./rate_design/ri/hp_rates/data/tariff_map +map-gas-tariff electric_utility upgrade_id output_dir: + uv run python {{project_root}}/utils/gas_tariff_mapper.py \ + --metadata_path "{{path_s3_resstock_metadata}}" \ + --state RI \ + --upgrade_id "{{upgrade_id}}" \ + --electric_utility "{{electric_utility}}" \ + --output_dir "{{output_dir}}" + +# Some useful enumerations: + +map-gas-coned-default: + just map-gas-tariff Coned 00 {{tariff_map_dir_ri}} + +map-gas-coned-seasonal: + just map-gas-tariff Coned 00 {{tariff_map_dir_ri}} + +map-gas-coned-class-specific-seasonal: + just map-gas-tariff Coned 00 {{tariff_map_dir_ri}} + +map-gas-national-grid-default: + just map-gas-tariff "National Grid" 00 {{tariff_map_dir_ri}} + +map-gas-national-grid-seasonal: + just map-gas-tariff "National Grid" 00 {{tariff_map_dir_ri}} + +map-gas-national-grid-class-specific-seasonal: + just map-gas-tariff "National Grid" 00 {{tariff_map_dir_ri}} + +map-gas-nyseg-default: + just map-gas-tariff NYSEG 00 {{tariff_map_dir_ri}} + +map-gas-nyseg-seasonal: + just map-gas-tariff NYSEG 00 {{tariff_map_dir_ri}} + +map-gas-nyseg-class-specific-seasonal: + just map-gas-tariff NYSEG 00 {{tariff_map_dir_ri}} \ No newline at end of file diff --git a/utils/__init__.py b/utils/__init__.py index 8323e2ac..71c9f5c1 100644 --- a/utils/__init__.py +++ b/utils/__init__.py @@ -1 +1,35 @@ """Utility helpers wrapping external packages (stub).""" + +import os +import subprocess +from pathlib import Path + + +def get_aws_region(default: str = "us-west-2") -> str: + """Return AWS region from env (AWS_REGION / AWS_DEFAULT_REGION) or from AWS config (~/.aws/config) via boto3.""" + region = os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") + if region: + return region + try: + import boto3 + session = boto3.Session() + if session.region_name: + return session.region_name + except ImportError: + pass + return default + + +def get_project_root() -> Path: + """Return the repository root as an absolute path (via git rev-parse --show-toplevel).""" + result = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + cwd=Path.cwd(), + ) + if result.returncode != 0: + raise RuntimeError( + "Could not find project root (run from inside the git repository)" + ) + return Path(result.stdout.strip()).resolve() diff --git a/utils/electric_tariff_mapper.py b/utils/electric_tariff_mapper.py index 64382ced..545c0454 100644 --- a/utils/electric_tariff_mapper.py +++ b/utils/electric_tariff_mapper.py @@ -5,15 +5,10 @@ import polars as pl from cloudpathlib import S3Path +from utils import get_aws_region from utils.types import SBScenario, electric_utility -# Project root (rate-design-platform); independent of cwd or caller -_PROJECT_ROOT = Path(__file__).resolve().parent.parent -RATE_DESIGN_DIR = _PROJECT_ROOT / "rate_design" - -AWS_REGION = "us-west-2" - -STORAGE_OPTIONS = {"aws_region": AWS_REGION} +STORAGE_OPTIONS = {"aws_region": get_aws_region()} def define_electrical_tariff_key( @@ -49,12 +44,12 @@ def generate_electrical_tariff_mapping( def map_electric_tariff( - SB_metadata_lazy_df: pl.LazyFrame, + SB_metadata_df: pl.LazyFrame, electric_utility: electric_utility, SB_scenario: SBScenario, state: str, ) -> pl.LazyFrame: - utility_metadata_df = SB_metadata_lazy_df.filter( + utility_metadata_df = SB_metadata_df.filter( pl.col("sb.electric_utility") == electric_utility ) @@ -75,7 +70,9 @@ def map_electric_tariff( if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Map electrical tariff.") + parser = argparse.ArgumentParser( + description="Utility to help assign electricity tariffs to utility customers." + ) parser.add_argument( "--metadata_path", required=True, @@ -92,12 +89,12 @@ def map_electric_tariff( help="SB scenario type (e.g. default, seasonal, class_specific_seasonal)", ) parser.add_argument( - "--SB_scenario_year", required=True, help="SB scenario year (e.g. 2024)" + "--SB_scenario_year", required=True, help="SB scenario year (e.g. 1 , 2, 3)" ) parser.add_argument( "--output_dir", - default=None, - help="Optional directory for output CSV; default is rate_design//hp_rates/data/tariff_map/", + required=True, + help="Output directory for output CSV", ) args = parser.parse_args() @@ -115,7 +112,7 @@ def map_electric_tariff( if not metadata_path.exists(): raise FileNotFoundError(f"Metadata path {metadata_path} does not exist") # Polars scan_parquet needs a string path; S3Path.as_uri() gives s3:// URL - SB_metadata_lazy_df = pl.scan_parquet( + SB_metadata_df = pl.scan_parquet( str(metadata_path), storage_options=STORAGE_OPTIONS ) except ValueError: @@ -129,10 +126,10 @@ def map_electric_tariff( ) if not metadata_path.exists(): raise FileNotFoundError(f"Metadata path {metadata_path} does not exist") - SB_metadata_lazy_df = pl.scan_parquet(str(metadata_path)) + SB_metadata_df = pl.scan_parquet(str(metadata_path)) # Add dummy electric utility column (deterministic by bldg_id). Later this column will be pre-existing in the SB metadata parquet. - SB_metadata_lazy_df_with_electric_utility = SB_metadata_lazy_df.with_columns( + SB_metadata_df_with_electric_utility = SB_metadata_df.with_columns( pl.when(pl.col("bldg_id").hash() % 3 == 0) .then(pl.lit("Coned")) .when(pl.col("bldg_id").hash() % 3 == 1) @@ -144,35 +141,22 @@ def map_electric_tariff( sb_scenario = SBScenario(args.SB_scenario_type, args.SB_scenario_year) electrical_tariff_mapping_df = map_electric_tariff( - SB_metadata_lazy_df=SB_metadata_lazy_df_with_electric_utility, + SB_metadata_df=SB_metadata_df_with_electric_utility, electric_utility=args.electric_utility, SB_scenario=sb_scenario, state=args.state, ) - if args.output_dir: - try: - base_path = S3Path(args.output_dir) - output_path = base_path / f"{args.electric_utility}_{sb_scenario}.csv" - if not output_path.parent.exists(): - output_path.parent.mkdir(parents=True) - electrical_tariff_mapping_df.sink_csv( - str(output_path), storage_options=STORAGE_OPTIONS - ) - except ValueError: - base_path = Path(args.output_dir) - output_path = base_path / f"{args.electric_utility}_{sb_scenario}.csv" - if not output_path.parent.exists(): - output_path.parent.mkdir(parents=True) - electrical_tariff_mapping_df.sink_csv(str(output_path)) - else: - output_path = ( - RATE_DESIGN_DIR - / args.state.lower() - / "hp_rates" - / "data" - / "tariff_map" - / f"{args.electric_utility}_{sb_scenario}.csv" + try: + base_path = S3Path(args.output_dir) + output_path = base_path / f"{args.electric_utility}_{sb_scenario}.csv" + if not output_path.parent.exists(): + output_path.parent.mkdir(parents=True) + electrical_tariff_mapping_df.sink_csv( + str(output_path), storage_options=STORAGE_OPTIONS ) + except ValueError: + base_path = Path(args.output_dir) + output_path = base_path / f"{args.electric_utility}_{sb_scenario}.csv" if not output_path.parent.exists(): output_path.parent.mkdir(parents=True) electrical_tariff_mapping_df.sink_csv(str(output_path)) diff --git a/utils/gas_tariff_mapper.py b/utils/gas_tariff_mapper.py new file mode 100644 index 00000000..22654154 --- /dev/null +++ b/utils/gas_tariff_mapper.py @@ -0,0 +1,123 @@ +import argparse +from pathlib import Path +from typing import cast + +import polars as pl +from cloudpathlib import S3Path + +from utils import get_aws_region +from utils.types import electric_utility + +STORAGE_OPTIONS = {"aws_region": get_aws_region()} + + +def map_gas_tariff( + SB_metadata_df: pl.LazyFrame, + electric_utility_name: electric_utility, +) -> pl.LazyFrame: + utility_metadata_df = SB_metadata_df.filter( + pl.col("sb.electric_utility") == electric_utility_name + ) + + # Check if there are any rows in the filtered dataframe + test_sample = cast(pl.DataFrame, utility_metadata_df.head(1).collect()) + if test_sample.is_empty(): + raise ValueError(f"No rows found for electric utility {electric_utility_name}") + + gas_tariff_mapping_df = ( + utility_metadata_df.select(pl.col("bldg_id", "sb.gas_utility")) + .with_columns( + pl.when(pl.col("sb.gas_utility") == "National Grid") + .then(pl.lit("national_grid")) + .otherwise(pl.lit("nyseg")) + .alias("tariff_key") + ) + .drop("sb.gas_utility") + ) + + return gas_tariff_mapping_df + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Utility to help assign gas tariffs to utility customers." + ) + parser.add_argument( + "--metadata_path", + required=True, + help="Absolute or s3 path to ResStock metadata", + ) + parser.add_argument("--state", required=True, help="State code (e.g. NY, RI)") + parser.add_argument("--upgrade_id", required=True, help="Upgrade id (e.g. 00)") + parser.add_argument( + "--electric_utility", required=True, help="Electric utility (e.g. Coned)" + ) + parser.add_argument( + "--output_dir", + required=True, + help="Output directory for output CSV", + ) + args = parser.parse_args() + + ######################################################### + # For now, we will manually add the electric and gas utility columns. Later the metadata parquet will include them. + # Electric: first ~1/3 Coned, next ~1/3 National Grid, last ~1/3 NYSEG. Gas: half National Grid, half NYSEG. + try: # If the metadata path is an S3 path, use the S3Path class. + base_path = S3Path(args.metadata_path) + metadata_path = ( + base_path + / f"state={args.state}" + / f"upgrade={args.upgrade_id}" + / "metadata-sb.parquet" + ) + if not metadata_path.exists(): + raise FileNotFoundError(f"Metadata path {metadata_path} does not exist") + SB_metadata_df = pl.scan_parquet( + str(metadata_path), storage_options=STORAGE_OPTIONS + ) + except ValueError: # If the metadata path is a local path, use the Path class. + base_path = Path(args.metadata_path) + metadata_path = ( + base_path + / f"state={args.state}" + / f"upgrade={args.upgrade_id}" + / "metadata-sb.parquet" + ) + if not metadata_path.exists(): + raise FileNotFoundError(f"Metadata path {metadata_path} does not exist") + SB_metadata_df = pl.scan_parquet(str(metadata_path)) + + SB_metadata_df_with_utilities = SB_metadata_df.with_columns( + pl.when(pl.col("bldg_id").hash() % 3 == 0) + .then(pl.lit("Coned")) + .when(pl.col("bldg_id").hash() % 3 == 1) + .then(pl.lit("National Grid")) + .otherwise(pl.lit("NYSEG")) + .alias("sb.electric_utility"), + pl.when((pl.col("bldg_id").hash() % 2) == 0) + .then(pl.lit("National Grid")) + .otherwise(pl.lit("NYSEG")) + .alias("sb.gas_utility"), + ) + ######################################################### + + gas_tariff_mapping_df = map_gas_tariff( + SB_metadata_df=SB_metadata_df_with_utilities, + electric_utility_name=args.electric_utility, + ) + + output_filename = f"{args.electric_utility}_gas.csv" + try: + out_base = S3Path(args.output_dir) + output_path = out_base / output_filename + if not output_path.parent.exists(): + output_path.parent.mkdir(parents=True) + gas_tariff_mapping_df.sink_csv( + str(output_path), storage_options=STORAGE_OPTIONS + ) + except ValueError: + out_base = Path(args.output_dir) + output_path = out_base / output_filename + if not output_path.parent.exists(): + out_base.mkdir(parents=True, exist_ok=True) + gas_tariff_mapping_df.sink_csv(str(output_path)) diff --git a/utils/types.py b/utils/types.py index b56c089b..49bcc16c 100644 --- a/utils/types.py +++ b/utils/types.py @@ -20,3 +20,5 @@ def __str__(self): electric_utility = Literal["Coned", "National Grid", "NYSEG"] + +gas_utility = Literal["National Grid", "NYSEG", "RIE"]