Skip to content

Commit b9f17e6

Browse files
alexhyunminleealexhyunminlee
committed
Address comments and reformat code
1 parent 2e602ea commit b9f17e6

7 files changed

Lines changed: 509 additions & 428 deletions

File tree

‎pyproject.toml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,10 @@ license = { text = "MIT" }
99
dependencies = [
1010
"buildstock-fetch>=1.6.1",
1111
"cairo",
12-
"polars",
12+
"cloudpathlib[s3]>=0.23.0",
1313
"numpy",
14+
"polars",
15+
"prek>=0.3.2",
1416
"pyarrow",
1517
]
1618

‎rate_design/ny/hp_rates/data/tariff_map/Justfile‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
1-
# Resolve project root as absolute path from this Justfile's location (works for any clone)
2-
project_root := absolute_path(justfile_directory() / ".." / ".." / ".." / ".." / "..")
3-
data_base := "s3://data.sb/nrel/resstock/res_2024_amy2018_2/metadata"
1+
# Resolve project root via git (works from any subdirectory in the repo)
2+
project_root := `git rev-parse --show-toplevel`
3+
path_s3_resstock_metadata := "s3://data.sb/nrel/resstock/res_2024_amy2018_2/metadata"
44

55
# Usage: just map-electric-tariff [electric_utility] [SB_scenario_type] [SB_scenario_year] [upgrade_id]
66
# Example: just map-electric-tariff Coned default 1 00
7-
map-electric-tariff electric_utility SB_scenario_name SB_scenario_year upgrade_id:
8-
cd {{project_root}} && uv run python {{project_root}}/utils/electric_tariff_mapper.py \
9-
--metadata_path "{{data_base}}/state=NY/upgrade={{upgrade_id}}/metadata-sb.parquet" \
7+
map-electric-tariff electric_utility SB_scenario_type SB_scenario_year upgrade_id output_dir="":
8+
uv run python {{project_root}}/utils/electric_tariff_mapper.py \
9+
--metadata_path "{{path_s3_resstock_metadata}}" \
1010
--state NY \
11+
--upgrade_id "{{upgrade_id}}" \
1112
--electric_utility "{{electric_utility}}" \
12-
--SB_scenario_name "{{SB_scenario_name}}" \
13-
--SB_scenario_year "{{SB_scenario_year}}"
13+
--SB_scenario_type "{{SB_scenario_type}}" \
14+
--SB_scenario_year "{{SB_scenario_year}}" \
15+
{{ if output_dir != "" { "--output_dir \"" + output_dir + "\"" } else { "" } }}

‎rate_design/ny/hp_rates/run_scenario.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
"""Entrypoint for running NY heat pump rate scenarios (stub)."""
22

33
import logging
4-
import pandas as pd
54
from pathlib import Path
5+
6+
import pandas as pd
67
from cairo.rates_tool.loads import _return_load, return_buildingstock
78
from cairo.rates_tool.systemsimulator import (
89
MeetRevenueSufficiencySystemWide,
@@ -164,7 +165,7 @@
164165
run_name=run_name,
165166
output_dir=path_results, # will default to this folder if not pass, mostly testing ability for user to pass arbitrary output directory
166167
)
167-
bs.tariff_map = tariff_map_name
168+
# bs.tariff_map = tariff_map_name
168169

169170
# TODO: add (maybe post-processing)module for delivered fuels bills
170171
bs.simulate(

‎rate_design/ri/hp_rates/data/tariff_map/Justfile‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
1-
# Resolve project root as absolute path from this Justfile's location (works for any clone)
2-
project_root := absolute_path(justfile_directory() / ".." / ".." / ".." / ".." / "..")
3-
data_base := "s3://data.sb/nrel/resstock/res_2024_amy2018_2/metadata"
1+
# Resolve project root via git (works from any subdirectory in the repo)
2+
project_root := `git rev-parse --show-toplevel`
3+
path_s3_resstock_metadata := "s3://data.sb/nrel/resstock/res_2024_amy2018_2/metadata"
44

55
# Usage: just map-electric-tariff [electric_utility] [SB_scenario_type] [SB_scenario_year] [upgrade_id]
66
# Example: just map-electric-tariff Coned default 1 00
7-
map-electric-tariff electric_utility SB_scenario_name SB_scenario_year upgrade_id:
8-
cd {{project_root}} && uv run python {{project_root}}/utils/electric_tariff_mapper.py \
9-
--metadata_path "{{data_base}}/state=RI/upgrade={{upgrade_id}}/metadata-sb.parquet" \
7+
map-electric-tariff electric_utility SB_scenario_type SB_scenario_year upgrade_id output_dir="":
8+
uv run python {{project_root}}/utils/electric_tariff_mapper.py \
9+
--metadata_path "{{path_s3_resstock_metadata}}" \
1010
--state RI \
11+
--upgrade_id "{{upgrade_id}}" \
1112
--electric_utility "{{electric_utility}}" \
12-
--SB_scenario_name "{{SB_scenario_name}}" \
13+
--SB_scenario_type "{{SB_scenario_type}}" \
1314
--SB_scenario_year "{{SB_scenario_year}}"
15+
{{ if output_dir != "" { "--output_dir \"" + output_dir + "\"" } else { "" } }}

‎utils/electric_tariff_mapper.py‎

Lines changed: 105 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,29 @@
11
import argparse
2-
import io
32
from pathlib import Path
3+
from typing import cast
44

55
import polars as pl
66
from cloudpathlib import S3Path
77

8-
from utils.types import SB_scenario, electric_utility
8+
from utils.types import SBScenario, electric_utility
99

1010
# Project root (rate-design-platform); independent of cwd or caller
1111
_PROJECT_ROOT = Path(__file__).resolve().parent.parent
1212
RATE_DESIGN_DIR = _PROJECT_ROOT / "rate_design"
1313

14+
AWS_REGION = "us-west-2"
15+
16+
STORAGE_OPTIONS = {"aws_region": AWS_REGION}
17+
1418

1519
def define_electrical_tariff_key(
16-
SB_scenario: SB_scenario,
20+
SB_scenario: SBScenario,
1721
electric_utility: electric_utility,
18-
has_hp: pl.Series,
22+
has_hp: pl.Expr,
1923
) -> pl.Expr:
2024
if SB_scenario.analysis_type == "default":
2125
return pl.lit(f"{electric_utility}_{SB_scenario}_default")
22-
elif (
23-
SB_scenario.analysis_type == "seasonal"
24-
or SB_scenario.analysis_type == "class_specific_seasonal"
25-
):
26+
elif SB_scenario.analysis_type in ["seasonal", "class_specific_seasonal"]:
2627
return (
2728
pl.when(has_hp)
2829
.then(pl.lit(f"{electric_utility}_{SB_scenario}_HP.csv"))
@@ -33,124 +34,145 @@ def define_electrical_tariff_key(
3334

3435

3536
def generate_electrical_tariff_mapping(
36-
metadata_has_hp: pl.DataFrame,
37-
SB_scenario: SB_scenario,
37+
lazy_metadata_has_hp: pl.LazyFrame,
38+
SB_scenario: SBScenario,
3839
electric_utility: electric_utility,
39-
) -> pl.DataFrame:
40-
has_hp = metadata_has_hp["postprocess_group.has_hp"]
41-
42-
electrical_tariff_mapping_df = metadata_has_hp.select(
43-
pl.col("bldg_id")
44-
).with_columns(
45-
define_electrical_tariff_key(SB_scenario, electric_utility, has_hp).alias(
46-
"tariff_key"
47-
)
40+
) -> pl.LazyFrame:
41+
electrical_tariff_mapping_df = lazy_metadata_has_hp.select(
42+
pl.col("bldg_id"),
43+
define_electrical_tariff_key(
44+
SB_scenario, electric_utility, pl.col("postprocess_group.has_hp")
45+
).alias("tariff_key"),
4846
)
4947

5048
return electrical_tariff_mapping_df
5149

5250

5351
def map_electric_tariff(
54-
SB_metadata_path: S3Path,
52+
SB_metadata_lazy_df: pl.LazyFrame,
5553
electric_utility: electric_utility,
56-
SB_scenario: SB_scenario,
54+
SB_scenario: SBScenario,
5755
state: str,
58-
):
59-
if not SB_metadata_path.exists():
60-
raise FileNotFoundError(f"SB metadata path {SB_metadata_path} does not exist")
61-
62-
metadata_df = pl.read_parquet(io.BytesIO(SB_metadata_path.read_bytes()))
63-
utility_metadata_df = metadata_df.filter(
56+
) -> pl.LazyFrame:
57+
utility_metadata_df = SB_metadata_lazy_df.filter(
6458
pl.col("sb.electric_utility") == electric_utility
6559
)
66-
if utility_metadata_df.is_empty():
67-
return
6860

6961
metadata_has_hp = utility_metadata_df.select(
7062
pl.col("bldg_id", "postprocess_group.has_hp")
7163
)
72-
electrical_tariff_mapping_df = generate_electrical_tariff_mapping(
73-
metadata_has_hp, SB_scenario, electric_utility
74-
)
7564

76-
print(electrical_tariff_mapping_df.head(20))
65+
# Check if there are any rows in the filtered dataframe
66+
test_sample = cast(pl.DataFrame, metadata_has_hp.head(1).collect())
67+
if test_sample.is_empty():
68+
raise ValueError(f"No rows found for electric utility {electric_utility}")
7769

78-
output_path = (
79-
RATE_DESIGN_DIR
80-
/ state.lower()
81-
/ "hp_rates"
82-
/ "data"
83-
/ "tariff_map"
84-
/ f"{electric_utility}_{SB_scenario}.csv"
70+
electrical_tariff_mapping_df = generate_electrical_tariff_mapping(
71+
metadata_has_hp, SB_scenario, electric_utility
8572
)
86-
if not output_path.parent.exists():
87-
output_path.parent.mkdir(parents=True)
88-
electrical_tariff_mapping_df.write_csv(output_path)
8973

90-
return
74+
return electrical_tariff_mapping_df
9175

9276

93-
def main():
77+
if __name__ == "__main__":
9478
parser = argparse.ArgumentParser(description="Map electrical tariff.")
9579
parser.add_argument(
96-
"--metadata_path", required=True, help="Base path for resstock data"
80+
"--metadata_path",
81+
required=True,
82+
help="Absolute or s3 path to ResStock metadata",
9783
)
9884
parser.add_argument("--state", required=True, help="State code (e.g. RI)")
85+
parser.add_argument("--upgrade_id", required=True, help="Upgrade id (e.g. 00)")
9986
parser.add_argument(
10087
"--electric_utility", required=True, help="Electric utility (e.g. Coned)"
10188
)
10289
parser.add_argument(
103-
"--SB_scenario_name", required=True, help="SB scenario name (e.g. default_1)"
90+
"--SB_scenario_type",
91+
required=True,
92+
help="SB scenario type (e.g. default, seasonal, class_specific_seasonal)",
10493
)
10594
parser.add_argument(
10695
"--SB_scenario_year", required=True, help="SB scenario year (e.g. 2024)"
10796
)
97+
parser.add_argument(
98+
"--output_dir",
99+
default=None,
100+
help="Optional directory for output CSV; default is rate_design/<state>/hp_rates/data/tariff_map/",
101+
)
108102
args = parser.parse_args()
109103

110104
#########################################################
111105
# For now, we will manually add the electric utility name column. Later on, the metadata parquet will be updated to include this column.
112106
# Assign first ~1/3 to Coned, next ~1/3 to National Grid, last ~1/3 to NYSEG.
113-
metadata_path = S3Path(args.metadata_path)
114-
metadata_df = pl.read_parquet(io.BytesIO(metadata_path.read_bytes()))
115-
n = len(metadata_df)
116-
metadata_df = (
117-
metadata_df.with_row_index("_row_idx")
118-
.with_columns(
119-
pl.when(pl.col("_row_idx") < n // 3)
120-
.then(pl.lit("Coned"))
121-
.when(pl.col("_row_idx") < 2 * n // 3)
122-
.then(pl.lit("National Grid"))
123-
.otherwise(pl.lit("NYSEG"))
124-
.alias("sb.electric_utility")
107+
try: # If the metadata path is an S3 path, use the S3Path class.
108+
base_path = S3Path(args.metadata_path)
109+
metadata_path = (
110+
base_path
111+
/ f"state={args.state}"
112+
/ f"upgrade={args.upgrade_id}"
113+
/ "metadata-sb.parquet"
114+
)
115+
if not metadata_path.exists():
116+
raise FileNotFoundError(f"Metadata path {metadata_path} does not exist")
117+
# Polars scan_parquet needs a string path; S3Path.as_uri() gives s3:// URL
118+
SB_metadata_lazy_df = pl.scan_parquet(
119+
str(metadata_path), storage_options=STORAGE_OPTIONS
120+
)
121+
except ValueError:
122+
# If the metadata path is a local path, use the Path class.
123+
base_path = Path(args.metadata_path)
124+
metadata_path = (
125+
base_path
126+
/ f"state={args.state}"
127+
/ f"upgrade={args.upgrade_id}"
128+
/ "metadata-sb.parquet"
125129
)
126-
.drop("_row_idx")
130+
if not metadata_path.exists():
131+
raise FileNotFoundError(f"Metadata path {metadata_path} does not exist")
132+
SB_metadata_lazy_df = pl.scan_parquet(str(metadata_path))
133+
134+
# Add dummy electric utility column (deterministic by bldg_id). Later this column will be pre-existing in the SB metadata parquet.
135+
SB_metadata_lazy_df_with_electric_utility = SB_metadata_lazy_df.with_columns(
136+
pl.when(pl.col("bldg_id").hash() % 3 == 0)
137+
.then(pl.lit("Coned"))
138+
.when(pl.col("bldg_id").hash() % 3 == 1)
139+
.then(pl.lit("National Grid"))
140+
.otherwise(pl.lit("NYSEG"))
141+
.alias("sb.electric_utility")
127142
)
128-
temp_path = metadata_path.parent / f"{metadata_path.stem}_temp.parquet"
129-
buf = io.BytesIO()
130-
metadata_df.write_parquet(buf)
131-
temp_path.write_bytes(buf.getvalue())
132143
#########################################################
133144

134-
if (
135-
args.SB_scenario_name != "default"
136-
and args.SB_scenario_name != "seasonal"
137-
and args.SB_scenario_name != "class_specific_seasonal"
138-
):
139-
raise ValueError(f"Invalid SB scenario name: {args.SB_scenario_name}")
140-
141-
sb_scenario = SB_scenario(args.SB_scenario_name, args.SB_scenario_year)
142-
map_electric_tariff(
143-
SB_metadata_path=temp_path,
145+
sb_scenario = SBScenario(args.SB_scenario_type, args.SB_scenario_year)
146+
electrical_tariff_mapping_df = map_electric_tariff(
147+
SB_metadata_lazy_df=SB_metadata_lazy_df_with_electric_utility,
144148
electric_utility=args.electric_utility,
145149
SB_scenario=sb_scenario,
146150
state=args.state,
147151
)
148-
149-
#########################################################
150-
# Remove the temp file that contains electric utility name column.
151-
temp_path.unlink()
152-
#########################################################
153-
154-
155-
if __name__ == "__main__":
156-
main()
152+
if args.output_dir:
153+
try:
154+
base_path = S3Path(args.output_dir)
155+
output_path = base_path / f"{args.electric_utility}_{sb_scenario}.csv"
156+
if not output_path.parent.exists():
157+
output_path.parent.mkdir(parents=True)
158+
electrical_tariff_mapping_df.sink_csv(
159+
str(output_path), storage_options=STORAGE_OPTIONS
160+
)
161+
except ValueError:
162+
base_path = Path(args.output_dir)
163+
output_path = base_path / f"{args.electric_utility}_{sb_scenario}.csv"
164+
if not output_path.parent.exists():
165+
output_path.parent.mkdir(parents=True)
166+
electrical_tariff_mapping_df.sink_csv(str(output_path))
167+
else:
168+
output_path = (
169+
RATE_DESIGN_DIR
170+
/ args.state.lower()
171+
/ "hp_rates"
172+
/ "data"
173+
/ "tariff_map"
174+
/ f"{args.electric_utility}_{sb_scenario}.csv"
175+
)
176+
if not output_path.parent.exists():
177+
output_path.parent.mkdir(parents=True)
178+
electrical_tariff_mapping_df.sink_csv(str(output_path))

‎utils/types.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
from typing import Literal
22

33

4-
class SB_scenario:
4+
class SBScenario:
55
analysis_type: Literal["default", "seasonal", "class_specific_seasonal"]
66
analysis_year: int
77

@@ -10,6 +10,8 @@ def __init__(
1010
analysis_type: Literal["default", "seasonal", "class_specific_seasonal"],
1111
analysis_year: int,
1212
):
13+
if analysis_type not in ["default", "seasonal", "class_specific_seasonal"]:
14+
raise ValueError(f"Invalid analysis type: {analysis_type}")
1315
self.analysis_type = analysis_type
1416
self.analysis_year = analysis_year
1517

0 commit comments

Comments
 (0)