Skip to content

Commit 1a7f682

Browse files
sherryzuosherryzuo
authored
Merge pull request #416 from switchbox-data/fair-default-rate-design
Add fair default rate design modules
2 parents eed3f0d + f7d9c02 commit 1a7f682

207 files changed

Lines changed: 739562 additions & 8308 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ __pycache__/
33
.mypy_cache/
44
.pytest_cache/
55
.ruff_cache/
6+
.tmp/
67
.DS_Store
78
dev_no_commit.py
89
# Local scratch / one-off scripts (not for CI or shared workflows)

‎context/README.md‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,12 @@ BAT framework, residual allocation, and how they connect to the literature.
6060

6161
Cost-reflective TOU design: theory and window optimization.
6262

63-
| File | Purpose |
64-
| ------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
65-
| cost_reflective_tou_rate_design.md | Theory and practice of cost-reflective TOU rate design: demand-weighted MC averages, cost-causation ratios, period selection, assumptions, demand flexibility implications, and partial vs. general equilibrium |
66-
| tou_window_optimization.md | TOU window width ($N$) sweep: welfare-loss metric derivation, HP-demand weighting proof, sweep algorithm, CLI/Justfile, NY results |
67-
| demand_flex_elasticity_calibration.md | Per-utility elasticity calibration: Arcturus 2.0 meta-analysis anchor, diagnostic methodology, results table (ε = -0.10 or -0.12 per utility), two savings mechanisms, known limitations |
63+
| File | Purpose |
64+
| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
65+
| cost_reflective_tou_rate_design.md | Theory and practice of cost-reflective TOU rate design: demand-weighted MC averages, cost-causation ratios, period selection, assumptions, demand flexibility implications, and partial vs. general equilibrium |
66+
| tou_window_optimization.md | TOU window width ($N$) sweep: welfare-loss metric derivation, HP-demand weighting proof, sweep algorithm, CLI/Justfile, NY results |
67+
| demand_flex_elasticity_calibration.md | Per-utility elasticity calibration: Arcturus 2.0 meta-analysis anchor, diagnostic methodology, results table (ε = -0.10 or -0.12 per utility), two savings mechanisms, known limitations |
68+
| fair_default_rate_design.md | Math for designing a single residential default tariff that eliminates the HP cross-subsidy $X_S$ from BAT outputs while collecting class RR. Three closed-form strategies (fixed-only, seasonal-only, combined cost-reflective ratio), uniqueness analysis (exactly one $(r_w, r_s)$ per chosen $F$), feasibility region, worked example. |
6869

6970
### methods/marginal_costs/
7071

‎context/methods/tou_and_rates/fair_default_rate_design.md‎

Lines changed: 413 additions & 0 deletions
Large diffs are not rendered by default.

‎rate_design/hp_rates/Justfile‎

Lines changed: 310 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
# Tier 1: Identity (set by state wrapper, dispatch, or manual source)
1414
# =============================================================================
1515

16-
set allow-duplicate-variables
16+
set allow-duplicate-variables := true
1717

1818
state := env_var_or_default('STATE', '')
1919
utility := env_var_or_default('UTILITY', '')
@@ -57,6 +57,7 @@ use_resstock_loads := env_var_or_default('USE_RESSTOCK_LOADS', 'false')
5757
# "default" = preserve the actual utility rate structure.
5858

5959
base_tariff_pattern := env_var_or_default('BASE_TARIFF_PATTERN', 'flat')
60+
mc_seasonal_ratio := env_var_or_default('MC_SEASONAL_RATIO', '')
6061

6162
# =============================================================================
6263
# Tier 3: Derived paths (all computed from Tier 1 + 2)
@@ -77,7 +78,7 @@ path_default_rev_requirement := path_rev_requirement / utility + ".yaml"
7778
path_differentiated_rev_requirement := path_rev_requirement / utility + "_hp_vs_nonhp.yaml"
7879
path_tariff_maps := path_config / "tariff_maps"
7980
path_dist_and_sub_tx_mc := "s3://data.sb/switchbox/marginal_costs/" + state + "/dist_and_sub_tx/utility=" + utility + "/year=" + mc_year + "/data.parquet"
80-
path_bulk_tx_mc := env_var_or_default('BULK_TX_MC', "")
81+
path_bulk_tx_mc := env_var_or_default('BULK_TX_MC', "s3://data.sb/switchbox/marginal_costs/" + state + "/bulk_tx/utility=" + utility + "/year=" + mc_year + "/data.parquet")
8182

8283
# Supply MC: default to per-utility S3 parquets (NY-style).
8384
# Cambium-based states (RI) override via SUPPLY_ENERGY_MC / SUPPLY_CAPACITY_MC / SUPPLY_ANCILLARY_MC in state.env.
@@ -430,6 +431,86 @@ create-flat-discount-tariff base_tariff_json flat_inputs_csv label output_path:
430431
"{{ label }}" \
431432
"{{ output_path }}"
432433

434+
compute-fair-default-inputs path_run_dir subclass_value cross_subsidy_col="BAT_percustomer" path_output_dir="" path_base_tariff_json="" group_col="has_hp" group_value_to_subclass="" mc_seasonal_ratio_value=mc_seasonal_ratio path_periods_yaml=path_periods_yaml fixed_charge_floor="0.0":
435+
#!/usr/bin/env bash
436+
set -euo pipefail
437+
path_effective_base_tariff="{{ path_base_tariff_json }}"
438+
if [ -z "${path_effective_base_tariff}" ]; then
439+
path_effective_base_tariff="{{ path_tariffs_electric }}/{{ utility }}_{{ base_tariff_pattern }}_calibrated.json"
440+
fi
441+
just -f {{ path_repo }}/utils/Justfile compute-fair-default-inputs \
442+
"{{ path_run_dir }}" \
443+
"{{ path_resstock_release }}" \
444+
"{{ state_upper }}" \
445+
"{{ upgrade }}" \
446+
"{{ subclass_value }}" \
447+
"{{ cross_subsidy_col }}" \
448+
"{{ path_output_dir }}" \
449+
"${path_effective_base_tariff}" \
450+
"{{ group_col }}" \
451+
"{{ group_value_to_subclass }}" \
452+
"{{ mc_seasonal_ratio_value }}" \
453+
"{{ path_periods_yaml }}" \
454+
"{{ fixed_charge_floor }}"
455+
456+
create-fair-default-tariff strategy label path_output_path path_inputs_csv path_base_tariff_json="" path_periods_yaml=path_periods_yaml allow_infeasible="false":
457+
#!/usr/bin/env bash
458+
set -euo pipefail
459+
path_effective_base_tariff="{{ path_base_tariff_json }}"
460+
if [ -z "${path_effective_base_tariff}" ]; then
461+
path_effective_base_tariff="{{ path_tariffs_electric }}/{{ utility }}_{{ base_tariff_pattern }}_calibrated.json"
462+
fi
463+
just -f {{ path_repo }}/utils/Justfile create-fair-default-tariff \
464+
"${path_effective_base_tariff}" \
465+
"{{ path_inputs_csv }}" \
466+
"{{ strategy }}" \
467+
"{{ label }}" \
468+
"{{ path_output_path }}" \
469+
"{{ path_periods_yaml }}" \
470+
"{{ allow_infeasible }}"
471+
472+
create-all-fair-default-tariffs path_run_dir subclass_value="true" path_tariff_output_dir=path_tariffs_electric path_base_tariff_json="" cross_subsidy_col="BAT_percustomer" group_col="has_hp" group_value_to_subclass="" path_output_dir="" mc_seasonal_ratio_value=mc_seasonal_ratio path_periods_yaml=path_periods_yaml fixed_charge_floor="0.0":
473+
#!/usr/bin/env bash
474+
set -euo pipefail
475+
path_effective_output_dir="{{ path_output_dir }}"
476+
if [ -z "${path_effective_output_dir}" ]; then
477+
path_effective_output_dir="{{ path_run_dir }}"
478+
fi
479+
path_effective_base_tariff="{{ path_base_tariff_json }}"
480+
if [ -z "${path_effective_base_tariff}" ]; then
481+
path_effective_base_tariff="{{ path_tariffs_electric }}/{{ utility }}_{{ base_tariff_pattern }}_calibrated.json"
482+
fi
483+
if [ -z "{{ mc_seasonal_ratio_value }}" ]; then
484+
echo "ERROR: create-all-fair-default-tariffs requires mc_seasonal_ratio (set MC_SEASONAL_RATIO or pass mc_seasonal_ratio_value)." >&2
485+
exit 1
486+
fi
487+
just compute-fair-default-inputs "{{ path_run_dir }}" "{{ subclass_value }}" \
488+
"{{ cross_subsidy_col }}" "${path_effective_output_dir}" \
489+
"${path_effective_base_tariff}" "{{ group_col }}" "{{ group_value_to_subclass }}" \
490+
"{{ mc_seasonal_ratio_value }}" "{{ path_periods_yaml }}" "{{ fixed_charge_floor }}"
491+
path_inputs_csv="${path_effective_output_dir}/fair_default_inputs.csv"
492+
just create-fair-default-tariff \
493+
fixed_charge_only \
494+
{{ utility }}_default_fair_fixed_charge_only \
495+
"{{ path_tariff_output_dir }}/{{ utility }}_default_fair_fixed_charge_only.json" \
496+
"${path_inputs_csv}" \
497+
"${path_effective_base_tariff}" \
498+
"{{ path_periods_yaml }}"
499+
just create-fair-default-tariff \
500+
seasonal_rates_only \
501+
{{ utility }}_default_fair_seasonal_rates_only \
502+
"{{ path_tariff_output_dir }}/{{ utility }}_default_fair_seasonal_rates_only.json" \
503+
"${path_inputs_csv}" \
504+
"${path_effective_base_tariff}" \
505+
"{{ path_periods_yaml }}"
506+
just create-fair-default-tariff \
507+
fixed_plus_seasonal_mc \
508+
{{ utility }}_default_fair_fixed_plus_seasonal_mc \
509+
"{{ path_tariff_output_dir }}/{{ utility }}_default_fair_fixed_plus_seasonal_mc.json" \
510+
"${path_inputs_csv}" \
511+
"${path_effective_base_tariff}" \
512+
"{{ path_periods_yaml }}"
513+
433514
copy-calibrated-tariff-from-run run_dir output_dir="":
434515
just -f {{ path_repo }}/utils/Justfile copy-calibrated-tariff-from-run \
435516
"{{ run_dir }}" \
@@ -458,8 +539,234 @@ compute-subclass-rev-requirements:
458539
"${run2_dir}"
459540

460541
# =============================================================================
461-
# RUNS: scenario execution (run-1 through run-16)
542+
# FAIR-DEFAULT: paths, tariff preparation, scenario generation, and runs 101-124
543+
# =============================================================================
544+
545+
path_scenarios_fair_default := path_config / "scenarios" / "fair_default"
546+
path_scenario_fair_default_config := path_scenarios_fair_default / ("scenarios_" + utility + ".yaml")
547+
548+
# Generate fair-default scenario YAMLs (one per utility, in fair_default/ subdir)
549+
create-fair-default-scenario-yamls:
550+
uv run python {{ path_repo }}/utils/pre/create_fair_default_scenario_yamls.py \
551+
--state {{ state }} --utility {{ utility }}
552+
553+
# Build the 12 uncalibrated fair-default tariff JSONs for one utility.
554+
555+
# Requires run-1 and run-2 outputs to already exist (resolved via latest_run_output.sh).
556+
prepare-fair-default-tariffs:
557+
#!/usr/bin/env bash
558+
set -euo pipefail
559+
run1_dir=$(bash "{{ latest_output }}" "{{ path_scenario_config }}" 1)
560+
run2_dir=$(bash "{{ latest_output }}" "{{ path_scenario_config }}" 2)
561+
echo ">> prepare-fair-default-tariffs: run-1 delivery dir → ${run1_dir}" >&2
562+
echo ">> prepare-fair-default-tariffs: run-2 supply dir → ${run2_dir}" >&2
563+
uv run python {{ path_repo }}/utils/mid/prepare_fair_default_tariffs.py \
564+
--state {{ state }} --utility {{ utility }} \
565+
--run-dir-delivery "${run1_dir}" \
566+
--run-dir-supply "${run2_dir}" \
567+
--output-dir "{{ path_tariffs_electric }}" \
568+
--resstock-base "{{ path_resstock_release }}" \
569+
--path-dist-and-sub-tx-mc "{{ path_dist_and_sub_tx_mc }}" \
570+
--path-bulk-tx-mc "{{ path_bulk_tx_mc }}" \
571+
--path-supply-energy-mc "{{ path_supply_energy_mc }}" \
572+
--path-supply-capacity-mc "{{ path_supply_capacity_mc }}" \
573+
--base-tariff-delivery "{{ path_tariffs_electric }}/{{ utility }}_{{ base_tariff_pattern }}_calibrated.json" \
574+
--base-tariff-supply "{{ path_tariffs_electric }}/{{ utility }}_{{ base_tariff_pattern }}_supply_calibrated.json" \
575+
--path-utility-assignment "{{ path_utility_assignment }}" \
576+
--path-electric-utility-stats "{{ path_electric_utility_stats }}" \
577+
--periods-yaml "{{ path_periods_yaml }}" \
578+
--allow-infeasible
579+
580+
# Generic dispatcher for fair-default runs (101-124): dispatches against
581+
582+
# the fair_default/ scenario YAML instead of the main scenarios_<util>.yaml.
583+
run-fd N:
584+
just run-scenario-from "{{ path_scenario_fair_default_config }}" {{ N }}
585+
586+
# Precalc fair-default runs (precalc delivery/supply): run then promote calibrated tariff.
587+
run-101:
588+
just run-fd 101
589+
run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 101); \
590+
just copy-calibrated-tariff-from-run "${run_dir}"
591+
592+
run-102:
593+
just run-fd 102
594+
run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 102); \
595+
just copy-calibrated-tariff-from-run "${run_dir}"
596+
597+
run-103:
598+
just run-fd 103
599+
600+
run-104:
601+
just run-fd 104
602+
603+
run-105:
604+
just run-fd 105
605+
run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 105); \
606+
just copy-calibrated-tariff-from-run "${run_dir}"
607+
608+
run-106:
609+
just run-fd 106
610+
run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 106); \
611+
just copy-calibrated-tariff-from-run "${run_dir}"
612+
613+
run-107:
614+
just run-fd 107
615+
616+
run-108:
617+
just run-fd 108
618+
619+
run-109:
620+
just run-fd 109
621+
run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 109); \
622+
just copy-calibrated-tariff-from-run "${run_dir}"
623+
624+
run-110:
625+
just run-fd 110
626+
run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 110); \
627+
just copy-calibrated-tariff-from-run "${run_dir}"
628+
629+
run-111:
630+
just run-fd 111
631+
632+
run-112:
633+
just run-fd 112
634+
635+
run-113:
636+
just run-fd 113
637+
run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 113); \
638+
just copy-calibrated-tariff-from-run "${run_dir}"
639+
640+
run-114:
641+
just run-fd 114
642+
run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 114); \
643+
just copy-calibrated-tariff-from-run "${run_dir}"
644+
645+
run-115:
646+
just run-fd 115
647+
648+
run-116:
649+
just run-fd 116
650+
651+
run-117:
652+
just run-fd 117
653+
run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 117); \
654+
just copy-calibrated-tariff-from-run "${run_dir}"
655+
656+
run-118:
657+
just run-fd 118
658+
run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 118); \
659+
just copy-calibrated-tariff-from-run "${run_dir}"
660+
661+
run-119:
662+
just run-fd 119
663+
664+
run-120:
665+
just run-fd 120
666+
667+
run-121:
668+
just run-fd 121
669+
run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 121); \
670+
just copy-calibrated-tariff-from-run "${run_dir}"
671+
672+
run-122:
673+
just run-fd 122
674+
run_dir=$(bash "{{ latest_output }}" "{{ path_scenario_fair_default_config }}" 122); \
675+
just copy-calibrated-tariff-from-run "${run_dir}"
676+
677+
run-123:
678+
just run-fd 123
679+
680+
run-124:
681+
just run-fd 124
682+
683+
# Pre-rate setup that includes fair-default scaffolding (generates scenario YAMLs
684+
685+
# and tariff maps for the fair_default/ subdir in addition to the baseline pre steps).
686+
all-pre-fair-rate:
687+
just all-pre
688+
just create-fair-default-scenario-yamls
689+
just -f {{ path_repo }}/utils/Justfile write-tariff-maps-all "{{ path_scenarios_fair_default }}"
690+
uv run python {{ path_repo }}/utils/pre/validate_config.py \
691+
--scenario-config "{{ path_scenario_fair_default_config }}" \
692+
--state "{{ state_upper }}" \
693+
--utility "{{ utility }}" \
694+
--upgrade "{{ upgrade }}" \
695+
--year "{{ year }}" \
696+
--path-dist-and-sub-tx-mc "{{ path_dist_and_sub_tx_mc }}" \
697+
{{ if path_bulk_tx_mc != "" { "--path-bulk-tx-mc \"" + path_bulk_tx_mc + "\"" } else { "" } }} \
698+
--path-supply-energy-mc "{{ path_supply_energy_mc }}" \
699+
--path-supply-capacity-mc "{{ path_supply_capacity_mc }}" \
700+
--path-electric-utility-stats "{{ path_electric_utility_stats }}" \
701+
--path-resstock-loads "{{ path_resstock_loads_00 }}" \
702+
--fair-default-dir "{{ path_scenarios_fair_default }}"
703+
704+
# Full end-to-end: pre-rate + run-1/2 + tariff prep + 24 fair-default runs (101-124).
705+
# Dependencies: run-1 → copy-calibrated, run-2 → copy-calibrated → prepare-fair-default-tariffs
706+
707+
# → runs 101-124 in groups of 4 (precalc-del, precalc-sup, u2-del, u2-sup).
708+
fair-default-rate-runs:
709+
#!/usr/bin/env bash
710+
set -euo pipefail
711+
just all-pre-fair-rate
712+
just run-1
713+
just run-2
714+
run1_dir=$(bash "{{ latest_output }}" "{{ path_scenario_config }}" 1)
715+
just copy-calibrated-tariff-from-run "${run1_dir}"
716+
run2_dir=$(bash "{{ latest_output }}" "{{ path_scenario_config }}" 2)
717+
just copy-calibrated-tariff-from-run "${run2_dir}"
718+
just prepare-fair-default-tariffs
719+
just run-101
720+
just run-102
721+
just run-103
722+
just run-104
723+
just run-105
724+
just run-106
725+
just run-107
726+
just run-108
727+
just run-109
728+
just run-110
729+
just run-111
730+
just run-112
731+
just run-113
732+
just run-114
733+
just run-115
734+
just run-116
735+
just run-117
736+
just run-118
737+
just run-119
738+
just run-120
739+
just run-121
740+
just run-122
741+
just run-123
742+
just run-124
743+
744+
# =============================================================================
745+
# RUNS: scenario execution (run-1 through run-36)
462746
# =============================================================================
747+
# Dispatch a single run from an arbitrary scenario YAML file (used by run-fd).
748+
749+
# Passes --scenario-config directly to run_scenario.py.
750+
run-scenario-from path_scenario_config run_num *extra_args:
751+
#!/usr/bin/env bash
752+
set -euo pipefail
753+
: "${RDP_BATCH:?Set RDP_BATCH before running (e.g. ny_20260305c_r1-8)}"
754+
log_dir="${HOME}/rdp_run_logs"
755+
mkdir -p "${log_dir}"
756+
log_file="${log_dir}/{{ utility }}_run{{ run_num }}_${RDP_BATCH}.log"
757+
echo ">> run-scenario-from {{ run_num }}: logging to ${log_file}" >&2
758+
echo "git_commit: $(git -C "{{ path_repo }}" rev-parse HEAD 2>/dev/null || echo unknown)" > "${log_file}"
759+
echo "git_dirty: $(git -C "{{ path_repo }}" status --porcelain 2>/dev/null | wc -l | tr -d ' ') files" >> "${log_file}"
760+
echo "timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "${log_file}"
761+
uv run python {{ path_repo }}/rate_design/hp_rates/run_scenario.py \
762+
--scenario-config "{{ path_scenario_config }}" \
763+
--state "{{ state }}" \
764+
--run-num "{{ run_num }}" \
765+
--output-dir "{{ path_outputs_base }}/${RDP_BATCH}" \
766+
{{ if env_var_or_default('RDP_NUM_WORKERS', '') != '' { "--num-workers " + env_var_or_default('RDP_NUM_WORKERS', '') } else { "" } }} \
767+
{{ if path_supply_ancillary_mc != "" { "--path-supply-ancillary-mc \"" + path_supply_ancillary_mc + "\"" } else { "" } }} \
768+
{{ extra_args }} \
769+
2>&1 | tee -a "${log_file}"
463770

464771
run-scenario run_num *extra_args:
465772
#!/usr/bin/env bash

0 commit comments

Comments
 (0)