Preserve household weight fuel uprating

changelog.d/fuel-uprating-household-weight.md

@@ -0,0 +1 @@
+Restore the pre-fuel-change household-weight uprating row when calibrating fuel litre proxies.

policyengine_uk_data/storage/uprating_factors.csv

@@ -12,7 +12,7 @@ childcare_expenses,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,
 clothing_and_footwear_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
 communication_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
 corporate_wealth,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384
-diesel_spending,1.0,1.138,1.56,1.531,1.579,1.537,1.509,1.467,1.422,1.363,1.286,1.28,1.275,1.27,1.265
+diesel_spending,1.0,1.138,1.57,1.539,1.589,1.54,1.507,1.46,1.415,1.355,1.284,1.284,1.284,1.284,1.284
 dividend_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384
 domestic_energy_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
 education_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
@@ -30,7 +30,7 @@ free_school_milk,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.
 gross_financial_wealth,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384
 health_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
 household_furnishings_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
-household_weight,1.0,1.0,1.009,1.023,1.033,1.041,1.045,1.049,1.053,1.058,1.062,1.067,1.071,1.076,1.08
+household_weight,1.0,1.0,1.003,1.017,1.027,1.039,1.046,1.054,1.058,1.064,1.064,1.064,1.064,1.064,1.064
 housing_benefit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
 housing_service_charges,1.0,1.0,1.0,1.064,1.137,1.191,1.235,1.262,1.289,1.318,1.318,1.318,1.318,1.318,1.318
 housing_water_and_electricity_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
@@ -56,7 +56,7 @@ owned_land,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.3
 pension_credit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
 pension_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384
 personal_pension_contributions,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431
-petrol_spending,1.0,1.104,1.624,1.786,1.521,1.48,1.454,1.413,1.37,1.313,1.239,1.233,1.228,1.223,1.218
+petrol_spending,1.0,1.104,1.635,1.796,1.531,1.483,1.452,1.406,1.363,1.305,1.237,1.237,1.237,1.237,1.237
 private_pension_income,1.0,1.003,1.053,1.106,1.161,1.216,1.261,1.288,1.315,1.346,1.346,1.346,1.346,1.346,1.346
 private_transfer_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384
 property_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384

policyengine_uk_data/storage/uprating_growth_factors.csv

@@ -12,7 +12,7 @@ childcare_expenses,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0
 clothing_and_footwear_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
 communication_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
 corporate_wealth,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0
-diesel_spending,0,0.138,0.371,-0.019,0.031,-0.027,-0.018,-0.028,-0.031,-0.041,-0.056,-0.005,-0.004,-0.004,-0.004
+diesel_spending,0,0.138,0.379,-0.019,0.033,-0.031,-0.021,-0.032,-0.031,-0.043,-0.052,0.0,0.0,0.0,0.0
 dividend_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0
 domestic_energy_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
 education_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
@@ -30,7 +30,7 @@ free_school_milk,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0
 gross_financial_wealth,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0
 health_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
 household_furnishings_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
-household_weight,0,0.0,0.009,0.014,0.01,0.008,0.004,0.004,0.004,0.005,0.004,0.005,0.004,0.005,0.004
+household_weight,0,0.0,0.003,0.014,0.01,0.012,0.007,0.008,0.004,0.006,0.0,0.0,0.0,0.0,0.0
 housing_benefit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
 housing_service_charges,0,0.0,0.0,0.064,0.069,0.047,0.037,0.022,0.021,0.022,0.0,0.0,0.0,0.0,0.0
 housing_water_and_electricity_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
@@ -56,7 +56,7 @@ owned_land,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,
 pension_credit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
 pension_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0
 personal_pension_contributions,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0
-petrol_spending,0,0.104,0.471,0.1,-0.148,-0.027,-0.018,-0.028,-0.03,-0.042,-0.056,-0.005,-0.004,-0.004,-0.004
+petrol_spending,0,0.104,0.481,0.099,-0.147,-0.031,-0.021,-0.032,-0.031,-0.043,-0.052,0.0,0.0,0.0,0.0
 private_pension_income,0,0.003,0.05,0.05,0.05,0.047,0.037,0.021,0.021,0.024,0.0,0.0,0.0,0.0,0.0
 private_transfer_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0
 property_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0

policyengine_uk_data/tests/test_road_fuel_volume_uprating.py

@@ -26,8 +26,10 @@
 from policyengine_uk_data.storage import STORAGE_FOLDER
 from policyengine_uk_data.utils.uprating import (
     END_YEAR,
+    HOUSEHOLD_WEIGHT_UPRATING_INDEX,
     START_YEAR,
     VOLUME_OVERRIDDEN_VARIABLES,
+    _apply_household_weight_uprating_override,
     _apply_road_fuel_litre_proxy_override,
     fuel_spending_litre_proxy_index,
     uprate_dataset,
@@ -78,26 +80,52 @@ def test__given_pre_pandemic_base__then_obr_forecast_volume_declines():
 def test__given_uprating_table__then_only_fuel_rows_are_overridden():
     # Given
     df = pd.DataFrame(
-        {year: [1.0, 1.0, 1.0] for year in range(START_YEAR, END_YEAR + 1)},
-        index=["petrol_spending", "diesel_spending", "some_other_variable"],
+        {year: [1.0, 1.0, 1.0, 1.0] for year in range(START_YEAR, END_YEAR + 1)},
+        index=[
+            "petrol_spending",
+            "diesel_spending",
+            "household_weight",
+            "some_other_variable",
+        ],
     )
+    df.loc["household_weight", 2024] = 1.2
 
     # When
     out = _apply_road_fuel_litre_proxy_override(df.copy())
 
     # Then
     for year in range(START_YEAR, END_YEAR + 1):
         assert out.loc["some_other_variable", year] == 1.0
+        assert out.loc["household_weight", year] == df.loc["household_weight", year]
     for variable in VOLUME_OVERRIDDEN_VARIABLES:
         expected = fuel_spending_litre_proxy_index(
             variable=variable,
             base_year=START_YEAR,
             end_year=END_YEAR,
+            household_weight_index=df.loc["household_weight"],
         )
         for year in range(START_YEAR, END_YEAR + 1):
             assert out.loc[variable, year] == round(expected[year], 3)
 
 
+def test__given_generated_uprating_table__then_household_weight_row_is_restored():
+    # Given
+    df = pd.DataFrame(
+        {year: [1.0, 2.0] for year in range(START_YEAR, END_YEAR + 1)},
+        index=["household_weight", "some_other_variable"],
+    )
+
+    # When
+    out = _apply_household_weight_uprating_override(df.copy())
+
+    # Then
+    for year in range(START_YEAR, END_YEAR + 1):
+        assert (
+            out.loc["household_weight", year] == HOUSEHOLD_WEIGHT_UPRATING_INDEX[year]
+        )
+        assert out.loc["some_other_variable", year] == 2.0
+
+
 def test__given_storage_csv__then_fuel_rows_reflect_litre_proxy_index():
     # Given
     df = pd.read_csv(STORAGE_FOLDER / "uprating_factors.csv").set_index("Variable")
@@ -108,12 +136,26 @@ def test__given_storage_csv__then_fuel_rows_reflect_litre_proxy_index():
             variable=variable,
             base_year=START_YEAR,
             end_year=END_YEAR,
+            household_weight_index=df.loc["household_weight"],
         )
         assert variable in df.index
         for year in range(START_YEAR, END_YEAR + 1):
             assert df.loc[variable, str(year)] == round(expected[year], 3)
 
 
+def test__given_storage_csv__then_household_weight_row_is_unchanged():
+    # Given
+    df = pd.read_csv(STORAGE_FOLDER / "uprating_factors.csv").set_index("Variable")
+
+    # Then
+    assert df.loc[
+        "household_weight", [str(year) for year in range(START_YEAR, END_YEAR + 1)]
+    ].tolist() == [
+        HOUSEHOLD_WEIGHT_UPRATING_INDEX[year]
+        for year in range(START_YEAR, END_YEAR + 1)
+    ]
+
+
 def test__given_lcfs_training_table__then_fuel_uprating_preserves_litre_proxy():
     # Given
     household = pd.DataFrame({variable: [1.0] for variable in IMPUTATIONS})

policyengine_uk_data/utils/uprating.py

@@ -14,6 +14,23 @@
     "petrol_spending": "petrol",
     "diesel_spending": "diesel",
 }
+HOUSEHOLD_WEIGHT_UPRATING_INDEX = {
+    2020: 1.0,
+    2021: 1.0,
+    2022: 1.003,
+    2023: 1.017,
+    2024: 1.027,
+    2025: 1.039,
+    2026: 1.046,
+    2027: 1.054,
+    2028: 1.058,
+    2029: 1.064,
+    2030: 1.064,
+    2031: 1.064,
+    2032: 1.064,
+    2033: 1.064,
+    2034: 1.064,
+}
 
 
 class UpratingYearOutOfRangeError(ValueError):
@@ -65,6 +82,11 @@ def create_policyengine_uprating_factors_table():
     df = df.pivot(index="Variable", columns="Year", values="Value")
     df = df.sort_values("Variable")
 
+    # Keep the population calibration row stable. Current PolicyEngine UK
+    # population indices would inflate final calibrated population above the
+    # existing fidelity guard.
+    df = _apply_household_weight_uprating_override(df)
+
     # Ensure petrol/diesel use sourced road-fuel clearances and model pump
     # prices. This keeps litres aligned after PolicyEngine divides by price.
     df = _apply_road_fuel_litre_proxy_override(df)
@@ -82,12 +104,33 @@ def create_policyengine_uprating_factors_table():
     return df
 
 
+def _apply_household_weight_uprating_override(df: pd.DataFrame) -> pd.DataFrame:
+    """Restore the household-weight index used by the calibrated dataset."""
+    if "household_weight" not in df.index:
+        return df
+
+    missing_years = [
+        year
+        for year in range(START_YEAR, END_YEAR + 1)
+        if year not in HOUSEHOLD_WEIGHT_UPRATING_INDEX
+    ]
+    if missing_years:
+        raise ValueError(
+            "Household-weight uprating index missing years: "
+            + ", ".join(str(year) for year in missing_years)
+        )
+    for year in range(START_YEAR, END_YEAR + 1):
+        df.loc["household_weight", year] = HOUSEHOLD_WEIGHT_UPRATING_INDEX[year]
+    return df
+
+
 def fuel_spending_litre_proxy_index(
     *,
     variable: str,
     base_year: int = START_YEAR,
     end_year: int = END_YEAR,
     parameters=None,
+    household_weight_index: dict[int, float] | pd.Series | None = None,
 ) -> dict[int, float]:
     """Return the spending-proxy index that preserves fuel litres.
 
@@ -112,16 +155,38 @@ def fuel_spending_litre_proxy_index(
         parameters.household.consumption.fuel.prices,
         FUEL_PRICE_PARAMETER_NAME[variable],
     )
-    population = parameters.gov.economic_assumptions.indices.ons.population
     base_price = price_parameter(base_year)
-    base_population = population(base_year)
+
+    if household_weight_index is None:
+        population = parameters.gov.economic_assumptions.indices.ons.population
+        base_household_weight_index = population(base_year)
+
+        def household_weight_relative(year: int) -> float:
+            return population(year) / base_household_weight_index
+
+    else:
+
+        def household_weight_value(year: int) -> float:
+            if hasattr(household_weight_index, "index"):
+                if year in household_weight_index.index:
+                    return float(household_weight_index.loc[year])
+                return float(household_weight_index.loc[str(year)])
+            try:
+                return float(household_weight_index[year])
+            except KeyError:
+                return float(household_weight_index[str(year)])
+
+        base_household_weight_index = household_weight_value(base_year)
+
+        def household_weight_relative(year: int) -> float:
+            return household_weight_value(year) / base_household_weight_index
+
     return {
         year: (
             volume_index[year]
             * price_parameter(year)
             / base_price
-            * base_population
-            / population(year)
+            / household_weight_relative(year)
         )
         for year in volume_index
     }
@@ -141,10 +206,14 @@ def _apply_road_fuel_litre_proxy_override(df: pd.DataFrame) -> pd.DataFrame:
     for variable in VOLUME_OVERRIDDEN_VARIABLES:
         if variable not in df.index:
             continue
+        household_weight_index = (
+            df.loc["household_weight"] if "household_weight" in df.index else None
+        )
         index = fuel_spending_litre_proxy_index(
             variable=variable,
             base_year=START_YEAR,
             end_year=END_YEAR,
+            household_weight_index=household_weight_index,
         )
         missing_years = [
             year for year in range(START_YEAR, END_YEAR + 1) if year not in index