Calibrate fuel litre proxies to road clearances

Author: MaxGhenis

changelog.d/fuel-litre-calibration.md

@@ -0,0 +1 @@
+- Calibrate final petrol and diesel litre proxies to HMRC/OBR road-fuel clearances, and uprate fuel-spending proxies by road-fuel litres and pump prices.

policyengine_uk_data/datasets/create_datasets.py

@@ -69,6 +69,7 @@ def main():
             "Calibrate constituency weights",
             "Calibrate local authority weights",
             "Downrate to 2023",
+            "Calibrate fuel litres",
             "Save final dataset",
             "Create tiny datasets",
         ]
@@ -217,6 +218,14 @@ def main():
             frs_calibrated = uprate_dataset(frs_calibrated_constituencies, 2023)
             update_dataset("Downrate to 2023", "completed")
 
+            update_dataset("Calibrate fuel litres", "processing")
+            from policyengine_uk_data.datasets.imputations.consumption import (
+                calibrate_dataset_fuel_litre_proxies_to_road_fuel,
+            )
+
+            calibrate_dataset_fuel_litre_proxies_to_road_fuel(frs_calibrated)
+            update_dataset("Calibrate fuel litres", "completed")
+
             update_dataset("Save final dataset", "processing")
             strip_internal_disability_reported_amounts(frs_calibrated).save(
                 STORAGE_FOLDER / "enhanced_frs_2023_24.h5"

policyengine_uk_data/datasets/imputations/consumption.py

@@ -610,13 +610,79 @@ def fuel_spending_litre_proxy_uprating(
         parameters.household.consumption.fuel.prices,
         FUEL_PRICE_PARAMETER_NAME[variable],
     )
+    population = parameters.gov.economic_assumptions.indices.ons.population
     return (
         road_fuel_volume_uprating(start_year=start_year, end_year=end_year)
+        / (population(end_year) / population(start_year))
         * model_price(end_year)
         / lcfs_price
     )
 
 
+def _fuel_litre_proxy_mlitres(
+    household: pd.DataFrame,
+    fiscal_year: int,
+    parameters=None,
+) -> float:
+    """Return weighted petrol + diesel litres represented by spending proxies."""
+    from policyengine_uk.system import system
+
+    if parameters is None:
+        parameters = system.parameters
+
+    total_litres = 0.0
+    for variable, price_parameter_name in FUEL_PRICE_PARAMETER_NAME.items():
+        price = getattr(
+            parameters.household.consumption.fuel.prices,
+            price_parameter_name,
+        )(fiscal_year)
+        total_litres += household[variable] / price
+    return (total_litres * household["household_weight"]).sum() / 1_000_000
+
+
+def calibrate_fuel_litre_proxies_to_road_fuel(
+    household: pd.DataFrame,
+    fiscal_year: int,
+    parameters=None,
+) -> float:
+    """Scale imputed fuel proxies to HMRC/OBR road-fuel litre controls.
+
+    PolicyEngine UK derives petrol and diesel litres from spending divided by
+    pump prices. Applying one multiplicative factor to petrol and diesel
+    spending preserves the household distribution while making the resulting
+    litres reconcile to the external fuel-duty volume benchmark.
+    """
+    from policyengine_uk_data.sources.road_fuel_volume import (
+        road_fuel_clearances_mlitres,
+    )
+
+    actual_mlitres = _fuel_litre_proxy_mlitres(
+        household,
+        fiscal_year,
+        parameters=parameters,
+    )
+    if actual_mlitres <= 0:
+        return 1.0
+
+    target_mlitres = road_fuel_clearances_mlitres(end_year=fiscal_year)[fiscal_year]
+    scale = target_mlitres / actual_mlitres
+    for variable in FUEL_PRICE_PARAMETER_NAME:
+        household[variable] *= scale
+    return scale
+
+
+def calibrate_dataset_fuel_litre_proxies_to_road_fuel(
+    dataset: UKSingleYearDataset,
+    parameters=None,
+) -> float:
+    """Scale a dataset's fuel proxies after final household weights are set."""
+    return calibrate_fuel_litre_proxies_to_road_fuel(
+        dataset.household,
+        int(dataset.time_period),
+        parameters=parameters,
+    )
+
+
 def save_imputation_models():
     from policyengine_uk_data.utils.qrf import QRF
 

policyengine_uk_data/storage/uprating_factors.csv

@@ -12,7 +12,7 @@ childcare_expenses,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,
 clothing_and_footwear_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
 communication_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
 corporate_wealth,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384
-diesel_spending,1.0,1.244,1.322,1.314,1.313,1.287,1.268,1.237,1.204,1.159,1.099,1.099,1.099,1.099,1.099
+diesel_spending,1.0,1.138,1.56,1.531,1.579,1.537,1.509,1.467,1.422,1.363,1.286,1.28,1.275,1.27,1.265
 dividend_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384
 domestic_energy_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
 education_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
@@ -30,7 +30,7 @@ free_school_milk,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.
 gross_financial_wealth,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384
 health_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
 household_furnishings_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
-household_weight,1.0,1.0,1.003,1.017,1.027,1.039,1.046,1.054,1.058,1.064,1.064,1.064,1.064,1.064,1.064
+household_weight,1.0,1.0,1.009,1.023,1.033,1.041,1.045,1.049,1.053,1.058,1.062,1.067,1.071,1.076,1.08
 housing_benefit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
 housing_service_charges,1.0,1.0,1.0,1.064,1.137,1.191,1.235,1.262,1.289,1.318,1.318,1.318,1.318,1.318,1.318
 housing_water_and_electricity_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
@@ -56,7 +56,7 @@ owned_land,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.3
 pension_credit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
 pension_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384
 personal_pension_contributions,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431
-petrol_spending,1.0,1.244,1.322,1.314,1.313,1.287,1.268,1.237,1.204,1.159,1.099,1.099,1.099,1.099,1.099
+petrol_spending,1.0,1.104,1.624,1.786,1.521,1.48,1.454,1.413,1.37,1.313,1.239,1.233,1.228,1.223,1.218
 private_pension_income,1.0,1.003,1.053,1.106,1.161,1.216,1.261,1.288,1.315,1.346,1.346,1.346,1.346,1.346,1.346
 private_transfer_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384
 property_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384

policyengine_uk_data/storage/uprating_growth_factors.csv

@@ -12,7 +12,7 @@ childcare_expenses,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0
 clothing_and_footwear_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
 communication_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
 corporate_wealth,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0
-diesel_spending,0,0.244,0.063,-0.006,-0.001,-0.02,-0.015,-0.024,-0.027,-0.037,-0.052,0.0,0.0,0.0,0.0
+diesel_spending,0,0.138,0.371,-0.019,0.031,-0.027,-0.018,-0.028,-0.031,-0.041,-0.056,-0.005,-0.004,-0.004,-0.004
 dividend_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0
 domestic_energy_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
 education_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
@@ -30,7 +30,7 @@ free_school_milk,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0
 gross_financial_wealth,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0
 health_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
 household_furnishings_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
-household_weight,0,0.0,0.003,0.014,0.01,0.012,0.007,0.008,0.004,0.006,0.0,0.0,0.0,0.0,0.0
+household_weight,0,0.0,0.009,0.014,0.01,0.008,0.004,0.004,0.004,0.005,0.004,0.005,0.004,0.005,0.004
 housing_benefit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
 housing_service_charges,0,0.0,0.0,0.064,0.069,0.047,0.037,0.022,0.021,0.022,0.0,0.0,0.0,0.0,0.0
 housing_water_and_electricity_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
@@ -56,7 +56,7 @@ owned_land,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,
 pension_credit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
 pension_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0
 personal_pension_contributions,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0
-petrol_spending,0,0.244,0.063,-0.006,-0.001,-0.02,-0.015,-0.024,-0.027,-0.037,-0.052,0.0,0.0,0.0,0.0
+petrol_spending,0,0.104,0.471,0.1,-0.148,-0.027,-0.018,-0.028,-0.03,-0.042,-0.056,-0.005,-0.004,-0.004,-0.004
 private_pension_income,0,0.003,0.05,0.05,0.05,0.047,0.037,0.021,0.021,0.024,0.0,0.0,0.0,0.0,0.0
 private_transfer_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0
 property_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0

policyengine_uk_data/tests/test_road_fuel_volume_uprating.py

@@ -1,10 +1,15 @@
 """Tests for road-fuel volume uprating. See #402."""
 
 import pandas as pd
+import pytest
+from policyengine_uk.data import UKSingleYearDataset
 
 from policyengine_uk_data.datasets.imputations.consumption import (
     CONSUMPTION_MODEL_FILENAME,
     IMPUTATIONS,
+    _fuel_litre_proxy_mlitres,
+    calibrate_dataset_fuel_litre_proxies_to_road_fuel,
+    calibrate_fuel_litre_proxies_to_road_fuel,
     fuel_spending_litre_proxy_uprating,
     uprate_lcfs_table,
 )
@@ -23,7 +28,9 @@
     END_YEAR,
     START_YEAR,
     VOLUME_OVERRIDDEN_VARIABLES,
-    _apply_road_fuel_volume_override,
+    _apply_road_fuel_litre_proxy_override,
+    fuel_spending_litre_proxy_index,
+    uprate_dataset,
 )
 
 
@@ -76,24 +83,32 @@ def test__given_uprating_table__then_only_fuel_rows_are_overridden():
     )
 
     # When
-    out = _apply_road_fuel_volume_override(df.copy())
-    expected = road_fuel_volume_index(base_year=START_YEAR, end_year=END_YEAR)
+    out = _apply_road_fuel_litre_proxy_override(df.copy())
 
     # Then
     for year in range(START_YEAR, END_YEAR + 1):
         assert out.loc["some_other_variable", year] == 1.0
     for variable in VOLUME_OVERRIDDEN_VARIABLES:
+        expected = fuel_spending_litre_proxy_index(
+            variable=variable,
+            base_year=START_YEAR,
+            end_year=END_YEAR,
+        )
         for year in range(START_YEAR, END_YEAR + 1):
             assert out.loc[variable, year] == round(expected[year], 3)
 
 
-def test__given_storage_csv__then_fuel_rows_reflect_volume_index():
+def test__given_storage_csv__then_fuel_rows_reflect_litre_proxy_index():
     # Given
     df = pd.read_csv(STORAGE_FOLDER / "uprating_factors.csv").set_index("Variable")
-    expected = road_fuel_volume_index(base_year=START_YEAR, end_year=END_YEAR)
 
     # Then
     for variable in VOLUME_OVERRIDDEN_VARIABLES:
+        expected = fuel_spending_litre_proxy_index(
+            variable=variable,
+            base_year=START_YEAR,
+            end_year=END_YEAR,
+        )
         assert variable in df.index
         for year in range(START_YEAR, END_YEAR + 1):
             assert df.loc[variable, str(year)] == round(expected[year], 3)
@@ -148,10 +163,124 @@ def test__given_obr_2027_volume__then_rate_difference_matches_cost_benchmark():
     assert round(benchmark_cost_gbp_bn, 2) == 3.12
 
 
+def test__given_imputed_fuel_proxies__then_calibration_matches_road_fuel_litres():
+    # Given
+    from policyengine_uk.system import system
+
+    year = 2027
+    target_mlitres = road_fuel_clearances_mlitres()[year]
+    petrol_price = system.parameters.household.consumption.fuel.prices.petrol(year)
+    diesel_price = system.parameters.household.consumption.fuel.prices.diesel(year)
+    household = pd.DataFrame(
+        {
+            "household_weight": [1.0, 1.0],
+            "petrol_spending": [
+                target_mlitres * 1_000_000 * petrol_price,
+                0.0,
+            ],
+            "diesel_spending": [
+                0.0,
+                target_mlitres * 1_000_000 * diesel_price,
+            ],
+        }
+    )
+
+    # When
+    scale = calibrate_fuel_litre_proxies_to_road_fuel(household, year)
+
+    # Then
+    assert scale == pytest.approx(0.5)
+    assert _fuel_litre_proxy_mlitres(household, year) == pytest.approx(target_mlitres)
+
+
+def test__given_final_weight_changes__then_dataset_calibration_matches_litres():
+    # Given
+    from policyengine_uk.system import system
+
+    year = 2023
+    target_mlitres = road_fuel_clearances_mlitres()[year]
+    petrol_price = system.parameters.household.consumption.fuel.prices.petrol(year)
+    dataset = _minimal_fuel_dataset(
+        fiscal_year=year,
+        petrol_spending=target_mlitres * 1_000_000 * petrol_price,
+        diesel_spending=0.0,
+        household_weight=2.0,
+    )
+
+    # When
+    scale = calibrate_dataset_fuel_litre_proxies_to_road_fuel(dataset)
+
+    # Then
+    assert scale == pytest.approx(0.5)
+    assert _fuel_litre_proxy_mlitres(dataset.household, year) == pytest.approx(
+        target_mlitres
+    )
+
+
+def test__given_calibrated_dataset_uprated__then_litres_track_forecast_target():
+    # Given
+    from policyengine_uk.system import system
+
+    start_year = 2023
+    target_year = 2027
+    start_mlitres = road_fuel_clearances_mlitres()[start_year]
+    petrol_price = system.parameters.household.consumption.fuel.prices.petrol(
+        start_year
+    )
+    dataset = _minimal_fuel_dataset(
+        fiscal_year=start_year,
+        petrol_spending=start_mlitres * 1_000_000 * petrol_price,
+        diesel_spending=0.0,
+    )
+    calibrate_dataset_fuel_litre_proxies_to_road_fuel(dataset)
+
+    # When
+    uprated = uprate_dataset(dataset, target_year)
+
+    # Then
+    assert _fuel_litre_proxy_mlitres(
+        uprated.household,
+        target_year,
+    ) == pytest.approx(road_fuel_clearances_mlitres()[target_year], rel=1e-3)
+
+
 def test__given_year_after_obr_horizon__then_last_forecast_is_carried_forward():
     # When
     series = road_fuel_clearances_mlitres(end_year=END_YEAR)
 
     # Then
     assert series[2031] == series[2030]
     assert series[2034] == series[2030]
+
+
+def _minimal_fuel_dataset(
+    *,
+    fiscal_year: int,
+    petrol_spending: float,
+    diesel_spending: float,
+    household_weight: float = 1.0,
+) -> UKSingleYearDataset:
+    return UKSingleYearDataset(
+        person=pd.DataFrame(
+            {
+                "person_id": [0],
+                "benunit_id": [0],
+                "household_id": [0],
+            }
+        ),
+        benunit=pd.DataFrame(
+            {
+                "benunit_id": [0],
+                "household_id": [0],
+            }
+        ),
+        household=pd.DataFrame(
+            {
+                "household_id": [0],
+                "household_weight": [household_weight],
+                "petrol_spending": [petrol_spending],
+                "diesel_spending": [diesel_spending],
+            }
+        ),
+        fiscal_year=fiscal_year,
+    )