Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fuel-litre-calibration.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Calibrate final petrol and diesel litre proxies to HMRC/OBR road-fuel clearances, and uprate fuel-spending proxies by road-fuel litres and pump prices.
9 changes: 9 additions & 0 deletions policyengine_uk_data/datasets/create_datasets.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ def main():
"Calibrate constituency weights",
"Calibrate local authority weights",
"Downrate to 2023",
"Calibrate fuel litres",
"Save final dataset",
"Create tiny datasets",
]
Expand Down Expand Up @@ -217,6 +218,14 @@ def main():
frs_calibrated = uprate_dataset(frs_calibrated_constituencies, 2023)
update_dataset("Downrate to 2023", "completed")

update_dataset("Calibrate fuel litres", "processing")
from policyengine_uk_data.datasets.imputations.consumption import (
calibrate_dataset_fuel_litre_proxies_to_road_fuel,
)

calibrate_dataset_fuel_litre_proxies_to_road_fuel(frs_calibrated)
update_dataset("Calibrate fuel litres", "completed")

update_dataset("Save final dataset", "processing")
strip_internal_disability_reported_amounts(frs_calibrated).save(
STORAGE_FOLDER / "enhanced_frs_2023_24.h5"
Expand Down
66 changes: 66 additions & 0 deletions policyengine_uk_data/datasets/imputations/consumption.py
Original file line number Diff line number Diff line change
Expand Up @@ -610,13 +610,79 @@ def fuel_spending_litre_proxy_uprating(
parameters.household.consumption.fuel.prices,
FUEL_PRICE_PARAMETER_NAME[variable],
)
population = parameters.gov.economic_assumptions.indices.ons.population
return (
road_fuel_volume_uprating(start_year=start_year, end_year=end_year)
/ (population(end_year) / population(start_year))
* model_price(end_year)
/ lcfs_price
)


def _fuel_litre_proxy_mlitres(
household: pd.DataFrame,
fiscal_year: int,
parameters=None,
) -> float:
"""Return weighted petrol + diesel litres represented by spending proxies."""
from policyengine_uk.system import system

if parameters is None:
parameters = system.parameters

total_litres = 0.0
for variable, price_parameter_name in FUEL_PRICE_PARAMETER_NAME.items():
price = getattr(
parameters.household.consumption.fuel.prices,
price_parameter_name,
)(fiscal_year)
total_litres += household[variable] / price
return (total_litres * household["household_weight"]).sum() / 1_000_000


def calibrate_fuel_litre_proxies_to_road_fuel(
household: pd.DataFrame,
fiscal_year: int,
parameters=None,
) -> float:
"""Scale imputed fuel proxies to HMRC/OBR road-fuel litre controls.

PolicyEngine UK derives petrol and diesel litres from spending divided by
pump prices. Applying one multiplicative factor to petrol and diesel
spending preserves the household distribution while making the resulting
litres reconcile to the external fuel-duty volume benchmark.
"""
from policyengine_uk_data.sources.road_fuel_volume import (
road_fuel_clearances_mlitres,
)

actual_mlitres = _fuel_litre_proxy_mlitres(
household,
fiscal_year,
parameters=parameters,
)
if actual_mlitres <= 0:
return 1.0

target_mlitres = road_fuel_clearances_mlitres(end_year=fiscal_year)[fiscal_year]
scale = target_mlitres / actual_mlitres
for variable in FUEL_PRICE_PARAMETER_NAME:
household[variable] *= scale
return scale


def calibrate_dataset_fuel_litre_proxies_to_road_fuel(
dataset: UKSingleYearDataset,
parameters=None,
) -> float:
"""Scale a dataset's fuel proxies after final household weights are set."""
return calibrate_fuel_litre_proxies_to_road_fuel(
dataset.household,
int(dataset.time_period),
parameters=parameters,
)


def save_imputation_models():
from policyengine_uk_data.utils.qrf import QRF

Expand Down
6 changes: 3 additions & 3 deletions policyengine_uk_data/storage/uprating_factors.csv
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ childcare_expenses,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,
clothing_and_footwear_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
communication_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
corporate_wealth,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384
diesel_spending,1.0,1.244,1.322,1.314,1.313,1.287,1.268,1.237,1.204,1.159,1.099,1.099,1.099,1.099,1.099
diesel_spending,1.0,1.138,1.56,1.531,1.579,1.537,1.509,1.467,1.422,1.363,1.286,1.28,1.275,1.27,1.265
dividend_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384
domestic_energy_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
education_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
Expand All @@ -30,7 +30,7 @@ free_school_milk,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.
gross_financial_wealth,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384
health_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
household_furnishings_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
household_weight,1.0,1.0,1.003,1.017,1.027,1.039,1.046,1.054,1.058,1.064,1.064,1.064,1.064,1.064,1.064
household_weight,1.0,1.0,1.009,1.023,1.033,1.041,1.045,1.049,1.053,1.058,1.062,1.067,1.071,1.076,1.08
housing_benefit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
housing_service_charges,1.0,1.0,1.0,1.064,1.137,1.191,1.235,1.262,1.289,1.318,1.318,1.318,1.318,1.318,1.318
housing_water_and_electricity_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
Expand All @@ -56,7 +56,7 @@ owned_land,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.3
pension_credit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
pension_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384
personal_pension_contributions,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431
petrol_spending,1.0,1.244,1.322,1.314,1.313,1.287,1.268,1.237,1.204,1.159,1.099,1.099,1.099,1.099,1.099
petrol_spending,1.0,1.104,1.624,1.786,1.521,1.48,1.454,1.413,1.37,1.313,1.239,1.233,1.228,1.223,1.218
private_pension_income,1.0,1.003,1.053,1.106,1.161,1.216,1.261,1.288,1.315,1.346,1.346,1.346,1.346,1.346,1.346
private_transfer_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384
property_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384
Expand Down
6 changes: 3 additions & 3 deletions policyengine_uk_data/storage/uprating_growth_factors.csv
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ childcare_expenses,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0
clothing_and_footwear_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
communication_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
corporate_wealth,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0
diesel_spending,0,0.244,0.063,-0.006,-0.001,-0.02,-0.015,-0.024,-0.027,-0.037,-0.052,0.0,0.0,0.0,0.0
diesel_spending,0,0.138,0.371,-0.019,0.031,-0.027,-0.018,-0.028,-0.031,-0.041,-0.056,-0.005,-0.004,-0.004,-0.004
dividend_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0
domestic_energy_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
education_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
Expand All @@ -30,7 +30,7 @@ free_school_milk,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0
gross_financial_wealth,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0
health_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
household_furnishings_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
household_weight,0,0.0,0.003,0.014,0.01,0.012,0.007,0.008,0.004,0.006,0.0,0.0,0.0,0.0,0.0
household_weight,0,0.0,0.009,0.014,0.01,0.008,0.004,0.004,0.004,0.005,0.004,0.005,0.004,0.005,0.004
housing_benefit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
housing_service_charges,0,0.0,0.0,0.064,0.069,0.047,0.037,0.022,0.021,0.022,0.0,0.0,0.0,0.0,0.0
housing_water_and_electricity_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
Expand All @@ -56,7 +56,7 @@ owned_land,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,
pension_credit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
pension_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0
personal_pension_contributions,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0
petrol_spending,0,0.244,0.063,-0.006,-0.001,-0.02,-0.015,-0.024,-0.027,-0.037,-0.052,0.0,0.0,0.0,0.0
petrol_spending,0,0.104,0.471,0.1,-0.148,-0.027,-0.018,-0.028,-0.03,-0.042,-0.056,-0.005,-0.004,-0.004,-0.004
private_pension_income,0,0.003,0.05,0.05,0.05,0.047,0.037,0.021,0.021,0.024,0.0,0.0,0.0,0.0,0.0
private_transfer_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0
property_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0
Expand Down
139 changes: 134 additions & 5 deletions policyengine_uk_data/tests/test_road_fuel_volume_uprating.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,15 @@
"""Tests for road-fuel volume uprating. See #402."""

import pandas as pd
import pytest
from policyengine_uk.data import UKSingleYearDataset

from policyengine_uk_data.datasets.imputations.consumption import (
CONSUMPTION_MODEL_FILENAME,
IMPUTATIONS,
_fuel_litre_proxy_mlitres,
calibrate_dataset_fuel_litre_proxies_to_road_fuel,
calibrate_fuel_litre_proxies_to_road_fuel,
fuel_spending_litre_proxy_uprating,
uprate_lcfs_table,
)
Expand All @@ -23,7 +28,9 @@
END_YEAR,
START_YEAR,
VOLUME_OVERRIDDEN_VARIABLES,
_apply_road_fuel_volume_override,
_apply_road_fuel_litre_proxy_override,
fuel_spending_litre_proxy_index,
uprate_dataset,
)


Expand Down Expand Up @@ -76,24 +83,32 @@ def test__given_uprating_table__then_only_fuel_rows_are_overridden():
)

# When
out = _apply_road_fuel_volume_override(df.copy())
expected = road_fuel_volume_index(base_year=START_YEAR, end_year=END_YEAR)
out = _apply_road_fuel_litre_proxy_override(df.copy())

# Then
for year in range(START_YEAR, END_YEAR + 1):
assert out.loc["some_other_variable", year] == 1.0
for variable in VOLUME_OVERRIDDEN_VARIABLES:
expected = fuel_spending_litre_proxy_index(
variable=variable,
base_year=START_YEAR,
end_year=END_YEAR,
)
for year in range(START_YEAR, END_YEAR + 1):
assert out.loc[variable, year] == round(expected[year], 3)


def test__given_storage_csv__then_fuel_rows_reflect_volume_index():
def test__given_storage_csv__then_fuel_rows_reflect_litre_proxy_index():
# Given
df = pd.read_csv(STORAGE_FOLDER / "uprating_factors.csv").set_index("Variable")
expected = road_fuel_volume_index(base_year=START_YEAR, end_year=END_YEAR)

# Then
for variable in VOLUME_OVERRIDDEN_VARIABLES:
expected = fuel_spending_litre_proxy_index(
variable=variable,
base_year=START_YEAR,
end_year=END_YEAR,
)
assert variable in df.index
for year in range(START_YEAR, END_YEAR + 1):
assert df.loc[variable, str(year)] == round(expected[year], 3)
Expand Down Expand Up @@ -148,10 +163,124 @@ def test__given_obr_2027_volume__then_rate_difference_matches_cost_benchmark():
assert round(benchmark_cost_gbp_bn, 2) == 3.12


def test__given_imputed_fuel_proxies__then_calibration_matches_road_fuel_litres():
# Given
from policyengine_uk.system import system

year = 2027
target_mlitres = road_fuel_clearances_mlitres()[year]
petrol_price = system.parameters.household.consumption.fuel.prices.petrol(year)
diesel_price = system.parameters.household.consumption.fuel.prices.diesel(year)
household = pd.DataFrame(
{
"household_weight": [1.0, 1.0],
"petrol_spending": [
target_mlitres * 1_000_000 * petrol_price,
0.0,
],
"diesel_spending": [
0.0,
target_mlitres * 1_000_000 * diesel_price,
],
}
)

# When
scale = calibrate_fuel_litre_proxies_to_road_fuel(household, year)

# Then
assert scale == pytest.approx(0.5)
assert _fuel_litre_proxy_mlitres(household, year) == pytest.approx(target_mlitres)


def test__given_final_weight_changes__then_dataset_calibration_matches_litres():
# Given
from policyengine_uk.system import system

year = 2023
target_mlitres = road_fuel_clearances_mlitres()[year]
petrol_price = system.parameters.household.consumption.fuel.prices.petrol(year)
dataset = _minimal_fuel_dataset(
fiscal_year=year,
petrol_spending=target_mlitres * 1_000_000 * petrol_price,
diesel_spending=0.0,
household_weight=2.0,
)

# When
scale = calibrate_dataset_fuel_litre_proxies_to_road_fuel(dataset)

# Then
assert scale == pytest.approx(0.5)
assert _fuel_litre_proxy_mlitres(dataset.household, year) == pytest.approx(
target_mlitres
)


def test__given_calibrated_dataset_uprated__then_litres_track_forecast_target():
# Given
from policyengine_uk.system import system

start_year = 2023
target_year = 2027
start_mlitres = road_fuel_clearances_mlitres()[start_year]
petrol_price = system.parameters.household.consumption.fuel.prices.petrol(
start_year
)
dataset = _minimal_fuel_dataset(
fiscal_year=start_year,
petrol_spending=start_mlitres * 1_000_000 * petrol_price,
diesel_spending=0.0,
)
calibrate_dataset_fuel_litre_proxies_to_road_fuel(dataset)

# When
uprated = uprate_dataset(dataset, target_year)

# Then
assert _fuel_litre_proxy_mlitres(
uprated.household,
target_year,
) == pytest.approx(road_fuel_clearances_mlitres()[target_year], rel=1e-3)


def test__given_year_after_obr_horizon__then_last_forecast_is_carried_forward():
# When
series = road_fuel_clearances_mlitres(end_year=END_YEAR)

# Then
assert series[2031] == series[2030]
assert series[2034] == series[2030]


def _minimal_fuel_dataset(
*,
fiscal_year: int,
petrol_spending: float,
diesel_spending: float,
household_weight: float = 1.0,
) -> UKSingleYearDataset:
return UKSingleYearDataset(
person=pd.DataFrame(
{
"person_id": [0],
"benunit_id": [0],
"household_id": [0],
}
),
benunit=pd.DataFrame(
{
"benunit_id": [0],
"household_id": [0],
}
),
household=pd.DataFrame(
{
"household_id": [0],
"household_weight": [household_weight],
"petrol_spending": [petrol_spending],
"diesel_spending": [diesel_spending],
}
),
fiscal_year=fiscal_year,
)
Loading
Loading