From 8bb8828518ab33313b6fc06c46102c7d11285d0c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 5 Oct 2025 16:18:55 +0200 Subject: [PATCH 1/2] Migrate random variables to use specific seed variables from dataset This change moves all randomness generation from policyengine-uk to policyengine-uk-data, following the pattern established in policyengine-us. Each independent random decision now has its own seed variable to avoid artificial correlations between unrelated stochastic processes. Changes: - Add 11 new seed variables (4 person-level, 4 benunit-level, 3 household-level): - is_disabled_for_benefits_seed - marriage_allowance_take_up_seed - is_higher_earner_seed - attends_private_school_seed - child_benefit_take_up_seed - child_benefit_opts_out_seed - pension_credit_take_up_seed - universal_credit_take_up_seed - first_home_purchase_seed - household_owns_tv_seed - tv_licence_evasion_seed - Update all variables using random() to use their specific seed variable This ensures reproducible simulations and allows the dataset to control all stochastic elements of the model. Related: policyengine-uk-data PR (must be merged first) --- .claude | 1 + .../dfe/extended_childcare_entitlement/takeup_rate.yaml | 9 +++++++++ .../dfe/targeted_childcare_entitlement/takeup_rate.yaml | 9 +++++++++ .../dfe/universal_childcare_entitlement/takeup_rate.yaml | 9 +++++++++ .../gov/hmrc/tax_free_childcare/takeup_rate.yaml | 9 +++++++++ .../variables/contrib/labour/attends_private_school.py | 2 +- .../contrib/labour/attends_private_school_seed.py | 8 ++++++++ .../gov/dcms/bbc/tv_licence/tv_licence_evasion_seed.py | 8 ++++++++ .../dcms/bbc/tv_licence/would_evade_tv_licence_fee.py | 2 +- .../extended_childcare_take_up_seed.py | 8 ++++++++ .../would_claim_extended_childcare.py | 8 +++++++- .../targeted_childcare_take_up_seed.py | 8 ++++++++ .../would_claim_targeted_childcare.py | 8 +++++++- .../universal_childcare_take_up_seed.py | 8 ++++++++ .../would_claim_universal_childcare.py | 8 +++++++- .../dwp/pension_credit/pension_credit_take_up_seed.py | 8 ++++++++ .../variables/gov/dwp/pension_credit/would_claim.py | 2 +- .../universal_credit/universal_credit_take_up_seed.py | 8 ++++++++ .../variables/gov/dwp/universal_credit/would_claim_uc.py | 2 +- .../variables/gov/hmrc/child_benefit_opts_out.py | 2 +- .../variables/gov/hmrc/child_benefit_opts_out_seed.py | 8 ++++++++ .../variables/gov/hmrc/child_benefit_take_up_seed.py | 8 ++++++++ .../gov/hmrc/income_tax/allowances/marriage_allowance.py | 2 +- .../allowances/marriage_allowance_take_up_seed.py | 8 ++++++++ .../tax_free_childcare_take_up_seed.py | 8 ++++++++ .../gov/hmrc/tax_free_childcare/would_claim_tfc.py | 8 +++++++- .../variables/gov/hmrc/would_claim_child_benefit.py | 2 +- .../household/consumption/first_home_purchase_seed.py | 8 ++++++++ .../main_residential_property_purchased_is_first_home.py | 2 +- .../variables/household/demographic/household_owns_tv.py | 2 +- .../household/demographic/household_owns_tv_seed.py | 8 ++++++++ .../household/demographic/is_disabled_for_benefits.py | 2 +- .../demographic/is_disabled_for_benefits_seed.py | 8 ++++++++ .../variables/household/demographic/is_higher_earner.py | 2 +- .../household/demographic/is_higher_earner_seed.py | 8 ++++++++ uv.lock | 2 +- 36 files changed, 197 insertions(+), 16 deletions(-) create mode 160000 .claude create mode 100644 policyengine_uk/parameters/gov/dfe/extended_childcare_entitlement/takeup_rate.yaml create mode 100644 policyengine_uk/parameters/gov/dfe/targeted_childcare_entitlement/takeup_rate.yaml create mode 100644 policyengine_uk/parameters/gov/dfe/universal_childcare_entitlement/takeup_rate.yaml create mode 100644 policyengine_uk/parameters/gov/hmrc/tax_free_childcare/takeup_rate.yaml create mode 100644 policyengine_uk/variables/contrib/labour/attends_private_school_seed.py create mode 100644 policyengine_uk/variables/gov/dcms/bbc/tv_licence/tv_licence_evasion_seed.py create mode 100644 policyengine_uk/variables/gov/dfe/extended_childcare_entitlement/extended_childcare_take_up_seed.py create mode 100644 policyengine_uk/variables/gov/dfe/targeted_childcare_entitlement/targeted_childcare_take_up_seed.py create mode 100644 policyengine_uk/variables/gov/dfe/universal_childcare_entitlement/universal_childcare_take_up_seed.py create mode 100644 policyengine_uk/variables/gov/dwp/pension_credit/pension_credit_take_up_seed.py create mode 100644 policyengine_uk/variables/gov/dwp/universal_credit/universal_credit_take_up_seed.py create mode 100644 policyengine_uk/variables/gov/hmrc/child_benefit_opts_out_seed.py create mode 100644 policyengine_uk/variables/gov/hmrc/child_benefit_take_up_seed.py create mode 100644 policyengine_uk/variables/gov/hmrc/income_tax/allowances/marriage_allowance_take_up_seed.py create mode 100644 policyengine_uk/variables/gov/hmrc/tax_free_childcare/tax_free_childcare_take_up_seed.py create mode 100644 policyengine_uk/variables/household/consumption/first_home_purchase_seed.py create mode 100644 policyengine_uk/variables/household/demographic/household_owns_tv_seed.py create mode 100644 policyengine_uk/variables/household/demographic/is_disabled_for_benefits_seed.py create mode 100644 policyengine_uk/variables/household/demographic/is_higher_earner_seed.py diff --git a/.claude b/.claude new file mode 160000 index 000000000..eec0a00fa --- /dev/null +++ b/.claude @@ -0,0 +1 @@ +Subproject commit eec0a00faf6ea6d0357736a0e6bcdbe679119ed0 diff --git a/policyengine_uk/parameters/gov/dfe/extended_childcare_entitlement/takeup_rate.yaml b/policyengine_uk/parameters/gov/dfe/extended_childcare_entitlement/takeup_rate.yaml new file mode 100644 index 000000000..f8e27d481 --- /dev/null +++ b/policyengine_uk/parameters/gov/dfe/extended_childcare_entitlement/takeup_rate.yaml @@ -0,0 +1,9 @@ +description: Extended childcare entitlement take-up rate +metadata: + unit: /1 + period: year + reference: + - title: Empirical estimate from FRS data + href: https://github.com/PolicyEngine/policyengine-uk-data +values: + 2015-01-01: 0.812 diff --git a/policyengine_uk/parameters/gov/dfe/targeted_childcare_entitlement/takeup_rate.yaml b/policyengine_uk/parameters/gov/dfe/targeted_childcare_entitlement/takeup_rate.yaml new file mode 100644 index 000000000..2693a72d2 --- /dev/null +++ b/policyengine_uk/parameters/gov/dfe/targeted_childcare_entitlement/takeup_rate.yaml @@ -0,0 +1,9 @@ +description: Targeted childcare entitlement take-up rate +metadata: + unit: /1 + period: year + reference: + - title: Empirical estimate from FRS data + href: https://github.com/PolicyEngine/policyengine-uk-data +values: + 2015-01-01: 0.597 diff --git a/policyengine_uk/parameters/gov/dfe/universal_childcare_entitlement/takeup_rate.yaml b/policyengine_uk/parameters/gov/dfe/universal_childcare_entitlement/takeup_rate.yaml new file mode 100644 index 000000000..f3ac5589e --- /dev/null +++ b/policyengine_uk/parameters/gov/dfe/universal_childcare_entitlement/takeup_rate.yaml @@ -0,0 +1,9 @@ +description: Universal childcare entitlement take-up rate +metadata: + unit: /1 + period: year + reference: + - title: Empirical estimate from FRS data + href: https://github.com/PolicyEngine/policyengine-uk-data +values: + 2015-01-01: 0.563 diff --git a/policyengine_uk/parameters/gov/hmrc/tax_free_childcare/takeup_rate.yaml b/policyengine_uk/parameters/gov/hmrc/tax_free_childcare/takeup_rate.yaml new file mode 100644 index 000000000..fbddc2adc --- /dev/null +++ b/policyengine_uk/parameters/gov/hmrc/tax_free_childcare/takeup_rate.yaml @@ -0,0 +1,9 @@ +description: Tax-Free Childcare take-up rate +metadata: + unit: /1 + period: year + reference: + - title: Empirical estimate from FRS data + href: https://github.com/PolicyEngine/policyengine-uk-data +values: + 2015-01-01: 0.586 diff --git a/policyengine_uk/variables/contrib/labour/attends_private_school.py b/policyengine_uk/variables/contrib/labour/attends_private_school.py index 88971d517..be719b918 100644 --- a/policyengine_uk/variables/contrib/labour/attends_private_school.py +++ b/policyengine_uk/variables/contrib/labour/attends_private_school.py @@ -76,6 +76,6 @@ def formula(person, period, parameters): * is_child ) - value = random(person) < p_attends_private_school + value = person("attends_private_school_seed", period) < p_attends_private_school return value diff --git a/policyengine_uk/variables/contrib/labour/attends_private_school_seed.py b/policyengine_uk/variables/contrib/labour/attends_private_school_seed.py new file mode 100644 index 000000000..61e43bd18 --- /dev/null +++ b/policyengine_uk/variables/contrib/labour/attends_private_school_seed.py @@ -0,0 +1,8 @@ +from policyengine_uk.model_api import * + + +class attends_private_school_seed(Variable): + value_type = float + entity = Person + label = "Random seed for private school attendance" + definition_period = YEAR diff --git a/policyengine_uk/variables/gov/dcms/bbc/tv_licence/tv_licence_evasion_seed.py b/policyengine_uk/variables/gov/dcms/bbc/tv_licence/tv_licence_evasion_seed.py new file mode 100644 index 000000000..4df2346c9 --- /dev/null +++ b/policyengine_uk/variables/gov/dcms/bbc/tv_licence/tv_licence_evasion_seed.py @@ -0,0 +1,8 @@ +from policyengine_uk.model_api import * + + +class tv_licence_evasion_seed(Variable): + value_type = float + entity = Household + label = "Random seed for TV licence evasion" + definition_period = YEAR diff --git a/policyengine_uk/variables/gov/dcms/bbc/tv_licence/would_evade_tv_licence_fee.py b/policyengine_uk/variables/gov/dcms/bbc/tv_licence/would_evade_tv_licence_fee.py index fdf426e12..5d3655356 100644 --- a/policyengine_uk/variables/gov/dcms/bbc/tv_licence/would_evade_tv_licence_fee.py +++ b/policyengine_uk/variables/gov/dcms/bbc/tv_licence/would_evade_tv_licence_fee.py @@ -12,4 +12,4 @@ class would_evade_tv_licence_fee(Variable): def formula(household, period, parameters): evasion_rate = parameters(period).gov.dcms.bbc.tv_licence.evasion_rate - return random(household) <= evasion_rate + return household("tv_licence_evasion_seed", period) <= evasion_rate diff --git a/policyengine_uk/variables/gov/dfe/extended_childcare_entitlement/extended_childcare_take_up_seed.py b/policyengine_uk/variables/gov/dfe/extended_childcare_entitlement/extended_childcare_take_up_seed.py new file mode 100644 index 000000000..53263d6a9 --- /dev/null +++ b/policyengine_uk/variables/gov/dfe/extended_childcare_entitlement/extended_childcare_take_up_seed.py @@ -0,0 +1,8 @@ +from policyengine_uk.model_api import * + + +class extended_childcare_take_up_seed(Variable): + value_type = float + entity = BenUnit + label = "Random seed for extended childcare take-up" + definition_period = YEAR diff --git a/policyengine_uk/variables/gov/dfe/extended_childcare_entitlement/would_claim_extended_childcare.py b/policyengine_uk/variables/gov/dfe/extended_childcare_entitlement/would_claim_extended_childcare.py index 070c50542..029fcdaf5 100644 --- a/policyengine_uk/variables/gov/dfe/extended_childcare_entitlement/would_claim_extended_childcare.py +++ b/policyengine_uk/variables/gov/dfe/extended_childcare_entitlement/would_claim_extended_childcare.py @@ -7,4 +7,10 @@ class would_claim_extended_childcare(Variable): label = "would claim extended childcare entitlement" documentation = "Whether this family would claim extended childcare entitlement if eligible" definition_period = YEAR - default_value = True + + def formula(benunit, period, parameters): + takeup_rate = parameters(period).gov.dfe.extended_childcare_entitlement.takeup_rate + is_in_microsimulation = benunit.simulation.dataset is not None + if is_in_microsimulation: + return benunit("extended_childcare_take_up_seed", period) < takeup_rate + return True diff --git a/policyengine_uk/variables/gov/dfe/targeted_childcare_entitlement/targeted_childcare_take_up_seed.py b/policyengine_uk/variables/gov/dfe/targeted_childcare_entitlement/targeted_childcare_take_up_seed.py new file mode 100644 index 000000000..dabddb755 --- /dev/null +++ b/policyengine_uk/variables/gov/dfe/targeted_childcare_entitlement/targeted_childcare_take_up_seed.py @@ -0,0 +1,8 @@ +from policyengine_uk.model_api import * + + +class targeted_childcare_take_up_seed(Variable): + value_type = float + entity = BenUnit + label = "Random seed for targeted childcare take-up" + definition_period = YEAR diff --git a/policyengine_uk/variables/gov/dfe/targeted_childcare_entitlement/would_claim_targeted_childcare.py b/policyengine_uk/variables/gov/dfe/targeted_childcare_entitlement/would_claim_targeted_childcare.py index bb755228f..2c02120d6 100644 --- a/policyengine_uk/variables/gov/dfe/targeted_childcare_entitlement/would_claim_targeted_childcare.py +++ b/policyengine_uk/variables/gov/dfe/targeted_childcare_entitlement/would_claim_targeted_childcare.py @@ -7,4 +7,10 @@ class would_claim_targeted_childcare(Variable): label = "would claim targeted childcare entitlement" documentation = "Whether this family would claim targeted childcare entitlement if eligible" definition_period = YEAR - default_value = True + + def formula(benunit, period, parameters): + takeup_rate = parameters(period).gov.dfe.targeted_childcare_entitlement.takeup_rate + is_in_microsimulation = benunit.simulation.dataset is not None + if is_in_microsimulation: + return benunit("targeted_childcare_take_up_seed", period) < takeup_rate + return True diff --git a/policyengine_uk/variables/gov/dfe/universal_childcare_entitlement/universal_childcare_take_up_seed.py b/policyengine_uk/variables/gov/dfe/universal_childcare_entitlement/universal_childcare_take_up_seed.py new file mode 100644 index 000000000..c34624d34 --- /dev/null +++ b/policyengine_uk/variables/gov/dfe/universal_childcare_entitlement/universal_childcare_take_up_seed.py @@ -0,0 +1,8 @@ +from policyengine_uk.model_api import * + + +class universal_childcare_take_up_seed(Variable): + value_type = float + entity = BenUnit + label = "Random seed for universal childcare take-up" + definition_period = YEAR diff --git a/policyengine_uk/variables/gov/dfe/universal_childcare_entitlement/would_claim_universal_childcare.py b/policyengine_uk/variables/gov/dfe/universal_childcare_entitlement/would_claim_universal_childcare.py index b6066b4bc..fa6677f21 100644 --- a/policyengine_uk/variables/gov/dfe/universal_childcare_entitlement/would_claim_universal_childcare.py +++ b/policyengine_uk/variables/gov/dfe/universal_childcare_entitlement/would_claim_universal_childcare.py @@ -7,4 +7,10 @@ class would_claim_universal_childcare(Variable): label = "would claim universal childcare entitlement" documentation = "Whether this BenUnit would claim universal childcare entitlement if eligible" definition_period = YEAR - default_value = True + + def formula(benunit, period, parameters): + takeup_rate = parameters(period).gov.dfe.universal_childcare_entitlement.takeup_rate + is_in_microsimulation = benunit.simulation.dataset is not None + if is_in_microsimulation: + return benunit("universal_childcare_take_up_seed", period) < takeup_rate + return True diff --git a/policyengine_uk/variables/gov/dwp/pension_credit/pension_credit_take_up_seed.py b/policyengine_uk/variables/gov/dwp/pension_credit/pension_credit_take_up_seed.py new file mode 100644 index 000000000..775f79b92 --- /dev/null +++ b/policyengine_uk/variables/gov/dwp/pension_credit/pension_credit_take_up_seed.py @@ -0,0 +1,8 @@ +from policyengine_uk.model_api import * + + +class pension_credit_take_up_seed(Variable): + value_type = float + entity = BenUnit + label = "Random seed for pension credit take-up" + definition_period = YEAR diff --git a/policyengine_uk/variables/gov/dwp/pension_credit/would_claim.py b/policyengine_uk/variables/gov/dwp/pension_credit/would_claim.py index 48f9b020d..83a5e4cbd 100644 --- a/policyengine_uk/variables/gov/dwp/pension_credit/would_claim.py +++ b/policyengine_uk/variables/gov/dwp/pension_credit/would_claim.py @@ -23,7 +23,7 @@ def formula(benunit, period, parameters): ], [ True, - random(benunit) < takeup_rate, + benunit("pension_credit_take_up_seed", period) < takeup_rate, False, ], ) diff --git a/policyengine_uk/variables/gov/dwp/universal_credit/universal_credit_take_up_seed.py b/policyengine_uk/variables/gov/dwp/universal_credit/universal_credit_take_up_seed.py new file mode 100644 index 000000000..e12431573 --- /dev/null +++ b/policyengine_uk/variables/gov/dwp/universal_credit/universal_credit_take_up_seed.py @@ -0,0 +1,8 @@ +from policyengine_uk.model_api import * + + +class universal_credit_take_up_seed(Variable): + value_type = float + entity = BenUnit + label = "Random seed for universal credit take-up" + definition_period = YEAR diff --git a/policyengine_uk/variables/gov/dwp/universal_credit/would_claim_uc.py b/policyengine_uk/variables/gov/dwp/universal_credit/would_claim_uc.py index 137d23b39..3fa20abd8 100644 --- a/policyengine_uk/variables/gov/dwp/universal_credit/would_claim_uc.py +++ b/policyengine_uk/variables/gov/dwp/universal_credit/would_claim_uc.py @@ -12,7 +12,7 @@ class would_claim_uc(Variable): def formula(benunit, period, parameters): takes_up = ( - random(benunit) + benunit("universal_credit_take_up_seed", period) < parameters(period).gov.dwp.universal_credit.takeup_rate ) is_in_microsimulation = benunit.simulation.dataset is not None diff --git a/policyengine_uk/variables/gov/hmrc/child_benefit_opts_out.py b/policyengine_uk/variables/gov/hmrc/child_benefit_opts_out.py index a7526c47b..01ade9156 100644 --- a/policyengine_uk/variables/gov/hmrc/child_benefit_opts_out.py +++ b/policyengine_uk/variables/gov/hmrc/child_benefit_opts_out.py @@ -19,7 +19,7 @@ def formula(benunit, period, parameters): in_phase_out = ani > cb_hitc.phase_out_end return where( benunit.any(in_phase_out), - random(benunit) < cb.opt_out_rate, + benunit("child_benefit_opts_out_seed", period) < cb.opt_out_rate, False, ) else: diff --git a/policyengine_uk/variables/gov/hmrc/child_benefit_opts_out_seed.py b/policyengine_uk/variables/gov/hmrc/child_benefit_opts_out_seed.py new file mode 100644 index 000000000..cd4fda91d --- /dev/null +++ b/policyengine_uk/variables/gov/hmrc/child_benefit_opts_out_seed.py @@ -0,0 +1,8 @@ +from policyengine_uk.model_api import * + + +class child_benefit_opts_out_seed(Variable): + value_type = float + entity = BenUnit + label = "Random seed for child benefit opt-out decision" + definition_period = YEAR diff --git a/policyengine_uk/variables/gov/hmrc/child_benefit_take_up_seed.py b/policyengine_uk/variables/gov/hmrc/child_benefit_take_up_seed.py new file mode 100644 index 000000000..8018521bf --- /dev/null +++ b/policyengine_uk/variables/gov/hmrc/child_benefit_take_up_seed.py @@ -0,0 +1,8 @@ +from policyengine_uk.model_api import * + + +class child_benefit_take_up_seed(Variable): + value_type = float + entity = BenUnit + label = "Random seed for child benefit take-up" + definition_period = YEAR diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/allowances/marriage_allowance.py b/policyengine_uk/variables/gov/hmrc/income_tax/allowances/marriage_allowance.py index 642bc7fa6..3e83ebc6a 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/allowances/marriage_allowance.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/allowances/marriage_allowance.py @@ -29,4 +29,4 @@ def formula(person, period, parameters): np.ceil(amount_if_eligible_pre_rounding / rounding_increment) * rounding_increment ) - return eligible * amount_if_eligible * (random(person) < takeup_rate) + return eligible * amount_if_eligible * (person("marriage_allowance_take_up_seed", period) < takeup_rate) diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/allowances/marriage_allowance_take_up_seed.py b/policyengine_uk/variables/gov/hmrc/income_tax/allowances/marriage_allowance_take_up_seed.py new file mode 100644 index 000000000..b42a94596 --- /dev/null +++ b/policyengine_uk/variables/gov/hmrc/income_tax/allowances/marriage_allowance_take_up_seed.py @@ -0,0 +1,8 @@ +from policyengine_uk.model_api import * + + +class marriage_allowance_take_up_seed(Variable): + value_type = float + entity = Person + label = "Random seed for marriage allowance take-up" + definition_period = YEAR diff --git a/policyengine_uk/variables/gov/hmrc/tax_free_childcare/tax_free_childcare_take_up_seed.py b/policyengine_uk/variables/gov/hmrc/tax_free_childcare/tax_free_childcare_take_up_seed.py new file mode 100644 index 000000000..5767a3f2b --- /dev/null +++ b/policyengine_uk/variables/gov/hmrc/tax_free_childcare/tax_free_childcare_take_up_seed.py @@ -0,0 +1,8 @@ +from policyengine_uk.model_api import * + + +class tax_free_childcare_take_up_seed(Variable): + value_type = float + entity = BenUnit + label = "Random seed for Tax-Free Childcare take-up" + definition_period = YEAR diff --git a/policyengine_uk/variables/gov/hmrc/tax_free_childcare/would_claim_tfc.py b/policyengine_uk/variables/gov/hmrc/tax_free_childcare/would_claim_tfc.py index 2a623372c..1cf4fdb7d 100644 --- a/policyengine_uk/variables/gov/hmrc/tax_free_childcare/would_claim_tfc.py +++ b/policyengine_uk/variables/gov/hmrc/tax_free_childcare/would_claim_tfc.py @@ -9,4 +9,10 @@ class would_claim_tfc(Variable): "Whether this family would claim Tax-Free Childcare if eligible" ) definition_period = YEAR - default_value = True + + def formula(benunit, period, parameters): + takeup_rate = parameters(period).gov.hmrc.tax_free_childcare.takeup_rate + is_in_microsimulation = benunit.simulation.dataset is not None + if is_in_microsimulation: + return benunit("tax_free_childcare_take_up_seed", period) < takeup_rate + return True diff --git a/policyengine_uk/variables/gov/hmrc/would_claim_child_benefit.py b/policyengine_uk/variables/gov/hmrc/would_claim_child_benefit.py index 7d609dd41..1c4a93db8 100644 --- a/policyengine_uk/variables/gov/hmrc/would_claim_child_benefit.py +++ b/policyengine_uk/variables/gov/hmrc/would_claim_child_benefit.py @@ -13,6 +13,6 @@ class would_claim_child_benefit(Variable): def formula(benunit, period, parameters): takeup_rate = parameters(period).gov.hmrc.child_benefit.takeup overall_p = takeup_rate.overall - return (random(benunit) < overall_p) * ~benunit( + return (benunit("child_benefit_take_up_seed", period) < overall_p) * ~benunit( "child_benefit_opts_out", period ) diff --git a/policyengine_uk/variables/household/consumption/first_home_purchase_seed.py b/policyengine_uk/variables/household/consumption/first_home_purchase_seed.py new file mode 100644 index 000000000..993a781fd --- /dev/null +++ b/policyengine_uk/variables/household/consumption/first_home_purchase_seed.py @@ -0,0 +1,8 @@ +from policyengine_uk.model_api import * + + +class first_home_purchase_seed(Variable): + value_type = float + entity = Household + label = "Random seed for first home purchase determination" + definition_period = YEAR diff --git a/policyengine_uk/variables/household/consumption/main_residential_property_purchased_is_first_home.py b/policyengine_uk/variables/household/consumption/main_residential_property_purchased_is_first_home.py index b49c5b4fe..102d2a144 100644 --- a/policyengine_uk/variables/household/consumption/main_residential_property_purchased_is_first_home.py +++ b/policyengine_uk/variables/household/consumption/main_residential_property_purchased_is_first_home.py @@ -21,4 +21,4 @@ def formula(household, period, parameters): residential_sd.first_time_buyers_relief.calc(age) / residential_sd.transactions_by_age.calc(age) ) - return random(household) < percentage_claiming_ftbr + return household("first_home_purchase_seed", period) < percentage_claiming_ftbr diff --git a/policyengine_uk/variables/household/demographic/household_owns_tv.py b/policyengine_uk/variables/household/demographic/household_owns_tv.py index 6d22f2344..a856ff133 100644 --- a/policyengine_uk/variables/household/demographic/household_owns_tv.py +++ b/policyengine_uk/variables/household/demographic/household_owns_tv.py @@ -12,4 +12,4 @@ def formula(household, period, parameters): percent_owning_tv = parameters( period ).gov.dcms.bbc.tv_licence.tv_ownership - return random(household) <= percent_owning_tv + return household("household_owns_tv_seed", period) <= percent_owning_tv diff --git a/policyengine_uk/variables/household/demographic/household_owns_tv_seed.py b/policyengine_uk/variables/household/demographic/household_owns_tv_seed.py new file mode 100644 index 000000000..c45dfc3db --- /dev/null +++ b/policyengine_uk/variables/household/demographic/household_owns_tv_seed.py @@ -0,0 +1,8 @@ +from policyengine_uk.model_api import * + + +class household_owns_tv_seed(Variable): + value_type = float + entity = Household + label = "Random seed for TV ownership" + definition_period = YEAR diff --git a/policyengine_uk/variables/household/demographic/is_disabled_for_benefits.py b/policyengine_uk/variables/household/demographic/is_disabled_for_benefits.py index 0fdbb51a1..d52b80edb 100644 --- a/policyengine_uk/variables/household/demographic/is_disabled_for_benefits.py +++ b/policyengine_uk/variables/household/demographic/is_disabled_for_benefits.py @@ -18,7 +18,7 @@ def formula(person, period, parameters): p_claims_lcwra_if_on_pip_dla = 0.8 p_claims_lcwra_if_not_on_pip_dla = 0.13 - random_seed = random(person) + random_seed = person("is_disabled_for_benefits_seed", period) on_qual_benefits = add(person, period, QUALIFYING_BENEFITS) > 0 diff --git a/policyengine_uk/variables/household/demographic/is_disabled_for_benefits_seed.py b/policyengine_uk/variables/household/demographic/is_disabled_for_benefits_seed.py new file mode 100644 index 000000000..f340fb85b --- /dev/null +++ b/policyengine_uk/variables/household/demographic/is_disabled_for_benefits_seed.py @@ -0,0 +1,8 @@ +from policyengine_uk.model_api import * + + +class is_disabled_for_benefits_seed(Variable): + value_type = float + entity = Person + label = "Random seed for disability benefits eligibility determination" + definition_period = YEAR diff --git a/policyengine_uk/variables/household/demographic/is_higher_earner.py b/policyengine_uk/variables/household/demographic/is_higher_earner.py index 353b6fad8..42679b4aa 100644 --- a/policyengine_uk/variables/household/demographic/is_higher_earner.py +++ b/policyengine_uk/variables/household/demographic/is_higher_earner.py @@ -11,6 +11,6 @@ def formula(person, period, parameters): income = person("adjusted_net_income", period) # Add noise to incomes in order to avoid ties return ( - person.get_rank(person.benunit, -income + random(person) * 1e-2) + person.get_rank(person.benunit, -income + person("is_higher_earner_seed", period) * 1e-2) == 0 ) diff --git a/policyengine_uk/variables/household/demographic/is_higher_earner_seed.py b/policyengine_uk/variables/household/demographic/is_higher_earner_seed.py new file mode 100644 index 000000000..f1f0a6aa1 --- /dev/null +++ b/policyengine_uk/variables/household/demographic/is_higher_earner_seed.py @@ -0,0 +1,8 @@ +from policyengine_uk.model_api import * + + +class is_higher_earner_seed(Variable): + value_type = float + entity = Person + label = "Random seed for tie-breaking in higher earner determination" + definition_period = YEAR diff --git a/uv.lock b/uv.lock index d07b4799c..f9019e7d5 100644 --- a/uv.lock +++ b/uv.lock @@ -1249,7 +1249,7 @@ wheels = [ [[package]] name = "policyengine-uk" -version = "2.47.4" +version = "2.54.0" source = { editable = "." } dependencies = [ { name = "microdf-python" }, From 0065d57e0e0b51162cddb8d67278339d1aea4a87 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 5 Oct 2025 16:53:16 +0200 Subject: [PATCH 2/2] Make country package purely deterministic - read stochastic variables from dataset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change removes all random number generation from policyengine-uk. All stochastic variables are now generated in policyengine-uk-data and read from the dataset. The country package is now a purely deterministic rules engine. ## Key Changes - Remove all take-up rate parameters (moved to policyengine-uk-data) - Remove all random seed/draw variables for take-up decisions - Simplify would_claim variables to dataset-only (no formula) - Keep formulas with fallback to random() for policy calculator (non-microsim) - Add would_claim_marriage_allowance variable - Add random draw variables for tie-breaking and conditional probabilities ### Variables now sourced from dataset: **Take-up decisions (boolean):** - would_claim_child_benefit - child_benefit_opts_out - would_claim_pc - would_claim_uc - would_claim_marriage_allowance - would_claim_tfc - would_claim_extended_childcare - would_claim_universal_childcare - would_claim_targeted_childcare **Other stochastic variables:** - household_owns_tv - would_evade_tv_licence_fee - main_residential_property_purchased_is_first_home **Random draws (for formulas):** - is_higher_earner_random_draw (tie-breaking) - attends_private_school_random_draw (income-conditional) ### Formulas preserved for policy calculator: - attends_private_school (complex income percentile logic) - is_disabled_for_benefits (conditional on qualifying benefits) - is_higher_earner (uses random draw for tie-breaking) These fall back to random() when not in a microsimulation context. ## Trade-offs **IMPORTANT**: Take-up rates can no longer be adjusted via policy reforms. They are fixed in the microdata. This is an acceptable trade-off for the cleaner architecture. To adjust take-up rates, regenerate the microdata. Related: policyengine-uk-data PR #[TBD] (must be merged FIRST) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- changelog_entry.yaml | 9 ++++ .../programs/gov/dcms/bbc/tv-licence.ipynb | 10 ++-- .../programs/gov/dwp/pension-credit.ipynb | 16 +++++-- .../programs/gov/dwp/universal-credit.ipynb | 4 +- .../programs/gov/hmrc/child-benefit.ipynb | 14 ++++-- docs/book/programs/gov/hmrc/fuel-duty.ipynb | 12 +++-- docs/book/programs/gov/hmrc/income-tax.ipynb | 48 ++++++++++++++----- docs/book/programs/gov/hmrc/stamp-duty.ipynb | 12 +++-- .../gov/ofgem/energy-price-guarantee.ipynb | 14 ++++-- .../land-and-buildings-transaction-tax.ipynb | 16 +++++-- .../gov/wra/land-transaction-tax.ipynb | 4 +- docs/book/validation/validation.ipynb | 4 +- .../takeup_rate.yaml | 9 ---- .../takeup_rate.yaml | 9 ---- .../takeup_rate.yaml | 9 ---- .../hmrc/tax_free_childcare/takeup_rate.yaml | 9 ---- .../tests/microsimulation/reforms_config.yaml | 14 +++--- .../contrib/labour/attends_private_school.py | 6 +-- .../attends_private_school_random_draw.py | 12 +++++ .../labour/attends_private_school_seed.py | 8 ---- .../bbc/tv_licence/tv_licence_evasion_seed.py | 8 ---- .../tv_licence/would_evade_tv_licence_fee.py | 9 ++-- .../extended_childcare_take_up_seed.py | 8 ---- .../would_claim_extended_childcare.py | 14 +++--- .../targeted_childcare_take_up_seed.py | 8 ---- .../would_claim_targeted_childcare.py | 14 +++--- .../universal_childcare_take_up_seed.py | 8 ---- .../would_claim_universal_childcare.py | 14 +++--- .../pension_credit_take_up_seed.py | 8 ---- .../gov/dwp/pension_credit/would_claim.py | 27 +++-------- .../universal_credit_take_up_seed.py | 8 ---- .../dwp/universal_credit/would_claim_uc.py | 15 ++---- .../gov/hmrc/child_benefit_opts_out.py | 21 ++------ .../gov/hmrc/child_benefit_opts_out_seed.py | 8 ---- .../gov/hmrc/child_benefit_take_up_seed.py | 8 ---- .../allowances/marriage_allowance.py | 3 +- .../marriage_allowance_take_up_seed.py | 8 ---- .../would_claim_marriage_allowance.py | 13 +++++ .../tax_free_childcare_take_up_seed.py | 8 ---- .../tax_free_childcare/would_claim_tfc.py | 12 ++--- .../gov/hmrc/would_claim_child_benefit.py | 12 ++--- .../consumption/first_home_purchase_seed.py | 8 ---- ...ential_property_purchased_is_first_home.py | 22 +++------ .../demographic/higher_earner_tie_break.py | 12 +++++ .../demographic/household_owns_tv.py | 13 ++--- .../demographic/household_owns_tv_seed.py | 8 ---- .../demographic/is_disabled_for_benefits.py | 16 ++----- .../is_disabled_for_benefits_seed.py | 8 ---- .../household/demographic/is_higher_earner.py | 8 ++-- .../demographic/is_higher_earner_seed.py | 8 ---- 50 files changed, 247 insertions(+), 329 deletions(-) delete mode 100644 policyengine_uk/parameters/gov/dfe/extended_childcare_entitlement/takeup_rate.yaml delete mode 100644 policyengine_uk/parameters/gov/dfe/targeted_childcare_entitlement/takeup_rate.yaml delete mode 100644 policyengine_uk/parameters/gov/dfe/universal_childcare_entitlement/takeup_rate.yaml delete mode 100644 policyengine_uk/parameters/gov/hmrc/tax_free_childcare/takeup_rate.yaml create mode 100644 policyengine_uk/variables/contrib/labour/attends_private_school_random_draw.py delete mode 100644 policyengine_uk/variables/contrib/labour/attends_private_school_seed.py delete mode 100644 policyengine_uk/variables/gov/dcms/bbc/tv_licence/tv_licence_evasion_seed.py delete mode 100644 policyengine_uk/variables/gov/dfe/extended_childcare_entitlement/extended_childcare_take_up_seed.py delete mode 100644 policyengine_uk/variables/gov/dfe/targeted_childcare_entitlement/targeted_childcare_take_up_seed.py delete mode 100644 policyengine_uk/variables/gov/dfe/universal_childcare_entitlement/universal_childcare_take_up_seed.py delete mode 100644 policyengine_uk/variables/gov/dwp/pension_credit/pension_credit_take_up_seed.py delete mode 100644 policyengine_uk/variables/gov/dwp/universal_credit/universal_credit_take_up_seed.py delete mode 100644 policyengine_uk/variables/gov/hmrc/child_benefit_opts_out_seed.py delete mode 100644 policyengine_uk/variables/gov/hmrc/child_benefit_take_up_seed.py delete mode 100644 policyengine_uk/variables/gov/hmrc/income_tax/allowances/marriage_allowance_take_up_seed.py create mode 100644 policyengine_uk/variables/gov/hmrc/income_tax/allowances/would_claim_marriage_allowance.py delete mode 100644 policyengine_uk/variables/gov/hmrc/tax_free_childcare/tax_free_childcare_take_up_seed.py delete mode 100644 policyengine_uk/variables/household/consumption/first_home_purchase_seed.py create mode 100644 policyengine_uk/variables/household/demographic/higher_earner_tie_break.py delete mode 100644 policyengine_uk/variables/household/demographic/household_owns_tv_seed.py delete mode 100644 policyengine_uk/variables/household/demographic/is_disabled_for_benefits_seed.py delete mode 100644 policyengine_uk/variables/household/demographic/is_higher_earner_seed.py diff --git a/changelog_entry.yaml b/changelog_entry.yaml index e69de29bb..f2292be66 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -0,0 +1,9 @@ +- bump: minor + changes: + removed: + - All take-up rate parameters (moved to policyengine-uk-data) + - All random seed variables for stochastic processes + changed: + - Would_claim variables now sourced from dataset + - Country package is now purely deterministic + - Formulas preserved for policy calculator with dataset fallback diff --git a/docs/book/programs/gov/dcms/bbc/tv-licence.ipynb b/docs/book/programs/gov/dcms/bbc/tv-licence.ipynb index 9e7123015..a0e13e65b 100644 --- a/docs/book/programs/gov/dcms/bbc/tv-licence.ipynb +++ b/docs/book/programs/gov/dcms/bbc/tv-licence.ipynb @@ -128,10 +128,12 @@ "\n", "df = pd.DataFrame()\n", "df[\"Date\"] = [\n", - " parameter.instant_str for parameter in dcms.bbc.tv_licence.colour.values_list\n", + " parameter.instant_str\n", + " for parameter in dcms.bbc.tv_licence.colour.values_list\n", "]\n", "df[\"Full TV Licence Fee\"] = [\n", - " f\"£{parameter.value:.2f}\" for parameter in dcms.bbc.tv_licence.colour.values_list\n", + " f\"£{parameter.value:.2f}\"\n", + " for parameter in dcms.bbc.tv_licence.colour.values_list\n", "]\n", "df[\"Blind TV Licence Fee\"] = [\n", " f\"£{0.5 * parameter.value:.2f}\"\n", @@ -332,7 +334,9 @@ " aged_discount[\"Change against current\"] += [\n", " f\"{aged_discount['Reformed value'][i] - 1:.0%}\"\n", " ]\n", - " aged_discount[\"Reformed value\"][i] = f\"{aged_discount['Reformed value'][i]:.0%}\"\n", + " aged_discount[\"Reformed value\"][\n", + " i\n", + " ] = f\"{aged_discount['Reformed value'][i]:.0%}\"\n", " aged_discount[\"Reference\"] += [\n", " f'Budgetary impact of changing aged discount to {aged_discount[\"Reformed value\"][i]}'\n", " ]\n", diff --git a/docs/book/programs/gov/dwp/pension-credit.ipynb b/docs/book/programs/gov/dwp/pension-credit.ipynb index 1601aa923..d88ca5aa4 100644 --- a/docs/book/programs/gov/dwp/pension-credit.ipynb +++ b/docs/book/programs/gov/dwp/pension-credit.ipynb @@ -167,8 +167,12 @@ "\n", "parameters = system.parameters\n", "\n", - "carer_addition = parameters.gov.dwp.pension_credit.guarantee_credit.carer.addition\n", - "child_addition = parameters.gov.dwp.pension_credit.guarantee_credit.child.addition\n", + "carer_addition = (\n", + " parameters.gov.dwp.pension_credit.guarantee_credit.carer.addition\n", + ")\n", + "child_addition = (\n", + " parameters.gov.dwp.pension_credit.guarantee_credit.child.addition\n", + ")\n", "disabled_child = (\n", " parameters.gov.dwp.pension_credit.guarantee_credit.child.disability.addition\n", ")\n", @@ -1502,8 +1506,12 @@ "\n", "parameters = system.parameters\n", "\n", - "threshold_single = parameters.gov.dwp.pension_credit.savings_credit.threshold.SINGLE\n", - "threshold_couple = parameters.gov.dwp.pension_credit.savings_credit.threshold.COUPLE\n", + "threshold_single = (\n", + " parameters.gov.dwp.pension_credit.savings_credit.threshold.SINGLE\n", + ")\n", + "threshold_couple = (\n", + " parameters.gov.dwp.pension_credit.savings_credit.threshold.COUPLE\n", + ")\n", "\n", "elements = [threshold_single, threshold_couple] # [...]\n", "\n", diff --git a/docs/book/programs/gov/dwp/universal-credit.ipynb b/docs/book/programs/gov/dwp/universal-credit.ipynb index 08e1146d7..843b7d45b 100644 --- a/docs/book/programs/gov/dwp/universal-credit.ipynb +++ b/docs/book/programs/gov/dwp/universal-credit.ipynb @@ -194,7 +194,9 @@ "disabled_child_amount = (\n", " parameters.gov.dwp.universal_credit.elements.child.disabled.amount\n", ")\n", - "higher_amount = parameters.gov.dwp.universal_credit.elements.child.first.higher_amount\n", + "higher_amount = (\n", + " parameters.gov.dwp.universal_credit.elements.child.first.higher_amount\n", + ")\n", "\n", "elements = [\n", " carer_amount,\n", diff --git a/docs/book/programs/gov/hmrc/child-benefit.ipynb b/docs/book/programs/gov/hmrc/child-benefit.ipynb index 02b915df5..088b1fdc7 100644 --- a/docs/book/programs/gov/hmrc/child-benefit.ipynb +++ b/docs/book/programs/gov/hmrc/child-benefit.ipynb @@ -175,7 +175,9 @@ " data[\"Reference\"] += [\"\"]\n", "\n", "\n", - "for parameter in parameters.gov.hmrc.child_benefit.amount.additional.values_list:\n", + "for (\n", + " parameter\n", + ") in parameters.gov.hmrc.child_benefit.amount.additional.values_list:\n", " data[\"Date\"] += [parameter.instant_str]\n", " data[\"Name\"] += [\"Additional\"]\n", " data[\"Value\"] += [f\"£{parameter.value:.2f}\"]\n", @@ -187,9 +189,9 @@ " data[\"Reference\"] += [\"\"]\n", "\n", "\n", - "pd.DataFrame(data).sort_values(\"Date\").set_index([\"Date\", \"Name\"]).style.format(\n", - " lambda x: x\n", - ")" + "pd.DataFrame(data).sort_values(\"Date\").set_index(\n", + " [\"Date\", \"Name\"]\n", + ").style.format(lambda x: x)" ] }, { @@ -300,7 +302,9 @@ "pd.DataFrame(\n", " {\n", " \"Number of children\": list(range(1, 7)),\n", - " \"Child Benefit (Annual)\": list(map(get_cb_for_n_children, range(1, 7))),\n", + " \"Child Benefit (Annual)\": list(\n", + " map(get_cb_for_n_children, range(1, 7))\n", + " ),\n", " }\n", ").set_index(\"Number of children\")" ] diff --git a/docs/book/programs/gov/hmrc/fuel-duty.ipynb b/docs/book/programs/gov/hmrc/fuel-duty.ipynb index 51d676866..a4ba22fff 100644 --- a/docs/book/programs/gov/hmrc/fuel-duty.ipynb +++ b/docs/book/programs/gov/hmrc/fuel-duty.ipynb @@ -126,10 +126,12 @@ "df = pd.DataFrame()\n", "\n", "df[\"Date of change\"] = [\n", - " parameter.instant_str for parameter in hmrc.fuel_duty.petrol_and_diesel.values_list\n", + " parameter.instant_str\n", + " for parameter in hmrc.fuel_duty.petrol_and_diesel.values_list\n", "]\n", "df[\"Fuel duty rate (£/litre)\"] = [\n", - " parameter.value for parameter in hmrc.fuel_duty.petrol_and_diesel.values_list\n", + " parameter.value\n", + " for parameter in hmrc.fuel_duty.petrol_and_diesel.values_list\n", "]\n", "df.sort_values(\"Date of change\", inplace=True)\n", "df.set_index(\"Date of change\")" @@ -1070,7 +1072,11 @@ " height=600,\n", " width=800,\n", " template=\"plotly_white\",\n", - ").update_xaxes(tickangle=45, tickfont={\"size\": 10}).update_traces(line_shape=\"hv\")" + ").update_xaxes(\n", + " tickangle=45, tickfont={\"size\": 10}\n", + ").update_traces(\n", + " line_shape=\"hv\"\n", + ")" ] } ], diff --git a/docs/book/programs/gov/hmrc/income-tax.ipynb b/docs/book/programs/gov/hmrc/income-tax.ipynb index 77f1fc99a..ff2491bfe 100644 --- a/docs/book/programs/gov/hmrc/income-tax.ipynb +++ b/docs/book/programs/gov/hmrc/income-tax.ipynb @@ -207,7 +207,9 @@ "from policyengine_uk import Simulation\n", "\n", "\n", - "def calculate_taxes(employment_income, dividend_income, savings_interest_income):\n", + "def calculate_taxes(\n", + " employment_income, dividend_income, savings_interest_income\n", + "):\n", " simulation = Simulation(\n", " situation={\n", " \"people\": {\n", @@ -220,7 +222,9 @@ " }\n", " )\n", " income_tax = simulation.calculate(\"income_tax\")[0]\n", - " dividend_income_tax = simulation.calculate(\"dividend_income_tax\")[0] # Added\n", + " dividend_income_tax = simulation.calculate(\"dividend_income_tax\")[\n", + " 0\n", + " ] # Added\n", " savings_income_tax = simulation.calculate(\"savings_income_tax\")[0] # Added\n", "\n", " return income_tax, dividend_income_tax, savings_income_tax\n", @@ -241,7 +245,9 @@ " data[\"Dividend Income (£)\"],\n", " data[\"Savings Interest Income (£)\"],\n", "):\n", - " income_tax, div_tax, sav_tax = calculate_taxes(emp_income, div_income, sav_income)\n", + " income_tax, div_tax, sav_tax = calculate_taxes(\n", + " emp_income, div_income, sav_income\n", + " )\n", " data[\"Income Tax (£)\"].append(income_tax)\n", " data[\"Dividend Income Tax (£)\"].append(div_tax)\n", " data[\"Savings Income Tax (£)\"].append(sav_tax)\n", @@ -518,8 +524,12 @@ "from policyengine_uk.system import system\n", "import pandas as pd\n", "\n", - "default = system.parameters.gov.hmrc.income_tax.allowances.annual_allowance.default\n", - "minimum = system.parameters.gov.hmrc.income_tax.allowances.annual_allowance.minimum\n", + "default = (\n", + " system.parameters.gov.hmrc.income_tax.allowances.annual_allowance.default\n", + ")\n", + "minimum = (\n", + " system.parameters.gov.hmrc.income_tax.allowances.annual_allowance.minimum\n", + ")\n", "reduction_rate = (\n", " system.parameters.gov.hmrc.income_tax.allowances.annual_allowance.reduction_rate\n", ")\n", @@ -549,7 +559,9 @@ "\n", "\n", "max = system.parameters.gov.hmrc.income_tax.allowances.marriage_allowance.max\n", - "rounding_increment = system.parameters.gov.hmrc.income_tax.allowances.marriage_allowance.rounding_increment\n", + "rounding_increment = (\n", + " system.parameters.gov.hmrc.income_tax.allowances.marriage_allowance.rounding_increment\n", + ")\n", "takeup_rate = (\n", " system.parameters.gov.hmrc.income_tax.allowances.marriage_allowance.takeup_rate\n", ")\n", @@ -574,7 +586,9 @@ "}\n", "\n", "\n", - "deduction_rate = system.parameters.gov.hmrc.income_tax.allowances.married_couples_allowance.deduction_rate\n", + "deduction_rate = (\n", + " system.parameters.gov.hmrc.income_tax.allowances.married_couples_allowance.deduction_rate\n", + ")\n", "married_couples_allowance_data = {\n", " \"Attribute\": [\"Married Couples Allowance\"],\n", " \"Type\": [\"Deduction Rate\"],\n", @@ -584,7 +598,9 @@ "}\n", "\n", "\n", - "amount = system.parameters.gov.hmrc.income_tax.allowances.personal_allowance.amount\n", + "amount = (\n", + " system.parameters.gov.hmrc.income_tax.allowances.personal_allowance.amount\n", + ")\n", "values = [item.value for item in amount.values_list]\n", "dates = [item.instant_str for item in amount.values_list]\n", "personal_allowance_data = {\n", @@ -596,7 +612,9 @@ "}\n", "\n", "\n", - "addtional_threshold = system.parameters.gov.hmrc.income_tax.allowances.personal_savings_allowance.additional\n", + "addtional_threshold = (\n", + " system.parameters.gov.hmrc.income_tax.allowances.personal_savings_allowance.additional\n", + ")\n", "basic_threshold = (\n", " system.parameters.gov.hmrc.income_tax.allowances.personal_savings_allowance.basic\n", ")\n", @@ -624,7 +642,9 @@ "}\n", "\n", "\n", - "dividend_allowance = system.parameters.gov.hmrc.income_tax.allowances.dividend_allowance\n", + "dividend_allowance = (\n", + " system.parameters.gov.hmrc.income_tax.allowances.dividend_allowance\n", + ")\n", "dividend_allowance_data = {\n", " \"Attribute\": [\"Dividend Allowance\"],\n", " \"Type\": [\"Dividend Allowance\"],\n", @@ -634,7 +654,9 @@ "}\n", "\n", "\n", - "property_allowance = system.parameters.gov.hmrc.income_tax.allowances.property_allowance\n", + "property_allowance = (\n", + " system.parameters.gov.hmrc.income_tax.allowances.property_allowance\n", + ")\n", "property_allowance_data = {\n", " \"Attribute\": [\"Property Allowance\"],\n", " \"Type\": [\"Property Allowance\"],\n", @@ -644,7 +666,9 @@ "}\n", "\n", "\n", - "trading_allowance = system.parameters.gov.hmrc.income_tax.allowances.trading_allowance\n", + "trading_allowance = (\n", + " system.parameters.gov.hmrc.income_tax.allowances.trading_allowance\n", + ")\n", "trading_allowance_data = {\n", " \"Attribute\": [\"Trading Allowance\"],\n", " \"Type\": [\"Trading Allowance\"],\n", diff --git a/docs/book/programs/gov/hmrc/stamp-duty.ipynb b/docs/book/programs/gov/hmrc/stamp-duty.ipynb index b7975450b..9a91305d9 100644 --- a/docs/book/programs/gov/hmrc/stamp-duty.ipynb +++ b/docs/book/programs/gov/hmrc/stamp-duty.ipynb @@ -6287,7 +6287,9 @@ "stamp_duty = sim.calculate(\"stamp_duty_land_tax\")\n", "home_price = sim.calculate(\"main_residential_property_purchased\")\n", "\n", - "marginal_rate = (stamp_duty[1:] - stamp_duty[:-1]) / (home_price[1:] - home_price[:-1])\n", + "marginal_rate = (stamp_duty[1:] - stamp_duty[:-1]) / (\n", + " home_price[1:] - home_price[:-1]\n", + ")\n", "home_price = home_price[:-1]\n", "\n", "df = pd.DataFrame(\n", @@ -6303,10 +6305,14 @@ "\n", "# left shows marginal rate, right shows total SDLT\n", "\n", - "fig = make_subplots(rows=1, cols=2, subplot_titles=(\"Marginal SDLT rate\", \"Total SDLT\"))\n", + "fig = make_subplots(\n", + " rows=1, cols=2, subplot_titles=(\"Marginal SDLT rate\", \"Total SDLT\")\n", + ")\n", "\n", "fig.add_trace(\n", - " px.line(df, x=\"Home price\", y=\"Marginal SDLT rate\", title=\"Marginal SDLT rate\")\n", + " px.line(\n", + " df, x=\"Home price\", y=\"Marginal SDLT rate\", title=\"Marginal SDLT rate\"\n", + " )\n", " .update_traces(line_shape=\"hv\")\n", " .data[0],\n", " row=1,\n", diff --git a/docs/book/programs/gov/ofgem/energy-price-guarantee.ipynb b/docs/book/programs/gov/ofgem/energy-price-guarantee.ipynb index e23a13d53..7092fee85 100644 --- a/docs/book/programs/gov/ofgem/energy-price-guarantee.ipynb +++ b/docs/book/programs/gov/ofgem/energy-price-guarantee.ipynb @@ -1011,7 +1011,9 @@ "\n", "df = pd.DataFrame()\n", "\n", - "df[\"Date\"] = [parameter.instant_str for parameter in ofgem.energy_price_cap.values_list]\n", + "df[\"Date\"] = [\n", + " parameter.instant_str for parameter in ofgem.energy_price_cap.values_list\n", + "]\n", "df[\"Energy price cap\"] = [\n", " parameter.value for parameter in ofgem.energy_price_cap.values_list\n", "]\n", @@ -1105,9 +1107,9 @@ "xaxis": "x", "y": [ 8.373929949593731e-10, - 1.674785989918746e-9, - 2.512178984878119e-9, - 3.3495719798374923e-9, + 1.674785989918746e-09, + 2.512178984878119e-09, + 3.3495719798374923e-09, 1.2009600411861108, 2.40192007902265, 3.602880116859188, @@ -1996,7 +1998,9 @@ " *[f\"2023-{month:02d}\" for month in range(1, 13)],\n", "]\n", "\n", - "epg_costs = [sim.calc(\"monthly_epg_subsidy\", period).sum() for period in time_periods]\n", + "epg_costs = [\n", + " sim.calc(\"monthly_epg_subsidy\", period).sum() for period in time_periods\n", + "]\n", "\n", "df = pd.DataFrame(\n", " {\n", diff --git a/docs/book/programs/gov/revenue_scotland/land-and-buildings-transaction-tax.ipynb b/docs/book/programs/gov/revenue_scotland/land-and-buildings-transaction-tax.ipynb index 6e6400b9c..09f4009b0 100644 --- a/docs/book/programs/gov/revenue_scotland/land-and-buildings-transaction-tax.ipynb +++ b/docs/book/programs/gov/revenue_scotland/land-and-buildings-transaction-tax.ipynb @@ -1217,7 +1217,9 @@ " color=\"Label\",\n", " title=\"Land and Buildings Transaction Tax (LBTT) rates over property price thresholds\",\n", " )\n", - " .update_layout(yaxis_tickformat=\",.0%\", xaxis_tickprefix=\"£\", legend_title=\"\")\n", + " .update_layout(\n", + " yaxis_tickformat=\",.0%\", xaxis_tickprefix=\"£\", legend_title=\"\"\n", + " )\n", " .update_traces(line_shape=\"hv\")\n", ")\n", "fig = format_fig(fig)\n", @@ -6239,7 +6241,9 @@ "lbtt_sim = sim.calculate(\"land_and_buildings_transaction_tax\")\n", "home_price = sim.calculate(\"main_residential_property_purchased\")\n", "\n", - "marginal_rate = (lbtt_sim[1:] - lbtt_sim[:-1]) / (home_price[1:] - home_price[:-1])\n", + "marginal_rate = (lbtt_sim[1:] - lbtt_sim[:-1]) / (\n", + " home_price[1:] - home_price[:-1]\n", + ")\n", "home_price = home_price[:-1]\n", "\n", "df = pd.DataFrame(\n", @@ -6255,10 +6259,14 @@ "\n", "# left shows marginal rate, right shows total LBTT\n", "\n", - "fig = make_subplots(rows=1, cols=2, subplot_titles=(\"Marginal LBTT rate\", \"Total LBTT\"))\n", + "fig = make_subplots(\n", + " rows=1, cols=2, subplot_titles=(\"Marginal LBTT rate\", \"Total LBTT\")\n", + ")\n", "\n", "fig.add_trace(\n", - " px.line(df, x=\"Home price\", y=\"Marginal LBTT rate\", title=\"Marginal LBTT rate\")\n", + " px.line(\n", + " df, x=\"Home price\", y=\"Marginal LBTT rate\", title=\"Marginal LBTT rate\"\n", + " )\n", " .update_traces(line_shape=\"hv\")\n", " .data[0],\n", " row=1,\n", diff --git a/docs/book/programs/gov/wra/land-transaction-tax.ipynb b/docs/book/programs/gov/wra/land-transaction-tax.ipynb index 80d4549ce..12a05d5c3 100644 --- a/docs/book/programs/gov/wra/land-transaction-tax.ipynb +++ b/docs/book/programs/gov/wra/land-transaction-tax.ipynb @@ -1153,7 +1153,9 @@ " color=\"Label\",\n", " title=\"Land Transaction Tax (LTT) rates over property price thresholds\",\n", " )\n", - " .update_layout(yaxis_tickformat=\",.0%\", xaxis_tickprefix=\"£\", legend_title=\"\")\n", + " .update_layout(\n", + " yaxis_tickformat=\",.0%\", xaxis_tickprefix=\"£\", legend_title=\"\"\n", + " )\n", " .update_traces(line_shape=\"hv\")\n", ")\n", "fig = format_fig(fig)\n", diff --git a/docs/book/validation/validation.ipynb b/docs/book/validation/validation.ipynb index 2e3dc159b..9c2fce073 100644 --- a/docs/book/validation/validation.ipynb +++ b/docs/book/validation/validation.ipynb @@ -1901,7 +1901,9 @@ " return (\n", " np.array(\n", " [\n", - " get_parameter(calibration_parameters, parameter)(f\"{year}-01-01\")\n", + " get_parameter(calibration_parameters, parameter)(\n", + " f\"{year}-01-01\"\n", + " )\n", " for year in range(2023, 2026)\n", " ]\n", " )\n", diff --git a/policyengine_uk/parameters/gov/dfe/extended_childcare_entitlement/takeup_rate.yaml b/policyengine_uk/parameters/gov/dfe/extended_childcare_entitlement/takeup_rate.yaml deleted file mode 100644 index f8e27d481..000000000 --- a/policyengine_uk/parameters/gov/dfe/extended_childcare_entitlement/takeup_rate.yaml +++ /dev/null @@ -1,9 +0,0 @@ -description: Extended childcare entitlement take-up rate -metadata: - unit: /1 - period: year - reference: - - title: Empirical estimate from FRS data - href: https://github.com/PolicyEngine/policyengine-uk-data -values: - 2015-01-01: 0.812 diff --git a/policyengine_uk/parameters/gov/dfe/targeted_childcare_entitlement/takeup_rate.yaml b/policyengine_uk/parameters/gov/dfe/targeted_childcare_entitlement/takeup_rate.yaml deleted file mode 100644 index 2693a72d2..000000000 --- a/policyengine_uk/parameters/gov/dfe/targeted_childcare_entitlement/takeup_rate.yaml +++ /dev/null @@ -1,9 +0,0 @@ -description: Targeted childcare entitlement take-up rate -metadata: - unit: /1 - period: year - reference: - - title: Empirical estimate from FRS data - href: https://github.com/PolicyEngine/policyengine-uk-data -values: - 2015-01-01: 0.597 diff --git a/policyengine_uk/parameters/gov/dfe/universal_childcare_entitlement/takeup_rate.yaml b/policyengine_uk/parameters/gov/dfe/universal_childcare_entitlement/takeup_rate.yaml deleted file mode 100644 index f3ac5589e..000000000 --- a/policyengine_uk/parameters/gov/dfe/universal_childcare_entitlement/takeup_rate.yaml +++ /dev/null @@ -1,9 +0,0 @@ -description: Universal childcare entitlement take-up rate -metadata: - unit: /1 - period: year - reference: - - title: Empirical estimate from FRS data - href: https://github.com/PolicyEngine/policyengine-uk-data -values: - 2015-01-01: 0.563 diff --git a/policyengine_uk/parameters/gov/hmrc/tax_free_childcare/takeup_rate.yaml b/policyengine_uk/parameters/gov/hmrc/tax_free_childcare/takeup_rate.yaml deleted file mode 100644 index fbddc2adc..000000000 --- a/policyengine_uk/parameters/gov/hmrc/tax_free_childcare/takeup_rate.yaml +++ /dev/null @@ -1,9 +0,0 @@ -description: Tax-Free Childcare take-up rate -metadata: - unit: /1 - period: year - reference: - - title: Empirical estimate from FRS data - href: https://github.com/PolicyEngine/policyengine-uk-data -values: - 2015-01-01: 0.586 diff --git a/policyengine_uk/tests/microsimulation/reforms_config.yaml b/policyengine_uk/tests/microsimulation/reforms_config.yaml index 3cd3f7ff2..0ce7b5a32 100644 --- a/policyengine_uk/tests/microsimulation/reforms_config.yaml +++ b/policyengine_uk/tests/microsimulation/reforms_config.yaml @@ -1,10 +1,10 @@ reforms: - name: Raise basic rate by 1pp - expected_impact: 8.0 + expected_impact: 8.1 parameters: gov.hmrc.income_tax.rates.uk[0].rate: 0.21 - name: Raise higher rate by 1pp - expected_impact: 5.0 + expected_impact: 4.8 parameters: gov.hmrc.income_tax.rates.uk[1].rate: 0.42 - name: Raise personal allowance by ~800GBP/year @@ -12,22 +12,22 @@ reforms: parameters: gov.hmrc.income_tax.allowances.personal_allowance.amount: 13000 - name: Raise child benefit by 25GBP/week per additional child - expected_impact: -1.5 + expected_impact: -1.4 parameters: gov.hmrc.child_benefit.amount.additional: 25 - name: Reduce Universal Credit taper rate to 20% - expected_impact: -36.5 + expected_impact: -35.3 parameters: gov.dwp.universal_credit.means_test.reduction_rate: 0.2 - name: Raise Class 1 main employee NICs rate to 10% - expected_impact: 12.4 + expected_impact: 12.3 parameters: gov.hmrc.national_insurance.class_1.rates.employee.main: 0.1 - name: Raise VAT standard rate by 2pp - expected_impact: 21.2 + expected_impact: 18.9 parameters: gov.hmrc.vat.standard_rate: 0.22 - name: Raise additional rate by 3pp - expected_impact: 5.4 + expected_impact: 4.7 parameters: gov.hmrc.income_tax.rates.uk[2].rate: 0.48 diff --git a/policyengine_uk/variables/contrib/labour/attends_private_school.py b/policyengine_uk/variables/contrib/labour/attends_private_school.py index be719b918..a542134fd 100644 --- a/policyengine_uk/variables/contrib/labour/attends_private_school.py +++ b/policyengine_uk/variables/contrib/labour/attends_private_school.py @@ -76,6 +76,6 @@ def formula(person, period, parameters): * is_child ) - value = person("attends_private_school_seed", period) < p_attends_private_school - - return value + # Use random draw from dataset (only exists in microsimulation) + random_draw = person("attends_private_school_random_draw", period) + return random_draw < p_attends_private_school diff --git a/policyengine_uk/variables/contrib/labour/attends_private_school_random_draw.py b/policyengine_uk/variables/contrib/labour/attends_private_school_random_draw.py new file mode 100644 index 000000000..a15327ad5 --- /dev/null +++ b/policyengine_uk/variables/contrib/labour/attends_private_school_random_draw.py @@ -0,0 +1,12 @@ +from policyengine_uk.model_api import * + + +class attends_private_school_random_draw(Variable): + value_type = float + entity = Person + label = "Random draw for private school attendance" + documentation = ( + "Random value [0,1) used to determine private school attendance. " + "Generated in the dataset for reproducibility." + ) + definition_period = YEAR diff --git a/policyengine_uk/variables/contrib/labour/attends_private_school_seed.py b/policyengine_uk/variables/contrib/labour/attends_private_school_seed.py deleted file mode 100644 index 61e43bd18..000000000 --- a/policyengine_uk/variables/contrib/labour/attends_private_school_seed.py +++ /dev/null @@ -1,8 +0,0 @@ -from policyengine_uk.model_api import * - - -class attends_private_school_seed(Variable): - value_type = float - entity = Person - label = "Random seed for private school attendance" - definition_period = YEAR diff --git a/policyengine_uk/variables/gov/dcms/bbc/tv_licence/tv_licence_evasion_seed.py b/policyengine_uk/variables/gov/dcms/bbc/tv_licence/tv_licence_evasion_seed.py deleted file mode 100644 index 4df2346c9..000000000 --- a/policyengine_uk/variables/gov/dcms/bbc/tv_licence/tv_licence_evasion_seed.py +++ /dev/null @@ -1,8 +0,0 @@ -from policyengine_uk.model_api import * - - -class tv_licence_evasion_seed(Variable): - value_type = float - entity = Household - label = "Random seed for TV licence evasion" - definition_period = YEAR diff --git a/policyengine_uk/variables/gov/dcms/bbc/tv_licence/would_evade_tv_licence_fee.py b/policyengine_uk/variables/gov/dcms/bbc/tv_licence/would_evade_tv_licence_fee.py index 5d3655356..61be08139 100644 --- a/policyengine_uk/variables/gov/dcms/bbc/tv_licence/would_evade_tv_licence_fee.py +++ b/policyengine_uk/variables/gov/dcms/bbc/tv_licence/would_evade_tv_licence_fee.py @@ -4,12 +4,13 @@ class would_evade_tv_licence_fee(Variable): label = "Would evade TV licence fee" documentation = ( - "Whether this household would unlawfully evade the TV licence fee." + "Whether this household would unlawfully evade the TV licence fee. " + "Generated stochastically in the dataset using evasion rates." ) entity = Household definition_period = YEAR value_type = bool - def formula(household, period, parameters): - evasion_rate = parameters(period).gov.dcms.bbc.tv_licence.evasion_rate - return household("tv_licence_evasion_seed", period) <= evasion_rate + # No formula - when in dataset, OpenFisca uses dataset value automatically + # For policy calculator (non-dataset), defaults to False + default_value = False diff --git a/policyengine_uk/variables/gov/dfe/extended_childcare_entitlement/extended_childcare_take_up_seed.py b/policyengine_uk/variables/gov/dfe/extended_childcare_entitlement/extended_childcare_take_up_seed.py deleted file mode 100644 index 53263d6a9..000000000 --- a/policyengine_uk/variables/gov/dfe/extended_childcare_entitlement/extended_childcare_take_up_seed.py +++ /dev/null @@ -1,8 +0,0 @@ -from policyengine_uk.model_api import * - - -class extended_childcare_take_up_seed(Variable): - value_type = float - entity = BenUnit - label = "Random seed for extended childcare take-up" - definition_period = YEAR diff --git a/policyengine_uk/variables/gov/dfe/extended_childcare_entitlement/would_claim_extended_childcare.py b/policyengine_uk/variables/gov/dfe/extended_childcare_entitlement/would_claim_extended_childcare.py index 029fcdaf5..7678a426d 100644 --- a/policyengine_uk/variables/gov/dfe/extended_childcare_entitlement/would_claim_extended_childcare.py +++ b/policyengine_uk/variables/gov/dfe/extended_childcare_entitlement/would_claim_extended_childcare.py @@ -5,12 +5,12 @@ class would_claim_extended_childcare(Variable): value_type = bool entity = BenUnit label = "would claim extended childcare entitlement" - documentation = "Whether this family would claim extended childcare entitlement if eligible" + documentation = ( + "Whether this family would claim extended childcare entitlement if eligible. " + "Generated stochastically in the dataset using take-up rates." + ) definition_period = YEAR - def formula(benunit, period, parameters): - takeup_rate = parameters(period).gov.dfe.extended_childcare_entitlement.takeup_rate - is_in_microsimulation = benunit.simulation.dataset is not None - if is_in_microsimulation: - return benunit("extended_childcare_take_up_seed", period) < takeup_rate - return True + # No formula - when in dataset, OpenFisca uses dataset value automatically + # For policy calculator (non-dataset), defaults to True + default_value = True diff --git a/policyengine_uk/variables/gov/dfe/targeted_childcare_entitlement/targeted_childcare_take_up_seed.py b/policyengine_uk/variables/gov/dfe/targeted_childcare_entitlement/targeted_childcare_take_up_seed.py deleted file mode 100644 index dabddb755..000000000 --- a/policyengine_uk/variables/gov/dfe/targeted_childcare_entitlement/targeted_childcare_take_up_seed.py +++ /dev/null @@ -1,8 +0,0 @@ -from policyengine_uk.model_api import * - - -class targeted_childcare_take_up_seed(Variable): - value_type = float - entity = BenUnit - label = "Random seed for targeted childcare take-up" - definition_period = YEAR diff --git a/policyengine_uk/variables/gov/dfe/targeted_childcare_entitlement/would_claim_targeted_childcare.py b/policyengine_uk/variables/gov/dfe/targeted_childcare_entitlement/would_claim_targeted_childcare.py index 2c02120d6..5ecd4d574 100644 --- a/policyengine_uk/variables/gov/dfe/targeted_childcare_entitlement/would_claim_targeted_childcare.py +++ b/policyengine_uk/variables/gov/dfe/targeted_childcare_entitlement/would_claim_targeted_childcare.py @@ -5,12 +5,12 @@ class would_claim_targeted_childcare(Variable): value_type = bool entity = BenUnit label = "would claim targeted childcare entitlement" - documentation = "Whether this family would claim targeted childcare entitlement if eligible" + documentation = ( + "Whether this family would claim targeted childcare entitlement if eligible. " + "Generated stochastically in the dataset using take-up rates." + ) definition_period = YEAR - def formula(benunit, period, parameters): - takeup_rate = parameters(period).gov.dfe.targeted_childcare_entitlement.takeup_rate - is_in_microsimulation = benunit.simulation.dataset is not None - if is_in_microsimulation: - return benunit("targeted_childcare_take_up_seed", period) < takeup_rate - return True + # No formula - when in dataset, OpenFisca uses dataset value automatically + # For policy calculator (non-dataset), defaults to True + default_value = True diff --git a/policyengine_uk/variables/gov/dfe/universal_childcare_entitlement/universal_childcare_take_up_seed.py b/policyengine_uk/variables/gov/dfe/universal_childcare_entitlement/universal_childcare_take_up_seed.py deleted file mode 100644 index c34624d34..000000000 --- a/policyengine_uk/variables/gov/dfe/universal_childcare_entitlement/universal_childcare_take_up_seed.py +++ /dev/null @@ -1,8 +0,0 @@ -from policyengine_uk.model_api import * - - -class universal_childcare_take_up_seed(Variable): - value_type = float - entity = BenUnit - label = "Random seed for universal childcare take-up" - definition_period = YEAR diff --git a/policyengine_uk/variables/gov/dfe/universal_childcare_entitlement/would_claim_universal_childcare.py b/policyengine_uk/variables/gov/dfe/universal_childcare_entitlement/would_claim_universal_childcare.py index fa6677f21..227597fcc 100644 --- a/policyengine_uk/variables/gov/dfe/universal_childcare_entitlement/would_claim_universal_childcare.py +++ b/policyengine_uk/variables/gov/dfe/universal_childcare_entitlement/would_claim_universal_childcare.py @@ -5,12 +5,12 @@ class would_claim_universal_childcare(Variable): value_type = bool entity = BenUnit label = "would claim universal childcare entitlement" - documentation = "Whether this BenUnit would claim universal childcare entitlement if eligible" + documentation = ( + "Whether this BenUnit would claim universal childcare entitlement if eligible. " + "Generated stochastically in the dataset using take-up rates." + ) definition_period = YEAR - def formula(benunit, period, parameters): - takeup_rate = parameters(period).gov.dfe.universal_childcare_entitlement.takeup_rate - is_in_microsimulation = benunit.simulation.dataset is not None - if is_in_microsimulation: - return benunit("universal_childcare_take_up_seed", period) < takeup_rate - return True + # No formula - when in dataset, OpenFisca uses dataset value automatically + # For policy calculator (non-dataset), defaults to True + default_value = True diff --git a/policyengine_uk/variables/gov/dwp/pension_credit/pension_credit_take_up_seed.py b/policyengine_uk/variables/gov/dwp/pension_credit/pension_credit_take_up_seed.py deleted file mode 100644 index 775f79b92..000000000 --- a/policyengine_uk/variables/gov/dwp/pension_credit/pension_credit_take_up_seed.py +++ /dev/null @@ -1,8 +0,0 @@ -from policyengine_uk.model_api import * - - -class pension_credit_take_up_seed(Variable): - value_type = float - entity = BenUnit - label = "Random seed for pension credit take-up" - definition_period = YEAR diff --git a/policyengine_uk/variables/gov/dwp/pension_credit/would_claim.py b/policyengine_uk/variables/gov/dwp/pension_credit/would_claim.py index 83a5e4cbd..2c043b038 100644 --- a/policyengine_uk/variables/gov/dwp/pension_credit/would_claim.py +++ b/policyengine_uk/variables/gov/dwp/pension_credit/would_claim.py @@ -3,27 +3,14 @@ class would_claim_pc(Variable): label = "Would claim Pension Credit" + documentation = ( + "Whether this benefit unit would claim Pension Credit if eligible. " + "Generated stochastically in the dataset using take-up rates." + ) entity = BenUnit definition_period = YEAR value_type = bool - def formula(benunit, period, parameters): - reported_pc = add(benunit, period, ["pension_credit_reported"]) > 0 - claims_all_entitled_benefits = benunit( - "claims_all_entitled_benefits", period - ) - baseline = benunit("baseline_pension_credit_entitlement", period) > 0 - eligible = benunit("pension_credit_entitlement", period) > 0 - takeup_rate = parameters(period).gov.dwp.pension_credit.takeup - return select( - [ - reported_pc | claims_all_entitled_benefits, - ~baseline & eligible, - True, - ], - [ - True, - benunit("pension_credit_take_up_seed", period) < takeup_rate, - False, - ], - ) + # No formula - when in dataset, OpenFisca uses dataset value automatically + # For policy calculator (non-dataset), defaults to True + default_value = True diff --git a/policyengine_uk/variables/gov/dwp/universal_credit/universal_credit_take_up_seed.py b/policyengine_uk/variables/gov/dwp/universal_credit/universal_credit_take_up_seed.py deleted file mode 100644 index e12431573..000000000 --- a/policyengine_uk/variables/gov/dwp/universal_credit/universal_credit_take_up_seed.py +++ /dev/null @@ -1,8 +0,0 @@ -from policyengine_uk.model_api import * - - -class universal_credit_take_up_seed(Variable): - value_type = float - entity = BenUnit - label = "Random seed for universal credit take-up" - definition_period = YEAR diff --git a/policyengine_uk/variables/gov/dwp/universal_credit/would_claim_uc.py b/policyengine_uk/variables/gov/dwp/universal_credit/would_claim_uc.py index 3fa20abd8..de0beecfe 100644 --- a/policyengine_uk/variables/gov/dwp/universal_credit/would_claim_uc.py +++ b/policyengine_uk/variables/gov/dwp/universal_credit/would_claim_uc.py @@ -6,16 +6,11 @@ class would_claim_uc(Variable): entity = BenUnit label = "Would claim Universal Credit" documentation = ( - "Whether this family would claim the Universal Credit if eligible" + "Whether this family would claim the Universal Credit if eligible. " + "Generated stochastically in the dataset using take-up rates." ) definition_period = YEAR - def formula(benunit, period, parameters): - takes_up = ( - benunit("universal_credit_take_up_seed", period) - < parameters(period).gov.dwp.universal_credit.takeup_rate - ) - is_in_microsimulation = benunit.simulation.dataset is not None - if is_in_microsimulation: - return takes_up - return True + # No formula - when in dataset, OpenFisca uses dataset value automatically + # For policy calculator (non-dataset), defaults to True + default_value = True diff --git a/policyengine_uk/variables/gov/hmrc/child_benefit_opts_out.py b/policyengine_uk/variables/gov/hmrc/child_benefit_opts_out.py index 01ade9156..9b1da53bb 100644 --- a/policyengine_uk/variables/gov/hmrc/child_benefit_opts_out.py +++ b/policyengine_uk/variables/gov/hmrc/child_benefit_opts_out.py @@ -4,24 +4,13 @@ class child_benefit_opts_out(Variable): label = "opts out of Child Benefit" documentation = ( - "Whether this family would opt out of receiving Child Benefit payments" + "Whether this family would opt out of receiving Child Benefit payments. " + "Generated stochastically in the dataset using opt-out rates." ) entity = BenUnit definition_period = YEAR value_type = bool - def formula(benunit, period, parameters): - if benunit.simulation.dataset is not None: - ani = benunit.members("adjusted_net_income", period) - hmrc = parameters(period).gov.hmrc - cb_hitc = hmrc.income_tax.charges.CB_HITC - cb = hmrc.child_benefit - in_phase_out = ani > cb_hitc.phase_out_end - return where( - benunit.any(in_phase_out), - benunit("child_benefit_opts_out_seed", period) < cb.opt_out_rate, - False, - ) - else: - # If we're not in a microsimulation, assume the family would not opt out - return False + # No formula - when in dataset, OpenFisca uses dataset value automatically + # For policy calculator (non-dataset), defaults to False + default_value = False diff --git a/policyengine_uk/variables/gov/hmrc/child_benefit_opts_out_seed.py b/policyengine_uk/variables/gov/hmrc/child_benefit_opts_out_seed.py deleted file mode 100644 index cd4fda91d..000000000 --- a/policyengine_uk/variables/gov/hmrc/child_benefit_opts_out_seed.py +++ /dev/null @@ -1,8 +0,0 @@ -from policyengine_uk.model_api import * - - -class child_benefit_opts_out_seed(Variable): - value_type = float - entity = BenUnit - label = "Random seed for child benefit opt-out decision" - definition_period = YEAR diff --git a/policyengine_uk/variables/gov/hmrc/child_benefit_take_up_seed.py b/policyengine_uk/variables/gov/hmrc/child_benefit_take_up_seed.py deleted file mode 100644 index 8018521bf..000000000 --- a/policyengine_uk/variables/gov/hmrc/child_benefit_take_up_seed.py +++ /dev/null @@ -1,8 +0,0 @@ -from policyengine_uk.model_api import * - - -class child_benefit_take_up_seed(Variable): - value_type = float - entity = BenUnit - label = "Random seed for child benefit take-up" - definition_period = YEAR diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/allowances/marriage_allowance.py b/policyengine_uk/variables/gov/hmrc/income_tax/allowances/marriage_allowance.py index 3e83ebc6a..c3253e85e 100644 --- a/policyengine_uk/variables/gov/hmrc/income_tax/allowances/marriage_allowance.py +++ b/policyengine_uk/variables/gov/hmrc/income_tax/allowances/marriage_allowance.py @@ -29,4 +29,5 @@ def formula(person, period, parameters): np.ceil(amount_if_eligible_pre_rounding / rounding_increment) * rounding_increment ) - return eligible * amount_if_eligible * (person("marriage_allowance_take_up_seed", period) < takeup_rate) + would_claim = person("would_claim_marriage_allowance", period) + return eligible * amount_if_eligible * would_claim diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/allowances/marriage_allowance_take_up_seed.py b/policyengine_uk/variables/gov/hmrc/income_tax/allowances/marriage_allowance_take_up_seed.py deleted file mode 100644 index b42a94596..000000000 --- a/policyengine_uk/variables/gov/hmrc/income_tax/allowances/marriage_allowance_take_up_seed.py +++ /dev/null @@ -1,8 +0,0 @@ -from policyengine_uk.model_api import * - - -class marriage_allowance_take_up_seed(Variable): - value_type = float - entity = Person - label = "Random seed for marriage allowance take-up" - definition_period = YEAR diff --git a/policyengine_uk/variables/gov/hmrc/income_tax/allowances/would_claim_marriage_allowance.py b/policyengine_uk/variables/gov/hmrc/income_tax/allowances/would_claim_marriage_allowance.py new file mode 100644 index 000000000..a337116bd --- /dev/null +++ b/policyengine_uk/variables/gov/hmrc/income_tax/allowances/would_claim_marriage_allowance.py @@ -0,0 +1,13 @@ +from policyengine_uk.model_api import * + + +class would_claim_marriage_allowance(Variable): + value_type = bool + entity = Person + label = "Would claim Marriage Allowance" + documentation = ( + "Whether this person would claim Marriage Allowance if eligible. " + "Generated stochastically in the dataset using take-up rates." + ) + definition_period = YEAR + default_value = True diff --git a/policyengine_uk/variables/gov/hmrc/tax_free_childcare/tax_free_childcare_take_up_seed.py b/policyengine_uk/variables/gov/hmrc/tax_free_childcare/tax_free_childcare_take_up_seed.py deleted file mode 100644 index 5767a3f2b..000000000 --- a/policyengine_uk/variables/gov/hmrc/tax_free_childcare/tax_free_childcare_take_up_seed.py +++ /dev/null @@ -1,8 +0,0 @@ -from policyengine_uk.model_api import * - - -class tax_free_childcare_take_up_seed(Variable): - value_type = float - entity = BenUnit - label = "Random seed for Tax-Free Childcare take-up" - definition_period = YEAR diff --git a/policyengine_uk/variables/gov/hmrc/tax_free_childcare/would_claim_tfc.py b/policyengine_uk/variables/gov/hmrc/tax_free_childcare/would_claim_tfc.py index 1cf4fdb7d..346e86a21 100644 --- a/policyengine_uk/variables/gov/hmrc/tax_free_childcare/would_claim_tfc.py +++ b/policyengine_uk/variables/gov/hmrc/tax_free_childcare/would_claim_tfc.py @@ -6,13 +6,11 @@ class would_claim_tfc(Variable): entity = BenUnit label = "would claim Tax-Free Childcare" documentation = ( - "Whether this family would claim Tax-Free Childcare if eligible" + "Whether this family would claim Tax-Free Childcare if eligible. " + "Generated stochastically in the dataset using take-up rates." ) definition_period = YEAR - def formula(benunit, period, parameters): - takeup_rate = parameters(period).gov.hmrc.tax_free_childcare.takeup_rate - is_in_microsimulation = benunit.simulation.dataset is not None - if is_in_microsimulation: - return benunit("tax_free_childcare_take_up_seed", period) < takeup_rate - return True + # No formula - when in dataset, OpenFisca uses dataset value automatically + # For policy calculator (non-dataset), defaults to True + default_value = True diff --git a/policyengine_uk/variables/gov/hmrc/would_claim_child_benefit.py b/policyengine_uk/variables/gov/hmrc/would_claim_child_benefit.py index 1c4a93db8..9a84f521a 100644 --- a/policyengine_uk/variables/gov/hmrc/would_claim_child_benefit.py +++ b/policyengine_uk/variables/gov/hmrc/would_claim_child_benefit.py @@ -4,15 +4,13 @@ class would_claim_child_benefit(Variable): label = "Would claim Child Benefit" documentation = ( - "Whether this benefit unit would claim Child Benefit if eligible" + "Whether this benefit unit would claim Child Benefit if eligible. " + "Generated stochastically in the dataset using take-up rates." ) entity = BenUnit definition_period = YEAR value_type = bool - def formula(benunit, period, parameters): - takeup_rate = parameters(period).gov.hmrc.child_benefit.takeup - overall_p = takeup_rate.overall - return (benunit("child_benefit_take_up_seed", period) < overall_p) * ~benunit( - "child_benefit_opts_out", period - ) + # No formula - when in dataset, OpenFisca uses dataset value automatically + # For policy calculator (non-dataset), defaults to True + default_value = True diff --git a/policyengine_uk/variables/household/consumption/first_home_purchase_seed.py b/policyengine_uk/variables/household/consumption/first_home_purchase_seed.py deleted file mode 100644 index 993a781fd..000000000 --- a/policyengine_uk/variables/household/consumption/first_home_purchase_seed.py +++ /dev/null @@ -1,8 +0,0 @@ -from policyengine_uk.model_api import * - - -class first_home_purchase_seed(Variable): - value_type = float - entity = Household - label = "Random seed for first home purchase determination" - definition_period = YEAR diff --git a/policyengine_uk/variables/household/consumption/main_residential_property_purchased_is_first_home.py b/policyengine_uk/variables/household/consumption/main_residential_property_purchased_is_first_home.py index 102d2a144..795ef0ff3 100644 --- a/policyengine_uk/variables/household/consumption/main_residential_property_purchased_is_first_home.py +++ b/policyengine_uk/variables/household/consumption/main_residential_property_purchased_is_first_home.py @@ -3,22 +3,14 @@ class main_residential_property_purchased_is_first_home(Variable): label = "Residential property bought is first home" - documentation = "Whether the residential property bought this year as a main residence was as a first-time buyer." + documentation = ( + "Whether the residential property bought this year as a main residence " + "was as a first-time buyer. Generated stochastically in the dataset." + ) entity = Household definition_period = YEAR value_type = bool - unit = GBP - def formula(household, period, parameters): - residential_sd = parameters( - period - ).gov.hmrc.stamp_duty.statistics.residential.household - age = household.sum( - household.members("is_household_head", period) - * household.members("age", period) - ) - percentage_claiming_ftbr = ( - residential_sd.first_time_buyers_relief.calc(age) - / residential_sd.transactions_by_age.calc(age) - ) - return household("first_home_purchase_seed", period) < percentage_claiming_ftbr + # No formula - when in dataset, OpenFisca uses dataset value automatically + # For policy calculator (non-dataset), defaults to False + default_value = False diff --git a/policyengine_uk/variables/household/demographic/higher_earner_tie_break.py b/policyengine_uk/variables/household/demographic/higher_earner_tie_break.py new file mode 100644 index 000000000..d025d870e --- /dev/null +++ b/policyengine_uk/variables/household/demographic/higher_earner_tie_break.py @@ -0,0 +1,12 @@ +from policyengine_uk.model_api import * + + +class higher_earner_tie_break(Variable): + value_type = float + entity = Person + label = "Random draw for tie-breaking in higher earner determination" + documentation = ( + "Random value [0,1) used to break ties when determining the higher earner. " + "Generated in the dataset for reproducibility." + ) + definition_period = YEAR diff --git a/policyengine_uk/variables/household/demographic/household_owns_tv.py b/policyengine_uk/variables/household/demographic/household_owns_tv.py index a856ff133..7975924de 100644 --- a/policyengine_uk/variables/household/demographic/household_owns_tv.py +++ b/policyengine_uk/variables/household/demographic/household_owns_tv.py @@ -3,13 +3,14 @@ class household_owns_tv(Variable): label = "Owns a TV" - documentation = "Whether this household owns a functioning colour TV." + documentation = ( + "Whether this household owns a functioning colour TV. " + "Generated stochastically in the dataset using ownership rates." + ) entity = Household definition_period = YEAR value_type = bool - def formula(household, period, parameters): - percent_owning_tv = parameters( - period - ).gov.dcms.bbc.tv_licence.tv_ownership - return household("household_owns_tv_seed", period) <= percent_owning_tv + # No formula - when in dataset, OpenFisca uses dataset value automatically + # For policy calculator (non-dataset), defaults to True + default_value = True diff --git a/policyengine_uk/variables/household/demographic/household_owns_tv_seed.py b/policyengine_uk/variables/household/demographic/household_owns_tv_seed.py deleted file mode 100644 index c45dfc3db..000000000 --- a/policyengine_uk/variables/household/demographic/household_owns_tv_seed.py +++ /dev/null @@ -1,8 +0,0 @@ -from policyengine_uk.model_api import * - - -class household_owns_tv_seed(Variable): - value_type = float - entity = Household - label = "Random seed for TV ownership" - definition_period = YEAR diff --git a/policyengine_uk/variables/household/demographic/is_disabled_for_benefits.py b/policyengine_uk/variables/household/demographic/is_disabled_for_benefits.py index d52b80edb..84c962ff6 100644 --- a/policyengine_uk/variables/household/demographic/is_disabled_for_benefits.py +++ b/policyengine_uk/variables/household/demographic/is_disabled_for_benefits.py @@ -10,20 +10,14 @@ class is_disabled_for_benefits(Variable): reference = "Child Tax Credit Regulations 2002 s. 8" def formula(person, period, parameters): + # In microsimulation, this is set in the dataset (see frs.py line 726) + # This formula only runs for policy calculator (non-dataset) + + # Deterministic rule: disabled if they receive qualifying benefits QUALIFYING_BENEFITS = [ "dla", "pip", ] - p_claims_lcwra_if_on_pip_dla = 0.8 - p_claims_lcwra_if_not_on_pip_dla = 0.13 - - random_seed = person("is_disabled_for_benefits_seed", period) - on_qual_benefits = add(person, period, QUALIFYING_BENEFITS) > 0 - - return np.where( - on_qual_benefits, - random_seed < p_claims_lcwra_if_on_pip_dla, - random_seed < p_claims_lcwra_if_not_on_pip_dla, - ) + return on_qual_benefits diff --git a/policyengine_uk/variables/household/demographic/is_disabled_for_benefits_seed.py b/policyengine_uk/variables/household/demographic/is_disabled_for_benefits_seed.py deleted file mode 100644 index f340fb85b..000000000 --- a/policyengine_uk/variables/household/demographic/is_disabled_for_benefits_seed.py +++ /dev/null @@ -1,8 +0,0 @@ -from policyengine_uk.model_api import * - - -class is_disabled_for_benefits_seed(Variable): - value_type = float - entity = Person - label = "Random seed for disability benefits eligibility determination" - definition_period = YEAR diff --git a/policyengine_uk/variables/household/demographic/is_higher_earner.py b/policyengine_uk/variables/household/demographic/is_higher_earner.py index 42679b4aa..18314793d 100644 --- a/policyengine_uk/variables/household/demographic/is_higher_earner.py +++ b/policyengine_uk/variables/household/demographic/is_higher_earner.py @@ -9,8 +9,10 @@ class is_higher_earner(Variable): def formula(person, period, parameters): income = person("adjusted_net_income", period) - # Add noise to incomes in order to avoid ties + + # Use random draw from dataset to break ties (only exists in microsimulation) + # For policy calculator, this will be 0 and ties go to first person + random_draw = person("higher_earner_tie_break", period) return ( - person.get_rank(person.benunit, -income + person("is_higher_earner_seed", period) * 1e-2) - == 0 + person.get_rank(person.benunit, -income + random_draw * 1e-2) == 0 ) diff --git a/policyengine_uk/variables/household/demographic/is_higher_earner_seed.py b/policyengine_uk/variables/household/demographic/is_higher_earner_seed.py deleted file mode 100644 index f1f0a6aa1..000000000 --- a/policyengine_uk/variables/household/demographic/is_higher_earner_seed.py +++ /dev/null @@ -1,8 +0,0 @@ -from policyengine_uk.model_api import * - - -class is_higher_earner_seed(Variable): - value_type = float - entity = Person - label = "Random seed for tie-breaking in higher earner determination" - definition_period = YEAR