diff --git a/changelog.d/add-srebate-net-of-rebates.added.md b/changelog.d/add-srebate-net-of-rebates.added.md new file mode 100644 index 0000000..5cb4722 --- /dev/null +++ b/changelog.d/add-srebate-net-of-rebates.added.md @@ -0,0 +1 @@ +Emit PolicyEngine one-time state rebates in the srebate output column and add a --net-of-rebates compare option that scores state tax as siitax plus srebate on both sides, removing the TAXSIM payout-year vs PolicyEngine liability-year rebate timing difference. diff --git a/policyengine_taxsim/cli.py b/policyengine_taxsim/cli.py index bba6358..bc6a2e6 100644 --- a/policyengine_taxsim/cli.py +++ b/policyengine_taxsim/cli.py @@ -321,6 +321,18 @@ def taxsim(input_file, output, sample, taxsim_path): "Default 0 uses the flat $15 absolute tolerance." ), ) +@click.option( + "--net-of-rebates", + is_flag=True, + default=False, + help=( + "Score state tax net of one-time rebates: compare siitax + srebate " + "on both sides. Removes the timing-convention difference between " + "TAXSIM (rebates in the payout year) and PolicyEngine (rebates in " + "the liability year) without changing either engine. Federal " + "comparison is unaffected." + ), +) def compare( input_file, sample, @@ -330,6 +342,7 @@ def compare( logs, assume_w2_wages, rel_tolerance, + net_of_rebates, ): """Compare PolicyEngine and TAXSIM results""" try: @@ -390,11 +403,17 @@ def compare( federal_tolerance=15.0, state_tolerance=15.0, relative_tolerance=rel_tolerance, + net_of_rebates=net_of_rebates, ) if rel_tolerance > 0: click.echo( f"Using income-scaled tolerance: max($15, {rel_tolerance:.3%} of |AGI|)" ) + if net_of_rebates: + click.echo( + "Scoring state tax net of one-time rebates (siitax + srebate " + "on both sides)" + ) comparator = TaxComparator(taxsim_results, pe_results, config) comparison_results = comparator.compare() diff --git a/policyengine_taxsim/comparison/comparator.py b/policyengine_taxsim/comparison/comparator.py index 072c79f..4c512cd 100644 --- a/policyengine_taxsim/comparison/comparator.py +++ b/policyengine_taxsim/comparison/comparator.py @@ -26,6 +26,15 @@ class ComparisonConfig: # behavior. `agi_col` is the column used for scaling (federal AGI, v10). relative_tolerance: float = 0.0 agi_col: str = "v10" + # Score state tax net of one-time rebates: compare siitax + srebate on + # each side (missing/NaN srebate treated as 0). TAXSIM includes one-time + # rebates in siitax in the payout year and reports them in `srebate`; + # PE books them to the liability year inside state_income_tax. Adding + # srebate back on both sides removes this timing-convention difference + # without changing either engine (see taxsim #1068, convention #716). + # Federal comparison is unaffected. Composable with relative_tolerance. + net_of_rebates: bool = False + rebate_col: str = "srebate" id_col: str = "taxsimid" @@ -150,11 +159,20 @@ def _compare_tax_column( agi = self.taxsim_results[self.config.agi_col].abs().fillna(0) eff_tol = np.maximum(tolerance, rel * agi.to_numpy()) + # Values to compare. For the state column with net_of_rebates, score + # siitax + srebate on each side so TAXSIM's payout-year rebate + # convention and PE's liability-year convention cancel out. + taxsim_values = self.taxsim_results[col_name] + pe_values = self.policyengine_results[col_name] + if tax_type == "state" and getattr(self.config, "net_of_rebates", False): + taxsim_values = taxsim_values + self._rebate_values(self.taxsim_results) + pe_values = pe_values + self._rebate_values(self.policyengine_results) + # Compare with tolerance. np.isclose applies atol elementwise when atol # is an array, giving the per-record income-scaled tolerance. matches = np.isclose( - self.taxsim_results[col_name], - self.policyengine_results[col_name], + taxsim_values, + pe_values, atol=eff_tol, rtol=0.0, equal_nan=True, @@ -163,8 +181,8 @@ def _compare_tax_column( # Collect mismatches for i, is_match in enumerate(matches): if not is_match: - taxsim_val = self.taxsim_results.iloc[i][col_name] - pe_val = self.policyengine_results.iloc[i][col_name] + taxsim_val = taxsim_values.iloc[i] + pe_val = pe_values.iloc[i] mismatch = MismatchRecord( taxsimid=self.taxsim_results.iloc[i][self.config.id_col], @@ -182,6 +200,13 @@ def _compare_tax_column( return matches, mismatches + def _rebate_values(self, df: pd.DataFrame) -> pd.Series: + """One-time rebate column as a numeric Series (missing/NaN -> 0).""" + rebate_col = getattr(self.config, "rebate_col", "srebate") + if rebate_col in df.columns: + return pd.to_numeric(df[rebate_col], errors="coerce").fillna(0.0) + return pd.Series(0.0, index=df.index) + def compare(self) -> "ComparisonResults": """Perform full comparison between TAXSIM and PolicyEngine results""" print("Comparing TAXSIM and PolicyEngine results") diff --git a/policyengine_taxsim/config/variable_mappings.yaml b/policyengine_taxsim/config/variable_mappings.yaml index cdf7ebd..51c60d5 100644 --- a/policyengine_taxsim/config/variable_mappings.yaml +++ b/policyengine_taxsim/config/variable_mappings.yaml @@ -551,6 +551,23 @@ policyengine_to_taxsim: group_order: 4 full_text_group: "State Tax Calculation" group_column: 1 + srebate: + # One-time state rebates that PE nets into state_income_tax for the + # year. TAXSIM reports these in `srebate` (payout-year convention, + # included in siitax); PE books them to the liability year inside + # state_income_tax. Computed as a difference — state_income_tax with the + # ONE_TIME_REBATE_VARIABLES (core/state_output_resolver.py) zeroed minus + # actual state_income_tax — so it equals exactly what PE netted into + # siitax. See taxsim #1068 / convention #716. + variable: srebate_computed + implemented: true + idtl: + - full: 2 + - full_text: 5 + text_description: "State One-Time Rebate" + group_order: 4 + full_text_group: "State Tax Calculation" + group_column: 1 v41: variable: na_pe implemented: false diff --git a/policyengine_taxsim/core/output_mapper.py b/policyengine_taxsim/core/output_mapper.py index 9f8437b..7b03ed2 100644 --- a/policyengine_taxsim/core/output_mapper.py +++ b/policyengine_taxsim/core/output_mapper.py @@ -4,6 +4,7 @@ to_roundedup_number, ) from .state_output_resolver import ( + ONE_TIME_REBATE_VARIABLES, calculate_output_adapter, get_state_mapped_variables, get_state_specific_variable_name, @@ -17,6 +18,35 @@ disable_salt_variable = False +def compute_srebate_single(simulation, year): + """One-time state rebates PE netted into state_income_tax this year. + + Computed as a difference: a twin simulation with the one-time rebate + variables forced to zero minus the actual simulation. This equals + exactly the rebate amount included in siitax (non-refundable caps, + credit pooling, and floors are handled automatically). TAXSIM reports + the same concept in `srebate` under its payout-year convention. + """ + try: + twin = Simulation(situation=simulation.situation_input) + if disable_salt_variable: + twin.set_input( + variable_name="state_and_local_sales_or_income_tax", + value=0.0, + period=year, + ) + for variable in ONE_TIME_REBATE_VARIABLES: + if twin.tax_benefit_system.variables.get(variable) is None: + continue + twin.set_input(variable_name=variable, value=0.0, period=year) + + actual_tax = float(simulation.calculate("state_income_tax", period=year)[0]) + rebate_free_tax = float(twin.calculate("state_income_tax", period=year)[0]) + return to_roundedup_number(rebate_free_tax - actual_tax) + except Exception: + return 0.00 + + def resolve_output_variable(simulation, variable, state_name): if simulation.tax_benefit_system.variables.get(variable) is not None: return variable @@ -59,6 +89,10 @@ def generate_non_description_output( for entry in each_item["idtl"]: if output_type in entry.values(): taxsim_output[key] = mtr_results.get(key, 0.0) + elif each_item.get("variable") == "srebate_computed": + for entry in each_item["idtl"]: + if output_type in entry.values(): + taxsim_output[key] = compute_srebate_single(simulation, year) elif is_output_adapter(each_item.get("variable", "")): for entry in each_item["idtl"]: if output_type in entry.values(): @@ -186,6 +220,8 @@ def generate_text_description_output( mtr_results = {"frate": 0.0, "srate": 0.0, "ficar": 0.0} mtr_computed = True value = mtr_results.get(var_name, 0.0) + elif each_item.get("variable") == "srebate_computed": + value = compute_srebate_single(simulation, year) elif is_output_adapter(each_item.get("variable", "")): value = to_roundedup_number( calculate_output_adapter( diff --git a/policyengine_taxsim/core/state_output_resolver.py b/policyengine_taxsim/core/state_output_resolver.py index 5f1fe18..b916977 100644 --- a/policyengine_taxsim/core/state_output_resolver.py +++ b/policyengine_taxsim/core/state_output_resolver.py @@ -13,6 +13,37 @@ "taxsim_v36_taxable_income", "taxsim_v37_property_tax_credit", } +# PE-US one-time state rebate variables (tax_unit level). TAXSIM reports +# these in `srebate` (payout-year convention, included in siitax); PE books +# them to the liability year inside state_income_tax. The emulator's srebate +# is computed as a difference — state_income_tax with these zeroed minus +# actual state_income_tax — so it equals exactly the amount PE netted into +# siitax (wiring, non-refundable caps, and floors are handled automatically). +# Only one-time rebates belong here; recurring annual rebates (e.g. +# nm_low_income_comprehensive_tax_rebate, nm_property_tax_rebate, +# pa_property_tax_or_rent_rebate, mt_property_tax_rebate) are excluded. +# See taxsim #1068 / convention #716. +ONE_TIME_REBATE_VARIABLES = ( + "az_families_tax_rebate", + "co_tabor_cash_back", + "ct_child_tax_rebate", + "de_relief_rebate", + "ga_surplus_tax_rebate", + "hi_act_115_rebate", + "id_2022_rebate", + "id_special_season_rebate", + "il_income_tax_rebate", + "in_automatic_refund_rebate", + "ma_taxpayer_refund_rebate", + "me_relief_rebate", + "mt_income_tax_rebate", + "nm_2021_income_rebate", + "nm_additional_2021_income_rebate", + "nm_supplemental_2021_income_rebate", + "ri_child_tax_rebate", + "sc_2022_rebate", + "va_rebate", +) OUTPUT_ADAPTER_OVERRIDES = { # MT: state_agi reads `gov.states.household.state_agis` which lists # `mt_agi_indiv` (Person, defined only for MFS-on-same-return). For all diff --git a/policyengine_taxsim/runners/policyengine_runner.py b/policyengine_taxsim/runners/policyengine_runner.py index 5afa45f..d4ec425 100644 --- a/policyengine_taxsim/runners/policyengine_runner.py +++ b/policyengine_taxsim/runners/policyengine_runner.py @@ -18,6 +18,7 @@ convert_taxsim32_dependents, ) from policyengine_taxsim.core.state_output_resolver import ( + ONE_TIME_REBATE_VARIABLES, calculate_output_adapter, calculate_state_mapped_output, get_state_specific_variable_name, @@ -1059,133 +1060,173 @@ def _run_chunk(self, chunk_df: pd.DataFrame) -> pd.DataFrame: try: dataset.generate() - sim = Microsimulation(dataset=dataset) - - # Resolve the state_and_local_sales_or_income_tax override for - # this chunk. Possible sources, in priority order: - # 1. self._state_tax_override (Pass B of three-pass: per-row - # values produced by Pass A, keyed by taxsimid) - # 2. self.disable_salt (zero out for state-only computation) - salt_override = None - if self._state_tax_override is not None: - ids = chunk_df["taxsimid"].astype(float).astype(int).values - # Look each id up in the override map; fall back to 0 if - # the id is unexpectedly missing. - salt_override = np.array( - [self._state_tax_override.get(int(i), 0.0) for i in ids], - dtype=float, + sim = self._build_configured_sim(dataset, chunk_df) + + def rebate_free_sim_factory(): + # Twin sim, identically configured, with the one-time state + # rebate variables forced to zero (set_input before any + # calculate). state_income_tax(twin) - state_income_tax(sim) + # is exactly the rebate amount PE netted into siitax. + twin = self._build_configured_sim(dataset, chunk_df) + self._zero_one_time_rebates(twin, chunk_df) + return twin + + return self._extract_vectorized_results( + sim, chunk_df, rebate_free_sim_factory + ) + + finally: + dataset.cleanup() + + def _build_configured_sim(self, dataset, chunk_df: pd.DataFrame): + """Build a Microsimulation from the chunk dataset and apply all + emulator overrides (SALT, QBID W-2 wages, rental QBID gate, MN CRP, + imputed-transfer zeroing, MD local tax zeroing).""" + sim = Microsimulation(dataset=dataset) + + # Resolve the state_and_local_sales_or_income_tax override for + # this chunk. Possible sources, in priority order: + # 1. self._state_tax_override (Pass B of three-pass: per-row + # values produced by Pass A, keyed by taxsimid) + # 2. self.disable_salt (zero out for state-only computation) + salt_override = None + if self._state_tax_override is not None: + ids = chunk_df["taxsimid"].astype(float).astype(int).values + # Look each id up in the override map; fall back to 0 if + # the id is unexpectedly missing. + salt_override = np.array( + [self._state_tax_override.get(int(i), 0.0) for i in ids], + dtype=float, + ) + elif self.disable_salt: + salt_override = np.zeros(len(chunk_df), dtype=float) + + if salt_override is not None: + years = sorted(set(chunk_df["year"].unique())) + for year in years: + year_mask = chunk_df["year"] == year + year_values = salt_override[year_mask.values] + sim.set_input( + variable_name="state_and_local_sales_or_income_tax", + value=year_values, + period=str( + int(year) if isinstance(year, (float, np.floating)) else year + ), ) - elif self.disable_salt: - salt_override = np.zeros(len(chunk_df), dtype=float) - if salt_override is not None: - years = sorted(set(chunk_df["year"].unique())) - for year in years: - year_mask = chunk_df["year"] == year - year_values = salt_override[year_mask.values] - sim.set_input( - variable_name="state_and_local_sales_or_income_tax", - value=year_values, - period=str( - int(year) - if isinstance(year, (float, np.floating)) - else year - ), - ) + if self.assume_w2_wages: + n_persons = sim.get_variable_population( + "w2_wages_from_qualified_business" + ).count + years = sorted(set(chunk_df["year"].unique())) + for year in years: + sim.set_input( + variable_name="w2_wages_from_qualified_business", + value=np.full(n_persons, 1e9), + period=str( + int(year) if isinstance(year, (float, np.floating)) else year + ), + ) - if self.assume_w2_wages: - n_persons = sim.get_variable_population( - "w2_wages_from_qualified_business" - ).count - years = sorted(set(chunk_df["year"].unique())) - for year in years: - sim.set_input( - variable_name="w2_wages_from_qualified_business", - value=np.full(n_persons, 1e9), - period=str( - int(year) - if isinstance(year, (float, np.floating)) - else year - ), - ) + # QBID gate on rental_income (TAXSIM `otherprop`): TAXSIM only + # triggers § 199A QBID via the explicit `pbusinc` input and never + # treats `otherprop` (Schedule E passive rents/royalties) as + # qualified. PE-US's `rental_income_would_be_qualified` defaults + # to True, which would generate a 20% QBID on emulator-routed + # rental income that TAXSIM does not produce. § 199A(c)(3)(A) + # requires the activity to rise to a § 162 trade or business; + # passive individual rentals generally do not qualify absent the + # § 1.199A-1(b)(14) safe harbor, which TAXSIM input never + # signals. We force the gate off for the whole chunk: every + # rental_income value in PE here originated as `otherprop` from + # TAXSIM, so the override has no side effect on non-emulated + # rental income. + if "otherprop" in chunk_df.columns and (chunk_df["otherprop"] != 0).any(): + n_persons = sim.get_variable_population( + "rental_income_would_be_qualified" + ).count + years = sorted(set(chunk_df["year"].unique())) + for year in years: + sim.set_input( + variable_name="rental_income_would_be_qualified", + value=np.zeros(n_persons, dtype=bool), + period=str( + int(year) if isinstance(year, (float, np.floating)) else year + ), + ) - # QBID gate on rental_income (TAXSIM `otherprop`): TAXSIM only - # triggers § 199A QBID via the explicit `pbusinc` input and never - # treats `otherprop` (Schedule E passive rents/royalties) as - # qualified. PE-US's `rental_income_would_be_qualified` defaults - # to True, which would generate a 20% QBID on emulator-routed - # rental income that TAXSIM does not produce. § 199A(c)(3)(A) - # requires the activity to rise to a § 162 trade or business; - # passive individual rentals generally do not qualify absent the - # § 1.199A-1(b)(14) safe harbor, which TAXSIM input never - # signals. We force the gate off for the whole chunk: every - # rental_income value in PE here originated as `otherprop` from - # TAXSIM, so the override has no side effect on non-emulated - # rental income. - if "otherprop" in chunk_df.columns and (chunk_df["otherprop"] != 0).any(): - n_persons = sim.get_variable_population( - "rental_income_would_be_qualified" - ).count + # MN Renter's Credit: PE-US gates the credit on a Certificate + # of Rent Paid (CRP) input variable that defaults to False. Per + # Minn. Stat. § 290.0693, the credit is allowed for renters who + # paid rent and meet income criteria; subd. 4 places the CRP + # issuance obligation on the landlord (not the renter), and + # subd. 9 accepts proof "including but not limited to" the CRP. + # TAXSIM input never carries CRP info, so we assume any MN + # tax unit with rent > 0 has the supporting documentation. + if "rentpaid" in chunk_df.columns and "state" in chunk_df.columns: + mn_mask = (chunk_df["state"] == 24) & (chunk_df["rentpaid"] > 0) + if mn_mask.any(): years = sorted(set(chunk_df["year"].unique())) for year in years: - sim.set_input( - variable_name="rental_income_would_be_qualified", - value=np.zeros(n_persons, dtype=bool), - period=str( - int(year) - if isinstance(year, (float, np.floating)) - else year - ), - ) + year_mask = (chunk_df["year"] == year) & mn_mask + if year_mask.any(): + # qualifying_crp is on TaxUnit; vector aligns + # with the chunk's tax-unit order, one row per + # tax unit. + sim.set_input( + variable_name="mn_renters_credit_qualifying_crp", + value=mn_mask[chunk_df["year"] == year].values, + period=str( + int(year) + if isinstance(year, (float, np.floating)) + else year + ), + ) - # MN Renter's Credit: PE-US gates the credit on a Certificate - # of Rent Paid (CRP) input variable that defaults to False. Per - # Minn. Stat. § 290.0693, the credit is allowed for renters who - # paid rent and meet income criteria; subd. 4 places the CRP - # issuance obligation on the landlord (not the renter), and - # subd. 9 accepts proof "including but not limited to" the CRP. - # TAXSIM input never carries CRP info, so we assume any MN - # tax unit with rent > 0 has the supporting documentation. - if "rentpaid" in chunk_df.columns and "state" in chunk_df.columns: - mn_mask = (chunk_df["state"] == 24) & (chunk_df["rentpaid"] > 0) - if mn_mask.any(): - years = sorted(set(chunk_df["year"].unique())) - for year in years: - year_mask = (chunk_df["year"] == year) & mn_mask - if year_mask.any(): - # qualifying_crp is on TaxUnit; vector aligns - # with the chunk's tax-unit order, one row per - # tax unit. - sim.set_input( - variable_name="mn_renters_credit_qualifying_crp", - value=mn_mask[chunk_df["year"] == year].values, - period=str( - int(year) - if isinstance(year, (float, np.floating)) - else year - ), - ) + # PE imputes means-tested transfers (SSI, SNAP, TANF, WIC, and + # state SSI supplements) from the microsimulation's low-income + # records. TAXSIM has no input columns for any of these, so they + # must be zeroed to match TAXSIM's transfer-free world — otherwise + # PE's imputed benefits leak into state calculations that count + # cash public assistance as income (e.g. the MA Senior Circuit + # Breaker base, MGL c.62 §6(k); taxsim #1031). These are + # formula-based benefit variables, so the dataset-level zeroing is + # silently recomputed by the Microsimulation; they must be forced + # off with set_input after the sim is built (same mechanism as the + # overrides above), which set_input holds as a fixed input. + years = sorted(set(chunk_df["year"].unique())) + for var in _ZERO_IMPUTED_TRANSFERS: + if var not in sim.tax_benefit_system.variables: + continue + n_entities = sim.get_variable_population(var).count + for year in years: + sim.set_input( + variable_name=var, + value=np.zeros(n_entities), + period=str( + int(year) if isinstance(year, (float, np.floating)) else year + ), + ) - # PE imputes means-tested transfers (SSI, SNAP, TANF, WIC, and - # state SSI supplements) from the microsimulation's low-income - # records. TAXSIM has no input columns for any of these, so they - # must be zeroed to match TAXSIM's transfer-free world — otherwise - # PE's imputed benefits leak into state calculations that count - # cash public assistance as income (e.g. the MA Senior Circuit - # Breaker base, MGL c.62 §6(k); taxsim #1031). These are - # formula-based benefit variables, so the dataset-level zeroing is - # silently recomputed by the Microsimulation; they must be forced - # off with set_input after the sim is built (same mechanism as the - # overrides above), which set_input holds as a fixed input. - years = sorted(set(chunk_df["year"].unique())) - for var in _ZERO_IMPUTED_TRANSFERS: - if var not in sim.tax_benefit_system.variables: - continue - n_entities = sim.get_variable_population(var).count + # Maryland county/local income tax: TAXSIM's MD `siitax` is + # state-only — it applies no county tax when the input carries no + # locality (verified against the binary: TAXSIM MD siitax equals + # PE's state-only MD tax to the dollar). PE, by contrast, applies + # MD's *residence-based* county tax (~2.25-3.20%) to every MD + # resident even with no county specified, systematically + # over-stating MD siitax vs TAXSIM. The TAXSIM input has no county, + # so zero PE's MD local income tax to match TAXSIM's coverage. MD + # is the only state affected: every other local-income-tax state + # (OH/PA/IN/KY/MI/NYC/…) requires a locality PE isn't given, so + # those already compute $0 local and match TAXSIM. + if "state" in chunk_df.columns and (chunk_df["state"] == 21).any(): + var = "md_local_income_tax_before_credits" + if var in sim.tax_benefit_system.variables: + n_md = sim.get_variable_population(var).count for year in years: sim.set_input( variable_name=var, - value=np.zeros(n_entities), + value=np.zeros(n_md), period=str( int(year) if isinstance(year, (float, np.floating)) @@ -1193,36 +1234,25 @@ def _run_chunk(self, chunk_df: pd.DataFrame) -> pd.DataFrame: ), ) - # Maryland county/local income tax: TAXSIM's MD `siitax` is - # state-only — it applies no county tax when the input carries no - # locality (verified against the binary: TAXSIM MD siitax equals - # PE's state-only MD tax to the dollar). PE, by contrast, applies - # MD's *residence-based* county tax (~2.25-3.20%) to every MD - # resident even with no county specified, systematically - # over-stating MD siitax vs TAXSIM. The TAXSIM input has no county, - # so zero PE's MD local income tax to match TAXSIM's coverage. MD - # is the only state affected: every other local-income-tax state - # (OH/PA/IN/KY/MI/NYC/…) requires a locality PE isn't given, so - # those already compute $0 local and match TAXSIM. - if "state" in chunk_df.columns and (chunk_df["state"] == 21).any(): - var = "md_local_income_tax_before_credits" - if var in sim.tax_benefit_system.variables: - n_md = sim.get_variable_population(var).count - for year in years: - sim.set_input( - variable_name=var, - value=np.zeros(n_md), - period=str( - int(year) - if isinstance(year, (float, np.floating)) - else year - ), - ) - - return self._extract_vectorized_results(sim, chunk_df) + return sim - finally: - dataset.cleanup() + def _zero_one_time_rebates(self, sim, chunk_df: pd.DataFrame) -> None: + """Force the one-time state rebate variables to zero. These are + formula variables, so they must be pinned with set_input before any + calculate on the sim (same mechanism as the transfer zeroing).""" + years = sorted(set(chunk_df["year"].unique())) + for var in ONE_TIME_REBATE_VARIABLES: + if var not in sim.tax_benefit_system.variables: + continue + n_entities = sim.get_variable_population(var).count + for year in years: + sim.set_input( + variable_name=var, + value=np.zeros(n_entities), + period=str( + int(year) if isinstance(year, (float, np.floating)) else year + ), + ) # Columns whose semantics belong to the state-side of PE-US. When # --disable-salt is set, we run PE twice: a full-SALT pass for the @@ -1448,17 +1478,25 @@ def _compute_marginal_rates(self, sim, year_str, year_data): } def _extract_vectorized_results( - self, sim: Microsimulation, input_df: pd.DataFrame + self, + sim: Microsimulation, + input_df: pd.DataFrame, + rebate_free_sim_factory=None, ) -> pd.DataFrame: """Extract results from Microsimulation and format as TAXSIM output. Uses vectorized sim.calculate() calls (one per variable, not per row) and builds the output DataFrame from arrays directly. + + ``rebate_free_sim_factory`` lazily builds a twin sim with the + one-time state rebate variables zeroed; the srebate output is the + state_income_tax difference between the twin and the actual sim. """ input_df = self._ensure_required_columns(input_df) pe_to_taxsim = self.mappings["policyengine_to_taxsim"] years = sorted(set(input_df["year"].unique())) year_frames = [] + rebate_free_sim = None for year in tqdm(years, desc="Processing years"): year_str = str(year) @@ -1535,6 +1573,23 @@ def _extract_vectorized_results( # Marginal rates are computed specially after this loop continue + if pe_var == "srebate_computed": + # srebate = state_income_tax with one-time rebates + # zeroed minus actual state_income_tax — exactly the + # amount PE netted into siitax this year (handles + # non-refundable caps, pooling, and floors). + if rebate_free_sim_factory is None: + columns[taxsim_var] = np.zeros(n) + continue + if rebate_free_sim is None: + rebate_free_sim = rebate_free_sim_factory() + actual_tax = self._calc_tax_unit(sim, "state_income_tax", year_str) + rebate_free_tax = self._calc_tax_unit( + rebate_free_sim, "state_income_tax", year_str + ) + columns[taxsim_var] = np.round(rebate_free_tax - actual_tax, 2) + continue + try: if is_output_adapter(pe_var): columns[taxsim_var] = np.round( diff --git a/tests/test_net_of_rebates.py b/tests/test_net_of_rebates.py new file mode 100644 index 0000000..318c3be --- /dev/null +++ b/tests/test_net_of_rebates.py @@ -0,0 +1,117 @@ +""" +Tests for rebate-aware state tax comparison (--net-of-rebates). + +TAXSIM includes one-time state rebates in `siitax` in the payout year and +reports them in the `srebate` output column; PolicyEngine books them to the +liability year inside `state_income_tax`. Scoring `siitax + srebate` on both +sides removes this timing-convention difference without changing either +engine (see taxsim #1068, convention #716). The emulator emits PE's one-time +rebates in `srebate` via the state_variables mapping in +variable_mappings.yaml. +""" + +import io + +import pandas as pd +from click.testing import CliRunner + +from policyengine_taxsim.cli import cli +from policyengine_taxsim.comparison.comparator import ( + TaxComparator, + ComparisonConfig, +) + + +def _frames(siitax_t, srebate_t, siitax_p, srebate_p): + ids = list(range(1, len(siitax_t) + 1)) + tx = pd.DataFrame( + { + "taxsimid": ids, + "fiitax": 100.0, + "siitax": siitax_t, + "srebate": srebate_t, + } + ) + pe = pd.DataFrame( + { + "taxsimid": ids, + "fiitax": 100.0, + "siitax": siitax_p, + "srebate": srebate_p, + } + ) + return tx, pe + + +def test_rebate_timing_difference_mismatches_by_default(): + """TAXSIM siitax includes a $250 payout-year rebate that PE booked to a + different year: raw siitax differs by $250 -> mismatch by default.""" + tx, pe = _frames([-500.0], [250.0], [-250.0], [0.0]) + r = TaxComparator(tx, pe, ComparisonConfig()).compare() + assert r.state_match_percentage == 0.0 + + +def test_net_of_rebates_matches_rebate_timing_difference(): + """Under net_of_rebates the same record matches: + -500 + 250 == -250 + 0.""" + tx, pe = _frames([-500.0], [250.0], [-250.0], [0.0]) + r = TaxComparator(tx, pe, ComparisonConfig(net_of_rebates=True)).compare() + assert r.state_match_percentage == 100.0 + + +def test_net_of_rebates_still_flags_real_differences(): + """net_of_rebates is not a blanket pass: a genuine $200 liability gap + remains a mismatch (-500 + 250 = -250 vs -50 + 0 = -50).""" + tx, pe = _frames([-500.0], [250.0], [-50.0], [0.0]) + r = TaxComparator(tx, pe, ComparisonConfig(net_of_rebates=True)).compare() + assert r.state_match_percentage == 0.0 + + +def test_net_of_rebates_treats_missing_srebate_as_zero(): + """A PE frame without an srebate column is treated as srebate=0.""" + tx, pe = _frames([-500.0], [250.0], [-250.0], [0.0]) + pe = pe.drop(columns=["srebate"]) + r = TaxComparator(tx, pe, ComparisonConfig(net_of_rebates=True)).compare() + assert r.state_match_percentage == 100.0 + + +def test_net_of_rebates_does_not_touch_federal(): + """The flag only adjusts the state comparison; a federal gap still + mismatches and a federal match still matches.""" + tx, pe = _frames([-500.0], [250.0], [-250.0], [0.0]) + tx["fiitax"] = 100.0 + pe["fiitax"] = 400.0 + r = TaxComparator(tx, pe, ComparisonConfig(net_of_rebates=True)).compare() + assert r.federal_match_percentage == 0.0 + assert r.state_match_percentage == 100.0 + + +def test_emulator_emits_me_2021_relief_rebate_in_srebate(): + """A ME 2021 record (SOI 20) should emit srebate ~= $850 + (me_relief_rebate), which PE books to the 2021 liability year. The same + record in 2022 should emit srebate = 0: PE's rebate variables can stay + nonzero after the one-time year, so srebate is gated by the years the + rebate sits in the state's credit lists (i.e. the years PE actually + nets it into siitax).""" + record = ( + "taxsimid,year,state,mstat,page,sage,depx,pwages,idtl\n" + "1,2021,20,1,40,0,0,30000,2\n" + "2,2022,20,1,40,0,0,30000,2\n" + ) + result = CliRunner().invoke(cli, [], input=record) + assert result.exit_code == 0, f"CLI failed: {result.output}\n{result.exception}" + out = result.output + idx = out.find("taxsimid,") + df = pd.read_csv(io.StringIO(out[idx:])) + # CliRunner interleaves progress lines with the CSV; keep data rows only. + df = df[pd.to_numeric(df["taxsimid"], errors="coerce").notna()] + df["taxsimid"] = df["taxsimid"].astype(float).astype(int) + df = df.set_index("taxsimid") + df["srebate"] = df["srebate"].astype(float) + assert abs(df.loc[1, "srebate"] - 850.0) <= 1.0, ( + f"ME 2021 srebate {df.loc[1, 'srebate']} != 850" + ) + assert df.loc[2, "srebate"] == 0.0, ( + f"ME 2022 srebate {df.loc[2, 'srebate']} should be 0 " + "(rebate booked to 2021 only)" + ) diff --git a/tests/test_performance.py b/tests/test_performance.py index 44db2bf..6c5333a 100644 --- a/tests/test_performance.py +++ b/tests/test_performance.py @@ -206,9 +206,9 @@ def test_extract_builds_dataframe_without_row_loop(self): orig_extract = runner._extract_vectorized_results.__func__ extract_time = {} - def timed_extract(self_runner, sim, input_df): + def timed_extract(self_runner, sim, input_df, *args, **kwargs): t0 = time.time() - result = orig_extract(self_runner, sim, input_df) + result = orig_extract(self_runner, sim, input_df, *args, **kwargs) extract_time["t"] = time.time() - t0 return result