Add income-scaled match tolerance to compare (--rel-tolerance) (#1061)

* Add income-scaled match tolerance to compare (--rel-tolerance)

The flat $15 absolute match tolerance flags negligible tax differences on
extreme-magnitude records (large S-corp income/losses), where PE and TAXSIM
agree to a tiny fraction of income. Add an optional income-scaled tolerance:
a record matches within max($15, rel-tolerance * |AGI|). Default 0.0 preserves
the absolute-only behavior, so nothing is hidden by default.

On the eCPS this reveals the true alignment (2025: federal 84% -> 97.7%,
state 82% -> 90.6% at 0.1% of AGI); the genuine remaining diffs are S-corp
loss/passive treatment (federal) and a per-state tail.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* Apply ruff format

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

Author: PavelMakarchuk

changelog.d/add-income-scaled-tolerance.added.md

@@ -0,0 +1 @@
+Add an income-scaled match tolerance option (--rel-tolerance) to the compare command so negligible tax differences on extreme-magnitude records (e.g. large S-corp income/losses) are not flagged as mismatches; default preserves the flat $15 absolute tolerance.

policyengine_taxsim/cli.py

@@ -309,7 +309,28 @@ def taxsim(input_file, output, sample, taxsim_path):
     default=False,
     help="Assume large W-2 wages for QBID (aligns with TAXSIM S-Corp handling)",
 )
-def compare(input_file, sample, output_dir, year, disable_salt, logs, assume_w2_wages):
+@click.option(
+    "--rel-tolerance",
+    type=float,
+    default=0.0,
+    help=(
+        "Income-scaled match tolerance as a fraction of |AGI| (e.g. 0.001 = "
+        "0.1%). A record matches if the tax difference is within "
+        "max($15, rel-tolerance * |AGI|), avoiding false mismatches on "
+        "extreme-magnitude records (e.g. large S-corp income/losses). "
+        "Default 0 uses the flat $15 absolute tolerance."
+    ),
+)
+def compare(
+    input_file,
+    sample,
+    output_dir,
+    year,
+    disable_salt,
+    logs,
+    assume_w2_wages,
+    rel_tolerance,
+):
     """Compare PolicyEngine and TAXSIM results"""
     try:
         # Load and optionally sample data
@@ -365,7 +386,15 @@ def compare(input_file, sample, output_dir, year, disable_salt, logs, assume_w2_
 
         # Compare results
         click.echo("Comparing results...")
-        config = ComparisonConfig(federal_tolerance=15.0, state_tolerance=15.0)
+        config = ComparisonConfig(
+            federal_tolerance=15.0,
+            state_tolerance=15.0,
+            relative_tolerance=rel_tolerance,
+        )
+        if rel_tolerance > 0:
+            click.echo(
+                f"Using income-scaled tolerance: max($15, {rel_tolerance:.3%} of |AGI|)"
+            )
 
         comparator = TaxComparator(taxsim_results, pe_results, config)
         comparison_results = comparator.compare()

policyengine_taxsim/comparison/comparator.py

@@ -18,6 +18,14 @@ class ComparisonConfig:
     state_tax_col: str = "siitax"
     federal_tolerance: float = 15.0  # $15 absolute tolerance
     state_tolerance: float = 15.0  # $15 absolute tolerance
+    # Optional income-scaled tolerance: a record matches if the tax difference
+    # is within max(absolute_tolerance, relative_tolerance * |AGI|). This avoids
+    # flagging negligible differences on extreme-magnitude records (e.g. large
+    # S-corp income/losses), where PE and TAXSIM agree to a tiny fraction of
+    # income but exceed a flat $15. Default 0.0 preserves the absolute-only
+    # behavior. `agi_col` is the column used for scaling (federal AGI, v10).
+    relative_tolerance: float = 0.0
+    agi_col: str = "v10"
     id_col: str = "taxsimid"
 
 
@@ -133,11 +141,22 @@ def _compare_tax_column(
             # Return empty results if column doesn't exist
             return np.array([]), mismatches
 
-        # Compare with tolerance
+        # Effective per-record tolerance: absolute, optionally raised to an
+        # income-scaled floor (relative_tolerance * |AGI|) so negligible
+        # differences at extreme income magnitudes are not flagged.
+        eff_tol = tolerance
+        rel = getattr(self.config, "relative_tolerance", 0.0) or 0.0
+        if rel > 0 and self.config.agi_col in self.taxsim_results.columns:
+            agi = self.taxsim_results[self.config.agi_col].abs().fillna(0)
+            eff_tol = np.maximum(tolerance, rel * agi.to_numpy())
+
+        # Compare with tolerance. np.isclose applies atol elementwise when atol
+        # is an array, giving the per-record income-scaled tolerance.
         matches = np.isclose(
             self.taxsim_results[col_name],
             self.policyengine_results[col_name],
-            atol=tolerance,
+            atol=eff_tol,
+            rtol=0.0,
             equal_nan=True,
         )
 

tests/test_income_scaled_tolerance.py

@@ -0,0 +1,56 @@
+"""
+Tests for the income-scaled match tolerance in the comparator.
+
+The flat $15 absolute tolerance flags negligible differences on
+extreme-magnitude records (e.g. large S-corp income/losses), where PE and
+TAXSIM agree to a tiny fraction of income. `relative_tolerance` lets a record
+match within max($15, relative_tolerance * |AGI|). Default 0.0 preserves the
+absolute-only behavior.
+"""
+
+import pandas as pd
+
+from policyengine_taxsim.comparison.comparator import (
+    TaxComparator,
+    ComparisonConfig,
+)
+
+
+def _frames(taxsimid, agi, fiitax_t, fiitax_p):
+    tx = pd.DataFrame(
+        {"taxsimid": taxsimid, "v10": agi, "fiitax": fiitax_t, "siitax": 0.0}
+    )
+    pe = pd.DataFrame(
+        {"taxsimid": taxsimid, "v10": agi, "fiitax": fiitax_p, "siitax": 0.0}
+    )
+    return tx, pe
+
+
+def test_default_absolute_tolerance_flags_large_dollar_diff():
+    """With default (absolute-only) tolerance, a $500 diff is a mismatch."""
+    tx, pe = _frames([1], [10_000_000], [1_000_000], [1_000_500])
+    r = TaxComparator(tx, pe, ComparisonConfig()).compare()
+    assert r.federal_match_percentage == 0.0
+
+
+def test_income_scaled_tolerance_absorbs_negligible_diff():
+    """A $500 diff on $10M AGI is 0.005% — within a 0.1% income-scaled
+    tolerance, so it matches; a genuinely large diff still fails."""
+    # negligible: $500 on $10M
+    tx, pe = _frames([1], [10_000_000], [1_000_000], [1_000_500])
+    r = TaxComparator(tx, pe, ComparisonConfig(relative_tolerance=0.001)).compare()
+    assert r.federal_match_percentage == 100.0
+
+    # material: $50k diff on $10M (0.5%) exceeds the 0.1% floor -> mismatch
+    tx2, pe2 = _frames([1], [10_000_000], [1_000_000], [1_050_000])
+    r2 = TaxComparator(tx2, pe2, ComparisonConfig(relative_tolerance=0.001)).compare()
+    assert r2.federal_match_percentage == 0.0
+
+
+def test_absolute_floor_preserved_for_small_income():
+    """The $15 absolute floor still applies for low-AGI records (the scaled
+    tolerance never lowers it below $15)."""
+    # $20 diff on $1,000 AGI: 0.1% of AGI = $1, so floor stays $15 -> mismatch
+    tx, pe = _frames([1], [1_000], [100], [120])
+    r = TaxComparator(tx, pe, ComparisonConfig(relative_tolerance=0.001)).compare()
+    assert r.federal_match_percentage == 0.0