Avoid overlapping state AGI top-tail targets

Author: MaxGhenis

policyengine_us_data/db/etl_irs_soi.py

@@ -78,6 +78,19 @@
     28: (10_000_000, np.inf),  # row 28
 }
 
+
+def _skip_coarse_state_agi_person_count_target(geo_type: str, agi_stub: int) -> bool:
+    """Skip the coarse state 500k+ count target when fine state bins are loaded.
+
+    The standard geography-file SOI feed only has a top-coded state AGI stub 9
+    (500k+). We separately load `in55cmcsv`, which splits that state tail into
+    500k-1m and 1m+. Keeping the coarse state count target alongside the fine
+    rows would double-constrain the same top-tail population in calibration.
+    """
+
+    return geo_type == "state" and agi_stub == 9
+
+
 # These variables map cleanly from Publication 1304 aggregate tables to the
 # existing national IRS-SOI domain strata. We intentionally leave `aca_ptc`
 # and `refundable_ctc` on the geography-file path for now because the
@@ -1244,6 +1257,9 @@ def load_soi_data(long_dfs, year, national_year: Optional[int] = None):
             geo_info = parse_ucgid(ucgid_i)
             person_count = agi_df.iloc[i][["target_value"]].values[0]
 
+            if _skip_coarse_state_agi_person_count_target(geo_info["type"], agi_stub):
+                continue
+
             if geo_info["type"] == "state":
                 parent_stratum_id = filer_strata["state"][geo_info["state_fips"]]
                 note = f"State FIPS {geo_info['state_fips']} filers, AGI >= {agi_income_lower}, AGI < {agi_income_upper}"

tests/unit/test_etl_irs_soi_overlay.py

@@ -11,6 +11,7 @@
     create_database,
 )
 from policyengine_us_data.db.etl_irs_soi import (
+    _skip_coarse_state_agi_person_count_target,
     _get_or_create_national_domain_stratum,
     _upsert_target,
     load_national_workbook_soi_targets,
@@ -180,3 +181,10 @@ def fake_get_tracked_soi_row(variable, requested_year, **kwargs):
     assert len(count_rows) == 1
     assert int(count_rows.iloc[0]["period"]) == 2023
     assert float(count_rows.iloc[0]["value"]) == 50.0
+
+
+def test_skip_coarse_state_agi_person_count_target_only_for_state_stub_9():
+    assert _skip_coarse_state_agi_person_count_target("state", 9) is True
+    assert _skip_coarse_state_agi_person_count_target("state", 8) is False
+    assert _skip_coarse_state_agi_person_count_target("district", 9) is False
+    assert _skip_coarse_state_agi_person_count_target("national", 9) is False