Fix stage 1 validation contracts

Author: MaxGhenis

changelog.d/1004.fixed.md

@@ -0,0 +1 @@
+Fixed Stage 1 production validation checks for ACA PTC targets, structural computed export variables, and additive calibration target expressions.

validation/stage_1/aca_calibration.py

@@ -0,0 +1,56 @@
+"""Shared ACA PTC state-target validation helpers."""
+
+from __future__ import annotations
+
+import logging
+from typing import Callable
+
+import numpy as np
+import pytest
+
+from policyengine_us_data.storage.calibration_targets.aca_ptc_targets import (
+    load_aca_ptc_state_targets,
+)
+
+ACA_PTC_STATE_TOLERANCE = 10.0
+
+
+def assert_aca_ptc_calibration(
+    sim,
+    *,
+    period: int = 2025,
+    emit: Callable[[str], None] | None = None,
+) -> None:
+    """Check state ACA PTC totals against the IRS SOI total-PTC target."""
+    targets = load_aca_ptc_state_targets(period)
+    if targets is None:
+        pytest.skip("ACA PTC state targets not available")
+
+    emit = emit or logging.info
+    state_code_hh = sim.calculate("state_code", map_to="household").values
+    aca_ptc = sim.calculate("aca_ptc", map_to="household", period=period)
+
+    failures = []
+    for row in targets.itertuples(index=False):
+        state = row.state
+        target_spending = float(row.TotalPTCAmount)
+        simulated = float(aca_ptc[state_code_hh == state].sum())
+        if target_spending <= 0:
+            pct_error = np.inf
+        else:
+            pct_error = abs(simulated - target_spending) / target_spending
+
+        message = (
+            f"{state}: simulated ${simulated / 1e9:.2f} bn  "
+            f"target ${target_spending / 1e9:.2f} bn  "
+            f"error {pct_error:.2%}"
+        )
+        emit(message)
+
+        if pct_error > ACA_PTC_STATE_TOLERANCE:
+            failures.append(message)
+
+    assert not failures, (
+        "One or more states exceeded tolerance of "
+        f"{ACA_PTC_STATE_TOLERANCE:.0%}:\n" + "\n".join(failures)
+    )

validation/stage_1/test_enhanced_cps.py

@@ -278,50 +278,14 @@ def test_has_tin_matches_identification_inputs(ecps_sim):
 
 
 def test_aca_calibration():
-    import pandas as pd
-    from pathlib import Path
     from policyengine_us import Microsimulation
     from policyengine_us_data.datasets.cps import EnhancedCPS_2024
+    from validation.stage_1.aca_calibration import assert_aca_ptc_calibration
 
-    TARGETS_PATH = Path(
-        "policyengine_us_data/storage/calibration_targets/aca_spending_and_enrollment_2024.csv"
-    )
-    targets = pd.read_csv(TARGETS_PATH)
-    # Monthly to yearly
-    targets["spending"] = targets["spending"] * 12
-    # Adjust to match national target
-    targets["spending"] = targets["spending"] * (98e9 / targets["spending"].sum())
-
+    # Use IRS SOI total premium tax credit targets. The older CMS APTC file is
+    # an outlay concept and is especially weak for Basic Health Program states.
     sim = Microsimulation(dataset=EnhancedCPS_2024)
-    state_code_hh = sim.calculate("state_code", map_to="household").values
-    aca_ptc = sim.calculate("aca_ptc", map_to="household", period=2025)
-
-    # Per-state CMS APTC targets mix outlay vs claimed-PTC concepts and
-    # do not account for ACA §1331 Basic Health Programs (NY Essential
-    # Plan, MN MinnesotaCare), which divert 138–200% FPL enrollees out
-    # of the Marketplace. Simulated aca_ptc is closer to total PTC
-    # claim than to CMS APTC paid. A full target-side redesign is in
-    # issue #805 (switch to IRS SOI A85770 total PTC claimed). Until
-    # that lands, hold a loose tolerance here so the build is not
-    # chronically blocked.
-    TOLERANCE = 10.0
-    failed = False
-    for _, row in targets.iterrows():
-        state = row["state"]
-        target_spending = row["spending"]
-        simulated = aca_ptc[state_code_hh == state].sum()
-
-        pct_error = abs(simulated - target_spending) / target_spending
-        print(
-            f"{state}: simulated ${simulated / 1e9:.2f} bn  "
-            f"target ${target_spending / 1e9:.2f} bn  "
-            f"error {pct_error:.2%}"
-        )
-
-        if pct_error > TOLERANCE:
-            failed = True
-
-    assert not failed, f"One or more states exceeded tolerance of {TOLERANCE:.0%}."
+    assert_aca_ptc_calibration(sim, emit=print)
 
 
 def test_aca_2025_takeup_override_helper():

validation/stage_1/test_sparse_enhanced_cps.py

@@ -193,38 +193,11 @@ def test_sparse_has_tin_matches_identification_inputs(sim):
 
 
 def test_sparse_aca_calibration(sim):
-    TARGETS_PATH = Path(
-        "policyengine_us_data/storage/calibration_targets/aca_spending_and_enrollment_2024.csv"
+    from validation.stage_1.aca_calibration import (
+        assert_aca_ptc_calibration,
     )
-    targets = pd.read_csv(TARGETS_PATH)
-    # Monthly to yearly
-    targets["spending"] = targets["spending"] * 12
-    # Adjust to match national target
-    targets["spending"] = targets["spending"] * (98e9 / targets["spending"].sum())
 
-    state_code_hh = sim.calculate("state_code", map_to="household").values
-    aca_ptc = sim.calculate("aca_ptc", map_to="household", period=2025)
-
-    # See test_aca_calibration in test_enhanced_cps.py for the full
-    # CMS-vs-IRS concept mismatch rationale; tracked in issue #805.
-    TOLERANCE = 10.0
-    failed = False
-    for _, row in targets.iterrows():
-        state = row["state"]
-        target_spending = row["spending"]
-        simulated = aca_ptc[state_code_hh == state].sum()
-
-        pct_error = abs(simulated - target_spending) / target_spending
-        logging.info(
-            f"{state}: simulated ${simulated / 1e9:.2f} bn  "
-            f"target ${target_spending / 1e9:.2f} bn  "
-            f"error {pct_error:.2%}"
-        )
-
-        if pct_error > TOLERANCE:
-            failed = True
-
-    assert not failed, f"One or more states exceeded tolerance of {TOLERANCE:.0%}."
+    assert_aca_ptc_calibration(sim, emit=logging.info)
 
 
 def test_sparse_medicaid_calibration(sim):