Merge pull request #1494 from PolicyEngine/warn-drop-deprecated-inputs

Warn and drop deprecated inputs instead of crashing

Author: anth-volk

changelog.d/warn-and-drop-deprecated-inputs.added.md

@@ -0,0 +1 @@
+Added a deprecation registry and warn-and-drop layer for inbound /calculate payloads. Removed model variables (e.g. `medical_out_of_pocket_expenses`, deleted in policyengine-us 1.673.0) are now stripped before the engine runs, and the response includes a structured warning telling partners which variable was ignored and how to migrate. Previously these requests returned HTTP 500 with `VariableNotFoundError`.

policyengine_household_api/endpoints/household.py

@@ -14,6 +14,9 @@
     HouseholdModelUK,
     HouseholdModelUS,
 )
+from policyengine_household_api.utils.deprecated_inputs import (
+    drop_deprecated_inputs,
+)
 from policyengine_household_api.utils.validate_country import validate_country
 
 
@@ -124,6 +127,11 @@ def get_calculate(country_id: str, add_missing: bool = False) -> Response:
 
     country = COUNTRIES.get(country_id)
 
+    # Strip deprecated inputs before validation so partners who still pass
+    # removed/renamed variables get a warning + working response instead
+    # of a `VariableNotFoundError` HTTP 500.
+    deprecation_warnings = drop_deprecated_inputs(household_json)
+
     # Validate inbound payload shape before reaching the compute layer.
     try:
         _validate_household_payload(country_id, household_json)
@@ -169,10 +177,13 @@ def get_calculate(country_id: str, add_missing: bool = False) -> Response:
         policyengine_bundle=dict(country.policyengine_bundle),
     )
 
-    if period_warnings:
+    warning_messages = [w.message for w in deprecation_warnings] + [
+        w.message for w in period_warnings
+    ]
+    if warning_messages:
         # Serialize to strings on the wire; the structured dataclasses
         # stay available for any future caller that wants the fields.
-        response_body["warnings"] = [w.message for w in period_warnings]
+        response_body["warnings"] = warning_messages
 
     if enable_ai_explainer:
         response_body["computation_tree_uuid"] = str(computation_tree_uuid)

policyengine_household_api/utils/deprecated_inputs.py

@@ -0,0 +1,147 @@
+"""Detect and drop deprecated input variables before they reach the engine.
+
+Without this, a partner who passes a removed model variable (e.g.
+``medical_out_of_pocket_expenses``, deleted in policyengine-us 1.673.0)
+crashes the simulation with ``VariableNotFoundError`` (HTTP 500). Dropping
+the input and surfacing a structured warning gives partners a soft
+landing — every other output computes normally; only outputs that
+depended on the deprecated input fall back to defaults.
+"""
+
+from dataclasses import dataclass
+
+
+# Registry of removed/renamed model variables that legacy partner traffic
+# may still pass. The value is a one-line migration hint surfaced verbatim
+# in the warning message — keep it actionable.
+DEPRECATED_VARIABLES: dict[str, str] = {
+    "medical_out_of_pocket_expenses": (
+        "Removed in policyengine-us 1.673.0. Migrate non-premium spending "
+        "to `other_medical_expenses` and premium spending to "
+        "`health_insurance_premiums`."
+    ),
+}
+
+
+@dataclass(frozen=True)
+class DeprecatedVariableWarning:
+    """A removed/renamed variable was supplied; dropped before the engine ran."""
+
+    variable: str
+    entity_plural: str
+    entity_id: str
+    hint: str
+
+    @property
+    def message(self) -> str:
+        location = f"`{self.entity_plural}/{self.entity_id}`"
+        if self.entity_plural == "axes":
+            location = f"`axes[{self.entity_id}].name`"
+        return (
+            f"Input `{self.variable}` on {location} is deprecated and was "
+            f"ignored for this calculation. {self.hint}"
+        )
+
+
+def drop_deprecated_inputs(
+    household: dict,
+) -> list[DeprecatedVariableWarning]:
+    """Strip deprecated input keys from ``household`` in place.
+
+    Returns one warning per (entity, deprecated-key) occurrence. Mutates
+    ``household`` so downstream validation and the simulation never see
+    the deprecated keys.
+
+    Non-dict inputs are returned unchanged with no warnings; the
+    Pydantic schema check that runs immediately after will reject the
+    bad shape with a 400.
+    """
+    warnings: list[DeprecatedVariableWarning] = []
+
+    if not isinstance(household, dict):
+        return warnings
+
+    for entity_plural, entity_group in household.items():
+        if entity_plural == "axes":
+            continue
+        if not isinstance(entity_group, dict):
+            continue
+        for entity_id, variables in entity_group.items():
+            if not isinstance(variables, dict):
+                continue
+            for variable_name in list(variables.keys()):
+                hint = DEPRECATED_VARIABLES.get(variable_name)
+                if hint is None:
+                    continue
+                warnings.append(
+                    DeprecatedVariableWarning(
+                        variable=variable_name,
+                        entity_plural=entity_plural,
+                        entity_id=entity_id,
+                        hint=hint,
+                    )
+                )
+                del variables[variable_name]
+
+    _drop_deprecated_axes(household, warnings)
+
+    return warnings
+
+
+def _drop_deprecated_axes(
+    household: dict, warnings: list[DeprecatedVariableWarning]
+) -> None:
+    axes = household.get("axes")
+    if not isinstance(axes, list):
+        return
+
+    changed = False
+    retained_entries = []
+
+    for entry_index, entry in enumerate(axes):
+        if isinstance(entry, list):
+            retained_axes = []
+            for axis_index, axis in enumerate(entry):
+                location = f"{entry_index}][{axis_index}"
+                if _is_deprecated_axis(axis, location, warnings):
+                    changed = True
+                    continue
+                retained_axes.append(axis)
+            if retained_axes:
+                retained_entries.append(retained_axes)
+            continue
+
+        location = str(entry_index)
+        if _is_deprecated_axis(entry, location, warnings):
+            changed = True
+            continue
+        retained_entries.append(entry)
+
+    if not changed:
+        return
+    if retained_entries:
+        household["axes"] = retained_entries
+    else:
+        del household["axes"]
+
+
+def _is_deprecated_axis(
+    axis, location: str, warnings: list[DeprecatedVariableWarning]
+) -> bool:
+    if not isinstance(axis, dict):
+        return False
+
+    variable_name = axis.get("name")
+    hint = DEPRECATED_VARIABLES.get(variable_name)
+    if hint is None:
+        return False
+
+    warnings.append(
+        DeprecatedVariableWarning(
+            variable=variable_name,
+            entity_plural="axes",
+            entity_id=location,
+            hint=hint,
+        )
+    )
+    return True

tests/unit/endpoints/test_household.py

@@ -149,6 +149,80 @@ def snap_with(employment_income: int) -> float:
             "reaching the engine."
         )
 
+    def test__deprecated_input__is_dropped_with_warning(self, client):
+        """A removed model variable in the payload must not crash the request.
+
+        `medical_out_of_pocket_expenses` was removed from policyengine-us in
+        1.673.0. Without warn-and-drop, partner traffic carrying it raises
+        `VariableNotFoundError` → HTTP 500 and the partner sees no output
+        at all. With warn-and-drop, the field is stripped before the engine
+        sees it and a structured warning is appended to the response.
+        """
+        household = {
+            **valid_household_requesting_ctc_calculation,
+            "people": {
+                "you": {
+                    "age": {"2024": 40},
+                    "medical_out_of_pocket_expenses": {"2024": 0},
+                }
+            },
+        }
+
+        response = client.post(
+            "/us/calculate",
+            json={"household": household},
+            headers=self.auth_headers,
+        )
+
+        assert response.status_code == 200
+        payload = json.loads(response.data)
+        assert payload["status"] == "ok"
+        # CTC still computes — non-medical outputs are unaffected.
+        assert (
+            payload["result"]["tax_units"]["tax_unit"]["ctc"]["2024"]
+            is not None
+        )
+        # The deprecation warning is surfaced in the response.
+        assert "warnings" in payload
+        assert any(
+            "medical_out_of_pocket_expenses" in w and "deprecated" in w.lower()
+            for w in payload["warnings"]
+        )
+
+    def test__deprecated_axis_name__is_dropped_with_warning(self, client):
+        household = {
+            **valid_household_requesting_ctc_calculation,
+            "axes": [
+                [
+                    {
+                        "name": "medical_out_of_pocket_expenses",
+                        "period": "2024",
+                        "min": 0,
+                        "max": 1000,
+                        "count": 2,
+                    }
+                ]
+            ],
+        }
+
+        response = client.post(
+            "/us/calculate",
+            json={"household": household},
+            headers=self.auth_headers,
+        )
+
+        assert response.status_code == 200
+        payload = json.loads(response.data)
+        assert payload["status"] == "ok"
+        assert (
+            payload["result"]["tax_units"]["tax_unit"]["ctc"]["2024"]
+            is not None
+        )
+        assert any(
+            "medical_out_of_pocket_expenses" in w and "axes[0][0].name" in w
+            for w in payload["warnings"]
+        )
+
     def test__given_ai_explainer_tracer_fails__returns_500(self, client):
         with patch(
             "policyengine_household_api.country.generate_computation_tree",