Merge pull request #846 from PolicyEngine/maria/indentation_error

Fix diagnostics upload script indentation

Author: anth-volk

changelog.d/modal-diagnostics-upload.fixed.md

@@ -0,0 +1 @@
+Fix the Modal pipeline diagnostics-upload helper so its generated `python -c` script is valid Python and covered by a unit test before deployment.

changelog.d/state-eitc-targets.changed.md

@@ -0,0 +1 @@
+Enable all state- and district-level EITC and CTC targets in the default calibration target configuration.

modal_app/pipeline.py

@@ -416,23 +416,7 @@ def upload_run_diagnostics(
     result = subprocess.run(
         _python_cmd(
             "-c",
-            f"""
-import json, os
-from huggingface_hub import HfApi
-
-	entries = json.loads('''{entries_json}''')
-	api = HfApi()
-	token = os.environ.get("HUGGING_FACE_TOKEN")
-	for local_path, repo_path in entries:
-    api.upload_file(
-        path_or_fileobj=local_path,
-        path_in_repo=repo_path,
-        repo_id="policyengine/policyengine-us-data",
-        repo_type="model",
-        token=token,
-	    )
-	    print(f"Uploaded {{repo_path}}")
-	""",
+            _build_diagnostics_upload_script(entries_json),
         ),
         cwd="/root/policyengine-us-data",
         capture_output=True,
@@ -444,6 +428,32 @@ def upload_run_diagnostics(
     print(f"  {result.stdout.strip()}")
 
 
+def _build_diagnostics_upload_script(entries_json: str) -> str:
+    """Build the isolated diagnostics-upload script.
+
+    Keep this snippet syntactically self-contained: it is passed directly to
+    ``python -c`` inside the Modal orchestrator container.
+    """
+    return f"""
+import json
+import os
+from huggingface_hub import HfApi
+
+entries = json.loads({entries_json!r})
+api = HfApi()
+token = os.environ.get("HUGGING_FACE_TOKEN")
+for local_path, repo_path in entries:
+    api.upload_file(
+        path_or_fileobj=local_path,
+        path_in_repo=repo_path,
+        repo_id="policyengine/policyengine-us-data",
+        repo_type="model",
+        token=token,
+    )
+    print(f"Uploaded {{repo_path}}")
+"""
+
+
 @app.function(
     image=image,
     timeout=300,

policyengine_us_data/calibration/target_config.yaml

@@ -26,6 +26,8 @@ include:
     geo_level: district
   - variable: taxable_pension_income
     geo_level: district
+  - variable: eitc
+    geo_level: district
   - variable: refundable_ctc
     geo_level: district
   - variable: non_refundable_ctc
@@ -58,6 +60,12 @@ include:
   - variable: spm_unit_count
     geo_level: state
     domain_variable: tanf
+  - variable: eitc
+    geo_level: state
+  - variable: refundable_ctc
+    geo_level: state
+  - variable: non_refundable_ctc
+    geo_level: state
 
   # === STATE — ACA marketplace APTC and bronze-plan enrollment counts ===
   - variable: tax_unit_count

tests/unit/test_pipeline.py

@@ -1,7 +1,9 @@
 """Tests for pipeline orchestrator metadata and helpers."""
 
 import json
+import sys
 import time
+from types import ModuleType
 from unittest.mock import MagicMock, patch
 
 import pytest
@@ -10,6 +12,7 @@
 
 from modal_app.pipeline import (
     RunMetadata,
+    _build_diagnostics_upload_script,
     _step_completed,
     _record_step,
     generate_run_id,
@@ -259,3 +262,43 @@ def test_read_nonexistent_raises(self):
         ):
             with pytest.raises(FileNotFoundError):
                 read_run_meta("fake_run", mock_vol)
+
+
+def test_diagnostics_upload_script_is_valid_python(monkeypatch, capsys):
+    entries = [
+        (
+            "/pipeline/runs/test/diagnostics/unified_diagnostics.csv",
+            "calibration/runs/test/diagnostics/unified_diagnostics.csv",
+        )
+    ]
+    entries_json = json.dumps(entries)
+
+    script = _build_diagnostics_upload_script(entries_json)
+
+    compile(script, "<diagnostics-upload>", "exec")
+    assert "\t" not in script
+    assert "api.upload_file(" in script
+
+    calls = []
+
+    class FakeHfApi:
+        def upload_file(self, **kwargs):
+            calls.append(kwargs)
+
+    fake_hub = ModuleType("huggingface_hub")
+    fake_hub.HfApi = FakeHfApi
+    monkeypatch.setitem(sys.modules, "huggingface_hub", fake_hub)
+    monkeypatch.setenv("HUGGING_FACE_TOKEN", "token")
+
+    exec(compile(script, "<diagnostics-upload>", "exec"), {})
+
+    assert calls == [
+        {
+            "path_or_fileobj": entries[0][0],
+            "path_in_repo": entries[0][1],
+            "repo_id": "policyengine/policyengine-us-data",
+            "repo_type": "model",
+            "token": "token",
+        }
+    ]
+    assert capsys.readouterr().out == f"Uploaded {entries[0][1]}\n"