Scope HF staging by run_id, decouple upload from validation, deduplicate Modal image

- Add run_id parameter to staging/promote/cleanup functions in data_upload.py
  so HF paths become staging/{run_id}/... instead of flat staging/
- Generate run_id in coordinate_publish/coordinate_national_publish if not provided
- Store run_id in manifest.json; promote_publish reads it back as fallback
- Downgrade manifest verification failure from hard error to warning so uploads
  proceed even if checksums have issues
- Add --run-id CLI arg to validate_staging, check_staging_sums, promote_local_h5s
- Thread run_id through pipeline.py spawn/promote calls
- Consolidate duplicated Modal image definition into images.py (addresses PR #611 review)
- All changes are backward-compatible: run_id="" preserves flat staging/ paths

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: baogorek

Makefile

@@ -211,11 +211,13 @@ promote:
 
 validate-staging:
 	python -m policyengine_us_data.calibration.validate_staging \
-		--area-type states --output validation_results.csv
+		--area-type states --output validation_results.csv \
+		$(if $(RUN_ID),--run-id $(RUN_ID))
 
 validate-staging-full:
 	python -m policyengine_us_data.calibration.validate_staging \
-		--area-type states,districts --output validation_results.csv
+		--area-type states,districts --output validation_results.csv \
+		$(if $(RUN_ID),--run-id $(RUN_ID))
 
 upload-validation:
 	python -c "from policyengine_us_data.utils.huggingface import upload; \
@@ -224,11 +226,13 @@ upload-validation:
 		'calibration/logs/validation_results.csv')"
 
 check-staging:
-	python -m policyengine_us_data.calibration.check_staging_sums
+	python -m policyengine_us_data.calibration.check_staging_sums \
+		$(if $(RUN_ID),--run-id $(RUN_ID))
 
 check-sanity:
 	python -m policyengine_us_data.calibration.validate_staging \
-		--sanity-only --area-type states --areas NC
+		--sanity-only --area-type states --areas NC \
+		$(if $(RUN_ID),--run-id $(RUN_ID))
 
 build-data-modal:
 	modal run --detach modal_app/data_build.py::main --branch $(BRANCH) --upload --skip-tests

modal_app/data_build.py

@@ -11,6 +11,8 @@
 
 import modal
 
+from modal_app.images import cpu_image as image
+
 app = modal.App("policyengine-us-data")
 
 hf_secret = modal.Secret.from_name("huggingface-token")
@@ -29,50 +31,6 @@
 )
 PIPELINE_MOUNT = "/pipeline"
 
-_REPO_ROOT = Path(__file__).resolve().parent.parent
-
-try:
-    _LOCAL_SHA = subprocess.check_output(
-        ["git", "rev-parse", "HEAD"],
-        text=True,
-        stderr=subprocess.DEVNULL,
-        cwd=str(_REPO_ROOT),
-    ).strip()
-except Exception:
-    _LOCAL_SHA = None
-
-_IGNORE = [
-    ".git",
-    "__pycache__",
-    "*.egg-info",
-    ".pytest_cache",
-    "*.h5",
-    "*.npy",
-    "*.pkl",
-    "*.db",
-    "node_modules",
-    "venv",
-    ".venv",
-    "docs/_build",
-    "paper",
-    "presentations",
-]
-image = (
-    modal.Image.debian_slim(python_version="3.13")
-    .apt_install("git")
-    .pip_install("uv>=0.8")
-    .add_local_dir(
-        str(_REPO_ROOT),
-        remote_path="/root/policyengine-us-data",
-        copy=True,
-        ignore=_IGNORE,
-    )
-    .env({"BUILD_COMMIT_SHA": _LOCAL_SHA or ""})
-    .run_commands(
-        "cd /root/policyengine-us-data && UV_HTTP_TIMEOUT=300 uv sync --frozen"
-    )
-)
-
 VOLUME_MOUNT = "/checkpoints"
 _volume_lock = threading.Lock()
 

modal_app/images.py

@@ -5,12 +5,32 @@
 changes, the image rebuilds; if not, the cached layer is reused.
 """
 
+import subprocess
 import modal
 from pathlib import Path
 
 REPO_ROOT = Path(__file__).resolve().parent.parent
 
-_ignore = [
+GIT_ENV = {}
+try:
+    GIT_ENV["GIT_COMMIT"] = (
+        subprocess.check_output(["git", "rev-parse", "HEAD"], stderr=subprocess.DEVNULL)
+        .decode()
+        .strip()
+    )
+    GIT_ENV["GIT_BRANCH"] = (
+        subprocess.check_output(
+            ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+            stderr=subprocess.DEVNULL,
+        )
+        .decode()
+        .strip()
+    )
+    GIT_ENV["BUILD_COMMIT_SHA"] = GIT_ENV["GIT_COMMIT"]
+except Exception:
+    pass
+
+_IGNORE = [
     ".git",
     "__pycache__",
     "*.egg-info",
@@ -38,8 +58,9 @@ def _base_image(extras: list[str] | None = None):
             str(REPO_ROOT),
             remote_path="/root/policyengine-us-data",
             copy=True,
-            ignore=_ignore,
+            ignore=_IGNORE,
         )
+        .env(GIT_ENV)
         .run_commands(
             f"cd /root/policyengine-us-data && "
             f"UV_HTTP_TIMEOUT=300 uv sync --frozen {extra_flags}"