Disable preemption on all Modal functions, log fc IDs at spawn points

Preemptible spot instances caused silent worker terminations that left
the pipeline hanging with no clear diagnostic trail. Every function
except pipeline_status (read-only, 60s) is now nonpreemptible. Spawn
points now print function-call IDs for coordinate_publish workers,
fit_weights, and H5 build orchestrators.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: baogorek

modal_app/local_area.py

@@ -276,6 +276,7 @@ def run_phase(
             calibration_inputs=calibration_inputs,
             validate=validate,
         )
+        print(f"    → fc: {handle.object_id}")
         handles.append(handle)
 
     print(f"Waiting for {phase_name} workers to complete...")
@@ -337,6 +338,7 @@ def run_phase(
     memory=16384,
     cpu=4.0,
     timeout=14400,
+    nonpreemptible=True,
 )
 def build_areas_worker(
     branch: str,
@@ -428,6 +430,7 @@ def build_areas_worker(
     volumes={VOLUME_MOUNT: staging_volume},
     memory=4096,
     timeout=1800,
+    nonpreemptible=True,
 )
 def validate_staging(branch: str, version: str) -> Dict:
     """Validate all expected files and generate manifest."""
@@ -480,6 +483,7 @@ def validate_staging(branch: str, version: str) -> Dict:
     volumes={VOLUME_MOUNT: staging_volume},
     memory=8192,
     timeout=14400,
+    nonpreemptible=True,
 )
 def upload_to_staging(branch: str, version: str, manifest: Dict) -> str:
     """
@@ -551,6 +555,7 @@ def upload_to_staging(branch: str, version: str, manifest: Dict) -> str:
     volumes={VOLUME_MOUNT: staging_volume},
     memory=4096,
     timeout=3600,
+    nonpreemptible=True,
 )
 def promote_publish(branch: str = "main", version: str = "") -> str:
     """
@@ -1084,6 +1089,7 @@ def main_national(branch: str = "main", n_clones: int = 430):
     volumes={VOLUME_MOUNT: staging_volume},
     memory=4096,
     timeout=3600,
+    nonpreemptible=True,
 )
 def promote_national_publish(
     branch: str = "main",

modal_app/pipeline.py

@@ -828,6 +828,7 @@ def run_pipeline(
                 lambda_l2=1e-8,
                 log_freq=500,
             )
+            print(f"    → regional fit fc: {regional_handle.object_id}")
 
             # Spawn national fit (if enabled)
             national_handle = None
@@ -848,6 +849,7 @@ def run_pipeline(
                     lambda_l2=1e-12,
                     log_freq=500,
                 )
+                print(f"    → national fit fc: {national_handle.object_id}")
 
             # Collect regional results
             print("  Waiting for regional fit...")
@@ -929,6 +931,7 @@ def run_pipeline(
                 n_clones=n_clones,
                 validate=True,
             )
+            print(f"    → coordinate_publish fc: {regional_h5_handle.object_id}")
 
             national_h5_handle = None
             if not skip_national:
@@ -938,6 +941,9 @@ def run_pipeline(
                     n_clones=n_clones,
                     validate=True,
                 )
+                print(
+                    f"    → coordinate_national_publish fc: {national_h5_handle.object_id}"
+                )
 
             # While H5 builds run, stage base datasets
             # and upload diagnostics in this container

modal_app/remote_calibration_runner.py

@@ -441,6 +441,7 @@ def _build_package_impl(
     cpu=8.0,
     timeout=50400,
     volumes={PIPELINE_MOUNT: pipeline_vol},
+    nonpreemptible=True,
 )
 def build_package_remote(
     branch: str = "main",
@@ -462,6 +463,7 @@ def build_package_remote(
     image=image,
     timeout=30,
     volumes={PIPELINE_MOUNT: pipeline_vol},
+    nonpreemptible=True,
 )
 def check_volume_package() -> dict:
     """Check if a calibration package exists on the volume.
@@ -515,6 +517,7 @@ def check_volume_package() -> dict:
     gpu="T4",
     timeout=14400,
     volumes={PIPELINE_MOUNT: pipeline_vol},
+    nonpreemptible=True,
 )
 def fit_weights_t4(
     branch: str = "main",
@@ -550,6 +553,7 @@ def fit_weights_t4(
     gpu="A10",
     timeout=14400,
     volumes={PIPELINE_MOUNT: pipeline_vol},
+    nonpreemptible=True,
 )
 def fit_weights_a10(
     branch: str = "main",
@@ -585,6 +589,7 @@ def fit_weights_a10(
     gpu="A100-40GB",
     timeout=14400,
     volumes={PIPELINE_MOUNT: pipeline_vol},
+    nonpreemptible=True,
 )
 def fit_weights_a100_40(
     branch: str = "main",
@@ -620,6 +625,7 @@ def fit_weights_a100_40(
     gpu="A100-80GB",
     timeout=14400,
     volumes={PIPELINE_MOUNT: pipeline_vol},
+    nonpreemptible=True,
 )
 def fit_weights_a100_80(
     branch: str = "main",
@@ -655,6 +661,7 @@ def fit_weights_a100_80(
     gpu="H100",
     timeout=14400,
     volumes={PIPELINE_MOUNT: pipeline_vol},
+    nonpreemptible=True,
 )
 def fit_weights_h100(
     branch: str = "main",
@@ -701,6 +708,7 @@ def fit_weights_h100(
     gpu="T4",
     timeout=14400,
     volumes={PIPELINE_MOUNT: pipeline_vol},
+    nonpreemptible=True,
 )
 def fit_from_package_t4(
     branch: str = "main",
@@ -733,6 +741,7 @@ def fit_from_package_t4(
     gpu="A10",
     timeout=14400,
     volumes={PIPELINE_MOUNT: pipeline_vol},
+    nonpreemptible=True,
 )
 def fit_from_package_a10(
     branch: str = "main",
@@ -765,6 +774,7 @@ def fit_from_package_a10(
     gpu="A100-40GB",
     timeout=14400,
     volumes={PIPELINE_MOUNT: pipeline_vol},
+    nonpreemptible=True,
 )
 def fit_from_package_a100_40(
     branch: str = "main",
@@ -797,6 +807,7 @@ def fit_from_package_a100_40(
     gpu="A100-80GB",
     timeout=14400,
     volumes={PIPELINE_MOUNT: pipeline_vol},
+    nonpreemptible=True,
 )
 def fit_from_package_a100_80(
     branch: str = "main",
@@ -829,6 +840,7 @@ def fit_from_package_a100_80(
     gpu="H100",
     timeout=14400,
     volumes={PIPELINE_MOUNT: pipeline_vol},
+    nonpreemptible=True,
 )
 def fit_from_package_h100(
     branch: str = "main",