Skip to content

Commit d709386

Browse files
baogorekclaude
andcommitted
Always fresh-download calibration inputs, clear stale builds
The Modal volume was caching old calibration inputs from previous runs. The code only checked file existence, not freshness, so new model fits on HuggingFace were never pulled. Also clear the version build directory to prevent stale h5 files (built from old weights) from being treated as completed by the volume checkpoint system. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent a6864b8 commit d709386

1 file changed

Lines changed: 23 additions & 19 deletions

File tree

modal_app/local_area.py

Lines changed: 23 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -430,11 +430,18 @@ def coordinate_publish(
430430
print(f"Publishing version {version} from branch {branch}")
431431
print(f"Using {num_workers} parallel workers")
432432

433+
import shutil
434+
433435
staging_dir = Path(VOLUME_MOUNT)
434436
version_dir = staging_dir / version
437+
if version_dir.exists():
438+
print(f"Clearing stale build directory: {version_dir}")
439+
shutil.rmtree(version_dir)
435440
version_dir.mkdir(parents=True, exist_ok=True)
436441

437442
calibration_dir = staging_dir / "calibration_inputs"
443+
if calibration_dir.exists():
444+
shutil.rmtree(calibration_dir)
438445
calibration_dir.mkdir(parents=True, exist_ok=True)
439446

440447
# hf_hub_download preserves directory structure, so files are in calibration/ subdir
@@ -446,29 +453,26 @@ def coordinate_publish(
446453
)
447454
db_path = calibration_dir / "calibration" / "policy_data.db"
448455

449-
if not all(p.exists() for p in [weights_path, dataset_path, db_path]):
450-
print("Downloading calibration inputs...")
451-
result = subprocess.run(
452-
[
453-
"uv",
454-
"run",
455-
"python",
456-
"-c",
457-
f"""
456+
print("Downloading calibration inputs from HuggingFace...")
457+
result = subprocess.run(
458+
[
459+
"uv",
460+
"run",
461+
"python",
462+
"-c",
463+
f"""
458464
from policyengine_us_data.utils.huggingface import download_calibration_inputs
459465
download_calibration_inputs("{calibration_dir}")
460466
print("Done")
461467
""",
462-
],
463-
text=True,
464-
env=os.environ.copy(),
465-
)
466-
if result.returncode != 0:
467-
raise RuntimeError(f"Download failed: {result.stderr}")
468-
staging_volume.commit()
469-
print("Calibration inputs downloaded and cached on volume")
470-
else:
471-
print("Using cached calibration inputs from volume")
468+
],
469+
text=True,
470+
env=os.environ.copy(),
471+
)
472+
if result.returncode != 0:
473+
raise RuntimeError(f"Download failed: {result.stderr}")
474+
staging_volume.commit()
475+
print("Calibration inputs downloaded")
472476

473477
calibration_inputs = {
474478
"weights": str(weights_path),

0 commit comments

Comments
 (0)