Merge remote-tracking branch 'origin/main' into enable-modal-memory-snapshot

# Conflicts:
#	tests/unit/modal_release/test_gateway.py
#	tests/unit/modal_release/test_worker_app.py

Author: hua7450

.github/scripts/modal-cleanup-apps.sh

@@ -16,5 +16,12 @@ for app_name in payload.get("app_names", []):
   if [[ -z "${app_name}" ]]; then
     continue
   fi
-  uv run modal app stop --env "${environment}" "${app_name}"
+  if ! output="$(uv run modal app stop --env "${environment}" "${app_name}" 2>&1)"; then
+    echo "${output}"
+    if [[ "${output}" == *"already stopped"* ]]; then
+      continue
+    fi
+    exit 1
+  fi
+  echo "${output}"
 done

.github/scripts/modal-deploy-release.sh

@@ -1,8 +1,14 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-config_json="${1:?Usage: modal-deploy-release.sh CONFIG_JSON}"
+config_json="${1:?Usage: modal-deploy-release.sh CONFIG_JSON [DEPLOY_MODE]}"
+deploy_mode="${2:-release}"
 output_file="${GITHUB_OUTPUT:-}"
+modal_extract_versions_script="${MODAL_EXTRACT_VERSIONS_SCRIPT:-.github/scripts/modal_extract_versions.py}"
+modal_sync_secrets_script="${MODAL_SYNC_SECRETS_SCRIPT:-.github/scripts/modal-sync-secrets.sh}"
+modal_active_worker_apps_script="${MODAL_ACTIVE_WORKER_APPS_SCRIPT:-.github/scripts/modal_active_worker_apps.py}"
+modal_cleanup_apps_script="${MODAL_CLEANUP_APPS_SCRIPT:-.github/scripts/modal-cleanup-apps.sh}"
+modal_get_url_script="${MODAL_GET_URL_SCRIPT:-.github/scripts/modal-get-url.sh}"
 
 require_env() {
   local missing=()
@@ -34,6 +40,18 @@ github_output() {
   fi
 }
 
+deploy_worker_app() {
+  local app_name="${1:?app name is required}"
+  local package_versions_json="${2:-}"
+
+  HOUSEHOLD_MODAL_WORKER_APP_NAME="${app_name}" \
+    HOUSEHOLD_MODAL_PACKAGE_VERSIONS_JSON="${package_versions_json}" \
+    MODAL_ENVIRONMENT="${modal_environment}" \
+    uv run modal deploy \
+      --env "${modal_environment}" \
+      -m policyengine_household_api.modal_release.worker_app
+}
+
 require_env \
   MODAL_ENVIRONMENT \
   USER_ANALYTICS_DB_USERNAME \
@@ -42,6 +60,15 @@ require_env \
 
 modal_environment="${MODAL_ENVIRONMENT}"
 
+case "${deploy_mode}" in
+  code|release)
+    ;;
+  *)
+    echo "::error::Unsupported Modal deploy mode: ${deploy_mode}"
+    exit 1
+    ;;
+esac
+
 uv run alembic upgrade head
 analytics_database_revision="$(
   uv run python -m policyengine_household_api.modal_release.analytics_revision
@@ -54,7 +81,7 @@ github_output "analytics_database_revision" "${analytics_database_revision}"
 
 versions_output="$(mktemp)"
 trap 'rm -f "${versions_output}"' EXIT
-uv run python .github/scripts/modal_extract_versions.py \
+uv run python "${modal_extract_versions_script}" \
   --github-output "${versions_output}"
 if [ -n "${output_file}" ]; then
   cat "${versions_output}" >> "${output_file}"
@@ -64,34 +91,47 @@ worker_app_name="$(
     "${versions_output}"
 )"
 
-bash .github/scripts/modal-sync-secrets.sh
+bash "${modal_sync_secrets_script}"
 
-new_app_target="$(config_value new_app_target)"
-if [ "${new_app_target}" != "none" ]; then
-  HOUSEHOLD_MODAL_WORKER_APP_NAME="${worker_app_name}" \
-    MODAL_ENVIRONMENT="${modal_environment}" \
-    uv run modal deploy \
-      --env "${modal_environment}" \
-      -m policyengine_household_api.modal_release.worker_app
-fi
+if [ "${deploy_mode}" = "code" ]; then
+  active_apps_tsv="$(mktemp)"
+  trap 'rm -f "${versions_output}" "${active_apps_tsv}"' EXIT
+
+  uv run python "${modal_active_worker_apps_script}" \
+    --modal-environment "${modal_environment}" \
+    --output-tsv "${active_apps_tsv}"
 
-uv run python -m policyengine_household_api.modal_release.update_manifest \
-  --config-json "${config_json}" \
-  --new-app-name "${worker_app_name}" \
-  --source-commit "${GITHUB_SHA}" \
-  --analytics-database-revision "${analytics_database_revision}" \
-  --modal-environment "${modal_environment}" \
-  --cleanup-output modal-cleanup.json \
-  --manifest-output modal-manifest.json
+  while IFS=$'\t' read -r active_app_name package_versions_json; do
+    if [ -z "${active_app_name}" ]; then
+      continue
+    fi
+    deploy_worker_app "${active_app_name}" "${package_versions_json}"
+  done < "${active_apps_tsv}"
+else
+  new_app_target="$(config_value new_app_target)"
+  if [ "${new_app_target}" != "none" ]; then
+    deploy_worker_app "${worker_app_name}" ""
+  fi
+
+  uv run python -m policyengine_household_api.modal_release.update_manifest \
+    --config-json "${config_json}" \
+    --new-app-name "${worker_app_name}" \
+    --analytics-database-revision "${analytics_database_revision}" \
+    --modal-environment "${modal_environment}" \
+    --cleanup-output modal-cleanup.json \
+    --manifest-output modal-manifest.json
+fi
 
 uv run modal deploy \
   --env "${modal_environment}" \
   -m policyengine_household_api.modal_release.gateway_app
 
-cleanup_target="$(config_value cleanup_target)"
-if [ "${cleanup_target}" != "none" ]; then
-  bash .github/scripts/modal-cleanup-apps.sh modal-cleanup.json
+if [ "${deploy_mode}" = "release" ]; then
+  cleanup_target="$(config_value cleanup_target)"
+  if [ "${cleanup_target}" != "none" ]; then
+    bash "${modal_cleanup_apps_script}" modal-cleanup.json
+  fi
 fi
 
-gateway_url="$(bash .github/scripts/modal-get-url.sh)"
+gateway_url="$(bash "${modal_get_url_script}")"
 curl -fsS "${gateway_url}/liveness_check"

.github/scripts/modal_active_worker_apps.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+import modal
+
+from policyengine_household_api.modal_release.manifest import (
+    MANIFEST_DICT_KEY,
+    MANIFEST_DICT_NAME,
+    active_app_deployments,
+)
+
+
+def main() -> None:
+    args = _parse_args()
+    manifest_dict = modal.Dict.from_name(
+        MANIFEST_DICT_NAME,
+        create_if_missing=False,
+        environment_name=args.modal_environment,
+    )
+    deployments = active_app_deployments(manifest_dict.get(MANIFEST_DICT_KEY))
+
+    if args.output_json:
+        Path(args.output_json).write_text(
+            json.dumps(deployments, indent=2, sort_keys=True) + "\n"
+        )
+    if args.output_tsv:
+        lines = [
+            "\t".join(
+                (
+                    deployment["app_name"],
+                    json.dumps(
+                        deployment["package_versions"],
+                        sort_keys=True,
+                        separators=(",", ":"),
+                    ),
+                )
+            )
+            for deployment in deployments
+        ]
+        Path(args.output_tsv).write_text("\n".join(lines) + "\n")
+
+    print(json.dumps(deployments, indent=2, sort_keys=True))
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Emit active Modal worker apps from the release manifest."
+    )
+    parser.add_argument("--modal-environment", required=True)
+    parser.add_argument("--output-json")
+    parser.add_argument("--output-tsv")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/resolve_modal_release_config.py

@@ -26,12 +26,14 @@
 class ResolvedModalRelease:
     should_deploy: bool
     source: str
+    deploy_mode: str
     config: ModalReleaseConfig | None = None
 
     def to_dict(self) -> dict[str, Any]:
         data = {
             "should_deploy": self.should_deploy,
             "source": self.source,
+            "deploy_mode": self.deploy_mode,
             "config": None,
         }
         if self.config:
@@ -82,15 +84,16 @@ def resolve_release_from_event(
         return ResolvedModalRelease(
             True,
             "workflow-dispatch-inputs",
+            "release",
             workflow_dispatch_config_from_inputs(event.get("inputs") or {}),
         )
 
     if "head_commit" not in event:
-        return ResolvedModalRelease(False, "unsupported-event")
+        return ResolvedModalRelease(False, "unsupported-event", "none")
 
     message = (event.get("head_commit") or {}).get("message")
     if message != WEEKLY_UPDATE_COMMIT_MESSAGE:
-        return ResolvedModalRelease(False, "push-not-release-commit")
+        return ResolvedModalRelease(False, "push-not-release-commit", "none")
 
     repository = (event.get("repository") or {}).get("full_name")
     source_sha = event.get("before") or event.get("after")
@@ -109,8 +112,8 @@ def resolve_release_from_event(
 
     return ResolvedModalRelease(
         True,
-        "weekly-default",
-        default_weekly_config(),
+        "code-only",
+        "code",
     )
 
 
@@ -121,10 +124,15 @@ def resolve_release_from_body(
     deploy_when_missing: bool,
 ) -> ResolvedModalRelease:
     if not body_contains_modal_release_config(body):
-        return ResolvedModalRelease(deploy_when_missing, f"{source}-missing")
+        deploy_mode = "code" if deploy_when_missing else "none"
+        return ResolvedModalRelease(
+            deploy_when_missing,
+            f"{source}-missing",
+            deploy_mode,
+        )
 
     config = parse_modal_release_config_from_body(body)
-    return ResolvedModalRelease(True, source, config)
+    return ResolvedModalRelease(True, source, "release", config)
 
 
 def workflow_dispatch_config_from_inputs(
@@ -200,6 +208,7 @@ def write_github_outputs(
     outputs = {
         "should_deploy": str(result["should_deploy"]).lower(),
         "source": result["source"],
+        "deploy_mode": result["deploy_mode"],
         "new_app_target": config.get("new_app_target", "none"),
         "promote_existing_frontier": str(
             config.get("promote_existing_frontier", False)

.github/scripts/test_modal_cleanup_apps.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import os
+import subprocess
+
+
+def test_modal_cleanup_treats_already_stopped_app_as_success(tmp_path):
+    cleanup_file = tmp_path / "cleanup.json"
+    cleanup_file.write_text('{"app_names":["old-app"]}\n')
+    _write_fake_uv(tmp_path, exit_code=1, output="App is already stopped.")
+
+    result = subprocess.run(
+        [
+            "bash",
+            ".github/scripts/modal-cleanup-apps.sh",
+            str(cleanup_file),
+        ],
+        capture_output=True,
+        env={
+            **os.environ,
+            "PATH": f"{tmp_path}:{os.environ['PATH']}",
+            "MODAL_ENVIRONMENT": "testing",
+        },
+        text=True,
+    )
+
+    assert result.returncode == 0
+    assert "already stopped" in result.stdout
+
+
+def test_modal_cleanup_fails_on_other_stop_errors(tmp_path):
+    cleanup_file = tmp_path / "cleanup.json"
+    cleanup_file.write_text('{"app_names":["old-app"]}\n')
+    _write_fake_uv(tmp_path, exit_code=1, output="Permission denied.")
+
+    result = subprocess.run(
+        [
+            "bash",
+            ".github/scripts/modal-cleanup-apps.sh",
+            str(cleanup_file),
+        ],
+        capture_output=True,
+        env={
+            **os.environ,
+            "PATH": f"{tmp_path}:{os.environ['PATH']}",
+            "MODAL_ENVIRONMENT": "testing",
+        },
+        text=True,
+    )
+
+    assert result.returncode == 1
+    assert "Permission denied" in result.stdout
+
+
+def _write_fake_uv(tmp_path, *, exit_code: int, output: str) -> None:
+    uv = tmp_path / "uv"
+    uv.write_text(
+        f"""#!/usr/bin/env bash
+echo "{output}" >&2
+exit {exit_code}
+"""
+    )
+    uv.chmod(0o755)