Benchmarks: bound orphaned infrastructure from killed runs

Killing the outer harbor process leaves the compose topology and any
in-flight Modal sandboxes running (each layer's cleanup lives in the
layer above). Cap sandbox idle lifetime at 1h via --ek, and add
scripts/cleanup_orphans.sh to tear down task__* containers and
terminate sandboxes in the dedicated Modal app.

Author: varunursekar

harness-engineering-bench/gaia/baseline/build.yaml

@@ -46,7 +46,7 @@ rescore_attempts: 1
 model: openai/gpt-5.4-mini-2026-03-17
 environment_name: ${inner_env:-modal}
 # inner eval sandboxes share a dedicated Modal app instead of the __harbor__ default
-extra_harbor_args: ["--ek", "app_name=harness-engineering-bench"]
+extra_harbor_args: ["--ek", "app_name=harness-engineering-bench", "--ek", "sandbox_idle_timeout_secs=3600"]
 harbor_python_version: "3.12"
 n_attempts: 1
 max_retries: 1

harness-engineering-bench/officeqa/baseline/build.yaml

@@ -54,7 +54,7 @@ model: openai/gpt-5.4-mini-2026-03-17
 environment_name: modal
 # --no-force-build pulls the prebuilt officeqa-corpus image (see task_source
 # note above); the --ek groups inner sandboxes under one dedicated Modal app.
-extra_harbor_args: ["--no-force-build", "--ek", "app_name=harness-engineering-bench"]
+extra_harbor_args: ["--no-force-build", "--ek", "app_name=harness-engineering-bench", "--ek", "sandbox_idle_timeout_secs=3600"]
 harbor_python_version: "3.12"
 n_attempts: 1
 max_retries: 1

harness-engineering-bench/scripts/cleanup_orphans.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Tear down orphaned benchmark infrastructure after a killed `vero harbor run`.
+#
+# Killing the outer harbor process orphans everything below it: the docker
+# compose topology (daemon-owned, survives its creator) and any in-flight
+# Modal sandboxes (server-side, default 24h timeout). This removes both.
+# Scope is strictly ours: `task__*` compose containers locally, and sandboxes
+# in the dedicated `harness-engineering-bench` Modal app.
+#
+# Usage: bash scripts/cleanup_orphans.sh [--dry-run]
+# Requires MODAL_TOKEN_ID/MODAL_TOKEN_SECRET in the environment (or a dotenv
+# sourced beforehand); skips Modal cleanup if they are absent.
+set -euo pipefail
+
+dry_run=false
+[ "${1:-}" = "--dry-run" ] && dry_run=true
+
+containers=$(docker ps --format '{{.Names}}' | grep -i '^task__' || true)
+if [ -n "$containers" ]; then
+    echo "orphaned containers:"; echo "$containers"
+    if ! $dry_run; then
+        echo "$containers" | xargs docker rm -f
+    fi
+else
+    echo "no orphaned task__* containers"
+fi
+
+if [ -z "${MODAL_TOKEN_ID:-}" ]; then
+    echo "MODAL_TOKEN_ID not set; skipping Modal sandbox cleanup"
+    exit 0
+fi
+
+uv run --quiet --python 3.12 --with modal python - "$dry_run" <<'PY'
+import sys
+import modal
+
+dry_run = sys.argv[1] == "true"
+try:
+    app = modal.App.lookup("harness-engineering-bench", create_if_missing=False)
+except Exception as error:
+    print(f"no harness-engineering-bench app: {error}")
+    raise SystemExit(0)
+count = 0
+for sandbox in modal.Sandbox.list(app_id=app.app_id):
+    print(("would terminate" if dry_run else "terminating"), sandbox.object_id)
+    if not dry_run:
+        sandbox.terminate()
+    count += 1
+print(f"{count} running sandbox(es) in harness-engineering-bench")
+PY

harness-engineering-bench/swe-atlas-qna/baseline/build.yaml

@@ -48,7 +48,7 @@ rescore_attempts: 1
 model: openai/gpt-5.4-mini-2026-03-17
 environment_name: modal
 # inner eval sandboxes share a dedicated Modal app instead of the __harbor__ default
-extra_harbor_args: ["--ek", "app_name=harness-engineering-bench"]
+extra_harbor_args: ["--ek", "app_name=harness-engineering-bench", "--ek", "sandbox_idle_timeout_secs=3600"]
 harbor_python_version: "3.12"
 n_attempts: 1
 max_retries: 1

harness-engineering-bench/tau3/baseline/build.yaml

@@ -47,7 +47,7 @@ rescore_attempts: 1
 model: openai/gpt-5.4-mini-2026-03-17
 environment_name: modal
 # inner eval sandboxes share a dedicated Modal app instead of the __harbor__ default
-extra_harbor_args: ["--ek", "app_name=harness-engineering-bench"]
+extra_harbor_args: ["--ek", "app_name=harness-engineering-bench", "--ek", "sandbox_idle_timeout_secs=3600"]
 harbor_python_version: "3.12"
 n_attempts: 1
 max_retries: 1