feat(dashboard): drain-and-restart from memory observability card (lidge-jun#563) (lidge-jun#580)

* feat(dashboard): drain-and-restart from memory observability card

Expose a longer 60s informed recycle on the memory card (POST /api/system/restart) that reuses drainAndShutdown and respawns via ensure/service without Codex teardown (lidge-jun#563).

* fix(dashboard): harden drain-and-restart respawn and exit cleanup

Address Codex review: use failure-exit for supervised service children, spawn start with the live port (not ensure), suppress injection teardown on recycle, and clear the GUI draining state on pid change.

* fix(dashboard): arm drain immediately and harden reconnect polling

Set draining before the 200ms flush delay, bound reconnect /healthz polls, announce restart status to assistive tech, and clarify docs around active-turn drain timeout abort.

Author: Wibias

docs-site/src/content/docs/troubleshooting/windows-memory.md

@@ -50,8 +50,13 @@ runtime the leak itself remains an upstream problem:
   whereas a flat `responseState` under rising observed memory points away from
   that store. The values are scalar-only — no request bodies, tokens, paths, or
   account identifiers — and the read is side-effect free (it never prunes or
-  evicts). The dashboard's read-only **Memory observability** card renders the
-  same fields.
+  evicts). The dashboard's **Memory observability** card renders the
+  same fields and offers a confirm-gated **Drain & restart** action: it shows
+  the current active-turn count, waits up to 60s for active turns (reusing
+  the existing 503 + `Retry-After` drain), then aborts any remaining turns and
+  restarts the proxy via `ocx start` on the live port (or a failure-only
+  service supervisor respawn) without tearing down Codex injection. That is a
+  longer, informed recycle than the short drain on `POST /api/stop`.
 - **A gated alternative stream path** — a bounded single-reader relay that
   removes the unbounded buffering shape entirely. It becomes the default
   automatically once a bundled Bun release verifiably carries the #32111 fix;

gui/src/components/MemoryObservabilityCard.tsx

@@ -4,12 +4,10 @@ import { IconActivity } from "../icons";
 import { useI18n, type Locale } from "../i18n/shared";
 
 /**
- * Read-only Memory observability card. Polls GET /api/system/memory (the #314 WP3
- * service-process introspection surface) every 5s and renders scalar diagnostics
- * only: no sliders, no restart toggle, no PUT. Observed memory is the largest
- * of RSS, external, and ArrayBuffers so Windows working-set trimming does not
- * hide committed retention; a rising continuation-store total under rising
- * observed memory points at conversation retention.
+ * Memory observability card. Polls GET /api/system/memory (#314 WP3) every 5s
+ * and renders scalar diagnostics. Also hosts the confirm-gated Drain & restart
+ * action (#563): longer 60s drain, then respawn via ensure/service — not the
+ * short /api/stop teardown path.
  */
 
 interface MemorySample {
@@ -33,6 +31,7 @@ interface ResponseState {
 }
 
 interface SystemMemory {
+  pid?: number;
   rss: number;
   heapUsed: number;
   heapTotal: number;
@@ -43,9 +42,14 @@ interface SystemMemory {
   jscHeap: { heapSize: number; heapCapacity: number; objectCount: number } | null;
   /** Absent on older proxies whose /api/system/memory predates the continuation-store metrics. */
   responseState?: ResponseState;
+  /** Absent on older proxies predating the drain-and-restart action (#563). */
+  activeTurnCount?: number;
+  isDraining?: boolean;
   watchdog: { warnThresholdBytes: number; lastWarnAt: number | null; observedBytes?: number; observedMetric?: MemoryMetric; samples: MemorySample[] } | null;
 }
 
+type RestartPhase = "idle" | "draining" | "reconnecting" | "error";
+
 /**
  * Render a byte count with a binary-scaled unit; non-finite/zero inputs render as "0 B".
  * The divisor is 1024, so the labels must be the binary ones (KiB/MiB/...). Labelling a
@@ -127,10 +131,34 @@ function Stat({ label, value }: { label: string; value: string }) {
   );
 }
 
+const DRAIN_TIMEOUT_S = 60;
+const RECONNECT_POLL_MS = 1500;
+const RECONNECT_GIVE_UP_MS = 120_000;
+
 export default function MemoryObservabilityCard({ apiBase }: { apiBase: string }) {
   const { locale, t } = useI18n();
   const [data, setData] = useState<SystemMemory | null>(null);
   const [unavailable, setUnavailable] = useState(false);
+  const [restartPhase, setRestartPhase] = useState<RestartPhase>("idle");
+  const [restartError, setRestartError] = useState<string | null>(null);
+  const [noSupervisor, setNoSupervisor] = useState(false);
+  const [supportsRestart, setSupportsRestart] = useState(false);
+  const [restartFromPid, setRestartFromPid] = useState<number | null>(null);
+
+  useEffect(() => {
+    let cancelled = false;
+    void (async () => {
+      try {
+        const res = await fetch(`${apiBase}/api/startup-health`);
+        if (!res.ok || cancelled) return;
+        const json = await res.json() as { protection?: string };
+        if (!cancelled) setNoSupervisor(json.protection === "none");
+      } catch {
+        /* older proxies / offline — leave warning off */
+      }
+    })();
+    return () => { cancelled = true; };
+  }, [apiBase]);
 
   useEffect(() => {
     let cancelled = false;
@@ -160,10 +188,31 @@ export default function MemoryObservabilityCard({ apiBase }: { apiBase: string }
         if (!cancelled) {
           setData(json);
           setUnavailable(false);
+          setSupportsRestart(typeof json.activeTurnCount === "number");
+          if (json.isDraining && restartPhase === "idle") setRestartPhase("draining");
+          // Fast recycle can finish between polls with no observed outage — detect pid change.
+          if (
+            (restartPhase === "draining" || restartPhase === "reconnecting")
+            && restartFromPid != null
+            && typeof json.pid === "number"
+            && json.pid !== restartFromPid
+            && !json.isDraining
+          ) {
+            setRestartPhase("idle");
+            setRestartFromPid(null);
+            setRestartError(null);
+          }
         }
       } catch {
         // Old servers (pre-#314) 404 this route; degrade to a quiet unavailable note.
-        if (!cancelled) setUnavailable(true);
+        // During drain/restart the proxy goes away — switch to reconnect polling.
+        if (!cancelled) {
+          if (restartPhase === "draining" || restartPhase === "reconnecting") {
+            setRestartPhase("reconnecting");
+          } else {
+            setUnavailable(true);
+          }
+        }
       } finally {
         if (timeoutId !== undefined) clearTimeout(timeoutId);
         if (activeController === controller) activeController = null;
@@ -177,9 +226,80 @@ export default function MemoryObservabilityCard({ apiBase }: { apiBase: string }
       activeController?.abort();
       clearInterval(interval);
     };
-  }, [apiBase]);
+  }, [apiBase, restartPhase, restartFromPid]);
 
-  if (unavailable && !data) {
+  useEffect(() => {
+    if (restartPhase !== "reconnecting") return;
+    let cancelled = false;
+    let inFlight = false;
+    const started = Date.now();
+    const tick = async () => {
+      if (inFlight) return;
+      inFlight = true;
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), 5_000);
+      try {
+        const res = await fetch(`${apiBase}/healthz`, { cache: "no-store", signal: controller.signal });
+        if (res.ok && !cancelled) {
+          let replaced = restartFromPid == null;
+          if (restartFromPid != null) {
+            try {
+              const health = await res.json() as { pid?: number };
+              replaced = typeof health.pid === "number" && health.pid !== restartFromPid;
+            } catch {
+              replaced = true;
+            }
+          }
+          if (replaced) {
+            setRestartPhase("idle");
+            setRestartFromPid(null);
+            setRestartError(null);
+            return;
+          }
+        }
+      } catch {
+        /* still down / aborted */
+      } finally {
+        clearTimeout(timeoutId);
+        inFlight = false;
+      }
+      if (!cancelled && Date.now() - started >= RECONNECT_GIVE_UP_MS) {
+        setRestartPhase("error");
+        setRestartError(t("dash.mem.restartFailed"));
+      }
+    };
+    void tick();
+    const interval = setInterval(() => void tick(), RECONNECT_POLL_MS);
+    return () => {
+      cancelled = true;
+      clearInterval(interval);
+    };
+  }, [apiBase, restartPhase, restartFromPid, t]);
+
+  const confirmRestart = () => {
+    const count = data?.activeTurnCount ?? 0;
+    const lines = [
+      t("dash.mem.restartConfirm", { count, seconds: DRAIN_TIMEOUT_S }),
+    ];
+    if (noSupervisor) lines.push(t("dash.mem.restartNoSupervisor"));
+    if (!window.confirm(lines.join("\n\n"))) return;
+    void (async () => {
+      setRestartError(null);
+      setRestartFromPid(typeof data?.pid === "number" ? data.pid : null);
+      setRestartPhase("draining");
+      try {
+        const res = await fetch(`${apiBase}/api/system/restart`, { method: "POST" });
+        if (!res.ok) throw new Error("restart_failed");
+        // Proxy will drain then exit; memory poll will trip reconnecting or pid change.
+      } catch {
+        setRestartPhase("error");
+        setRestartFromPid(null);
+        setRestartError(t("dash.mem.restartFailed"));
+      }
+    })();
+  };
+
+  if (unavailable && !data && restartPhase === "idle") {
     return (
       <div className="panel" style={{ marginBottom: 24 }}>
         <div className="font-semibold" style={{ display: "flex", alignItems: "center", gap: 8 }}>
@@ -196,6 +316,8 @@ export default function MemoryObservabilityCard({ apiBase }: { apiBase: string }
   const observedBy = data ? observedMetric(data) : null;
   // Optional on purpose: a 200 from an older proxy may lack the responseState field.
   const responseState = data?.responseState;
+  const activeTurns = data?.activeTurnCount;
+  const busy = restartPhase === "draining" || restartPhase === "reconnecting";
 
   return (
     <div className="panel" style={{ marginBottom: 24 }}>
@@ -249,6 +371,37 @@ export default function MemoryObservabilityCard({ apiBase }: { apiBase: string }
           </div>
         )}
       </details>
+
+      {supportsRestart && (
+        <div style={{ marginTop: 14, display: "flex", flexWrap: "wrap", alignItems: "center", gap: 12 }} aria-live="polite">
+          <Stat
+            label={t("dash.mem.inFlight")}
+            value={typeof activeTurns === "number" ? plainNumberFormat(locale).format(activeTurns) : "—"}
+          />
+          <button
+            type="button"
+            className="btn btn-ghost btn-sm"
+            disabled={busy}
+            onClick={confirmRestart}
+          >
+            {t("dash.mem.restart")}
+          </button>
+          {restartPhase === "draining" && (
+            <span className="muted text-control">
+              {t("dash.mem.draining", { count: typeof activeTurns === "number" ? activeTurns : 0 })}
+            </span>
+          )}
+          {restartPhase === "reconnecting" && (
+            <span className="muted text-control">{t("dash.mem.reconnecting")}</span>
+          )}
+          {restartPhase === "error" && restartError && (
+            <span className="text-control" style={{ color: "var(--danger, #c44)" }}>{restartError}</span>
+          )}
+          {noSupervisor && restartPhase === "idle" && (
+            <span className="muted text-control">{t("dash.mem.restartNoSupervisor")}</span>
+          )}
+        </div>
+      )}
     </div>
   );
 }

gui/src/i18n/de.ts

@@ -136,6 +136,13 @@ export const de: Record<TKey, string> = {
   "dash.mem.never": "Nie",
   "dash.mem.details": "Details",
   "dash.mem.unavailable": "Speicherdiagnose nicht verfügbar (älterer Proxy).",
+  "dash.mem.inFlight": "Laufende Anfragen",
+  "dash.mem.restart": "Abwarten & neu starten",
+  "dash.mem.restartConfirm": "Auf {count} laufende Anfrage(n) warten, dann neu starten (bis zu {seconds}s; Rest wird bei Timeout abgebrochen).",
+  "dash.mem.draining": "{count} Anfrage(n) werden abgewartet… Neustart danach",
+  "dash.mem.reconnecting": "Proxy wird neu gestartet… warte auf Verbindung",
+  "dash.mem.restartFailed": "Abwarten & Neustart fehlgeschlagen. Prüfen Sie, ob der Proxy läuft.",
+  "dash.mem.restartNoSupervisor": "Kein Neustartschutz erkannt. Der Proxy bleibt nach dem Neustart möglicherweise aus, bis Sie ihn erneut starten.",
   "dash.activeProviders": "Aktive Anbieter",
   "dash.noProviders": "Keine Anbieter konfiguriert. Führe {cmd} aus.",
   "dash.col.name": "Name",

gui/src/i18n/en.ts

@@ -146,6 +146,13 @@ export const en = {
   "dash.mem.never": "Never",
   "dash.mem.details": "Details",
   "dash.mem.unavailable": "Memory diagnostics unavailable (older proxy).",
+  "dash.mem.inFlight": "In-flight requests",
+  "dash.mem.restart": "Drain & restart",
+  "dash.mem.restartConfirm": "Wait for {count} in-flight request(s), then restart (up to {seconds}s; remaining requests are cut on timeout).",
+  "dash.mem.draining": "Draining {count} request(s)… restarting when complete",
+  "dash.mem.reconnecting": "Proxy restarting… waiting to reconnect",
+  "dash.mem.restartFailed": "Drain & restart failed. Check that the proxy is running.",
+  "dash.mem.restartNoSupervisor": "No restart protection detected. The proxy may stay down after restart unless you start it again.",
   "dash.activeProviders": "Active providers",
   "dash.noProviders": "No providers configured. Run {cmd}.",
   "dash.col.name": "Name",

gui/src/i18n/ja.ts

@@ -145,6 +145,13 @@ export const ja: Record<TKey, string> = {
   "dash.mem.never": "なし",
   "dash.mem.details": "詳細",
   "dash.mem.unavailable": "メモリ診断は利用できません（旧バージョンのプロキシ）。",
+  "dash.mem.inFlight": "処理中のリクエスト",
+  "dash.mem.restart": "完了後に再起動",
+  "dash.mem.restartConfirm": "処理中のリクエスト {count} 件の完了を待ってから再起動します（最大 {seconds} 秒。タイムアウト時は残りを打ち切ります）。",
+  "dash.mem.draining": "リクエスト {count} 件の完了を待機中… 完了後に再起動",
+  "dash.mem.reconnecting": "プロキシを再起動中… 再接続を待機",
+  "dash.mem.restartFailed": "完了後の再起動に失敗しました。プロキシが起動しているか確認してください。",
+  "dash.mem.restartNoSupervisor": "再起動保護がありません。再起動後、プロキシが自動で戻らない可能性があります。",
   "dash.activeProviders": "アクティブなプロバイダー",
   "dash.noProviders": "プロバイダーが設定されていません。{cmd} を実行してください。",
   "dash.col.name": "名前",

gui/src/i18n/ko.ts

@@ -140,6 +140,13 @@ export const ko: Record<TKey, string> = {
   "dash.mem.never": "없음",
   "dash.mem.details": "상세 정보",
   "dash.mem.unavailable": "메모리 진단을 사용할 수 없음 (구버전 프록시).",
+  "dash.mem.inFlight": "진행 중 요청",
+  "dash.mem.restart": "작업 완료 후 재시작",
+  "dash.mem.restartConfirm": "진행 중 요청 {count}개가 끝날 때까지 기다린 뒤 재시작합니다(최대 {seconds}초; 시간이 지나면 남은 요청은 중단됩니다).",
+  "dash.mem.draining": "요청 {count}개 완료 대기 중… 끝나면 재시작",
+  "dash.mem.reconnecting": "프록시 재시작 중… 다시 연결하는 중",
+  "dash.mem.restartFailed": "작업 완료 후 재시작에 실패했습니다. 프록시가 실행 중인지 확인하세요.",
+  "dash.mem.restartNoSupervisor": "재시작 보호가 없습니다. 재시작 후 프록시가 자동으로 올라오지 않을 수 있습니다.",
   "dash.activeProviders": "활성 프로바이더",
   "dash.noProviders": "설정된 프로바이더가 없습니다. {cmd} 를 실행하세요.",
   "dash.col.name": "이름",

gui/src/i18n/ru.ts

@@ -145,6 +145,13 @@ export const ru: Record<TKey, string> = {
   "dash.mem.never": "Никогда",
   "dash.mem.details": "Подробности",
   "dash.mem.unavailable": "Диагностика памяти недоступна (старая версия прокси).",
+  "dash.mem.inFlight": "Активные запросы",
+  "dash.mem.restart": "Дождаться и перезапустить",
+  "dash.mem.restartConfirm": "Дождаться завершения {count} активных запросов, затем перезапустить (до {seconds} с; оставшиеся при таймауте прервутся).",
+  "dash.mem.draining": "Ожидание {count} запрос(ов)… перезапуск после завершения",
+  "dash.mem.reconnecting": "Прокси перезапускается… ожидание подключения",
+  "dash.mem.restartFailed": "Не удалось дождаться и перезапустить. Проверьте, что прокси запущен.",
+  "dash.mem.restartNoSupervisor": "Защита перезапуска не обнаружена. После перезапуска прокси может остаться выключенным, пока вы не запустите его снова.",
   "dash.activeProviders": "Активные провайдеры",
   "dash.noProviders": "Провайдеры не настроены. Выполните {cmd}.",
   "dash.col.name": "Название",

gui/src/i18n/zh.ts

@@ -140,6 +140,13 @@ export const zh: Record<TKey, string> = {
   "dash.mem.never": "从不",
   "dash.mem.details": "详情",
   "dash.mem.unavailable": "内存诊断不可用（旧版代理）。",
+  "dash.mem.inFlight": "进行中的请求",
+  "dash.mem.restart": "排空并重启",
+  "dash.mem.restartConfirm": "等待 {count} 个进行中的请求结束后再重启（最多 {seconds} 秒；超时将中断剩余请求）。",
+  "dash.mem.draining": "正在等待 {count} 个请求完成… 完成后重启",
+  "dash.mem.reconnecting": "代理正在重启… 等待重新连接",
+  "dash.mem.restartFailed": "排空并重启失败。请确认代理正在运行。",
+  "dash.mem.restartNoSupervisor": "未检测到重启保护。重启后代理可能不会自动恢复，需手动启动。",
   "dash.activeProviders": "活跃提供方",
   "dash.noProviders": "尚未配置提供方。请运行 {cmd}。",
   "dash.col.name": "名称",