fix: measure memory while containers are running, not after teardown

Previously benchmarkMemory() ran `echo measuring_memory` which completed
instantly, so containers were already stopped before docker stats ran,
always reporting 0 MB. Now spawns awf with `sleep 30` in the background,
polls until containers are healthy, then samples docker stats while
containers are alive.

Closes #1758

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: Mossaka

scripts/ci/benchmark-performance.ts

@@ -11,7 +11,7 @@
  * Outputs structured JSON with mean, median, p95, p99 per metric.
  */
 
-import { execSync, ExecSyncOptions } from "child_process";
+import { execSync, ExecSyncOptions, spawn, ChildProcess } from "child_process";
 
 // ── Configuration ──────────────────────────────────────────────────
 
@@ -159,45 +159,115 @@ function benchmarkHttpsLatency(): BenchmarkResult {
   return { metric: "squid_https_latency", unit: "ms", values, ...stats(values) };
 }
 
-function benchmarkMemory(): BenchmarkResult {
+/**
+ * Wait for Docker containers to be running, polling at 500ms intervals.
+ * Throws if containers are not running within timeoutMs.
+ */
+function waitForContainers(containerNames: string[], timeoutMs: number): Promise<void> {
+  const start = Date.now();
+  return new Promise((resolve, reject) => {
+    const poll = (): void => {
+      if (Date.now() - start > timeoutMs) {
+        reject(new Error(`Containers not running after ${timeoutMs}ms`));
+        return;
+      }
+      try {
+        const allRunning = containerNames.every((name) => {
+          const result = execSync(
+            `sudo docker ps --filter name=${name} --filter status=running --format '{{.Names}}' 2>/dev/null`,
+            { encoding: "utf-8", timeout: 5_000 }
+          ).trim();
+          return result.includes(name);
+        });
+        if (allRunning) {
+          resolve();
+          return;
+        }
+      } catch {
+        // container not ready yet
+      }
+      setTimeout(poll, 500);
+    };
+    poll();
+  });
+}
+
+/**
+ * Parse a Docker memory usage string like "123.4MiB / 7.773GiB" into MB.
+ */
+function parseMb(s: string): number {
+  const match = s.match(/([\d.]+)\s*(MiB|GiB|KiB)/i);
+  if (!match) return 0;
+  const val = parseFloat(match[1]);
+  const unit = match[2].toLowerCase();
+  if (unit === "gib") return val * 1024;
+  if (unit === "kib") return val / 1024;
+  return val;
+}
+
+/**
+ * Kill a spawned background process and its process group, best-effort.
+ */
+function killBackground(child: ChildProcess): void {
+  try {
+    if (child.pid) {
+      // Kill the process group (negative PID) to catch child processes
+      process.kill(-child.pid, "SIGTERM");
+    }
+  } catch {
+    // Process may have already exited
+  }
+  try {
+    child.kill("SIGKILL");
+  } catch {
+    // best-effort
+  }
+}
+
+async function benchmarkMemory(): Promise<BenchmarkResult> {
   console.error("  Benchmarking memory footprint...");
   const values: number[] = [];
 
   for (let i = 0; i < ITERATIONS; i++) {
     cleanup();
-    // Start containers, measure memory, then stop
+    let child: ChildProcess | null = null;
     try {
-      // Run a sleep command so containers stay up, then check memory
-      const output = exec(
-        `${AWF_CMD} --allow-domains ${ALLOWED_DOMAIN} --log-level error --keep-containers -- ` +
-          `echo measuring_memory`
+      // Start awf with a long-running command in the background so containers stay alive
+      child = spawn(
+        "sudo",
+        ["awf", "--allow-domains", ALLOWED_DOMAIN, "--log-level", "error", "--", "sleep", "30"],
+        {
+          detached: true,
+          stdio: "ignore",
+        }
       );
-      // Get memory stats for both containers
+
+      // Wait for both containers to be running (up to 30s)
+      await waitForContainers(["awf-squid", "awf-agent"], 30_000);
+
+      // Give containers a moment to stabilize memory usage
+      await new Promise((resolve) => setTimeout(resolve, 2000));
+
+      // Get memory stats while containers are alive
       const squidMem = exec(
         "sudo docker stats awf-squid --no-stream --format '{{.MemUsage}}' 2>/dev/null || echo '0MiB'"
       );
       const agentMem = exec(
         "sudo docker stats awf-agent --no-stream --format '{{.MemUsage}}' 2>/dev/null || echo '0MiB'"
       );
 
-      // Parse memory values (format: "123.4MiB / 7.773GiB")
-      const parseMb = (s: string): number => {
-        const match = s.match(/([\d.]+)\s*(MiB|GiB|KiB)/i);
-        if (!match) return 0;
-        const val = parseFloat(match[1]);
-        const unit = match[2].toLowerCase();
-        if (unit === "gib") return val * 1024;
-        if (unit === "kib") return val / 1024;
-        return val;
-      };
-
       const totalMb = Math.round(parseMb(squidMem) + parseMb(agentMem));
       values.push(totalMb);
       console.error(`    Iteration ${i + 1}/${ITERATIONS}: ${totalMb}MB (squid: ${squidMem}, agent: ${agentMem})`);
-    } catch {
-      console.error(`    Iteration ${i + 1}/${ITERATIONS}: failed (skipped)`);
+    } catch (err) {
+      console.error(`    Iteration ${i + 1}/${ITERATIONS}: failed (skipped) - ${err}`);
+    } finally {
+      // Always clean up the background process and containers
+      if (child) {
+        killBackground(child);
+      }
+      cleanup();
     }
-    cleanup();
   }
 
   if (values.length === 0) {
@@ -248,7 +318,7 @@ async function main(): Promise<void> {
   results.push(benchmarkWarmStart());
   results.push(benchmarkColdStart());
   results.push(benchmarkHttpsLatency());
-  results.push(benchmarkMemory());
+  results.push(await benchmarkMemory());
 
   // Final cleanup
   cleanup();