Skip to content

Commit 95c3bb3

Browse files
authored
fix: reclaim orphaned temp runtimes safely (#2089)
1 parent 20d60dc commit 95c3bb3

6 files changed

Lines changed: 356 additions & 7 deletions

File tree

package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,7 @@
188188
"test:browser-scenarios": "tsx tests/browser-multi-actor-scenario.test.ts",
189189
"test:reviewer-access": "tsx tests/reviewer-access.test.ts",
190190
"test:command-router": "tsx tests/command-router.test.ts",
191+
"test:temp-runtime-cleanup": "tsx tests/temp-runtime-cleanup.test.ts",
191192
"test:command-agent-run": "tsx tests/command-agent-run.test.ts",
192193
"smoke:cli-version": "tsx scripts/cli-version-smoke.ts",
193194
"test:host-command-executor": "tsx tests/host-command-executor.test.ts",

packages/cli/src/commands/doctor.ts

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { existsSync } from "node:fs"
44
import { opendir, readFile, realpath, stat, unlink } from "node:fs/promises"
55
import { homedir } from "node:os"
66
import { dirname, join, resolve } from "node:path"
7+
import { inventoryTempRuntimeDirectories } from "./temp-runtime-cleanup.js"
78

89
type HealthStatus = "ok" | "warning" | "error"
910

@@ -19,6 +20,7 @@ interface DoctorOptions {
1920
cleanup: boolean
2021
staleAfterSeconds: number
2122
archiveRoots: string[]
23+
runRegistryRoots: string[]
2224
}
2325

2426
interface DoctorOutput {
@@ -48,6 +50,7 @@ function parseDoctorOptions(args: string[], cleanup: boolean): DoctorOptions {
4850
let json = false
4951
let staleAfterSeconds = Number.parseInt(process.env.WP_CODEBOX_STALE_AFTER_SECONDS ?? "3600", 10)
5052
const archiveRoots = archiveRootsFromEnv()
53+
const runRegistryRoots = runRegistryRootsFromEnv()
5154

5255
for (let index = 0; index < args.length; index++) {
5356
const arg = args[index]
@@ -75,6 +78,12 @@ function parseDoctorOptions(args: string[], cleanup: boolean): DoctorOptions {
7578
archiveRoots.push(value)
7679
continue
7780
}
81+
if (arg === "--run-registry") {
82+
const value = args[++index]
83+
if (!value) throw new Error("--run-registry requires a directory")
84+
runRegistryRoots.push(value)
85+
continue
86+
}
7887

7988
throw new Error(`Unknown option: ${arg}`)
8089
}
@@ -84,6 +93,7 @@ function parseDoctorOptions(args: string[], cleanup: boolean): DoctorOptions {
8493
cleanup,
8594
staleAfterSeconds: Number.isFinite(staleAfterSeconds) ? staleAfterSeconds : 3600,
8695
archiveRoots,
96+
runRegistryRoots,
8797
}
8898
}
8999

@@ -94,6 +104,7 @@ async function buildDoctorOutput(options: DoctorOptions): Promise<DoctorOutput>
94104
await binaryCheck(),
95105
await sourceCheck(),
96106
await staleRecipeRunCheck(options),
107+
await tempRuntimeCheck(options),
97108
await archiveCheck(options),
98109
]
99110
const summary = {
@@ -112,6 +123,23 @@ async function buildDoctorOutput(options: DoctorOptions): Promise<DoctorOutput>
112123
}
113124
}
114125

126+
async function tempRuntimeCheck(options: DoctorOptions): Promise<HealthCheck> {
127+
const inventory = await inventoryTempRuntimeDirectories({
128+
cleanup: options.cleanup,
129+
staleAfterSeconds: options.staleAfterSeconds,
130+
runRegistryRoots: unique([...options.runRegistryRoots, resolve("artifacts", "runs")]),
131+
})
132+
const failures = inventory.entries.filter((row) => row.state === "failed")
133+
const removed = inventory.entries.filter((row) => row.state === "removed").length
134+
const status: HealthStatus = failures.length > 0 ? "error" : !options.cleanup && inventory.candidateCount > 0 ? "warning" : "ok"
135+
const message = options.cleanup
136+
? `removed ${removed}/${inventory.candidateCount} stale owned temp runtime director${inventory.candidateCount === 1 ? "y" : "ies"}`
137+
: inventory.candidateCount > 0
138+
? `${inventory.candidateCount} stale owned temp runtime director${inventory.candidateCount === 1 ? "y" : "ies"} found`
139+
: `no stale owned temp runtime directories found; retained ${inventory.retainedCount}`
140+
return { id: "wp-codebox.temp-runtime", status, message, details: inventory as unknown as Record<string, unknown> }
141+
}
142+
115143
async function commandToolCheck(id: string, args: string[]): Promise<HealthCheck> {
116144
try {
117145
const { stdout } = await execFile(id, args)
@@ -230,6 +258,10 @@ function archiveRootsFromEnv(): string[] {
230258
return (process.env.WP_CODEBOX_ARCHIVE_ROOTS ?? "").split(":").map((root) => root.trim()).filter(Boolean)
231259
}
232260

261+
function runRegistryRootsFromEnv(): string[] {
262+
return (process.env.WP_CODEBOX_RUN_REGISTRY_ROOTS ?? "").split(":").map((root) => root.trim()).filter(Boolean)
263+
}
264+
233265
function defaultArchiveRoots(): string[] {
234266
return [join(homedir(), ".cache", "wp-codebox"), join(homedir(), ".wp-codebox"), join(homedir(), ".cache", "wordpress-playground"), join(homedir(), ".wordpress-playground")]
235267
}
Lines changed: 232 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,232 @@
1+
import { lstat, opendir, readFile, readdir, realpath, rm, stat } from "node:fs/promises"
2+
import { tmpdir } from "node:os"
3+
import { basename, join, resolve, sep } from "node:path"
4+
5+
export type TempRuntimeRetentionReason = "recent" | "live-process" | "active-lease" | "recent-run" | "changed-before-cleanup"
6+
7+
export interface TempRuntimeCandidate {
8+
path: string
9+
kind: string
10+
ownershipEvidence: string
11+
ageSeconds: number
12+
allocatedBytes: number
13+
state: "candidate" | "retained" | "removed" | "failed"
14+
retainedReason?: TempRuntimeRetentionReason
15+
error?: string
16+
}
17+
18+
export interface TempRuntimeInventory {
19+
tempRoot: string
20+
scannedCount: number
21+
ownedCount: number
22+
candidateCount: number
23+
retainedCount: number
24+
retainedReasons: Record<string, number>
25+
estimatedAllocatedBytes: number
26+
reclaimedAllocatedBytes: number
27+
entries: TempRuntimeCandidate[]
28+
}
29+
30+
interface TempRuntimeCleanupOptions {
31+
cleanup: boolean
32+
staleAfterSeconds: number
33+
runRegistryRoots: string[]
34+
now?: number
35+
processRows?: ProcessRow[]
36+
}
37+
38+
interface ProcessRow { pid: number; command: string; references: string[] }
39+
interface DirectoryIdentity { device: number; inode: number }
40+
41+
const OWNED_PREFIXES: ReadonlyArray<readonly [prefix: string, kind: string]> = [
42+
["wp-codebox-agent-task-artifacts-", "agent-task-artifacts"],
43+
["wp-codebox-agent-task-recipe-", "agent-task-recipe"],
44+
["wp-codebox-dependency-consumer-", "dependency-consumer"],
45+
["wp-codebox-dependency-overlay-", "dependency-overlay"],
46+
["wp-codebox-fuzz-command-", "fuzz-command"],
47+
["wp-codebox-fuzz-workload-", "fuzz-workload"],
48+
["wp-codebox-git-head-", "git-head"],
49+
["wp-codebox-input-mount-baseline-", "input-mount-baseline"],
50+
["wp-codebox-mariadb-", "native-mariadb"],
51+
["wp-codebox-overlay-php-ai-client-", "runtime-overlay"],
52+
["wp-codebox-plugin-", "plugin-overlay"],
53+
["wp-codebox-readonly-mounts-", "readonly-mounts"],
54+
["wp-codebox-release-coverage-", "release-coverage"],
55+
["wp-codebox-release-", "release-package"],
56+
["wp-codebox-source-package-", "source-package"],
57+
["wp-codebox-source-", "source"],
58+
["wp-codebox-staged-file-", "staged-file"],
59+
["wp-codebox-workload-cli-", "workload-cli"],
60+
["wp-codebox-workspace-preload-", "workspace-preload"],
61+
]
62+
63+
export async function inventoryTempRuntimeDirectories(options: TempRuntimeCleanupOptions): Promise<TempRuntimeInventory> {
64+
const tempRoot = await realpath(tmpdir())
65+
const rootStat = await stat(tempRoot)
66+
const now = options.now ?? Date.now()
67+
const processes = options.processRows ?? await runtimeProcessRows()
68+
const globalProtection = await globalRuntimeProtection(processes, options.runRegistryRoots, now, options.staleAfterSeconds)
69+
const entries: TempRuntimeCandidate[] = []
70+
let scannedCount = 0
71+
72+
const directory = await opendir(tempRoot)
73+
for await (const entry of directory) {
74+
scannedCount++
75+
if (!entry.isDirectory() || entry.isSymbolicLink()) continue
76+
const path = join(tempRoot, entry.name)
77+
const ownership = await ownershipEvidence(path, entry.name)
78+
if (!ownership) continue
79+
80+
const before = await stat(path)
81+
if (!before.isDirectory() || before.dev !== rootStat.dev || !isDirectChild(tempRoot, path)) continue
82+
const usage = await allocatedUsage(path)
83+
const ageSeconds = Math.max(0, Math.floor((now - usage.latestMtimeMs) / 1000))
84+
const row: TempRuntimeCandidate = {
85+
path,
86+
kind: ownership.kind,
87+
ownershipEvidence: ownership.evidence,
88+
ageSeconds,
89+
allocatedBytes: usage.allocatedBytes,
90+
state: "candidate",
91+
}
92+
const retainedReason = ageSeconds < options.staleAfterSeconds
93+
? "recent"
94+
: processReferencesPath(processes, path)
95+
? "live-process"
96+
: globalProtection
97+
if (retainedReason) {
98+
row.state = "retained"
99+
row.retainedReason = retainedReason
100+
} else if (options.cleanup) {
101+
await removeCandidate(row, { device: before.dev, inode: before.ino }, tempRoot)
102+
}
103+
entries.push(row)
104+
}
105+
106+
const retainedReasons: Record<string, number> = {}
107+
for (const row of entries) {
108+
if (row.retainedReason) retainedReasons[row.retainedReason] = (retainedReasons[row.retainedReason] ?? 0) + 1
109+
}
110+
return {
111+
tempRoot,
112+
scannedCount,
113+
ownedCount: entries.length,
114+
candidateCount: entries.filter((row) => row.state === "candidate" || row.state === "removed" || row.state === "failed").length,
115+
retainedCount: entries.filter((row) => row.state === "retained").length,
116+
retainedReasons,
117+
estimatedAllocatedBytes: entries.filter((row) => row.state !== "retained").reduce((total, row) => total + row.allocatedBytes, 0),
118+
reclaimedAllocatedBytes: entries.filter((row) => row.state === "removed").reduce((total, row) => total + row.allocatedBytes, 0),
119+
entries,
120+
}
121+
}
122+
123+
async function ownershipEvidence(path: string, name: string): Promise<{ kind: string; evidence: string } | undefined> {
124+
const owned = OWNED_PREFIXES.find(([prefix]) => name.startsWith(prefix) && name.length > prefix.length)
125+
if (owned) return { kind: owned[1], evidence: `registered-producer-prefix:${owned[0]}` }
126+
if (!name.startsWith("node-playground-cli-site-") || name.length === "node-playground-cli-site-".length) return undefined
127+
const children = new Set(await readdir(path).catch(() => []))
128+
if (!children.has("wp-content") || (!children.has("wp-includes") && !children.has("wp-config.php"))) return undefined
129+
return { kind: "playground-site", evidence: "registered-upstream-prefix+wordpress-layout" }
130+
}
131+
132+
async function allocatedUsage(root: string): Promise<{ allocatedBytes: number; latestMtimeMs: number }> {
133+
const metadata = await stat(root)
134+
let allocatedBytes = metadata.blocks * 512
135+
let latestMtimeMs = metadata.mtimeMs
136+
const directory = await opendir(root)
137+
for await (const entry of directory) {
138+
const path = join(root, entry.name)
139+
const child = await lstat(path).catch(() => undefined)
140+
if (!child) continue
141+
allocatedBytes += child.blocks * 512
142+
latestMtimeMs = Math.max(latestMtimeMs, child.mtimeMs)
143+
if (entry.isDirectory() && !entry.isSymbolicLink()) {
144+
const nested = await allocatedUsage(path)
145+
allocatedBytes += nested.allocatedBytes - child.blocks * 512
146+
latestMtimeMs = Math.max(latestMtimeMs, nested.latestMtimeMs)
147+
}
148+
}
149+
return { allocatedBytes, latestMtimeMs }
150+
}
151+
152+
async function removeCandidate(row: TempRuntimeCandidate, identity: DirectoryIdentity, tempRoot: string): Promise<void> {
153+
try {
154+
const canonical = await realpath(row.path)
155+
const current = await stat(row.path)
156+
if (canonical !== resolve(row.path) || !isDirectChild(tempRoot, canonical) || current.dev !== identity.device || current.ino !== identity.inode) {
157+
row.state = "retained"
158+
row.retainedReason = "changed-before-cleanup"
159+
return
160+
}
161+
await rm(row.path, { recursive: true })
162+
row.state = "removed"
163+
} catch (error) {
164+
row.state = "failed"
165+
row.error = errorMessage(error)
166+
}
167+
}
168+
169+
async function globalRuntimeProtection(processes: ProcessRow[], registryRoots: string[], now: number, staleAfterSeconds: number): Promise<TempRuntimeRetentionReason | undefined> {
170+
if (processes.some((row) => row.pid !== process.pid && /wp-codebox/.test(row.command) && /(recipe-run|agent-task-run|run-agent-task|preview-lease-child)/.test(row.command))) return "live-process"
171+
for (const root of registryRoots) {
172+
const leaseProtection = await activeLeaseInRegistry(root, now)
173+
if (leaseProtection) return "active-lease"
174+
if (await recentRunInRegistry(root, now, staleAfterSeconds)) return "recent-run"
175+
}
176+
return undefined
177+
}
178+
179+
async function activeLeaseInRegistry(root: string, now: number): Promise<boolean> {
180+
const leaseDirectory = join(resolve(root), "preview-leases")
181+
for (const name of await readdir(leaseDirectory).catch(() => [])) {
182+
if (!name.endsWith(".json")) continue
183+
const lease = await readJson(join(leaseDirectory, name))
184+
if (!lease || !["starting", "available", "release_requested"].includes(String(lease.status))) continue
185+
const expiresAt = Date.parse(String(lease.expiresAt ?? ""))
186+
if (!Number.isFinite(expiresAt) || expiresAt > now) return true
187+
}
188+
return false
189+
}
190+
191+
async function recentRunInRegistry(root: string, now: number, staleAfterSeconds: number): Promise<boolean> {
192+
for (const name of await readdir(resolve(root)).catch(() => [])) {
193+
if (!name.endsWith(".json")) continue
194+
const run = await readJson(join(resolve(root), name))
195+
if (run?.schema !== "wp-codebox/run-registry-entry/v1") continue
196+
const heartbeat = Date.parse(String(run.heartbeatAt ?? run.updatedAt ?? ""))
197+
if (Number.isFinite(heartbeat) && now - heartbeat < staleAfterSeconds * 1000) return true
198+
}
199+
return false
200+
}
201+
202+
async function readJson(path: string): Promise<Record<string, unknown> | undefined> {
203+
try { return JSON.parse(await readFile(path, "utf8")) as Record<string, unknown> } catch { return undefined }
204+
}
205+
206+
async function runtimeProcessRows(): Promise<ProcessRow[]> {
207+
if (process.platform !== "linux") return []
208+
const rows: ProcessRow[] = []
209+
for (const name of await readdir("/proc").catch(() => [])) {
210+
if (!/^\d+$/.test(name)) continue
211+
const pid = Number.parseInt(name, 10)
212+
const command = (await readFile(`/proc/${pid}/cmdline`, "utf8").catch(() => "")).replace(/\0/g, " ").trim()
213+
if (!command) continue
214+
const references = [await realpath(`/proc/${pid}/cwd`).catch(() => "")]
215+
for (const fd of await readdir(`/proc/${pid}/fd`).catch(() => [])) references.push(await realpath(`/proc/${pid}/fd/${fd}`).catch(() => ""))
216+
rows.push({ pid, command, references: references.filter(Boolean) })
217+
}
218+
return rows
219+
}
220+
221+
function processReferencesPath(processes: ProcessRow[], path: string): boolean {
222+
const prefix = `${resolve(path)}${sep}`
223+
return processes.some((row) => row.pid !== process.pid && (row.command.includes(path) || row.references.some((reference) => reference === path || reference.startsWith(prefix))))
224+
}
225+
226+
function isDirectChild(root: string, path: string): boolean {
227+
return resolve(path).startsWith(`${resolve(root)}${sep}`) && basename(path) !== "" && resolve(join(path, "..")) === resolve(root)
228+
}
229+
230+
function errorMessage(error: unknown): string {
231+
return error instanceof Error ? error.message : String(error)
232+
}

packages/cli/src/output.ts

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -303,8 +303,8 @@ export function printHelp(): void {
303303
wp-codebox commands [--json]
304304
wp-codebox runtime descriptor [--json]
305305
wp-codebox schema recipe [--json]
306-
wp-codebox doctor [--json] [--fix] [--archive-root <dir>] [--stale-after-seconds <n>]
307-
wp-codebox cleanup [--json] [--archive-root <dir>] [--stale-after-seconds <n>]
306+
wp-codebox doctor [--json] [--fix] [--archive-root <dir>] [--run-registry <dir>] [--stale-after-seconds <n>]
307+
wp-codebox cleanup [--json] [--archive-root <dir>] [--run-registry <dir>] [--stale-after-seconds <n>]
308308
wp-codebox workspace-policy check --workspace-root <path> --writable-root <path> [options]
309309
wp-codebox recipe build phpunit|bench|template|generic-ability-runtime-run|runtime-package-run --options <path> [--output <path>]
310310
wp-codebox recipe validate --recipe <path> [--policy <json|file>] [--json]
@@ -437,12 +437,14 @@ Options:
437437
--json Emit machine-readable JSON.
438438
439439
Doctor and cleanup:
440-
doctor Report binary/source fingerprint, Node/npm availability, stale recipe-run processes, and corrupt archives.
441-
cleanup Run doctor checks and remove safe stale/corrupt runtime state.
440+
doctor Report binary/source fingerprint, stale processes, owned temp runtimes, and corrupt archives.
441+
cleanup Remove only age-gated, inactive owned temp runtimes and safe corrupt runtime state.
442442
--fix Allow mutating cleanup when command is doctor.
443443
--stale-after-seconds <n>
444-
Age threshold for stale recipe-run processes. Defaults to 3600.
444+
Age threshold for stale recipe-run processes and orphaned temp runtimes. Defaults to 3600.
445445
--archive-root <dir> Additional archive/cache root to scan for invalid .zip files. Repeatable.
446+
--run-registry <dir> Run registry checked for active preview leases and recent run heartbeats. Repeatable.
447+
Defaults to ./artifacts/runs; additional defaults may be set with WP_CODEBOX_RUN_REGISTRY_ROOTS.
446448
447449
Workspace policy:
448450
--workspace-root <dir>

0 commit comments

Comments
 (0)