feat: auto-delete stopped workspaces after configurable TTL (#916)

* task: move stopped-workspace-auto-delete to active

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* feat: auto-delete stopped workspaces after configurable TTL

Extend NodeLifecycle DO to schedule workspace deletion 5 minutes after
a workspace stops. This prevents disk exhaustion on nodes caused by
accumulated stopped workspaces (Docker volumes with git repos, node_modules).

- Add DEFAULT_WORKSPACE_STOPPED_TTL_MS (300000ms) to shared constants
- Add scheduleWorkspaceDeletion/cancelWorkspaceDeletion to NodeLifecycle DO
- Alarm handler processes expired deletions (VM agent DELETE + D1 update)
- recalculateAlarm picks earliest of warm timeout and pending deletions
- Schedule deletion from: stop route, cleanupTaskRun, cleanupOnFailure
- Cancel deletion from: restart route (before TTL expires)
- Cron safety-net in node-cleanup.ts for missed DO alarms
- Configurable via WORKSPACE_STOPPED_TTL_MS env var

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix(test): update existing tests for recalculateAlarm and new cleanup counter

- warm-node-pooling integration test: setAlarm → recalculateAlarm, deleteAlarm → recalculateAlarm(null)
- node-cleanup unit test: add stoppedWorkspacesDeleted to expected result, mock node-agent and project-data

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* task: archive stopped-workspace-auto-delete task

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix: address cloudflare-specialist review findings

- [HIGH] Use deleteWorkspaceOnNode shared helper with proper JWT auth
  instead of raw IP fetch with X-User-ID header
- [HIGH] Hoist getStoredState() before the deletion loop to avoid N
  redundant DO storage reads
- [MEDIUM] Add status='stopped' guard to cron sweep D1 update (TOCTOU)
- [MEDIUM] Use recalculateAlarm in destroying branch so pending workspace
  deletions are not delayed by retry window

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* feat: wire scheduleWorkspaceDeletion into idle cleanup path

The processExpiredCleanups idle timeout path stops workspaces but
wasn't scheduling automatic deletion. Workspaces stopped via idle
timeout now get deletion scheduled via the NodeLifecycle DO, matching
the lifecycle route and task-runner paths.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* ci: trigger fresh CI run with updated PR body

---------

Co-authored-by: Raphaël Titsworth-Morin <raphael@raphaeltm.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: 3 people

apps/api/.env.example

@@ -229,6 +229,7 @@ BASE_DOMAIN=workspaces.example.com
 
 # Workspace idle timeout (ProjectData DO)
 # WORKSPACE_IDLE_TIMEOUT_MS=7200000      # 2 hours — global default idle timeout before workspace is stopped (overridable per-project)
+# WORKSPACE_STOPPED_TTL_MS=300000        # 5 minutes — auto-delete stopped workspaces after this TTL
 
 # Task agent configuration
 # DEFAULT_TASK_AGENT_TYPE=opencode     # Agent used for autonomous task execution

apps/api/src/durable-objects/node-lifecycle.ts

@@ -14,25 +14,34 @@
  *   alarm()       → on `warm`: sets `destroying`, updates D1, cron handles teardown
  *                 → on `active`: no-op (was claimed between schedule and fire)
  *
+ * Workspace auto-deletion:
+ *   scheduleWorkspaceDeletion(workspaceId, userId) → stores pending deletion, recalculates alarm
+ *   cancelWorkspaceDeletion(workspaceId) → removes pending deletion, recalculates alarm
+ *   alarm() → also processes expired workspace deletions (calls VM agent, updates D1)
+ *
  * Actual infrastructure destruction (Hetzner API, DNS) is handled by the
  * cron sweep, NOT by this DO — because user credentials are encrypted in D1
  * and must be decrypted with CREDENTIAL_ENCRYPTION_KEY (or ENCRYPTION_KEY
  * fallback) in the worker context via getCredentialEncryptionKey(env).
  *
  * See: specs/021-task-chat-architecture/tasks.md (Phase 5)
  */
-import type { NodeLifecycleState,NodeLifecycleStatus } from '@simple-agent-manager/shared';
+import type { NodeLifecycleState, NodeLifecycleStatus } from '@simple-agent-manager/shared';
 import {
   DEFAULT_NODE_LIFECYCLE_ALARM_RETRY_MS,
   DEFAULT_NODE_WARM_TIMEOUT_MS,
+  DEFAULT_WORKSPACE_STOPPED_TTL_MS,
 } from '@simple-agent-manager/shared';
 import { DurableObject } from 'cloudflare:workers';
 
+import type { Env } from '../env';
 import { log } from '../lib/logger';
+import { deleteWorkspaceOnNode } from '../services/node-agent';
 
 type NodeLifecycleEnv = {
   DATABASE: D1Database;
   NODE_WARM_TIMEOUT_MS?: string;
+  WORKSPACE_STOPPED_TTL_MS?: string;
 };
 
 interface StoredState {
@@ -45,6 +54,12 @@ interface StoredState {
   warmTimeoutOverrideMs?: number | null;
 }
 
+interface PendingWorkspaceDeletion {
+  workspaceId: string;
+  userId: string;
+  deleteAt: number;
+}
+
 export class NodeLifecycle extends DurableObject<NodeLifecycleEnv> {
   /**
    * Mark a node as idle (warm). Called after the last workspace on the node
@@ -73,8 +88,8 @@ export class NodeLifecycle extends DurableObject<NodeLifecycleEnv> {
     };
     await this.ctx.storage.put('state', newState);
 
-    // Schedule (or reschedule) alarm
-    await this.ctx.storage.setAlarm(now + warmTimeout);
+    // Recalculate alarm considering both warm timeout and pending workspace deletions
+    await this.recalculateAlarm(now + warmTimeout);
 
     // Update D1 warm_since column
     await this.updateD1WarmSince(nodeId, new Date(now).toISOString());
@@ -84,7 +99,7 @@ export class NodeLifecycle extends DurableObject<NodeLifecycleEnv> {
 
   /**
    * Mark a node as active. Called when a workspace starts on the node.
-   * Cancels any pending warm timeout alarm.
+   * Cancels any pending warm timeout alarm but preserves workspace deletion alarms.
    */
   async markActive(): Promise<NodeLifecycleState> {
     const state = await this.getStoredState();
@@ -97,8 +112,8 @@ export class NodeLifecycle extends DurableObject<NodeLifecycleEnv> {
     state.warmSince = null;
     await this.ctx.storage.put('state', state);
 
-    // Cancel any pending alarm
-    await this.ctx.storage.deleteAlarm();
+    // Recalculate alarm — pending workspace deletions still need to fire
+    await this.recalculateAlarm(null);
 
     // Clear D1 warm_since
     await this.updateD1WarmSince(state.nodeId, null);
@@ -128,8 +143,8 @@ export class NodeLifecycle extends DurableObject<NodeLifecycleEnv> {
     state.warmSince = null;
     await this.ctx.storage.put('state', state);
 
-    // Cancel alarm
-    await this.ctx.storage.deleteAlarm();
+    // Recalculate alarm — pending workspace deletions still need to fire
+    await this.recalculateAlarm(null);
 
     // Clear D1 warm_since
     await this.updateD1WarmSince(state.nodeId, null);
@@ -148,19 +163,65 @@ export class NodeLifecycle extends DurableObject<NodeLifecycleEnv> {
     return this.toPublicState(state);
   }
 
+  // =========================================================================
+  // Workspace auto-deletion scheduling
+  // =========================================================================
+
+  /**
+   * Schedule a stopped workspace for automatic deletion after the configured TTL.
+   * Called when a workspace transitions to 'stopped' status.
+   */
+  async scheduleWorkspaceDeletion(workspaceId: string, userId: string): Promise<void> {
+    const ttl = this.getWorkspaceStoppedTtlMs();
+    const deleteAt = Date.now() + ttl;
+
+    const entry: PendingWorkspaceDeletion = { workspaceId, userId, deleteAt };
+    await this.ctx.storage.put(`ws-delete:${workspaceId}`, entry);
+
+    log.info('node_lifecycle.workspace_deletion_scheduled', {
+      workspaceId,
+      userId,
+      deleteAt: new Date(deleteAt).toISOString(),
+      ttlMs: ttl,
+    });
+
+    await this.recalculateAlarm(await this.getWarmAlarmTime());
+  }
+
   /**
-   * Alarm handler. Fires when the warm timeout expires.
+   * Cancel a pending workspace deletion. Called when a workspace is restarted
+   * before the TTL expires.
+   */
+  async cancelWorkspaceDeletion(workspaceId: string): Promise<void> {
+    await this.ctx.storage.delete(`ws-delete:${workspaceId}`);
+
+    log.info('node_lifecycle.workspace_deletion_cancelled', { workspaceId });
+
+    await this.recalculateAlarm(await this.getWarmAlarmTime());
+  }
+
+  // =========================================================================
+  // Alarm handler
+  // =========================================================================
+
+  /**
+   * Alarm handler. Fires when either:
+   * 1. The warm timeout expires (node should be destroyed)
+   * 2. A workspace deletion is due
    *
-   * If the node is still warm, transitions to `destroying` and marks D1
-   * for the cron sweep to handle actual infrastructure teardown.
-   * If the node was claimed between schedule and fire, this is a no-op.
+   * Processes expired workspace deletions first, then handles warm timeout.
    */
   async alarm(): Promise<void> {
+    // Process any expired workspace deletions
+    await this.processExpiredDeletions();
+
     const state = await this.getStoredState();
     if (!state) return;
 
     // No-op if node was claimed (active) or already destroying
     if (state.status === 'active') {
+      // Still recalculate alarm for any remaining pending workspace deletions
+      await this.recalculateAlarm(null);
       return;
     }
 
@@ -171,7 +232,18 @@ export class NodeLifecycle extends DurableObject<NodeLifecycleEnv> {
       return;
     }
 
-    // status === 'warm' → transition to destroying
+    // status === 'warm' → check if warm timeout has actually expired
+    if (state.warmSince) {
+      const warmTimeout = state.warmTimeoutOverrideMs ?? this.getWarmTimeoutMs();
+      const warmExpiry = state.warmSince + warmTimeout;
+      if (Date.now() < warmExpiry) {
+        // Warm timeout hasn't expired yet — alarm fired for workspace deletion only
+        await this.recalculateAlarm(warmExpiry);
+        return;
+      }
+    }
+
+    // Warm timeout expired → transition to destroying
     state.status = 'destroying';
     await this.ctx.storage.put('state', state);
 
@@ -193,8 +265,8 @@ export class NodeLifecycle extends DurableObject<NodeLifecycleEnv> {
         nodeId: state.nodeId,
         error: err instanceof Error ? err.message : String(err),
       });
-      // Schedule retry
-      await this.ctx.storage.setAlarm(Date.now() + DEFAULT_NODE_LIFECYCLE_ALARM_RETRY_MS);
+      // Schedule retry (use recalculateAlarm to not delay pending workspace deletions)
+      await this.recalculateAlarm(Date.now() + DEFAULT_NODE_LIFECYCLE_ALARM_RETRY_MS);
     }
   }
 
@@ -215,6 +287,15 @@ export class NodeLifecycle extends DurableObject<NodeLifecycleEnv> {
     return DEFAULT_NODE_WARM_TIMEOUT_MS;
   }
 
+  private getWorkspaceStoppedTtlMs(): number {
+    const envValue = this.env.WORKSPACE_STOPPED_TTL_MS;
+    if (envValue) {
+      const parsed = parseInt(envValue, 10);
+      if (Number.isFinite(parsed) && parsed > 0) return parsed;
+    }
+    return DEFAULT_WORKSPACE_STOPPED_TTL_MS;
+  }
+
   private toPublicState(state: StoredState): NodeLifecycleState {
     return {
       nodeId: state.nodeId,
@@ -238,4 +319,114 @@ export class NodeLifecycle extends DurableObject<NodeLifecycleEnv> {
       });
     }
   }
+
+  /**
+   * Get all pending workspace deletions from DO storage.
+   */
+  private async getPendingDeletions(): Promise<Map<string, PendingWorkspaceDeletion>> {
+    return await this.ctx.storage.list<PendingWorkspaceDeletion>({ prefix: 'ws-delete:' });
+  }
+
+  /**
+   * Process all workspace deletions whose deleteAt time has passed.
+   */
+  private async processExpiredDeletions(): Promise<void> {
+    const pending = await this.getPendingDeletions();
+    const now = Date.now();
+
+    // Load state once to get nodeId — avoids N storage reads in the loop
+    const state = await this.getStoredState();
+    if (!state) return;
+
+    for (const [key, entry] of pending) {
+      if (entry.deleteAt > now) continue;
+
+      try {
+        await this.deleteWorkspace(state.nodeId, entry.workspaceId, entry.userId);
+        await this.ctx.storage.delete(key);
+
+        log.info('node_lifecycle.workspace_auto_deleted', {
+          workspaceId: entry.workspaceId,
+          userId: entry.userId,
+        });
+      } catch (err) {
+        log.error('node_lifecycle.workspace_deletion_failed', {
+          workspaceId: entry.workspaceId,
+          userId: entry.userId,
+          error: err instanceof Error ? err.message : String(err),
+        });
+        // Leave the entry for retry on next alarm. Push deleteAt forward slightly
+        // to avoid tight retry loops.
+        entry.deleteAt = now + DEFAULT_NODE_LIFECYCLE_ALARM_RETRY_MS;
+        await this.ctx.storage.put(key, entry);
+      }
+    }
+  }
+
+  /**
+   * Delete a workspace: call VM agent to remove Docker container + volume,
+   * then update D1 status to 'deleted'.
+   */
+  private async deleteWorkspace(nodeId: string, workspaceId: string, userId: string): Promise<void> {
+    // Call VM agent DELETE endpoint via shared helper (handles JWT auth, proper URL routing)
+    try {
+      await deleteWorkspaceOnNode(nodeId, workspaceId, this.env as unknown as Env, userId);
+    } catch (err) {
+      // If the node is unreachable (already destroyed), log but don't fail
+      // The D1 status update below still marks the workspace as deleted
+      log.warn('node_lifecycle.workspace_delete_vm_agent_failed', {
+        workspaceId,
+        nodeId,
+        error: err instanceof Error ? err.message : String(err),
+      });
+    }
+
+    // Update D1 workspace status to 'deleted'
+    const now = new Date().toISOString();
+    await this.env.DATABASE.prepare(
+      `UPDATE workspaces SET status = 'deleted', updated_at = ? WHERE id = ? AND status = 'stopped'`
+    ).bind(now, workspaceId).run();
+
+    // Clean up any agent_sessions referencing this workspace (best-effort)
+    try {
+      await this.env.DATABASE.prepare(
+        `UPDATE agent_sessions SET status = 'completed', updated_at = ? WHERE workspace_id = ? AND status NOT IN ('completed', 'failed')`
+      ).bind(now, workspaceId).run();
+    } catch {
+      // best-effort
+    }
+  }
+
+  /**
+   * Get the warm alarm time if the node is in warm state.
+   */
+  private async getWarmAlarmTime(): Promise<number | null> {
+    const state = await this.getStoredState();
+    if (!state || state.status !== 'warm' || !state.warmSince) return null;
+    const warmTimeout = state.warmTimeoutOverrideMs ?? this.getWarmTimeoutMs();
+    return state.warmSince + warmTimeout;
+  }
+
+  /**
+   * Recalculate and set the alarm to the earliest time needed:
+   * either the warm timeout expiry or the earliest pending workspace deletion.
+   *
+   * @param warmAlarmTime - The warm timeout expiry time, or null if not applicable
+   */
+  private async recalculateAlarm(warmAlarmTime: number | null): Promise<void> {
+    let earliest = warmAlarmTime;
+
+    const pending = await this.getPendingDeletions();
+    for (const [, entry] of pending) {
+      if (earliest === null || entry.deleteAt < earliest) {
+        earliest = entry.deleteAt;
+      }
+    }
+
+    if (earliest !== null) {
+      await this.ctx.storage.setAlarm(earliest);
+    } else {
+      await this.ctx.storage.deleteAlarm();
+    }
+  }
 }

apps/api/src/durable-objects/project-data/index.ts

@@ -350,7 +350,24 @@ export class ProjectData extends DurableObject<Env> {
       (type, payload, sid) => this.broadcastEvent(type, payload, sid), () => this.scheduleSummarySync());
     await idleCleanup.processExpiredCleanups(this.sql, this.env,
       (taskId) => idleCleanup.completeTaskInD1(this.env.DATABASE, taskId),
-      (workspaceId) => idleCleanup.stopWorkspaceInD1(this.env.DATABASE, workspaceId),
+      async (workspaceId) => {
+        await idleCleanup.stopWorkspaceInD1(this.env.DATABASE, workspaceId);
+        // Schedule automatic deletion after TTL (best-effort)
+        try {
+          const workerEnv = this.env as unknown as import('../../env').Env;
+          const wsRow = await workerEnv.DATABASE.prepare(
+            'SELECT node_id, user_id FROM workspaces WHERE id = ?'
+          ).bind(workspaceId).first<{ node_id: string | null; user_id: string }>();
+          if (wsRow?.node_id) {
+            const doId = workerEnv.NODE_LIFECYCLE.idFromName(wsRow.node_id);
+            const stub = workerEnv.NODE_LIFECYCLE.get(doId);
+            await (stub as unknown as import('../node-lifecycle').NodeLifecycle)
+              .scheduleWorkspaceDeletion(workspaceId, wsRow.user_id);
+          }
+        } catch {
+          // Best-effort — cron safety-net will catch it
+        }
+      },
       (type, payload, sid) => this.broadcastEvent(type, payload, sid), () => this.scheduleSummarySync());
 
     // Mailbox delivery sweep: expire stale messages and re-queue unacked ones

apps/api/src/durable-objects/task-runner/state-machine.ts

@@ -350,6 +350,20 @@ export async function cleanupOnFailure(
         error: err instanceof Error ? err.message : String(err),
       });
     }
+
+    // Schedule automatic deletion after TTL (best-effort)
+    try {
+      const doId = rc.env.NODE_LIFECYCLE.idFromName(state.stepResults.nodeId);
+      const stub = rc.env.NODE_LIFECYCLE.get(doId);
+      await (stub as unknown as import('../node-lifecycle').NodeLifecycle)
+        .scheduleWorkspaceDeletion(state.stepResults.workspaceId, state.userId);
+    } catch (err) {
+      log.warn('task_runner_do.cleanup.schedule_deletion_failed', {
+        taskId: state.taskId,
+        workspaceId: state.stepResults.workspaceId,
+        error: err instanceof Error ? err.message : String(err),
+      });
+    }
   }
 
   // Clean up auto-provisioned node. If a workspace exists, cleanupTaskRun

apps/api/src/env.ts

@@ -139,6 +139,8 @@ export interface Env {
   ORPHANED_WORKSPACE_GRACE_PERIOD_MS?: string;
   // Workspace idle timeout (global default, overridable per-project)
   WORKSPACE_IDLE_TIMEOUT_MS?: string;
+  // Auto-delete stopped workspaces after this TTL (default: 300000 = 5 minutes)
+  WORKSPACE_STOPPED_TTL_MS?: string;
   // Task agent configuration
   DEFAULT_TASK_AGENT_TYPE?: string;
   // Built-in profile model overrides (defaults: claude-sonnet-4-5-20250929, claude-opus-4-6)