Phase 3a: runtime-neutral cf-container session hibernate/wake/restore (#1562)

Phase 3a: runtime-neutral session hibernate/wake

Author: simple-agent-manager[bot]

.claude/skills/env-reference/SKILL.md

@@ -66,7 +66,13 @@ See `apps/api/.env.example` for the full list. Key variables:
 - `CF_CONTAINER_SLEEP_AFTER` — Container idle sleep duration for instant-session runtime (default: `10m`)
 - `CF_CONTAINER_VM_AGENT_PORT` — vm-agent standalone HTTP port inside the raw container (default: `8080`)
 - `CF_CONTAINER_PORT_READY_TIMEOUT_MS` — Max wait for vm-agent port readiness (default: `30000`)
-- `CF_CONTAINER_WORKSPACE_BASE_DIR` — Base checkout directory inside raw containers (default: `/workspaces`)
+- `$1
+- `SESSION_SNAPSHOT_TTL_DAYS` — Retention for hibernated session snapshots; deployment also provisions matching R2 prefix expiration (default: `7`)
+- `SESSION_SNAPSHOT_TOTAL_BUDGET_BYTES` — Maximum bytes accepted for one snapshot artifact (default: `104857600`)
+- `SESSION_SNAPSHOT_ENTRY_THRESHOLD_BYTES` — Per-file threshold before snapshot content is visibly skipped (default: `52428800`)
+- `SESSION_SNAPSHOT_TRANSFER_IDLE_TIMEOUT_MS` — Progress-idle timeout for snapshot upload/download (default: `30000`)
+- `SESSION_SNAPSHOT_JSON_BODY_MAX_BYTES` — Maximum snapshot coordination JSON body (default: `262144`)
+- `SESSION_SNAPSHOT_R2_PREFIX` — Private R2 object prefix for session snapshots (default: `session-snapshots`)
 
 ### Devcontainer Cache
 

.github/workflows/deploy-reusable.yml

@@ -358,7 +358,9 @@ jobs:
           HETZNER_BASE_IMAGE: ${{ vars.HETZNER_BASE_IMAGE }}
           ARTIFACTS_BINDING_ENABLED: ${{ vars.ARTIFACTS_BINDING_ENABLED }}
           CF_CONTAINER_ENABLED: ${{ vars.CF_CONTAINER_ENABLED }}
+          CF_CONTAINER_SLEEP_AFTER: ${{ vars.CF_CONTAINER_SLEEP_AFTER }}
           CF_CONTAINER_PORT_READY_TIMEOUT_MS: ${{ vars.CF_CONTAINER_PORT_READY_TIMEOUT_MS }}
+          CF_CONTAINER_WAKE_TIMEOUT_MS: ${{ vars.CF_CONTAINER_WAKE_TIMEOUT_MS }}
           CF_CONTAINER_VM_AGENT_PORT: ${{ vars.CF_CONTAINER_VM_AGENT_PORT }}
           SANDBOX_ENABLED: ${{ vars.SANDBOX_ENABLED }}
           SANDBOX_EXEC_TIMEOUT_MS: ${{ vars.SANDBOX_EXEC_TIMEOUT_MS }}

apps/api/.env.example

@@ -52,6 +52,12 @@ BASE_DOMAIN=workspaces.example.com
 # CF_CONTAINER_VM_AGENT_PORT=8080
 # CF_CONTAINER_PORT_READY_TIMEOUT_MS=30000
 # CF_CONTAINER_WORKSPACE_BASE_DIR=/workspaces
+# SESSION_SNAPSHOT_TTL_DAYS=7
+# SESSION_SNAPSHOT_TOTAL_BUDGET_BYTES=104857600
+# SESSION_SNAPSHOT_ENTRY_THRESHOLD_BYTES=52428800
+# SESSION_SNAPSHOT_TRANSFER_IDLE_TIMEOUT_MS=30000
+# SESSION_SNAPSHOT_JSON_BODY_MAX_BYTES=262144
+# SESSION_SNAPSHOT_R2_PREFIX=session-snapshots
 
 # User approval / invite-only mode
 # When "true", new users require admin approval before accessing the platform.

apps/api/src/db/migrations/0091_session_snapshots.sql

@@ -0,0 +1,33 @@
+-- Runtime-neutral hibernate/restore snapshots for agent sessions.
+CREATE TABLE session_snapshots (
+  id TEXT PRIMARY KEY,
+  project_id TEXT REFERENCES projects(id) ON DELETE SET NULL,
+  workspace_id TEXT REFERENCES workspaces(id) ON DELETE SET NULL,
+  node_id TEXT REFERENCES nodes(id) ON DELETE SET NULL,
+  user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+  chat_session_id TEXT NOT NULL,
+  agent_session_id TEXT,
+  runtime TEXT NOT NULL,
+  status TEXT NOT NULL DEFAULT 'pending',
+  degradation TEXT NOT NULL DEFAULT 'none',
+  home_r2_key TEXT,
+  wip_r2_key TEXT,
+  manifest_r2_key TEXT NOT NULL,
+  base_commit TEXT,
+  expires_at TEXT NOT NULL,
+  manifest_json TEXT,
+  restore_status TEXT,
+  restore_message TEXT,
+  restored_at TEXT,
+  created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE UNIQUE INDEX idx_session_snapshots_chat_session_id
+  ON session_snapshots(chat_session_id);
+
+CREATE INDEX idx_session_snapshots_workspace_id
+  ON session_snapshots(workspace_id);
+
+CREATE INDEX idx_session_snapshots_expires_at
+  ON session_snapshots(expires_at);

apps/api/src/db/schema.ts

@@ -1044,6 +1044,47 @@ export const agentSessions = sqliteTable(
   })
 );
 
+// =============================================================================
+// Runtime-neutral Session Snapshots
+// =============================================================================
+export const sessionSnapshots = sqliteTable(
+  'session_snapshots',
+  {
+    id: text('id').primaryKey(),
+    projectId: text('project_id').references(() => projects.id, { onDelete: 'set null' }),
+    workspaceId: text('workspace_id').references(() => workspaces.id, { onDelete: 'set null' }),
+    nodeId: text('node_id').references(() => nodes.id, { onDelete: 'set null' }),
+    userId: text('user_id')
+      .notNull()
+      .references(() => users.id, { onDelete: 'cascade' }),
+    chatSessionId: text('chat_session_id').notNull(),
+    agentSessionId: text('agent_session_id'),
+    runtime: text('runtime').notNull(),
+    status: text('status').notNull().default('pending'),
+    degradation: text('degradation').notNull().default('none'),
+    homeR2Key: text('home_r2_key'),
+    wipR2Key: text('wip_r2_key'),
+    manifestR2Key: text('manifest_r2_key').notNull(),
+    baseCommit: text('base_commit'),
+    expiresAt: text('expires_at').notNull(),
+    manifestJson: text('manifest_json'),
+    restoreStatus: text('restore_status'),
+    restoreMessage: text('restore_message'),
+    restoredAt: text('restored_at'),
+    createdAt: text('created_at')
+      .notNull()
+      .default(sql`CURRENT_TIMESTAMP`),
+    updatedAt: text('updated_at')
+      .notNull()
+      .default(sql`CURRENT_TIMESTAMP`),
+  },
+  (table) => ({
+    chatSessionIdUnique: uniqueIndex('idx_session_snapshots_chat_session_id').on(table.chatSessionId),
+    workspaceIdIdx: index('idx_session_snapshots_workspace_id').on(table.workspaceId),
+    expiresAtIdx: index('idx_session_snapshots_expires_at').on(table.expiresAt),
+  })
+);
+
 // =============================================================================
 // Agent Settings (per-user, per-agent configuration)
 // =============================================================================
@@ -1545,6 +1586,8 @@ export type Workspace = typeof workspaces.$inferSelect;
 export type NewWorkspace = typeof workspaces.$inferInsert;
 export type AgentSession = typeof agentSessions.$inferSelect;
 export type NewAgentSession = typeof agentSessions.$inferInsert;
+export type SessionSnapshot = typeof sessionSnapshots.$inferSelect;
+export type NewSessionSnapshot = typeof sessionSnapshots.$inferInsert;
 export type UIStandard = typeof uiStandards.$inferSelect;
 export type NewUIStandard = typeof uiStandards.$inferInsert;
 export type ThemeToken = typeof themeTokens.$inferSelect;

apps/api/src/durable-objects/vm-agent-container.ts

@@ -1,10 +1,11 @@
 import { Container, switchPort } from '@cloudflare/containers';
-import { eq } from 'drizzle-orm';
+import { desc, eq } from 'drizzle-orm';
 import { drizzle } from 'drizzle-orm/d1';
 
 import * as schema from '../db/schema';
 import type { Env } from '../env';
 import { log } from '../lib/logger';
+import { signCallbackToken, signNodeCallbackToken, signNodeManagementToken } from '../services/jwt';
 
 export const DEFAULT_CF_CONTAINER_SLEEP_AFTER = '1h';
 export const DEFAULT_CF_CONTAINER_ACTIVE_WORK_MAX_MS = 2 * 60 * 60 * 1000;
@@ -45,14 +46,20 @@ interface ActiveWorkState {
 
 const ACTIVE_WORK_KEY = 'activeWork';
 const KEEPALIVE_CALLBACK = 'renewActiveWorkKeepalive';
-const SLEEPING_RESPONSE = 'Container is asleep; wake/rehydrate is not implemented yet.';
+const WAKE_DEGRADED_RESPONSE = 'Workspace woke with degraded snapshot restore; retry the prompt or fork from transcript history.';
 
 export class VmAgentContainer extends Container<Env> {
   defaultPort = 8080;
   requiredPorts = [8080];
   sleepAfter = DEFAULT_CF_CONTAINER_SLEEP_AFTER;
   enableInternet = true;
 
+  // Serializes wake-from-snapshot so two concurrent requests to a sleeping
+  // container do not both launch a fresh container + restore. DO request
+  // handlers interleave across `await`, so the sleeping-check + wake must run
+  // as one critical section (see .claude/rules/45).
+  private wakeChain: Promise<unknown> = Promise.resolve();
+
   constructor(ctx: DurableObjectState<Record<string, never>>, env: Env) {
     super(ctx, env);
     const configuredPort = Number.parseInt(env.CF_CONTAINER_VM_AGENT_PORT || env.SANDBOX_VM_AGENT_PORT || '', 10);
@@ -108,12 +115,18 @@ export class VmAgentContainer extends Container<Env> {
   }
 
   async proxyHttp(request: Request, port?: number): Promise<Response> {
-    const state = await this.getState();
+    let state = await this.getState();
     const lifecycleStatus = await this.ctx.storage.get<LifecycleStatus>('lifecycleStatus');
     if (lifecycleStatus === 'sleeping') {
-      // Phase 3 of idea 01KX4KSXEXQMP41KS34TW9EN01 will replace this temporary
-      // response with wake/rehydrate before proxying the request.
-      return new Response(SLEEPING_RESPONSE, { status: 503 });
+      const wake = await this.ensureAwake();
+      if (!wake.ok) {
+        return new Response(wake.message || WAKE_DEGRADED_RESPONSE, { status: 503 });
+      }
+      // wakeFromSnapshot launched a fresh container and restored the session.
+      // Re-read the container state so the stopped-check below reflects the
+      // now-running container instead of the pre-wake stopped snapshot,
+      // otherwise a successfully-woken session is wrongly rejected with 410.
+      state = await this.getState();
     }
     if (state.status === 'stopped' || state.status === 'stopped_with_code') {
       return new Response('Container is stopped; create a new instant session.', { status: 410 });
@@ -232,12 +245,16 @@ export class VmAgentContainer extends Container<Env> {
   }
 
   override async fetch(request: Request): Promise<Response> {
-    const state = await this.getState();
+    let state = await this.getState();
     const lifecycleStatus = await this.ctx.storage.get<LifecycleStatus>('lifecycleStatus');
     if (lifecycleStatus === 'sleeping') {
-      // Phase 3 of idea 01KX4KSXEXQMP41KS34TW9EN01 will replace this temporary
-      // response with wake/rehydrate before proxying the request.
-      return new Response(SLEEPING_RESPONSE, { status: 503 });
+      const wake = await this.ensureAwake();
+      if (!wake.ok) {
+        return new Response(wake.message || WAKE_DEGRADED_RESPONSE, { status: 503 });
+      }
+      // Re-read the container state after wake so the stopped-check reflects
+      // the freshly-launched container, not the pre-wake stopped snapshot.
+      state = await this.getState();
     }
     if (state.status === 'stopped' || state.status === 'stopped_with_code') {
       return new Response('Container is stopped; create a new instant session.', { status: 410 });
@@ -299,6 +316,165 @@ export class VmAgentContainer extends Container<Env> {
     return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS;
   }
 
+  /**
+   * Wake a sleeping container exactly once under concurrency. Serializes on
+   * `wakeChain` and re-reads `lifecycleStatus` inside the critical section, so a
+   * second request that arrives while the first is waking sees `running` and
+   * skips a duplicate launch/restore instead of racing it (see rule 45).
+   */
+  private async ensureAwake(): Promise<{ ok: boolean; message?: string }> {
+    const run = this.wakeChain.then(async () => {
+      const status = await this.ctx.storage.get<LifecycleStatus>('lifecycleStatus');
+      if (status !== 'sleeping') {
+        return { ok: true };
+      }
+      return this.wakeFromSnapshot();
+    });
+    // Keep the chain alive even if this wake rejects, so a failure does not
+    // permanently wedge all future wakes.
+    this.wakeChain = run.then(
+      () => undefined,
+      () => undefined
+    );
+    return run;
+  }
+
+  private async wakeFromSnapshot(): Promise<{ ok: boolean; message?: string }> {
+    const config = await this.ctx.storage.get<VmAgentContainerLaunchConfig>('launchConfig');
+    if (!config) {
+      return { ok: false, message: 'Container launch configuration is unavailable.' };
+    }
+    const db = drizzle(this.env.DATABASE, { schema });
+    const workspace = await db
+      .select({
+        userId: schema.workspaces.userId,
+        chatSessionId: schema.workspaces.chatSessionId,
+      })
+      .from(schema.workspaces)
+      .where(eq(schema.workspaces.id, config.workspaceId))
+      .get();
+    if (!workspace?.chatSessionId) {
+      return { ok: false, message: 'Workspace session metadata is unavailable.' };
+    }
+    const agentSession = await db
+      .select({ id: schema.agentSessions.id, agentType: schema.agentSessions.agentType })
+      .from(schema.agentSessions)
+      .where(eq(schema.agentSessions.workspaceId, config.workspaceId))
+      .orderBy(desc(schema.agentSessions.updatedAt))
+      .get();
+    if (!agentSession) {
+      return { ok: false, message: 'Agent session metadata is unavailable.' };
+    }
+
+    log.info('vm_agent_container_wake_started', {
+      nodeId: config.nodeId,
+      workspaceId: config.workspaceId,
+      chatSessionId: workspace.chatSessionId,
+      agentSessionId: agentSession.id,
+    });
+
+    await this.ctx.storage.put('lifecycleStatus', 'launching' satisfies LifecycleStatus);
+    // The container's CALLBACK_TOKEN must be node-scoped to match the initial
+    // launch (see launchInstantSession): the vm-agent uses it for node callbacks
+    // (error/activity/message reporting) which reject workspace-scoped tokens.
+    // Using a workspace-scoped token here caused restored sessions to accept a
+    // prompt (200) but silently fail to report the agent's reply back (403
+    // "Insufficient token scope"), so no answer appeared after wake.
+    const callbackToken = await signNodeCallbackToken(config.nodeId, this.env);
+    await this.launch(config, { nodeCallbackToken: callbackToken });
+
+    // The fresh container never ran create-workspace, so its workspace-scoped
+    // runtime.CallbackToken is unset. The message reporter and snapshot
+    // callbacks require it (they do NOT fall back to the node-scoped token), so
+    // pass it on the restore request; without it, restored sessions accept a
+    // prompt but silently discard the agent's reply ("no auth token").
+    const workspaceCallbackToken = await signCallbackToken(config.workspaceId, this.env);
+    const { token } = await signNodeManagementToken(workspace.userId, config.nodeId, config.workspaceId, this.env);
+    const restoreUrl = new URL(`http://localhost:${config.vmAgentPort}/workspaces/${config.workspaceId}/agent-sessions/${agentSession.id}/restore`);
+    const restoreResponse = await this.containerFetch(
+      new Request(restoreUrl.toString(), {
+        method: 'POST',
+        headers: {
+          Authorization: `Bearer ${token}`,
+          'Content-Type': 'application/json',
+          'X-SAM-Node-Id': config.nodeId,
+          'X-SAM-Workspace-Id': config.workspaceId,
+        },
+        body: JSON.stringify({
+          chatSessionId: workspace.chatSessionId,
+          runtime: 'cf-container',
+          agentType: agentSession.agentType,
+          workspaceCallbackToken,
+        }),
+      }),
+      config.vmAgentPort
+    );
+    const restoreBody = await restoreResponse.text().catch(() => '');
+    if (!restoreResponse.ok) {
+      await this.markWakeDegraded(config, restoreBody || `restore failed with HTTP ${restoreResponse.status}`);
+      return { ok: false, message: restoreBody || 'Session restore failed.' };
+    }
+    let restoreStatus = '';
+    try {
+      const parsed = JSON.parse(restoreBody) as { status?: unknown };
+      restoreStatus = typeof parsed.status === 'string' ? parsed.status : '';
+    } catch {
+      // A successful restore must provide an explicit machine-readable status.
+    }
+    if (restoreStatus !== 'restored') {
+      const message = restoreBody || 'Session restore did not report restored status.';
+      await this.markWakeDegraded(config, message);
+      return { ok: false, message };
+    }
+
+    const now = new Date().toISOString();
+    await db.update(schema.nodes).set({
+      status: 'running',
+      healthStatus: 'healthy',
+      errorMessage: null,
+      updatedAt: now,
+    }).where(eq(schema.nodes.id, config.nodeId));
+    await db.update(schema.workspaces).set({
+      status: 'running',
+      errorMessage: null,
+      updatedAt: now,
+    }).where(eq(schema.workspaces.id, config.workspaceId));
+    await db.update(schema.agentSessions).set({
+      status: 'running',
+      errorMessage: null,
+      updatedAt: now,
+    }).where(eq(schema.agentSessions.id, agentSession.id));
+    await this.ctx.storage.put('lifecycleStatus', 'running' satisfies LifecycleStatus);
+
+    log.info('vm_agent_container_wake_completed', {
+      nodeId: config.nodeId,
+      workspaceId: config.workspaceId,
+      chatSessionId: workspace.chatSessionId,
+      agentSessionId: agentSession.id,
+    });
+    return { ok: true };
+  }
+
+  private async markWakeDegraded(config: VmAgentContainerLaunchConfig, message: string): Promise<void> {
+    const now = new Date().toISOString();
+    const db = drizzle(this.env.DATABASE, { schema });
+    await db.update(schema.workspaces).set({
+      status: 'recovery',
+      errorMessage: message,
+      updatedAt: now,
+    }).where(eq(schema.workspaces.id, config.workspaceId));
+    await db.update(schema.agentSessions).set({
+      status: 'error',
+      errorMessage: message,
+      updatedAt: now,
+    }).where(eq(schema.agentSessions.workspaceId, config.workspaceId));
+    log.warn('vm_agent_container_wake_degraded', {
+      nodeId: config.nodeId,
+      workspaceId: config.workspaceId,
+      message,
+    });
+  }
+
   private async replaceKeepaliveSchedule(delayMs: number): Promise<void> {
     await this.clearKeepaliveSchedule();
     await this.schedule(Math.max(1, Math.ceil(delayMs / 1000)), KEEPALIVE_CALLBACK);

apps/api/src/env.ts

@@ -140,6 +140,12 @@ export interface Env {
   COMPOSE_IMAGE_ARTIFACT_CLEANUP_BATCH_SIZE?: string; // Max abandoned compose artifacts to delete per cleanup run (default: 50)
   COMPOSE_IMAGE_ARTIFACT_CLEANUP_INTERVAL_HOURS?: string; // Minimum hours between R2 cleanup scans from cron (default: 24)
   COMPOSE_IMAGE_ARTIFACT_CLEANUP_LAST_RUN_KV_KEY?: string; // KV key for cleanup interval gating
+  SESSION_SNAPSHOT_TTL_DAYS?: string; // Runtime hibernate snapshot retention (default: 7)
+  SESSION_SNAPSHOT_R2_PREFIX?: string; // R2 key prefix for runtime hibernate snapshots
+  SESSION_SNAPSHOT_TOTAL_BUDGET_BYTES?: string; // Max combined home/WIP snapshot size (default: 104857600)
+  SESSION_SNAPSHOT_ENTRY_THRESHOLD_BYTES?: string; // Max individual file/dir included by vm-agent scanner (default: 52428800)
+  SESSION_SNAPSHOT_TRANSFER_IDLE_TIMEOUT_MS?: string; // No-progress upload/download watchdog window (default: 30000)
+  SESSION_SNAPSHOT_JSON_BODY_MAX_BYTES?: string; // Max snapshot control-plane JSON request size (default: 262144)
   DEPLOY_ACME_EMAIL?: string; // Contact email for deployment-node ACME certificates
   DEPLOY_ACME_CA?: string; // ACME CA directory override for deployment nodes
   DEPLOY_COMPOSE_CMD?: string; // Docker Compose command override on deployment nodes
@@ -772,6 +778,7 @@ export interface Env {
   CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS?: string; // Active-work renewActivityTimeout interval (default: 300000)
   CF_CONTAINER_VM_AGENT_PORT?: string; // vm-agent standalone HTTP port inside the raw container (default: 8080)
   CF_CONTAINER_PORT_READY_TIMEOUT_MS?: string; // Max time to wait for vm-agent port readiness (default: 30000)
+  CF_CONTAINER_WAKE_TIMEOUT_MS?: string; // Max time for launch + restore before forwarding a wake request (default: 120000)
   CF_CONTAINER_WORKSPACE_BASE_DIR?: string; // Base checkout dir inside raw container (default: /workspaces)
   // Legacy Sandbox SDK prototype (admin-only)
   SANDBOX_ENABLED?: string; // Legacy/fallback kill switch for sandbox routes and older staging config (default: false)