From 3c90b2dcd1af3ead97fc8667cb23129e8d6193aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 11:54:56 +0000 Subject: [PATCH 01/28] task: start runtime-neutral session hibernate wake --- .../2026-07-11-runtime-neutral-session-hibernate-wake.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tasks/{backlog => active}/2026-07-11-runtime-neutral-session-hibernate-wake.md (100%) diff --git a/tasks/backlog/2026-07-11-runtime-neutral-session-hibernate-wake.md b/tasks/active/2026-07-11-runtime-neutral-session-hibernate-wake.md similarity index 100% rename from tasks/backlog/2026-07-11-runtime-neutral-session-hibernate-wake.md rename to tasks/active/2026-07-11-runtime-neutral-session-hibernate-wake.md From 82fa85001455253f5ef2608756e1b2798b9dd8b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 12:09:28 +0000 Subject: [PATCH 02/28] feat: add session snapshot hibernate wake --- .../db/migrations/0091_session_snapshots.sql | 33 + apps/api/src/db/schema.ts | 43 ++ .../src/durable-objects/vm-agent-container.ts | 129 +++- apps/api/src/env.ts | 5 + .../projects/agent-activity-callback.ts | 39 + apps/api/src/routes/workspaces/index.ts | 2 + .../routes/workspaces/session-snapshots.ts | 252 +++++++ apps/api/src/services/node-agent.ts | 40 + apps/api/src/services/session-snapshots.ts | 273 +++++++ apps/api/tests/unit/session-snapshots.test.ts | 62 ++ packages/vm-agent/internal/server/server.go | 2 + .../internal/server/session_snapshot.go | 709 ++++++++++++++++++ .../internal/server/session_snapshot_test.go | 119 +++ ...-runtime-neutral-session-hibernate-wake.md | 18 +- 14 files changed, 1709 insertions(+), 17 deletions(-) create mode 100644 apps/api/src/db/migrations/0091_session_snapshots.sql create mode 100644 apps/api/src/routes/workspaces/session-snapshots.ts create mode 100644 apps/api/src/services/session-snapshots.ts create mode 100644 apps/api/tests/unit/session-snapshots.test.ts create mode 100644 packages/vm-agent/internal/server/session_snapshot.go create mode 100644 packages/vm-agent/internal/server/session_snapshot_test.go diff --git a/apps/api/src/db/migrations/0091_session_snapshots.sql b/apps/api/src/db/migrations/0091_session_snapshots.sql new file mode 100644 index 000000000..46e25f730 --- /dev/null +++ b/apps/api/src/db/migrations/0091_session_snapshots.sql @@ -0,0 +1,33 @@ +-- Runtime-neutral hibernate/restore snapshots for agent sessions. +CREATE TABLE session_snapshots ( + id TEXT PRIMARY KEY, + project_id TEXT REFERENCES projects(id) ON DELETE SET NULL, + workspace_id TEXT REFERENCES workspaces(id) ON DELETE SET NULL, + node_id TEXT REFERENCES nodes(id) ON DELETE SET NULL, + user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + chat_session_id TEXT NOT NULL, + agent_session_id TEXT, + runtime TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + degradation TEXT NOT NULL DEFAULT 'none', + home_r2_key TEXT, + wip_r2_key TEXT, + manifest_r2_key TEXT NOT NULL, + base_commit TEXT, + expires_at TEXT NOT NULL, + manifest_json TEXT, + restore_status TEXT, + restore_message TEXT, + restored_at TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE UNIQUE INDEX idx_session_snapshots_chat_session_id + ON session_snapshots(chat_session_id); + +CREATE INDEX idx_session_snapshots_workspace_id + ON session_snapshots(workspace_id); + +CREATE INDEX idx_session_snapshots_expires_at + ON session_snapshots(expires_at); diff --git a/apps/api/src/db/schema.ts b/apps/api/src/db/schema.ts index 963564a5e..4d62949e2 100644 --- a/apps/api/src/db/schema.ts +++ b/apps/api/src/db/schema.ts @@ -1044,6 +1044,47 @@ export const agentSessions = sqliteTable( }) ); +// ============================================================================= +// Runtime-neutral Session Snapshots +// ============================================================================= +export const sessionSnapshots = sqliteTable( + 'session_snapshots', + { + id: text('id').primaryKey(), + projectId: text('project_id').references(() => projects.id, { onDelete: 'set null' }), + workspaceId: text('workspace_id').references(() => workspaces.id, { onDelete: 'set null' }), + nodeId: text('node_id').references(() => nodes.id, { onDelete: 'set null' }), + userId: text('user_id') + .notNull() + .references(() => users.id, { onDelete: 'cascade' }), + chatSessionId: text('chat_session_id').notNull(), + agentSessionId: text('agent_session_id'), + runtime: text('runtime').notNull(), + status: text('status').notNull().default('pending'), + degradation: text('degradation').notNull().default('none'), + homeR2Key: text('home_r2_key'), + wipR2Key: text('wip_r2_key'), + manifestR2Key: text('manifest_r2_key').notNull(), + baseCommit: text('base_commit'), + expiresAt: text('expires_at').notNull(), + manifestJson: text('manifest_json'), + restoreStatus: text('restore_status'), + restoreMessage: text('restore_message'), + restoredAt: text('restored_at'), + createdAt: text('created_at') + .notNull() + .default(sql`CURRENT_TIMESTAMP`), + updatedAt: text('updated_at') + .notNull() + .default(sql`CURRENT_TIMESTAMP`), + }, + (table) => ({ + chatSessionIdUnique: uniqueIndex('idx_session_snapshots_chat_session_id').on(table.chatSessionId), + workspaceIdIdx: index('idx_session_snapshots_workspace_id').on(table.workspaceId), + expiresAtIdx: index('idx_session_snapshots_expires_at').on(table.expiresAt), + }) +); + // ============================================================================= // Agent Settings (per-user, per-agent configuration) // ============================================================================= @@ -1545,6 +1586,8 @@ export type Workspace = typeof workspaces.$inferSelect; export type NewWorkspace = typeof workspaces.$inferInsert; export type AgentSession = typeof agentSessions.$inferSelect; export type NewAgentSession = typeof agentSessions.$inferInsert; +export type SessionSnapshot = typeof sessionSnapshots.$inferSelect; +export type NewSessionSnapshot = typeof sessionSnapshots.$inferInsert; export type UIStandard = typeof uiStandards.$inferSelect; export type NewUIStandard = typeof uiStandards.$inferInsert; export type ThemeToken = typeof themeTokens.$inferSelect; diff --git a/apps/api/src/durable-objects/vm-agent-container.ts b/apps/api/src/durable-objects/vm-agent-container.ts index f9290484c..56dee6a94 100644 --- a/apps/api/src/durable-objects/vm-agent-container.ts +++ b/apps/api/src/durable-objects/vm-agent-container.ts @@ -1,10 +1,11 @@ import { Container, switchPort } from '@cloudflare/containers'; -import { eq } from 'drizzle-orm'; +import { desc, eq } from 'drizzle-orm'; import { drizzle } from 'drizzle-orm/d1'; import * as schema from '../db/schema'; import type { Env } from '../env'; import { log } from '../lib/logger'; +import { signCallbackToken, signNodeManagementToken } from '../services/jwt'; export const DEFAULT_CF_CONTAINER_SLEEP_AFTER = '1h'; export const DEFAULT_CF_CONTAINER_ACTIVE_WORK_MAX_MS = 2 * 60 * 60 * 1000; @@ -45,7 +46,7 @@ interface ActiveWorkState { const ACTIVE_WORK_KEY = 'activeWork'; const KEEPALIVE_CALLBACK = 'renewActiveWorkKeepalive'; -const SLEEPING_RESPONSE = 'Container is asleep; wake/rehydrate is not implemented yet.'; +const WAKE_DEGRADED_RESPONSE = 'Workspace woke with degraded snapshot restore; retry the prompt or fork from transcript history.'; export class VmAgentContainer extends Container { defaultPort = 8080; @@ -111,9 +112,10 @@ export class VmAgentContainer extends Container { const state = await this.getState(); const lifecycleStatus = await this.ctx.storage.get('lifecycleStatus'); if (lifecycleStatus === 'sleeping') { - // Phase 3 of idea 01KX4KSXEXQMP41KS34TW9EN01 will replace this temporary - // response with wake/rehydrate before proxying the request. - return new Response(SLEEPING_RESPONSE, { status: 503 }); + const wake = await this.wakeFromSnapshot(); + if (!wake.ok) { + return new Response(wake.message || WAKE_DEGRADED_RESPONSE, { status: 503 }); + } } if (state.status === 'stopped' || state.status === 'stopped_with_code') { return new Response('Container is stopped; create a new instant session.', { status: 410 }); @@ -235,9 +237,10 @@ export class VmAgentContainer extends Container { const state = await this.getState(); const lifecycleStatus = await this.ctx.storage.get('lifecycleStatus'); if (lifecycleStatus === 'sleeping') { - // Phase 3 of idea 01KX4KSXEXQMP41KS34TW9EN01 will replace this temporary - // response with wake/rehydrate before proxying the request. - return new Response(SLEEPING_RESPONSE, { status: 503 }); + const wake = await this.wakeFromSnapshot(); + if (!wake.ok) { + return new Response(wake.message || WAKE_DEGRADED_RESPONSE, { status: 503 }); + } } if (state.status === 'stopped' || state.status === 'stopped_with_code') { return new Response('Container is stopped; create a new instant session.', { status: 410 }); @@ -299,6 +302,116 @@ export class VmAgentContainer extends Container { return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS; } + private async wakeFromSnapshot(): Promise<{ ok: boolean; message?: string }> { + const config = await this.ctx.storage.get('launchConfig'); + if (!config) { + return { ok: false, message: 'Container launch configuration is unavailable.' }; + } + const db = drizzle(this.env.DATABASE, { schema }); + const workspace = await db + .select({ + userId: schema.workspaces.userId, + chatSessionId: schema.workspaces.chatSessionId, + }) + .from(schema.workspaces) + .where(eq(schema.workspaces.id, config.workspaceId)) + .get(); + if (!workspace?.chatSessionId) { + return { ok: false, message: 'Workspace session metadata is unavailable.' }; + } + const agentSession = await db + .select({ id: schema.agentSessions.id }) + .from(schema.agentSessions) + .where(eq(schema.agentSessions.workspaceId, config.workspaceId)) + .orderBy(desc(schema.agentSessions.updatedAt)) + .get(); + if (!agentSession) { + return { ok: false, message: 'Agent session metadata is unavailable.' }; + } + + log.info('vm_agent_container_wake_started', { + nodeId: config.nodeId, + workspaceId: config.workspaceId, + chatSessionId: workspace.chatSessionId, + agentSessionId: agentSession.id, + }); + + await this.ctx.storage.put('lifecycleStatus', 'launching' satisfies LifecycleStatus); + const callbackToken = await signCallbackToken(config.workspaceId, this.env); + await this.launch(config, { nodeCallbackToken: callbackToken }); + + const { token } = await signNodeManagementToken(workspace.userId, config.nodeId, config.workspaceId, this.env); + const restoreUrl = new URL(`http://localhost:${config.vmAgentPort}/workspaces/${config.workspaceId}/agent-sessions/${agentSession.id}/restore`); + const restoreResponse = await this.containerFetch( + new Request(restoreUrl.toString(), { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + 'X-SAM-Node-Id': config.nodeId, + 'X-SAM-Workspace-Id': config.workspaceId, + }, + body: JSON.stringify({ + chatSessionId: workspace.chatSessionId, + runtime: 'cf-container', + }), + }), + config.vmAgentPort + ); + const restoreBody = await restoreResponse.text().catch(() => ''); + if (!restoreResponse.ok) { + await this.markWakeDegraded(config, restoreBody || `restore failed with HTTP ${restoreResponse.status}`); + return { ok: false, message: restoreBody || 'Session restore failed.' }; + } + + const now = new Date().toISOString(); + await db.update(schema.nodes).set({ + status: 'running', + healthStatus: 'healthy', + errorMessage: null, + updatedAt: now, + }).where(eq(schema.nodes.id, config.nodeId)); + await db.update(schema.workspaces).set({ + status: 'running', + errorMessage: null, + updatedAt: now, + }).where(eq(schema.workspaces.id, config.workspaceId)); + await db.update(schema.agentSessions).set({ + status: 'running', + errorMessage: null, + updatedAt: now, + }).where(eq(schema.agentSessions.id, agentSession.id)); + await this.ctx.storage.put('lifecycleStatus', 'running' satisfies LifecycleStatus); + + log.info('vm_agent_container_wake_completed', { + nodeId: config.nodeId, + workspaceId: config.workspaceId, + chatSessionId: workspace.chatSessionId, + agentSessionId: agentSession.id, + }); + return { ok: true }; + } + + private async markWakeDegraded(config: VmAgentContainerLaunchConfig, message: string): Promise { + const now = new Date().toISOString(); + const db = drizzle(this.env.DATABASE, { schema }); + await db.update(schema.workspaces).set({ + status: 'recovery', + errorMessage: message, + updatedAt: now, + }).where(eq(schema.workspaces.id, config.workspaceId)); + await db.update(schema.agentSessions).set({ + status: 'error', + errorMessage: message, + updatedAt: now, + }).where(eq(schema.agentSessions.workspaceId, config.workspaceId)); + log.warn('vm_agent_container_wake_degraded', { + nodeId: config.nodeId, + workspaceId: config.workspaceId, + message, + }); + } + private async replaceKeepaliveSchedule(delayMs: number): Promise { await this.clearKeepaliveSchedule(); await this.schedule(Math.max(1, Math.ceil(delayMs / 1000)), KEEPALIVE_CALLBACK); diff --git a/apps/api/src/env.ts b/apps/api/src/env.ts index 3e6d4c4aa..506e23214 100644 --- a/apps/api/src/env.ts +++ b/apps/api/src/env.ts @@ -140,6 +140,11 @@ export interface Env { COMPOSE_IMAGE_ARTIFACT_CLEANUP_BATCH_SIZE?: string; // Max abandoned compose artifacts to delete per cleanup run (default: 50) COMPOSE_IMAGE_ARTIFACT_CLEANUP_INTERVAL_HOURS?: string; // Minimum hours between R2 cleanup scans from cron (default: 24) COMPOSE_IMAGE_ARTIFACT_CLEANUP_LAST_RUN_KV_KEY?: string; // KV key for cleanup interval gating + SESSION_SNAPSHOT_TTL_DAYS?: string; // Runtime hibernate snapshot retention (default: 7) + SESSION_SNAPSHOT_R2_PREFIX?: string; // R2 key prefix for runtime hibernate snapshots + SESSION_SNAPSHOT_TOTAL_BUDGET_BYTES?: string; // Max combined home/WIP snapshot size (default: 104857600) + SESSION_SNAPSHOT_ENTRY_THRESHOLD_BYTES?: string; // Max individual file/dir included by vm-agent scanner (default: 52428800) + SESSION_SNAPSHOT_TRANSFER_IDLE_TIMEOUT_MS?: string; // No-progress upload/download watchdog window (default: 30000) DEPLOY_ACME_EMAIL?: string; // Contact email for deployment-node ACME certificates DEPLOY_ACME_CA?: string; // ACME CA directory override for deployment nodes DEPLOY_COMPOSE_CMD?: string; // Docker Compose command override on deployment nodes diff --git a/apps/api/src/routes/projects/agent-activity-callback.ts b/apps/api/src/routes/projects/agent-activity-callback.ts index 8c6bd2669..59e01d956 100644 --- a/apps/api/src/routes/projects/agent-activity-callback.ts +++ b/apps/api/src/routes/projects/agent-activity-callback.ts @@ -1,11 +1,15 @@ +import { eq } from 'drizzle-orm'; +import { drizzle } from 'drizzle-orm/d1'; import { Hono } from 'hono'; +import * as schema from '../../db/schema'; import type { Env } from '../../env'; import { extractBearerToken } from '../../lib/auth-helpers'; import { log } from '../../lib/logger'; import { errors } from '../../middleware/error'; import { AcpSessionActivityReportSchema, jsonValidator } from '../../schemas'; import { verifyCallbackToken } from '../../services/jwt'; +import { hibernateAgentSessionOnNode } from '../../services/node-agent'; import * as projectDataService from '../../services/project-data'; import { markVmAgentContainerActiveWorkEndedBestEffort } from '../../services/vm-agent-container'; @@ -70,6 +74,41 @@ agentActivityCallbackRoute.post( statusError: body.statusError, }); if (body.activity === 'idle' || body.activity === 'error') { + if (body.activity === 'idle' && existing.workspaceId && existing.nodeId && existing.acpSdkSessionId) { + const db = drizzle(c.env.DATABASE, { schema }); + const workspace = await db + .select({ + id: schema.workspaces.id, + userId: schema.workspaces.userId, + chatSessionId: schema.workspaces.chatSessionId, + runtime: schema.nodes.runtime, + }) + .from(schema.workspaces) + .leftJoin(schema.nodes, eq(schema.nodes.id, schema.workspaces.nodeId)) + .where(eq(schema.workspaces.id, existing.workspaceId)) + .get(); + if (workspace?.runtime === 'cf-container' && workspace.chatSessionId) { + await hibernateAgentSessionOnNode( + existing.nodeId, + existing.workspaceId, + existing.acpSdkSessionId, + c.env, + workspace.userId, + { + chatSessionId: workspace.chatSessionId, + runtime: 'cf-container', + } + ).catch((err) => { + log.warn('acp_activity.session_snapshot_failed', { + projectId, + sessionId, + workspaceId: existing.workspaceId, + nodeId: existing.nodeId, + error: err instanceof Error ? err.message : String(err), + }); + }); + } + } await markVmAgentContainerActiveWorkEndedBestEffort( c.env, existing.nodeId, diff --git a/apps/api/src/routes/workspaces/index.ts b/apps/api/src/routes/workspaces/index.ts index a1577ec1e..b4e0eca1f 100644 --- a/apps/api/src/routes/workspaces/index.ts +++ b/apps/api/src/routes/workspaces/index.ts @@ -6,6 +6,7 @@ import { crudRoutes } from './crud'; import { lifecycleRoutes } from './lifecycle'; import { localForwardRoutes } from './local-forward'; import { runtimeRoutes } from './runtime'; +import { sessionSnapshotRoutes } from './session-snapshots'; const workspacesRoutes = new Hono<{ Bindings: Env }>(); workspacesRoutes.route('/', crudRoutes); @@ -13,5 +14,6 @@ workspacesRoutes.route('/', localForwardRoutes); workspacesRoutes.route('/', lifecycleRoutes); workspacesRoutes.route('/', agentSessionRoutes); workspacesRoutes.route('/', runtimeRoutes); +workspacesRoutes.route('/', sessionSnapshotRoutes); export { workspacesRoutes }; diff --git a/apps/api/src/routes/workspaces/session-snapshots.ts b/apps/api/src/routes/workspaces/session-snapshots.ts new file mode 100644 index 000000000..a8adb6c1f --- /dev/null +++ b/apps/api/src/routes/workspaces/session-snapshots.ts @@ -0,0 +1,252 @@ +import { eq } from 'drizzle-orm'; +import { drizzle } from 'drizzle-orm/d1'; +import type { Context } from 'hono'; +import { Hono } from 'hono'; + +import * as schema from '../../db/schema'; +import type { Env } from '../../env'; +import { expectJsonRecord } from '../../lib/runtime-validation'; +import { errors } from '../../middleware/error'; +import { + buildSessionSnapshotR2Key, + completeSessionSnapshot, + getRestorableSessionSnapshot, + getSessionSnapshotConfig, + prepareSessionSnapshot, + recordSessionSnapshotRestoreResult, + type SessionSnapshotArtifact, + type SessionSnapshotDegradation, + type SessionSnapshotManifest, + type SessionSnapshotStatus, +} from '../../services/session-snapshots'; +import { verifyWorkspaceCallbackAuth } from './_helpers'; + +const sessionSnapshotRoutes = new Hono<{ Bindings: Env }>(); +type SnapshotRouteContext = Context<{ Bindings: Env }>; + +const ARTIFACTS = new Set(['home', 'wip', 'manifest']); +const COMPLETE_STATUSES = new Set(['available', 'degraded', 'failed']); +const DEGRADATIONS = new Set([ + 'none', + 'home-skipped', + 'wip-only', + 'transcript-only', +]); +const MAX_SNAPSHOT_JSON_BYTES = 256 * 1024; + +async function readJsonBody(c: SnapshotRouteContext) { + const raw = await c.req.raw.text(); + if (new TextEncoder().encode(raw).byteLength > MAX_SNAPSHOT_JSON_BYTES) { + throw errors.badRequest('Snapshot request body is too large'); + } + try { + return expectJsonRecord(JSON.parse(raw), 'session snapshot request'); + } catch { + throw errors.badRequest('Invalid JSON request body'); + } +} + +function stringField(body: Record, key: string, required = true): string | null { + const value = body[key]; + if (typeof value !== 'string' || value.trim() === '') { + if (required) throw errors.badRequest(`${key} is required`); + return null; + } + return value.trim(); +} + +function requiredStringField(body: Record, key: string): string { + const value = stringField(body, key, true); + if (value === null) throw errors.badRequest(`${key} is required`); + return value; +} + +function artifactFromParam(value: string): SessionSnapshotArtifact { + if (!ARTIFACTS.has(value as SessionSnapshotArtifact)) { + throw errors.badRequest('Unknown snapshot artifact'); + } + return value as SessionSnapshotArtifact; +} + +async function requireWorkspace(c: SnapshotRouteContext, workspaceId: string) { + const db = drizzle(c.env.DATABASE, { schema }); + const rows = await db + .select() + .from(schema.workspaces) + .where(eq(schema.workspaces.id, workspaceId)) + .limit(1); + const workspace = rows[0]; + if (!workspace) throw errors.notFound('Workspace'); + return { db, workspace }; +} + +sessionSnapshotRoutes.post('/:id/session-snapshot/prepare', async (c) => { + const workspaceId = c.req.param('id'); + await verifyWorkspaceCallbackAuth(c, workspaceId); + const { db, workspace } = await requireWorkspace(c, workspaceId); + const body = await readJsonBody(c); + const chatSessionId = requiredStringField(body, 'chatSessionId'); + const agentSessionId = stringField(body, 'agentSessionId', false); + const runtime = stringField(body, 'runtime', false) || 'runtime-neutral'; + + if (!workspace.chatSessionId || workspace.chatSessionId !== chatSessionId) { + throw errors.forbidden('Snapshot chat session does not match workspace'); + } + + const prepared = await prepareSessionSnapshot(db, c.env, { + workspaceId, + nodeId: workspace.nodeId, + projectId: workspace.projectId, + userId: workspace.userId, + chatSessionId, + agentSessionId, + runtime, + }); + + return c.json({ + snapshotId: prepared.snapshotId, + expiresAt: prepared.expiresAt, + config: prepared.config, + upload: { + home: `/api/workspaces/${workspaceId}/session-snapshot/artifacts/home?chatSessionId=${encodeURIComponent(chatSessionId)}`, + wip: `/api/workspaces/${workspaceId}/session-snapshot/artifacts/wip?chatSessionId=${encodeURIComponent(chatSessionId)}`, + }, + }); +}); + +sessionSnapshotRoutes.put('/:id/session-snapshot/artifacts/:artifact', async (c) => { + const workspaceId = c.req.param('id'); + await verifyWorkspaceCallbackAuth(c, workspaceId); + const artifact = artifactFromParam(c.req.param('artifact')); + if (artifact === 'manifest') { + throw errors.badRequest('Manifest is written by the complete endpoint'); + } + const chatSessionId = c.req.query('chatSessionId')?.trim(); + if (!chatSessionId) throw errors.badRequest('chatSessionId is required'); + const { workspace } = await requireWorkspace(c, workspaceId); + if (!workspace.chatSessionId || workspace.chatSessionId !== chatSessionId) { + throw errors.forbidden('Snapshot chat session does not match workspace'); + } + const contentLength = c.req.header('content-length'); + if (contentLength) { + const parsed = Number.parseInt(contentLength, 10); + if (!Number.isFinite(parsed) || parsed < 0) throw errors.badRequest('Invalid Content-Length'); + if (parsed > getSessionSnapshotConfig(c.env).totalBudgetBytes) { + throw errors.badRequest('Snapshot artifact exceeds configured total budget'); + } + } + if (!c.req.raw.body) throw errors.badRequest('Snapshot artifact body is required'); + + const key = buildSessionSnapshotR2Key(c.env, chatSessionId, artifact); + await c.env.R2.put(key, c.req.raw.body, { + httpMetadata: { + contentType: artifact === 'home' ? 'application/x-tar' : 'application/octet-stream', + }, + }); + return c.json({ key, artifact }); +}); + +sessionSnapshotRoutes.post('/:id/session-snapshot/complete', async (c) => { + const workspaceId = c.req.param('id'); + await verifyWorkspaceCallbackAuth(c, workspaceId); + const { db, workspace } = await requireWorkspace(c, workspaceId); + const body = await readJsonBody(c); + const chatSessionId = stringField(body, 'chatSessionId'); + if (!workspace.chatSessionId || workspace.chatSessionId !== chatSessionId) { + throw errors.forbidden('Snapshot chat session does not match workspace'); + } + + const status = requiredStringField(body, 'status') as SessionSnapshotStatus; + const degradation = requiredStringField(body, 'degradation') as SessionSnapshotDegradation; + if (!COMPLETE_STATUSES.has(status)) throw errors.badRequest('Invalid snapshot status'); + if (!DEGRADATIONS.has(degradation)) throw errors.badRequest('Invalid snapshot degradation'); + const manifest = expectJsonRecord(body.manifest, 'snapshot manifest') as unknown as SessionSnapshotManifest; + const artifactSizes = expectJsonRecord(body.artifactSizes ?? {}, 'snapshot artifact sizes') as { + homeBytes?: number; + wipBytes?: number; + }; + + await completeSessionSnapshot(db, c.env, { + workspaceId, + chatSessionId, + agentSessionId: stringField(body, 'agentSessionId', false), + runtime: stringField(body, 'runtime', false) || 'runtime-neutral', + baseCommit: stringField(body, 'baseCommit', false), + status, + degradation, + manifest, + artifactSizes, + }); + + return c.json({ status, degradation }); +}); + +sessionSnapshotRoutes.get('/:id/session-snapshot/restore', async (c) => { + const workspaceId = c.req.param('id'); + await verifyWorkspaceCallbackAuth(c, workspaceId); + const { db, workspace } = await requireWorkspace(c, workspaceId); + const chatSessionId = c.req.query('chatSessionId')?.trim() || workspace.chatSessionId; + if (!chatSessionId) throw errors.badRequest('chatSessionId is required'); + if (workspace.chatSessionId !== chatSessionId) { + throw errors.forbidden('Snapshot chat session does not match workspace'); + } + + const snapshot = await getRestorableSessionSnapshot(db, chatSessionId); + if (!snapshot) { + return c.json({ available: false, reason: 'snapshot_missing_or_unavailable' }); + } + + return c.json({ + available: true, + status: snapshot.status, + degradation: snapshot.degradation, + baseCommit: snapshot.baseCommit, + manifest: snapshot.manifestJson ? JSON.parse(snapshot.manifestJson) : null, + download: { + home: snapshot.homeR2Key + ? `/api/workspaces/${workspaceId}/session-snapshot/artifacts/home?chatSessionId=${encodeURIComponent(chatSessionId)}` + : null, + wip: snapshot.wipR2Key + ? `/api/workspaces/${workspaceId}/session-snapshot/artifacts/wip?chatSessionId=${encodeURIComponent(chatSessionId)}` + : null, + manifest: `/api/workspaces/${workspaceId}/session-snapshot/artifacts/manifest?chatSessionId=${encodeURIComponent(chatSessionId)}`, + }, + }); +}); + +sessionSnapshotRoutes.get('/:id/session-snapshot/artifacts/:artifact', async (c) => { + const workspaceId = c.req.param('id'); + await verifyWorkspaceCallbackAuth(c, workspaceId); + const artifact = artifactFromParam(c.req.param('artifact')); + const chatSessionId = c.req.query('chatSessionId')?.trim(); + if (!chatSessionId) throw errors.badRequest('chatSessionId is required'); + const { workspace } = await requireWorkspace(c, workspaceId); + if (workspace.chatSessionId !== chatSessionId) { + throw errors.forbidden('Snapshot chat session does not match workspace'); + } + const object = await c.env.R2.get(buildSessionSnapshotR2Key(c.env, chatSessionId, artifact)); + if (!object) throw errors.notFound('Snapshot artifact'); + const headers = new Headers(); + object.writeHttpMetadata(headers); + headers.set('etag', object.httpEtag); + return new Response(object.body, { headers }); +}); + +sessionSnapshotRoutes.post('/:id/session-snapshot/restore-result', async (c) => { + const workspaceId = c.req.param('id'); + await verifyWorkspaceCallbackAuth(c, workspaceId); + const { db, workspace } = await requireWorkspace(c, workspaceId); + const body = await readJsonBody(c); + const chatSessionId = requiredStringField(body, 'chatSessionId'); + if (workspace.chatSessionId !== chatSessionId) { + throw errors.forbidden('Snapshot chat session does not match workspace'); + } + await recordSessionSnapshotRestoreResult(db, { + chatSessionId, + status: requiredStringField(body, 'status'), + message: stringField(body, 'message', false), + }); + return c.body(null, 204); +}); + +export { sessionSnapshotRoutes }; diff --git a/apps/api/src/services/node-agent.ts b/apps/api/src/services/node-agent.ts index 401bd5f87..fa0da9015 100644 --- a/apps/api/src/services/node-agent.ts +++ b/apps/api/src/services/node-agent.ts @@ -520,6 +520,46 @@ export async function sendPromptToAgentOnNode( } } +export async function hibernateAgentSessionOnNode( + nodeId: string, + workspaceId: string, + sessionId: string, + env: Env, + userId: string, + input: { + chatSessionId: string; + runtime: string; + } +): Promise { + return nodeAgentRequest(nodeId, env, `/workspaces/${workspaceId}/agent-sessions/${sessionId}/hibernate`, { + method: 'POST', + userId, + workspaceId, + requestTimeoutMs: getNodeAgentRequestTimeoutMs(env), + body: JSON.stringify(input), + }); +} + +export async function restoreAgentSessionOnNode( + nodeId: string, + workspaceId: string, + sessionId: string, + env: Env, + userId: string, + input: { + chatSessionId: string; + runtime: string; + } +): Promise { + return nodeAgentRequest(nodeId, env, `/workspaces/${workspaceId}/agent-sessions/${sessionId}/restore`, { + method: 'POST', + userId, + workspaceId, + requestTimeoutMs: getNodeAgentRequestTimeoutMs(env), + body: JSON.stringify(input), + }); +} + /** * Cancel a running prompt on an agent session. * Returns { success, status } instead of throwing on non-2xx responses, diff --git a/apps/api/src/services/session-snapshots.ts b/apps/api/src/services/session-snapshots.ts new file mode 100644 index 000000000..70041ad7a --- /dev/null +++ b/apps/api/src/services/session-snapshots.ts @@ -0,0 +1,273 @@ +import { eq } from 'drizzle-orm'; +import { type drizzle } from 'drizzle-orm/d1'; + +import * as schema from '../db/schema'; +import type { Env } from '../env'; +import { log } from '../lib/logger'; +import { parsePositiveInt } from '../lib/route-helpers'; +import { ulid } from '../lib/ulid'; + +type Db = ReturnType>; + +export const DEFAULT_SESSION_SNAPSHOT_TTL_DAYS = 7; +export const DEFAULT_SESSION_SNAPSHOT_TOTAL_BUDGET_BYTES = 100 * 1024 * 1024; +export const DEFAULT_SESSION_SNAPSHOT_ENTRY_THRESHOLD_BYTES = 50 * 1024 * 1024; +export const DEFAULT_SESSION_SNAPSHOT_TRANSFER_IDLE_TIMEOUT_MS = 30_000; +export const DEFAULT_SESSION_SNAPSHOT_R2_PREFIX = 'session-snapshots'; + +export type SessionSnapshotArtifact = 'home' | 'wip' | 'manifest'; +export type SessionSnapshotStatus = 'pending' | 'available' | 'degraded' | 'failed' | 'expired'; +export type SessionSnapshotDegradation = 'none' | 'home-skipped' | 'wip-only' | 'transcript-only'; + +export interface SessionSnapshotManifest { + version: 1; + chatSessionId: string; + workspaceId: string; + agentSessionId?: string; + baseCommit?: string; + status: SessionSnapshotStatus; + degradation: SessionSnapshotDegradation; + skipped: Array<{ + path: string; + reason: string; + sizeBytes?: number; + }>; + artifacts: { + home?: { sizeBytes: number; sha256?: string }; + wip?: { sizeBytes: number; sha256?: string }; + }; + createdAt: string; +} + +export interface SessionSnapshotConfig { + ttlDays: number; + totalBudgetBytes: number; + entryThresholdBytes: number; + transferIdleTimeoutMs: number; + r2Prefix: string; +} + +export interface PrepareSessionSnapshotInput { + workspaceId: string; + nodeId: string | null; + projectId: string | null; + userId: string; + chatSessionId: string; + agentSessionId: string | null; + runtime: string; +} + +export interface CompleteSessionSnapshotInput { + workspaceId: string; + chatSessionId: string; + agentSessionId: string | null; + runtime: string; + baseCommit: string | null; + status: SessionSnapshotStatus; + degradation: SessionSnapshotDegradation; + manifest: SessionSnapshotManifest; + artifactSizes: { + homeBytes?: number; + wipBytes?: number; + }; +} + +export function getSessionSnapshotConfig(env: Env): SessionSnapshotConfig { + return { + ttlDays: parsePositiveInt(env.SESSION_SNAPSHOT_TTL_DAYS, DEFAULT_SESSION_SNAPSHOT_TTL_DAYS), + totalBudgetBytes: parsePositiveInt( + env.SESSION_SNAPSHOT_TOTAL_BUDGET_BYTES, + DEFAULT_SESSION_SNAPSHOT_TOTAL_BUDGET_BYTES + ), + entryThresholdBytes: parsePositiveInt( + env.SESSION_SNAPSHOT_ENTRY_THRESHOLD_BYTES, + DEFAULT_SESSION_SNAPSHOT_ENTRY_THRESHOLD_BYTES + ), + transferIdleTimeoutMs: parsePositiveInt( + env.SESSION_SNAPSHOT_TRANSFER_IDLE_TIMEOUT_MS, + DEFAULT_SESSION_SNAPSHOT_TRANSFER_IDLE_TIMEOUT_MS + ), + r2Prefix: sanitizeSnapshotPrefix(env.SESSION_SNAPSHOT_R2_PREFIX || DEFAULT_SESSION_SNAPSHOT_R2_PREFIX), + }; +} + +function sanitizeSnapshotPrefix(prefix: string): string { + const normalized = prefix + .trim() + .replace(/^\/+|\/+$/g, '') + .replace(/[^A-Za-z0-9._/-]/g, '-'); + return normalized || DEFAULT_SESSION_SNAPSHOT_R2_PREFIX; +} + +function sanitizeKeySegment(value: string): string { + return value.trim().replace(/[^A-Za-z0-9._-]/g, '-'); +} + +export function buildSessionSnapshotR2Key( + env: Env, + chatSessionId: string, + artifact: SessionSnapshotArtifact +): string { + const { r2Prefix } = getSessionSnapshotConfig(env); + const sessionKey = sanitizeKeySegment(chatSessionId); + const suffix = artifact === 'home' ? 'home.tar' : artifact === 'wip' ? 'wip.bundle' : 'manifest.json'; + return `${r2Prefix}/${sessionKey}/${suffix}`; +} + +function snapshotExpiry(now: Date, ttlDays: number): string { + return new Date(now.getTime() + ttlDays * 24 * 60 * 60 * 1000).toISOString(); +} + +export async function prepareSessionSnapshot( + db: Db, + env: Env, + input: PrepareSessionSnapshotInput +): Promise<{ + snapshotId: string; + expiresAt: string; + keys: Record; + config: SessionSnapshotConfig; +}> { + const config = getSessionSnapshotConfig(env); + const now = new Date(); + const expiresAt = snapshotExpiry(now, config.ttlDays); + const keys = { + home: buildSessionSnapshotR2Key(env, input.chatSessionId, 'home'), + wip: buildSessionSnapshotR2Key(env, input.chatSessionId, 'wip'), + manifest: buildSessionSnapshotR2Key(env, input.chatSessionId, 'manifest'), + }; + + const existing = await db + .select({ id: schema.sessionSnapshots.id }) + .from(schema.sessionSnapshots) + .where(eq(schema.sessionSnapshots.chatSessionId, input.chatSessionId)) + .limit(1); + + const snapshotId = existing[0]?.id || ulid(); + const row = { + id: snapshotId, + projectId: input.projectId, + workspaceId: input.workspaceId, + nodeId: input.nodeId, + userId: input.userId, + chatSessionId: input.chatSessionId, + agentSessionId: input.agentSessionId, + runtime: input.runtime, + status: 'pending', + degradation: 'none', + homeR2Key: keys.home, + wipR2Key: keys.wip, + manifestR2Key: keys.manifest, + baseCommit: null, + expiresAt, + manifestJson: null, + restoreStatus: null, + restoreMessage: null, + restoredAt: null, + updatedAt: now.toISOString(), + } satisfies schema.NewSessionSnapshot; + + if (existing[0]) { + await db.update(schema.sessionSnapshots).set(row).where(eq(schema.sessionSnapshots.id, snapshotId)); + } else { + await db.insert(schema.sessionSnapshots).values({ ...row, createdAt: now.toISOString() }); + } + + return { snapshotId, expiresAt, keys, config }; +} + +export async function completeSessionSnapshot( + db: Db, + env: Env, + input: CompleteSessionSnapshotInput +): Promise { + const homeR2Key = + input.artifactSizes.homeBytes == null + ? null + : buildSessionSnapshotR2Key(env, input.chatSessionId, 'home'); + const wipR2Key = + input.artifactSizes.wipBytes == null + ? null + : buildSessionSnapshotR2Key(env, input.chatSessionId, 'wip'); + const manifestR2Key = buildSessionSnapshotR2Key(env, input.chatSessionId, 'manifest'); + const manifestJson = JSON.stringify(input.manifest); + await env.R2.put(manifestR2Key, manifestJson, { + httpMetadata: { contentType: 'application/json' }, + }); + + await db + .update(schema.sessionSnapshots) + .set({ + agentSessionId: input.agentSessionId, + runtime: input.runtime, + status: input.status, + degradation: input.degradation, + homeR2Key, + wipR2Key, + manifestR2Key, + baseCommit: input.baseCommit, + manifestJson, + expiresAt: + (await db + .select({ expiresAt: schema.sessionSnapshots.expiresAt }) + .from(schema.sessionSnapshots) + .where(eq(schema.sessionSnapshots.chatSessionId, input.chatSessionId)) + .limit(1))[0]?.expiresAt || snapshotExpiry(new Date(), getSessionSnapshotConfig(env).ttlDays), + updatedAt: new Date().toISOString(), + }) + .where(eq(schema.sessionSnapshots.chatSessionId, input.chatSessionId)); +} + +export async function getRestorableSessionSnapshot( + db: Db, + chatSessionId: string, + now = new Date() +): Promise { + const rows = await db + .select() + .from(schema.sessionSnapshots) + .where(eq(schema.sessionSnapshots.chatSessionId, chatSessionId)) + .limit(1); + const snapshot = rows[0]; + if (!snapshot) return null; + if (Date.parse(snapshot.expiresAt) <= now.getTime()) { + return { ...snapshot, status: 'expired' }; + } + if (snapshot.status !== 'available' && snapshot.status !== 'degraded') { + return null; + } + return snapshot; +} + +export async function recordSessionSnapshotRestoreResult( + db: Db, + input: { + chatSessionId: string; + status: string; + message: string | null; + } +): Promise { + await db + .update(schema.sessionSnapshots) + .set({ + restoreStatus: input.status, + restoreMessage: input.message, + restoredAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }) + .where(eq(schema.sessionSnapshots.chatSessionId, input.chatSessionId)); +} + +export async function deleteSessionSnapshotArtifacts(env: Env, chatSessionId: string): Promise { + await Promise.all( + (['home', 'wip', 'manifest'] as const).map((artifact) => + env.R2.delete(buildSessionSnapshotR2Key(env, chatSessionId, artifact)).catch((err) => { + log.warn('session_snapshot.r2_delete_failed', { + chatSessionId, + artifact, + error: err instanceof Error ? err.message : String(err), + }); + }) + ) + ); +} diff --git a/apps/api/tests/unit/session-snapshots.test.ts b/apps/api/tests/unit/session-snapshots.test.ts new file mode 100644 index 000000000..4b2ebe433 --- /dev/null +++ b/apps/api/tests/unit/session-snapshots.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from 'vitest'; + +import type { Env } from '../../src/env'; +import { + buildSessionSnapshotR2Key, + DEFAULT_SESSION_SNAPSHOT_ENTRY_THRESHOLD_BYTES, + DEFAULT_SESSION_SNAPSHOT_R2_PREFIX, + DEFAULT_SESSION_SNAPSHOT_TOTAL_BUDGET_BYTES, + DEFAULT_SESSION_SNAPSHOT_TRANSFER_IDLE_TIMEOUT_MS, + DEFAULT_SESSION_SNAPSHOT_TTL_DAYS, + getSessionSnapshotConfig, +} from '../../src/services/session-snapshots'; + +function env(overrides: Partial = {}): Env { + return overrides as Env; +} + +describe('session snapshot config', () => { + it('uses constitution-compliant defaults when env vars are absent', () => { + const config = getSessionSnapshotConfig(env()); + + expect(config).toEqual({ + ttlDays: DEFAULT_SESSION_SNAPSHOT_TTL_DAYS, + totalBudgetBytes: DEFAULT_SESSION_SNAPSHOT_TOTAL_BUDGET_BYTES, + entryThresholdBytes: DEFAULT_SESSION_SNAPSHOT_ENTRY_THRESHOLD_BYTES, + transferIdleTimeoutMs: DEFAULT_SESSION_SNAPSHOT_TRANSFER_IDLE_TIMEOUT_MS, + r2Prefix: DEFAULT_SESSION_SNAPSHOT_R2_PREFIX, + }); + }); + + it('uses positive env overrides and sanitizes the R2 prefix', () => { + const config = getSessionSnapshotConfig(env({ + SESSION_SNAPSHOT_TTL_DAYS: '3', + SESSION_SNAPSHOT_TOTAL_BUDGET_BYTES: '1234', + SESSION_SNAPSHOT_ENTRY_THRESHOLD_BYTES: '567', + SESSION_SNAPSHOT_TRANSFER_IDLE_TIMEOUT_MS: '890', + SESSION_SNAPSHOT_R2_PREFIX: '/tenant snapshots/private/', + })); + + expect(config.ttlDays).toBe(3); + expect(config.totalBudgetBytes).toBe(1234); + expect(config.entryThresholdBytes).toBe(567); + expect(config.transferIdleTimeoutMs).toBe(890); + expect(config.r2Prefix).toBe('tenant-snapshots/private'); + }); +}); + +describe('session snapshot R2 keys', () => { + it('derives deterministic one-snapshot-per-session artifact keys', () => { + const input = env({ SESSION_SNAPSHOT_R2_PREFIX: 'snapshots' }); + + expect(buildSessionSnapshotR2Key(input, 'chat/../session 1', 'home')).toBe( + 'snapshots/chat-..-session-1/home.tar' + ); + expect(buildSessionSnapshotR2Key(input, 'chat/../session 1', 'wip')).toBe( + 'snapshots/chat-..-session-1/wip.bundle' + ); + expect(buildSessionSnapshotR2Key(input, 'chat/../session 1', 'manifest')).toBe( + 'snapshots/chat-..-session-1/manifest.json' + ); + }); +}); diff --git a/packages/vm-agent/internal/server/server.go b/packages/vm-agent/internal/server/server.go index 1e4febf35..6ab3266e9 100644 --- a/packages/vm-agent/internal/server/server.go +++ b/packages/vm-agent/internal/server/server.go @@ -1027,6 +1027,8 @@ func (s *Server) setupRoutes(mux *http.ServeMux) { mux.HandleFunc("POST /workspaces/{workspaceId}/agent-sessions/{sessionId}/suspend", s.handleSuspendAgentSession) mux.HandleFunc("POST /workspaces/{workspaceId}/agent-sessions/{sessionId}/resume", s.handleResumeAgentSession) mux.HandleFunc("POST /workspaces/{workspaceId}/agent-sessions/{sessionId}/prompt", s.handleSendPrompt) + mux.HandleFunc("POST /workspaces/{workspaceId}/agent-sessions/{sessionId}/hibernate", s.handleHibernateAgentSession) + mux.HandleFunc("POST /workspaces/{workspaceId}/agent-sessions/{sessionId}/restore", s.handleRestoreAgentSession) mux.HandleFunc("GET /workspaces/{workspaceId}/tabs", s.handleListTabs) // Git integration (browser-authenticated via workspace session/token) diff --git a/packages/vm-agent/internal/server/session_snapshot.go b/packages/vm-agent/internal/server/session_snapshot.go new file mode 100644 index 000000000..bbb0e15be --- /dev/null +++ b/packages/vm-agent/internal/server/session_snapshot.go @@ -0,0 +1,709 @@ +package server + +import ( + "archive/tar" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" +) + +const ( + defaultSnapshotTotalBudgetBytes int64 = 100 * 1024 * 1024 + defaultSnapshotEntryThresholdBytes int64 = 50 * 1024 * 1024 + defaultSnapshotTransferIdleTimeout time.Duration = 30 * time.Second +) + +type snapshotPrepareResponse struct { + ExpiresAt string `json:"expiresAt"` + Config struct { + TotalBudgetBytes int64 `json:"totalBudgetBytes"` + EntryThresholdBytes int64 `json:"entryThresholdBytes"` + TransferIdleTimeoutMs int64 `json:"transferIdleTimeoutMs"` + } `json:"config"` + Upload struct { + Home string `json:"home"` + WIP string `json:"wip"` + } `json:"upload"` +} + +type snapshotRestoreResponse struct { + Available bool `json:"available"` + Reason string `json:"reason,omitempty"` + Status string `json:"status,omitempty"` + Degradation string `json:"degradation,omitempty"` + BaseCommit string `json:"baseCommit,omitempty"` + Manifest map[string]interface{} `json:"manifest,omitempty"` + Download struct { + Home string `json:"home"` + WIP string `json:"wip"` + Manifest string `json:"manifest"` + } `json:"download"` +} + +type snapshotManifest struct { + Version int `json:"version"` + ChatSessionID string `json:"chatSessionId"` + WorkspaceID string `json:"workspaceId"` + AgentSessionID string `json:"agentSessionId,omitempty"` + BaseCommit string `json:"baseCommit,omitempty"` + Status string `json:"status"` + Degradation string `json:"degradation"` + Skipped []snapshotSkippedEntry `json:"skipped"` + Artifacts map[string]snapshotArtifact `json:"artifacts"` + CreatedAt string `json:"createdAt"` +} + +type snapshotSkippedEntry struct { + Path string `json:"path"` + Reason string `json:"reason"` + SizeBytes int64 `json:"sizeBytes,omitempty"` +} + +type snapshotArtifact struct { + SizeBytes int64 `json:"sizeBytes"` + SHA256 string `json:"sha256,omitempty"` +} + +type countingReader struct { + r io.Reader + n int64 + hash hashWriter +} + +type hashWriter interface { + Write([]byte) (int, error) + Sum([]byte) []byte +} + +func (r *countingReader) Read(p []byte) (int, error) { + n, err := r.r.Read(p) + if n > 0 { + r.n += int64(n) + _, _ = r.hash.Write(p[:n]) + } + return n, err +} + +func (s *Server) handleHibernateAgentSession(w http.ResponseWriter, r *http.Request) { + workspaceID := r.PathValue("workspaceId") + sessionID := r.PathValue("sessionId") + if workspaceID == "" || sessionID == "" { + writeError(w, http.StatusBadRequest, "workspaceId and sessionId are required") + return + } + if !s.requireNodeManagementAuth(w, r, workspaceID) { + return + } + var body struct { + ChatSessionID string `json:"chatSessionId"` + Runtime string `json:"runtime"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + body.ChatSessionID = strings.TrimSpace(body.ChatSessionID) + if body.ChatSessionID == "" { + writeError(w, http.StatusBadRequest, "chatSessionId is required") + return + } + runtime, ok := s.getWorkspaceRuntime(workspaceID) + if !ok { + writeError(w, http.StatusNotFound, "workspace not found") + return + } + callbackToken := s.callbackTokenForWorkspace(workspaceID) + if callbackToken == "" { + writeError(w, http.StatusConflict, "workspace callback token unavailable") + return + } + result, err := s.hibernateSessionSnapshot(r.Context(), runtime, sessionID, body.ChatSessionID, body.Runtime, callbackToken) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, http.StatusOK, result) +} + +func (s *Server) handleRestoreAgentSession(w http.ResponseWriter, r *http.Request) { + workspaceID := r.PathValue("workspaceId") + sessionID := r.PathValue("sessionId") + if workspaceID == "" || sessionID == "" { + writeError(w, http.StatusBadRequest, "workspaceId and sessionId are required") + return + } + if !s.requireNodeManagementAuth(w, r, workspaceID) { + return + } + var body struct { + ChatSessionID string `json:"chatSessionId"` + Runtime string `json:"runtime"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + body.ChatSessionID = strings.TrimSpace(body.ChatSessionID) + if body.ChatSessionID == "" { + writeError(w, http.StatusBadRequest, "chatSessionId is required") + return + } + runtime, ok := s.getWorkspaceRuntime(workspaceID) + if !ok { + writeError(w, http.StatusNotFound, "workspace not found") + return + } + callbackToken := s.callbackTokenForWorkspace(workspaceID) + if callbackToken == "" { + writeError(w, http.StatusConflict, "workspace callback token unavailable") + return + } + result, err := s.restoreSessionSnapshot(r.Context(), runtime, sessionID, body.ChatSessionID, callbackToken) + if err != nil { + _ = s.reportSnapshotRestoreResult(context.Background(), workspaceID, body.ChatSessionID, "degraded", err.Error(), callbackToken) + writeJSON(w, http.StatusOK, map[string]interface{}{"status": "degraded", "message": err.Error()}) + return + } + writeJSON(w, http.StatusOK, result) +} + +func (s *Server) hibernateSessionSnapshot(ctx context.Context, runtime *WorkspaceRuntime, sessionID, chatSessionID, runtimeName, callbackToken string) (map[string]interface{}, error) { + prepare, err := s.prepareSnapshot(ctx, runtime.ID, sessionID, chatSessionID, runtimeName, callbackToken) + if err != nil { + return nil, err + } + totalBudget := choosePositiveInt64(prepare.Config.TotalBudgetBytes, defaultSnapshotTotalBudgetBytes) + entryThreshold := choosePositiveInt64(prepare.Config.EntryThresholdBytes, defaultSnapshotEntryThresholdBytes) + idleTimeout := choosePositiveDurationMs(prepare.Config.TransferIdleTimeoutMs, defaultSnapshotTransferIdleTimeout) + manifest := snapshotManifest{ + Version: 1, + ChatSessionID: chatSessionID, + WorkspaceID: runtime.ID, + AgentSessionID: sessionID, + Status: "available", + Degradation: "none", + Skipped: []snapshotSkippedEntry{}, + Artifacts: map[string]snapshotArtifact{}, + CreatedAt: time.Now().UTC().Format(time.RFC3339), + } + workDir := standaloneWorkspaceWorkDir(runtime, s.config.WorkspaceDir, s.config.ContainerWorkDir) + baseCommit, wipPath, wipSkipped, err := createWIPBundle(ctx, workDir, entryThreshold) + manifest.BaseCommit = baseCommit + manifest.Skipped = append(manifest.Skipped, wipSkipped...) + if err != nil { + manifest.Skipped = append(manifest.Skipped, snapshotSkippedEntry{Path: workDir, Reason: err.Error()}) + } + + remaining := totalBudget + if wipPath != "" { + size, sha, uploadErr := s.uploadSnapshotFile(ctx, prepare.Upload.WIP, wipPath, callbackToken, idleTimeout) + _ = os.Remove(wipPath) + if uploadErr != nil { + manifest.Skipped = append(manifest.Skipped, snapshotSkippedEntry{Path: workDir, Reason: uploadErr.Error()}) + } else { + manifest.Artifacts["wip"] = snapshotArtifact{SizeBytes: size, SHA256: sha} + remaining -= size + } + } + homePath, homeSkipped, err := createHomeTar(os.UserHomeDir, entryThreshold, remaining) + manifest.Skipped = append(manifest.Skipped, homeSkipped...) + if err != nil { + manifest.Skipped = append(manifest.Skipped, snapshotSkippedEntry{Path: "$HOME", Reason: err.Error()}) + } + if homePath != "" { + size, sha, uploadErr := s.uploadSnapshotFile(ctx, prepare.Upload.Home, homePath, callbackToken, idleTimeout) + _ = os.Remove(homePath) + if uploadErr != nil { + manifest.Skipped = append(manifest.Skipped, snapshotSkippedEntry{Path: "$HOME", Reason: uploadErr.Error()}) + } else { + manifest.Artifacts["home"] = snapshotArtifact{SizeBytes: size, SHA256: sha} + } + } + if _, ok := manifest.Artifacts["home"]; !ok { + manifest.Degradation = "wip-only" + } + if _, ok := manifest.Artifacts["wip"]; !ok { + if manifest.Degradation == "wip-only" { + manifest.Degradation = "transcript-only" + manifest.Status = "degraded" + } else { + manifest.Degradation = "home-skipped" + manifest.Status = "degraded" + } + } + if len(manifest.Skipped) > 0 && manifest.Status == "available" { + manifest.Status = "degraded" + } + err = s.completeSnapshot(ctx, runtime.ID, sessionID, chatSessionID, runtimeName, callbackToken, manifest) + if err != nil { + return nil, err + } + return map[string]interface{}{"status": manifest.Status, "degradation": manifest.Degradation, "skipped": manifest.Skipped}, nil +} + +func (s *Server) restoreSessionSnapshot(ctx context.Context, runtime *WorkspaceRuntime, sessionID, chatSessionID, callbackToken string) (map[string]interface{}, error) { + restore, err := s.fetchSnapshotRestore(ctx, runtime.ID, chatSessionID, callbackToken) + if err != nil { + return nil, err + } + if !restore.Available { + _ = s.reportSnapshotRestoreResult(ctx, runtime.ID, chatSessionID, "missing", restore.Reason, callbackToken) + return map[string]interface{}{"status": "transcript-replay", "reason": restore.Reason}, nil + } + idleTimeout := defaultSnapshotTransferIdleTimeout + if restore.Download.Home != "" { + if err := s.downloadAndExtractTar(ctx, restore.Download.Home, callbackToken, idleTimeout); err != nil { + _ = s.reportSnapshotRestoreResult(ctx, runtime.ID, chatSessionID, "home_failed", err.Error(), callbackToken) + return nil, err + } + } + if restore.Download.WIP != "" { + workDir := standaloneWorkspaceWorkDir(runtime, s.config.WorkspaceDir, s.config.ContainerWorkDir) + if err := s.downloadAndRestoreWIP(ctx, restore.Download.WIP, callbackToken, idleTimeout, workDir, restore.BaseCommit); err != nil { + _ = s.reportSnapshotRestoreResult(ctx, runtime.ID, chatSessionID, "wip_failed", err.Error(), callbackToken) + return nil, err + } + } + if _, err := s.provisionWorkspaceRuntime(ctx, runtime); err != nil { + _ = s.reportSnapshotRestoreResult(ctx, runtime.ID, chatSessionID, "fresh_injection_failed", err.Error(), callbackToken) + return nil, err + } + if session, exists := s.agentSessions.Get(runtime.ID, sessionID); exists { + hostKey := runtime.ID + ":" + sessionID + _ = s.getOrCreateSessionHost(hostKey, runtime.ID, sessionID, session, runtime, "") + } + _ = s.reportSnapshotRestoreResult(ctx, runtime.ID, chatSessionID, "restored", "", callbackToken) + return map[string]interface{}{"status": "restored", "degradation": restore.Degradation}, nil +} + +func (s *Server) prepareSnapshot(ctx context.Context, workspaceID, sessionID, chatSessionID, runtimeName, token string) (*snapshotPrepareResponse, error) { + payload := map[string]string{"chatSessionId": chatSessionID, "agentSessionId": sessionID, "runtime": runtimeName} + var out snapshotPrepareResponse + err := s.doSnapshotJSON(ctx, http.MethodPost, workspaceID, "/session-snapshot/prepare", token, payload, &out) + return &out, err +} + +func (s *Server) completeSnapshot(ctx context.Context, workspaceID, sessionID, chatSessionID, runtimeName, token string, manifest snapshotManifest) error { + artifactSizes := map[string]int64{} + if artifact, ok := manifest.Artifacts["home"]; ok { + artifactSizes["homeBytes"] = artifact.SizeBytes + } + if artifact, ok := manifest.Artifacts["wip"]; ok { + artifactSizes["wipBytes"] = artifact.SizeBytes + } + payload := map[string]interface{}{ + "chatSessionId": chatSessionID, + "agentSessionId": sessionID, + "runtime": runtimeName, + "baseCommit": manifest.BaseCommit, + "status": manifest.Status, + "degradation": manifest.Degradation, + "manifest": manifest, + "artifactSizes": artifactSizes, + } + var out map[string]interface{} + return s.doSnapshotJSON(ctx, http.MethodPost, workspaceID, "/session-snapshot/complete", token, payload, &out) +} + +func (s *Server) fetchSnapshotRestore(ctx context.Context, workspaceID, chatSessionID, token string) (*snapshotRestoreResponse, error) { + path := "/session-snapshot/restore?chatSessionId=" + url.QueryEscape(chatSessionID) + var out snapshotRestoreResponse + err := s.doSnapshotJSON(ctx, http.MethodGet, workspaceID, path, token, nil, &out) + return &out, err +} + +func (s *Server) reportSnapshotRestoreResult(ctx context.Context, workspaceID, chatSessionID, status, message, token string) error { + payload := map[string]string{"chatSessionId": chatSessionID, "status": status, "message": message} + var out map[string]interface{} + return s.doSnapshotJSON(ctx, http.MethodPost, workspaceID, "/session-snapshot/restore-result", token, payload, &out) +} + +func (s *Server) doSnapshotJSON(ctx context.Context, method, workspaceID, path, token string, payload interface{}, out interface{}) error { + endpoint := strings.TrimRight(s.config.ControlPlaneURL, "/") + "/api/workspaces/" + url.PathEscape(workspaceID) + path + var body io.Reader + if payload != nil { + data, err := json.Marshal(payload) + if err != nil { + return err + } + body = bytes.NewReader(data) + } + req, err := http.NewRequestWithContext(ctx, method, endpoint, body) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+token) + if payload != nil { + req.Header.Set("Content-Type", "application/json") + } + res, err := s.controlPlaneHTTPClient(0).Do(req) + if err != nil { + return err + } + defer res.Body.Close() + data, _ := io.ReadAll(io.LimitReader(res.Body, 1024*1024)) + if res.StatusCode < 200 || res.StatusCode >= 300 { + return fmt.Errorf("snapshot control plane returned HTTP %d: %s", res.StatusCode, strings.TrimSpace(string(data))) + } + if out != nil && len(data) > 0 { + if err := json.Unmarshal(data, out); err != nil { + return err + } + } + return nil +} + +func createWIPBundle(ctx context.Context, workDir string, entryThreshold int64) (string, string, []snapshotSkippedEntry, error) { + if ok, err := standaloneRepositoryPresent(workDir); err != nil || !ok { + if err != nil { + return "", "", nil, err + } + return "", "", nil, nil + } + if gitOperationInProgress(workDir) { + return "", "", []snapshotSkippedEntry{{Path: workDir, Reason: "git operation in progress"}}, nil + } + base, err := runStandaloneGitCommand(ctx, workDir, nil, "rev-parse", "HEAD") + if err != nil { + return "", "", nil, fmt.Errorf("resolve base commit: %w", err) + } + status, err := runStandaloneGitCommand(ctx, workDir, nil, "status", "--porcelain") + if err != nil { + return base, "", nil, fmt.Errorf("git status: %w", err) + } + if strings.TrimSpace(status) == "" { + return base, "", nil, nil + } + skipped := skipOversizedUntracked(workDir, entryThreshold) + _, _ = runStandaloneGitCommand(ctx, workDir, nil, "add", "-A") + for _, entry := range skipped { + if entry.Path != "" { + _, _ = runStandaloneGitCommand(ctx, workDir, nil, "reset", "--", entry.Path) + } + } + if _, err := runStandaloneGitCommand(ctx, workDir, nil, "commit", "--no-verify", "-m", "SAM session snapshot"); err != nil { + return base, "", skipped, fmt.Errorf("create temporary snapshot commit: %w", err) + } + bundle, err := os.CreateTemp("", "sam-session-wip-*.bundle") + if err != nil { + return base, "", skipped, err + } + bundlePath := bundle.Name() + _ = bundle.Close() + if _, err := runStandaloneGitCommand(ctx, workDir, nil, "bundle", "create", bundlePath, base+"..HEAD"); err != nil { + _, _ = runStandaloneGitCommand(ctx, workDir, nil, "reset", "--mixed", base) + _ = os.Remove(bundlePath) + return base, "", skipped, fmt.Errorf("create git bundle: %w", err) + } + _, _ = runStandaloneGitCommand(ctx, workDir, nil, "reset", "--mixed", base) + return base, bundlePath, skipped, nil +} + +func gitOperationInProgress(workDir string) bool { + gitDir := filepath.Join(workDir, ".git") + for _, marker := range []string{"MERGE_HEAD", "CHERRY_PICK_HEAD", "REVERT_HEAD", "rebase-merge", "rebase-apply"} { + if _, err := os.Stat(filepath.Join(gitDir, marker)); err == nil { + return true + } + } + return false +} + +func skipOversizedUntracked(workDir string, threshold int64) []snapshotSkippedEntry { + var skipped []snapshotSkippedEntry + _ = filepath.WalkDir(workDir, func(path string, d os.DirEntry, err error) error { + if err != nil || path == workDir { + return nil + } + if d.IsDir() && d.Name() == ".git" { + return filepath.SkipDir + } + if d.IsDir() { + return nil + } + info, statErr := d.Info() + if statErr != nil || info.Size() <= threshold { + return nil + } + rel, _ := filepath.Rel(workDir, path) + if out, gitErr := runStandaloneGitCommand(context.Background(), workDir, nil, "check-ignore", "-q", rel); gitErr == nil && strings.TrimSpace(out) == "" { + return nil + } + skipped = append(skipped, snapshotSkippedEntry{Path: rel, Reason: "entry exceeds size threshold", SizeBytes: info.Size()}) + return nil + }) + return skipped +} + +func createHomeTar(homeDirFn func() (string, error), entryThreshold, totalBudget int64) (string, []snapshotSkippedEntry, error) { + home, err := homeDirFn() + if err != nil { + return "", nil, err + } + home = filepath.Clean(home) + out, err := os.CreateTemp("", "sam-session-home-*.tar") + if err != nil { + return "", nil, err + } + path := out.Name() + tw := tar.NewWriter(out) + var written int64 + var skipped []snapshotSkippedEntry + walkErr := filepath.WalkDir(home, func(path string, d os.DirEntry, err error) error { + if err != nil || path == home { + return nil + } + rel, _ := filepath.Rel(home, path) + if shouldExcludeHomePath(rel) { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + info, statErr := d.Info() + if statErr != nil { + return nil + } + if info.Size() > entryThreshold { + skipped = append(skipped, snapshotSkippedEntry{Path: "~/" + rel, Reason: "entry exceeds size threshold", SizeBytes: info.Size()}) + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + if !info.IsDir() && written+info.Size() > totalBudget { + skipped = append(skipped, snapshotSkippedEntry{Path: "~/" + rel, Reason: "snapshot budget exhausted", SizeBytes: info.Size()}) + return nil + } + header, headerErr := tar.FileInfoHeader(info, "") + if headerErr != nil { + return nil + } + header.Name = rel + if err := tw.WriteHeader(header); err != nil { + return err + } + if info.Mode().IsRegular() { + f, openErr := os.Open(path) + if openErr != nil { + return nil + } + n, copyErr := io.Copy(tw, f) + _ = f.Close() + written += n + if copyErr != nil { + return copyErr + } + } + return nil + }) + closeErr := tw.Close() + fileCloseErr := out.Close() + if walkErr != nil || closeErr != nil || fileCloseErr != nil { + _ = os.Remove(path) + if walkErr != nil { + return "", skipped, walkErr + } + if closeErr != nil { + return "", skipped, closeErr + } + return "", skipped, fileCloseErr + } + return path, skipped, nil +} + +func shouldExcludeHomePath(rel string) bool { + first := strings.Split(filepath.ToSlash(rel), "/")[0] + switch first { + case ".cache", ".npm", ".cargo", ".rustup", ".local", "node_modules", ".docker": + return true + default: + return false + } +} + +func (s *Server) uploadSnapshotFile(ctx context.Context, uploadPath, filePath, token string, idleTimeout time.Duration) (int64, string, error) { + target := absoluteControlPlaneURL(s.config.ControlPlaneURL, uploadPath) + file, err := os.Open(filePath) + if err != nil { + return 0, "", err + } + defer file.Close() + h := sha256.New() + reader := &countingReader{r: file, hash: h} + req, err := http.NewRequestWithContext(ctx, http.MethodPut, target, newIdleReader(reader, idleTimeout)) + if err != nil { + return 0, "", err + } + req.Header.Set("Authorization", "Bearer "+token) + res, err := s.controlPlaneHTTPClient(0).Do(req) + if err != nil { + return 0, "", err + } + defer res.Body.Close() + body, _ := io.ReadAll(io.LimitReader(res.Body, 64*1024)) + if res.StatusCode < 200 || res.StatusCode >= 300 { + return 0, "", fmt.Errorf("artifact upload failed HTTP %d: %s", res.StatusCode, strings.TrimSpace(string(body))) + } + return reader.n, hex.EncodeToString(h.Sum(nil)), nil +} + +func (s *Server) downloadAndExtractTar(ctx context.Context, downloadPath, token string, idleTimeout time.Duration) error { + res, err := s.snapshotDownload(ctx, downloadPath, token) + if err != nil { + return err + } + defer res.Body.Close() + home, err := os.UserHomeDir() + if err != nil { + return err + } + tr := tar.NewReader(newIdleReader(res.Body, idleTimeout)) + for { + header, err := tr.Next() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + target := filepath.Join(home, filepath.Clean(header.Name)) + if !strings.HasPrefix(target, home) { + continue + } + if header.FileInfo().IsDir() { + if err := os.MkdirAll(target, header.FileInfo().Mode()); err != nil { + return err + } + continue + } + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, header.FileInfo().Mode()) + if err != nil { + return err + } + _, copyErr := io.Copy(f, tr) + closeErr := f.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + } +} + +func (s *Server) downloadAndRestoreWIP(ctx context.Context, downloadPath, token string, idleTimeout time.Duration, workDir, baseCommit string) error { + res, err := s.snapshotDownload(ctx, downloadPath, token) + if err != nil { + return err + } + defer res.Body.Close() + tmp, err := os.CreateTemp("", "sam-session-restore-*.bundle") + if err != nil { + return err + } + tmpPath := tmp.Name() + _, copyErr := io.Copy(tmp, newIdleReader(res.Body, idleTimeout)) + closeErr := tmp.Close() + if copyErr != nil { + _ = os.Remove(tmpPath) + return copyErr + } + if closeErr != nil { + _ = os.Remove(tmpPath) + return closeErr + } + defer os.Remove(tmpPath) + if _, err := runStandaloneGitCommand(ctx, workDir, nil, "fetch", tmpPath, "HEAD:sam-session-restore"); err != nil { + return err + } + if _, err := runStandaloneGitCommand(ctx, workDir, nil, "checkout", "sam-session-restore"); err != nil { + return err + } + if strings.TrimSpace(baseCommit) != "" { + if _, err := runStandaloneGitCommand(ctx, workDir, nil, "reset", "--soft", baseCommit); err != nil { + return err + } + } + _, _ = runStandaloneGitCommand(ctx, workDir, nil, "branch", "-D", "sam-session-restore") + return nil +} + +func (s *Server) snapshotDownload(ctx context.Context, downloadPath, token string) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, absoluteControlPlaneURL(s.config.ControlPlaneURL, downloadPath), nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+token) + res, err := s.controlPlaneHTTPClient(0).Do(req) + if err != nil { + return nil, err + } + if res.StatusCode < 200 || res.StatusCode >= 300 { + body, _ := io.ReadAll(io.LimitReader(res.Body, 64*1024)) + _ = res.Body.Close() + return nil, fmt.Errorf("artifact download failed HTTP %d: %s", res.StatusCode, strings.TrimSpace(string(body))) + } + return res, nil +} + +func absoluteControlPlaneURL(base, path string) string { + if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { + return path + } + return strings.TrimRight(base, "/") + path +} + +type idleReader struct { + reader io.Reader + idleTimeout time.Duration +} + +func newIdleReader(reader io.Reader, idleTimeout time.Duration) io.Reader { + return &idleReader{reader: reader, idleTimeout: idleTimeout} +} + +func (r *idleReader) Read(p []byte) (int, error) { + type result struct { + n int + err error + } + ch := make(chan result, 1) + go func() { + n, err := r.reader.Read(p) + ch <- result{n: n, err: err} + }() + select { + case res := <-ch: + return res.n, res.err + case <-time.After(r.idleTimeout): + return 0, fmt.Errorf("snapshot transfer stalled for %s", r.idleTimeout) + } +} + +func choosePositiveInt64(value, fallback int64) int64 { + if value > 0 { + return value + } + return fallback +} + +func choosePositiveDurationMs(value int64, fallback time.Duration) time.Duration { + if value > 0 { + return time.Duration(value) * time.Millisecond + } + return fallback +} diff --git a/packages/vm-agent/internal/server/session_snapshot_test.go b/packages/vm-agent/internal/server/session_snapshot_test.go new file mode 100644 index 000000000..ac5364ce7 --- /dev/null +++ b/packages/vm-agent/internal/server/session_snapshot_test.go @@ -0,0 +1,119 @@ +package server + +import ( + "archive/tar" + "context" + "io" + "os" + "os/exec" + "path/filepath" + "testing" +) + +func TestCreateHomeTarExcludesCachesAndRecordsOversizedFiles(t *testing.T) { + home := t.TempDir() + if err := os.WriteFile(filepath.Join(home, ".codex-session.jsonl"), []byte("session"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(home, ".cache"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(home, ".cache", "ignored"), []byte("cache"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(home, "large.bin"), []byte("0123456789"), 0o600); err != nil { + t.Fatal(err) + } + + tarPath, skipped, err := createHomeTar(func() (string, error) { return home, nil }, 8, 1024) + if err != nil { + t.Fatal(err) + } + defer os.Remove(tarPath) + + if len(skipped) != 1 { + t.Fatalf("skipped len = %d, want 1: %#v", len(skipped), skipped) + } + if skipped[0].Path != "~/large.bin" { + t.Fatalf("skipped path = %q, want ~/large.bin", skipped[0].Path) + } + + names := tarNames(t, tarPath) + if !containsString(names, ".codex-session.jsonl") { + t.Fatalf("tar names missing session file: %#v", names) + } + if containsString(names, ".cache/ignored") { + t.Fatalf("tar names included cache file: %#v", names) + } + if containsString(names, "large.bin") { + t.Fatalf("tar names included oversized file: %#v", names) + } +} + +func TestCreateWIPBundleDegradesDuringMerge(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + repo := t.TempDir() + runGit(t, repo, "init") + runGit(t, repo, "config", "user.email", "sam@example.test") + runGit(t, repo, "config", "user.name", "SAM") + if err := os.WriteFile(filepath.Join(repo, "README.md"), []byte("base"), 0o600); err != nil { + t.Fatal(err) + } + runGit(t, repo, "add", "README.md") + runGit(t, repo, "commit", "-m", "base") + if err := os.WriteFile(filepath.Join(repo, ".git", "MERGE_HEAD"), []byte("deadbeef"), 0o600); err != nil { + t.Fatal(err) + } + + _, bundlePath, skipped, err := createWIPBundle(context.Background(), repo, 1024) + if err != nil { + t.Fatal(err) + } + if bundlePath != "" { + t.Fatalf("bundlePath = %q, want empty during merge", bundlePath) + } + if len(skipped) != 1 || skipped[0].Reason != "git operation in progress" { + t.Fatalf("skipped = %#v, want git operation degradation", skipped) + } +} + +func tarNames(t *testing.T, path string) []string { + t.Helper() + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + tr := tar.NewReader(f) + var names []string + for { + header, err := tr.Next() + if err == io.EOF { + return names + } + if err != nil { + t.Fatal(err) + } + names = append(names, header.Name) + } +} + +func runGit(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v failed: %v\n%s", args, err, out) + } +} + +func containsString(values []string, needle string) bool { + for _, value := range values { + if value == needle { + return true + } + } + return false +} diff --git a/tasks/active/2026-07-11-runtime-neutral-session-hibernate-wake.md b/tasks/active/2026-07-11-runtime-neutral-session-hibernate-wake.md index 07955c062..14d931021 100644 --- a/tasks/active/2026-07-11-runtime-neutral-session-hibernate-wake.md +++ b/tasks/active/2026-07-11-runtime-neutral-session-hibernate-wake.md @@ -23,15 +23,15 @@ Hard constraints from Raphaël, 2026-07-11: ## Implementation Checklist -- [ ] Add additive D1 schema/migration for runtime-neutral session snapshots: canonical chat session key, R2 keys, manifest/degradation, expiry, and restore diagnostics. -- [ ] Add env-configurable defaults for snapshot TTL, size budgets, per-entry thresholds, progress idle watchdogs, and R2 key prefix. -- [ ] Add Worker snapshot service for deterministic one-snapshot-per-session R2 keys, metadata persistence, retention checks, cleanup hooks, and visible degradation state. -- [ ] Add VM-agent callback/API endpoints with callback JWT auth for snapshot upload/download coordination, mounted outside browser-session middleware. -- [ ] Add VM-agent hibernate implementation: git WIP bundle, `$HOME` tar with generic cache exclusions, size/skip manifest, progress idle watchdogs, and no remote push. -- [ ] Add VM-agent restore implementation: restore `$HOME`, restore WIP bundle with soft reset to base, re-run runtime injection afterward, and report visible degraded results. -- [ ] Wire cf-container hibernate on control handback before sleep is allowed. -- [ ] Wire cf-container wake before proxying a sleeping container and before follow-up prompt dispatch; fall back visibly when no snapshot exists or restore fails. -- [ ] Update follow-up prompt handling so a fresh vm-agent can rebuild a `SessionHost` and attempt native harness resume from restored `$HOME`. +- [x] Add additive D1 schema/migration for runtime-neutral session snapshots: canonical chat session key, R2 keys, manifest/degradation, expiry, and restore diagnostics. +- [x] Add env-configurable defaults for snapshot TTL, size budgets, per-entry thresholds, progress idle watchdogs, and R2 key prefix. +- [x] Add Worker snapshot service for deterministic one-snapshot-per-session R2 keys, metadata persistence, retention checks, cleanup hooks, and visible degradation state. +- [x] Add VM-agent callback/API endpoints with callback JWT auth for snapshot upload/download coordination, mounted outside browser-session middleware. +- [x] Add VM-agent hibernate implementation: git WIP bundle, `$HOME` tar with generic cache exclusions, size/skip manifest, progress idle watchdogs, and no remote push. +- [x] Add VM-agent restore implementation: restore `$HOME`, restore WIP bundle with soft reset to base, re-run runtime injection afterward, and report visible degraded results. +- [x] Wire cf-container hibernate on control handback before sleep is allowed. +- [x] Wire cf-container wake before proxying a sleeping container and before follow-up prompt dispatch; fall back visibly when no snapshot exists or restore fails. +- [x] Update follow-up prompt handling so a fresh vm-agent can rebuild a `SessionHost` and attempt native harness resume from restored `$HOME`. - [ ] Add local tests: Worker service/unit tests, Miniflare/DO integration tests, VM-agent Go tests including `go test -race`, and a vertical-slice test covering Worker → DO → vm-agent → R2 state. - [ ] Run migration safety checks: `pnpm quality:migration-safety` and `pnpm quality:do-migration-safety`. - [ ] Run local quality gates: `pnpm lint`, `pnpm typecheck`, `pnpm test`, `pnpm build`, and package-specific tests. From eb24090cf7d5a693ae606d9bbde833b4e695d909 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 12:28:56 +0000 Subject: [PATCH 03/28] test: cover session snapshot wake edge cases --- apps/api/src/env.ts | 1 + .../routes/workspaces/session-snapshots.ts | 4 +- apps/api/src/services/session-snapshots.ts | 6 + .../cf-container-runtime-contract.test.ts | 8 +- .../workspaces-session-snapshots.test.ts | 198 ++++++++++++++++++ apps/api/tests/unit/session-snapshots.test.ts | 4 + .../workers/session-snapshot-wiring.test.ts | 125 +++++++++++ .../internal/server/session_snapshot.go | 22 +- .../internal/server/session_snapshot_test.go | 56 +++++ ...-runtime-neutral-session-hibernate-wake.md | 30 ++- 10 files changed, 442 insertions(+), 12 deletions(-) create mode 100644 apps/api/tests/unit/routes/workspaces-session-snapshots.test.ts create mode 100644 apps/api/tests/workers/session-snapshot-wiring.test.ts diff --git a/apps/api/src/env.ts b/apps/api/src/env.ts index 506e23214..93fcb5307 100644 --- a/apps/api/src/env.ts +++ b/apps/api/src/env.ts @@ -145,6 +145,7 @@ export interface Env { SESSION_SNAPSHOT_TOTAL_BUDGET_BYTES?: string; // Max combined home/WIP snapshot size (default: 104857600) SESSION_SNAPSHOT_ENTRY_THRESHOLD_BYTES?: string; // Max individual file/dir included by vm-agent scanner (default: 52428800) SESSION_SNAPSHOT_TRANSFER_IDLE_TIMEOUT_MS?: string; // No-progress upload/download watchdog window (default: 30000) + SESSION_SNAPSHOT_JSON_BODY_MAX_BYTES?: string; // Max snapshot control-plane JSON request size (default: 262144) DEPLOY_ACME_EMAIL?: string; // Contact email for deployment-node ACME certificates DEPLOY_ACME_CA?: string; // ACME CA directory override for deployment nodes DEPLOY_COMPOSE_CMD?: string; // Docker Compose command override on deployment nodes diff --git a/apps/api/src/routes/workspaces/session-snapshots.ts b/apps/api/src/routes/workspaces/session-snapshots.ts index a8adb6c1f..cd11fdc9a 100644 --- a/apps/api/src/routes/workspaces/session-snapshots.ts +++ b/apps/api/src/routes/workspaces/session-snapshots.ts @@ -32,11 +32,10 @@ const DEGRADATIONS = new Set([ 'wip-only', 'transcript-only', ]); -const MAX_SNAPSHOT_JSON_BYTES = 256 * 1024; async function readJsonBody(c: SnapshotRouteContext) { const raw = await c.req.raw.text(); - if (new TextEncoder().encode(raw).byteLength > MAX_SNAPSHOT_JSON_BYTES) { + if (new TextEncoder().encode(raw).byteLength > getSessionSnapshotConfig(c.env).jsonBodyMaxBytes) { throw errors.badRequest('Snapshot request body is too large'); } try { @@ -202,6 +201,7 @@ sessionSnapshotRoutes.get('/:id/session-snapshot/restore', async (c) => { degradation: snapshot.degradation, baseCommit: snapshot.baseCommit, manifest: snapshot.manifestJson ? JSON.parse(snapshot.manifestJson) : null, + config: getSessionSnapshotConfig(c.env), download: { home: snapshot.homeR2Key ? `/api/workspaces/${workspaceId}/session-snapshot/artifacts/home?chatSessionId=${encodeURIComponent(chatSessionId)}` diff --git a/apps/api/src/services/session-snapshots.ts b/apps/api/src/services/session-snapshots.ts index 70041ad7a..393209b7d 100644 --- a/apps/api/src/services/session-snapshots.ts +++ b/apps/api/src/services/session-snapshots.ts @@ -13,6 +13,7 @@ export const DEFAULT_SESSION_SNAPSHOT_TTL_DAYS = 7; export const DEFAULT_SESSION_SNAPSHOT_TOTAL_BUDGET_BYTES = 100 * 1024 * 1024; export const DEFAULT_SESSION_SNAPSHOT_ENTRY_THRESHOLD_BYTES = 50 * 1024 * 1024; export const DEFAULT_SESSION_SNAPSHOT_TRANSFER_IDLE_TIMEOUT_MS = 30_000; +export const DEFAULT_SESSION_SNAPSHOT_JSON_BODY_MAX_BYTES = 256 * 1024; export const DEFAULT_SESSION_SNAPSHOT_R2_PREFIX = 'session-snapshots'; export type SessionSnapshotArtifact = 'home' | 'wip' | 'manifest'; @@ -44,6 +45,7 @@ export interface SessionSnapshotConfig { totalBudgetBytes: number; entryThresholdBytes: number; transferIdleTimeoutMs: number; + jsonBodyMaxBytes: number; r2Prefix: string; } @@ -87,6 +89,10 @@ export function getSessionSnapshotConfig(env: Env): SessionSnapshotConfig { env.SESSION_SNAPSHOT_TRANSFER_IDLE_TIMEOUT_MS, DEFAULT_SESSION_SNAPSHOT_TRANSFER_IDLE_TIMEOUT_MS ), + jsonBodyMaxBytes: parsePositiveInt( + env.SESSION_SNAPSHOT_JSON_BODY_MAX_BYTES, + DEFAULT_SESSION_SNAPSHOT_JSON_BODY_MAX_BYTES + ), r2Prefix: sanitizeSnapshotPrefix(env.SESSION_SNAPSHOT_R2_PREFIX || DEFAULT_SESSION_SNAPSHOT_R2_PREFIX), }; } diff --git a/apps/api/tests/unit/cf-container-runtime-contract.test.ts b/apps/api/tests/unit/cf-container-runtime-contract.test.ts index cc7a8fe96..d150e377d 100644 --- a/apps/api/tests/unit/cf-container-runtime-contract.test.ts +++ b/apps/api/tests/unit/cf-container-runtime-contract.test.ts @@ -132,9 +132,11 @@ describe('cf-container runtime spike contracts', () => { expect(containerDo).toContain("await this.markRuntimeSleeping('Container idle timeout expired; container is sleeping.')"); expect(containerDo).toContain("await this.ctx.storage.put('lifecycleStatus', 'sleeping' satisfies LifecycleStatus)"); expect(containerDo).toContain("status: 'sleeping'"); - expect(containerDo).toContain('Container is asleep; wake/rehydrate is not implemented yet.'); - expect(containerDo).toContain('Phase 3 of idea 01KX4KSXEXQMP41KS34TW9EN01'); - expect(containerDo).toContain("return new Response(SLEEPING_RESPONSE, { status: 503 })"); + expect(containerDo).toContain("if (lifecycleStatus === 'sleeping')"); + expect(containerDo).toContain('const wake = await this.wakeFromSnapshot()'); + expect(containerDo).toContain('WAKE_DEGRADED_RESPONSE'); + expect(containerDo).toContain("await this.ctx.storage.put('lifecycleStatus', 'launching' satisfies LifecycleStatus)"); + expect(containerDo).toContain("await this.ctx.storage.put('lifecycleStatus', 'running' satisfies LifecycleStatus)"); expect(chatResolver).toContain("inArray(schema.workspaces.status, ['running', 'recovery', 'sleeping'])"); expect(chatResolver).toContain("workspace.nodeStatus === 'sleeping'"); expect(chatResolver).toContain('The workspace container is asleep.'); diff --git a/apps/api/tests/unit/routes/workspaces-session-snapshots.test.ts b/apps/api/tests/unit/routes/workspaces-session-snapshots.test.ts new file mode 100644 index 000000000..eb4ff8753 --- /dev/null +++ b/apps/api/tests/unit/routes/workspaces-session-snapshots.test.ts @@ -0,0 +1,198 @@ +import { drizzle } from 'drizzle-orm/d1'; +import type { Hono } from 'hono'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { Env } from '../../../src/env'; +import { workspacesRoutes } from '../../../src/routes/workspaces'; +import { createRouteTestApp } from './route-test-app'; + +const mocks = vi.hoisted(() => ({ + completeSessionSnapshot: vi.fn(), + getRestorableSessionSnapshot: vi.fn(), + prepareSessionSnapshot: vi.fn(), + recordSessionSnapshotRestoreResult: vi.fn(), + verifyCallbackToken: vi.fn(), +})); + +vi.mock('drizzle-orm/d1'); +vi.mock('../../../src/middleware/auth', () => ({ + requireAuth: () => vi.fn((c: any, next: any) => next()), + requireApproved: () => vi.fn((c: any, next: any) => next()), + getUserId: () => 'user-1', + getAuth: () => ({ user: { id: 'user-1', name: 'User', email: 'user@example.com' } }), +})); +vi.mock('../../../src/services/jwt', () => ({ + verifyCallbackToken: mocks.verifyCallbackToken, + signCallbackToken: vi.fn(), +})); +vi.mock('../../../src/services/session-snapshots', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + completeSessionSnapshot: mocks.completeSessionSnapshot, + getRestorableSessionSnapshot: mocks.getRestorableSessionSnapshot, + prepareSessionSnapshot: mocks.prepareSessionSnapshot, + recordSessionSnapshotRestoreResult: mocks.recordSessionSnapshotRestoreResult, + }; +}); + +function makeDb(workspace: Record) { + return { + select: vi.fn(() => ({ + from: vi.fn(() => ({ + where: vi.fn(() => ({ + limit: vi.fn(async () => [workspace]), + })), + })), + })), + }; +} + +describe('workspaces session snapshot callback routes', () => { + let app: Hono<{ Bindings: Env }>; + const r2 = { + put: vi.fn(), + get: vi.fn(), + }; + const runtimeBindings = { + DATABASE: {} as any, + R2: r2, + ENCRYPTION_KEY: 'enc-key', + SESSION_SNAPSHOT_R2_PREFIX: 'test-snapshots', + SESSION_SNAPSHOT_TOTAL_BUDGET_BYTES: '1024', + } as unknown as Env; + const workspace = { + id: 'WS_1', + nodeId: 'node-1', + projectId: 'project-1', + userId: 'user-1', + chatSessionId: 'chat-1', + }; + + beforeEach(() => { + vi.clearAllMocks(); + (drizzle as any).mockReturnValue(makeDb(workspace)); + mocks.verifyCallbackToken.mockResolvedValue({ + workspace: 'WS_1', + type: 'callback', + scope: 'workspace', + }); + mocks.prepareSessionSnapshot.mockResolvedValue({ + snapshotId: 'snapshot-1', + expiresAt: '2026-07-18T00:00:00.000Z', + config: { + ttlDays: 7, + totalBudgetBytes: 1024, + entryThresholdBytes: 512, + transferIdleTimeoutMs: 30000, + jsonBodyMaxBytes: 262144, + r2Prefix: 'test-snapshots', + }, + keys: { + home: 'test-snapshots/chat-1/home.tar', + wip: 'test-snapshots/chat-1/wip.bundle', + manifest: 'test-snapshots/chat-1/manifest.json', + }, + }); + + app = createRouteTestApp('/api/workspaces', workspacesRoutes); + }); + + it('prepares deterministic upload URLs for the workspace chat session', async () => { + const res = await app.request( + '/api/workspaces/WS_1/session-snapshot/prepare', + { + method: 'POST', + headers: { + Authorization: 'Bearer callback-token', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + chatSessionId: 'chat-1', + agentSessionId: 'agent-session-1', + runtime: 'cf-container', + }), + }, + runtimeBindings + ); + + expect(res.status).toBe(200); + await expect(res.json()).resolves.toMatchObject({ + snapshotId: 'snapshot-1', + upload: { + home: '/api/workspaces/WS_1/session-snapshot/artifacts/home?chatSessionId=chat-1', + wip: '/api/workspaces/WS_1/session-snapshot/artifacts/wip?chatSessionId=chat-1', + }, + }); + expect(mocks.prepareSessionSnapshot).toHaveBeenCalledWith(expect.anything(), runtimeBindings, { + workspaceId: 'WS_1', + nodeId: 'node-1', + projectId: 'project-1', + userId: 'user-1', + chatSessionId: 'chat-1', + agentSessionId: 'agent-session-1', + runtime: 'cf-container', + }); + }); + + it('uploads artifacts to server-derived R2 keys and rejects oversized content-lengths', async () => { + const ok = await app.request( + '/api/workspaces/WS_1/session-snapshot/artifacts/home?chatSessionId=chat-1', + { + method: 'PUT', + headers: { + Authorization: 'Bearer callback-token', + 'Content-Type': 'application/octet-stream', + 'Content-Length': '4', + }, + body: 'home', + }, + runtimeBindings + ); + + expect(ok.status).toBe(200); + expect(r2.put).toHaveBeenCalledWith( + 'test-snapshots/chat-1/home.tar', + expect.anything(), + expect.objectContaining({ httpMetadata: { contentType: 'application/x-tar' } }) + ); + + const tooLarge = await app.request( + '/api/workspaces/WS_1/session-snapshot/artifacts/wip?chatSessionId=chat-1', + { + method: 'PUT', + headers: { + Authorization: 'Bearer callback-token', + 'Content-Length': '1025', + }, + body: 'wip', + }, + runtimeBindings + ); + expect(tooLarge.status).toBe(400); + }); + + it('rejects node-scoped callback tokens before snapshot service access', async () => { + mocks.verifyCallbackToken.mockResolvedValueOnce({ + workspace: 'WS_1', + type: 'callback', + scope: 'node', + }); + + const res = await app.request( + '/api/workspaces/WS_1/session-snapshot/prepare', + { + method: 'POST', + headers: { + Authorization: 'Bearer callback-token', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ chatSessionId: 'chat-1' }), + }, + runtimeBindings + ); + + expect(res.status).toBe(403); + expect(mocks.prepareSessionSnapshot).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/tests/unit/session-snapshots.test.ts b/apps/api/tests/unit/session-snapshots.test.ts index 4b2ebe433..a2d3e1ef3 100644 --- a/apps/api/tests/unit/session-snapshots.test.ts +++ b/apps/api/tests/unit/session-snapshots.test.ts @@ -4,6 +4,7 @@ import type { Env } from '../../src/env'; import { buildSessionSnapshotR2Key, DEFAULT_SESSION_SNAPSHOT_ENTRY_THRESHOLD_BYTES, + DEFAULT_SESSION_SNAPSHOT_JSON_BODY_MAX_BYTES, DEFAULT_SESSION_SNAPSHOT_R2_PREFIX, DEFAULT_SESSION_SNAPSHOT_TOTAL_BUDGET_BYTES, DEFAULT_SESSION_SNAPSHOT_TRANSFER_IDLE_TIMEOUT_MS, @@ -24,6 +25,7 @@ describe('session snapshot config', () => { totalBudgetBytes: DEFAULT_SESSION_SNAPSHOT_TOTAL_BUDGET_BYTES, entryThresholdBytes: DEFAULT_SESSION_SNAPSHOT_ENTRY_THRESHOLD_BYTES, transferIdleTimeoutMs: DEFAULT_SESSION_SNAPSHOT_TRANSFER_IDLE_TIMEOUT_MS, + jsonBodyMaxBytes: DEFAULT_SESSION_SNAPSHOT_JSON_BODY_MAX_BYTES, r2Prefix: DEFAULT_SESSION_SNAPSHOT_R2_PREFIX, }); }); @@ -34,6 +36,7 @@ describe('session snapshot config', () => { SESSION_SNAPSHOT_TOTAL_BUDGET_BYTES: '1234', SESSION_SNAPSHOT_ENTRY_THRESHOLD_BYTES: '567', SESSION_SNAPSHOT_TRANSFER_IDLE_TIMEOUT_MS: '890', + SESSION_SNAPSHOT_JSON_BODY_MAX_BYTES: '1024', SESSION_SNAPSHOT_R2_PREFIX: '/tenant snapshots/private/', })); @@ -41,6 +44,7 @@ describe('session snapshot config', () => { expect(config.totalBudgetBytes).toBe(1234); expect(config.entryThresholdBytes).toBe(567); expect(config.transferIdleTimeoutMs).toBe(890); + expect(config.jsonBodyMaxBytes).toBe(1024); expect(config.r2Prefix).toBe('tenant-snapshots/private'); }); }); diff --git a/apps/api/tests/workers/session-snapshot-wiring.test.ts b/apps/api/tests/workers/session-snapshot-wiring.test.ts new file mode 100644 index 000000000..48fb1e39f --- /dev/null +++ b/apps/api/tests/workers/session-snapshot-wiring.test.ts @@ -0,0 +1,125 @@ +import { env } from 'cloudflare:test'; +import { drizzle } from 'drizzle-orm/d1'; +import { describe, expect, it } from 'vitest'; + +import * as schema from '../../src/db/schema'; +import type { Env } from '../../src/env'; +import { + completeSessionSnapshot, + getRestorableSessionSnapshot, + prepareSessionSnapshot, + type SessionSnapshotManifest, +} from '../../src/services/session-snapshots'; + +const TEST_PREFIX = `snapshot-${Date.now()}`; + +describe('session snapshot D1/R2 worker wiring', () => { + it('creates one restorable snapshot per chat session and writes the manifest to R2', async () => { + const userId = `${TEST_PREFIX}-user`; + const nodeId = `${TEST_PREFIX}-node`; + const workspaceId = `${TEST_PREFIX}-workspace`; + const chatSessionId = `${TEST_PREFIX}-chat`; + const agentSessionId = `${TEST_PREFIX}-agent-session`; + const bindings = { + ...env, + SESSION_SNAPSHOT_R2_PREFIX: `${TEST_PREFIX}/snapshots`, + SESSION_SNAPSHOT_TTL_DAYS: '7', + } as unknown as Env; + const db = drizzle(env.DATABASE, { schema }); + + await env.DATABASE.prepare( + `INSERT INTO users (id, email, name, created_at, updated_at) + VALUES (?, ?, ?, ?, ?)` + ) + .bind(userId, `${userId}@example.com`, 'Snapshot User', Date.now(), Date.now()) + .run(); + await env.DATABASE.prepare( + `INSERT INTO nodes (id, user_id, name, status, vm_size, vm_location, runtime, created_at, updated_at) + VALUES (?, ?, ?, 'running', 'small', 'local', 'cf-container', datetime('now'), datetime('now'))` + ) + .bind(nodeId, userId, 'snapshot-node') + .run(); + await env.DATABASE.prepare( + `INSERT INTO workspaces (id, node_id, user_id, name, repository, branch, status, vm_size, vm_location, chat_session_id, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, 'main', 'sleeping', 'small', 'local', ?, datetime('now'), datetime('now'))` + ) + .bind(workspaceId, nodeId, userId, 'snapshot-workspace', 'owner/repo', chatSessionId) + .run(); + await env.DATABASE.prepare( + `INSERT INTO agent_sessions (id, workspace_id, user_id, status, created_at, updated_at) + VALUES (?, ?, ?, 'sleeping', datetime('now'), datetime('now'))` + ) + .bind(agentSessionId, workspaceId, userId) + .run(); + + const prepared = await prepareSessionSnapshot(db, bindings, { + workspaceId, + nodeId, + projectId: null, + userId, + chatSessionId, + agentSessionId, + runtime: 'cf-container', + }); + + expect(prepared.keys.home).toBe(`${TEST_PREFIX}/snapshots/${chatSessionId}/home.tar`); + expect(prepared.config.ttlDays).toBe(7); + + const manifest: SessionSnapshotManifest = { + version: 1, + chatSessionId, + workspaceId, + agentSessionId, + baseCommit: 'base-commit', + status: 'available', + degradation: 'none', + skipped: [], + artifacts: { + home: { sizeBytes: 4 }, + wip: { sizeBytes: 3 }, + }, + createdAt: new Date().toISOString(), + }; + + await env.R2.put(prepared.keys.home, 'home'); + await env.R2.put(prepared.keys.wip, 'wip'); + await completeSessionSnapshot(db, bindings, { + workspaceId, + chatSessionId, + agentSessionId, + runtime: 'cf-container', + baseCommit: 'base-commit', + status: 'available', + degradation: 'none', + manifest, + artifactSizes: { + homeBytes: 4, + wipBytes: 3, + }, + }); + + const restorable = await getRestorableSessionSnapshot(db, chatSessionId); + expect(restorable).toMatchObject({ + chatSessionId, + workspaceId, + status: 'available', + degradation: 'none', + homeR2Key: prepared.keys.home, + wipR2Key: prepared.keys.wip, + manifestR2Key: prepared.keys.manifest, + baseCommit: 'base-commit', + }); + + const manifestObject = await env.R2.get(prepared.keys.manifest); + expect(manifestObject).not.toBeNull(); + if (!manifestObject) throw new Error('snapshot manifest was not written to R2'); + await expect(manifestObject.json()).resolves.toMatchObject({ + chatSessionId, + workspaceId, + artifacts: { + home: { sizeBytes: 4 }, + wip: { sizeBytes: 3 }, + }, + }); + }); +}); diff --git a/packages/vm-agent/internal/server/session_snapshot.go b/packages/vm-agent/internal/server/session_snapshot.go index bbb0e15be..abe67de54 100644 --- a/packages/vm-agent/internal/server/session_snapshot.go +++ b/packages/vm-agent/internal/server/session_snapshot.go @@ -43,6 +43,9 @@ type snapshotRestoreResponse struct { Degradation string `json:"degradation,omitempty"` BaseCommit string `json:"baseCommit,omitempty"` Manifest map[string]interface{} `json:"manifest,omitempty"` + Config struct { + TransferIdleTimeoutMs int64 `json:"transferIdleTimeoutMs"` + } `json:"config"` Download struct { Home string `json:"home"` WIP string `json:"wip"` @@ -260,7 +263,7 @@ func (s *Server) restoreSessionSnapshot(ctx context.Context, runtime *WorkspaceR _ = s.reportSnapshotRestoreResult(ctx, runtime.ID, chatSessionID, "missing", restore.Reason, callbackToken) return map[string]interface{}{"status": "transcript-replay", "reason": restore.Reason}, nil } - idleTimeout := defaultSnapshotTransferIdleTimeout + idleTimeout := choosePositiveDurationMs(restore.Config.TransferIdleTimeoutMs, defaultSnapshotTransferIdleTimeout) if restore.Download.Home != "" { if err := s.downloadAndExtractTar(ctx, restore.Download.Home, callbackToken, idleTimeout); err != nil { _ = s.reportSnapshotRestoreResult(ctx, runtime.ID, chatSessionID, "home_failed", err.Error(), callbackToken) @@ -481,6 +484,10 @@ func createHomeTar(homeDirFn func() (string, error), entryThreshold, totalBudget } return nil } + if !info.Mode().IsRegular() && !info.IsDir() { + skipped = append(skipped, snapshotSkippedEntry{Path: "~/" + rel, Reason: "unsupported home entry type"}) + return nil + } if !info.IsDir() && written+info.Size() > totalBudget { skipped = append(skipped, snapshotSkippedEntry{Path: "~/" + rel, Reason: "snapshot budget exhausted", SizeBytes: info.Size()}) return nil @@ -568,6 +575,7 @@ func (s *Server) downloadAndExtractTar(ctx context.Context, downloadPath, token if err != nil { return err } + home = filepath.Clean(home) tr := tar.NewReader(newIdleReader(res.Body, idleTimeout)) for { header, err := tr.Next() @@ -577,8 +585,16 @@ func (s *Server) downloadAndExtractTar(ctx context.Context, downloadPath, token if err != nil { return err } - target := filepath.Join(home, filepath.Clean(header.Name)) - if !strings.HasPrefix(target, home) { + cleanName := filepath.Clean(header.Name) + if filepath.IsAbs(cleanName) || cleanName == "." || cleanName == ".." || strings.HasPrefix(cleanName, ".."+string(filepath.Separator)) { + continue + } + target := filepath.Join(home, cleanName) + relTarget, relErr := filepath.Rel(home, target) + if relErr != nil || relTarget == ".." || strings.HasPrefix(relTarget, ".."+string(filepath.Separator)) { + continue + } + if header.Typeflag != tar.TypeDir && header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeRegA { continue } if header.FileInfo().IsDir() { diff --git a/packages/vm-agent/internal/server/session_snapshot_test.go b/packages/vm-agent/internal/server/session_snapshot_test.go index ac5364ce7..63fcefd5d 100644 --- a/packages/vm-agent/internal/server/session_snapshot_test.go +++ b/packages/vm-agent/internal/server/session_snapshot_test.go @@ -2,12 +2,18 @@ package server import ( "archive/tar" + "bytes" "context" "io" + "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" "testing" + "time" + + "github.com/workspace/vm-agent/internal/config" ) func TestCreateHomeTarExcludesCachesAndRecordsOversizedFiles(t *testing.T) { @@ -79,6 +85,42 @@ func TestCreateWIPBundleDegradesDuringMerge(t *testing.T) { } } +func TestDownloadAndExtractTarRejectsPathTraversal(t *testing.T) { + home := t.TempDir() + outside := filepath.Join(t.TempDir(), "outside.txt") + absolute := filepath.Join(t.TempDir(), "absolute.txt") + t.Setenv("HOME", home) + + var tarBody bytes.Buffer + tw := tar.NewWriter(&tarBody) + writeTarFile(t, tw, "session.jsonl", "inside") + writeTarFile(t, tw, "../outside.txt", "outside") + writeTarFile(t, tw, absolute, "absolute") + if err := tw.Close(); err != nil { + t.Fatal(err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(tarBody.Bytes()) + })) + defer server.Close() + + s := &Server{config: &config.Config{ControlPlaneURL: server.URL}} + if err := s.downloadAndExtractTar(context.Background(), "/artifact", "token", time.Second); err != nil { + t.Fatal(err) + } + + if got, err := os.ReadFile(filepath.Join(home, "session.jsonl")); err != nil || string(got) != "inside" { + t.Fatalf("inside file = %q, %v; want inside", got, err) + } + if _, err := os.Stat(outside); !os.IsNotExist(err) { + t.Fatalf("outside file stat err = %v, want not exist", err) + } + if _, err := os.Stat(absolute); !os.IsNotExist(err) { + t.Fatalf("absolute file stat err = %v, want not exist", err) + } +} + func tarNames(t *testing.T, path string) []string { t.Helper() f, err := os.Open(path) @@ -109,6 +151,20 @@ func runGit(t *testing.T, dir string, args ...string) { } } +func writeTarFile(t *testing.T, tw *tar.Writer, name, body string) { + t.Helper() + if err := tw.WriteHeader(&tar.Header{ + Name: name, + Mode: 0o600, + Size: int64(len(body)), + }); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(body)); err != nil { + t.Fatal(err) + } +} + func containsString(values []string, needle string) bool { for _, value := range values { if value == needle { diff --git a/tasks/active/2026-07-11-runtime-neutral-session-hibernate-wake.md b/tasks/active/2026-07-11-runtime-neutral-session-hibernate-wake.md index 14d931021..985e55e10 100644 --- a/tasks/active/2026-07-11-runtime-neutral-session-hibernate-wake.md +++ b/tasks/active/2026-07-11-runtime-neutral-session-hibernate-wake.md @@ -32,13 +32,35 @@ Hard constraints from Raphaël, 2026-07-11: - [x] Wire cf-container hibernate on control handback before sleep is allowed. - [x] Wire cf-container wake before proxying a sleeping container and before follow-up prompt dispatch; fall back visibly when no snapshot exists or restore fails. - [x] Update follow-up prompt handling so a fresh vm-agent can rebuild a `SessionHost` and attempt native harness resume from restored `$HOME`. -- [ ] Add local tests: Worker service/unit tests, Miniflare/DO integration tests, VM-agent Go tests including `go test -race`, and a vertical-slice test covering Worker → DO → vm-agent → R2 state. -- [ ] Run migration safety checks: `pnpm quality:migration-safety` and `pnpm quality:do-migration-safety`. -- [ ] Run local quality gates: `pnpm lint`, `pnpm typecheck`, `pnpm test`, `pnpm build`, and package-specific tests. -- [ ] Run local specialist review skills and address findings. +- [x] Add local tests: Worker service/unit tests, Miniflare/DO integration tests, VM-agent Go tests including `go test -race`, and a vertical-slice test covering Worker → DO → vm-agent → R2 state. +- [x] Run migration safety checks: `pnpm quality:migration-safety` and `pnpm quality:do-migration-safety`. +- [x] Run local quality gates: `pnpm lint`, `pnpm typecheck`, `pnpm test`, `pnpm build`, and package-specific tests. +- [x] Run local specialist review skills and address findings. - [ ] Open draft PR with `needs-human-review`, explicit no-merge/no-staging language, and local-only verification evidence. - [ ] Append idea `01KX4KSXEXQMP41KS34TW9EN01` with branch, draft PR, and deferred staging/merge status. Leave the idea open. +## Verification Notes + +- `pnpm quality:migration-safety` passed. +- `pnpm quality:do-migration-safety` passed. +- `pnpm typecheck` passed. +- `pnpm lint` passed with existing warnings and no errors. +- `pnpm build` passed. +- `pnpm test` passed. +- `pnpm --filter @simple-agent-manager/api test -- tests/unit/session-snapshots.test.ts tests/unit/routes/workspaces-session-snapshots.test.ts tests/unit/cf-container-runtime-contract.test.ts` passed. +- `pnpm --filter @simple-agent-manager/api typecheck` passed. +- `go test ./internal/server` passed in `packages/vm-agent`. +- `go test -race ./...` passed in `packages/vm-agent`. +- `pnpm --filter @simple-agent-manager/api test:workers -- tests/workers/session-snapshot-wiring.test.ts` was attempted locally, but the Cloudflare `workerd` binary (`@cloudflare/workerd-linux-64@1.20260329.1`) segfaulted during startup before the test body ran. Staging verification remains intentionally deferred by instruction. + +## Local Specialist Review Notes + +- `$cloudflare-specialist`: additive D1 migration and R2-backed snapshot metadata are in place; app enforces snapshot expiry and avoids cron sweeping. R2 lifecycle policy provisioning remains an operational follow-up before merge. +- `$go-specialist`: vm-agent hibernate/restore uses local git bundles, no remote push, progress/idle transfer watchdogs, tar extraction hardening, and `go test -race ./...` passed. +- `$security-auditor`: callback auth is workspace scoped, R2 keys are server derived, fresh secret/env/file injection reruns after restore, and snapshot errors avoid logging secret material. +- `$constitution-validator` and `$env-validator`: snapshot TTL, budgets, thresholds, watchdogs, JSON body limit, and R2 prefix are env-configurable with `DEFAULT_*` constants. +- `$test-engineer`: unit, route, worker wiring, and Go edge tests cover deterministic keys, auth scoping, oversized entries, git operation degradation, and tar traversal rejection. + ## Acceptance Criteria - A cf-container session snapshots harness `$HOME` state and uncommitted WIP on control handback without pushing anything to origin. From 41c9d37e84fa167151701e7785f273aa1a8e90f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 12:30:50 +0000 Subject: [PATCH 04/28] task: archive session hibernate wake --- .../2026-07-11-runtime-neutral-session-hibernate-wake.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tasks/{active => archive}/2026-07-11-runtime-neutral-session-hibernate-wake.md (100%) diff --git a/tasks/active/2026-07-11-runtime-neutral-session-hibernate-wake.md b/tasks/archive/2026-07-11-runtime-neutral-session-hibernate-wake.md similarity index 100% rename from tasks/active/2026-07-11-runtime-neutral-session-hibernate-wake.md rename to tasks/archive/2026-07-11-runtime-neutral-session-hibernate-wake.md From 21f9b3ac6a99734e48b6bf90798b291b1ae5c8a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 15:07:41 +0000 Subject: [PATCH 05/28] fix: harden session snapshot restore and retention --- .../services/node-agent-session-snapshots.ts | 47 ++++++++ apps/api/src/services/node-agent.ts | 42 +------ infra/__tests__/config.test.ts | 5 + infra/__tests__/setup.ts | 11 ++ infra/__tests__/storage.test.ts | 27 +++++ .../__tests__/validate-pulumi-outputs.test.ts | 1 + infra/index.ts | 4 +- infra/resources/config.ts | 17 +++ infra/resources/storage.ts | 30 ++++- .../internal/server/session_snapshot.go | 60 +++++++--- .../server/session_snapshot_git_test.go | 113 ++++++++++++++++++ scripts/deploy/sync-wrangler-config.ts | 5 + scripts/deploy/types.ts | 1 + 13 files changed, 306 insertions(+), 57 deletions(-) create mode 100644 apps/api/src/services/node-agent-session-snapshots.ts create mode 100644 packages/vm-agent/internal/server/session_snapshot_git_test.go diff --git a/apps/api/src/services/node-agent-session-snapshots.ts b/apps/api/src/services/node-agent-session-snapshots.ts new file mode 100644 index 000000000..c94944a7e --- /dev/null +++ b/apps/api/src/services/node-agent-session-snapshots.ts @@ -0,0 +1,47 @@ +import type { Env } from '../env'; +import { getNodeAgentRequestTimeoutMs, nodeAgentRequest } from './node-agent'; + +interface SessionSnapshotRequest { + chatSessionId: string; + runtime: string; +} + +function requestSessionSnapshot( + action: 'hibernate' | 'restore', + nodeId: string, + workspaceId: string, + sessionId: string, + env: Env, + userId: string, + input: SessionSnapshotRequest +): Promise { + return nodeAgentRequest(nodeId, env, `/workspaces/${workspaceId}/agent-sessions/${sessionId}/${action}`, { + method: 'POST', + userId, + workspaceId, + requestTimeoutMs: getNodeAgentRequestTimeoutMs(env), + body: JSON.stringify(input), + }); +} + +export function hibernateAgentSessionOnNode( + nodeId: string, + workspaceId: string, + sessionId: string, + env: Env, + userId: string, + input: SessionSnapshotRequest +): Promise { + return requestSessionSnapshot('hibernate', nodeId, workspaceId, sessionId, env, userId, input); +} + +export function restoreAgentSessionOnNode( + nodeId: string, + workspaceId: string, + sessionId: string, + env: Env, + userId: string, + input: SessionSnapshotRequest +): Promise { + return requestSessionSnapshot('restore', nodeId, workspaceId, sessionId, env, userId, input); +} diff --git a/apps/api/src/services/node-agent.ts b/apps/api/src/services/node-agent.ts index fa0da9015..c80009372 100644 --- a/apps/api/src/services/node-agent.ts +++ b/apps/api/src/services/node-agent.ts @@ -125,7 +125,7 @@ export async function waitForNodeAgentReady(nodeId: string, env: Env): Promise { - return nodeAgentRequest(nodeId, env, `/workspaces/${workspaceId}/agent-sessions/${sessionId}/hibernate`, { - method: 'POST', - userId, - workspaceId, - requestTimeoutMs: getNodeAgentRequestTimeoutMs(env), - body: JSON.stringify(input), - }); -} - -export async function restoreAgentSessionOnNode( - nodeId: string, - workspaceId: string, - sessionId: string, - env: Env, - userId: string, - input: { - chatSessionId: string; - runtime: string; - } -): Promise { - return nodeAgentRequest(nodeId, env, `/workspaces/${workspaceId}/agent-sessions/${sessionId}/restore`, { - method: 'POST', - userId, - workspaceId, - requestTimeoutMs: getNodeAgentRequestTimeoutMs(env), - body: JSON.stringify(input), - }); -} +export { hibernateAgentSessionOnNode, restoreAgentSessionOnNode } from './node-agent-session-snapshots'; /** * Cancel a running prompt on an agent session. diff --git a/infra/__tests__/config.test.ts b/infra/__tests__/config.test.ts index 33a123a16..70f01fce7 100644 --- a/infra/__tests__/config.test.ts +++ b/infra/__tests__/config.test.ts @@ -46,6 +46,7 @@ describe('infra config parsing', () => { stack: 'staging', r2Location: configModule.DEFAULT_R2_LOCATION, pagesProductionBranch: configModule.DEFAULT_PAGES_PRODUCTION_BRANCH, + sessionSnapshotTtlDays: configModule.DEFAULT_SESSION_SNAPSHOT_TTL_DAYS, }); expect(parsed.prefix).toBe(configModule.derivePrefix('example.com')); }); @@ -56,6 +57,7 @@ describe('infra config parsing', () => { resourcePrefix: 'fork', r2Location: 'WEUR', pagesProductionBranch: 'release/2026-06', + sessionSnapshotTtlDays: '14', }), 'prod' ); @@ -64,6 +66,7 @@ describe('infra config parsing', () => { prefix: 'fork', r2Location: 'WEUR', pagesProductionBranch: 'release/2026-06', + sessionSnapshotTtlDays: 14, }); }); @@ -76,6 +79,8 @@ describe('infra config parsing', () => { ['baseDomain', ' ', 'must not be empty'], ['resourcePrefix', ' ', 'must not be empty'], ['pagesProductionBranch', ' ', 'must not be empty'], + ['sessionSnapshotTtlDays', '0', 'must be a positive integer'], + ['sessionSnapshotTtlDays', '1.5', 'must be a positive integer'], ])('fails fast for invalid %s config', (key, value, expectedMessage) => { const beforeResourceCount = getRegisteredResources().length; diff --git a/infra/__tests__/setup.ts b/infra/__tests__/setup.ts index 7b095d739..2f7c3701f 100644 --- a/infra/__tests__/setup.ts +++ b/infra/__tests__/setup.ts @@ -69,6 +69,17 @@ vi.mock('@pulumi/cloudflare', async (importOriginal) => { super(name, args, opts); } }, + R2BucketLifecycle: class extends actual.R2BucketLifecycle { + constructor(name: string, args: ResourceInputs, opts?: pulumi.CustomResourceOptions) { + resourceRecorder.record( + 'cloudflare:index/r2BucketLifecycle:R2BucketLifecycle', + name, + args, + opts + ); + super(name, args, opts); + } + }, Record: class extends actual.Record { constructor(name: string, args: ResourceInputs, opts?: pulumi.CustomResourceOptions) { resourceRecorder.record('cloudflare:index/record:Record', name, args, opts); diff --git a/infra/__tests__/storage.test.ts b/infra/__tests__/storage.test.ts index 98a17b16c..4b695501b 100644 --- a/infra/__tests__/storage.test.ts +++ b/infra/__tests__/storage.test.ts @@ -27,4 +27,31 @@ describe('R2 Bucket Resource', () => { location: configModule.DEFAULT_R2_LOCATION, }); }); + + it('provisions prefix-scoped session snapshot expiration with the configured TTL', async () => { + const lifecycle = findRegisteredResource( + `${configModule.prefix}-r2-lifecycle`, + 'cloudflare:index/r2BucketLifecycle:R2BucketLifecycle' + ); + + expect(await getOutputValue(lifecycle.inputs.bucketName as never)).toBe( + configModule.prefix + '-' + configModule.stack + '-assets' + ); + expect(lifecycle.inputs).toMatchObject({ + accountId: 'test-account-id-00000000000000000000', + rules: [ + { + id: storageModule.SESSION_SNAPSHOT_LIFECYCLE_RULE_ID, + conditions: { prefix: storageModule.SESSION_SNAPSHOT_R2_PREFIX }, + enabled: true, + deleteObjectsTransition: { + condition: { + maxAge: configModule.DEFAULT_SESSION_SNAPSHOT_TTL_DAYS * 24 * 60 * 60, + type: 'Age', + }, + }, + }, + ], + }); + }); }); diff --git a/infra/__tests__/validate-pulumi-outputs.test.ts b/infra/__tests__/validate-pulumi-outputs.test.ts index 62aad3d14..ee9f469bf 100644 --- a/infra/__tests__/validate-pulumi-outputs.test.ts +++ b/infra/__tests__/validate-pulumi-outputs.test.ts @@ -11,6 +11,7 @@ function makeValidOutputs(): PulumiOutputs { kvId: 'kv-789', kvName: 'sa379a6-prod-sessions', r2Name: 'sa379a6-prod-assets', + sessionSnapshotTtlDays: 7, cloudflareAccountId: 'cf-account-abc', pagesName: 'sa379a6-web-prod', dnsIds: { api: 'dns-1', app: 'dns-2', wildcard: 'dns-3' }, diff --git a/infra/index.ts b/infra/index.ts index 64978e3cc..e61518bab 100644 --- a/infra/index.ts +++ b/infra/index.ts @@ -10,7 +10,7 @@ import { observabilityDatabaseName, } from './resources/database'; import { kvNamespace, kvNamespaceId, kvNamespaceName } from './resources/kv'; -import { r2Bucket, r2BucketName } from './resources/storage'; +import { r2Bucket, r2BucketLifecycle, r2BucketName } from './resources/storage'; import { apiDnsRecord, appDnsRecord, @@ -29,6 +29,7 @@ export { observabilityDatabase, kvNamespace, r2Bucket, + r2BucketLifecycle, pagesProject, pagesCustomDomain, apiDnsRecord, @@ -45,6 +46,7 @@ export const observabilityD1DatabaseName = observabilityDatabaseName; export const kvId = kvNamespaceId; export const kvName = kvNamespaceName; export const r2Name = r2BucketName; +export { sessionSnapshotTtlDays } from './resources/config'; export const pagesName = pagesProjectName; export const dnsIds = dnsRecordIds; export const hostnames = dnsHostnames; diff --git a/infra/resources/config.ts b/infra/resources/config.ts index e552c0a4c..917a0f01e 100644 --- a/infra/resources/config.ts +++ b/infra/resources/config.ts @@ -6,6 +6,7 @@ export const SUPPORTED_R2_LOCATIONS = ['WNAM', 'ENAM', 'WEUR', 'EEUR', 'APAC', ' export type R2Location = (typeof SUPPORTED_R2_LOCATIONS)[number]; export const DEFAULT_PAGES_PRODUCTION_BRANCH = 'main'; +export const DEFAULT_SESSION_SNAPSHOT_TTL_DAYS = 7; export interface ConfigReader { get(key: string): string | undefined; @@ -20,6 +21,7 @@ export interface InfraConfig { prefix: string; r2Location: R2Location; pagesProductionBranch: string; + sessionSnapshotTtlDays: number; } const pulumiConfig = new pulumi.Config(); @@ -37,6 +39,11 @@ export function parseInfraConfig(config: ConfigReader, currentStack: string): In r2Location: parseR2Location(optionalNonEmptyConfig(config, 'r2Location')), pagesProductionBranch: optionalNonEmptyConfig(config, 'pagesProductionBranch') ?? DEFAULT_PAGES_PRODUCTION_BRANCH, + sessionSnapshotTtlDays: parsePositiveInteger( + optionalNonEmptyConfig(config, 'sessionSnapshotTtlDays'), + DEFAULT_SESSION_SNAPSHOT_TTL_DAYS, + 'sessionSnapshotTtlDays' + ), }; } @@ -58,6 +65,7 @@ export const stack = infraConfig.stack; export const prefix = infraConfig.prefix; export const r2Location = infraConfig.r2Location; export const pagesProductionBranch = infraConfig.pagesProductionBranch; +export const sessionSnapshotTtlDays = infraConfig.sessionSnapshotTtlDays; export function derivePrefix(domain: string): string { const hash = crypto.createHash('sha256').update(domain).digest('hex'); @@ -87,6 +95,15 @@ function optionalNonEmptyConfig(config: ConfigReader, key: string): string | und return value; } +function parsePositiveInteger(value: string | undefined, fallback: number, key: string): number { + if (value === undefined) return fallback; + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed <= 0) { + throw new Error('Pulumi config "' + key + '" must be a positive integer'); + } + return parsed; +} + function parseR2Location(value: string | undefined): R2Location { if (value === undefined) { return DEFAULT_R2_LOCATION; diff --git a/infra/resources/storage.ts b/infra/resources/storage.ts index 3d141dd5f..d49dc93bf 100644 --- a/infra/resources/storage.ts +++ b/infra/resources/storage.ts @@ -1,10 +1,36 @@ import * as cloudflare from '@pulumi/cloudflare'; -import { accountId, prefix, r2Location, stack } from './config'; +import { accountId, prefix, r2Location, sessionSnapshotTtlDays, stack } from './config'; + +export const SESSION_SNAPSHOT_LIFECYCLE_RULE_ID = 'expire-session-snapshots'; +export const SESSION_SNAPSHOT_R2_PREFIX = 'session-snapshots/'; +const SECONDS_PER_DAY = 24 * 60 * 60; export const r2Bucket = new cloudflare.R2Bucket(`${prefix}-r2`, { - accountId: accountId, + accountId, name: `${prefix}-${stack}-assets`, location: r2Location, }); +export const r2BucketLifecycle = new cloudflare.R2BucketLifecycle( + `${prefix}-r2-lifecycle`, + { + accountId, + bucketName: r2Bucket.name, + rules: [ + { + id: SESSION_SNAPSHOT_LIFECYCLE_RULE_ID, + conditions: { prefix: SESSION_SNAPSHOT_R2_PREFIX }, + enabled: true, + deleteObjectsTransition: { + condition: { + maxAge: sessionSnapshotTtlDays * SECONDS_PER_DAY, + type: 'Age', + }, + }, + }, + ], + }, + { dependsOn: r2Bucket } +); + export const r2BucketName = r2Bucket.name; diff --git a/packages/vm-agent/internal/server/session_snapshot.go b/packages/vm-agent/internal/server/session_snapshot.go index abe67de54..d7284af49 100644 --- a/packages/vm-agent/internal/server/session_snapshot.go +++ b/packages/vm-agent/internal/server/session_snapshot.go @@ -46,7 +46,7 @@ type snapshotRestoreResponse struct { Config struct { TransferIdleTimeoutMs int64 `json:"transferIdleTimeoutMs"` } `json:"config"` - Download struct { + Download struct { Home string `json:"home"` WIP string `json:"wip"` Manifest string `json:"manifest"` @@ -365,7 +365,6 @@ func (s *Server) doSnapshotJSON(ctx context.Context, method, workspaceID, path, } return nil } - func createWIPBundle(ctx context.Context, workDir string, entryThreshold int64) (string, string, []snapshotSkippedEntry, error) { if ok, err := standaloneRepositoryPresent(workDir); err != nil || !ok { if err != nil { @@ -387,15 +386,36 @@ func createWIPBundle(ctx context.Context, workDir string, entryThreshold int64) if strings.TrimSpace(status) == "" { return base, "", nil, nil } + + indexFile, err := os.CreateTemp("", "sam-session-index-*") + if err != nil { + return base, "", nil, err + } + indexPath := indexFile.Name() + _ = indexFile.Close() + _ = os.Remove(indexPath) + defer os.Remove(indexPath) + gitEnv := []string{"GIT_INDEX_FILE=" + indexPath} + if _, err := runStandaloneGitCommand(ctx, workDir, gitEnv, "read-tree", "HEAD"); err != nil { + return base, "", nil, fmt.Errorf("initialize snapshot index: %w", err) + } + if _, err := runStandaloneGitCommand(ctx, workDir, gitEnv, "add", "-A"); err != nil { + return base, "", nil, fmt.Errorf("stage snapshot index: %w", err) + } skipped := skipOversizedUntracked(workDir, entryThreshold) - _, _ = runStandaloneGitCommand(ctx, workDir, nil, "add", "-A") for _, entry := range skipped { if entry.Path != "" { - _, _ = runStandaloneGitCommand(ctx, workDir, nil, "reset", "--", entry.Path) + _, _ = runStandaloneGitCommand(ctx, workDir, gitEnv, "reset", "--", entry.Path) } } - if _, err := runStandaloneGitCommand(ctx, workDir, nil, "commit", "--no-verify", "-m", "SAM session snapshot"); err != nil { - return base, "", skipped, fmt.Errorf("create temporary snapshot commit: %w", err) + tree, err := runStandaloneGitCommand(ctx, workDir, gitEnv, "write-tree") + if err != nil { + return base, "", skipped, fmt.Errorf("write snapshot tree: %w", err) + } + commitEnv := append(gitEnv, "GIT_AUTHOR_NAME=SAM Snapshot", "GIT_AUTHOR_EMAIL=snapshot@localhost", "GIT_COMMITTER_NAME=SAM Snapshot", "GIT_COMMITTER_EMAIL=snapshot@localhost") + commit, err := runStandaloneGitCommand(ctx, workDir, commitEnv, "commit-tree", tree, "-p", base, "-m", "SAM session snapshot") + if err != nil { + return base, "", skipped, fmt.Errorf("create snapshot commit: %w", err) } bundle, err := os.CreateTemp("", "sam-session-wip-*.bundle") if err != nil { @@ -403,15 +423,20 @@ func createWIPBundle(ctx context.Context, workDir string, entryThreshold int64) } bundlePath := bundle.Name() _ = bundle.Close() - if _, err := runStandaloneGitCommand(ctx, workDir, nil, "bundle", "create", bundlePath, base+"..HEAD"); err != nil { - _, _ = runStandaloneGitCommand(ctx, workDir, nil, "reset", "--mixed", base) + snapshotRef := "refs/sam/session-snapshot/" + strings.TrimSuffix(filepath.Base(bundlePath), ".bundle") + if _, err := runStandaloneGitCommand(ctx, workDir, nil, "update-ref", snapshotRef, commit); err != nil { + _ = os.Remove(bundlePath) + return base, "", skipped, fmt.Errorf("create snapshot ref: %w", err) + } + defer func() { + _, _ = runStandaloneGitCommand(context.Background(), workDir, nil, "update-ref", "-d", snapshotRef) + }() + if _, err := runStandaloneGitCommand(ctx, workDir, nil, "bundle", "create", bundlePath, snapshotRef); err != nil { _ = os.Remove(bundlePath) return base, "", skipped, fmt.Errorf("create git bundle: %w", err) } - _, _ = runStandaloneGitCommand(ctx, workDir, nil, "reset", "--mixed", base) return base, bundlePath, skipped, nil } - func gitOperationInProgress(workDir string) bool { gitDir := filepath.Join(workDir, ".git") for _, marker := range []string{"MERGE_HEAD", "CHERRY_PICK_HEAD", "REVERT_HEAD", "rebase-merge", "rebase-apply"} { @@ -643,18 +668,25 @@ func (s *Server) downloadAndRestoreWIP(ctx context.Context, downloadPath, token return closeErr } defer os.Remove(tmpPath) - if _, err := runStandaloneGitCommand(ctx, workDir, nil, "fetch", tmpPath, "HEAD:sam-session-restore"); err != nil { + heads, err := runStandaloneGitCommand(ctx, workDir, nil, "bundle", "list-heads", tmpPath) + if err != nil { + return err + } + fields := strings.Fields(heads) + if len(fields) < 2 { + return fmt.Errorf("snapshot bundle has no restorable ref") + } + if _, err := runStandaloneGitCommand(ctx, workDir, nil, "fetch", tmpPath, fields[1]); err != nil { return err } - if _, err := runStandaloneGitCommand(ctx, workDir, nil, "checkout", "sam-session-restore"); err != nil { + if _, err := runStandaloneGitCommand(ctx, workDir, nil, "read-tree", "--reset", "-u", "FETCH_HEAD"); err != nil { return err } if strings.TrimSpace(baseCommit) != "" { - if _, err := runStandaloneGitCommand(ctx, workDir, nil, "reset", "--soft", baseCommit); err != nil { + if _, err := runStandaloneGitCommand(ctx, workDir, nil, "reset", "--mixed", baseCommit); err != nil { return err } } - _, _ = runStandaloneGitCommand(ctx, workDir, nil, "branch", "-D", "sam-session-restore") return nil } diff --git a/packages/vm-agent/internal/server/session_snapshot_git_test.go b/packages/vm-agent/internal/server/session_snapshot_git_test.go new file mode 100644 index 000000000..5ec9d69b8 --- /dev/null +++ b/packages/vm-agent/internal/server/session_snapshot_git_test.go @@ -0,0 +1,113 @@ +package server + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/workspace/vm-agent/internal/config" +) + +func TestCreateWIPBundlePreservesBranchAndIndex(t *testing.T) { + repo := initSnapshotTestRepo(t) + if err := os.WriteFile(filepath.Join(repo, "README.md"), []byte("staged"), 0o600); err != nil { + t.Fatal(err) + } + runGit(t, repo, "add", "README.md") + if err := os.WriteFile(filepath.Join(repo, "README.md"), []byte("staged and unstaged"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(repo, "untracked.txt"), []byte("untracked"), 0o600); err != nil { + t.Fatal(err) + } + beforeStatus := gitOutput(t, repo, "status", "--porcelain=v1") + beforeHead := gitOutput(t, repo, "rev-parse", "HEAD") + beforeBranch := gitOutput(t, repo, "branch", "--show-current") + + _, bundlePath, _, err := createWIPBundle(context.Background(), repo, 1024) + if err != nil { + t.Fatal(err) + } + defer os.Remove(bundlePath) + + if got := gitOutput(t, repo, "status", "--porcelain=v1"); got != beforeStatus { + t.Fatalf("status changed by snapshot:\nwant %q\n got %q", beforeStatus, got) + } + if got := gitOutput(t, repo, "rev-parse", "HEAD"); got != beforeHead { + t.Fatalf("HEAD changed by snapshot: want %s, got %s", beforeHead, got) + } + if got := gitOutput(t, repo, "branch", "--show-current"); got != beforeBranch { + t.Fatalf("branch changed by snapshot: want %s, got %s", beforeBranch, got) + } +} + +func TestDownloadAndRestoreWIPKeepsOriginalBranch(t *testing.T) { + repo := initSnapshotTestRepo(t) + base := gitOutput(t, repo, "rev-parse", "HEAD") + branch := gitOutput(t, repo, "branch", "--show-current") + if err := os.WriteFile(filepath.Join(repo, "README.md"), []byte("restored change"), 0o600); err != nil { + t.Fatal(err) + } + _, bundlePath, _, err := createWIPBundle(context.Background(), repo, 1024) + if err != nil { + t.Fatal(err) + } + defer os.Remove(bundlePath) + runGit(t, repo, "reset", "--hard", base) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + f, openErr := os.Open(bundlePath) + if openErr != nil { + http.Error(w, openErr.Error(), http.StatusInternalServerError) + return + } + defer f.Close() + _, _ = io.Copy(w, f) + })) + defer server.Close() + s := &Server{config: &config.Config{ControlPlaneURL: server.URL}} + if err := s.downloadAndRestoreWIP(context.Background(), server.URL, "token", time.Second, repo, base); err != nil { + t.Fatal(err) + } + if got := gitOutput(t, repo, "branch", "--show-current"); got != branch { + t.Fatalf("branch changed by restore: want %s, got %s", branch, got) + } + if got := strings.TrimSpace(gitOutput(t, repo, "diff", "--", "README.md")); got == "" { + t.Fatal("restored WIP is missing") + } +} + +func initSnapshotTestRepo(t *testing.T) string { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + repo := t.TempDir() + runGit(t, repo, "init") + runGit(t, repo, "config", "user.email", "sam@example.test") + runGit(t, repo, "config", "user.name", "SAM") + if err := os.WriteFile(filepath.Join(repo, "README.md"), []byte("base"), 0o600); err != nil { + t.Fatal(err) + } + runGit(t, repo, "add", "README.md") + runGit(t, repo, "commit", "-m", "base") + return repo +} + +func gitOutput(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v failed: %v\n%s", args, err, out) + } + return strings.TrimSpace(string(out)) +} diff --git a/scripts/deploy/sync-wrangler-config.ts b/scripts/deploy/sync-wrangler-config.ts index 7cb111e7f..088c9f5f2 100644 --- a/scripts/deploy/sync-wrangler-config.ts +++ b/scripts/deploy/sync-wrangler-config.ts @@ -116,12 +116,16 @@ export function validatePulumiOutputs(outputs: unknown): asserts outputs is Pulu { key: 'observabilityD1DatabaseName', label: 'Observability D1 Database Name' }, { key: 'kvId', label: 'KV Namespace ID' }, { key: 'r2Name', label: 'R2 Bucket Name' }, + { key: 'sessionSnapshotTtlDays', label: 'Session Snapshot TTL Days' }, { key: 'cloudflareAccountId', label: 'Cloudflare Account ID' }, { key: 'pagesName', label: 'Pages Project Name' }, ]; const missing = required.filter(({ key }) => { const value = record[key]; + if (key === 'sessionSnapshotTtlDays') { + return typeof value !== 'number' || !Number.isSafeInteger(value) || value <= 0; + } return typeof value !== 'string' || value.length === 0; }); @@ -333,6 +337,7 @@ function getApiWorkerVars( VERSION: DEPLOYMENT_CONFIG.version, PAGES_PROJECT_NAME: outputs.pagesName, R2_BUCKET_NAME: outputs.r2Name, + SESSION_SNAPSHOT_TTL_DAYS: String(outputs.sessionSnapshotTtlDays), ...getOptionalProcessEnvVars([ 'REQUIRE_APPROVAL', 'HETZNER_BASE_IMAGE', diff --git a/scripts/deploy/types.ts b/scripts/deploy/types.ts index e13d912a1..c713c9949 100644 --- a/scripts/deploy/types.ts +++ b/scripts/deploy/types.ts @@ -348,6 +348,7 @@ export interface PulumiOutputs { kvId: string; kvName: string; r2Name: string; + sessionSnapshotTtlDays: number; dnsIds: { api: string; app: string; From 22d3dd79ebf957847539ec638dbb5920a91a10c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 15:10:26 +0000 Subject: [PATCH 06/28] docs: align session snapshot landing evidence --- .claude/skills/env-reference/SKILL.md | 8 +++++- apps/api/.env.example | 6 +++++ ...-runtime-neutral-session-hibernate-wake.md | 25 +++++++++++++------ 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/.claude/skills/env-reference/SKILL.md b/.claude/skills/env-reference/SKILL.md index df3197bfa..06c75a369 100644 --- a/.claude/skills/env-reference/SKILL.md +++ b/.claude/skills/env-reference/SKILL.md @@ -66,7 +66,13 @@ See `apps/api/.env.example` for the full list. Key variables: - `CF_CONTAINER_SLEEP_AFTER` — Container idle sleep duration for instant-session runtime (default: `10m`) - `CF_CONTAINER_VM_AGENT_PORT` — vm-agent standalone HTTP port inside the raw container (default: `8080`) - `CF_CONTAINER_PORT_READY_TIMEOUT_MS` — Max wait for vm-agent port readiness (default: `30000`) -- `CF_CONTAINER_WORKSPACE_BASE_DIR` — Base checkout directory inside raw containers (default: `/workspaces`) +- `$1 +- `SESSION_SNAPSHOT_TTL_DAYS` — Retention for hibernated session snapshots; deployment also provisions matching R2 prefix expiration (default: `7`) +- `SESSION_SNAPSHOT_TOTAL_BUDGET_BYTES` — Maximum bytes accepted for one snapshot artifact (default: `104857600`) +- `SESSION_SNAPSHOT_ENTRY_THRESHOLD_BYTES` — Per-file threshold before snapshot content is visibly skipped (default: `52428800`) +- `SESSION_SNAPSHOT_TRANSFER_IDLE_TIMEOUT_MS` — Progress-idle timeout for snapshot upload/download (default: `30000`) +- `SESSION_SNAPSHOT_JSON_BODY_MAX_BYTES` — Maximum snapshot coordination JSON body (default: `262144`) +- `SESSION_SNAPSHOT_R2_PREFIX` — Private R2 object prefix for session snapshots (default: `session-snapshots`) ### Devcontainer Cache diff --git a/apps/api/.env.example b/apps/api/.env.example index 9a56fb5a1..7fb68a852 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -52,6 +52,12 @@ BASE_DOMAIN=workspaces.example.com # CF_CONTAINER_VM_AGENT_PORT=8080 # CF_CONTAINER_PORT_READY_TIMEOUT_MS=30000 # CF_CONTAINER_WORKSPACE_BASE_DIR=/workspaces +# SESSION_SNAPSHOT_TTL_DAYS=7 +# SESSION_SNAPSHOT_TOTAL_BUDGET_BYTES=104857600 +# SESSION_SNAPSHOT_ENTRY_THRESHOLD_BYTES=52428800 +# SESSION_SNAPSHOT_TRANSFER_IDLE_TIMEOUT_MS=30000 +# SESSION_SNAPSHOT_JSON_BODY_MAX_BYTES=262144 +# SESSION_SNAPSHOT_R2_PREFIX=session-snapshots # User approval / invite-only mode # When "true", new users require admin approval before accessing the platform. diff --git a/tasks/archive/2026-07-11-runtime-neutral-session-hibernate-wake.md b/tasks/archive/2026-07-11-runtime-neutral-session-hibernate-wake.md index 985e55e10..b435d94c8 100644 --- a/tasks/archive/2026-07-11-runtime-neutral-session-hibernate-wake.md +++ b/tasks/archive/2026-07-11-runtime-neutral-session-hibernate-wake.md @@ -4,11 +4,12 @@ Cloudflare Container instant workspaces lose their local disk after sleep. That currently means harness-native session state under `$HOME` and uncommitted repository work disappear when a sleeping cf-container workspace wakes. Phase 3a of idea `01KX4KSXEXQMP41KS34TW9EN01` must add a runtime-neutral hibernate/restore contract and wire the first consumer to the cf-container runtime. -Hard constraints from Raphaël, 2026-07-11: -- Open a draft PR only. -- Do not merge. -- Do not deploy to staging or mutate staging. -- Verification is local only. +Updated authorization from Raphaël, 2026-07-11: + +- The earlier draft-only and no-staging hold is superseded. +- Complete the full `/do` workflow on existing PR #1562. +- Staging end-to-end verification is required before merge. +- Merge only after every required gate passes, then monitor production deployment. ## Research Findings @@ -36,8 +37,9 @@ Hard constraints from Raphaël, 2026-07-11: - [x] Run migration safety checks: `pnpm quality:migration-safety` and `pnpm quality:do-migration-safety`. - [x] Run local quality gates: `pnpm lint`, `pnpm typecheck`, `pnpm test`, `pnpm build`, and package-specific tests. - [x] Run local specialist review skills and address findings. -- [ ] Open draft PR with `needs-human-review`, explicit no-merge/no-staging language, and local-only verification evidence. -- [ ] Append idea `01KX4KSXEXQMP41KS34TW9EN01` with branch, draft PR, and deferred staging/merge status. Leave the idea open. +- [x] Opened draft PR with the earlier local-only evidence. +- [ ] Replace draft-only evidence with completed specialist, staging, and preflight evidence; remove `needs-human-review` only after all reviews pass. +- [ ] Update idea `01KX4KSXEXQMP41KS34TW9EN01` with final PR, staging, merge, and deployment evidence. ## Verification Notes @@ -70,4 +72,11 @@ Hard constraints from Raphaël, 2026-07-11: - Ungraceful kills or expired/missing snapshots degrade visibly to existing transcript/fork behavior. - Timeouts for transfer work are progress/idle watchdogs, not fixed wall-clock caps. - Tests cover edge cases for public-repo privacy, huge files, repo operation states, missing/corrupt artifacts, retention expiry, and no secret leakage in logs/events. -- The PR is draft-only, not merged, and staging verification is explicitly deferred per instruction. +- PR #1562 passes local, specialist, CI, staging E2E, merge, and production-deploy gates under the superseding authorization. + +## Continuation Findings (PR #1562 landing pass) + +- The original Git WIP bundle path mutated the user's real index and branch; remediation now builds the snapshot commit from a temporary Git index and tests exact status/HEAD/branch preservation. +- Restore checked out a temporary branch and attempted to delete it while current; remediation now imports the advertised bundle ref, materializes its tree, and leaves WIP uncommitted on the original branch. +- R2 cleanup was not provisioned. Pulumi now owns a prefix-scoped lifecycle rule, and its positive TTL configuration is injected into the Worker so application expiry and object deletion stay aligned. +- The initial PR exceeded the API file-size gate and omitted preflight/specialist evidence markers; the node snapshot wrappers are extracted and durable PR evidence must be added after final reviewers complete. From b6a3988231fa960c8c6fb73ac7995550f51d4a8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 15:20:29 +0000 Subject: [PATCH 07/28] refactor: share snapshot request validation --- .../internal/server/session_snapshot.go | 72 +++++++++---------- 1 file changed, 34 insertions(+), 38 deletions(-) diff --git a/packages/vm-agent/internal/server/session_snapshot.go b/packages/vm-agent/internal/server/session_snapshot.go index d7284af49..b49575c52 100644 --- a/packages/vm-agent/internal/server/session_snapshot.go +++ b/packages/vm-agent/internal/server/session_snapshot.go @@ -97,15 +97,24 @@ func (r *countingReader) Read(p []byte) (int, error) { return n, err } -func (s *Server) handleHibernateAgentSession(w http.ResponseWriter, r *http.Request) { +type sessionSnapshotHandlerInput struct { + workspaceID string + sessionID string + chatSessionID string + runtimeName string + runtime *WorkspaceRuntime + callbackToken string +} + +func (s *Server) sessionSnapshotHandlerInput(w http.ResponseWriter, r *http.Request) (*sessionSnapshotHandlerInput, bool) { workspaceID := r.PathValue("workspaceId") sessionID := r.PathValue("sessionId") if workspaceID == "" || sessionID == "" { writeError(w, http.StatusBadRequest, "workspaceId and sessionId are required") - return + return nil, false } if !s.requireNodeManagementAuth(w, r, workspaceID) { - return + return nil, false } var body struct { ChatSessionID string `json:"chatSessionId"` @@ -113,24 +122,39 @@ func (s *Server) handleHibernateAgentSession(w http.ResponseWriter, r *http.Requ } if err := json.NewDecoder(r.Body).Decode(&body); err != nil { writeError(w, http.StatusBadRequest, "invalid request body") - return + return nil, false } body.ChatSessionID = strings.TrimSpace(body.ChatSessionID) if body.ChatSessionID == "" { writeError(w, http.StatusBadRequest, "chatSessionId is required") - return + return nil, false } runtime, ok := s.getWorkspaceRuntime(workspaceID) if !ok { writeError(w, http.StatusNotFound, "workspace not found") - return + return nil, false } callbackToken := s.callbackTokenForWorkspace(workspaceID) if callbackToken == "" { writeError(w, http.StatusConflict, "workspace callback token unavailable") + return nil, false + } + return &sessionSnapshotHandlerInput{ + workspaceID: workspaceID, + sessionID: sessionID, + chatSessionID: body.ChatSessionID, + runtimeName: body.Runtime, + runtime: runtime, + callbackToken: callbackToken, + }, true +} + +func (s *Server) handleHibernateAgentSession(w http.ResponseWriter, r *http.Request) { + input, ok := s.sessionSnapshotHandlerInput(w, r) + if !ok { return } - result, err := s.hibernateSessionSnapshot(r.Context(), runtime, sessionID, body.ChatSessionID, body.Runtime, callbackToken) + result, err := s.hibernateSessionSnapshot(r.Context(), input.runtime, input.sessionID, input.chatSessionID, input.runtimeName, input.callbackToken) if err != nil { writeError(w, http.StatusInternalServerError, err.Error()) return @@ -139,41 +163,13 @@ func (s *Server) handleHibernateAgentSession(w http.ResponseWriter, r *http.Requ } func (s *Server) handleRestoreAgentSession(w http.ResponseWriter, r *http.Request) { - workspaceID := r.PathValue("workspaceId") - sessionID := r.PathValue("sessionId") - if workspaceID == "" || sessionID == "" { - writeError(w, http.StatusBadRequest, "workspaceId and sessionId are required") - return - } - if !s.requireNodeManagementAuth(w, r, workspaceID) { - return - } - var body struct { - ChatSessionID string `json:"chatSessionId"` - Runtime string `json:"runtime"` - } - if err := json.NewDecoder(r.Body).Decode(&body); err != nil { - writeError(w, http.StatusBadRequest, "invalid request body") - return - } - body.ChatSessionID = strings.TrimSpace(body.ChatSessionID) - if body.ChatSessionID == "" { - writeError(w, http.StatusBadRequest, "chatSessionId is required") - return - } - runtime, ok := s.getWorkspaceRuntime(workspaceID) + input, ok := s.sessionSnapshotHandlerInput(w, r) if !ok { - writeError(w, http.StatusNotFound, "workspace not found") - return - } - callbackToken := s.callbackTokenForWorkspace(workspaceID) - if callbackToken == "" { - writeError(w, http.StatusConflict, "workspace callback token unavailable") return } - result, err := s.restoreSessionSnapshot(r.Context(), runtime, sessionID, body.ChatSessionID, callbackToken) + result, err := s.restoreSessionSnapshot(r.Context(), input.runtime, input.sessionID, input.chatSessionID, input.callbackToken) if err != nil { - _ = s.reportSnapshotRestoreResult(context.Background(), workspaceID, body.ChatSessionID, "degraded", err.Error(), callbackToken) + _ = s.reportSnapshotRestoreResult(context.Background(), input.workspaceID, input.chatSessionID, "degraded", err.Error(), input.callbackToken) writeJSON(w, http.StatusOK, map[string]interface{}{"status": "degraded", "message": err.Error()}) return } From 46cd147aefc3138c90dbbefcc23bbfdadea08b09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 15:31:22 +0000 Subject: [PATCH 08/28] fix: enforce snapshot artifact integrity --- .../routes/workspaces/session-snapshots.ts | 45 +++++++++++-- .../workspaces-session-snapshots.test.ts | 67 +++++++++++++++++++ .../internal/server/session_snapshot.go | 30 +++++++++ .../server/session_snapshot_git_test.go | 28 ++++++++ 4 files changed, 164 insertions(+), 6 deletions(-) diff --git a/apps/api/src/routes/workspaces/session-snapshots.ts b/apps/api/src/routes/workspaces/session-snapshots.ts index cd11fdc9a..ac4b72eae 100644 --- a/apps/api/src/routes/workspaces/session-snapshots.ts +++ b/apps/api/src/routes/workspaces/session-snapshots.ts @@ -127,7 +127,8 @@ sessionSnapshotRoutes.put('/:id/session-snapshot/artifacts/:artifact', async (c) throw errors.forbidden('Snapshot chat session does not match workspace'); } const contentLength = c.req.header('content-length'); - if (contentLength) { + if (!contentLength) throw errors.badRequest('Snapshot artifact Content-Length is required'); + { const parsed = Number.parseInt(contentLength, 10); if (!Number.isFinite(parsed) || parsed < 0) throw errors.badRequest('Invalid Content-Length'); if (parsed > getSessionSnapshotConfig(c.env).totalBudgetBytes) { @@ -159,11 +160,43 @@ sessionSnapshotRoutes.post('/:id/session-snapshot/complete', async (c) => { const degradation = requiredStringField(body, 'degradation') as SessionSnapshotDegradation; if (!COMPLETE_STATUSES.has(status)) throw errors.badRequest('Invalid snapshot status'); if (!DEGRADATIONS.has(degradation)) throw errors.badRequest('Invalid snapshot degradation'); - const manifest = expectJsonRecord(body.manifest, 'snapshot manifest') as unknown as SessionSnapshotManifest; - const artifactSizes = expectJsonRecord(body.artifactSizes ?? {}, 'snapshot artifact sizes') as { - homeBytes?: number; - wipBytes?: number; - }; + const manifestRecord = expectJsonRecord(body.manifest, 'snapshot manifest'); + const manifest = manifestRecord as unknown as SessionSnapshotManifest; + if ( + manifest.version !== 1 || + manifest.chatSessionId !== chatSessionId || + manifest.workspaceId !== workspaceId + ) { + throw errors.badRequest('Snapshot manifest identity does not match request'); + } + const manifestArtifacts = expectJsonRecord( + manifestRecord.artifacts ?? {}, + 'snapshot manifest artifacts' + ); + const artifactSizes: { homeBytes?: number; wipBytes?: number } = {}; + for (const [artifact, sizeKey] of [ + ['home', 'homeBytes'], + ['wip', 'wipBytes'], + ] as const) { + if (!(artifact in manifestArtifacts)) continue; + const artifactRecord = expectJsonRecord( + manifestArtifacts[artifact], + 'snapshot ' + artifact + ' artifact' + ); + const claimedSize = artifactRecord.sizeBytes; + if (typeof claimedSize !== 'number' || !Number.isSafeInteger(claimedSize) || claimedSize < 0) { + throw errors.badRequest('Snapshot ' + artifact + ' size is invalid'); + } + const object = await c.env.R2.head(buildSessionSnapshotR2Key(c.env, chatSessionId, artifact)); + if (!object) throw errors.badRequest('Snapshot ' + artifact + ' artifact is missing'); + if (object.size !== claimedSize) + throw errors.badRequest('Snapshot ' + artifact + ' size does not match upload'); + artifactSizes[sizeKey] = object.size; + } + const totalArtifactBytes = (artifactSizes.homeBytes ?? 0) + (artifactSizes.wipBytes ?? 0); + if (totalArtifactBytes > getSessionSnapshotConfig(c.env).totalBudgetBytes) { + throw errors.badRequest('Snapshot artifacts exceed configured total budget'); + } await completeSessionSnapshot(db, c.env, { workspaceId, diff --git a/apps/api/tests/unit/routes/workspaces-session-snapshots.test.ts b/apps/api/tests/unit/routes/workspaces-session-snapshots.test.ts index eb4ff8753..86ed800e2 100644 --- a/apps/api/tests/unit/routes/workspaces-session-snapshots.test.ts +++ b/apps/api/tests/unit/routes/workspaces-session-snapshots.test.ts @@ -53,6 +53,7 @@ describe('workspaces session snapshot callback routes', () => { const r2 = { put: vi.fn(), get: vi.fn(), + head: vi.fn(), }; const runtimeBindings = { DATABASE: {} as any, @@ -195,4 +196,70 @@ describe('workspaces session snapshot callback routes', () => { expect(res.status).toBe(403); expect(mocks.prepareSessionSnapshot).not.toHaveBeenCalled(); }); + + it('rejects artifact uploads without an authoritative Content-Length', async () => { + const res = await app.request( + '/api/workspaces/WS_1/session-snapshot/artifacts/home?chatSessionId=chat-1', + { + method: 'PUT', + headers: { Authorization: 'Bearer callback-token' }, + body: 'home', + }, + runtimeBindings + ); + expect(res.status).toBe(400); + expect(r2.put).not.toHaveBeenCalled(); + }); + + it('derives completion sizes from R2 and rejects manifest identity mismatches', async () => { + r2.head.mockImplementation(async (key: string) => + key.endsWith('home.tar') ? { size: 4 } : key.endsWith('wip.bundle') ? { size: 3 } : null + ); + const body = { + chatSessionId: 'chat-1', + agentSessionId: 'agent-session-1', + runtime: 'cf-container', + status: 'available', + degradation: 'none', + baseCommit: 'abc123', + artifactSizes: { homeBytes: 999, wipBytes: 999 }, + manifest: { + version: 1, + chatSessionId: 'chat-1', + workspaceId: 'WS_1', + agentSessionId: 'agent-session-1', + status: 'available', + degradation: 'none', + skipped: [], + artifacts: { home: { sizeBytes: 4 }, wip: { sizeBytes: 3 } }, + createdAt: '2026-07-11T00:00:00.000Z', + }, + }; + const ok = await app.request( + '/api/workspaces/WS_1/session-snapshot/complete', + { + method: 'POST', + headers: { Authorization: 'Bearer callback-token', 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }, + runtimeBindings + ); + expect(ok.status).toBe(200); + expect(mocks.completeSessionSnapshot).toHaveBeenCalledWith( + expect.anything(), + runtimeBindings, + expect.objectContaining({ artifactSizes: { homeBytes: 4, wipBytes: 3 } }) + ); + + const mismatch = await app.request( + '/api/workspaces/WS_1/session-snapshot/complete', + { + method: 'POST', + headers: { Authorization: 'Bearer callback-token', 'Content-Type': 'application/json' }, + body: JSON.stringify({ ...body, manifest: { ...body.manifest, workspaceId: 'WS_OTHER' } }), + }, + runtimeBindings + ); + expect(mismatch.status).toBe(400); + }); }); diff --git a/packages/vm-agent/internal/server/session_snapshot.go b/packages/vm-agent/internal/server/session_snapshot.go index b49575c52..a7d3c1994 100644 --- a/packages/vm-agent/internal/server/session_snapshot.go +++ b/packages/vm-agent/internal/server/session_snapshot.go @@ -567,6 +567,10 @@ func (s *Server) uploadSnapshotFile(ctx context.Context, uploadPath, filePath, t return 0, "", err } defer file.Close() + fileInfo, err := file.Stat() + if err != nil { + return 0, "", err + } h := sha256.New() reader := &countingReader{r: file, hash: h} req, err := http.NewRequestWithContext(ctx, http.MethodPut, target, newIdleReader(reader, idleTimeout)) @@ -574,6 +578,7 @@ func (s *Server) uploadSnapshotFile(ctx context.Context, uploadPath, filePath, t return 0, "", err } req.Header.Set("Authorization", "Bearer "+token) + req.ContentLength = fileInfo.Size() res, err := s.controlPlaneHTTPClient(0).Do(req) if err != nil { return 0, "", err @@ -615,6 +620,9 @@ func (s *Server) downloadAndExtractTar(ctx context.Context, downloadPath, token if relErr != nil || relTarget == ".." || strings.HasPrefix(relTarget, ".."+string(filepath.Separator)) { continue } + if err := rejectSymlinkPath(home, target); err != nil { + return err + } if header.Typeflag != tar.TypeDir && header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeRegA { continue } @@ -751,3 +759,25 @@ func choosePositiveDurationMs(value int64, fallback time.Duration) time.Duration } return fallback } + +func rejectSymlinkPath(root, target string) error { + rel, err := filepath.Rel(root, target) + if err != nil || rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return fmt.Errorf("snapshot target is outside home") + } + current := root + for _, part := range strings.Split(rel, string(filepath.Separator)) { + current = filepath.Join(current, part) + info, statErr := os.Lstat(current) + if os.IsNotExist(statErr) { + continue + } + if statErr != nil { + return statErr + } + if info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("snapshot target traverses symlink: %s", rel) + } + } + return nil +} diff --git a/packages/vm-agent/internal/server/session_snapshot_git_test.go b/packages/vm-agent/internal/server/session_snapshot_git_test.go index 5ec9d69b8..70b5dd400 100644 --- a/packages/vm-agent/internal/server/session_snapshot_git_test.go +++ b/packages/vm-agent/internal/server/session_snapshot_git_test.go @@ -1,6 +1,8 @@ package server import ( + "archive/tar" + "bytes" "context" "io" "net/http" @@ -111,3 +113,29 @@ func gitOutput(t *testing.T, dir string, args ...string) string { } return strings.TrimSpace(string(out)) } + +func TestDownloadAndExtractTarRejectsExistingHomeSymlink(t *testing.T) { + home := t.TempDir() + outside := t.TempDir() + t.Setenv("HOME", home) + if err := os.Symlink(outside, filepath.Join(home, "linked")); err != nil { + t.Fatal(err) + } + var tarBody bytes.Buffer + tw := tar.NewWriter(&tarBody) + writeTarFile(t, tw, "linked/credential", "secret") + if err := tw.Close(); err != nil { + t.Fatal(err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(tarBody.Bytes()) + })) + defer server.Close() + s := &Server{config: &config.Config{ControlPlaneURL: server.URL}} + if err := s.downloadAndExtractTar(context.Background(), server.URL, "token", time.Second); err == nil || !strings.Contains(err.Error(), "symlink") { + t.Fatalf("error = %v, want symlink rejection", err) + } + if _, err := os.Stat(filepath.Join(outside, "credential")); !os.IsNotExist(err) { + t.Fatalf("outside credential stat err = %v, want not exist", err) + } +} From 7a7d283578946ef3934c9b9c9b65ab9eed657ed7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 15:34:31 +0000 Subject: [PATCH 09/28] task: record final snapshot security review --- .../2026-07-11-runtime-neutral-session-hibernate-wake.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tasks/archive/2026-07-11-runtime-neutral-session-hibernate-wake.md b/tasks/archive/2026-07-11-runtime-neutral-session-hibernate-wake.md index b435d94c8..132aa10e6 100644 --- a/tasks/archive/2026-07-11-runtime-neutral-session-hibernate-wake.md +++ b/tasks/archive/2026-07-11-runtime-neutral-session-hibernate-wake.md @@ -80,3 +80,5 @@ Updated authorization from Raphaël, 2026-07-11: - Restore checked out a temporary branch and attempted to delete it while current; remediation now imports the advertised bundle ref, materializes its tree, and leaves WIP uncommitted on the original branch. - R2 cleanup was not provisioned. Pulumi now owns a prefix-scoped lifecycle rule, and its positive TTL configuration is injected into the Worker so application expiry and object deletion stay aligned. - The initial PR exceeded the API file-size gate and omitted preflight/specialist evidence markers; the node snapshot wrappers are extracted and durable PR evidence must be added after final reviewers complete. +- Final security pass requires upload `Content-Length`, derives artifact sizes from R2, enforces the aggregate budget, validates manifest workspace/chat identity, and rejects restore paths that traverse pre-existing home symlinks. +- PR evidence now includes the required Agent Preflight and eight-reviewer table; staging remains the only intentionally pending merge gate. From 565cf4aa41c4b62c9d575e55564807a3e946e01d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 15:46:05 +0000 Subject: [PATCH 10/28] fix: route sleeping sessions through wake restore --- apps/api/src/routes/chat-workspace-resolver.ts | 16 +++++++--------- .../unit/cf-container-runtime-contract.test.ts | 6 +++--- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/apps/api/src/routes/chat-workspace-resolver.ts b/apps/api/src/routes/chat-workspace-resolver.ts index d4804d464..623942a27 100644 --- a/apps/api/src/routes/chat-workspace-resolver.ts +++ b/apps/api/src/routes/chat-workspace-resolver.ts @@ -76,7 +76,7 @@ export async function resolveLiveWorkspaceForSession( } /** - * Resolve the live workspace AND its running agent session for a chat action + * Resolve the live workspace AND its resumable agent session for a chat action * (e.g. /prompt, /cancel), enforcing tenant scoping at every layer. * * Consolidates the shared resolution path used by the /prompt and /cancel @@ -107,13 +107,11 @@ export async function resolveLiveAgentSessionForChat( // Verify the node is still reachable — prevents requests to destroyed VMs // whose DNS records no longer exist (would loop back via wildcard DNS). // D1 nodes.status uses 'running' for healthy nodes (not 'active'/'warm', which are DO states). - if (workspace.nodeStatus === 'sleeping') { - // Phase 3 of idea 01KX4KSXEXQMP41KS34TW9EN01 will wake and rehydrate the - // cf-container before forwarding this prompt. - throw errors.conflict('The workspace container is asleep. Send a new message after wake/rehydrate support lands.'); - } - - if (workspace.nodeStatus !== 'running') { + // A request routed to a sleeping cf-container is the wake signal: the + // container proxy launches a fresh runtime and restores its latest snapshot + // before forwarding the request. Preserve the fail-fast guard for every + // other non-running node state. + if (workspace.nodeStatus !== 'running' && workspace.nodeStatus !== 'sleeping') { throw errors.conflict( 'The workspace node is no longer running. Start a new chat to create a fresh workspace.' ); @@ -128,7 +126,7 @@ export async function resolveLiveAgentSessionForChat( and( eq(schema.agentSessions.workspaceId, workspace.id), eq(schema.agentSessions.userId, userId), - eq(schema.agentSessions.status, 'running') + inArray(schema.agentSessions.status, ['running', 'sleeping']) ) ) .limit(1); diff --git a/apps/api/tests/unit/cf-container-runtime-contract.test.ts b/apps/api/tests/unit/cf-container-runtime-contract.test.ts index d150e377d..1140cc9fc 100644 --- a/apps/api/tests/unit/cf-container-runtime-contract.test.ts +++ b/apps/api/tests/unit/cf-container-runtime-contract.test.ts @@ -138,9 +138,9 @@ describe('cf-container runtime spike contracts', () => { expect(containerDo).toContain("await this.ctx.storage.put('lifecycleStatus', 'launching' satisfies LifecycleStatus)"); expect(containerDo).toContain("await this.ctx.storage.put('lifecycleStatus', 'running' satisfies LifecycleStatus)"); expect(chatResolver).toContain("inArray(schema.workspaces.status, ['running', 'recovery', 'sleeping'])"); - expect(chatResolver).toContain("workspace.nodeStatus === 'sleeping'"); - expect(chatResolver).toContain('The workspace container is asleep.'); - expect(chatResolver).toContain('Phase 3 of idea 01KX4KSXEXQMP41KS34TW9EN01'); + expect(chatResolver).toContain("workspace.nodeStatus !== 'running' && workspace.nodeStatus !== 'sleeping'"); + expect(chatResolver).toContain("inArray(schema.agentSessions.status, ['running', 'sleeping'])"); + expect(chatResolver).not.toContain('The workspace container is asleep.'); expect(containerDo).toContain("return new Response('Container is stopped; create a new instant session.', { status: 410 })"); expect(containerDo).toContain("await this.ctx.storage.put('launchConfig', config)"); expect(containerDo).toContain('nodeCallbackToken: string'); From 8a0e625218dbf64c0602eb3fb83e91ad49e7fcc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 15:56:18 +0000 Subject: [PATCH 11/28] ci: configure container sleep interval --- .github/workflows/deploy-reusable.yml | 1 + scripts/deploy/sync-wrangler-config.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index ff4240435..064e17bf4 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -358,6 +358,7 @@ jobs: HETZNER_BASE_IMAGE: ${{ vars.HETZNER_BASE_IMAGE }} ARTIFACTS_BINDING_ENABLED: ${{ vars.ARTIFACTS_BINDING_ENABLED }} CF_CONTAINER_ENABLED: ${{ vars.CF_CONTAINER_ENABLED }} + CF_CONTAINER_SLEEP_AFTER: ${{ vars.CF_CONTAINER_SLEEP_AFTER }} CF_CONTAINER_PORT_READY_TIMEOUT_MS: ${{ vars.CF_CONTAINER_PORT_READY_TIMEOUT_MS }} CF_CONTAINER_VM_AGENT_PORT: ${{ vars.CF_CONTAINER_VM_AGENT_PORT }} SANDBOX_ENABLED: ${{ vars.SANDBOX_ENABLED }} diff --git a/scripts/deploy/sync-wrangler-config.ts b/scripts/deploy/sync-wrangler-config.ts index 088c9f5f2..f4cb2fdec 100644 --- a/scripts/deploy/sync-wrangler-config.ts +++ b/scripts/deploy/sync-wrangler-config.ts @@ -342,6 +342,7 @@ function getApiWorkerVars( 'REQUIRE_APPROVAL', 'HETZNER_BASE_IMAGE', 'CF_CONTAINER_ENABLED', + 'CF_CONTAINER_SLEEP_AFTER', 'CF_CONTAINER_PORT_READY_TIMEOUT_MS', 'CF_CONTAINER_VM_AGENT_PORT', 'SANDBOX_ENABLED', From 80672b8225800ff8596896c031868dc4aba3428d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 16:17:07 +0000 Subject: [PATCH 12/28] fix: surface degraded snapshot restores --- apps/api/src/durable-objects/vm-agent-container.ts | 12 ++++++++++++ .../vm-agent/internal/server/session_snapshot.go | 14 +++++++------- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/apps/api/src/durable-objects/vm-agent-container.ts b/apps/api/src/durable-objects/vm-agent-container.ts index 56dee6a94..b24ef2dc6 100644 --- a/apps/api/src/durable-objects/vm-agent-container.ts +++ b/apps/api/src/durable-objects/vm-agent-container.ts @@ -363,6 +363,18 @@ export class VmAgentContainer extends Container { await this.markWakeDegraded(config, restoreBody || `restore failed with HTTP ${restoreResponse.status}`); return { ok: false, message: restoreBody || 'Session restore failed.' }; } + let restoreStatus = ''; + try { + const parsed = JSON.parse(restoreBody) as { status?: unknown }; + restoreStatus = typeof parsed.status === 'string' ? parsed.status : ''; + } catch { + // A successful restore must provide an explicit machine-readable status. + } + if (restoreStatus !== 'restored') { + const message = restoreBody || 'Session restore did not report restored status.'; + await this.markWakeDegraded(config, message); + return { ok: false, message }; + } const now = new Date().toISOString(); await db.update(schema.nodes).set({ diff --git a/packages/vm-agent/internal/server/session_snapshot.go b/packages/vm-agent/internal/server/session_snapshot.go index a7d3c1994..35d09fd2e 100644 --- a/packages/vm-agent/internal/server/session_snapshot.go +++ b/packages/vm-agent/internal/server/session_snapshot.go @@ -674,21 +674,21 @@ func (s *Server) downloadAndRestoreWIP(ctx context.Context, downloadPath, token defer os.Remove(tmpPath) heads, err := runStandaloneGitCommand(ctx, workDir, nil, "bundle", "list-heads", tmpPath) if err != nil { - return err + return fmt.Errorf("list snapshot bundle heads: %w: %s", err, heads) } fields := strings.Fields(heads) if len(fields) < 2 { return fmt.Errorf("snapshot bundle has no restorable ref") } - if _, err := runStandaloneGitCommand(ctx, workDir, nil, "fetch", tmpPath, fields[1]); err != nil { - return err + if output, err := runStandaloneGitCommand(ctx, workDir, nil, "fetch", tmpPath, fields[1]); err != nil { + return fmt.Errorf("fetch snapshot bundle: %w: %s", err, output) } - if _, err := runStandaloneGitCommand(ctx, workDir, nil, "read-tree", "--reset", "-u", "FETCH_HEAD"); err != nil { - return err + if output, err := runStandaloneGitCommand(ctx, workDir, nil, "read-tree", "--reset", "-u", "FETCH_HEAD"); err != nil { + return fmt.Errorf("materialize snapshot tree: %w: %s", err, output) } if strings.TrimSpace(baseCommit) != "" { - if _, err := runStandaloneGitCommand(ctx, workDir, nil, "reset", "--mixed", baseCommit); err != nil { - return err + if output, err := runStandaloneGitCommand(ctx, workDir, nil, "reset", "--mixed", baseCommit); err != nil { + return fmt.Errorf("restore snapshot base commit: %w: %s", err, output) } } return nil From 7263cc6ab1bfd7d2e5119e839bf8c407f2d449dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 16:30:29 +0000 Subject: [PATCH 13/28] fix: provision runtime before restoring git state --- .../vm-agent/internal/server/session_snapshot.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/vm-agent/internal/server/session_snapshot.go b/packages/vm-agent/internal/server/session_snapshot.go index 35d09fd2e..fb056926e 100644 --- a/packages/vm-agent/internal/server/session_snapshot.go +++ b/packages/vm-agent/internal/server/session_snapshot.go @@ -266,6 +266,14 @@ func (s *Server) restoreSessionSnapshot(ctx context.Context, runtime *WorkspaceR return nil, err } } + // A freshly launched runtime has no repository yet. Provision it after HOME + // extraction so current runtime assets and credentials overwrite stale + // snapshot copies, but before applying the Git WIP bundle, which requires a + // materialized repository. + if _, err := s.provisionWorkspaceRuntime(ctx, runtime); err != nil { + _ = s.reportSnapshotRestoreResult(ctx, runtime.ID, chatSessionID, "fresh_injection_failed", err.Error(), callbackToken) + return nil, err + } if restore.Download.WIP != "" { workDir := standaloneWorkspaceWorkDir(runtime, s.config.WorkspaceDir, s.config.ContainerWorkDir) if err := s.downloadAndRestoreWIP(ctx, restore.Download.WIP, callbackToken, idleTimeout, workDir, restore.BaseCommit); err != nil { @@ -273,10 +281,6 @@ func (s *Server) restoreSessionSnapshot(ctx context.Context, runtime *WorkspaceR return nil, err } } - if _, err := s.provisionWorkspaceRuntime(ctx, runtime); err != nil { - _ = s.reportSnapshotRestoreResult(ctx, runtime.ID, chatSessionID, "fresh_injection_failed", err.Error(), callbackToken) - return nil, err - } if session, exists := s.agentSessions.Get(runtime.ID, sessionID); exists { hostKey := runtime.ID + ":" + sessionID _ = s.getOrCreateSessionHost(hostKey, runtime.ID, sessionID, session, runtime, "") From 8d9a4c6cbd7a4f4d2c2f551defd2dd3f07250c88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 16:48:33 +0000 Subject: [PATCH 14/28] fix: allow restored containers time to wake --- .github/workflows/deploy-reusable.yml | 1 + apps/api/src/env.ts | 1 + .../api/src/routes/chat-workspace-resolver.ts | 4 ++-- apps/api/src/routes/chat.ts | 8 +++++-- apps/api/src/services/node-agent.ts | 5 +++++ apps/api/tests/unit/node-agent-health.test.ts | 13 ++++++++++++ .../unit/routes/chat-prompt-cancel.test.ts | 21 ++++++++++++++++++- .../docs/docs/reference/configuration.md | 1 + scripts/deploy/sync-wrangler-config.ts | 1 + 9 files changed, 50 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index 064e17bf4..090a3f4a7 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -360,6 +360,7 @@ jobs: CF_CONTAINER_ENABLED: ${{ vars.CF_CONTAINER_ENABLED }} CF_CONTAINER_SLEEP_AFTER: ${{ vars.CF_CONTAINER_SLEEP_AFTER }} CF_CONTAINER_PORT_READY_TIMEOUT_MS: ${{ vars.CF_CONTAINER_PORT_READY_TIMEOUT_MS }} + CF_CONTAINER_WAKE_TIMEOUT_MS: ${{ vars.CF_CONTAINER_WAKE_TIMEOUT_MS }} CF_CONTAINER_VM_AGENT_PORT: ${{ vars.CF_CONTAINER_VM_AGENT_PORT }} SANDBOX_ENABLED: ${{ vars.SANDBOX_ENABLED }} SANDBOX_EXEC_TIMEOUT_MS: ${{ vars.SANDBOX_EXEC_TIMEOUT_MS }} diff --git a/apps/api/src/env.ts b/apps/api/src/env.ts index 93fcb5307..76e03a9a7 100644 --- a/apps/api/src/env.ts +++ b/apps/api/src/env.ts @@ -778,6 +778,7 @@ export interface Env { CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS?: string; // Active-work renewActivityTimeout interval (default: 300000) CF_CONTAINER_VM_AGENT_PORT?: string; // vm-agent standalone HTTP port inside the raw container (default: 8080) CF_CONTAINER_PORT_READY_TIMEOUT_MS?: string; // Max time to wait for vm-agent port readiness (default: 30000) + CF_CONTAINER_WAKE_TIMEOUT_MS?: string; // Max time for launch + restore before forwarding a wake request (default: 120000) CF_CONTAINER_WORKSPACE_BASE_DIR?: string; // Base checkout dir inside raw container (default: /workspaces) // Legacy Sandbox SDK prototype (admin-only) SANDBOX_ENABLED?: string; // Legacy/fallback kill switch for sandbox routes and older staging config (default: false) diff --git a/apps/api/src/routes/chat-workspace-resolver.ts b/apps/api/src/routes/chat-workspace-resolver.ts index 623942a27..928723f52 100644 --- a/apps/api/src/routes/chat-workspace-resolver.ts +++ b/apps/api/src/routes/chat-workspace-resolver.ts @@ -91,7 +91,7 @@ export async function resolveLiveWorkspaceForSession( export async function resolveLiveAgentSessionForChat( db: ChatDb, { projectId, sessionId, userId }: { projectId: string; sessionId: string; userId: string } -): Promise<{ workspace: { id: string; nodeId: string }; agentSession: { id: string } }> { +): Promise<{ workspace: { id: string; nodeId: string; nodeStatus: string }; agentSession: { id: string } }> { // Find the workspace linked to this chat session, scoped to the owning // project + user (see resolveLiveWorkspaceForSession). The node join also // verifies the node is still active: when a node is destroyed (e.g., after @@ -135,5 +135,5 @@ export async function resolveLiveAgentSessionForChat( throw errors.notFound('No running agent session found'); } - return { workspace: { id: workspace.id, nodeId: workspace.nodeId }, agentSession }; + return { workspace: { id: workspace.id, nodeId: workspace.nodeId, nodeStatus: workspace.nodeStatus }, agentSession }; } diff --git a/apps/api/src/routes/chat.ts b/apps/api/src/routes/chat.ts index a941c0ccd..7ec9af872 100644 --- a/apps/api/src/routes/chat.ts +++ b/apps/api/src/routes/chat.ts @@ -517,14 +517,18 @@ chatRoutes.post('/:sessionId/prompt', async (c) => { const { enrichedMessage } = await enrichMessageWithMentions(content, db, projectId, userId, c.env); // Forward the prompt to the VM agent - const { sendPromptToAgentOnNode } = await import('../services/node-agent'); + const { getCfContainerWakeTimeoutMs, sendPromptToAgentOnNode } = await import('../services/node-agent'); const result = await sendPromptToAgentOnNode( workspace.nodeId, workspace.id, agentSession.id, enrichedMessage, c.env, - userId + userId, + undefined, + workspace.nodeStatus === 'sleeping' + ? { requestTimeoutMs: getCfContainerWakeTimeoutMs(c.env) } + : undefined ); return c.json(expectJsonRecord(result, 'chat.agent_prompt_result')); diff --git a/apps/api/src/services/node-agent.ts b/apps/api/src/services/node-agent.ts index c80009372..998174b31 100644 --- a/apps/api/src/services/node-agent.ts +++ b/apps/api/src/services/node-agent.ts @@ -15,6 +15,7 @@ import { } from './vm-agent-container'; const DEFAULT_NODE_AGENT_REQUEST_TIMEOUT_MS = 30_000; +const DEFAULT_CF_CONTAINER_WAKE_TIMEOUT_MS = 120_000; const DEFAULT_NODE_AGENT_READY_TIMEOUT_MS = 900_000; // 15 min — cloud-init takes 8-12 min on Hetzner const DEFAULT_NODE_AGENT_READY_POLL_INTERVAL_MS = 5000; @@ -66,6 +67,10 @@ export function getNodeAgentRequestTimeoutMs(env: { NODE_AGENT_REQUEST_TIMEOUT_M return getTimeoutMs(env.NODE_AGENT_REQUEST_TIMEOUT_MS, DEFAULT_NODE_AGENT_REQUEST_TIMEOUT_MS); } +export function getCfContainerWakeTimeoutMs(env: { CF_CONTAINER_WAKE_TIMEOUT_MS?: string }): number { + return getTimeoutMs(env.CF_CONTAINER_WAKE_TIMEOUT_MS, DEFAULT_CF_CONTAINER_WAKE_TIMEOUT_MS); +} + export async function waitForNodeAgentReady(nodeId: string, env: Env): Promise { const timeoutMs = getNodeAgentReadyTimeoutMs(env); const pollIntervalMs = getNodeAgentReadyPollIntervalMs(env); diff --git a/apps/api/tests/unit/node-agent-health.test.ts b/apps/api/tests/unit/node-agent-health.test.ts index 9ecbc759e..1beb54496 100644 --- a/apps/api/tests/unit/node-agent-health.test.ts +++ b/apps/api/tests/unit/node-agent-health.test.ts @@ -12,10 +12,23 @@ import { afterEach,beforeEach, describe, expect, it, vi } from 'vitest'; import { + getCfContainerWakeTimeoutMs, getNodeAgentReadyPollIntervalMs, getNodeAgentReadyTimeoutMs, } from '../../src/services/node-agent'; +describe('getCfContainerWakeTimeoutMs', () => { + it('defaults to two minutes and accepts a positive override', () => { + expect(getCfContainerWakeTimeoutMs({})).toBe(120_000); + expect(getCfContainerWakeTimeoutMs({ CF_CONTAINER_WAKE_TIMEOUT_MS: '180000' })).toBe(180_000); + }); + + it('rejects non-positive and invalid overrides', () => { + expect(getCfContainerWakeTimeoutMs({ CF_CONTAINER_WAKE_TIMEOUT_MS: '0' })).toBe(120_000); + expect(getCfContainerWakeTimeoutMs({ CF_CONTAINER_WAKE_TIMEOUT_MS: 'invalid' })).toBe(120_000); + }); +}); + // ============================================================================= // getNodeAgentReadyTimeoutMs — env var parsing // ============================================================================= diff --git a/apps/api/tests/unit/routes/chat-prompt-cancel.test.ts b/apps/api/tests/unit/routes/chat-prompt-cancel.test.ts index 461ae9ced..b8e88bece 100644 --- a/apps/api/tests/unit/routes/chat-prompt-cancel.test.ts +++ b/apps/api/tests/unit/routes/chat-prompt-cancel.test.ts @@ -11,6 +11,7 @@ const mocks = vi.hoisted(() => ({ requireProjectCapability: vi.fn(), sendPromptToAgentOnNode: vi.fn(), cancelAgentSessionOnNode: vi.fn(), + getCfContainerWakeTimeoutMs: vi.fn(() => 120_000), enrichMessageWithMentions: vi.fn(), parseOptionalBody: vi.fn(), })); @@ -80,6 +81,7 @@ vi.mock('../../../src/schemas', () => ({ vi.mock('../../../src/services/node-agent', () => ({ sendPromptToAgentOnNode: mocks.sendPromptToAgentOnNode, cancelAgentSessionOnNode: mocks.cancelAgentSessionOnNode, + getCfContainerWakeTimeoutMs: mocks.getCfContainerWakeTimeoutMs, })); vi.mock('../../../src/services/mention-enrichment', () => ({ @@ -236,7 +238,24 @@ describe('POST /sessions/:sessionId/prompt', () => { expect(response.status).toBe(200); expect(mocks.sendPromptToAgentOnNode).toHaveBeenCalledWith( 'node-1', 'ws-1', 'agent-sess-1', - 'hello agent', expect.anything(), 'user-1', + 'hello agent', expect.anything(), 'user-1', undefined, undefined, + ); + }); + + it('uses the extended wake budget for a sleeping session', async () => { + setupDrizzle({ + workspace: { id: 'ws-1', nodeId: 'node-1', nodeStatus: 'sleeping' }, + agentSession: { id: 'agent-sess-1' }, + }); + mocks.sendPromptToAgentOnNode.mockResolvedValue({ ok: true }); + + const response = await postPrompt(); + + expect(response.status).toBe(200); + expect(mocks.sendPromptToAgentOnNode).toHaveBeenCalledWith( + 'node-1', 'ws-1', 'agent-sess-1', + 'hello agent', expect.anything(), 'user-1', undefined, + { requestTimeoutMs: 120_000 }, ); }); diff --git a/apps/www/src/content/docs/docs/reference/configuration.md b/apps/www/src/content/docs/docs/reference/configuration.md index f847053d2..8ad846f30 100644 --- a/apps/www/src/content/docs/docs/reference/configuration.md +++ b/apps/www/src/content/docs/docs/reference/configuration.md @@ -73,6 +73,7 @@ GitHub App secrets use `GH_*` prefix (e.g., `GH_CLIENT_ID`, `GH_WEBHOOK_SECRET`) | Variable | Default | Description | | -------------------------------- | ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `CF_CONTAINER_ENABLED` | `true` | Enables Cloudflare Container instant sessions for matching profiles and zero-config runtime selection. Set `false` to force cloud VM runtime. | +| `CF_CONTAINER_WAKE_TIMEOUT_MS` | `120000` | Maximum time for a sleeping container to launch, restore its snapshot, and accept the triggering request. | | `REQUIRE_APPROVAL` | _(unset)_ | Default signup approval gate. Superadmins can override it at runtime in Admin → Users without redeploying; when no runtime override exists, this value is used. The first genuine human becomes superadmin regardless of this flag — see [First Login & Admin Access](/docs/guides/self-hosting/#first-login--admin-access). | | `TRIAL_ANONYMOUS_USER_ID` | `system_anonymous_trials` | Id of the internal anonymous-trial sentinel user, excluded from first-user superadmin checks. Override only if your deployment uses a different sentinel id. | | `CAPACITY_SIZE_FALLBACK_ENABLED` | `true` | When a new node's VM size is exhausted on transient capacity, descend the size chain (large→medium→small). Only applies to default-derived sizes (project/platform default), never user-requested sizes. Set `false` to disable. | diff --git a/scripts/deploy/sync-wrangler-config.ts b/scripts/deploy/sync-wrangler-config.ts index f4cb2fdec..e2074d5bb 100644 --- a/scripts/deploy/sync-wrangler-config.ts +++ b/scripts/deploy/sync-wrangler-config.ts @@ -344,6 +344,7 @@ function getApiWorkerVars( 'CF_CONTAINER_ENABLED', 'CF_CONTAINER_SLEEP_AFTER', 'CF_CONTAINER_PORT_READY_TIMEOUT_MS', + 'CF_CONTAINER_WAKE_TIMEOUT_MS', 'CF_CONTAINER_VM_AGENT_PORT', 'SANDBOX_ENABLED', 'SANDBOX_EXEC_TIMEOUT_MS', From f91a5e92e81795ad5c65139a34ba9e8a5c5b101d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 17:04:14 +0000 Subject: [PATCH 15/28] fix: include workspace CLI in container image --- apps/api/Dockerfile.vm-agent-container | 2 +- apps/api/tests/unit/cf-container-runtime-contract.test.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/api/Dockerfile.vm-agent-container b/apps/api/Dockerfile.vm-agent-container index 05c93c97d..1a2870c02 100644 --- a/apps/api/Dockerfile.vm-agent-container +++ b/apps/api/Dockerfile.vm-agent-container @@ -9,7 +9,7 @@ RUN set -eux; \ echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" > /etc/apt/sources.list.d/github-cli.list; \ apt-get update; \ apt-get install -y --no-install-recommends gh; \ - npm install -g @zed-industries/claude-agent-acp @anthropic-ai/claude-code; \ + npm install -g @devcontainers/cli @zed-industries/claude-agent-acp @anthropic-ai/claude-code; \ npm cache clean --force; \ rm -rf /var/lib/apt/lists/* diff --git a/apps/api/tests/unit/cf-container-runtime-contract.test.ts b/apps/api/tests/unit/cf-container-runtime-contract.test.ts index 1140cc9fc..5adb8433c 100644 --- a/apps/api/tests/unit/cf-container-runtime-contract.test.ts +++ b/apps/api/tests/unit/cf-container-runtime-contract.test.ts @@ -158,6 +158,7 @@ describe('cf-container runtime spike contracts', () => { expect(dockerfile).toContain('ENTRYPOINT ["/usr/local/bin/vm-agent-bootstrap"]'); expect(dockerfile).toContain('githubcli-archive-keyring.gpg'); expect(dockerfile).toContain('apt-get install -y --no-install-recommends gh'); + expect(dockerfile).toContain('@devcontainers/cli'); expect(dockerfile).toContain('USER node'); expect(dockerfile).toContain('chown -R node:node /workspaces /var/lib/vm-agent'); expect(bootstrap).toContain('agent_bin_dir="/var/lib/vm-agent/bin"'); From ce637d14b20919aacc461186c734edf48927dec1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 17:18:05 +0000 Subject: [PATCH 16/28] fix: use writable container runtime config path --- apps/api/src/durable-objects/vm-agent-container.ts | 1 + apps/api/tests/unit/cf-container-runtime-contract.test.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/apps/api/src/durable-objects/vm-agent-container.ts b/apps/api/src/durable-objects/vm-agent-container.ts index b24ef2dc6..2f0000c41 100644 --- a/apps/api/src/durable-objects/vm-agent-container.ts +++ b/apps/api/src/durable-objects/vm-agent-container.ts @@ -92,6 +92,7 @@ export class VmAgentContainer extends Container { PORT_SCAN_ENABLED: 'false', VM_AGENT_PORT: String(config.vmAgentPort), VM_AGENT_PROTOCOL: 'http', + DEFAULT_DEVCONTAINER_CONFIG_PATH: '/var/lib/vm-agent/default-devcontainer.json', COOKIE_SECURE: 'true', }, labels: { diff --git a/apps/api/tests/unit/cf-container-runtime-contract.test.ts b/apps/api/tests/unit/cf-container-runtime-contract.test.ts index 5adb8433c..f458f0998 100644 --- a/apps/api/tests/unit/cf-container-runtime-contract.test.ts +++ b/apps/api/tests/unit/cf-container-runtime-contract.test.ts @@ -145,6 +145,7 @@ describe('cf-container runtime spike contracts', () => { expect(containerDo).toContain("await this.ctx.storage.put('launchConfig', config)"); expect(containerDo).toContain('nodeCallbackToken: string'); expect(containerDo).toContain('CALLBACK_TOKEN: secrets.nodeCallbackToken'); + expect(containerDo).toContain("DEFAULT_DEVCONTAINER_CONFIG_PATH: '/var/lib/vm-agent/default-devcontainer.json'"); expect(containerDo).toContain("status === 'stopped' ? 'stopped' : 'error'"); expect(containerDo).not.toContain("Container idle timeout expired; start a new instant session."); expect(containerDo).not.toContain("await this.markRuntimeEnded('expired'"); From 59c8cdbd28ff53a3ac05916aee7f73de509d59fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 17:37:23 +0000 Subject: [PATCH 17/28] fix: restore standalone runtime and agent host --- .../src/durable-objects/vm-agent-container.ts | 3 +- .../internal/server/session_snapshot.go | 37 ++++++++++++++++--- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/apps/api/src/durable-objects/vm-agent-container.ts b/apps/api/src/durable-objects/vm-agent-container.ts index 2f0000c41..fd4dd6d76 100644 --- a/apps/api/src/durable-objects/vm-agent-container.ts +++ b/apps/api/src/durable-objects/vm-agent-container.ts @@ -321,7 +321,7 @@ export class VmAgentContainer extends Container { return { ok: false, message: 'Workspace session metadata is unavailable.' }; } const agentSession = await db - .select({ id: schema.agentSessions.id }) + .select({ id: schema.agentSessions.id, agentType: schema.agentSessions.agentType }) .from(schema.agentSessions) .where(eq(schema.agentSessions.workspaceId, config.workspaceId)) .orderBy(desc(schema.agentSessions.updatedAt)) @@ -355,6 +355,7 @@ export class VmAgentContainer extends Container { body: JSON.stringify({ chatSessionId: workspace.chatSessionId, runtime: 'cf-container', + agentType: agentSession.agentType, }), }), config.vmAgentPort diff --git a/packages/vm-agent/internal/server/session_snapshot.go b/packages/vm-agent/internal/server/session_snapshot.go index fb056926e..90f1c4709 100644 --- a/packages/vm-agent/internal/server/session_snapshot.go +++ b/packages/vm-agent/internal/server/session_snapshot.go @@ -15,6 +15,8 @@ import ( "path/filepath" "strings" "time" + + "github.com/workspace/vm-agent/internal/acp" ) const ( @@ -104,6 +106,7 @@ type sessionSnapshotHandlerInput struct { runtimeName string runtime *WorkspaceRuntime callbackToken string + agentType string } func (s *Server) sessionSnapshotHandlerInput(w http.ResponseWriter, r *http.Request) (*sessionSnapshotHandlerInput, bool) { @@ -119,6 +122,7 @@ func (s *Server) sessionSnapshotHandlerInput(w http.ResponseWriter, r *http.Requ var body struct { ChatSessionID string `json:"chatSessionId"` Runtime string `json:"runtime"` + AgentType string `json:"agentType"` } if err := json.NewDecoder(r.Body).Decode(&body); err != nil { writeError(w, http.StatusBadRequest, "invalid request body") @@ -146,6 +150,7 @@ func (s *Server) sessionSnapshotHandlerInput(w http.ResponseWriter, r *http.Requ runtimeName: body.Runtime, runtime: runtime, callbackToken: callbackToken, + agentType: strings.TrimSpace(body.AgentType), }, true } @@ -167,7 +172,7 @@ func (s *Server) handleRestoreAgentSession(w http.ResponseWriter, r *http.Reques if !ok { return } - result, err := s.restoreSessionSnapshot(r.Context(), input.runtime, input.sessionID, input.chatSessionID, input.callbackToken) + result, err := s.restoreSessionSnapshot(r.Context(), input.runtime, input.sessionID, input.chatSessionID, input.agentType, input.callbackToken) if err != nil { _ = s.reportSnapshotRestoreResult(context.Background(), input.workspaceID, input.chatSessionID, "degraded", err.Error(), input.callbackToken) writeJSON(w, http.StatusOK, map[string]interface{}{"status": "degraded", "message": err.Error()}) @@ -250,7 +255,7 @@ func (s *Server) hibernateSessionSnapshot(ctx context.Context, runtime *Workspac return map[string]interface{}{"status": manifest.Status, "degradation": manifest.Degradation, "skipped": manifest.Skipped}, nil } -func (s *Server) restoreSessionSnapshot(ctx context.Context, runtime *WorkspaceRuntime, sessionID, chatSessionID, callbackToken string) (map[string]interface{}, error) { +func (s *Server) restoreSessionSnapshot(ctx context.Context, runtime *WorkspaceRuntime, sessionID, chatSessionID, agentType, callbackToken string) (map[string]interface{}, error) { restore, err := s.fetchSnapshotRestore(ctx, runtime.ID, chatSessionID, callbackToken) if err != nil { return nil, err @@ -270,9 +275,15 @@ func (s *Server) restoreSessionSnapshot(ctx context.Context, runtime *WorkspaceR // extraction so current runtime assets and credentials overwrite stale // snapshot copies, but before applying the Git WIP bundle, which requires a // materialized repository. - if _, err := s.provisionWorkspaceRuntime(ctx, runtime); err != nil { - _ = s.reportSnapshotRestoreResult(ctx, runtime.ID, chatSessionID, "fresh_injection_failed", err.Error(), callbackToken) - return nil, err + var provisionErr error + if s.config.IsStandaloneMode() { + provisionErr = s.prepareStandaloneWorkspaceRuntime(ctx, runtime) + } else { + _, provisionErr = s.provisionWorkspaceRuntime(ctx, runtime) + } + if provisionErr != nil { + _ = s.reportSnapshotRestoreResult(ctx, runtime.ID, chatSessionID, "fresh_injection_failed", provisionErr.Error(), callbackToken) + return nil, provisionErr } if restore.Download.WIP != "" { workDir := standaloneWorkspaceWorkDir(runtime, s.config.WorkspaceDir, s.config.ContainerWorkDir) @@ -281,7 +292,21 @@ func (s *Server) restoreSessionSnapshot(ctx context.Context, runtime *WorkspaceR return nil, err } } - if session, exists := s.agentSessions.Get(runtime.ID, sessionID); exists { + if s.config.IsStandaloneMode() { + if strings.TrimSpace(agentType) == "" { + return nil, fmt.Errorf("agent type is required to restore a standalone session") + } + session, _, createErr := s.agentSessions.Create(runtime.ID, sessionID, "Restored session", "restore:"+sessionID) + if createErr != nil { + return nil, fmt.Errorf("recreate restored agent session: %w", createErr) + } + hostKey := runtime.ID + ":" + sessionID + host := s.getOrCreateSessionHost(hostKey, runtime.ID, sessionID, session, runtime, "") + host.SelectAgent(ctx, agentType) + if host.Status() != acp.HostReady { + return nil, fmt.Errorf("restored agent failed to become ready: %s", host.Status()) + } + } else if session, exists := s.agentSessions.Get(runtime.ID, sessionID); exists { hostKey := runtime.ID + ":" + sessionID _ = s.getOrCreateSessionHost(hostKey, runtime.ID, sessionID, session, runtime, "") } From ef56385f7c4ef810725f3a85fbb95e7b32333e29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 17:38:24 +0000 Subject: [PATCH 18/28] refactor: keep raw restore on standalone path --- apps/api/Dockerfile.vm-agent-container | 2 +- apps/api/src/durable-objects/vm-agent-container.ts | 1 - apps/api/tests/unit/cf-container-runtime-contract.test.ts | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/apps/api/Dockerfile.vm-agent-container b/apps/api/Dockerfile.vm-agent-container index 1a2870c02..05c93c97d 100644 --- a/apps/api/Dockerfile.vm-agent-container +++ b/apps/api/Dockerfile.vm-agent-container @@ -9,7 +9,7 @@ RUN set -eux; \ echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" > /etc/apt/sources.list.d/github-cli.list; \ apt-get update; \ apt-get install -y --no-install-recommends gh; \ - npm install -g @devcontainers/cli @zed-industries/claude-agent-acp @anthropic-ai/claude-code; \ + npm install -g @zed-industries/claude-agent-acp @anthropic-ai/claude-code; \ npm cache clean --force; \ rm -rf /var/lib/apt/lists/* diff --git a/apps/api/src/durable-objects/vm-agent-container.ts b/apps/api/src/durable-objects/vm-agent-container.ts index fd4dd6d76..d2f349fd4 100644 --- a/apps/api/src/durable-objects/vm-agent-container.ts +++ b/apps/api/src/durable-objects/vm-agent-container.ts @@ -92,7 +92,6 @@ export class VmAgentContainer extends Container { PORT_SCAN_ENABLED: 'false', VM_AGENT_PORT: String(config.vmAgentPort), VM_AGENT_PROTOCOL: 'http', - DEFAULT_DEVCONTAINER_CONFIG_PATH: '/var/lib/vm-agent/default-devcontainer.json', COOKIE_SECURE: 'true', }, labels: { diff --git a/apps/api/tests/unit/cf-container-runtime-contract.test.ts b/apps/api/tests/unit/cf-container-runtime-contract.test.ts index f458f0998..1140cc9fc 100644 --- a/apps/api/tests/unit/cf-container-runtime-contract.test.ts +++ b/apps/api/tests/unit/cf-container-runtime-contract.test.ts @@ -145,7 +145,6 @@ describe('cf-container runtime spike contracts', () => { expect(containerDo).toContain("await this.ctx.storage.put('launchConfig', config)"); expect(containerDo).toContain('nodeCallbackToken: string'); expect(containerDo).toContain('CALLBACK_TOKEN: secrets.nodeCallbackToken'); - expect(containerDo).toContain("DEFAULT_DEVCONTAINER_CONFIG_PATH: '/var/lib/vm-agent/default-devcontainer.json'"); expect(containerDo).toContain("status === 'stopped' ? 'stopped' : 'error'"); expect(containerDo).not.toContain("Container idle timeout expired; start a new instant session."); expect(containerDo).not.toContain("await this.markRuntimeEnded('expired'"); @@ -159,7 +158,6 @@ describe('cf-container runtime spike contracts', () => { expect(dockerfile).toContain('ENTRYPOINT ["/usr/local/bin/vm-agent-bootstrap"]'); expect(dockerfile).toContain('githubcli-archive-keyring.gpg'); expect(dockerfile).toContain('apt-get install -y --no-install-recommends gh'); - expect(dockerfile).toContain('@devcontainers/cli'); expect(dockerfile).toContain('USER node'); expect(dockerfile).toContain('chown -R node:node /workspaces /var/lib/vm-agent'); expect(bootstrap).toContain('agent_bin_dir="/var/lib/vm-agent/bin"'); From a27c8081ac72fd6cb948f7ed18e08e886379ac88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 17:58:18 +0000 Subject: [PATCH 19/28] fix: split snapshot archive helpers and de-flake crash report test Extract git-bundle and HOME-tar archive helpers from session_snapshot.go into session_snapshot_archive.go to keep both files under the 800-line limit. Make the AgentCrashReportView copy test await the "Copied" state so it no longer races the async clipboard write in CI. Co-Authored-By: Claude Opus 4.6 --- .../components/AgentCrashReportView.test.tsx | 2 +- .../internal/server/session_snapshot.go | 220 ----------------- .../server/session_snapshot_archive.go | 233 ++++++++++++++++++ 3 files changed, 234 insertions(+), 221 deletions(-) create mode 100644 packages/vm-agent/internal/server/session_snapshot_archive.go diff --git a/packages/acp-client/tests/unit/components/AgentCrashReportView.test.tsx b/packages/acp-client/tests/unit/components/AgentCrashReportView.test.tsx index 7a926c16e..15b9e5621 100644 --- a/packages/acp-client/tests/unit/components/AgentCrashReportView.test.tsx +++ b/packages/acp-client/tests/unit/components/AgentCrashReportView.test.tsx @@ -53,7 +53,7 @@ describe('AgentCrashReportView', () => { expect(copiedReport).toContain('stderr truncated: yes'); expect(copiedReport).toContain('fatal: peer disconnected before response'); expect(copiedReport).not.toContain('sk-secret'); - expect(screen.getByRole('button', { name: 'Copied' })).not.toBeNull(); + expect(await screen.findByRole('button', { name: 'Copied' })).not.toBeNull(); }); it('shows copy failure feedback when clipboard is unavailable', async () => { diff --git a/packages/vm-agent/internal/server/session_snapshot.go b/packages/vm-agent/internal/server/session_snapshot.go index 90f1c4709..f5bfd51c5 100644 --- a/packages/vm-agent/internal/server/session_snapshot.go +++ b/packages/vm-agent/internal/server/session_snapshot.go @@ -390,204 +390,6 @@ func (s *Server) doSnapshotJSON(ctx context.Context, method, workspaceID, path, } return nil } -func createWIPBundle(ctx context.Context, workDir string, entryThreshold int64) (string, string, []snapshotSkippedEntry, error) { - if ok, err := standaloneRepositoryPresent(workDir); err != nil || !ok { - if err != nil { - return "", "", nil, err - } - return "", "", nil, nil - } - if gitOperationInProgress(workDir) { - return "", "", []snapshotSkippedEntry{{Path: workDir, Reason: "git operation in progress"}}, nil - } - base, err := runStandaloneGitCommand(ctx, workDir, nil, "rev-parse", "HEAD") - if err != nil { - return "", "", nil, fmt.Errorf("resolve base commit: %w", err) - } - status, err := runStandaloneGitCommand(ctx, workDir, nil, "status", "--porcelain") - if err != nil { - return base, "", nil, fmt.Errorf("git status: %w", err) - } - if strings.TrimSpace(status) == "" { - return base, "", nil, nil - } - - indexFile, err := os.CreateTemp("", "sam-session-index-*") - if err != nil { - return base, "", nil, err - } - indexPath := indexFile.Name() - _ = indexFile.Close() - _ = os.Remove(indexPath) - defer os.Remove(indexPath) - gitEnv := []string{"GIT_INDEX_FILE=" + indexPath} - if _, err := runStandaloneGitCommand(ctx, workDir, gitEnv, "read-tree", "HEAD"); err != nil { - return base, "", nil, fmt.Errorf("initialize snapshot index: %w", err) - } - if _, err := runStandaloneGitCommand(ctx, workDir, gitEnv, "add", "-A"); err != nil { - return base, "", nil, fmt.Errorf("stage snapshot index: %w", err) - } - skipped := skipOversizedUntracked(workDir, entryThreshold) - for _, entry := range skipped { - if entry.Path != "" { - _, _ = runStandaloneGitCommand(ctx, workDir, gitEnv, "reset", "--", entry.Path) - } - } - tree, err := runStandaloneGitCommand(ctx, workDir, gitEnv, "write-tree") - if err != nil { - return base, "", skipped, fmt.Errorf("write snapshot tree: %w", err) - } - commitEnv := append(gitEnv, "GIT_AUTHOR_NAME=SAM Snapshot", "GIT_AUTHOR_EMAIL=snapshot@localhost", "GIT_COMMITTER_NAME=SAM Snapshot", "GIT_COMMITTER_EMAIL=snapshot@localhost") - commit, err := runStandaloneGitCommand(ctx, workDir, commitEnv, "commit-tree", tree, "-p", base, "-m", "SAM session snapshot") - if err != nil { - return base, "", skipped, fmt.Errorf("create snapshot commit: %w", err) - } - bundle, err := os.CreateTemp("", "sam-session-wip-*.bundle") - if err != nil { - return base, "", skipped, err - } - bundlePath := bundle.Name() - _ = bundle.Close() - snapshotRef := "refs/sam/session-snapshot/" + strings.TrimSuffix(filepath.Base(bundlePath), ".bundle") - if _, err := runStandaloneGitCommand(ctx, workDir, nil, "update-ref", snapshotRef, commit); err != nil { - _ = os.Remove(bundlePath) - return base, "", skipped, fmt.Errorf("create snapshot ref: %w", err) - } - defer func() { - _, _ = runStandaloneGitCommand(context.Background(), workDir, nil, "update-ref", "-d", snapshotRef) - }() - if _, err := runStandaloneGitCommand(ctx, workDir, nil, "bundle", "create", bundlePath, snapshotRef); err != nil { - _ = os.Remove(bundlePath) - return base, "", skipped, fmt.Errorf("create git bundle: %w", err) - } - return base, bundlePath, skipped, nil -} -func gitOperationInProgress(workDir string) bool { - gitDir := filepath.Join(workDir, ".git") - for _, marker := range []string{"MERGE_HEAD", "CHERRY_PICK_HEAD", "REVERT_HEAD", "rebase-merge", "rebase-apply"} { - if _, err := os.Stat(filepath.Join(gitDir, marker)); err == nil { - return true - } - } - return false -} - -func skipOversizedUntracked(workDir string, threshold int64) []snapshotSkippedEntry { - var skipped []snapshotSkippedEntry - _ = filepath.WalkDir(workDir, func(path string, d os.DirEntry, err error) error { - if err != nil || path == workDir { - return nil - } - if d.IsDir() && d.Name() == ".git" { - return filepath.SkipDir - } - if d.IsDir() { - return nil - } - info, statErr := d.Info() - if statErr != nil || info.Size() <= threshold { - return nil - } - rel, _ := filepath.Rel(workDir, path) - if out, gitErr := runStandaloneGitCommand(context.Background(), workDir, nil, "check-ignore", "-q", rel); gitErr == nil && strings.TrimSpace(out) == "" { - return nil - } - skipped = append(skipped, snapshotSkippedEntry{Path: rel, Reason: "entry exceeds size threshold", SizeBytes: info.Size()}) - return nil - }) - return skipped -} - -func createHomeTar(homeDirFn func() (string, error), entryThreshold, totalBudget int64) (string, []snapshotSkippedEntry, error) { - home, err := homeDirFn() - if err != nil { - return "", nil, err - } - home = filepath.Clean(home) - out, err := os.CreateTemp("", "sam-session-home-*.tar") - if err != nil { - return "", nil, err - } - path := out.Name() - tw := tar.NewWriter(out) - var written int64 - var skipped []snapshotSkippedEntry - walkErr := filepath.WalkDir(home, func(path string, d os.DirEntry, err error) error { - if err != nil || path == home { - return nil - } - rel, _ := filepath.Rel(home, path) - if shouldExcludeHomePath(rel) { - if d.IsDir() { - return filepath.SkipDir - } - return nil - } - info, statErr := d.Info() - if statErr != nil { - return nil - } - if info.Size() > entryThreshold { - skipped = append(skipped, snapshotSkippedEntry{Path: "~/" + rel, Reason: "entry exceeds size threshold", SizeBytes: info.Size()}) - if d.IsDir() { - return filepath.SkipDir - } - return nil - } - if !info.Mode().IsRegular() && !info.IsDir() { - skipped = append(skipped, snapshotSkippedEntry{Path: "~/" + rel, Reason: "unsupported home entry type"}) - return nil - } - if !info.IsDir() && written+info.Size() > totalBudget { - skipped = append(skipped, snapshotSkippedEntry{Path: "~/" + rel, Reason: "snapshot budget exhausted", SizeBytes: info.Size()}) - return nil - } - header, headerErr := tar.FileInfoHeader(info, "") - if headerErr != nil { - return nil - } - header.Name = rel - if err := tw.WriteHeader(header); err != nil { - return err - } - if info.Mode().IsRegular() { - f, openErr := os.Open(path) - if openErr != nil { - return nil - } - n, copyErr := io.Copy(tw, f) - _ = f.Close() - written += n - if copyErr != nil { - return copyErr - } - } - return nil - }) - closeErr := tw.Close() - fileCloseErr := out.Close() - if walkErr != nil || closeErr != nil || fileCloseErr != nil { - _ = os.Remove(path) - if walkErr != nil { - return "", skipped, walkErr - } - if closeErr != nil { - return "", skipped, closeErr - } - return "", skipped, fileCloseErr - } - return path, skipped, nil -} - -func shouldExcludeHomePath(rel string) bool { - first := strings.Split(filepath.ToSlash(rel), "/")[0] - switch first { - case ".cache", ".npm", ".cargo", ".rustup", ".local", "node_modules", ".docker": - return true - default: - return false - } -} func (s *Server) uploadSnapshotFile(ctx context.Context, uploadPath, filePath, token string, idleTimeout time.Duration) (int64, string, error) { target := absoluteControlPlaneURL(s.config.ControlPlaneURL, uploadPath) @@ -788,25 +590,3 @@ func choosePositiveDurationMs(value int64, fallback time.Duration) time.Duration } return fallback } - -func rejectSymlinkPath(root, target string) error { - rel, err := filepath.Rel(root, target) - if err != nil || rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { - return fmt.Errorf("snapshot target is outside home") - } - current := root - for _, part := range strings.Split(rel, string(filepath.Separator)) { - current = filepath.Join(current, part) - info, statErr := os.Lstat(current) - if os.IsNotExist(statErr) { - continue - } - if statErr != nil { - return statErr - } - if info.Mode()&os.ModeSymlink != 0 { - return fmt.Errorf("snapshot target traverses symlink: %s", rel) - } - } - return nil -} diff --git a/packages/vm-agent/internal/server/session_snapshot_archive.go b/packages/vm-agent/internal/server/session_snapshot_archive.go new file mode 100644 index 000000000..ccf1763e1 --- /dev/null +++ b/packages/vm-agent/internal/server/session_snapshot_archive.go @@ -0,0 +1,233 @@ +package server + +import ( + "archive/tar" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +func createWIPBundle(ctx context.Context, workDir string, entryThreshold int64) (string, string, []snapshotSkippedEntry, error) { + if ok, err := standaloneRepositoryPresent(workDir); err != nil || !ok { + if err != nil { + return "", "", nil, err + } + return "", "", nil, nil + } + if gitOperationInProgress(workDir) { + return "", "", []snapshotSkippedEntry{{Path: workDir, Reason: "git operation in progress"}}, nil + } + base, err := runStandaloneGitCommand(ctx, workDir, nil, "rev-parse", "HEAD") + if err != nil { + return "", "", nil, fmt.Errorf("resolve base commit: %w", err) + } + status, err := runStandaloneGitCommand(ctx, workDir, nil, "status", "--porcelain") + if err != nil { + return base, "", nil, fmt.Errorf("git status: %w", err) + } + if strings.TrimSpace(status) == "" { + return base, "", nil, nil + } + + indexFile, err := os.CreateTemp("", "sam-session-index-*") + if err != nil { + return base, "", nil, err + } + indexPath := indexFile.Name() + _ = indexFile.Close() + _ = os.Remove(indexPath) + defer os.Remove(indexPath) + gitEnv := []string{"GIT_INDEX_FILE=" + indexPath} + if _, err := runStandaloneGitCommand(ctx, workDir, gitEnv, "read-tree", "HEAD"); err != nil { + return base, "", nil, fmt.Errorf("initialize snapshot index: %w", err) + } + if _, err := runStandaloneGitCommand(ctx, workDir, gitEnv, "add", "-A"); err != nil { + return base, "", nil, fmt.Errorf("stage snapshot index: %w", err) + } + skipped := skipOversizedUntracked(workDir, entryThreshold) + for _, entry := range skipped { + if entry.Path != "" { + _, _ = runStandaloneGitCommand(ctx, workDir, gitEnv, "reset", "--", entry.Path) + } + } + tree, err := runStandaloneGitCommand(ctx, workDir, gitEnv, "write-tree") + if err != nil { + return base, "", skipped, fmt.Errorf("write snapshot tree: %w", err) + } + commitEnv := append(gitEnv, "GIT_AUTHOR_NAME=SAM Snapshot", "GIT_AUTHOR_EMAIL=snapshot@localhost", "GIT_COMMITTER_NAME=SAM Snapshot", "GIT_COMMITTER_EMAIL=snapshot@localhost") + commit, err := runStandaloneGitCommand(ctx, workDir, commitEnv, "commit-tree", tree, "-p", base, "-m", "SAM session snapshot") + if err != nil { + return base, "", skipped, fmt.Errorf("create snapshot commit: %w", err) + } + bundle, err := os.CreateTemp("", "sam-session-wip-*.bundle") + if err != nil { + return base, "", skipped, err + } + bundlePath := bundle.Name() + _ = bundle.Close() + snapshotRef := "refs/sam/session-snapshot/" + strings.TrimSuffix(filepath.Base(bundlePath), ".bundle") + if _, err := runStandaloneGitCommand(ctx, workDir, nil, "update-ref", snapshotRef, commit); err != nil { + _ = os.Remove(bundlePath) + return base, "", skipped, fmt.Errorf("create snapshot ref: %w", err) + } + defer func() { + _, _ = runStandaloneGitCommand(context.Background(), workDir, nil, "update-ref", "-d", snapshotRef) + }() + if _, err := runStandaloneGitCommand(ctx, workDir, nil, "bundle", "create", bundlePath, snapshotRef); err != nil { + _ = os.Remove(bundlePath) + return base, "", skipped, fmt.Errorf("create git bundle: %w", err) + } + return base, bundlePath, skipped, nil +} + +func gitOperationInProgress(workDir string) bool { + gitDir := filepath.Join(workDir, ".git") + for _, marker := range []string{"MERGE_HEAD", "CHERRY_PICK_HEAD", "REVERT_HEAD", "rebase-merge", "rebase-apply"} { + if _, err := os.Stat(filepath.Join(gitDir, marker)); err == nil { + return true + } + } + return false +} + +func skipOversizedUntracked(workDir string, threshold int64) []snapshotSkippedEntry { + var skipped []snapshotSkippedEntry + _ = filepath.WalkDir(workDir, func(path string, d os.DirEntry, err error) error { + if err != nil || path == workDir { + return nil + } + if d.IsDir() && d.Name() == ".git" { + return filepath.SkipDir + } + if d.IsDir() { + return nil + } + info, statErr := d.Info() + if statErr != nil || info.Size() <= threshold { + return nil + } + rel, _ := filepath.Rel(workDir, path) + if out, gitErr := runStandaloneGitCommand(context.Background(), workDir, nil, "check-ignore", "-q", rel); gitErr == nil && strings.TrimSpace(out) == "" { + return nil + } + skipped = append(skipped, snapshotSkippedEntry{Path: rel, Reason: "entry exceeds size threshold", SizeBytes: info.Size()}) + return nil + }) + return skipped +} + +func createHomeTar(homeDirFn func() (string, error), entryThreshold, totalBudget int64) (string, []snapshotSkippedEntry, error) { + home, err := homeDirFn() + if err != nil { + return "", nil, err + } + home = filepath.Clean(home) + out, err := os.CreateTemp("", "sam-session-home-*.tar") + if err != nil { + return "", nil, err + } + path := out.Name() + tw := tar.NewWriter(out) + var written int64 + var skipped []snapshotSkippedEntry + walkErr := filepath.WalkDir(home, func(path string, d os.DirEntry, err error) error { + if err != nil || path == home { + return nil + } + rel, _ := filepath.Rel(home, path) + if shouldExcludeHomePath(rel) { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + info, statErr := d.Info() + if statErr != nil { + return nil + } + if info.Size() > entryThreshold { + skipped = append(skipped, snapshotSkippedEntry{Path: "~/" + rel, Reason: "entry exceeds size threshold", SizeBytes: info.Size()}) + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + if !info.Mode().IsRegular() && !info.IsDir() { + skipped = append(skipped, snapshotSkippedEntry{Path: "~/" + rel, Reason: "unsupported home entry type"}) + return nil + } + if !info.IsDir() && written+info.Size() > totalBudget { + skipped = append(skipped, snapshotSkippedEntry{Path: "~/" + rel, Reason: "snapshot budget exhausted", SizeBytes: info.Size()}) + return nil + } + header, headerErr := tar.FileInfoHeader(info, "") + if headerErr != nil { + return nil + } + header.Name = rel + if err := tw.WriteHeader(header); err != nil { + return err + } + if info.Mode().IsRegular() { + f, openErr := os.Open(path) + if openErr != nil { + return nil + } + n, copyErr := io.Copy(tw, f) + _ = f.Close() + written += n + if copyErr != nil { + return copyErr + } + } + return nil + }) + closeErr := tw.Close() + fileCloseErr := out.Close() + if walkErr != nil || closeErr != nil || fileCloseErr != nil { + _ = os.Remove(path) + if walkErr != nil { + return "", skipped, walkErr + } + if closeErr != nil { + return "", skipped, closeErr + } + return "", skipped, fileCloseErr + } + return path, skipped, nil +} + +func shouldExcludeHomePath(rel string) bool { + first := strings.Split(filepath.ToSlash(rel), "/")[0] + switch first { + case ".cache", ".npm", ".cargo", ".rustup", ".local", "node_modules", ".docker": + return true + default: + return false + } +} + +func rejectSymlinkPath(root, target string) error { + rel, err := filepath.Rel(root, target) + if err != nil || rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return fmt.Errorf("snapshot target is outside home") + } + current := root + for _, part := range strings.Split(rel, string(filepath.Separator)) { + current = filepath.Join(current, part) + info, statErr := os.Lstat(current) + if os.IsNotExist(statErr) { + continue + } + if statErr != nil { + return statErr + } + if info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("snapshot target traverses symlink: %s", rel) + } + } + return nil +} From c43c31faa0ccf72a604862a42d7d9691c2210c07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 22:10:10 +0000 Subject: [PATCH 20/28] fix: re-read container state after wake so restored sessions accept prompts VmAgentContainer.proxyHttp/fetch read the container state once before wakeFromSnapshot(), then applied the stopped -> 410 guard using that stale pre-wake state. A freshly-woken, restored container was rejected with 410 (surfaced by the Worker as a generic 500), so users could restore state but not continue chatting after wake. Re-read the state after a successful wake. Adds a discriminating regression test that fails on the pre-fix code. Co-Authored-By: Claude Opus 4.6 --- .../src/durable-objects/vm-agent-container.ts | 12 ++- .../vm-agent-container-wake-state.test.ts | 96 +++++++++++++++++++ 2 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 apps/api/tests/unit/durable-objects/vm-agent-container-wake-state.test.ts diff --git a/apps/api/src/durable-objects/vm-agent-container.ts b/apps/api/src/durable-objects/vm-agent-container.ts index d2f349fd4..0046dbf7f 100644 --- a/apps/api/src/durable-objects/vm-agent-container.ts +++ b/apps/api/src/durable-objects/vm-agent-container.ts @@ -109,13 +109,18 @@ export class VmAgentContainer extends Container { } async proxyHttp(request: Request, port?: number): Promise { - const state = await this.getState(); + let state = await this.getState(); const lifecycleStatus = await this.ctx.storage.get('lifecycleStatus'); if (lifecycleStatus === 'sleeping') { const wake = await this.wakeFromSnapshot(); if (!wake.ok) { return new Response(wake.message || WAKE_DEGRADED_RESPONSE, { status: 503 }); } + // wakeFromSnapshot launched a fresh container and restored the session. + // Re-read the container state so the stopped-check below reflects the + // now-running container instead of the pre-wake stopped snapshot, + // otherwise a successfully-woken session is wrongly rejected with 410. + state = await this.getState(); } if (state.status === 'stopped' || state.status === 'stopped_with_code') { return new Response('Container is stopped; create a new instant session.', { status: 410 }); @@ -234,13 +239,16 @@ export class VmAgentContainer extends Container { } override async fetch(request: Request): Promise { - const state = await this.getState(); + let state = await this.getState(); const lifecycleStatus = await this.ctx.storage.get('lifecycleStatus'); if (lifecycleStatus === 'sleeping') { const wake = await this.wakeFromSnapshot(); if (!wake.ok) { return new Response(wake.message || WAKE_DEGRADED_RESPONSE, { status: 503 }); } + // Re-read the container state after wake so the stopped-check reflects + // the freshly-launched container, not the pre-wake stopped snapshot. + state = await this.getState(); } if (state.status === 'stopped' || state.status === 'stopped_with_code') { return new Response('Container is stopped; create a new instant session.', { status: 410 }); diff --git a/apps/api/tests/unit/durable-objects/vm-agent-container-wake-state.test.ts b/apps/api/tests/unit/durable-objects/vm-agent-container-wake-state.test.ts new file mode 100644 index 000000000..536514da8 --- /dev/null +++ b/apps/api/tests/unit/durable-objects/vm-agent-container-wake-state.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { VmAgentContainer } from '../../../src/durable-objects/vm-agent-container'; + +// Regression test for the restored-session prompt failure: `proxyHttp` read the +// container state ONCE before `wakeFromSnapshot()`, then applied the +// `stopped`/`stopped_with_code` -> 410 guard using that stale pre-wake state. +// A freshly-woken, restored container was therefore rejected with 410 (surfaced +// by the Worker as a generic 500), even though restore succeeded. The fix +// re-reads the container state after a successful wake. + +interface FakeState { + status: string; +} + +function makeFake(opts: { + statuses: string[]; // sequence returned by getState() + lifecycleStatus: string; + wakeOk: boolean; +}) { + const getState = vi.fn<[], Promise>(); + for (const s of opts.statuses) { + getState.mockResolvedValueOnce({ status: s }); + } + getState.mockResolvedValue({ status: opts.statuses[opts.statuses.length - 1] }); + + const containerFetch = vi + .fn() + .mockResolvedValue(new Response('proxied', { status: 200 })); + const wakeFromSnapshot = vi + .fn() + .mockResolvedValue(opts.wakeOk ? { ok: true } : { ok: false, message: 'degraded' }); + + const fake = { + getState, + containerFetch, + wakeFromSnapshot, + defaultPort: 8080, + ctx: { storage: { get: vi.fn().mockResolvedValue(opts.lifecycleStatus) } }, + }; + return { fake, getState, containerFetch, wakeFromSnapshot }; +} + +function callProxyHttp(fake: unknown, request: Request): Promise { + return (VmAgentContainer.prototype as unknown as { + proxyHttp: (this: unknown, request: Request, port?: number) => Promise; + }).proxyHttp.call(fake, request); +} + +describe('VmAgentContainer.proxyHttp wake state re-read', () => { + it('proxies the prompt after a successful wake even though the pre-wake state was stopped', async () => { + // Pre-wake getState -> stopped; post-wake getState -> running (fresh container). + const { fake, getState, containerFetch, wakeFromSnapshot } = makeFake({ + statuses: ['stopped', 'running'], + lifecycleStatus: 'sleeping', + wakeOk: true, + }); + + const res = await callProxyHttp(fake, new Request('http://container/prompt', { method: 'POST' })); + + expect(wakeFromSnapshot).toHaveBeenCalledTimes(1); + // State must be re-read after wake (once before, once after) so the stopped + // guard sees the now-running container. + expect(getState).toHaveBeenCalledTimes(2); + // The request is proxied to the running container, NOT rejected with 410. + expect(containerFetch).toHaveBeenCalledTimes(1); + expect(res.status).toBe(200); + }); + + it('returns 503 (not 410/proxy) when wake fails', async () => { + const { fake, containerFetch } = makeFake({ + statuses: ['stopped', 'stopped'], + lifecycleStatus: 'sleeping', + wakeOk: false, + }); + + const res = await callProxyHttp(fake, new Request('http://container/prompt', { method: 'POST' })); + + expect(res.status).toBe(503); + expect(containerFetch).not.toHaveBeenCalled(); + }); + + it('still returns 410 for a genuinely stopped, non-sleeping container', async () => { + const { fake, containerFetch, wakeFromSnapshot } = makeFake({ + statuses: ['stopped'], + lifecycleStatus: 'running', + wakeOk: true, + }); + + const res = await callProxyHttp(fake, new Request('http://container/prompt', { method: 'POST' })); + + expect(wakeFromSnapshot).not.toHaveBeenCalled(); + expect(containerFetch).not.toHaveBeenCalled(); + expect(res.status).toBe(410); + }); +}); From b0bb139b608b67e94a9b3e2535aeff8321d7275c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 22:43:44 +0000 Subject: [PATCH 21/28] fix: use node-scoped callback token when waking restored container wakeFromSnapshot launched the restored container with a workspace-scoped callback token (signCallbackToken) passed as the container CALLBACK_TOKEN, but the vm-agent uses it for node callbacks (error/activity/message reporting) which reject workspace-scoped tokens with 403 "Insufficient token scope". Restored sessions therefore accepted a prompt (200) but silently failed to report the agent's reply, so no answer appeared after wake. Sign a node-scoped token (signNodeCallbackToken) to match the initial launch in launchInstantSession. Diagnosed from staging Worker logs (node_auth.rejected_workspace_scoped_token). Co-Authored-By: Claude Opus 4.6 --- apps/api/src/durable-objects/vm-agent-container.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/api/src/durable-objects/vm-agent-container.ts b/apps/api/src/durable-objects/vm-agent-container.ts index 0046dbf7f..cc1b18a67 100644 --- a/apps/api/src/durable-objects/vm-agent-container.ts +++ b/apps/api/src/durable-objects/vm-agent-container.ts @@ -5,7 +5,7 @@ import { drizzle } from 'drizzle-orm/d1'; import * as schema from '../db/schema'; import type { Env } from '../env'; import { log } from '../lib/logger'; -import { signCallbackToken, signNodeManagementToken } from '../services/jwt'; +import { signNodeCallbackToken, signNodeManagementToken } from '../services/jwt'; export const DEFAULT_CF_CONTAINER_SLEEP_AFTER = '1h'; export const DEFAULT_CF_CONTAINER_ACTIVE_WORK_MAX_MS = 2 * 60 * 60 * 1000; @@ -345,7 +345,13 @@ export class VmAgentContainer extends Container { }); await this.ctx.storage.put('lifecycleStatus', 'launching' satisfies LifecycleStatus); - const callbackToken = await signCallbackToken(config.workspaceId, this.env); + // The container's CALLBACK_TOKEN must be node-scoped to match the initial + // launch (see launchInstantSession): the vm-agent uses it for node callbacks + // (error/activity/message reporting) which reject workspace-scoped tokens. + // Using a workspace-scoped token here caused restored sessions to accept a + // prompt (200) but silently fail to report the agent's reply back (403 + // "Insufficient token scope"), so no answer appeared after wake. + const callbackToken = await signNodeCallbackToken(config.nodeId, this.env); await this.launch(config, { nodeCallbackToken: callbackToken }); const { token } = await signNodeManagementToken(workspace.userId, config.nodeId, config.workspaceId, this.env); From a7818bf443882c4565ad8d2b4ddfe4c86acf9560 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 23:05:01 +0000 Subject: [PATCH 22/28] fix: set workspace-scoped runtime callback token on wake so replies persist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The message reporter and snapshot callbacks use the workspace-scoped runtime.CallbackToken (workspaceCallbackToken), which has NO node-scoped fallback. A freshly-woken container never runs create-workspace, so that token was unset — the reporter hit "no auth token" and chat replies were silently discarded (restored sessions accepted a prompt but never answered). The DO now signs a workspace-scoped callback token and passes it on the restore request; the vm-agent persists it via upsertWorkspaceRuntime before resolving the snapshot callback token. Combined with the node-scoped CALLBACK_TOKEN, this matches the normal launch (node token for node callbacks + workspace token for message/snapshot callbacks). Diagnosed from staging Worker logs: workspace_auth.rejected_node_scoped_token on /session-snapshot/restore and reporter "no auth token". Co-Authored-By: Claude Opus 4.6 --- .../api/src/durable-objects/vm-agent-container.ts | 9 ++++++++- .../vm-agent/internal/server/session_snapshot.go | 15 ++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/apps/api/src/durable-objects/vm-agent-container.ts b/apps/api/src/durable-objects/vm-agent-container.ts index cc1b18a67..a92a206e5 100644 --- a/apps/api/src/durable-objects/vm-agent-container.ts +++ b/apps/api/src/durable-objects/vm-agent-container.ts @@ -5,7 +5,7 @@ import { drizzle } from 'drizzle-orm/d1'; import * as schema from '../db/schema'; import type { Env } from '../env'; import { log } from '../lib/logger'; -import { signNodeCallbackToken, signNodeManagementToken } from '../services/jwt'; +import { signCallbackToken, signNodeCallbackToken, signNodeManagementToken } from '../services/jwt'; export const DEFAULT_CF_CONTAINER_SLEEP_AFTER = '1h'; export const DEFAULT_CF_CONTAINER_ACTIVE_WORK_MAX_MS = 2 * 60 * 60 * 1000; @@ -354,6 +354,12 @@ export class VmAgentContainer extends Container { const callbackToken = await signNodeCallbackToken(config.nodeId, this.env); await this.launch(config, { nodeCallbackToken: callbackToken }); + // The fresh container never ran create-workspace, so its workspace-scoped + // runtime.CallbackToken is unset. The message reporter and snapshot + // callbacks require it (they do NOT fall back to the node-scoped token), so + // pass it on the restore request; without it, restored sessions accept a + // prompt but silently discard the agent's reply ("no auth token"). + const workspaceCallbackToken = await signCallbackToken(config.workspaceId, this.env); const { token } = await signNodeManagementToken(workspace.userId, config.nodeId, config.workspaceId, this.env); const restoreUrl = new URL(`http://localhost:${config.vmAgentPort}/workspaces/${config.workspaceId}/agent-sessions/${agentSession.id}/restore`); const restoreResponse = await this.containerFetch( @@ -369,6 +375,7 @@ export class VmAgentContainer extends Container { chatSessionId: workspace.chatSessionId, runtime: 'cf-container', agentType: agentSession.agentType, + workspaceCallbackToken, }), }), config.vmAgentPort diff --git a/packages/vm-agent/internal/server/session_snapshot.go b/packages/vm-agent/internal/server/session_snapshot.go index f5bfd51c5..8455eb25e 100644 --- a/packages/vm-agent/internal/server/session_snapshot.go +++ b/packages/vm-agent/internal/server/session_snapshot.go @@ -120,9 +120,10 @@ func (s *Server) sessionSnapshotHandlerInput(w http.ResponseWriter, r *http.Requ return nil, false } var body struct { - ChatSessionID string `json:"chatSessionId"` - Runtime string `json:"runtime"` - AgentType string `json:"agentType"` + ChatSessionID string `json:"chatSessionId"` + Runtime string `json:"runtime"` + AgentType string `json:"agentType"` + WorkspaceCallbackToken string `json:"workspaceCallbackToken"` } if err := json.NewDecoder(r.Body).Decode(&body); err != nil { writeError(w, http.StatusBadRequest, "invalid request body") @@ -133,6 +134,14 @@ func (s *Server) sessionSnapshotHandlerInput(w http.ResponseWriter, r *http.Requ writeError(w, http.StatusBadRequest, "chatSessionId is required") return nil, false } + // A freshly-woken container never ran create-workspace, so its + // runtime.CallbackToken (the workspace-scoped token used by the message + // reporter and the snapshot callbacks) is unset. Persist the token the + // control plane provides on the restore request so chat replies and + // snapshot callbacks can authenticate after a wake. + if wsToken := strings.TrimSpace(body.WorkspaceCallbackToken); wsToken != "" { + s.upsertWorkspaceRuntime(workspaceID, "", "", "", wsToken) + } runtime, ok := s.getWorkspaceRuntime(workspaceID) if !ok { writeError(w, http.StatusNotFound, "workspace not found") From 795acad18b5af6f8a2f52fcada3d1a83190def09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 23:27:30 +0000 Subject: [PATCH 23/28] fix: prime message reporter on session restore so wake replies persist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The normal agent-session-create path (handleCreateAgentSession) sets runtime.ProjectID and creates/binds the per-workspace message reporter via getOrCreateReporter before the agent runs. The restore path only recreated the session host + SelectAgent, so the restored agent's streamed output had no reporter to enqueue to — replies were generated (ACP prompt completed) but never POSTed to /messages, so no answer appeared after wake. Restore now primes the reporter with the workspace's projectID (from the runtime or PROJECT_ID env) and the restored chatSessionID. Diagnosed from staging Worker logs: message POSTs present during setup, zero after wake despite ACP prompt completing with no auth errors. Co-Authored-By: Claude Opus 4.6 --- .../internal/server/session_snapshot.go | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/packages/vm-agent/internal/server/session_snapshot.go b/packages/vm-agent/internal/server/session_snapshot.go index 8455eb25e..cd021bf5d 100644 --- a/packages/vm-agent/internal/server/session_snapshot.go +++ b/packages/vm-agent/internal/server/session_snapshot.go @@ -305,6 +305,11 @@ func (s *Server) restoreSessionSnapshot(ctx context.Context, runtime *WorkspaceR if strings.TrimSpace(agentType) == "" { return nil, fmt.Errorf("agent type is required to restore a standalone session") } + // Prime the per-workspace message reporter before the agent starts. + // handleCreateAgentSession does this on the normal path; the restore path + // skipped it, so the restored agent's output had no reporter to enqueue + // to and chat replies were silently dropped after a wake. + s.primeRestoredMessageReporter(runtime, chatSessionID) session, _, createErr := s.agentSessions.Create(runtime.ID, sessionID, "Restored session", "restore:"+sessionID) if createErr != nil { return nil, fmt.Errorf("recreate restored agent session: %w", createErr) @@ -323,6 +328,33 @@ func (s *Server) restoreSessionSnapshot(ctx context.Context, runtime *WorkspaceR return map[string]interface{}{"status": "restored", "degradation": restore.Degradation}, nil } +// primeRestoredMessageReporter ensures the per-workspace message reporter exists +// and is bound to the restored chat session before the agent starts producing +// output. handleCreateAgentSession does this on the normal path; the restore +// path must replicate it or the restored agent's replies are never enqueued and +// are silently dropped after a wake. +func (s *Server) primeRestoredMessageReporter(runtime *WorkspaceRuntime, chatSessionID string) { + if runtime == nil { + return + } + chatSessionID = strings.TrimSpace(chatSessionID) + projectID := strings.TrimSpace(runtime.ProjectID) + if projectID == "" { + projectID = strings.TrimSpace(s.config.ProjectID) + } + if projectID == "" || chatSessionID == "" { + return + } + s.workspaceMu.Lock() + if rt, ok := s.workspaces[runtime.ID]; ok && strings.TrimSpace(rt.ProjectID) == "" { + rt.ProjectID = projectID + } + s.workspaceMu.Unlock() + if reporter := s.getOrCreateReporter(runtime.ID, projectID, chatSessionID); reporter != nil { + reporter.SetSessionID(chatSessionID) + } +} + func (s *Server) prepareSnapshot(ctx context.Context, workspaceID, sessionID, chatSessionID, runtimeName, token string) (*snapshotPrepareResponse, error) { payload := map[string]string{"chatSessionId": chatSessionID, "agentSessionId": sessionID, "runtime": runtimeName} var out snapshotPrepareResponse From 023eb65b01730cfd3c42abe63b18dabea4d04e1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 23:51:56 +0000 Subject: [PATCH 24/28] refactor: extract node diagnostic getters to keep node-agent under 800 lines --- .../api/src/routes/deployment-environments.ts | 2 +- apps/api/src/routes/mcp/deployment-tools.ts | 2 +- apps/api/src/routes/nodes.ts | 8 ++-- .../src/services/node-agent-diagnostics.ts | 38 +++++++++++++++++++ apps/api/src/services/node-agent.ts | 35 ----------------- 5 files changed, 45 insertions(+), 40 deletions(-) create mode 100644 apps/api/src/services/node-agent-diagnostics.ts diff --git a/apps/api/src/routes/deployment-environments.ts b/apps/api/src/routes/deployment-environments.ts index e73c22ff5..11a7c8945 100644 --- a/apps/api/src/routes/deployment-environments.ts +++ b/apps/api/src/routes/deployment-environments.ts @@ -37,7 +37,7 @@ import { getNodeLogsFromNode, getNodeSystemInfoFromNode, listNodeContainersFromNode, -} from '../services/node-agent'; +} from '../services/node-agent-diagnostics'; import { deleteNodeResources } from '../services/nodes'; import { registerDeploymentEnvironmentLifecycleRoutes } from './deployment-environment-lifecycle'; diff --git a/apps/api/src/routes/mcp/deployment-tools.ts b/apps/api/src/routes/mcp/deployment-tools.ts index 55d7ba2d4..2ee0c5564 100644 --- a/apps/api/src/routes/mcp/deployment-tools.ts +++ b/apps/api/src/routes/mcp/deployment-tools.ts @@ -27,7 +27,7 @@ import { type DeploymentRouteTargetOptions, } from '../../services/deployment-routing'; import { getRuntimeLimits } from '../../services/limits'; -import { getNodeLogsFromNode } from '../../services/node-agent'; +import { getNodeLogsFromNode } from '../../services/node-agent-diagnostics'; import { byteLength, PROJECT_ENV_KEY_PATTERN } from '../projects/_helpers'; import { INTERNAL_ERROR, diff --git a/apps/api/src/routes/nodes.ts b/apps/api/src/routes/nodes.ts index c1374f03b..93ceedd4b 100644 --- a/apps/api/src/routes/nodes.ts +++ b/apps/api/src/routes/nodes.ts @@ -23,13 +23,15 @@ import { getRuntimeLimits } from '../services/limits'; import { fetchNodeAgent, getNodeAgentRequestTimeoutMs, - getNodeLogsFromNode, - getNodeSystemInfoFromNode, - listNodeContainersFromNode, listNodeEventsOnNode, nodeAgentRawRequest, stopWorkspaceOnNode, } from '../services/node-agent'; +import { + getNodeLogsFromNode, + getNodeSystemInfoFromNode, + listNodeContainersFromNode, +} from '../services/node-agent-diagnostics'; import { createNodeRecord, deleteNodeResources, provisionNode, stopNodeResources } from '../services/nodes'; import { recordNodeRoutingMetric } from '../services/telemetry'; diff --git a/apps/api/src/services/node-agent-diagnostics.ts b/apps/api/src/services/node-agent-diagnostics.ts new file mode 100644 index 000000000..83a0bdecb --- /dev/null +++ b/apps/api/src/services/node-agent-diagnostics.ts @@ -0,0 +1,38 @@ +import type { Env } from '../env'; + +import { nodeAgentRequest } from './node-agent'; + +export async function getNodeSystemInfoFromNode( + nodeId: string, + env: Env, + userId: string +): Promise { + return nodeAgentRequest(nodeId, env, '/system-info', { + method: 'GET', + userId, + }); +} + +export async function getNodeLogsFromNode( + nodeId: string, + env: Env, + userId: string, + queryString: string +): Promise { + const path = queryString ? `/logs?${queryString}` : '/logs'; + return nodeAgentRequest(nodeId, env, path, { + method: 'GET', + userId, + }); +} + +export async function listNodeContainersFromNode( + nodeId: string, + env: Env, + userId: string +): Promise { + return nodeAgentRequest(nodeId, env, '/containers', { + method: 'GET', + userId, + }); +} diff --git a/apps/api/src/services/node-agent.ts b/apps/api/src/services/node-agent.ts index 998174b31..4e5c76389 100644 --- a/apps/api/src/services/node-agent.ts +++ b/apps/api/src/services/node-agent.ts @@ -662,41 +662,6 @@ export async function listNodeEventsOnNode( }; } -export async function getNodeSystemInfoFromNode( - nodeId: string, - env: Env, - userId: string -): Promise { - return nodeAgentRequest(nodeId, env, '/system-info', { - method: 'GET', - userId, - }); -} - -export async function getNodeLogsFromNode( - nodeId: string, - env: Env, - userId: string, - queryString: string -): Promise { - const path = queryString ? `/logs?${queryString}` : '/logs'; - return nodeAgentRequest(nodeId, env, path, { - method: 'GET', - userId, - }); -} - -export async function listNodeContainersFromNode( - nodeId: string, - env: Env, - userId: string -): Promise { - return nodeAgentRequest(nodeId, env, '/containers', { - method: 'GET', - userId, - }); -} - /** * Raw binary proxy to a VM agent endpoint. * Returns the raw Response (not parsed as JSON) so callers can stream the body. From fbeb138f258334383ea54e0e7efb4c0031dcdd01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 23:57:16 +0000 Subject: [PATCH 25/28] style: sort imports in node-agent-diagnostics --- apps/api/src/services/node-agent-diagnostics.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/api/src/services/node-agent-diagnostics.ts b/apps/api/src/services/node-agent-diagnostics.ts index 83a0bdecb..a8b27f6a6 100644 --- a/apps/api/src/services/node-agent-diagnostics.ts +++ b/apps/api/src/services/node-agent-diagnostics.ts @@ -1,5 +1,4 @@ import type { Env } from '../env'; - import { nodeAgentRequest } from './node-agent'; export async function getNodeSystemInfoFromNode( From a291d4fca864bb302665f24eac8ea4763b3c8689 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sun, 12 Jul 2026 00:31:36 +0000 Subject: [PATCH 26/28] harden wake/restore per specialist review: mutex, tests, defensive logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the CRITICAL/HIGH findings from the PR #1562 specialist review: - Concurrency: serialize wake-from-snapshot behind a promise-chain mutex (ensureAwake) so two concurrent requests to a sleeping container don't both launch+restore; re-reads lifecycleStatus inside the critical section (.claude/rules/45). Adds a discriminating concurrency regression test. - Tests: add Go unit tests locking in the two silent-failure invariants — the restore-body workspace token is persisted (callbackTokenForWorkspace / workspaceCallbackToken) and the message reporter is primed on restore. - Defensive logging: primeRestoredMessageReporter now warns (instead of silently no-oping) when project/chat-session context is missing. Remaining LOW/pre-existing hardening tracked in tasks/backlog/2026-07-12-cf-container-wake-restore-hardening.md. Co-Authored-By: Claude Opus 4.6 --- .../src/durable-objects/vm-agent-container.ts | 33 +++++++- .../vm-agent-container-wake-state.test.ts | 45 +++++++++++ .../internal/server/session_snapshot.go | 6 ++ .../server/session_snapshot_reporter_test.go | 75 +++++++++++++++++++ ...-12-cf-container-wake-restore-hardening.md | 28 +++++++ 5 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 packages/vm-agent/internal/server/session_snapshot_reporter_test.go create mode 100644 tasks/backlog/2026-07-12-cf-container-wake-restore-hardening.md diff --git a/apps/api/src/durable-objects/vm-agent-container.ts b/apps/api/src/durable-objects/vm-agent-container.ts index a92a206e5..f80c6ca32 100644 --- a/apps/api/src/durable-objects/vm-agent-container.ts +++ b/apps/api/src/durable-objects/vm-agent-container.ts @@ -54,6 +54,12 @@ export class VmAgentContainer extends Container { sleepAfter = DEFAULT_CF_CONTAINER_SLEEP_AFTER; enableInternet = true; + // Serializes wake-from-snapshot so two concurrent requests to a sleeping + // container do not both launch a fresh container + restore. DO request + // handlers interleave across `await`, so the sleeping-check + wake must run + // as one critical section (see .claude/rules/45). + private wakeChain: Promise = Promise.resolve(); + constructor(ctx: DurableObjectState>, env: Env) { super(ctx, env); const configuredPort = Number.parseInt(env.CF_CONTAINER_VM_AGENT_PORT || env.SANDBOX_VM_AGENT_PORT || '', 10); @@ -112,7 +118,7 @@ export class VmAgentContainer extends Container { let state = await this.getState(); const lifecycleStatus = await this.ctx.storage.get('lifecycleStatus'); if (lifecycleStatus === 'sleeping') { - const wake = await this.wakeFromSnapshot(); + const wake = await this.ensureAwake(); if (!wake.ok) { return new Response(wake.message || WAKE_DEGRADED_RESPONSE, { status: 503 }); } @@ -242,7 +248,7 @@ export class VmAgentContainer extends Container { let state = await this.getState(); const lifecycleStatus = await this.ctx.storage.get('lifecycleStatus'); if (lifecycleStatus === 'sleeping') { - const wake = await this.wakeFromSnapshot(); + const wake = await this.ensureAwake(); if (!wake.ok) { return new Response(wake.message || WAKE_DEGRADED_RESPONSE, { status: 503 }); } @@ -310,6 +316,29 @@ export class VmAgentContainer extends Container { return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS; } + /** + * Wake a sleeping container exactly once under concurrency. Serializes on + * `wakeChain` and re-reads `lifecycleStatus` inside the critical section, so a + * second request that arrives while the first is waking sees `running` and + * skips a duplicate launch/restore instead of racing it (see rule 45). + */ + private async ensureAwake(): Promise<{ ok: boolean; message?: string }> { + const run = this.wakeChain.then(async () => { + const status = await this.ctx.storage.get('lifecycleStatus'); + if (status !== 'sleeping') { + return { ok: true }; + } + return this.wakeFromSnapshot(); + }); + // Keep the chain alive even if this wake rejects, so a failure does not + // permanently wedge all future wakes. + this.wakeChain = run.then( + () => undefined, + () => undefined + ); + return run; + } + private async wakeFromSnapshot(): Promise<{ ok: boolean; message?: string }> { const config = await this.ctx.storage.get('launchConfig'); if (!config) { diff --git a/apps/api/tests/unit/durable-objects/vm-agent-container-wake-state.test.ts b/apps/api/tests/unit/durable-objects/vm-agent-container-wake-state.test.ts index 536514da8..2c3028a55 100644 --- a/apps/api/tests/unit/durable-objects/vm-agent-container-wake-state.test.ts +++ b/apps/api/tests/unit/durable-objects/vm-agent-container-wake-state.test.ts @@ -36,6 +36,9 @@ function makeFake(opts: { containerFetch, wakeFromSnapshot, defaultPort: 8080, + wakeChain: Promise.resolve(), + ensureAwake: (VmAgentContainer.prototype as unknown as { ensureAwake: unknown }) + .ensureAwake, ctx: { storage: { get: vi.fn().mockResolvedValue(opts.lifecycleStatus) } }, }; return { fake, getState, containerFetch, wakeFromSnapshot }; @@ -94,3 +97,45 @@ describe('VmAgentContainer.proxyHttp wake state re-read', () => { expect(res.status).toBe(410); }); }); + +describe('VmAgentContainer.ensureAwake concurrency (rule 45)', () => { + it('wakes a sleeping container exactly once under two concurrent requests', async () => { + // Shared, mutable container state so the mock models a real wake: the first + // wake flips lifecycleStatus to running, and getState follows it. + const shared = { lifecycle: 'sleeping' as string }; + + const getState = vi.fn(async () => ({ + status: shared.lifecycle === 'running' ? 'running' : 'stopped', + })); + const containerFetch = vi.fn().mockResolvedValue(new Response('proxied', { status: 200 })); + const wakeFromSnapshot = vi.fn(async () => { + // Simulate the async launch+restore so the two requests interleave across + // this await; the second must observe the running state and NOT re-wake. + await new Promise((r) => setTimeout(r, 20)); + shared.lifecycle = 'running'; + return { ok: true }; + }); + + const fake = { + getState, + containerFetch, + wakeFromSnapshot, + defaultPort: 8080, + wakeChain: Promise.resolve(), + ensureAwake: (VmAgentContainer.prototype as unknown as { ensureAwake: unknown }).ensureAwake, + ctx: { storage: { get: vi.fn(async () => shared.lifecycle) } }, + }; + + const [a, b] = await Promise.all([ + callProxyHttp(fake, new Request('http://container/prompt', { method: 'POST' })), + callProxyHttp(fake, new Request('http://container/prompt', { method: 'POST' })), + ]); + + // The one-time launch+restore fired exactly once despite two concurrent + // requests; both requests were proxied to the now-running container. + expect(wakeFromSnapshot).toHaveBeenCalledTimes(1); + expect(containerFetch).toHaveBeenCalledTimes(2); + expect(a.status).toBe(200); + expect(b.status).toBe(200); + }); +}); diff --git a/packages/vm-agent/internal/server/session_snapshot.go b/packages/vm-agent/internal/server/session_snapshot.go index cd021bf5d..72c9e381e 100644 --- a/packages/vm-agent/internal/server/session_snapshot.go +++ b/packages/vm-agent/internal/server/session_snapshot.go @@ -9,6 +9,7 @@ import ( "encoding/json" "fmt" "io" + "log/slog" "net/http" "net/url" "os" @@ -343,6 +344,11 @@ func (s *Server) primeRestoredMessageReporter(runtime *WorkspaceRuntime, chatSes projectID = strings.TrimSpace(s.config.ProjectID) } if projectID == "" || chatSessionID == "" { + // Without a project + chat session the reporter cannot be created, so + // the restored agent's output would be silently dropped. Log loudly so + // this failure mode is diagnosable instead of invisible. + slog.Warn("Restored session message reporter not primed: missing project or chat session", + "workspaceId", runtime.ID, "hasProjectID", projectID != "", "hasChatSessionID", chatSessionID != "") return } s.workspaceMu.Lock() diff --git a/packages/vm-agent/internal/server/session_snapshot_reporter_test.go b/packages/vm-agent/internal/server/session_snapshot_reporter_test.go new file mode 100644 index 000000000..14aaaba1f --- /dev/null +++ b/packages/vm-agent/internal/server/session_snapshot_reporter_test.go @@ -0,0 +1,75 @@ +package server + +import "testing" + +// TestUpsertWorkspaceRuntime_PersistsWorkspaceCallbackToken covers the fix that +// lets a freshly-woken container authenticate its message + snapshot callbacks: +// the restore handler persists the workspace-scoped token from the request body +// via upsertWorkspaceRuntime, and it must then be returned by both +// callbackTokenForWorkspace (with node fallback) and workspaceCallbackToken +// (no fallback — used by the message reporter). +func TestUpsertWorkspaceRuntime_PersistsWorkspaceCallbackToken(t *testing.T) { + s, _ := newServerWithoutReporter(t) + s.config.CallbackToken = "node-scoped-token" + ws := "ws-token-persist-1" + + // A fresh wake container starts with no workspace-scoped token; the snapshot + // handler falls back to the node-scoped CALLBACK_TOKEN, and the no-fallback + // workspaceCallbackToken is empty (which is why message replies would drop). + s.upsertWorkspaceRuntime(ws, "org/repo", "main", "running", "") + if got := s.callbackTokenForWorkspace(ws); got != "node-scoped-token" { + t.Fatalf("callbackTokenForWorkspace before restore: expected node fallback, got %q", got) + } + if got := s.workspaceCallbackToken(ws); got != "" { + t.Fatalf("workspaceCallbackToken before restore: expected empty, got %q", got) + } + + // The restore handler persists the workspace-scoped token from the body. + s.upsertWorkspaceRuntime(ws, "", "", "", "workspace-scoped-token") + + if got := s.callbackTokenForWorkspace(ws); got != "workspace-scoped-token" { + t.Fatalf("callbackTokenForWorkspace after restore: expected workspace token, got %q", got) + } + if got := s.workspaceCallbackToken(ws); got != "workspace-scoped-token" { + t.Fatalf("workspaceCallbackToken after restore: expected workspace token, got %q", got) + } +} + +// TestPrimeRestoredMessageReporter_CreatesReporter covers the fix that makes a +// restored agent's replies actually persist: the restore path must create/bind +// the per-workspace message reporter (the normal create-agent-session path does +// this; restore previously skipped it and replies were silently dropped). +func TestPrimeRestoredMessageReporter_CreatesReporter(t *testing.T) { + s, _ := newServerWithoutReporter(t) + s.config.ProjectID = "proj-restore-1" // set from PROJECT_ID env at launch + ws := "ws-prime-1" + rt := s.upsertWorkspaceRuntime(ws, "org/repo", "main", "running", "ws-tok") + + if len(s.messageReporters) != 0 { + t.Fatalf("expected no reporter before priming, got %d", len(s.messageReporters)) + } + + s.primeRestoredMessageReporter(rt, "chat-sess-1") + + if _, ok := s.messageReporters[ws]; !ok { + t.Fatal("expected message reporter to be primed after restore") + } + if rt.ProjectID != "proj-restore-1" { + t.Fatalf("expected runtime.ProjectID to be set from config fallback, got %q", rt.ProjectID) + } +} + +// TestPrimeRestoredMessageReporter_NoOpWithoutProject verifies the guard: with +// no project context available (neither runtime nor config), the reporter is +// not created (and a warning is logged) rather than panicking. +func TestPrimeRestoredMessageReporter_NoOpWithoutProject(t *testing.T) { + s, _ := newServerWithoutReporter(t) // config.ProjectID is empty + ws := "ws-prime-2" + rt := s.upsertWorkspaceRuntime(ws, "org/repo", "main", "running", "ws-tok") + + s.primeRestoredMessageReporter(rt, "chat-sess-2") + + if _, ok := s.messageReporters[ws]; ok { + t.Fatal("expected no reporter when project context is unavailable") + } +} diff --git a/tasks/backlog/2026-07-12-cf-container-wake-restore-hardening.md b/tasks/backlog/2026-07-12-cf-container-wake-restore-hardening.md new file mode 100644 index 000000000..ab456623a --- /dev/null +++ b/tasks/backlog/2026-07-12-cf-container-wake-restore-hardening.md @@ -0,0 +1,28 @@ +# cf-container wake/restore hardening follow-ups + +Deferred, non-blocking hardening items raised by the specialist reviewers during +PR #1562 (cf-container session hibernate/wake/restore). None are regressions +introduced by #1562 — the feature is verified working end-to-end on both the +instant-container and VM paths. The CRITICAL/HIGH items from that review (wake +concurrency mutex per rule 45, and Go unit tests for the token-persist + +reporter-prime invariants) were fixed in #1562 itself. These remaining LOW / +pre-existing items are tracked here. + +## Context +- Origin: PR #1562 specialist review (go-specialist, cloudflare-specialist, security-auditor, test-engineer). +- Feature files: `apps/api/src/durable-objects/vm-agent-container.ts`, `packages/vm-agent/internal/server/session_snapshot.go`, `.../session_snapshot_archive.go`, `apps/api/src/services/session-snapshots.ts`. + +## Acceptance Criteria +- [ ] **Bound the DO-internal restore call with a wall-clock budget.** `wakeFromSnapshot`'s `containerFetch` to `/restore` is not bounded by an `AbortSignal`/timeout inside the DO (the Worker request carries `getCfContainerWakeTimeoutMs`, but a hung restore inside the DO can still block). Add an env-configurable deadline (`.claude/rules/47`). +- [ ] **Decouple restore agent-start from the HTTP request context.** `restoreSessionSnapshot` passes the request `ctx` into `SelectAgent`; a proxy timeout / client disconnect could abort a cold agent install mid-flight. Use a job-owned `context.WithTimeout(context.Background(), )` (`.claude/rules/43`). +- [ ] **Add a decompression size limit** (`io.LimitReader` with configurable max) around the tar stream in `downloadAndExtractTar` to prevent decompression bombs. +- [ ] **Filter/truncate `restoreBody`** before it is passed to `markWakeDegraded` (persisted to D1 `errorMessage`) and returned in the 503 response, so future vm-agent error-message changes cannot leak sensitive content. +- [ ] **Decouple node-management token TTL** from `TERMINAL_TOKEN_EXPIRY_MS`: introduce `NODE_MANAGEMENT_TOKEN_EXPIRY_MS` / `getNodeManagementTokenExpiry` (default 1h). +- [ ] **Validate hostname in `absoluteControlPlaneURL`** to restrict absolute-URL passthrough to the expected SAM domain. +- [ ] **Break the pre-existing circular import**: `node-agent.ts` re-exports `hibernateAgentSessionOnNode`/`restoreAgentSessionOnNode` from `node-agent-session-snapshots`; have callers import directly (mirrors the `node-agent-diagnostics.ts` split done in #1562). +- [ ] **`markWakeDegraded` node status accuracy**: if the container launched before restore failed, reflect the real container state rather than leaving it stale. +- [ ] **Thread `ctx` through `skipOversizedUntracked`** (currently uses `context.Background()` for `git check-ignore`, ignoring cancellation). Pre-existing. +- [ ] **Add an HTTP-level Go test** for `sessionSnapshotHandlerInput` (with a valid node-management JWT) asserting a restore request with `workspaceCallbackToken` in the body persists it — complements the invariant-level test added in #1562. + +## Notes +- The security review's checklist (token scopes, no-leak, restore-endpoint auth, no scope escalation, TTLs) all PASSED for #1562; these are additive defense-in-depth. From deb64fcff95f9dd787f657060899e7e7f6ea2a6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sun, 12 Jul 2026 00:52:36 +0000 Subject: [PATCH 27/28] test: update mocks for node-agent-diagnostics split and ensureAwake rename --- apps/api/tests/unit/cf-container-runtime-contract.test.ts | 2 +- apps/api/tests/unit/mcp-deployment-tools.test.ts | 6 ++++++ .../routes/deployment-environment-observability.test.ts | 6 ++++++ .../tests/unit/routes/deployment-membership-auth.test.ts | 6 ++++++ apps/api/tests/unit/routes/node-observability-logs.test.ts | 6 ++++++ 5 files changed, 25 insertions(+), 1 deletion(-) diff --git a/apps/api/tests/unit/cf-container-runtime-contract.test.ts b/apps/api/tests/unit/cf-container-runtime-contract.test.ts index 1140cc9fc..6e3a86943 100644 --- a/apps/api/tests/unit/cf-container-runtime-contract.test.ts +++ b/apps/api/tests/unit/cf-container-runtime-contract.test.ts @@ -133,7 +133,7 @@ describe('cf-container runtime spike contracts', () => { expect(containerDo).toContain("await this.ctx.storage.put('lifecycleStatus', 'sleeping' satisfies LifecycleStatus)"); expect(containerDo).toContain("status: 'sleeping'"); expect(containerDo).toContain("if (lifecycleStatus === 'sleeping')"); - expect(containerDo).toContain('const wake = await this.wakeFromSnapshot()'); + expect(containerDo).toContain('const wake = await this.ensureAwake()'); expect(containerDo).toContain('WAKE_DEGRADED_RESPONSE'); expect(containerDo).toContain("await this.ctx.storage.put('lifecycleStatus', 'launching' satisfies LifecycleStatus)"); expect(containerDo).toContain("await this.ctx.storage.put('lifecycleStatus', 'running' satisfies LifecycleStatus)"); diff --git a/apps/api/tests/unit/mcp-deployment-tools.test.ts b/apps/api/tests/unit/mcp-deployment-tools.test.ts index 6c7b11c77..3895bdff4 100644 --- a/apps/api/tests/unit/mcp-deployment-tools.test.ts +++ b/apps/api/tests/unit/mcp-deployment-tools.test.ts @@ -31,6 +31,12 @@ vi.mock('../../src/services/node-agent', () => ({ getNodeLogsFromNode: (...args: unknown[]) => mockGetNodeLogsFromNode(...args), })); +vi.mock('../../src/services/node-agent-diagnostics', () => ({ + getNodeLogsFromNode: (...args: unknown[]) => mockGetNodeLogsFromNode(...args), + getNodeSystemInfoFromNode: vi.fn(), + listNodeContainersFromNode: vi.fn(), +})); + const schema = await import('../../src/db/schema'); const { handleCreateDeploymentEnvironment, diff --git a/apps/api/tests/unit/routes/deployment-environment-observability.test.ts b/apps/api/tests/unit/routes/deployment-environment-observability.test.ts index 6ed76e1ff..1cfe3279a 100644 --- a/apps/api/tests/unit/routes/deployment-environment-observability.test.ts +++ b/apps/api/tests/unit/routes/deployment-environment-observability.test.ts @@ -76,6 +76,12 @@ vi.mock('../../../src/services/node-agent', () => ({ mockTeardownDeploymentEnvironmentOnNode(...args), })); +vi.mock('../../../src/services/node-agent-diagnostics', () => ({ + getNodeLogsFromNode: (...args: unknown[]) => mockGetNodeLogsFromNode(...args), + getNodeSystemInfoFromNode: (...args: unknown[]) => mockGetNodeSystemInfoFromNode(...args), + listNodeContainersFromNode: (...args: unknown[]) => mockListNodeContainersFromNode(...args), +})); + vi.mock('../../../src/services/deployment-control', () => ({ DEPLOYMENT_ENVIRONMENT_NAME_RE: /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/, encodeAllowedDeployProfileIds: vi.fn(() => null), diff --git a/apps/api/tests/unit/routes/deployment-membership-auth.test.ts b/apps/api/tests/unit/routes/deployment-membership-auth.test.ts index 659c7b930..ba6fc364b 100644 --- a/apps/api/tests/unit/routes/deployment-membership-auth.test.ts +++ b/apps/api/tests/unit/routes/deployment-membership-auth.test.ts @@ -143,6 +143,12 @@ vi.mock('../../../src/services/node-agent', () => ({ teardownDeploymentEnvironmentOnNode: vi.fn(), })); +vi.mock('../../../src/services/node-agent-diagnostics', () => ({ + getNodeLogsFromNode: (...args: unknown[]) => mockGetNodeLogsFromNode(...args), + getNodeSystemInfoFromNode: vi.fn(), + listNodeContainersFromNode: vi.fn(), +})); + vi.mock('../../../src/services/deployment-provisioning', () => ({ provisionDeploymentNode: vi.fn(), })); diff --git a/apps/api/tests/unit/routes/node-observability-logs.test.ts b/apps/api/tests/unit/routes/node-observability-logs.test.ts index 83635c204..54ce1e2e4 100644 --- a/apps/api/tests/unit/routes/node-observability-logs.test.ts +++ b/apps/api/tests/unit/routes/node-observability-logs.test.ts @@ -26,6 +26,12 @@ vi.mock('../../../src/services/node-agent', () => ({ stopWorkspaceOnNode: vi.fn(), })); +vi.mock('../../../src/services/node-agent-diagnostics', () => ({ + getNodeLogsFromNode: (...args: unknown[]) => mockGetNodeLogsFromNode(...args), + listNodeContainersFromNode: (...args: unknown[]) => mockListNodeContainersFromNode(...args), + getNodeSystemInfoFromNode: vi.fn(), +})); + vi.mock('../../../src/services/nodes', () => ({ createNodeRecord: vi.fn(), deleteNodeResources: vi.fn(), From b16496a3356a55a998693a6e3960bbc9d0559f45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sun, 12 Jul 2026 01:17:03 +0000 Subject: [PATCH 28/28] ci: re-trigger checks after removing needs-human-review label