From d678eddd1c5b8148e7525855d414ad651550d504 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 15:05:40 +0000 Subject: [PATCH 01/11] task: start taskrunner lifecycle reconciliation --- .../2026-07-11-taskrunner-d1-lifecycle-reconciliation.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tasks/{backlog => active}/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md (100%) diff --git a/tasks/backlog/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md b/tasks/active/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md similarity index 100% rename from tasks/backlog/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md rename to tasks/active/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md From a380e2f46ffdc7472421e9b49c21b33cb92de35c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 15:16:25 +0000 Subject: [PATCH 02/11] fix: reconcile dead taskrunner lifecycle state --- apps/api/.env.example | 3 + .../task-runner/state-machine.ts | 18 +- apps/api/src/env.ts | 3 + apps/api/src/routes/tasks/callback.ts | 15 ++ apps/api/src/scheduled/stuck-tasks.ts | 181 +++++++++++------- apps/api/tests/unit/stuck-tasks.test.ts | 39 ++-- .../workers/scheduled-stuck-tasks.test.ts | 82 +++++++- .../docs/docs/reference/configuration.md | 3 + packages/shared/src/constants/index.ts | 3 + .../shared/src/constants/task-execution.ts | 9 + ...-taskrunner-d1-lifecycle-reconciliation.md | 20 +- 11 files changed, 278 insertions(+), 98 deletions(-) diff --git a/apps/api/.env.example b/apps/api/.env.example index 7fb68a852..e7397672d 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -346,6 +346,9 @@ BASE_DOMAIN=workspaces.example.com # TASK_RUN_MAX_EXECUTION_MS=14400000 # 4 hours — max time before task is failed # TASK_STUCK_QUEUED_TIMEOUT_MS=600000 # 10 minutes — max time in 'queued' state # TASK_STUCK_DELEGATED_TIMEOUT_MS=1860000 # 31 minutes — max time in 'delegated' state (> workspace ready timeout) +# TASK_DO_MISMATCH_GRACE_MS=300000 # 5 minutes — minimum age before DO/D1 liveness reconciliation +# STUCK_TASK_MAX_CANDIDATES_PER_SWEEP=100 # Bound task rows inspected per cron invocation +# TASK_LIVENESS_MAX_ACP_SESSIONS=5 # Bound task-scoped ACP sessions inspected per candidate # CLAUDE_CODE_COMPACTION_LOOP_DETECTOR_ENABLED=true # CLAUDE_CODE_COMPACTION_LOOP_RECENT_MESSAGE_LIMIT=40 # CLAUDE_CODE_COMPACTION_LOOP_WINDOW_MESSAGES=20 diff --git a/apps/api/src/durable-objects/task-runner/state-machine.ts b/apps/api/src/durable-objects/task-runner/state-machine.ts index dd408db15..df8e82aeb 100644 --- a/apps/api/src/durable-objects/task-runner/state-machine.ts +++ b/apps/api/src/durable-objects/task-runner/state-machine.ts @@ -121,12 +121,26 @@ export async function transitionToInProgress( ).bind(now, now, state.taskId).run(); if (!result.meta.changes || result.meta.changes === 0) { + const authoritative = await rc.env.DATABASE.prepare( + `SELECT status FROM tasks WHERE id = ?` + ).bind(state.taskId).first<{ status: string }>(); log.warn('task_runner_do.aborted_by_recovery', { taskId: state.taskId, step: 'in_progress_transition', + authoritativeStatus: authoritative?.status ?? null, }); - state.completed = true; - await rc.ctx.storage.put('state', state); + if (authoritative?.status === 'in_progress') { + state.currentStep = 'running'; + state.completed = true; + await rc.ctx.storage.put('state', state); + return; + } + if (!authoritative || ['completed', 'failed', 'cancelled'].includes(authoritative.status)) { + state.completed = true; + await rc.ctx.storage.put('state', state); + return; + } + await failTask(state, 'Task orchestration was superseded before agent handoff completed.', rc); return; } diff --git a/apps/api/src/env.ts b/apps/api/src/env.ts index 76e03a9a7..5e5e3bcaf 100644 --- a/apps/api/src/env.ts +++ b/apps/api/src/env.ts @@ -217,6 +217,9 @@ export interface Env { TASK_RUN_HARD_TIMEOUT_MS?: string; TASK_STUCK_QUEUED_TIMEOUT_MS?: string; TASK_STUCK_DELEGATED_TIMEOUT_MS?: string; + TASK_DO_MISMATCH_GRACE_MS?: string; + STUCK_TASK_MAX_CANDIDATES_PER_SWEEP?: string; + TASK_LIVENESS_MAX_ACP_SESSIONS?: string; CLAUDE_CODE_COMPACTION_LOOP_DETECTOR_ENABLED?: string; CLAUDE_CODE_COMPACTION_LOOP_RECENT_MESSAGE_LIMIT?: string; CLAUDE_CODE_COMPACTION_LOOP_WINDOW_MESSAGES?: string; diff --git a/apps/api/src/routes/tasks/callback.ts b/apps/api/src/routes/tasks/callback.ts index 0a47b1a21..4fc2bb098 100644 --- a/apps/api/src/routes/tasks/callback.ts +++ b/apps/api/src/routes/tasks/callback.ts @@ -197,6 +197,21 @@ taskCallbackRoute.post('/:projectId/tasks/:taskId/status/callback', jsonValidato throw errors.badRequest(`Invalid task status in database: ${task.status}`); } + if ( + task.status === body.toStatus && + (body.toStatus === 'completed' || body.toStatus === 'failed' || body.toStatus === 'cancelled') + ) { + await cleanupTerminalTaskResourcesOrThrow(c.env, taskId, { + status: body.toStatus, + errorMessage: task.errorMessage, + projectId, + failureLogEvent: 'task.callback_terminal_cleanup_failed', + logContext: { projectId, source: 'task.callback.idempotent' }, + }); + const blocked = await computeBlockedForTask(db, task.id); + return c.json(toTaskResponse(task, blocked)); + } + if (!canTransitionTaskStatus(task.status, body.toStatus)) { throw errors.conflict( `Invalid transition ${task.status} -> ${body.toStatus}. Allowed: ${getAllowedTaskTransitions(task.status).join(', ') || 'none'}` diff --git a/apps/api/src/scheduled/stuck-tasks.ts b/apps/api/src/scheduled/stuck-tasks.ts index 36a69c456..236834590 100644 --- a/apps/api/src/scheduled/stuck-tasks.ts +++ b/apps/api/src/scheduled/stuck-tasks.ts @@ -24,6 +24,9 @@ */ import { DEFAULT_NODE_HEARTBEAT_STALE_SECONDS, + DEFAULT_STUCK_TASK_MAX_CANDIDATES_PER_SWEEP, + DEFAULT_TASK_DO_MISMATCH_GRACE_MS, + DEFAULT_TASK_LIVENESS_MAX_ACP_SESSIONS, DEFAULT_TASK_RUN_HARD_TIMEOUT_MS, DEFAULT_TASK_RUN_MAX_EXECUTION_MS, DEFAULT_TASK_STUCK_DELEGATED_TIMEOUT_MS, @@ -102,6 +105,83 @@ export interface RecoveryDiagnostics { } | null; } +export interface TaskRuntimeLiveness { + live: boolean; + conclusive: boolean; + reason: string; + workspaceStatus: string | null; + nodeId: string | null; + activeAcpSessionId: string | null; +} + +const LIVE_WORKSPACE_STATUSES = new Set(['running', 'recovery']); +const ACTIVE_ACP_STATUSES = new Set(['assigned', 'running']); + +/** Prove task-scoped liveness. A shared-node heartbeat is never sufficient. */ +export async function getTaskRuntimeLiveness( + env: Env, + task: { project_id: string; workspace_id: string | null }, +): Promise { + const dead = (reason: string, workspaceStatus: string | null, nodeId: string | null): TaskRuntimeLiveness => ({ + live: false, conclusive: true, reason, workspaceStatus, nodeId, activeAcpSessionId: null, + }); + if (!task.workspace_id) return dead('workspace_missing', null, null); + + const row = await env.DATABASE.prepare( + `SELECT w.status AS workspace_status, w.chat_session_id, w.node_id, + n.status AS node_status, n.health_status, n.last_heartbeat_at + FROM workspaces w + LEFT JOIN nodes n ON n.id = w.node_id + WHERE w.id = ? + LIMIT 1` + ).bind(task.workspace_id).first<{ + workspace_status: string; + chat_session_id: string | null; + node_id: string | null; + node_status: string | null; + health_status: string | null; + last_heartbeat_at: string | null; + }>(); + + if (!row) return dead('workspace_missing', null, null); + if (!LIVE_WORKSPACE_STATUSES.has(row.workspace_status)) { + return dead(`workspace_${row.workspace_status}`, row.workspace_status, row.node_id); + } + if (!row.chat_session_id || !row.node_id) { + return { live: false, conclusive: false, reason: 'workspace_runtime_identity_incomplete', workspaceStatus: row.workspace_status, nodeId: row.node_id, activeAcpSessionId: null }; + } + + const staleSeconds = parseInt(env.NODE_HEARTBEAT_STALE_SECONDS || '', 10) || DEFAULT_NODE_HEARTBEAT_STALE_SECONDS; + const staleMs = staleSeconds * 1000; + const nodeHeartbeatAt = row.last_heartbeat_at ? new Date(row.last_heartbeat_at).getTime() : Number.NaN; + if (row.node_status !== 'running' || row.health_status !== 'healthy' || !Number.isFinite(nodeHeartbeatAt) || Date.now() - nodeHeartbeatAt > staleMs) { + return dead('node_not_live', row.workspace_status, row.node_id); + } + + try { + const limit = parseMs(env.TASK_LIVENESS_MAX_ACP_SESSIONS, DEFAULT_TASK_LIVENESS_MAX_ACP_SESSIONS); + const { sessions } = await projectDataService.listAcpSessions(env, task.project_id, { + chatSessionId: row.chat_session_id, + limit, + }); + const active = sessions.find((session) => { + if (!ACTIVE_ACP_STATUSES.has(session.status) || session.workspaceId !== task.workspace_id) return false; + const heartbeatAt = session.lastHeartbeatAt ?? session.updatedAt ?? session.startedAt ?? session.createdAt; + return Number.isFinite(heartbeatAt) && Date.now() - heartbeatAt <= staleMs; + }); + if (active) { + return { live: true, conclusive: true, reason: 'task_acp_session_live', workspaceStatus: row.workspace_status, nodeId: row.node_id, activeAcpSessionId: active.id }; + } + return dead('task_acp_session_not_live', row.workspace_status, row.node_id); + } catch (err) { + log.warn('stuck_task.liveness_probe_failed', { + workspaceId: task.workspace_id, + error: err instanceof Error ? err.message : String(err), + }); + return { live: false, conclusive: false, reason: 'task_liveness_unknown', workspaceStatus: row.workspace_status, nodeId: row.node_id, activeAcpSessionId: null }; + } +} + /** * Query diagnostic context for a stuck task — workspace status, node status, * and TaskRunner DO state. Best-effort: returns whatever context is available. @@ -204,6 +284,8 @@ export async function recoverStuckTasks(env: Env): Promise { const delegatedTimeoutMs = parseMs(env.TASK_STUCK_DELEGATED_TIMEOUT_MS, DEFAULT_TASK_STUCK_DELEGATED_TIMEOUT_MS); const maxExecutionMs = parseMs(env.TASK_RUN_MAX_EXECUTION_MS, DEFAULT_TASK_RUN_MAX_EXECUTION_MS); const hardTimeoutMs = parseMs(env.TASK_RUN_HARD_TIMEOUT_MS, DEFAULT_TASK_RUN_HARD_TIMEOUT_MS); + const mismatchGraceMs = parseMs(env.TASK_DO_MISMATCH_GRACE_MS, DEFAULT_TASK_DO_MISMATCH_GRACE_MS); + const maxCandidates = parseMs(env.STUCK_TASK_MAX_CANDIDATES_PER_SWEEP, DEFAULT_STUCK_TASK_MAX_CANDIDATES_PER_SWEEP); if (hardTimeoutMs <= maxExecutionMs) { log.warn('stuck_task.misconfigured_hard_timeout', { @@ -219,9 +301,10 @@ export async function recoverStuckTasks(env: Env): Promise { `SELECT id, project_id, user_id, status, execution_step, updated_at, started_at, workspace_id, auto_provisioned_node_id FROM tasks - WHERE status IN ('queued', 'delegated', 'in_progress') - ORDER BY updated_at ASC` - ).all<{ + WHERE (status IN ('queued', 'delegated', 'in_progress') OR status = 'awaiting_followup') + ORDER BY updated_at ASC + LIMIT ?` + ).bind(maxCandidates).all<{ id: string; project_id: string; user_id: string; @@ -290,28 +373,18 @@ export async function recoverStuckTasks(env: Env): Promise { reason = `Task stuck in 'delegated' for ${Math.round(elapsedMs / 1000)}s (threshold: ${Math.round(delegatedTimeoutMs / 1000)}s).${stepInfo} Workspace may have failed to start.`; } break; - case 'in_progress': { + case 'in_progress': + case 'awaiting_followup': { const startedAt = task.started_at ? new Date(task.started_at).getTime() : updatedAt; const executionMs = now.getTime() - startedAt; if (executionMs > maxExecutionMs) { - // Hard timeout: absolute ceiling that cannot be bypassed by heartbeat. - // Past this point, the task is killed regardless of node health. - if (executionMs > hardTimeoutMs) { - isStuck = true; - reason = `Task exceeded hard timeout of ${Math.round(hardTimeoutMs / 60000)} minutes (no heartbeat grace).${stepInfo}`; - break; - } - - // Soft timeout (4h-8h window): check if the VM agent is still alive via heartbeat. - // A recent heartbeat means the agent is actively working — allow grace period. - const nodeIdToCheck = await getTaskNodeId(env, task); - if (nodeIdToCheck) { - const staleSeconds = parseInt(env.NODE_HEARTBEAT_STALE_SECONDS || '', 10) || DEFAULT_NODE_HEARTBEAT_STALE_SECONDS; - const heartbeatRecent = await isNodeHeartbeatRecent(env, nodeIdToCheck, staleSeconds); - if (heartbeatRecent) { + const liveness = await getTaskRuntimeLiveness(env, task); + if (liveness.live || !liveness.conclusive) { + if (liveness.live) { log.info('stuck_task.skipped_active_heartbeat', { taskId: task.id, - nodeId: nodeIdToCheck, + nodeId: liveness.nodeId, + activeAcpSessionId: liveness.activeAcpSessionId, executionMs, maxExecutionMs, hardTimeoutMs, @@ -324,22 +397,22 @@ export async function recoverStuckTasks(env: Env): Promise { context: { recoveryType: 'stuck_task_heartbeat_skip', taskId: task.id, - nodeId: nodeIdToCheck, + nodeId: liveness.nodeId, + activeAcpSessionId: liveness.activeAcpSessionId, executionMs, maxExecutionMs, hardTimeoutMs, }, userId: task.user_id, - nodeId: nodeIdToCheck, + nodeId: liveness.nodeId, }); - result.heartbeatSkipped++; - break; } + break; } - isStuck = true; - reason = `Task exceeded max execution time of ${Math.round(maxExecutionMs / 60000)} minutes.${stepInfo}`; + const threshold = executionMs > hardTimeoutMs ? hardTimeoutMs : maxExecutionMs; + reason = `Task runtime is no longer live after ${Math.round(threshold / 60000)} minutes. Last liveness result: ${liveness.reason}.${stepInfo}`; } break; } @@ -366,13 +439,18 @@ export async function recoverStuckTasks(env: Env): Promise { halfThreshold = maxExecutionMs / 2; } - if (timeForCheck > halfThreshold) { + if (timeForCheck > Math.min(halfThreshold, mismatchGraceMs)) { try { const doId = env.TASK_RUNNER.idFromName(task.id); const stub = env.TASK_RUNNER.get(doId) as DurableObjectStub; const doStatus = await stub.getStatus(); if (doStatus && doStatus.completed && task.status !== 'failed' && task.status !== 'completed') { + const liveness = await getTaskRuntimeLiveness(env, task); + if (liveness.conclusive && !liveness.live) { + isStuck = true; + reason = `TaskRunner orchestration completed but task runtime is gone (${liveness.reason}).`; + } // DO thinks it's done but D1 status is still transient — log for investigation. // Only record once: check if we already have a recent mismatch record for this task. // The stuck timeout will eventually fail the task, so this is informational only. @@ -414,7 +492,7 @@ export async function recoverStuckTasks(env: Env): Promise { // DO unreachable — not necessarily an error (may not have been created yet) } } - continue; + if (!isStuck) continue; } try { @@ -489,7 +567,7 @@ export async function recoverStuckTasks(env: Env): Promise { await db.insert(schema.taskStatusEvents).values({ id: ulid(), taskId: task.id, - fromStatus: task.status as 'queued' | 'delegated' | 'in_progress', + fromStatus: task.status as 'queued' | 'delegated' | 'in_progress' | 'awaiting_followup', toStatus: 'failed', actorType: 'system', actorId: null, @@ -550,6 +628,7 @@ export async function recoverStuckTasks(env: Env): Promise { case 'queued': result.failedQueued++; break; case 'delegated': result.failedDelegated++; break; case 'in_progress': + case 'awaiting_followup': result.failedInProgress++; if (compactionLoopRecovery) result.failedCompactionLoops++; break; @@ -581,47 +660,3 @@ export async function recoverStuckTasks(env: Env): Promise { return result; } - -/** - * Look up the node ID for a task — via its workspace or auto-provisioned node. - * Best-effort: returns null if no node can be found. - */ -async function getTaskNodeId( - env: Env, - task: { workspace_id: string | null; auto_provisioned_node_id: string | null } -): Promise { - if (task.workspace_id) { - try { - const ws = await env.DATABASE.prepare( - `SELECT node_id FROM workspaces WHERE id = ?` - ).bind(task.workspace_id).first<{ node_id: string | null }>(); - if (ws?.node_id) return ws.node_id; - } catch { - // Best-effort - } - } - return task.auto_provisioned_node_id; -} - -/** - * Check whether a node's heartbeat is recent (within staleSeconds). - * Returns false if the node has no heartbeat or it's stale. - */ -async function isNodeHeartbeatRecent( - env: Env, - nodeId: string, - staleSeconds: number -): Promise { - try { - const node = await env.DATABASE.prepare( - `SELECT last_heartbeat_at FROM nodes WHERE id = ?` - ).bind(nodeId).first<{ last_heartbeat_at: string | null }>(); - - if (!node?.last_heartbeat_at) return false; - - const heartbeatAge = (Date.now() - new Date(node.last_heartbeat_at).getTime()) / 1000; - return heartbeatAge < staleSeconds; - } catch { - return false; - } -} diff --git a/apps/api/tests/unit/stuck-tasks.test.ts b/apps/api/tests/unit/stuck-tasks.test.ts index 9426e53f5..ef70a63b8 100644 --- a/apps/api/tests/unit/stuck-tasks.test.ts +++ b/apps/api/tests/unit/stuck-tasks.test.ts @@ -28,6 +28,7 @@ const { projectDataMocks } = vi.hoisted(() => ({ projectDataMocks: { getMessages: vi.fn(), listSessions: vi.fn(), + listAcpSessions: vi.fn(), failSession: vi.fn(), }, })); @@ -110,6 +111,15 @@ describe('recoverStuckTasks', () => { vi.clearAllMocks(); projectDataMocks.getMessages.mockResolvedValue({ messages: [], hasMore: false }); projectDataMocks.listSessions.mockResolvedValue({ sessions: [], total: 0 }); + projectDataMocks.listAcpSessions.mockResolvedValue({ + sessions: [{ + id: 'acp-live', + status: 'running', + workspaceId: 'ws-1', + lastHeartbeatAt: Date.now(), + }], + total: 1, + }); projectDataMocks.failSession.mockResolvedValue(undefined); }); @@ -317,6 +327,9 @@ describe('recoverStuckTasks', () => { responses.set('last_heartbeat_at FROM nodes', { results: [{ last_heartbeat_at: recentHeartbeat }], }); + responses.set('w.chat_session_id', { + results: [{ workspace_status: 'running', chat_session_id: 'chat-1', node_id: 'node-1', node_status: 'running', health_status: 'healthy', last_heartbeat_at: recentHeartbeat }], + }); const env = createMockEnv(responses); const result = await recoverStuckTasks(env); @@ -354,6 +367,9 @@ describe('recoverStuckTasks', () => { responses.set('last_heartbeat_at FROM nodes', { results: [{ last_heartbeat_at: staleHeartbeat }], }); + responses.set('w.chat_session_id', { + results: [{ workspace_status: 'running', chat_session_id: 'chat-1', node_id: 'node-1', node_status: 'running', health_status: 'healthy', last_heartbeat_at: staleHeartbeat }], + }); // Workspace status for diagnostics responses.set('node_id, status FROM workspaces', { results: [{ id: 'ws-1', node_id: 'node-1', status: 'running' }], @@ -379,7 +395,7 @@ describe('recoverStuckTasks', () => { env.DATABASE, 'task-1', 'failed', - expect.stringContaining('max execution time'), + expect.stringContaining('runtime is no longer live'), ); }); @@ -423,7 +439,7 @@ describe('recoverStuckTasks', () => { }); describe('hard timeout enforcement', () => { - it('kills tasks past hard timeout even with fresh heartbeat', async () => { + it('preserves a genuinely live task past the hard timeout', async () => { const now = Date.now(); // Task started 9 hours ago (past 8h hard timeout) const startedAt = new Date(now - 9 * 60 * 60 * 1000).toISOString(); @@ -455,6 +471,9 @@ describe('recoverStuckTasks', () => { responses.set('last_heartbeat_at FROM nodes', { results: [{ last_heartbeat_at: recentHeartbeat }], }); + responses.set('w.chat_session_id', { + results: [{ workspace_status: 'running', chat_session_id: 'chat-1', node_id: 'node-1', node_status: 'running', health_status: 'healthy', last_heartbeat_at: recentHeartbeat }], + }); // Workspace status for diagnostics responses.set('node_id, status FROM workspaces', { results: [{ id: 'ws-1', node_id: 'node-1', status: 'running' }], @@ -472,16 +491,9 @@ describe('recoverStuckTasks', () => { const env = createMockEnv(responses); const result = await recoverStuckTasks(env); - // Hard timeout should override the heartbeat grace - expect(result.failedInProgress).toBe(1); - expect(result.heartbeatSkipped).toBe(0); - - // Verify heartbeat was NOT consulted — the hard timeout short-circuits - const db = env.DATABASE as unknown as { prepare: ReturnType }; - const heartbeatCall = db.prepare.mock.calls.find( - ([sql]: [string]) => sql.includes('last_heartbeat_at'), - ); - expect(heartbeatCall).toBeUndefined(); + expect(result.failedInProgress).toBe(0); + expect(result.heartbeatSkipped).toBe(1); + expect(projectDataMocks.listAcpSessions).toHaveBeenCalled(); }); it('preserves heartbeat grace between soft and hard timeout', async () => { @@ -514,6 +526,9 @@ describe('recoverStuckTasks', () => { responses.set('last_heartbeat_at FROM nodes', { results: [{ last_heartbeat_at: recentHeartbeat }], }); + responses.set('w.chat_session_id', { + results: [{ workspace_status: 'running', chat_session_id: 'chat-1', node_id: 'node-1', node_status: 'running', health_status: 'healthy', last_heartbeat_at: recentHeartbeat }], + }); const env = createMockEnv(responses); const result = await recoverStuckTasks(env); diff --git a/apps/api/tests/workers/scheduled-stuck-tasks.test.ts b/apps/api/tests/workers/scheduled-stuck-tasks.test.ts index 9243b95a4..1aa4c5fc9 100644 --- a/apps/api/tests/workers/scheduled-stuck-tasks.test.ts +++ b/apps/api/tests/workers/scheduled-stuck-tasks.test.ts @@ -10,10 +10,11 @@ * - Heartbeat grace period for in_progress tasks * - Diagnostic gathering (workspace/node status from D1) */ -import { env } from 'cloudflare:test'; +import { env, runInDurableObject } from 'cloudflare:test'; import { describe, expect, it } from 'vitest'; import type { Env } from '../../src/env'; +import type { TaskRunner, TaskRunnerState } from '../../src/durable-objects/task-runner'; import { gatherDiagnostics, recoverStuckTasks } from '../../src/scheduled/stuck-tasks'; import { seedInstallation, @@ -69,6 +70,85 @@ async function getObservabilityEvents(taskId: string): Promise<{ message: string } describe('recoverStuckTasks — vertical slice', () => { + describe('DO-completed D1-active reconciliation', () => { + it('fails promptly when the task workspace is demonstrably gone', async () => { + await seedBaseData(); + const taskId = 'task-st-do-mismatch-dead'; + const nodeId = 'node-st-do-mismatch-dead'; + const workspaceId = 'ws-st-do-mismatch-dead'; + const oldDate = new Date(Date.now() - 10 * 60 * 1000).toISOString(); + + await seedNode(nodeId, USER_ID, { status: 'deleted', healthStatus: 'stale' }); + await seedWorkspace(workspaceId, nodeId, USER_ID, { + projectId: PROJECT_ID, + status: 'deleted', + }); + await seedTask(taskId, PROJECT_ID, USER_ID, { + status: 'in_progress', + executionStep: 'awaiting_followup', + startedAt: oldDate, + updatedAt: oldDate, + workspaceId, + taskMode: 'conversation', + }); + + const stub = env.TASK_RUNNER.get( + env.TASK_RUNNER.idFromName(taskId), + ) as DurableObjectStub; + await stub.start({ + taskId, + projectId: PROJECT_ID, + userId: USER_ID, + config: { + vmSize: 'small', + vmLocation: 'nbg1', + branch: 'main', + defaultBranch: 'main', + preferredNodeId: null, + userName: null, + userEmail: null, + githubId: null, + taskTitle: 'Dead conversation', + taskDescription: null, + repository: 'test/repo', + installationId: INSTALL_ID, + outputBranch: null, + projectDefaultVmSize: null, + chatSessionId: null, + agentType: 'codex', + workspaceProfile: null, + devcontainerConfigName: null, + cloudProvider: null, + taskMode: 'conversation', + model: null, + permissionMode: null, + opencodeProvider: null, + opencodeBaseUrl: null, + systemPromptAppend: null, + attachments: null, + }, + }); + await runInDurableObject(stub, async (instance) => { + const state = await instance.ctx.storage.get('state'); + if (!state) throw new Error('TaskRunner state missing'); + state.completed = true; + state.currentStep = 'running'; + await instance.ctx.storage.put('state', state); + }); + + const result = await recoverStuckTasks({ + ...env, + TASK_DO_MISMATCH_GRACE_MS: '60000', + } as unknown as Env); + + expect(result.failedInProgress).toBe(1); + const task = await getTaskStatus(taskId); + expect(task?.status).toBe('failed'); + expect(task?.error_message).toContain('runtime is gone'); + expect(task?.error_message).toContain('workspace_deleted'); + }); + }); + describe('stuck queued task detection', () => { it('fails a task stuck in queued past timeout and records status event', async () => { await seedBaseData(); diff --git a/apps/www/src/content/docs/docs/reference/configuration.md b/apps/www/src/content/docs/docs/reference/configuration.md index 8ad846f30..84935f636 100644 --- a/apps/www/src/content/docs/docs/reference/configuration.md +++ b/apps/www/src/content/docs/docs/reference/configuration.md @@ -203,6 +203,9 @@ SAM loads OpenCode Zen and OpenCode Go model choices through the authenticated m | `TASK_RUN_MAX_EXECUTION_MS` | `14400000` (4 hr) | Max task execution time | | `TASK_STUCK_QUEUED_TIMEOUT_MS` | `600000` (10 min) | Timeout for tasks stuck in queued state | | `TASK_STUCK_DELEGATED_TIMEOUT_MS` | `1860000` (31 min) | Timeout for tasks stuck in delegated state | +| `TASK_DO_MISMATCH_GRACE_MS` | `300000` (5 min) | Minimum age before reconciling completed TaskRunner state with task-scoped liveness | +| `STUCK_TASK_MAX_CANDIDATES_PER_SWEEP` | `100` | Maximum active tasks inspected by each recovery sweep | +| `TASK_LIVENESS_MAX_ACP_SESSIONS` | `5` | Maximum task-scoped ACP sessions inspected per liveness probe | | `CLAUDE_CODE_COMPACTION_LOOP_DETECTOR_ENABLED` | `true` | Enable Claude Code compaction-loop shutdown from recent message evidence | | `CLAUDE_CODE_COMPACTION_LOOP_RECENT_MESSAGE_LIMIT` | `40` | Recent task-session messages to inspect for compaction-loop evidence | | `CLAUDE_CODE_COMPACTION_LOOP_WINDOW_MESSAGES` | `20` | Rolling recent-message window used for compaction-loop detection | diff --git a/packages/shared/src/constants/index.ts b/packages/shared/src/constants/index.ts index d3d6d8e58..3106e41f3 100644 --- a/packages/shared/src/constants/index.ts +++ b/packages/shared/src/constants/index.ts @@ -70,6 +70,9 @@ export { export { DEFAULT_MAX_WORKSPACES_PER_NODE, DEFAULT_TASK_RUN_CLEANUP_DELAY_MS, + DEFAULT_STUCK_TASK_MAX_CANDIDATES_PER_SWEEP, + DEFAULT_TASK_DO_MISMATCH_GRACE_MS, + DEFAULT_TASK_LIVENESS_MAX_ACP_SESSIONS, DEFAULT_TASK_RUN_HARD_TIMEOUT_MS, DEFAULT_TASK_RUN_MAX_EXECUTION_MS, DEFAULT_TASK_RUN_NODE_CPU_THRESHOLD_PERCENT, diff --git a/packages/shared/src/constants/task-execution.ts b/packages/shared/src/constants/task-execution.ts index f4abda676..29d888031 100644 --- a/packages/shared/src/constants/task-execution.ts +++ b/packages/shared/src/constants/task-execution.ts @@ -41,6 +41,15 @@ export const DEFAULT_TASK_STUCK_QUEUED_TIMEOUT_MS = 20 * 60 * 1000; // 20 minute * Set to 31 minutes (1 min buffer above workspace ready timeout). */ export const DEFAULT_TASK_STUCK_DELEGATED_TIMEOUT_MS = 31 * 60 * 1000; // 31 minutes +/** Minimum age before reconciling a completed TaskRunner DO against active D1 state. */ +export const DEFAULT_TASK_DO_MISMATCH_GRACE_MS = 5 * 60 * 1000; + +/** Maximum active task rows inspected by one stuck-task cron invocation. */ +export const DEFAULT_STUCK_TASK_MAX_CANDIDATES_PER_SWEEP = 100; + +/** Maximum ACP sessions read while proving task-scoped runtime liveness. */ +export const DEFAULT_TASK_LIVENESS_MAX_ACP_SESSIONS = 5; + // ============================================================================= // TaskRunner DO Defaults (Alarm-Driven Orchestration — TDF-2) // ============================================================================= diff --git a/tasks/active/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md b/tasks/active/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md index ee1e96a58..139b416d1 100644 --- a/tasks/active/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md +++ b/tasks/active/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md @@ -22,16 +22,16 @@ TaskRunner Durable Objects mark orchestration complete after handing an agent a ## Implementation checklist -- [ ] Add a bounded task-liveness probe correlating workspace state, task-scoped agent-session freshness, and node heartbeat; shared-node heartbeat is never sufficient alone. -- [ ] Keep thresholds and per-run candidate/I/O limits configurable with shared `DEFAULT_*` constants and environment fallbacks. -- [ ] Promptly reconcile DO-completed+D1-active work only when composite liveness proves workspace/agent gone, with sanitized context and a clear terminal state. -- [ ] Preserve live long-running task/conversation sessions and priority 2 recoverable `awaiting_followup`. -- [ ] Cover queued, delegated, in_progress, and awaiting_followup status/step semantics with task/conversation distinctions. -- [ ] Close `aborted_by_recovery` by re-reading D1 and ensuring it is terminal before DO completion without overwriting a concurrent legitimate terminal callback. -- [ ] Make terminal side effects idempotent across cron, DO recovery, and callbacks: event, trigger sync, session state, cleanup, repeated callback. -- [ ] Correct diagnostic timing and bound/deduplicate control-loop I/O. -- [ ] Add unit and cross-runtime Miniflare/D1/DO tests for dead mismatch, live negative case, recovery interleaving, terminal callback/idempotency, and mode/status variants. -- [ ] Update public docs/environment references if configuration changes. +- [x] Add a bounded task-liveness probe correlating workspace state, task-scoped agent-session freshness, and node heartbeat; shared-node heartbeat is never sufficient alone. +- [x] Keep thresholds and per-run candidate/I/O limits configurable with shared `DEFAULT_*` constants and environment fallbacks. +- [x] Promptly reconcile DO-completed+D1-active work only when composite liveness proves workspace/agent gone, with sanitized context and a clear terminal state. +- [x] Preserve live long-running task/conversation sessions and priority 2 recoverable `awaiting_followup`. +- [x] Cover queued, delegated, in_progress, and awaiting_followup status/step semantics with task/conversation distinctions. +- [x] Close `aborted_by_recovery` by re-reading D1 and ensuring it is terminal before DO completion without overwriting a concurrent legitimate terminal callback. +- [x] Make repeated same-terminal callbacks idempotent and reuse terminal cleanup; remaining cross-path verification is tracked below. +- [ ] Correct diagnostic timing and finish bounding/deduplicating control-loop I/O. +- [ ] Complete unit and cross-runtime regressions for recovery interleaving and callback idempotency (dead mismatch and live negative coverage added). +- [x] Update public docs/environment references for new configuration. - [ ] Rebase after priority 2 merges and preserve—not duplicate—its recovery behavior. - [ ] Run focused/full validation, control-loop checks, and Cloudflare/security/constitution/test/docs-sync/control-loop/task-completion reviews. - [ ] Wait for staging turns 1–4 and unrelated deployments, query staging state/logs, deploy, and verify. From 2c77ccf9a38f8afd1298aa7b6a8007ab4da749bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 15:19:25 +0000 Subject: [PATCH 03/11] test: cover task lifecycle recovery races --- apps/api/src/scheduled/stuck-tasks.ts | 10 +++++-- .../task-runner-state-machine.test.ts | 28 +++++++++++++++++++ .../task-callback-recoverable-error.test.ts | 26 +++++++++++++++++ ...-taskrunner-d1-lifecycle-reconciliation.md | 4 +-- 4 files changed, 63 insertions(+), 5 deletions(-) diff --git a/apps/api/src/scheduled/stuck-tasks.ts b/apps/api/src/scheduled/stuck-tasks.ts index 236834590..abeb42b0b 100644 --- a/apps/api/src/scheduled/stuck-tasks.ts +++ b/apps/api/src/scheduled/stuck-tasks.ts @@ -497,7 +497,11 @@ export async function recoverStuckTasks(env: Env): Promise { try { // Gather diagnostic context before recovery - const diagnostics = await gatherDiagnostics(env, task, elapsedMs, reason); + const diagnosticElapsedMs = + (task.status === 'in_progress' || task.status === 'awaiting_followup') && task.started_at + ? now.getTime() - new Date(task.started_at).getTime() + : elapsedMs; + const diagnostics = await gatherDiagnostics(env, task, diagnosticElapsedMs, reason); log.warn('stuck_task.recovering', { taskId: task.id, @@ -505,7 +509,7 @@ export async function recoverStuckTasks(env: Env): Promise { userId: task.user_id, status: task.status, executionStep: task.execution_step, - elapsedMs, + elapsedMs: diagnosticElapsedMs, reason, recoveryType: compactionLoopRecovery ? 'claude_code_compaction_loop' : 'stuck_task', compactionLoop: compactionLoopRecovery?.evidence, @@ -525,7 +529,7 @@ export async function recoverStuckTasks(env: Env): Promise { : { recoveryType: 'stuck_task' }), taskStatus: task.status, executionStep: task.execution_step, - elapsedMs, + elapsedMs: diagnosticElapsedMs, compactionLoop: compactionLoopRecovery ? { sessionId: compactionLoopRecovery.sessionId, agentSessionId: compactionLoopRecovery.agentSessionId, diff --git a/apps/api/tests/unit/durable-objects/task-runner-state-machine.test.ts b/apps/api/tests/unit/durable-objects/task-runner-state-machine.test.ts index a88f4e71e..c35174821 100644 --- a/apps/api/tests/unit/durable-objects/task-runner-state-machine.test.ts +++ b/apps/api/tests/unit/durable-objects/task-runner-state-machine.test.ts @@ -99,6 +99,10 @@ function createD1Database(state: ReturnType) { prepare: vi.fn((sql: string) => ({ bind: (...params: unknown[]) => ({ first: async () => { + if (sql.includes('SELECT status FROM tasks WHERE id = ?')) { + const task = state.tasks.get(String(params[0])); + return task ? { status: task.status } : null; + } if (sql.includes('SELECT status, mission_id FROM tasks WHERE id = ?')) { const task = state.tasks.get(String(params[0])); return task ? { status: task.status, mission_id: task.mission_id } : null; @@ -321,6 +325,30 @@ describe('transitionToInProgress', () => { expect(state.completed).toBe(true); expect(storageWrites.at(-1)).toMatchObject({ currentStep: 'agent_session', completed: true }); }); + + it('makes D1 terminal before completing the DO when recovery leaves an active non-delegated row', async () => { + const { dbState, rc, storageWrites } = createContext(); + seedTask(dbState, { + status: 'queued', + execution_step: 'agent_session', + }); + const state = makeState(); + + await transitionToInProgress(state, rc); + + expect(dbState.tasks.get('task-1')).toMatchObject({ + status: 'failed', + execution_step: null, + error_message: 'Task orchestration was superseded before agent handoff completed.', + }); + expect(dbState.statusEvents).toContainEqual(expect.objectContaining({ + task_id: 'task-1', + from_status: 'queued', + to_status: 'failed', + })); + expect(state.completed).toBe(true); + expect(storageWrites.at(-1)).toMatchObject({ completed: true }); + }); }); describe('failTask', () => { diff --git a/apps/api/tests/unit/routes/task-callback-recoverable-error.test.ts b/apps/api/tests/unit/routes/task-callback-recoverable-error.test.ts index e9e4f1777..f634f5060 100644 --- a/apps/api/tests/unit/routes/task-callback-recoverable-error.test.ts +++ b/apps/api/tests/unit/routes/task-callback-recoverable-error.test.ts @@ -255,4 +255,30 @@ describe('task callback recoverable errors', () => { ); expect(projectDataService.stopSession).not.toHaveBeenCalled(); }); + + it('accepts a repeated same-terminal callback and reruns idempotent cleanup', async () => { + const { cleanupTerminalTaskResourcesOrThrow } = + await import('../../../src/services/task-terminal-cleanup'); + mocks.task.status = 'failed'; + mocks.task.errorMessage = 'fatal error'; + + const res = await postCallback(app, { + toStatus: 'failed', + reason: 'duplicate delivery', + errorMessage: 'fatal error', + }); + + await expectOk(res); + expect(cleanupTerminalTaskResourcesOrThrow).toHaveBeenCalledWith( + expect.anything(), + 'task-recoverable', + { + status: 'failed', + errorMessage: 'fatal error', + projectId: 'proj-recoverable', + failureLogEvent: 'task.callback_terminal_cleanup_failed', + logContext: { projectId: 'proj-recoverable', source: 'task.callback.idempotent' }, + } + ); + }); }); diff --git a/tasks/active/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md b/tasks/active/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md index 139b416d1..c86e0c0b3 100644 --- a/tasks/active/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md +++ b/tasks/active/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md @@ -29,8 +29,8 @@ TaskRunner Durable Objects mark orchestration complete after handing an agent a - [x] Cover queued, delegated, in_progress, and awaiting_followup status/step semantics with task/conversation distinctions. - [x] Close `aborted_by_recovery` by re-reading D1 and ensuring it is terminal before DO completion without overwriting a concurrent legitimate terminal callback. - [x] Make repeated same-terminal callbacks idempotent and reuse terminal cleanup; remaining cross-path verification is tracked below. -- [ ] Correct diagnostic timing and finish bounding/deduplicating control-loop I/O. -- [ ] Complete unit and cross-runtime regressions for recovery interleaving and callback idempotency (dead mismatch and live negative coverage added). +- [x] Correct diagnostic timing and bound control-loop candidates, ACP-session reads, and observability dedupe lookups. +- [x] Add unit and cross-runtime regressions for recovery interleaving, callback idempotency, dead mismatch, and live negative behavior. - [x] Update public docs/environment references for new configuration. - [ ] Rebase after priority 2 merges and preserve—not duplicate—its recovery behavior. - [ ] Run focused/full validation, control-loop checks, and Cloudflare/security/constitution/test/docs-sync/control-loop/task-completion reviews. From dfb08549ca6e6ad35893da7b77ad531b6bbe6c7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 15:31:36 +0000 Subject: [PATCH 04/11] fix: align lifecycle quality assertions --- apps/api/tests/unit/recovery-resilience.test.ts | 2 +- packages/shared/src/constants/index.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/api/tests/unit/recovery-resilience.test.ts b/apps/api/tests/unit/recovery-resilience.test.ts index 45acb523a..6bc862834 100644 --- a/apps/api/tests/unit/recovery-resilience.test.ts +++ b/apps/api/tests/unit/recovery-resilience.test.ts @@ -155,7 +155,7 @@ describe('stuck-tasks DO health checks (TDF-7)', () => { it('checks DO health for non-stuck tasks at half threshold', () => { expect(stuckTasksSource).toContain('halfThreshold'); - expect(stuckTasksSource).toContain('timeForCheck > halfThreshold'); + expect(stuckTasksSource).toContain('timeForCheck > Math.min(halfThreshold, mismatchGraceMs)'); }); it('uses started_at for in_progress tasks (consistent time base)', () => { diff --git a/packages/shared/src/constants/index.ts b/packages/shared/src/constants/index.ts index 3106e41f3..add2e026d 100644 --- a/packages/shared/src/constants/index.ts +++ b/packages/shared/src/constants/index.ts @@ -69,10 +69,10 @@ export { // Task Execution export { DEFAULT_MAX_WORKSPACES_PER_NODE, - DEFAULT_TASK_RUN_CLEANUP_DELAY_MS, DEFAULT_STUCK_TASK_MAX_CANDIDATES_PER_SWEEP, DEFAULT_TASK_DO_MISMATCH_GRACE_MS, DEFAULT_TASK_LIVENESS_MAX_ACP_SESSIONS, + DEFAULT_TASK_RUN_CLEANUP_DELAY_MS, DEFAULT_TASK_RUN_HARD_TIMEOUT_MS, DEFAULT_TASK_RUN_MAX_EXECUTION_MS, DEFAULT_TASK_RUN_NODE_CPU_THRESHOLD_PERCENT, From 5ed62571ea86c2ca2e74815e26c6f39ac0ca8a74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 15:46:31 +0000 Subject: [PATCH 05/11] test: harden lifecycle liveness fallback --- apps/api/src/scheduled/stuck-tasks.ts | 6 ++--- apps/api/tests/unit/stuck-tasks.test.ts | 30 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/apps/api/src/scheduled/stuck-tasks.ts b/apps/api/src/scheduled/stuck-tasks.ts index abeb42b0b..a5ce4ef11 100644 --- a/apps/api/src/scheduled/stuck-tasks.ts +++ b/apps/api/src/scheduled/stuck-tasks.ts @@ -451,9 +451,9 @@ export async function recoverStuckTasks(env: Env): Promise { isStuck = true; reason = `TaskRunner orchestration completed but task runtime is gone (${liveness.reason}).`; } - // DO thinks it's done but D1 status is still transient — log for investigation. - // Only record once: check if we already have a recent mismatch record for this task. - // The stuck timeout will eventually fail the task, so this is informational only. + // Reconcile only with conclusive dead-runtime evidence. Live or unknown + // task-scoped runtime state remains active and is logged for investigation. + // Deduplicate the persisted mismatch signal independently of reconciliation. log.warn('stuck_task.do_completed_but_task_active', { taskId: task.id, taskStatus: task.status, diff --git a/apps/api/tests/unit/stuck-tasks.test.ts b/apps/api/tests/unit/stuck-tasks.test.ts index ef70a63b8..3a5a27724 100644 --- a/apps/api/tests/unit/stuck-tasks.test.ts +++ b/apps/api/tests/unit/stuck-tasks.test.ts @@ -294,6 +294,36 @@ describe('recoverStuckTasks', () => { }); describe('heartbeat-aware in_progress recovery', () => { + it('preserves the task when task-scoped liveness cannot be read', async () => { + const now = Date.now(); + const old = new Date(now - 5 * 60 * 60 * 1000).toISOString(); + const recentHeartbeat = new Date(now - 30 * 1000).toISOString(); + const responses = new Map(); + responses.set('status IN (\'queued\', \'delegated\', \'in_progress\')', { + results: [{ + id: 'task-liveness-unknown', + project_id: 'proj-1', + user_id: 'user-1', + status: 'in_progress', + execution_step: 'running', + updated_at: old, + started_at: old, + workspace_id: 'ws-1', + auto_provisioned_node_id: 'node-1', + }], + }); + responses.set('w.chat_session_id', { + results: [{ workspace_status: 'running', chat_session_id: 'chat-1', node_id: 'node-1', node_status: 'running', health_status: 'healthy', last_heartbeat_at: recentHeartbeat }], + }); + projectDataMocks.listAcpSessions.mockRejectedValueOnce(new Error('ProjectData unavailable')); + + const env = createMockEnv(responses); + const result = await recoverStuckTasks(env); + + expect(result.failedInProgress).toBe(0); + expect(result.errors).toBe(0); + }); + it('skips in_progress tasks when node heartbeat is recent', async () => { const now = Date.now(); // Task started 5 hours ago (past 4h limit) From 04f9695e6953e9d5d7da838d1f73b0c7358abb98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 18:23:41 +0000 Subject: [PATCH 06/11] fix: address review findings on taskrunner lifecycle reconciliation Remove dead awaiting_followup task-STATUS handling (it is a TaskExecutionStep, never written to tasks.status; the OR also defeated the candidate index). Add rule-47 timeout (TASK_LIVENESS_PROBE_TIMEOUT_MS) bounding the ProjectData DO liveness probe (inconclusive on timeout, never fatal) and cache the per-candidate liveness so both gates probe at most once. Fix two pre-existing workers-pool tests silently broken by the code change, add CI-verified node-pool regressions (live awaiting_followup preserved; aborted_by_recovery in_progress branch), fix lint, and document the liveness-gated recovery semantic. File backlog for workers-pool tests not running in CI. Co-Authored-By: Claude Opus 4.6 --- apps/api/.env.example | 1 + apps/api/src/env.ts | 1 + apps/api/src/scheduled/stuck-tasks.ts | 54 ++++++++++++++---- .../task-runner-state-machine.test.ts | 24 ++++++++ apps/api/tests/unit/stuck-tasks.test.ts | 37 +++++++++++++ .../workers/scheduled-stuck-tasks.test.ts | 26 ++++++--- .../docs/docs/reference/configuration.md | 3 + packages/shared/src/constants/index.ts | 1 + .../shared/src/constants/task-execution.ts | 8 +++ ...-taskrunner-d1-lifecycle-reconciliation.md | 33 ++++++++++- ...-07-11-workers-pool-tests-not-run-in-ci.md | 55 +++++++++++++++++++ 11 files changed, 219 insertions(+), 24 deletions(-) create mode 100644 tasks/backlog/2026-07-11-workers-pool-tests-not-run-in-ci.md diff --git a/apps/api/.env.example b/apps/api/.env.example index e7397672d..cf5973cb9 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -349,6 +349,7 @@ BASE_DOMAIN=workspaces.example.com # TASK_DO_MISMATCH_GRACE_MS=300000 # 5 minutes — minimum age before DO/D1 liveness reconciliation # STUCK_TASK_MAX_CANDIDATES_PER_SWEEP=100 # Bound task rows inspected per cron invocation # TASK_LIVENESS_MAX_ACP_SESSIONS=5 # Bound task-scoped ACP sessions inspected per candidate +# TASK_LIVENESS_PROBE_TIMEOUT_MS=5000 # Per-candidate timeout for the ACP liveness DO probe (inconclusive on timeout, never fatal) # CLAUDE_CODE_COMPACTION_LOOP_DETECTOR_ENABLED=true # CLAUDE_CODE_COMPACTION_LOOP_RECENT_MESSAGE_LIMIT=40 # CLAUDE_CODE_COMPACTION_LOOP_WINDOW_MESSAGES=20 diff --git a/apps/api/src/env.ts b/apps/api/src/env.ts index 5e5e3bcaf..080469d65 100644 --- a/apps/api/src/env.ts +++ b/apps/api/src/env.ts @@ -220,6 +220,7 @@ export interface Env { TASK_DO_MISMATCH_GRACE_MS?: string; STUCK_TASK_MAX_CANDIDATES_PER_SWEEP?: string; TASK_LIVENESS_MAX_ACP_SESSIONS?: string; + TASK_LIVENESS_PROBE_TIMEOUT_MS?: string; CLAUDE_CODE_COMPACTION_LOOP_DETECTOR_ENABLED?: string; CLAUDE_CODE_COMPACTION_LOOP_RECENT_MESSAGE_LIMIT?: string; CLAUDE_CODE_COMPACTION_LOOP_WINDOW_MESSAGES?: string; diff --git a/apps/api/src/scheduled/stuck-tasks.ts b/apps/api/src/scheduled/stuck-tasks.ts index a5ce4ef11..b17c09ba0 100644 --- a/apps/api/src/scheduled/stuck-tasks.ts +++ b/apps/api/src/scheduled/stuck-tasks.ts @@ -27,6 +27,7 @@ import { DEFAULT_STUCK_TASK_MAX_CANDIDATES_PER_SWEEP, DEFAULT_TASK_DO_MISMATCH_GRACE_MS, DEFAULT_TASK_LIVENESS_MAX_ACP_SESSIONS, + DEFAULT_TASK_LIVENESS_PROBE_TIMEOUT_MS, DEFAULT_TASK_RUN_HARD_TIMEOUT_MS, DEFAULT_TASK_RUN_MAX_EXECUTION_MS, DEFAULT_TASK_STUCK_DELEGATED_TIMEOUT_MS, @@ -160,10 +161,29 @@ export async function getTaskRuntimeLiveness( try { const limit = parseMs(env.TASK_LIVENESS_MAX_ACP_SESSIONS, DEFAULT_TASK_LIVENESS_MAX_ACP_SESSIONS); - const { sessions } = await projectDataService.listAcpSessions(env, task.project_id, { - chatSessionId: row.chat_session_id, - limit, - }); + // Bound the ProjectData DO probe so a slow/unresponsive DO cannot stall the + // control-loop sweep (rule 47). A timeout is inconclusive, never fatal. + const probeTimeoutMs = parseMs(env.TASK_LIVENESS_PROBE_TIMEOUT_MS, DEFAULT_TASK_LIVENESS_PROBE_TIMEOUT_MS); + const TIMEOUT = Symbol('liveness_probe_timeout'); + let timer: ReturnType | undefined; + const probe = await Promise.race([ + projectDataService.listAcpSessions(env, task.project_id, { + chatSessionId: row.chat_session_id, + limit, + }), + new Promise((resolve) => { + timer = setTimeout(() => resolve(TIMEOUT), probeTimeoutMs); + }), + ]); + if (timer) clearTimeout(timer); + if (probe === TIMEOUT) { + log.warn('stuck_task.liveness_probe_timeout', { + workspaceId: task.workspace_id, + probeTimeoutMs, + }); + return { live: false, conclusive: false, reason: 'task_liveness_timeout', workspaceStatus: row.workspace_status, nodeId: row.node_id, activeAcpSessionId: null }; + } + const { sessions } = probe; const active = sessions.find((session) => { if (!ACTIVE_ACP_STATUSES.has(session.status) || session.workspaceId !== task.workspace_id) return false; const heartbeatAt = session.lastHeartbeatAt ?? session.updatedAt ?? session.startedAt ?? session.createdAt; @@ -301,7 +321,7 @@ export async function recoverStuckTasks(env: Env): Promise { `SELECT id, project_id, user_id, status, execution_step, updated_at, started_at, workspace_id, auto_provisioned_node_id FROM tasks - WHERE (status IN ('queued', 'delegated', 'in_progress') OR status = 'awaiting_followup') + WHERE status IN ('queued', 'delegated', 'in_progress') ORDER BY updated_at ASC LIMIT ?` ).bind(maxCandidates).all<{ @@ -359,6 +379,15 @@ export async function recoverStuckTasks(env: Env): Promise { }); } + // Compute task-scoped liveness at most once per candidate: both the + // in_progress timeout gate and the DO-completed mismatch gate below may need + // it, and each probe is a bounded ProjectData DO call (rule 47 — I/O budget). + let cachedLiveness: TaskRuntimeLiveness | null = null; + const probeLiveness = async (): Promise => { + cachedLiveness ??= await getTaskRuntimeLiveness(env, task); + return cachedLiveness; + }; + if (!isStuck) { switch (task.status) { case 'queued': @@ -373,12 +402,14 @@ export async function recoverStuckTasks(env: Env): Promise { reason = `Task stuck in 'delegated' for ${Math.round(elapsedMs / 1000)}s (threshold: ${Math.round(delegatedTimeoutMs / 1000)}s).${stepInfo} Workspace may have failed to start.`; } break; - case 'in_progress': - case 'awaiting_followup': { + case 'in_progress': { + // A task-mode task legitimately paused at execution_step + // 'awaiting_followup' keeps status 'in_progress'; it is protected here + // by the same liveness gate (a live workspace/agent is never failed). const startedAt = task.started_at ? new Date(task.started_at).getTime() : updatedAt; const executionMs = now.getTime() - startedAt; if (executionMs > maxExecutionMs) { - const liveness = await getTaskRuntimeLiveness(env, task); + const liveness = await probeLiveness(); if (liveness.live || !liveness.conclusive) { if (liveness.live) { log.info('stuck_task.skipped_active_heartbeat', { @@ -446,7 +477,7 @@ export async function recoverStuckTasks(env: Env): Promise { const doStatus = await stub.getStatus(); if (doStatus && doStatus.completed && task.status !== 'failed' && task.status !== 'completed') { - const liveness = await getTaskRuntimeLiveness(env, task); + const liveness = await probeLiveness(); if (liveness.conclusive && !liveness.live) { isStuck = true; reason = `TaskRunner orchestration completed but task runtime is gone (${liveness.reason}).`; @@ -498,7 +529,7 @@ export async function recoverStuckTasks(env: Env): Promise { try { // Gather diagnostic context before recovery const diagnosticElapsedMs = - (task.status === 'in_progress' || task.status === 'awaiting_followup') && task.started_at + task.status === 'in_progress' && task.started_at ? now.getTime() - new Date(task.started_at).getTime() : elapsedMs; const diagnostics = await gatherDiagnostics(env, task, diagnosticElapsedMs, reason); @@ -571,7 +602,7 @@ export async function recoverStuckTasks(env: Env): Promise { await db.insert(schema.taskStatusEvents).values({ id: ulid(), taskId: task.id, - fromStatus: task.status as 'queued' | 'delegated' | 'in_progress' | 'awaiting_followup', + fromStatus: task.status as 'queued' | 'delegated' | 'in_progress', toStatus: 'failed', actorType: 'system', actorId: null, @@ -632,7 +663,6 @@ export async function recoverStuckTasks(env: Env): Promise { case 'queued': result.failedQueued++; break; case 'delegated': result.failedDelegated++; break; case 'in_progress': - case 'awaiting_followup': result.failedInProgress++; if (compactionLoopRecovery) result.failedCompactionLoops++; break; diff --git a/apps/api/tests/unit/durable-objects/task-runner-state-machine.test.ts b/apps/api/tests/unit/durable-objects/task-runner-state-machine.test.ts index c35174821..c1ca62a61 100644 --- a/apps/api/tests/unit/durable-objects/task-runner-state-machine.test.ts +++ b/apps/api/tests/unit/durable-objects/task-runner-state-machine.test.ts @@ -349,6 +349,30 @@ describe('transitionToInProgress', () => { expect(state.completed).toBe(true); expect(storageWrites.at(-1)).toMatchObject({ completed: true }); }); + + it('completes the DO as running when a concurrent recovery already advanced D1 to in_progress', async () => { + // aborted_by_recovery Branch 1: the optimistic delegated->in_progress UPDATE + // finds 0 rows because a concurrent path already set the row to 'in_progress'. + // The DO must converge on 'running' WITHOUT failing the task or overwriting D1. + const { dbState, rc, storageWrites } = createContext(); + seedTask(dbState, { + status: 'in_progress', + execution_step: 'running', + }); + const state = makeState(); + + await transitionToInProgress(state, rc); + + expect(dbState.tasks.get('task-1')).toMatchObject({ + status: 'in_progress', + execution_step: 'running', + }); + // No new status event and no failTask side effect — D1 is left as-is. + expect(dbState.statusEvents).toHaveLength(0); + expect(state.currentStep).toBe('running'); + expect(state.completed).toBe(true); + expect(storageWrites.at(-1)).toMatchObject({ currentStep: 'running', completed: true }); + }); }); describe('failTask', () => { diff --git a/apps/api/tests/unit/stuck-tasks.test.ts b/apps/api/tests/unit/stuck-tasks.test.ts index 3a5a27724..7fdcb0bd7 100644 --- a/apps/api/tests/unit/stuck-tasks.test.ts +++ b/apps/api/tests/unit/stuck-tasks.test.ts @@ -368,6 +368,43 @@ describe('recoverStuckTasks', () => { expect(result.failedInProgress).toBe(0); }); + it('preserves a live task-mode task paused at the awaiting_followup step', async () => { + // A task-mode task that called complete_task in conversation flow keeps + // status 'in_progress' with execution_step 'awaiting_followup'. With a live + // workspace + node + task-scoped ACP session it must NOT be failed, even + // past the execution limit — awaiting-followup is an intentional pause. + const now = Date.now(); + const startedAt = new Date(now - 5 * 60 * 60 * 1000).toISOString(); + const recentHeartbeat = new Date(now - 30 * 1000).toISOString(); + + const responses = new Map(); + responses.set('status IN (\'queued\', \'delegated\', \'in_progress\')', { + results: [ + { + id: 'task-1', + project_id: 'proj-1', + user_id: 'user-1', + status: 'in_progress', + execution_step: 'awaiting_followup', + updated_at: startedAt, + started_at: startedAt, + workspace_id: 'ws-1', + auto_provisioned_node_id: 'node-1', + }, + ], + }); + responses.set('w.chat_session_id', { + results: [{ workspace_status: 'running', chat_session_id: 'chat-1', node_id: 'node-1', node_status: 'running', health_status: 'healthy', last_heartbeat_at: recentHeartbeat }], + }); + + const env = createMockEnv(responses); + const result = await recoverStuckTasks(env); + + // Default listAcpSessions mock returns a live running session for ws-1. + expect(result.failedInProgress).toBe(0); + expect(result.heartbeatSkipped).toBe(1); + }); + it('fails in_progress tasks when node heartbeat is stale', async () => { const now = Date.now(); const startedAt = new Date(now - 5 * 60 * 60 * 1000).toISOString(); diff --git a/apps/api/tests/workers/scheduled-stuck-tasks.test.ts b/apps/api/tests/workers/scheduled-stuck-tasks.test.ts index 1aa4c5fc9..0fda15597 100644 --- a/apps/api/tests/workers/scheduled-stuck-tasks.test.ts +++ b/apps/api/tests/workers/scheduled-stuck-tasks.test.ts @@ -13,8 +13,8 @@ import { env, runInDurableObject } from 'cloudflare:test'; import { describe, expect, it } from 'vitest'; -import type { Env } from '../../src/env'; import type { TaskRunner, TaskRunnerState } from '../../src/durable-objects/task-runner'; +import type { Env } from '../../src/env'; import { gatherDiagnostics, recoverStuckTasks } from '../../src/scheduled/stuck-tasks'; import { seedInstallation, @@ -249,20 +249,28 @@ describe('recoverStuckTasks — vertical slice', () => { const task = await getTaskStatus(taskId); expect(task?.status).toBe('failed'); - expect(task?.error_message).toContain('hard timeout'); + // Past the hard timeout with no provable live runtime (no workspace), the + // task is failed through the liveness gate with a sanitized reason. + expect(task?.error_message).toContain('no longer live'); + expect(task?.error_message).toContain('workspace_missing'); }); }); - describe('heartbeat grace period', () => { - it('skips in_progress task with recent heartbeat', async () => { + describe('liveness grace period', () => { + it('preserves an in_progress task whose runtime identity is incomplete (fail-safe)', async () => { + // A running workspace with a fresh node heartbeat but no resolvable + // task-scoped runtime identity yields an INCONCLUSIVE liveness result. + // Fail-safe: the task must be left untouched, never failed on node + // heartbeat alone (the production regression this fix targets). await seedBaseData(); - const taskId = 'task-st-heartbeat-skip'; + const taskId = 'task-st-liveness-inconclusive'; const nodeId = 'node-st-heartbeat'; const fiveHoursAgo = new Date(Date.now() - 5 * 60 * 60 * 1000).toISOString(); await seedNode(nodeId, USER_ID, { - lastHeartbeatAt: new Date().toISOString(), // very recent heartbeat + lastHeartbeatAt: new Date().toISOString(), // very recent node heartbeat }); + // Running workspace but no chat_session_id → runtime identity incomplete. await seedWorkspace('ws-st-heartbeat', nodeId, USER_ID, { projectId: PROJECT_ID, status: 'running', @@ -286,9 +294,9 @@ describe('recoverStuckTasks — vertical slice', () => { const result = await recoverStuckTasks(testEnv); - expect(result.heartbeatSkipped).toBeGreaterThanOrEqual(1); - - // Task should still be in_progress + // Inconclusive liveness is never fatal: the task is preserved, and a + // shared-node heartbeat alone does NOT count as a task-scoped skip. + expect(result.failedInProgress).toBe(0); const task = await getTaskStatus(taskId); expect(task?.status).toBe('in_progress'); }); diff --git a/apps/www/src/content/docs/docs/reference/configuration.md b/apps/www/src/content/docs/docs/reference/configuration.md index 84935f636..154ed5bfb 100644 --- a/apps/www/src/content/docs/docs/reference/configuration.md +++ b/apps/www/src/content/docs/docs/reference/configuration.md @@ -206,6 +206,7 @@ SAM loads OpenCode Zen and OpenCode Go model choices through the authenticated m | `TASK_DO_MISMATCH_GRACE_MS` | `300000` (5 min) | Minimum age before reconciling completed TaskRunner state with task-scoped liveness | | `STUCK_TASK_MAX_CANDIDATES_PER_SWEEP` | `100` | Maximum active tasks inspected by each recovery sweep | | `TASK_LIVENESS_MAX_ACP_SESSIONS` | `5` | Maximum task-scoped ACP sessions inspected per liveness probe | +| `TASK_LIVENESS_PROBE_TIMEOUT_MS` | `5000` (5 sec) | Per-candidate timeout for the ACP liveness probe; a timeout is inconclusive (never fails a task) | | `CLAUDE_CODE_COMPACTION_LOOP_DETECTOR_ENABLED` | `true` | Enable Claude Code compaction-loop shutdown from recent message evidence | | `CLAUDE_CODE_COMPACTION_LOOP_RECENT_MESSAGE_LIMIT` | `40` | Recent task-session messages to inspect for compaction-loop evidence | | `CLAUDE_CODE_COMPACTION_LOOP_WINDOW_MESSAGES` | `20` | Rolling recent-message window used for compaction-loop detection | @@ -219,6 +220,8 @@ SAM loads OpenCode Zen and OpenCode Go model choices through the authenticated m | `TASK_RECONCILIATION_PROMPT_HARD_STALL_MS` | `7200000` (2 hr) | In-flight prompt hard-stall threshold before SAM requests prompt cancellation | | `TASK_RECONCILIATION_MIN_ALARM_DELAY_MS` | `10000` (10 sec) | Minimum delay before the next reconciliation alarm can fire | +> **Liveness-gated recovery.** Stuck-task recovery for `in_progress` tasks (including task-mode work paused at the `awaiting_followup` execution step) is gated on **task-scoped** liveness — a live workspace, a healthy node with a recent heartbeat, **and** an active task-scoped ACP session. A shared-node heartbeat alone is never sufficient. Consequently, `TASK_RUN_HARD_TIMEOUT_MS` and `TASK_RUN_MAX_EXECUTION_MS` bound the point at which a task with **no** proven live runtime is failed; a task with a demonstrably live runtime is preserved past those thresholds rather than terminated on elapsed time alone. When liveness cannot be determined (probe timeout or error), the task is left untouched (fail-safe). + ## Node & Workspace Readiness | Variable | Default | Description | diff --git a/packages/shared/src/constants/index.ts b/packages/shared/src/constants/index.ts index add2e026d..96c51cb99 100644 --- a/packages/shared/src/constants/index.ts +++ b/packages/shared/src/constants/index.ts @@ -72,6 +72,7 @@ export { DEFAULT_STUCK_TASK_MAX_CANDIDATES_PER_SWEEP, DEFAULT_TASK_DO_MISMATCH_GRACE_MS, DEFAULT_TASK_LIVENESS_MAX_ACP_SESSIONS, + DEFAULT_TASK_LIVENESS_PROBE_TIMEOUT_MS, DEFAULT_TASK_RUN_CLEANUP_DELAY_MS, DEFAULT_TASK_RUN_HARD_TIMEOUT_MS, DEFAULT_TASK_RUN_MAX_EXECUTION_MS, diff --git a/packages/shared/src/constants/task-execution.ts b/packages/shared/src/constants/task-execution.ts index 29d888031..4a1f9ba3a 100644 --- a/packages/shared/src/constants/task-execution.ts +++ b/packages/shared/src/constants/task-execution.ts @@ -50,6 +50,14 @@ export const DEFAULT_STUCK_TASK_MAX_CANDIDATES_PER_SWEEP = 100; /** Maximum ACP sessions read while proving task-scoped runtime liveness. */ export const DEFAULT_TASK_LIVENESS_MAX_ACP_SESSIONS = 5; +/** + * Per-candidate timeout for the task-scoped ACP liveness probe (a ProjectData DO + * call) inside the stuck-task control loop. A healthy runtime answers in + * milliseconds; past this bound the probe is treated as inconclusive (fail-safe: + * never fails a task on a slow/unresponsive DO). Keeps the sweep's worst-case + * wall time bounded (rule 47 — control-loop I/O budget). */ +export const DEFAULT_TASK_LIVENESS_PROBE_TIMEOUT_MS = 5 * 1000; + // ============================================================================= // TaskRunner DO Defaults (Alarm-Driven Orchestration — TDF-2) // ============================================================================= diff --git a/tasks/active/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md b/tasks/active/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md index c86e0c0b3..6a9560e92 100644 --- a/tasks/active/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md +++ b/tasks/active/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md @@ -32,10 +32,37 @@ TaskRunner Durable Objects mark orchestration complete after handing an agent a - [x] Correct diagnostic timing and bound control-loop candidates, ACP-session reads, and observability dedupe lookups. - [x] Add unit and cross-runtime regressions for recovery interleaving, callback idempotency, dead mismatch, and live negative behavior. - [x] Update public docs/environment references for new configuration. -- [ ] Rebase after priority 2 merges and preserve—not duplicate—its recovery behavior. -- [ ] Run focused/full validation, control-loop checks, and Cloudflare/security/constitution/test/docs-sync/control-loop/task-completion reviews. +- [ ] Rebase after priority 2 merges and preserve—not duplicate—its recovery behavior. (P2 `01KX8SWC9DEMHCA8RSPZN5W1V1` has NOT landed; rebased onto current main instead. Coordination flagged on PR #1567.) +- [x] Run focused/full validation, control-loop checks, and Cloudflare/security/constitution/test/docs-sync/control-loop/task-completion reviews. (See "Review fixes" below.) - [ ] Wait for staging turns 1–4 and unrelated deployments, query staging state/logs, deploy, and verify. -- [ ] Open PR, pass CI, merge, monitor production deploy/evidence, and update idea `01KT90PKF6167SXZ9YZY0R26MM`. +- [ ] Open PR (#1567), pass CI, merge, monitor production deploy/evidence, and update idea `01KT90PKF6167SXZ9YZY0R26MM`. (PR open; CI/staging/merge pending.) + +## Review fixes (2026-07-11, local specialist reviewers on PR #1567) + +Constitution: PASS. Cloudflare + task-completion: FAIL with confirmed real bugs in +the recovered work, all fixed: + +1. `awaiting_followup` is a `TaskExecutionStep`, NOT a `TaskStatus` + (`packages/shared/src/types/task.ts`; writers set `execution_step`, never + `status` — `task-tools.ts:264`, `callback.ts:157`). Removed the dead + `status='awaiting_followup'` candidate-query OR (which also defeated the index), + switch case, `fromStatus` cast, `failedInProgress` counter case, and + diagnostic-timing branch. Real production rows are `in_progress` + + `execution_step='awaiting_followup'`, still handled by the `in_progress` case. +2. Rule 47: added `TASK_LIVENESS_PROBE_TIMEOUT_MS` (default 5s) bounding the + ProjectData DO liveness probe; a timeout is inconclusive (fail-safe), never + fatal. Cached the per-candidate liveness result so the in_progress gate and the + DO-mismatch gate probe at most once. +3. Two pre-existing workers-pool tests were silently broken by the code change and + never caught because the workers pool does not run in CI (see backlog + `2026-07-11-workers-pool-tests-not-run-in-ci.md`). Fixed both assertions; the + live-skip vertical slice is deferred to that backlog task (workerd could not run + in the task workspace to verify DO-seeded ACP sessions). +4. Added CI-verified node-pool regressions: a live task-mode `awaiting_followup` + task is preserved; state-machine `aborted_by_recovery` Branch 1 (concurrent + recovery advanced D1 to `in_progress` → DO completes as running, no `failTask`). +5. Fixed lint (import sort). Documented the liveness-gated recovery semantic + (live tasks are preserved past the hard ceiling) in `configuration.md`. ## Acceptance criteria diff --git a/tasks/backlog/2026-07-11-workers-pool-tests-not-run-in-ci.md b/tasks/backlog/2026-07-11-workers-pool-tests-not-run-in-ci.md new file mode 100644 index 000000000..92d32913e --- /dev/null +++ b/tasks/backlog/2026-07-11-workers-pool-tests-not-run-in-ci.md @@ -0,0 +1,55 @@ +# Workers-pool vitest tests do not run in CI + +## Problem + +The `@cloudflare/vitest-pool-workers` test suite under `apps/api/tests/workers/**` +(real D1 + Durable Object vertical-slice tests, run via +`vitest.workers.config.ts` / `pnpm --filter @simple-agent-manager/api test:workers`) +is **not executed by any CI workflow**: + +- `apps/api/vitest.config.ts` sets `exclude: ['tests/workers/**']`, so the + default `pnpm test` / `pnpm test:coverage` (what CI runs — `ci.yml` lines ~199 + and ~371) skips the entire workers pool. +- No workflow in `.github/workflows/` invokes `test:workers`. + +Consequence: the workers-pool "cross-runtime" / vertical-slice tests (rule 10 / +rule 35 capability coverage) provide **zero CI protection**. They only run when a +developer manually runs `test:workers` locally. + +## Evidence (discovered 2026-07-11, during Priority 5 — PR #1567) + +Two pre-existing tests in `apps/api/tests/workers/scheduled-stuck-tasks.test.ts` +silently broke when the TaskRunner liveness logic changed, and nothing caught it: + +- "fails in_progress task past hard timeout" — asserted `error_message` contains + `'hard timeout'`; the new liveness-gated reason no longer contains that string. +- "skips in_progress task with recent heartbeat" — asserted + `heartbeatSkipped >= 1`; under the new task-scoped liveness a node heartbeat + alone no longer counts as a skip, so the counter stays `0`. + +Both were fixed in PR #1567, but only by manual code reading — CI was green the +whole time because the workers pool never ran. + +## Acceptance criteria + +- [ ] The workers-pool suite (`test:workers`) runs in CI on every PR that touches + `apps/api/**` (either as a dedicated job or folded into the Test job). +- [ ] The job is reliable in the CI runner (investigate `workerd` "Worker exited + unexpectedly" crashes seen in some sandboxed environments; may need a + runner/resource or `@cloudflare/vitest-pool-workers` config adjustment). +- [ ] A deliberately-broken workers-pool assertion fails the CI job (prove the + gate works). +- [ ] Restore the workers vertical-slice for the "genuinely-live task is skipped" + path in `scheduled-stuck-tasks.test.ts` (seed a live task-scoped ACP session + in the ProjectData DO — `createAcpSession` + `transitionAcpSession` to + `running` with a matching `workspaceId` and recent heartbeat). PR #1567 + converted that test to the fail-safe "inconclusive → preserved" path because + the workers pool could not be run in the task workspace; the live-skip + vertical slice should be re-added once the pool runs in CI. + +## Context + +- Discovered while completing Priority 5 (idea `01KT90PKF6167SXZ9YZY0R26MM`, + PR #1567) — TaskRunner DO/D1 lifecycle reconciliation. +- Related rules: `.claude/rules/10-e2e-verification.md`, + `.claude/rules/35-vertical-slice-testing.md`. From f19cb44f02b8bfafbc210bc8d75d9c4c94d5bd4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 18:28:38 +0000 Subject: [PATCH 07/11] test: add discriminating coverage from test-engineer review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - C-1: node healthy + recent heartbeat but task-scoped ACP session gone -> task is failed (proves node heartbeat alone cannot suppress reconciliation — the exact production regression; previously every failure path short-circuited at node_not_live so listAcpSessions was never on a failing path). - H-1: make the custom-hard-timeout test discriminating (assert the overridden ceiling surfaces as the failure threshold, not the default 480 min). - M-1: assert the idempotent terminal callback does NOT re-write task status. Co-Authored-By: Claude Opus 4.6 --- .../task-callback-recoverable-error.test.ts | 3 + apps/api/tests/unit/stuck-tasks.test.ts | 86 +++++++++++++++---- 2 files changed, 70 insertions(+), 19 deletions(-) diff --git a/apps/api/tests/unit/routes/task-callback-recoverable-error.test.ts b/apps/api/tests/unit/routes/task-callback-recoverable-error.test.ts index f634f5060..aec1b45fd 100644 --- a/apps/api/tests/unit/routes/task-callback-recoverable-error.test.ts +++ b/apps/api/tests/unit/routes/task-callback-recoverable-error.test.ts @@ -259,6 +259,7 @@ describe('task callback recoverable errors', () => { it('accepts a repeated same-terminal callback and reruns idempotent cleanup', async () => { const { cleanupTerminalTaskResourcesOrThrow } = await import('../../../src/services/task-terminal-cleanup'); + const { setTaskStatus } = await import('../../../src/routes/tasks/_helpers'); mocks.task.status = 'failed'; mocks.task.errorMessage = 'fatal error'; @@ -280,5 +281,7 @@ describe('task callback recoverable errors', () => { logContext: { projectId: 'proj-recoverable', source: 'task.callback.idempotent' }, } ); + // Idempotency invariant: the already-terminal row is NOT written again. + expect(setTaskStatus).not.toHaveBeenCalled(); }); }); diff --git a/apps/api/tests/unit/stuck-tasks.test.ts b/apps/api/tests/unit/stuck-tasks.test.ts index 7fdcb0bd7..97024ee16 100644 --- a/apps/api/tests/unit/stuck-tasks.test.ts +++ b/apps/api/tests/unit/stuck-tasks.test.ts @@ -466,6 +466,59 @@ describe('recoverStuckTasks', () => { ); }); + it('fails an in_progress task when the node is healthy but the task-scoped ACP session is gone', async () => { + // The exact production regression: a HEALTHY node with a RECENT heartbeat + // is not proof the task is alive. When the task-scoped ACP session is + // absent/stale, reconciliation must still fail the task — a shared-node + // heartbeat cannot independently suppress it. + const now = Date.now(); + const startedAt = new Date(now - 5 * 60 * 60 * 1000).toISOString(); + const recentHeartbeat = new Date(now - 30 * 1000).toISOString(); // node very much alive + + const responses = new Map(); + responses.set('status IN (\'queued\', \'delegated\', \'in_progress\')', { + results: [ + { + id: 'task-1', + project_id: 'proj-1', + user_id: 'user-1', + status: 'in_progress', + execution_step: 'running', + updated_at: startedAt, + started_at: startedAt, + workspace_id: 'ws-1', + auto_provisioned_node_id: 'node-1', + }, + ], + }); + responses.set('w.chat_session_id', { + results: [{ workspace_status: 'running', chat_session_id: 'chat-1', node_id: 'node-1', node_status: 'running', health_status: 'healthy', last_heartbeat_at: recentHeartbeat }], + }); + responses.set('node_id, status FROM workspaces', { + results: [{ id: 'ws-1', node_id: 'node-1', status: 'running' }], + }); + responses.set('status, health_status FROM nodes', { + results: [{ id: 'node-1', status: 'running', health_status: 'healthy' }], + }); + responses.set('UPDATE tasks SET status = \'failed\'', { results: [], changes: 1 }); + + // No live task-scoped ACP session, despite the healthy node. + projectDataMocks.listAcpSessions.mockResolvedValueOnce({ sessions: [], total: 0 }); + + const env = createMockEnv(responses); + const result = await recoverStuckTasks(env); + + expect(result.failedInProgress).toBe(1); + expect(result.heartbeatSkipped).toBe(0); + // Failure is driven by the ACP-session check, not the node heartbeat. + expect(syncTriggerExecutionMock).toHaveBeenCalledWith( + env.DATABASE, + 'task-1', + 'failed', + expect.stringContaining('task_acp_session_not_live'), + ); + }); + it('fails in_progress tasks with no node (no heartbeat to check)', async () => { const now = Date.now(); const startedAt = new Date(now - 5 * 60 * 60 * 1000).toISOString(); @@ -654,14 +707,14 @@ describe('recoverStuckTasks', () => { expect(result.heartbeatSkipped).toBe(0); }); - it('respects custom hard timeout from env var', async () => { + it('respects custom hard timeout from env var in the failure threshold', async () => { const now = Date.now(); - // Custom hard timeout: 2 hours (7200000ms) - // Task started 3 hours ago — past custom hard timeout + // Custom hard timeout: 2 hours (7200000ms → 120 min). + // Task started 3 hours ago — past custom hard timeout, with a dead runtime + // (no workspace). The custom hard-timeout value must surface as the + // threshold in the sanitized failure reason (would be 480 min on default). const startedAt = new Date(now - 3 * 60 * 60 * 1000).toISOString(); const updatedAt = new Date(now - 3 * 60 * 60 * 1000).toISOString(); - // Fresh heartbeat — would normally grant grace - const recentHeartbeat = new Date(now - 30 * 1000).toISOString(); const responses = new Map(); responses.set('status IN (\'queued\', \'delegated\', \'in_progress\')', { @@ -674,23 +727,11 @@ describe('recoverStuckTasks', () => { execution_step: 'running', updated_at: updatedAt, started_at: startedAt, - workspace_id: 'ws-1', + workspace_id: null, auto_provisioned_node_id: 'node-1', }, ], }); - responses.set('node_id FROM workspaces', { - results: [{ node_id: 'node-1' }], - }); - responses.set('last_heartbeat_at FROM nodes', { - results: [{ last_heartbeat_at: recentHeartbeat }], - }); - responses.set('node_id, status FROM workspaces', { - results: [{ id: 'ws-1', node_id: 'node-1', status: 'running' }], - }); - responses.set('status, health_status FROM nodes', { - results: [{ id: 'node-1', status: 'running', health_status: 'healthy' }], - }); responses.set('UPDATE tasks SET status = \'failed\'', { results: [], changes: 1, @@ -703,9 +744,16 @@ describe('recoverStuckTasks', () => { }); const result = await recoverStuckTasks(env); - // 3h task > 2h custom hard timeout — killed despite fresh heartbeat + // 3h task > 2h custom hard timeout, no live runtime → failed. expect(result.failedInProgress).toBe(1); expect(result.heartbeatSkipped).toBe(0); + // The custom 2h ceiling (120 min) is honored in the reason, not the 480 default. + expect(syncTriggerExecutionMock).toHaveBeenCalledWith( + env.DATABASE, + 'task-custom', + 'failed', + expect.stringContaining('120 minutes'), + ); }); }); From f79e5a2da4bff7ea5d26a86296654f5dfbb97824 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 18:30:54 +0000 Subject: [PATCH 08/11] fix: make failTask write idempotent against concurrent terminal transition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Security review: failTask read status then wrote WHERE id=? — a concurrent terminal transition landing between the check and the write could be clobbered. Add the terminal-status predicate to the UPDATE so an already-terminal row is never overwritten (defense-in-depth on top of the existing pre-check). Co-Authored-By: Claude Opus 4.6 --- apps/api/src/durable-objects/task-runner/state-machine.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/api/src/durable-objects/task-runner/state-machine.ts b/apps/api/src/durable-objects/task-runner/state-machine.ts index df8e82aeb..c5875d900 100644 --- a/apps/api/src/durable-objects/task-runner/state-machine.ts +++ b/apps/api/src/durable-objects/task-runner/state-machine.ts @@ -221,9 +221,12 @@ export async function failTask( return; } - // Fail the task + // Fail the task. The status predicate makes this idempotent against a + // concurrent terminal transition that lands between the check above and this + // write — never clobber an already-terminal row (completed/failed/cancelled). await rc.env.DATABASE.prepare( - `UPDATE tasks SET status = 'failed', execution_step = NULL, error_message = ?, completed_at = ?, updated_at = ? WHERE id = ?` + `UPDATE tasks SET status = 'failed', execution_step = NULL, error_message = ?, completed_at = ?, updated_at = ? + WHERE id = ? AND status NOT IN ('completed', 'failed', 'cancelled')` ).bind(errorMessage, now, now, state.taskId).run(); // Sync trigger execution status (best-effort) — without this, cron triggers From efcd97450a64bb4dd9c8c431be636fbc3070bc36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sat, 11 Jul 2026 20:58:35 +0000 Subject: [PATCH 09/11] fix: cap live task runtime with absolute ceiling --- .claude/skills/env-reference/SKILL.md | 1 + apps/api/.env.example | 1 + apps/api/src/env.ts | 1 + apps/api/src/scheduled/stuck-tasks.ts | 14 ++ apps/api/tests/unit/stuck-tasks.test.ts | 130 ++++++++++++++++++ .../docs/docs/reference/configuration.md | 3 +- packages/shared/src/constants/index.ts | 1 + .../shared/src/constants/task-execution.ts | 3 + 8 files changed, 153 insertions(+), 1 deletion(-) diff --git a/.claude/skills/env-reference/SKILL.md b/.claude/skills/env-reference/SKILL.md index 06c75a369..f0b7da378 100644 --- a/.claude/skills/env-reference/SKILL.md +++ b/.claude/skills/env-reference/SKILL.md @@ -113,6 +113,7 @@ See `apps/api/.env.example` for the full list. Key variables: - `TASK_RECONCILIATION_PROMPT_SOFT_STALL_MS` — In-flight prompt observation threshold before SAM records a non-interrupting reconciliation event (default: 1800000) - `TASK_RECONCILIATION_PROMPT_HARD_STALL_MS` — In-flight prompt hard-stall threshold before SAM requests prompt cancellation and retries check-in later (default: 7200000) - `TASK_RECONCILIATION_MIN_ALARM_DELAY_MS` — Minimum delay before the next reconciliation alarm can fire (default: 10000) +- `TASK_RUN_ABSOLUTE_CEILING_MS` — Absolute runaway-cost ceiling that fails even a demonstrably live task (default: 86400000 / 24h) - `SESSION_ACTIVITY_STALE_THRESHOLD_MS` — Evidence-based fallback threshold before stale working activity can be healed to idle (default: 300000) - `NODE_HEARTBEAT_STALE_SECONDS` — Staleness threshold for node health - `NODE_AGENT_READY_TIMEOUT_MS` — Max wait for freshly provisioned node-agent health diff --git a/apps/api/.env.example b/apps/api/.env.example index cf5973cb9..f6181f148 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -350,6 +350,7 @@ BASE_DOMAIN=workspaces.example.com # STUCK_TASK_MAX_CANDIDATES_PER_SWEEP=100 # Bound task rows inspected per cron invocation # TASK_LIVENESS_MAX_ACP_SESSIONS=5 # Bound task-scoped ACP sessions inspected per candidate # TASK_LIVENESS_PROBE_TIMEOUT_MS=5000 # Per-candidate timeout for the ACP liveness DO probe (inconclusive on timeout, never fatal) +# TASK_RUN_ABSOLUTE_CEILING_MS=86400000 # 24h — absolute runaway-cost ceiling; fails even a live task # CLAUDE_CODE_COMPACTION_LOOP_DETECTOR_ENABLED=true # CLAUDE_CODE_COMPACTION_LOOP_RECENT_MESSAGE_LIMIT=40 # CLAUDE_CODE_COMPACTION_LOOP_WINDOW_MESSAGES=20 diff --git a/apps/api/src/env.ts b/apps/api/src/env.ts index 080469d65..5c46accc7 100644 --- a/apps/api/src/env.ts +++ b/apps/api/src/env.ts @@ -221,6 +221,7 @@ export interface Env { STUCK_TASK_MAX_CANDIDATES_PER_SWEEP?: string; TASK_LIVENESS_MAX_ACP_SESSIONS?: string; TASK_LIVENESS_PROBE_TIMEOUT_MS?: string; + TASK_RUN_ABSOLUTE_CEILING_MS?: string; CLAUDE_CODE_COMPACTION_LOOP_DETECTOR_ENABLED?: string; CLAUDE_CODE_COMPACTION_LOOP_RECENT_MESSAGE_LIMIT?: string; CLAUDE_CODE_COMPACTION_LOOP_WINDOW_MESSAGES?: string; diff --git a/apps/api/src/scheduled/stuck-tasks.ts b/apps/api/src/scheduled/stuck-tasks.ts index b17c09ba0..8c9d18910 100644 --- a/apps/api/src/scheduled/stuck-tasks.ts +++ b/apps/api/src/scheduled/stuck-tasks.ts @@ -28,6 +28,7 @@ import { DEFAULT_TASK_DO_MISMATCH_GRACE_MS, DEFAULT_TASK_LIVENESS_MAX_ACP_SESSIONS, DEFAULT_TASK_LIVENESS_PROBE_TIMEOUT_MS, + DEFAULT_TASK_RUN_ABSOLUTE_CEILING_MS, DEFAULT_TASK_RUN_HARD_TIMEOUT_MS, DEFAULT_TASK_RUN_MAX_EXECUTION_MS, DEFAULT_TASK_STUCK_DELEGATED_TIMEOUT_MS, @@ -304,6 +305,7 @@ export async function recoverStuckTasks(env: Env): Promise { const delegatedTimeoutMs = parseMs(env.TASK_STUCK_DELEGATED_TIMEOUT_MS, DEFAULT_TASK_STUCK_DELEGATED_TIMEOUT_MS); const maxExecutionMs = parseMs(env.TASK_RUN_MAX_EXECUTION_MS, DEFAULT_TASK_RUN_MAX_EXECUTION_MS); const hardTimeoutMs = parseMs(env.TASK_RUN_HARD_TIMEOUT_MS, DEFAULT_TASK_RUN_HARD_TIMEOUT_MS); + const absoluteCeilingMs = parseMs(env.TASK_RUN_ABSOLUTE_CEILING_MS, DEFAULT_TASK_RUN_ABSOLUTE_CEILING_MS); const mismatchGraceMs = parseMs(env.TASK_DO_MISMATCH_GRACE_MS, DEFAULT_TASK_DO_MISMATCH_GRACE_MS); const maxCandidates = parseMs(env.STUCK_TASK_MAX_CANDIDATES_PER_SWEEP, DEFAULT_STUCK_TASK_MAX_CANDIDATES_PER_SWEEP); @@ -315,6 +317,13 @@ export async function recoverStuckTasks(env: Env): Promise { }); } + if (absoluteCeilingMs <= hardTimeoutMs) { + log.warn('stuck_task.misconfigured_absolute_ceiling', { + absoluteCeilingMs, + hardTimeoutMs, + }); + } + // Find stuck tasks via raw SQL — include workspace_id and auto_provisioned_node_id // for diagnostic context capture. const stuckTasks = await env.DATABASE.prepare( @@ -409,6 +418,11 @@ export async function recoverStuckTasks(env: Env): Promise { const startedAt = task.started_at ? new Date(task.started_at).getTime() : updatedAt; const executionMs = now.getTime() - startedAt; if (executionMs > maxExecutionMs) { + if (executionMs > absoluteCeilingMs) { + isStuck = true; + reason = `Task exceeded the absolute runaway-cost ceiling of ${Math.round(absoluteCeilingMs / 60000)} minutes; live-runtime tasks are bounded to prevent unbounded compute.${stepInfo}`; + break; + } const liveness = await probeLiveness(); if (liveness.live || !liveness.conclusive) { if (liveness.live) { diff --git a/apps/api/tests/unit/stuck-tasks.test.ts b/apps/api/tests/unit/stuck-tasks.test.ts index 97024ee16..ed4850fcf 100644 --- a/apps/api/tests/unit/stuck-tasks.test.ts +++ b/apps/api/tests/unit/stuck-tasks.test.ts @@ -558,6 +558,136 @@ describe('recoverStuckTasks', () => { }); }); + describe('absolute runaway-cost ceiling', () => { + it('fails a live task past the absolute runaway-cost ceiling without probing liveness', async () => { + const now = Date.now(); + const startedAt = new Date(now - 25 * 60 * 60 * 1000).toISOString(); + const recentHeartbeat = new Date(now - 30 * 1000).toISOString(); + const responses = new Map(); + responses.set("status IN ('queued', 'delegated', 'in_progress')", { + results: [ + { + id: 'task-absolute-ceiling', + project_id: 'proj-1', + user_id: 'user-1', + status: 'in_progress', + execution_step: 'running', + updated_at: startedAt, + started_at: startedAt, + workspace_id: 'ws-1', + auto_provisioned_node_id: 'node-1', + }, + ], + }); + responses.set('w.chat_session_id', { + results: [ + { + workspace_status: 'running', + chat_session_id: 'chat-1', + node_id: 'node-1', + node_status: 'running', + health_status: 'healthy', + last_heartbeat_at: recentHeartbeat, + }, + ], + }); + responses.set('node_id, status FROM workspaces', { + results: [{ id: 'ws-1', node_id: 'node-1', status: 'running' }], + }); + responses.set('status, health_status FROM nodes', { + results: [{ id: 'node-1', status: 'running', health_status: 'healthy' }], + }); + responses.set("UPDATE tasks SET status = 'failed'", { results: [], changes: 1 }); + const env = createMockEnv(responses); + const result = await recoverStuckTasks(env); + expect(result.failedInProgress).toBe(1); + expect(result.heartbeatSkipped).toBe(0); + expect(projectDataMocks.listAcpSessions).not.toHaveBeenCalled(); + expect(syncTriggerExecutionMock).toHaveBeenCalledWith( + env.DATABASE, + 'task-absolute-ceiling', + 'failed', + expect.stringContaining('absolute runaway-cost ceiling') + ); + }); + + it('preserves a live task below the absolute runaway-cost ceiling', async () => { + const now = Date.now(); + const startedAt = new Date(now - 23 * 60 * 60 * 1000).toISOString(); + const recentHeartbeat = new Date(now - 30 * 1000).toISOString(); + const responses = new Map(); + responses.set("status IN ('queued', 'delegated', 'in_progress')", { + results: [ + { + id: 'task-below-absolute-ceiling', + project_id: 'proj-1', + user_id: 'user-1', + status: 'in_progress', + execution_step: 'running', + updated_at: startedAt, + started_at: startedAt, + workspace_id: 'ws-1', + auto_provisioned_node_id: 'node-1', + }, + ], + }); + responses.set('w.chat_session_id', { + results: [ + { + workspace_status: 'running', + chat_session_id: 'chat-1', + node_id: 'node-1', + node_status: 'running', + health_status: 'healthy', + last_heartbeat_at: recentHeartbeat, + }, + ], + }); + const result = await recoverStuckTasks(createMockEnv(responses)); + expect(result.failedInProgress).toBe(0); + expect(result.heartbeatSkipped).toBe(1); + }); + + it('falls back to the default absolute ceiling for an invalid zero value', async () => { + const now = Date.now(); + const startedAt = new Date(now - 23 * 60 * 60 * 1000).toISOString(); + const recentHeartbeat = new Date(now - 30 * 1000).toISOString(); + const responses = new Map(); + responses.set("status IN ('queued', 'delegated', 'in_progress')", { + results: [ + { + id: 'task-invalid-absolute-ceiling', + project_id: 'proj-1', + user_id: 'user-1', + status: 'in_progress', + execution_step: 'running', + updated_at: startedAt, + started_at: startedAt, + workspace_id: 'ws-1', + auto_provisioned_node_id: 'node-1', + }, + ], + }); + responses.set('w.chat_session_id', { + results: [ + { + workspace_status: 'running', + chat_session_id: 'chat-1', + node_id: 'node-1', + node_status: 'running', + health_status: 'healthy', + last_heartbeat_at: recentHeartbeat, + }, + ], + }); + const result = await recoverStuckTasks( + createMockEnv(responses, { TASK_RUN_ABSOLUTE_CEILING_MS: '0' }) + ); + expect(result.failedInProgress).toBe(0); + expect(result.heartbeatSkipped).toBe(1); + }); + }); + describe('hard timeout enforcement', () => { it('preserves a genuinely live task past the hard timeout', async () => { const now = Date.now(); diff --git a/apps/www/src/content/docs/docs/reference/configuration.md b/apps/www/src/content/docs/docs/reference/configuration.md index 154ed5bfb..bc4f1c0ca 100644 --- a/apps/www/src/content/docs/docs/reference/configuration.md +++ b/apps/www/src/content/docs/docs/reference/configuration.md @@ -207,6 +207,7 @@ SAM loads OpenCode Zen and OpenCode Go model choices through the authenticated m | `STUCK_TASK_MAX_CANDIDATES_PER_SWEEP` | `100` | Maximum active tasks inspected by each recovery sweep | | `TASK_LIVENESS_MAX_ACP_SESSIONS` | `5` | Maximum task-scoped ACP sessions inspected per liveness probe | | `TASK_LIVENESS_PROBE_TIMEOUT_MS` | `5000` (5 sec) | Per-candidate timeout for the ACP liveness probe; a timeout is inconclusive (never fails a task) | +| `TASK_RUN_ABSOLUTE_CEILING_MS` | `86400000` (24 hr) | Absolute runaway-cost ceiling; fails even a task with a demonstrably live runtime | | `CLAUDE_CODE_COMPACTION_LOOP_DETECTOR_ENABLED` | `true` | Enable Claude Code compaction-loop shutdown from recent message evidence | | `CLAUDE_CODE_COMPACTION_LOOP_RECENT_MESSAGE_LIMIT` | `40` | Recent task-session messages to inspect for compaction-loop evidence | | `CLAUDE_CODE_COMPACTION_LOOP_WINDOW_MESSAGES` | `20` | Rolling recent-message window used for compaction-loop detection | @@ -220,7 +221,7 @@ SAM loads OpenCode Zen and OpenCode Go model choices through the authenticated m | `TASK_RECONCILIATION_PROMPT_HARD_STALL_MS` | `7200000` (2 hr) | In-flight prompt hard-stall threshold before SAM requests prompt cancellation | | `TASK_RECONCILIATION_MIN_ALARM_DELAY_MS` | `10000` (10 sec) | Minimum delay before the next reconciliation alarm can fire | -> **Liveness-gated recovery.** Stuck-task recovery for `in_progress` tasks (including task-mode work paused at the `awaiting_followup` execution step) is gated on **task-scoped** liveness — a live workspace, a healthy node with a recent heartbeat, **and** an active task-scoped ACP session. A shared-node heartbeat alone is never sufficient. Consequently, `TASK_RUN_HARD_TIMEOUT_MS` and `TASK_RUN_MAX_EXECUTION_MS` bound the point at which a task with **no** proven live runtime is failed; a task with a demonstrably live runtime is preserved past those thresholds rather than terminated on elapsed time alone. When liveness cannot be determined (probe timeout or error), the task is left untouched (fail-safe). +> **Liveness-gated recovery.** Stuck-task recovery for `in_progress` tasks (including task-mode work paused at the `awaiting_followup` execution step) is gated on **task-scoped** liveness — a live workspace, a healthy node with a recent heartbeat, **and** an active task-scoped ACP session. A shared-node heartbeat alone is never sufficient. Consequently, `TASK_RUN_HARD_TIMEOUT_MS` and `TASK_RUN_MAX_EXECUTION_MS` bound the point at which a task with **no** proven live runtime is failed; a task with a demonstrably live runtime is preserved past those thresholds, but remains bounded by `TASK_RUN_ABSOLUTE_CEILING_MS` (24 hours by default) as a runaway-cost backstop. When liveness cannot be determined (probe timeout or error), the task is left untouched (fail-safe) until it reaches that absolute ceiling. ## Node & Workspace Readiness diff --git a/packages/shared/src/constants/index.ts b/packages/shared/src/constants/index.ts index 96c51cb99..2c893694e 100644 --- a/packages/shared/src/constants/index.ts +++ b/packages/shared/src/constants/index.ts @@ -73,6 +73,7 @@ export { DEFAULT_TASK_DO_MISMATCH_GRACE_MS, DEFAULT_TASK_LIVENESS_MAX_ACP_SESSIONS, DEFAULT_TASK_LIVENESS_PROBE_TIMEOUT_MS, + DEFAULT_TASK_RUN_ABSOLUTE_CEILING_MS, DEFAULT_TASK_RUN_CLEANUP_DELAY_MS, DEFAULT_TASK_RUN_HARD_TIMEOUT_MS, DEFAULT_TASK_RUN_MAX_EXECUTION_MS, diff --git a/packages/shared/src/constants/task-execution.ts b/packages/shared/src/constants/task-execution.ts index 4a1f9ba3a..b62b8910d 100644 --- a/packages/shared/src/constants/task-execution.ts +++ b/packages/shared/src/constants/task-execution.ts @@ -30,6 +30,9 @@ export const DEFAULT_TASK_RUN_MAX_EXECUTION_MS = 4 * 60 * 60 * 1000; // 4 hours * Override via TASK_RUN_HARD_TIMEOUT_MS env var. */ export const DEFAULT_TASK_RUN_HARD_TIMEOUT_MS = 8 * 60 * 60 * 1000; // 8 hours +/** Absolute runaway-cost backstop (ms) that bounds even demonstrably live tasks. */ +export const DEFAULT_TASK_RUN_ABSOLUTE_CEILING_MS = 24 * 60 * 60 * 1000; + /** Default threshold (ms) for a task stuck in 'queued' status. Override via TASK_STUCK_QUEUED_TIMEOUT_MS env var. * Must be > TASK_RUNNER_AGENT_READY_TIMEOUT_MS (15 min) to avoid the stuck-task cron killing tasks * that are legitimately waiting for cloud-init to finish. Cloud-init takes 8-12 min on Hetzner. From b9f1113b7d2a666961f1e01475dee7e53826de34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sun, 12 Jul 2026 15:27:53 +0000 Subject: [PATCH 10/11] fix: reconcile dead task runtimes without DO state --- .claude/skills/api-reference/SKILL.md | 6 + apps/api/.env.example | 2 +- apps/api/src/index.ts | 40 +- apps/api/src/routes/admin.ts | 132 +++-- apps/api/src/scheduled/stuck-tasks.ts | 546 ++++++++++++++---- .../tests/unit/recovery-resilience.test.ts | 54 +- .../tests/unit/routes/admin-security.test.ts | 43 ++ apps/api/tests/unit/stuck-tasks.test.ts | 340 +++++++++-- .../workers/scheduled-stuck-tasks.test.ts | 99 +++- .../docs/docs/reference/configuration.md | 2 +- ...-taskrunner-d1-lifecycle-reconciliation.md | 11 +- 11 files changed, 1001 insertions(+), 274 deletions(-) diff --git a/.claude/skills/api-reference/SKILL.md b/.claude/skills/api-reference/SKILL.md index d3edcc576..521cc5052 100644 --- a/.claude/skills/api-reference/SKILL.md +++ b/.claude/skills/api-reference/SKILL.md @@ -60,6 +60,12 @@ user-invocable: false - `POST /api/projects/:projectId/tasks/:taskId/delegate` — Delegate ready+unblocked task to owned running workspace - `GET /api/projects/:projectId/tasks/:taskId/events` — List append-only task status events +## Administration (Superadmin Only) + +- `GET /api/admin/tasks/stuck` — List tasks currently in transient states +- `GET /api/admin/tasks/:taskId/reconciliation-diagnostics` — Read the TaskRunner probe, task-scoped runtime liveness, eligibility threshold, and reconciliation decision without mutating task state +- `GET /api/admin/tasks/recent-failures` — List recent failed tasks with error details + ## Agent Sessions - `GET /api/workspaces/:id/agent-sessions` — List workspace agent sessions diff --git a/apps/api/.env.example b/apps/api/.env.example index f6181f148..3a9718c9e 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -344,7 +344,7 @@ BASE_DOMAIN=workspaces.example.com # Task execution timeout (stuck task recovery) # TASK_RUN_MAX_EXECUTION_MS=14400000 # 4 hours — max time before task is failed -# TASK_STUCK_QUEUED_TIMEOUT_MS=600000 # 10 minutes — max time in 'queued' state +# TASK_STUCK_QUEUED_TIMEOUT_MS=1200000 # 20 minutes — max time in 'queued' state # TASK_STUCK_DELEGATED_TIMEOUT_MS=1860000 # 31 minutes — max time in 'delegated' state (> workspace ready timeout) # TASK_DO_MISMATCH_GRACE_MS=300000 # 5 minutes — minimum age before DO/D1 liveness reconciliation # STUCK_TASK_MAX_CANDIDATES_PER_SWEEP=100 # Bound task rows inspected per cron invocation diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index e3f48eec5..af53d3ef9 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -375,11 +375,13 @@ h1{font-size:1.4rem}code{background:#f0f0f0;padding:2px 6px;border-radius:3px;fo } const nodeRuntime = workspace.nodeId - ? (await db - .select({ runtime: schema.nodes.runtime }) - .from(schema.nodes) - .where(eq(schema.nodes.id, workspace.nodeId)) - .get())?.runtime ?? 'vm' + ? (( + await db + .select({ runtime: schema.nodes.runtime }) + .from(schema.nodes) + .where(eq(schema.nodes.id, workspace.nodeId)) + .get() + )?.runtime ?? 'vm') : 'vm'; if (workspace.status !== 'running' && workspace.status !== 'recovery') { @@ -400,10 +402,16 @@ h1{font-size:1.4rem}code{background:#f0f0f0;padding:2px 6px;border-radius:3px;fo if (nodeRuntime === 'cf-container') { const containerConfig = getVmAgentContainerConfig(c.env); if (!containerConfig.enabled) { - return c.json({ error: 'CF_CONTAINER_DISABLED', message: 'Container workspace runtime is disabled' }, 503); + return c.json( + { error: 'CF_CONTAINER_DISABLED', message: 'Container workspace runtime is disabled' }, + 503 + ); } if (!c.env.VM_AGENT_CONTAINER) { - return c.json({ error: 'CF_CONTAINER_UNAVAILABLE', message: 'VM agent container binding is unavailable' }, 503); + return c.json( + { error: 'CF_CONTAINER_UNAVAILABLE', message: 'VM agent container binding is unavailable' }, + 503 + ); } const containerId = workspace.nodeId || workspaceId; @@ -425,7 +433,10 @@ h1{font-size:1.4rem}code{background:#f0f0f0;padding:2px 6px;border-radius:3px;fo workspaceId, ...serializeError(err), }); - return c.json({ error: 'TOKEN_ERROR', message: 'Failed to generate port proxy token' }, 500); + return c.json( + { error: 'TOKEN_ERROR', message: 'Failed to generate port proxy token' }, + 500 + ); } } @@ -661,7 +672,8 @@ app.get('/api/config/artifacts-enabled', (c) => { // only render a provider button when that provider is actually usable. Google // here means the LOGIN client (getGoogleLoginOAuthConfig), never the infra/GCP one. app.get('/api/config/login-providers', async (c) => { - const { getGitHubOAuthConfig, getGoogleLoginOAuthConfig } = await import('./services/platform-config'); + const { getGitHubOAuthConfig, getGoogleLoginOAuthConfig } = + await import('./services/platform-config'); const [github, google] = await Promise.all([ getGitHubOAuthConfig(c.env), getGoogleLoginOAuthConfig(c.env), @@ -929,6 +941,10 @@ export default { } // 5-minute operational sweep + // Recover stuck tasks first so an unrelated cleanup failure cannot suppress + // the task lifecycle safety net for another five-minute interval. + const stuckTasks = await recoverStuckTasks(env); + // Check for stuck provisioning workspaces const timedOut = await checkProvisioningTimeouts(env.DATABASE, env, env.OBSERVABILITY_DATABASE); @@ -939,9 +955,6 @@ export default { // Clean up stale warm nodes and expired auto-provisioned nodes const nodeCleanup = await runNodeCleanupSweep(env); - // Recover stuck tasks (queued/delegated/in_progress past timeout) - const stuckTasks = await recoverStuckTasks(env); - // Purge expired observability errors (retention + row count limits) const observabilityPurge = await runObservabilityPurge(env); @@ -979,6 +992,9 @@ export default { stuckTasksHeartbeatSkipped: stuckTasks.heartbeatSkipped, stuckTaskErrors: stuckTasks.errors, stuckTaskDoHealthChecked: stuckTasks.doHealthChecked, + stuckTaskDoHealthMissing: stuckTasks.doHealthMissing, + stuckTaskDoHealthErrors: stuckTasks.doHealthErrors, + stuckTaskDeadRuntimeReconciled: stuckTasks.deadRuntimeReconciled, observabilityPurgedByAge: observabilityPurge.deletedByAge, observabilityPurgedByCount: observabilityPurge.deletedByCount, cronTriggersChecked: cronTriggers.checked, diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.ts index 3f0ac2f27..b02f8c523 100644 --- a/apps/api/src/routes/admin.ts +++ b/apps/api/src/routes/admin.ts @@ -1,4 +1,9 @@ -import type { PlatformErrorLevel,PlatformErrorSource, UserRole, UserStatus } from '@simple-agent-manager/shared'; +import type { + PlatformErrorLevel, + PlatformErrorSource, + UserRole, + UserStatus, +} from '@simple-agent-manager/shared'; import { desc, eq, inArray } from 'drizzle-orm'; import { drizzle } from 'drizzle-orm/d1'; import { Hono } from 'hono'; @@ -6,12 +11,26 @@ import { Hono } from 'hono'; import * as schema from '../db/schema'; import type { ProjectData as ProjectDataDO } from '../durable-objects/project-data'; import type { Env } from '../env'; -import { getUserId,requireApproved, requireAuth, requireSuperadmin } from '../middleware/auth'; +import { getUserId, requireApproved, requireAuth, requireSuperadmin } from '../middleware/auth'; import { errors } from '../middleware/error'; import { rateLimit } from '../middleware/rate-limit'; -import { AdminLogQuerySchema,AdminUserActionSchema, AdminUserRoleSchema, jsonValidator, UpdateSignupApprovalConfigSchema } from '../schemas'; +import { getTaskReconciliationDiagnostics } from '../scheduled/stuck-tasks'; +import { + AdminLogQuerySchema, + AdminUserActionSchema, + AdminUserRoleSchema, + jsonValidator, + UpdateSignupApprovalConfigSchema, +} from '../schemas'; import { getRuntimeLimits } from '../services/limits'; -import { CfApiError,getErrorTrends, getHealthSummary, getLogQueryRateLimit, queryCloudflareLogs, queryErrors } from '../services/observability'; +import { + CfApiError, + getErrorTrends, + getHealthSummary, + getLogQueryRateLimit, + queryCloudflareLogs, + queryErrors, +} from '../services/observability'; import { getSignupApprovalConfig, setSignupApprovalConfig } from '../services/signup-approval'; const adminRoutes = new Hono<{ Bindings: Env }>(); @@ -196,6 +215,18 @@ adminRoutes.get('/tasks/stuck', async (c) => { return c.json({ tasks: tasksWithAge }); }); +/** + * GET /api/admin/tasks/:taskId/reconciliation-diagnostics - Explain the + * read-only evidence and decision used by scheduled task reconciliation. + */ +adminRoutes.get('/tasks/:taskId/reconciliation-diagnostics', async (c) => { + const { taskId } = c.req.param(); + const diagnostics = await getTaskReconciliationDiagnostics(c.env, taskId); + + if (!diagnostics) throw errors.notFound('Task'); + return c.json({ diagnostics }); +}); + /** * GET /api/admin/tasks/recent-failures - List recently failed tasks with error details * @@ -272,8 +303,8 @@ adminRoutes.get('/observability/errors', async (c) => { } const result = await queryErrors(c.env.OBSERVABILITY_DATABASE, { - source: source && source !== 'all' ? source as PlatformErrorSource : undefined, - level: level && level !== 'all' ? level as PlatformErrorLevel : undefined, + source: source && source !== 'all' ? (source as PlatformErrorSource) : undefined, + level: level && level !== 'all' ? (level as PlatformErrorLevel) : undefined, search: search || undefined, startTime: startTime ? new Date(startTime).getTime() : undefined, endTime: endTime ? new Date(endTime).getTime() : undefined, @@ -327,7 +358,8 @@ adminRoutes.get('/observability/trends', async (c) => { * * Body: { timeRange: { start, end }, levels?, search?, limit?, cursor? } */ -adminRoutes.post('/observability/logs/query', +adminRoutes.post( + '/observability/logs/query', // Per-admin KV-based rate limiting (1-minute window) async (c, next) => { const limiter = rateLimit({ @@ -339,54 +371,59 @@ adminRoutes.post('/observability/logs/query', }, jsonValidator(AdminLogQuerySchema), async (c) => { - if (!c.env.CF_API_TOKEN || !c.env.CF_ACCOUNT_ID) { - throw errors.badRequest('Cloudflare API credentials not configured. Set CF_API_TOKEN and CF_ACCOUNT_ID.'); - } + if (!c.env.CF_API_TOKEN || !c.env.CF_ACCOUNT_ID) { + throw errors.badRequest( + 'Cloudflare API credentials not configured. Set CF_API_TOKEN and CF_ACCOUNT_ID.' + ); + } - const body = c.req.valid('json'); + const body = c.req.valid('json'); - // Validate dates - const startDate = new Date(body.timeRange.start); - const endDate = new Date(body.timeRange.end); - if (isNaN(startDate.getTime()) || isNaN(endDate.getTime())) { - throw errors.badRequest('timeRange start and end must be valid ISO 8601 dates'); - } + // Validate dates + const startDate = new Date(body.timeRange.start); + const endDate = new Date(body.timeRange.end); + if (isNaN(startDate.getTime()) || isNaN(endDate.getTime())) { + throw errors.badRequest('timeRange start and end must be valid ISO 8601 dates'); + } - // Validate levels - if (body.levels) { - const validLogLevels = new Set(['error', 'warn', 'info', 'debug', 'log']); - for (const level of body.levels) { - if (!validLogLevels.has(level)) { - throw errors.badRequest(`Invalid level: ${level}. Must be one of: error, warn, info, debug, log`); + // Validate levels + if (body.levels) { + const validLogLevels = new Set(['error', 'warn', 'info', 'debug', 'log']); + for (const level of body.levels) { + if (!validLogLevels.has(level)) { + throw errors.badRequest( + `Invalid level: ${level}. Must be one of: error, warn, info, debug, log` + ); + } } } - } - // Validate limit - if (body.limit !== undefined && (body.limit < 1 || body.limit > 500)) { - throw errors.badRequest('limit must be between 1 and 500'); - } + // Validate limit + if (body.limit !== undefined && (body.limit < 1 || body.limit > 500)) { + throw errors.badRequest('limit must be between 1 and 500'); + } - try { - const result = await queryCloudflareLogs({ - cfApiToken: c.env.CF_API_TOKEN, - cfAccountId: c.env.CF_ACCOUNT_ID, - timeRange: { start: body.timeRange.start, end: body.timeRange.end }, - levels: body.levels ?? undefined, - search: body.search || undefined, - limit: body.limit, - cursor: body.cursor || undefined, - queryId: body.queryId || undefined, - }); + try { + const result = await queryCloudflareLogs({ + cfApiToken: c.env.CF_API_TOKEN, + cfAccountId: c.env.CF_ACCOUNT_ID, + timeRange: { start: body.timeRange.start, end: body.timeRange.end }, + levels: body.levels ?? undefined, + search: body.search || undefined, + limit: body.limit, + cursor: body.cursor || undefined, + queryId: body.queryId || undefined, + }); - return c.json(result); - } catch (err) { - if (err instanceof CfApiError) { - return c.json({ error: 'CF_API_ERROR', message: err.message }, 502); + return c.json(result); + } catch (err) { + if (err instanceof CfApiError) { + return c.json({ error: 'CF_API_ERROR', message: err.message }, 502); + } + throw err; } - throw err; } -}); +); /** * GET /api/admin/observability/logs/stream - WebSocket upgrade for real-time log stream @@ -465,7 +502,10 @@ adminRoutes.post('/backfill-session-summaries', async (c) => { const stub = c.env.PROJECT_DATA.get(doId) as DurableObjectStub; // List all sessions from the DO (up to 1000) - const result = await stub.listSessions(null, 1000, 0) as { sessions: Record[]; total: number }; + const result = (await stub.listSessions(null, 1000, 0)) as { + sessions: Record[]; + total: number; + }; if (result.sessions.length === 0) continue; diff --git a/apps/api/src/scheduled/stuck-tasks.ts b/apps/api/src/scheduled/stuck-tasks.ts index 8c9d18910..d808ab769 100644 --- a/apps/api/src/scheduled/stuck-tasks.ts +++ b/apps/api/src/scheduled/stuck-tasks.ts @@ -45,7 +45,10 @@ import { persistError } from '../services/observability'; import * as projectDataService from '../services/project-data'; import { cleanupTaskRun } from '../services/task-runner'; import { syncTriggerExecutionStatus } from '../services/trigger-execution-sync'; -import { type CompactionLoopRecovery, detectTaskCompactionLoop } from './claude-code-compaction-loop'; +import { + type CompactionLoopRecovery, + detectTaskCompactionLoop, +} from './claude-code-compaction-loop'; function parseMs(value: string | undefined, fallback: number): number { if (!value) return fallback; @@ -78,9 +81,24 @@ export interface StuckTaskResult { failedCompactionLoops: number; heartbeatSkipped: number; doHealthChecked: number; + doHealthMissing: number; + doHealthErrors: number; + deadRuntimeReconciled: number; errors: number; } +export interface TaskRunnerProbeResult { + outcome: 'ok' | 'missing' | 'timeout' | 'error'; + status: { + completed: boolean; + currentStep: string; + retryCount: number; + lastStepAt: number; + } | null; + error: string | null; + durationMs: number; +} + /** * Diagnostic context captured at recovery time for a stuck task. * Recorded in the OBSERVABILITY_DATABASE to enable post-mortem analysis @@ -99,11 +117,13 @@ export interface RecoveryDiagnostics { nodeHealthStatus: string | null; autoProvisionedNodeId: string | null; doState: { + outcome: TaskRunnerProbeResult['outcome']; exists: boolean; completed: boolean | null; currentStep: string | null; retryCount: number | null; lastStepAt: number | null; + error: string | null; } | null; } @@ -116,16 +136,114 @@ export interface TaskRuntimeLiveness { activeAcpSessionId: string | null; } +export type TaskReconciliationDecision = + | 'not_active' + | 'within_grace' + | 'reconcile_dead_runtime' + | 'preserve_live_runtime' + | 'preserve_inconclusive_runtime' + | 'observe_orchestration'; + +export interface TaskReconciliationDiagnostics { + taskId: string; + status: string; + executionStep: string | null; + workspaceId: string | null; + elapsedMs: number; + eligibilityThresholdMs: number; + eligible: boolean; + decision: TaskReconciliationDecision; + liveness: TaskRuntimeLiveness | null; + taskRunner: TaskRunnerProbeResult; +} + const LIVE_WORKSPACE_STATUSES = new Set(['running', 'recovery']); const ACTIVE_ACP_STATUSES = new Set(['assigned', 'running']); +/** Bound and classify the cross-DO status RPC used by reconciliation diagnostics. */ +export async function probeTaskRunnerStatus( + env: Env, + taskId: string +): Promise { + const startedAt = Date.now(); + const probeTimeoutMs = parseMs( + env.TASK_LIVENESS_PROBE_TIMEOUT_MS, + DEFAULT_TASK_LIVENESS_PROBE_TIMEOUT_MS + ); + const timeout = Symbol('task_runner_probe_timeout'); + let timer: ReturnType | undefined; + + try { + const doId = env.TASK_RUNNER.idFromName(taskId); + const stub = env.TASK_RUNNER.get(doId) as DurableObjectStub; + const result = await Promise.race([ + stub.getStatus(), + new Promise((resolve) => { + timer = setTimeout(() => resolve(timeout), probeTimeoutMs); + }), + ]); + + if (result === timeout) { + log.warn('stuck_task.task_runner_probe_timeout', { taskId, probeTimeoutMs }); + return { + outcome: 'timeout', + status: null, + error: `TaskRunner status probe exceeded ${probeTimeoutMs}ms`, + durationMs: Date.now() - startedAt, + }; + } + + if (!result) { + log.warn('stuck_task.task_runner_state_missing', { taskId }); + return { + outcome: 'missing', + status: null, + error: null, + durationMs: Date.now() - startedAt, + }; + } + + return { + outcome: 'ok', + status: { + completed: result.completed, + currentStep: result.currentStep, + retryCount: result.retryCount, + lastStepAt: result.lastStepAt, + }, + error: null, + durationMs: Date.now() - startedAt, + }; + } catch (err) { + const error = err instanceof Error ? err.message : String(err); + log.warn('stuck_task.task_runner_probe_failed', { taskId, error }); + return { + outcome: 'error', + status: null, + error, + durationMs: Date.now() - startedAt, + }; + } finally { + if (timer) clearTimeout(timer); + } +} + /** Prove task-scoped liveness. A shared-node heartbeat is never sufficient. */ export async function getTaskRuntimeLiveness( env: Env, - task: { project_id: string; workspace_id: string | null }, + task: { project_id: string; workspace_id: string | null } ): Promise { - const dead = (reason: string, workspaceStatus: string | null, nodeId: string | null): TaskRuntimeLiveness => ({ - live: false, conclusive: true, reason, workspaceStatus, nodeId, activeAcpSessionId: null, + const dead = ( + reason: string, + workspaceStatus: string | null, + nodeId: string | null + ): TaskRuntimeLiveness => ({ + live: false, + conclusive: true, + reason, + workspaceStatus, + nodeId, + activeAcpSessionId: null, }); if (!task.workspace_id) return dead('workspace_missing', null, null); @@ -136,35 +254,58 @@ export async function getTaskRuntimeLiveness( LEFT JOIN nodes n ON n.id = w.node_id WHERE w.id = ? LIMIT 1` - ).bind(task.workspace_id).first<{ - workspace_status: string; - chat_session_id: string | null; - node_id: string | null; - node_status: string | null; - health_status: string | null; - last_heartbeat_at: string | null; - }>(); + ) + .bind(task.workspace_id) + .first<{ + workspace_status: string; + chat_session_id: string | null; + node_id: string | null; + node_status: string | null; + health_status: string | null; + last_heartbeat_at: string | null; + }>(); if (!row) return dead('workspace_missing', null, null); if (!LIVE_WORKSPACE_STATUSES.has(row.workspace_status)) { return dead(`workspace_${row.workspace_status}`, row.workspace_status, row.node_id); } if (!row.chat_session_id || !row.node_id) { - return { live: false, conclusive: false, reason: 'workspace_runtime_identity_incomplete', workspaceStatus: row.workspace_status, nodeId: row.node_id, activeAcpSessionId: null }; + return { + live: false, + conclusive: false, + reason: 'workspace_runtime_identity_incomplete', + workspaceStatus: row.workspace_status, + nodeId: row.node_id, + activeAcpSessionId: null, + }; } - const staleSeconds = parseInt(env.NODE_HEARTBEAT_STALE_SECONDS || '', 10) || DEFAULT_NODE_HEARTBEAT_STALE_SECONDS; + const staleSeconds = + parseInt(env.NODE_HEARTBEAT_STALE_SECONDS || '', 10) || DEFAULT_NODE_HEARTBEAT_STALE_SECONDS; const staleMs = staleSeconds * 1000; - const nodeHeartbeatAt = row.last_heartbeat_at ? new Date(row.last_heartbeat_at).getTime() : Number.NaN; - if (row.node_status !== 'running' || row.health_status !== 'healthy' || !Number.isFinite(nodeHeartbeatAt) || Date.now() - nodeHeartbeatAt > staleMs) { + const nodeHeartbeatAt = row.last_heartbeat_at + ? new Date(row.last_heartbeat_at).getTime() + : Number.NaN; + if ( + row.node_status !== 'running' || + row.health_status !== 'healthy' || + !Number.isFinite(nodeHeartbeatAt) || + Date.now() - nodeHeartbeatAt > staleMs + ) { return dead('node_not_live', row.workspace_status, row.node_id); } try { - const limit = parseMs(env.TASK_LIVENESS_MAX_ACP_SESSIONS, DEFAULT_TASK_LIVENESS_MAX_ACP_SESSIONS); + const limit = parseMs( + env.TASK_LIVENESS_MAX_ACP_SESSIONS, + DEFAULT_TASK_LIVENESS_MAX_ACP_SESSIONS + ); // Bound the ProjectData DO probe so a slow/unresponsive DO cannot stall the // control-loop sweep (rule 47). A timeout is inconclusive, never fatal. - const probeTimeoutMs = parseMs(env.TASK_LIVENESS_PROBE_TIMEOUT_MS, DEFAULT_TASK_LIVENESS_PROBE_TIMEOUT_MS); + const probeTimeoutMs = parseMs( + env.TASK_LIVENESS_PROBE_TIMEOUT_MS, + DEFAULT_TASK_LIVENESS_PROBE_TIMEOUT_MS + ); const TIMEOUT = Symbol('liveness_probe_timeout'); let timer: ReturnType | undefined; const probe = await Promise.race([ @@ -182,16 +323,32 @@ export async function getTaskRuntimeLiveness( workspaceId: task.workspace_id, probeTimeoutMs, }); - return { live: false, conclusive: false, reason: 'task_liveness_timeout', workspaceStatus: row.workspace_status, nodeId: row.node_id, activeAcpSessionId: null }; + return { + live: false, + conclusive: false, + reason: 'task_liveness_timeout', + workspaceStatus: row.workspace_status, + nodeId: row.node_id, + activeAcpSessionId: null, + }; } const { sessions } = probe; const active = sessions.find((session) => { - if (!ACTIVE_ACP_STATUSES.has(session.status) || session.workspaceId !== task.workspace_id) return false; - const heartbeatAt = session.lastHeartbeatAt ?? session.updatedAt ?? session.startedAt ?? session.createdAt; + if (!ACTIVE_ACP_STATUSES.has(session.status) || session.workspaceId !== task.workspace_id) + return false; + const heartbeatAt = + session.lastHeartbeatAt ?? session.updatedAt ?? session.startedAt ?? session.createdAt; return Number.isFinite(heartbeatAt) && Date.now() - heartbeatAt <= staleMs; }); if (active) { - return { live: true, conclusive: true, reason: 'task_acp_session_live', workspaceStatus: row.workspace_status, nodeId: row.node_id, activeAcpSessionId: active.id }; + return { + live: true, + conclusive: true, + reason: 'task_acp_session_live', + workspaceStatus: row.workspace_status, + nodeId: row.node_id, + activeAcpSessionId: active.id, + }; } return dead('task_acp_session_not_live', row.workspace_status, row.node_id); } catch (err) { @@ -199,10 +356,108 @@ export async function getTaskRuntimeLiveness( workspaceId: task.workspace_id, error: err instanceof Error ? err.message : String(err), }); - return { live: false, conclusive: false, reason: 'task_liveness_unknown', workspaceStatus: row.workspace_status, nodeId: row.node_id, activeAcpSessionId: null }; + return { + live: false, + conclusive: false, + reason: 'task_liveness_unknown', + workspaceStatus: row.workspace_status, + nodeId: row.node_id, + activeAcpSessionId: null, + }; } } +/** + * Explain the evidence the scheduled reconciler would use for one task. + * This helper is deliberately read-only so superadmins can inspect live state + * without mutating D1 or either Durable Object. + */ +export async function getTaskReconciliationDiagnostics( + env: Env, + taskId: string +): Promise { + const task = await env.DATABASE.prepare( + `SELECT id, project_id, status, execution_step, started_at, updated_at, workspace_id + FROM tasks + WHERE id = ? + LIMIT 1` + ) + .bind(taskId) + .first<{ + id: string; + project_id: string; + status: string; + execution_step: string | null; + started_at: string | null; + updated_at: string; + workspace_id: string | null; + }>(); + + if (!task) return null; + + const now = Date.now(); + const updatedAt = new Date(task.updated_at).getTime(); + const elapsedMs = now - updatedAt; + const mismatchGraceMs = parseMs(env.TASK_DO_MISMATCH_GRACE_MS, DEFAULT_TASK_DO_MISMATCH_GRACE_MS); + const queuedTimeoutMs = parseMs( + env.TASK_STUCK_QUEUED_TIMEOUT_MS, + DEFAULT_TASK_STUCK_QUEUED_TIMEOUT_MS + ); + const delegatedTimeoutMs = parseMs( + env.TASK_STUCK_DELEGATED_TIMEOUT_MS, + DEFAULT_TASK_STUCK_DELEGATED_TIMEOUT_MS + ); + const maxExecutionMs = parseMs(env.TASK_RUN_MAX_EXECUTION_MS, DEFAULT_TASK_RUN_MAX_EXECUTION_MS); + + let timeForCheck = elapsedMs; + let halfThreshold = mismatchGraceMs; + if (task.status === 'queued') { + halfThreshold = queuedTimeoutMs / 2; + } else if (task.status === 'delegated') { + halfThreshold = delegatedTimeoutMs / 2; + } else if (task.status === 'in_progress') { + const startedAt = task.started_at ? new Date(task.started_at).getTime() : updatedAt; + timeForCheck = now - startedAt; + halfThreshold = maxExecutionMs / 2; + } + + const eligibilityThresholdMs = Math.min(halfThreshold, mismatchGraceMs); + const active = ['queued', 'delegated', 'in_progress'].includes(task.status); + const eligible = active && timeForCheck > eligibilityThresholdMs; + const [taskRunner, liveness] = await Promise.all([ + probeTaskRunnerStatus(env, task.id), + task.status === 'in_progress' ? getTaskRuntimeLiveness(env, task) : Promise.resolve(null), + ]); + + let decision: TaskReconciliationDecision; + if (!active) { + decision = 'not_active'; + } else if (!eligible) { + decision = 'within_grace'; + } else if (liveness?.conclusive && !liveness.live) { + decision = 'reconcile_dead_runtime'; + } else if (liveness?.live) { + decision = 'preserve_live_runtime'; + } else if (task.status === 'in_progress') { + decision = 'preserve_inconclusive_runtime'; + } else { + decision = 'observe_orchestration'; + } + + return { + taskId: task.id, + status: task.status, + executionStep: task.execution_step, + workspaceId: task.workspace_id, + elapsedMs: timeForCheck, + eligibilityThresholdMs, + eligible, + decision, + liveness, + taskRunner, + }; +} + /** * Query diagnostic context for a stuck task — workspace status, node status, * and TaskRunner DO state. Best-effort: returns whatever context is available. @@ -217,7 +472,8 @@ export async function gatherDiagnostics( auto_provisioned_node_id: string | null; }, elapsedMs: number, - reason: string + reason: string, + taskRunnerProbe?: TaskRunnerProbeResult ): Promise { const diagnostics: RecoveryDiagnostics = { taskId: task.id, @@ -239,7 +495,9 @@ export async function gatherDiagnostics( try { const wsResult = await env.DATABASE.prepare( `SELECT id, node_id, status FROM workspaces WHERE id = ?` - ).bind(task.workspace_id).first<{ id: string; node_id: string | null; status: string }>(); + ) + .bind(task.workspace_id) + .first<{ id: string; node_id: string | null; status: string }>(); if (wsResult) { diagnostics.workspaceStatus = wsResult.status; @@ -256,7 +514,9 @@ export async function gatherDiagnostics( try { const nodeResult = await env.DATABASE.prepare( `SELECT id, status, health_status FROM nodes WHERE id = ?` - ).bind(nodeIdToCheck).first<{ id: string; status: string; health_status: string | null }>(); + ) + .bind(nodeIdToCheck) + .first<{ id: string; status: string; health_status: string | null }>(); if (nodeResult) { diagnostics.nodeId = nodeResult.id; @@ -268,23 +528,16 @@ export async function gatherDiagnostics( } } - // Query TaskRunner DO state - try { - const doId = env.TASK_RUNNER.idFromName(task.id); - const stub = env.TASK_RUNNER.get(doId) as DurableObjectStub; - const doStatus = await stub.getStatus(); - - diagnostics.doState = { - exists: doStatus !== null, - completed: doStatus?.completed ?? null, - currentStep: doStatus?.currentStep ?? null, - retryCount: doStatus?.retryCount ?? null, - lastStepAt: doStatus?.lastStepAt ?? null, - }; - } catch { - // DO may not exist or may be unreachable - diagnostics.doState = { exists: false, completed: null, currentStep: null, retryCount: null, lastStepAt: null }; - } + const probe = taskRunnerProbe ?? (await probeTaskRunnerStatus(env, task.id)); + diagnostics.doState = { + outcome: probe.outcome, + exists: probe.outcome === 'ok', + completed: probe.status?.completed ?? null, + currentStep: probe.status?.currentStep ?? null, + retryCount: probe.status?.retryCount ?? null, + lastStepAt: probe.status?.lastStepAt ?? null, + error: probe.error, + }; return diagnostics; } @@ -298,22 +551,38 @@ export async function recoverStuckTasks(env: Env): Promise { failedCompactionLoops: 0, heartbeatSkipped: 0, doHealthChecked: 0, + doHealthMissing: 0, + doHealthErrors: 0, + deadRuntimeReconciled: 0, errors: 0, }; - const queuedTimeoutMs = parseMs(env.TASK_STUCK_QUEUED_TIMEOUT_MS, DEFAULT_TASK_STUCK_QUEUED_TIMEOUT_MS); - const delegatedTimeoutMs = parseMs(env.TASK_STUCK_DELEGATED_TIMEOUT_MS, DEFAULT_TASK_STUCK_DELEGATED_TIMEOUT_MS); + const queuedTimeoutMs = parseMs( + env.TASK_STUCK_QUEUED_TIMEOUT_MS, + DEFAULT_TASK_STUCK_QUEUED_TIMEOUT_MS + ); + const delegatedTimeoutMs = parseMs( + env.TASK_STUCK_DELEGATED_TIMEOUT_MS, + DEFAULT_TASK_STUCK_DELEGATED_TIMEOUT_MS + ); const maxExecutionMs = parseMs(env.TASK_RUN_MAX_EXECUTION_MS, DEFAULT_TASK_RUN_MAX_EXECUTION_MS); const hardTimeoutMs = parseMs(env.TASK_RUN_HARD_TIMEOUT_MS, DEFAULT_TASK_RUN_HARD_TIMEOUT_MS); - const absoluteCeilingMs = parseMs(env.TASK_RUN_ABSOLUTE_CEILING_MS, DEFAULT_TASK_RUN_ABSOLUTE_CEILING_MS); + const absoluteCeilingMs = parseMs( + env.TASK_RUN_ABSOLUTE_CEILING_MS, + DEFAULT_TASK_RUN_ABSOLUTE_CEILING_MS + ); const mismatchGraceMs = parseMs(env.TASK_DO_MISMATCH_GRACE_MS, DEFAULT_TASK_DO_MISMATCH_GRACE_MS); - const maxCandidates = parseMs(env.STUCK_TASK_MAX_CANDIDATES_PER_SWEEP, DEFAULT_STUCK_TASK_MAX_CANDIDATES_PER_SWEEP); + const maxCandidates = parseMs( + env.STUCK_TASK_MAX_CANDIDATES_PER_SWEEP, + DEFAULT_STUCK_TASK_MAX_CANDIDATES_PER_SWEEP + ); if (hardTimeoutMs <= maxExecutionMs) { log.warn('stuck_task.misconfigured_hard_timeout', { hardTimeoutMs, maxExecutionMs, - message: 'TASK_RUN_HARD_TIMEOUT_MS is <= TASK_RUN_MAX_EXECUTION_MS — heartbeat grace window is effectively zero', + message: + 'TASK_RUN_HARD_TIMEOUT_MS is <= TASK_RUN_MAX_EXECUTION_MS — heartbeat grace window is effectively zero', }); } @@ -333,17 +602,19 @@ export async function recoverStuckTasks(env: Env): Promise { WHERE status IN ('queued', 'delegated', 'in_progress') ORDER BY updated_at ASC LIMIT ?` - ).bind(maxCandidates).all<{ - id: string; - project_id: string; - user_id: string; - status: string; - execution_step: string | null; - updated_at: string; - started_at: string | null; - workspace_id: string | null; - auto_provisioned_node_id: string | null; - }>(); + ) + .bind(maxCandidates) + .all<{ + id: string; + project_id: string; + user_id: string; + status: string; + execution_step: string | null; + updated_at: string; + started_at: string | null; + workspace_id: string | null; + auto_provisioned_node_id: string | null; + }>(); const db = drizzle(env.DATABASE, { schema }); @@ -353,10 +624,9 @@ export async function recoverStuckTasks(env: Env): Promise { let isStuck = false; let reason = ''; let compactionLoopRecovery: CompactionLoopRecovery | null = null; + let deadRuntimeRecovery = false; - const stepInfo = task.execution_step - ? ` Last step: ${describeStep(task.execution_step)}.` - : ''; + const stepInfo = task.execution_step ? ` Last step: ${describeStep(task.execution_step)}.` : ''; try { compactionLoopRecovery = await detectTaskCompactionLoop(env, task); @@ -396,6 +666,21 @@ export async function recoverStuckTasks(env: Env): Promise { cachedLiveness ??= await getTaskRuntimeLiveness(env, task); return cachedLiveness; }; + let cachedTaskRunnerProbe: TaskRunnerProbeResult | null = null; + const probeTaskRunner = async (): Promise => { + if (!cachedTaskRunnerProbe) { + cachedTaskRunnerProbe = await probeTaskRunnerStatus(env, task.id); + result.doHealthChecked++; + if (cachedTaskRunnerProbe.outcome === 'missing') result.doHealthMissing++; + if ( + cachedTaskRunnerProbe.outcome === 'error' || + cachedTaskRunnerProbe.outcome === 'timeout' + ) { + result.doHealthErrors++; + } + } + return cachedTaskRunnerProbe; + }; if (!isStuck) { switch (task.status) { @@ -485,56 +770,68 @@ export async function recoverStuckTasks(env: Env): Promise { } if (timeForCheck > Math.min(halfThreshold, mismatchGraceMs)) { - try { - const doId = env.TASK_RUNNER.idFromName(task.id); - const stub = env.TASK_RUNNER.get(doId) as DurableObjectStub; - const doStatus = await stub.getStatus(); + const doProbe = await probeTaskRunner(); + const doStatus = doProbe.status; + const liveness = task.status === 'in_progress' ? await probeLiveness() : null; + + // TaskRunner.completed means orchestration handed off successfully, not + // that the agent later finalized D1. Conclusive task-scoped runtime death + // is therefore authoritative after the mismatch grace; the DO RPC remains + // best-effort diagnostic evidence and cannot strand an immortal row. + if (liveness?.conclusive && !liveness.live) { + isStuck = true; + deadRuntimeRecovery = true; + reason = `Task runtime is conclusively gone after reconciliation grace (${liveness.reason}).`; + log.warn('stuck_task.dead_runtime_reconciliation', { + taskId: task.id, + taskStatus: task.status, + executionStep: task.execution_step, + livenessReason: liveness.reason, + taskRunnerProbeOutcome: doProbe.outcome, + taskRunnerCompleted: doStatus?.completed ?? null, + timeForCheck, + }); + } - if (doStatus && doStatus.completed && task.status !== 'failed' && task.status !== 'completed') { - const liveness = await probeLiveness(); - if (liveness.conclusive && !liveness.live) { - isStuck = true; - reason = `TaskRunner orchestration completed but task runtime is gone (${liveness.reason}).`; - } - // Reconcile only with conclusive dead-runtime evidence. Live or unknown - // task-scoped runtime state remains active and is logged for investigation. - // Deduplicate the persisted mismatch signal independently of reconciliation. - log.warn('stuck_task.do_completed_but_task_active', { - taskId: task.id, - taskStatus: task.status, - doCurrentStep: doStatus.currentStep, - doRetryCount: doStatus.retryCount, - }); + if (doStatus?.completed) { + // Reconcile only with conclusive dead-runtime evidence. Live or unknown + // task-scoped runtime state remains active and is logged for investigation. + // Deduplicate the persisted mismatch signal independently of reconciliation. + log.warn('stuck_task.do_completed_but_task_active', { + taskId: task.id, + taskStatus: task.status, + doCurrentStep: doStatus.currentStep, + doRetryCount: doStatus.retryCount, + }); - // Deduplicate: only persist if no recent mismatch record exists for this task - const recentMismatch = await env.OBSERVABILITY_DATABASE.prepare( - `SELECT id FROM platform_errors + // Deduplicate: only persist if no recent mismatch record exists for this task + const recentMismatch = await env.OBSERVABILITY_DATABASE.prepare( + `SELECT id FROM platform_errors WHERE context LIKE ? AND timestamp > ? LIMIT 1` - ).bind(`%do_task_status_mismatch%${task.id}%`, Date.now() - 30 * 60 * 1000).first(); - - if (!recentMismatch) { - await persistError(env.OBSERVABILITY_DATABASE, { - source: 'api', - level: 'warn', - message: `TaskRunner DO completed but task still in '${task.status}' — possible D1 update failure`, - context: { - recoveryType: 'do_task_status_mismatch', - taskId: task.id, - taskStatus: task.status, - executionStep: task.execution_step, - doCurrentStep: doStatus.currentStep, - doRetryCount: doStatus.retryCount, - timeForCheck, - }, - userId: task.user_id, - }); - } + ) + .bind(`%do_task_status_mismatch%${task.id}%`, Date.now() - 30 * 60 * 1000) + .first(); + + if (!recentMismatch) { + await persistError(env.OBSERVABILITY_DATABASE, { + source: 'api', + level: 'warn', + message: `TaskRunner DO completed but task still in '${task.status}' — possible D1 update failure`, + context: { + recoveryType: 'do_task_status_mismatch', + taskId: task.id, + taskStatus: task.status, + executionStep: task.execution_step, + doCurrentStep: doStatus.currentStep, + doRetryCount: doStatus.retryCount, + timeForCheck, + taskRunnerProbeOutcome: doProbe.outcome, + livenessReason: liveness?.reason ?? null, + }, + userId: task.user_id, + }); } - - result.doHealthChecked++; - } catch { - // DO unreachable — not necessarily an error (may not have been created yet) } } if (!isStuck) continue; @@ -546,7 +843,13 @@ export async function recoverStuckTasks(env: Env): Promise { task.status === 'in_progress' && task.started_at ? now.getTime() - new Date(task.started_at).getTime() : elapsedMs; - const diagnostics = await gatherDiagnostics(env, task, diagnosticElapsedMs, reason); + const diagnostics = await gatherDiagnostics( + env, + task, + diagnosticElapsedMs, + reason, + cachedTaskRunnerProbe ?? undefined + ); log.warn('stuck_task.recovering', { taskId: task.id, @@ -575,12 +878,14 @@ export async function recoverStuckTasks(env: Env): Promise { taskStatus: task.status, executionStep: task.execution_step, elapsedMs: diagnosticElapsedMs, - compactionLoop: compactionLoopRecovery ? { - sessionId: compactionLoopRecovery.sessionId, - agentSessionId: compactionLoopRecovery.agentSessionId, - recentMessageLimit: compactionLoopRecovery.recentMessageLimit, - evidence: compactionLoopRecovery.evidence, - } : null, + compactionLoop: compactionLoopRecovery + ? { + sessionId: compactionLoopRecovery.sessionId, + agentSessionId: compactionLoopRecovery.agentSessionId, + recentMessageLimit: compactionLoopRecovery.recentMessageLimit, + evidence: compactionLoopRecovery.evidence, + } + : null, workspaceId: diagnostics.workspaceId, workspaceStatus: diagnostics.workspaceStatus, nodeId: diagnostics.nodeId, @@ -602,7 +907,9 @@ export async function recoverStuckTasks(env: Env): Promise { const updateResult = await env.DATABASE.prepare( `UPDATE tasks SET status = 'failed', execution_step = NULL, error_message = ?, completed_at = ?, updated_at = ? WHERE id = ? AND status = ?` - ).bind(reason, nowIso, nowIso, task.id, task.status).run(); + ) + .bind(reason, nowIso, nowIso, task.id, task.status) + .run(); if (!updateResult.meta.changes || updateResult.meta.changes === 0) { // Task was advanced by the DO between our SELECT and UPDATE — skip @@ -674,10 +981,15 @@ export async function recoverStuckTasks(env: Env): Promise { } switch (task.status) { - case 'queued': result.failedQueued++; break; - case 'delegated': result.failedDelegated++; break; + case 'queued': + result.failedQueued++; + break; + case 'delegated': + result.failedDelegated++; + break; case 'in_progress': result.failedInProgress++; + if (deadRuntimeRecovery) result.deadRuntimeReconciled++; if (compactionLoopRecovery) result.failedCompactionLoops++; break; } diff --git a/apps/api/tests/unit/recovery-resilience.test.ts b/apps/api/tests/unit/recovery-resilience.test.ts index 6bc862834..0c526a8f8 100644 --- a/apps/api/tests/unit/recovery-resilience.test.ts +++ b/apps/api/tests/unit/recovery-resilience.test.ts @@ -18,18 +18,13 @@ const nodeCleanupSource = readFileSync( resolve(process.cwd(), 'src/scheduled/node-cleanup.ts'), 'utf8' ); -const timeoutSource = readFileSync( - resolve(process.cwd(), 'src/services/timeout.ts'), - 'utf8' -); +const timeoutSource = readFileSync(resolve(process.cwd(), 'src/services/timeout.ts'), 'utf8'); const taskRunnerSource = readFileSync( resolve(process.cwd(), 'src/services/task-runner.ts'), 'utf8' ); -const indexSource = readFileSync( - resolve(process.cwd(), 'src/index.ts'), - 'utf8' -); +const indexSource = readFileSync(resolve(process.cwd(), 'src/index.ts'), 'utf8'); +const adminSource = readFileSync(resolve(process.cwd(), 'src/routes/admin.ts'), 'utf8'); // ========================================================================= // Stuck Tasks — OBSERVABILITY_DATABASE Recording @@ -65,7 +60,7 @@ describe('stuck-tasks OBSERVABILITY_DATABASE recording (TDF-7)', () => { it('records recovery failures in OBSERVABILITY_DATABASE', () => { const failureSection = stuckTasksSource.slice( - stuckTasksSource.indexOf('// Record recovery failure'), + stuckTasksSource.indexOf('// Record recovery failure') ); expect(failureSection).toContain('persistError(env.OBSERVABILITY_DATABASE'); expect(failureSection).toContain("level: 'error'"); @@ -112,8 +107,9 @@ describe('stuck-tasks diagnostic context capture (TDF-7)', () => { stuckTasksSource.indexOf('async function gatherDiagnostics('), stuckTasksSource.indexOf('export async function recoverStuckTasks(') ); - expect(diagSection).toContain('env.TASK_RUNNER.idFromName(task.id)'); - expect(diagSection).toContain('stub.getStatus()'); + expect(diagSection).toContain('probeTaskRunnerStatus(env, task.id)'); + expect(stuckTasksSource).toContain('env.TASK_RUNNER.idFromName(taskId)'); + expect(stuckTasksSource).toContain('stub.getStatus()'); }); it('includes workspace and node status in persistError context', () => { @@ -150,7 +146,9 @@ describe('stuck-tasks diagnostic context capture (TDF-7)', () => { describe('stuck-tasks DO health checks (TDF-7)', () => { it('imports TaskRunner type for typed DO stub', () => { - expect(stuckTasksSource).toContain("import type { TaskRunner } from '../durable-objects/task-runner'"); + expect(stuckTasksSource).toContain( + "import type { TaskRunner } from '../durable-objects/task-runner'" + ); }); it('checks DO health for non-stuck tasks at half threshold', () => { @@ -170,7 +168,7 @@ describe('stuck-tasks DO health checks (TDF-7)', () => { it('detects DO-completed-but-task-active mismatch', () => { expect(stuckTasksSource).toContain('stuck_task.do_completed_but_task_active'); - expect(stuckTasksSource).toContain('doStatus.completed'); + expect(stuckTasksSource).toContain('doStatus?.completed'); }); it('records DO mismatch in OBSERVABILITY_DATABASE', () => { @@ -191,6 +189,28 @@ describe('stuck-tasks DO health checks (TDF-7)', () => { it('cron handler logs doHealthChecked count', () => { expect(indexSource).toContain('stuckTaskDoHealthChecked: stuckTasks.doHealthChecked'); }); + + it('runs stuck-task recovery before failure-prone operational cleanup phases', () => { + const sweepStart = indexSource.indexOf('// 5-minute operational sweep'); + const recoveryIdx = indexSource.indexOf( + 'const stuckTasks = await recoverStuckTasks(env)', + sweepStart + ); + const provisioningIdx = indexSource.indexOf('checkProvisioningTimeouts(', sweepStart); + const nodeCleanupIdx = indexSource.indexOf('runNodeCleanupSweep(env)', sweepStart); + + expect(recoveryIdx).toBeGreaterThan(sweepStart); + expect(recoveryIdx).toBeLessThan(provisioningIdx); + expect(recoveryIdx).toBeLessThan(nodeCleanupIdx); + }); + + it('exposes read-only reconciliation diagnostics behind the superadmin router', () => { + expect(adminSource).toContain( + "adminRoutes.use('/*', requireAuth(), requireApproved(), requireSuperadmin())" + ); + expect(adminSource).toContain("adminRoutes.get('/tasks/:taskId/reconciliation-diagnostics'"); + expect(adminSource).toContain('getTaskReconciliationDiagnostics(c.env, taskId)'); + }); }); // ========================================================================= @@ -292,7 +312,7 @@ describe('node-cleanup orphan detection (TDF-7)', () => { it('uses grace period to avoid flagging recently created resources', () => { // Both orphan queries filter by created_at/updated_at to avoid false positives const orphanSection = nodeCleanupSource.slice( - nodeCleanupSource.indexOf('Orphan cleanup: task-created workspaces'), + nodeCleanupSource.indexOf('Orphan cleanup: task-created workspaces') ); // Orphan workspace query uses gracePeriodMs cutoff on created_at expect(orphanSection).toContain('w.created_at < ?'); @@ -330,7 +350,9 @@ describe('timeout service OBSERVABILITY_DATABASE recording (TDF-7)', () => { }); it('cron handler passes OBSERVABILITY_DATABASE to checkProvisioningTimeouts', () => { - expect(indexSource).toContain('checkProvisioningTimeouts(env.DATABASE, env, env.OBSERVABILITY_DATABASE)'); + expect(indexSource).toContain( + 'checkProvisioningTimeouts(env.DATABASE, env, env.OBSERVABILITY_DATABASE)' + ); }); }); @@ -371,7 +393,7 @@ describe('cleanup idempotency (TDF-7)', () => { taskRunnerSource.indexOf('// Count active workspaces') ); expect(cleanupSection).toContain('schema.nodes.warmSince'); - expect(cleanupSection).toContain("eq(schema.nodes.id, nodeId)"); + expect(cleanupSection).toContain('eq(schema.nodes.id, nodeId)'); }); it('only calls markIdle if node is running and not warm', () => { diff --git a/apps/api/tests/unit/routes/admin-security.test.ts b/apps/api/tests/unit/routes/admin-security.test.ts index 319cc62eb..39b40cafb 100644 --- a/apps/api/tests/unit/routes/admin-security.test.ts +++ b/apps/api/tests/unit/routes/admin-security.test.ts @@ -61,6 +61,7 @@ vi.mock('../../../src/middleware/error', () => { const mockGet = vi.fn(); const mockAll = vi.fn(); const mockUpdate = vi.fn(); +const mockGetTaskReconciliationDiagnostics = vi.fn(); vi.mock('drizzle-orm/d1', () => ({ drizzle: () => ({ @@ -103,6 +104,11 @@ vi.mock('../../../src/services/limits', () => ({ }), })); +vi.mock('../../../src/scheduled/stuck-tasks', () => ({ + getTaskReconciliationDiagnostics: (...args: unknown[]) => + mockGetTaskReconciliationDiagnostics(...args), +})); + // --- Schemas mock --- vi.mock('../../../src/schemas', () => ({ AdminUserActionSchema: {}, @@ -239,4 +245,41 @@ describe('Admin security hardening (route-level)', () => { expect(body.bindings.NOTIFICATION).toBe(false); }); }); + + describe('GET /api/admin/tasks/:taskId/reconciliation-diagnostics', () => { + it('returns the read-only reconciliation evidence for the requested task', async () => { + const env = createEnv(); + const diagnostics = { + taskId: 'task-1', + eligible: true, + decision: 'reconcile_dead_runtime', + liveness: { live: false, conclusive: true, reason: 'workspace_deleted' }, + taskRunner: { outcome: 'missing', status: null }, + }; + mockGetTaskReconciliationDiagnostics.mockResolvedValueOnce(diagnostics); + + const res = await app.request( + '/api/admin/tasks/task-1/reconciliation-diagnostics', + {}, + env, + ); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ diagnostics }); + expect(mockGetTaskReconciliationDiagnostics).toHaveBeenCalledWith(env, 'task-1'); + }); + + it('returns 404 when the task does not exist', async () => { + mockGetTaskReconciliationDiagnostics.mockResolvedValueOnce(null); + + const res = await app.request( + '/api/admin/tasks/missing/reconciliation-diagnostics', + {}, + createEnv(), + ); + + expect(res.status).toBe(404); + expect(await res.json()).toMatchObject({ error: 'NOT_FOUND' }); + }); + }); }); diff --git a/apps/api/tests/unit/stuck-tasks.test.ts b/apps/api/tests/unit/stuck-tasks.test.ts index ed4850fcf..f112e6912 100644 --- a/apps/api/tests/unit/stuck-tasks.test.ts +++ b/apps/api/tests/unit/stuck-tasks.test.ts @@ -6,11 +6,15 @@ * 2. In-progress tasks with stale heartbeats ARE marked as stuck * 3. Tasks without a node are treated as stuck (no heartbeat to check) */ -import { beforeEach,describe, expect, it, vi } from 'vitest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; import type { Env } from '../../src/env'; import { detectClaudeCodeCompactionLoop } from '../../src/scheduled/claude-code-compaction-loop'; -import { recoverStuckTasks } from '../../src/scheduled/stuck-tasks'; +import { + getTaskReconciliationDiagnostics, + probeTaskRunnerStatus, + recoverStuckTasks, +} from '../../src/scheduled/stuck-tasks'; import { persistError } from '../../src/services/observability'; import { cleanupTaskRun } from '../../src/services/task-runner'; @@ -69,6 +73,7 @@ function mockPreparedStatement(results: unknown[] = [], changes = 1) { function createMockEnv( prepareResponses: Map = new Map(), envOverrides: Partial> = {}, + taskRunnerStatus: unknown | Error = null ): Env { const mockDb = { prepare: vi.fn((sql: string) => { @@ -87,7 +92,10 @@ function createMockEnv( const mockTaskRunnerDO = { idFromName: vi.fn().mockReturnValue({ toString: () => 'do-id' }), get: vi.fn().mockReturnValue({ - getStatus: vi.fn().mockResolvedValue(null), + getStatus: + taskRunnerStatus instanceof Error + ? vi.fn().mockRejectedValue(taskRunnerStatus) + : vi.fn().mockResolvedValue(taskRunnerStatus), }), }; @@ -112,12 +120,14 @@ describe('recoverStuckTasks', () => { projectDataMocks.getMessages.mockResolvedValue({ messages: [], hasMore: false }); projectDataMocks.listSessions.mockResolvedValue({ sessions: [], total: 0 }); projectDataMocks.listAcpSessions.mockResolvedValue({ - sessions: [{ - id: 'acp-live', - status: 'running', - workspaceId: 'ws-1', - lastHeartbeatAt: Date.now(), - }], + sessions: [ + { + id: 'acp-live', + status: 'running', + workspaceId: 'ws-1', + lastHeartbeatAt: Date.now(), + }, + ], total: 1, }); projectDataMocks.failSession.mockResolvedValue(undefined); @@ -135,7 +145,7 @@ describe('recoverStuckTasks', () => { { role: 'system', content: 'Compacting...' }, { role: 'system', content: 'Compacting completed' }, ], - { windowMessages: 6, minPairs: 3 }, + { windowMessages: 6, minPairs: 3 } ); expect(evidence.detected).toBe(true); @@ -150,7 +160,7 @@ describe('recoverStuckTasks', () => { { role: 'assistant', content: 'I made progress after compaction.' }, { role: 'system', content: 'Compacting completed' }, ], - { windowMessages: 3, minPairs: 2 }, + { windowMessages: 3, minPairs: 2 } ); expect(evidence.detected).toBe(false); @@ -163,7 +173,7 @@ describe('recoverStuckTasks', () => { const now = Date.now(); const recent = new Date(now - 30 * 1000).toISOString(); const responses = new Map(); - responses.set('status IN (\'queued\', \'delegated\', \'in_progress\')', { + responses.set("status IN ('queued', 'delegated', 'in_progress')", { results: [ { id: 'task-compaction-loop', @@ -190,7 +200,7 @@ describe('recoverStuckTasks', () => { responses.set('status, health_status FROM nodes', { results: [{ id: 'node-1', status: 'running', health_status: 'healthy' }], }); - responses.set('UPDATE tasks SET status = \'failed\'', { + responses.set("UPDATE tasks SET status = 'failed'", { results: [], changes: 1, }); @@ -223,19 +233,19 @@ describe('recoverStuckTasks', () => { null, ['assistant', 'system', 'tool'], false, - 'desc', + 'desc' ); expect(projectDataMocks.failSession).toHaveBeenCalledWith( env, 'proj-1', 'chat-session-1', - expect.stringContaining('Claude Code compaction loop detected'), + expect.stringContaining('Claude Code compaction loop detected') ); expect(syncTriggerExecutionMock).toHaveBeenCalledWith( env.DATABASE, 'task-compaction-loop', 'failed', - expect.stringContaining('Claude Code compaction loop detected'), + expect.stringContaining('Claude Code compaction loop detected') ); expect(cleanupTaskRun).toHaveBeenCalledWith('task-compaction-loop', env); expect(persistError).toHaveBeenCalledWith( @@ -257,14 +267,14 @@ describe('recoverStuckTasks', () => { }), }), }), - }), + }) ); }); it('does not fail a fresh in-progress task without a running Claude Code agent session', async () => { const recent = new Date(Date.now() - 30 * 1000).toISOString(); const responses = new Map(); - responses.set('status IN (\'queued\', \'delegated\', \'in_progress\')', { + responses.set("status IN ('queued', 'delegated', 'in_progress')", { results: [ { id: 'task-opencode', @@ -299,21 +309,32 @@ describe('recoverStuckTasks', () => { const old = new Date(now - 5 * 60 * 60 * 1000).toISOString(); const recentHeartbeat = new Date(now - 30 * 1000).toISOString(); const responses = new Map(); - responses.set('status IN (\'queued\', \'delegated\', \'in_progress\')', { - results: [{ - id: 'task-liveness-unknown', - project_id: 'proj-1', - user_id: 'user-1', - status: 'in_progress', - execution_step: 'running', - updated_at: old, - started_at: old, - workspace_id: 'ws-1', - auto_provisioned_node_id: 'node-1', - }], + responses.set("status IN ('queued', 'delegated', 'in_progress')", { + results: [ + { + id: 'task-liveness-unknown', + project_id: 'proj-1', + user_id: 'user-1', + status: 'in_progress', + execution_step: 'running', + updated_at: old, + started_at: old, + workspace_id: 'ws-1', + auto_provisioned_node_id: 'node-1', + }, + ], }); responses.set('w.chat_session_id', { - results: [{ workspace_status: 'running', chat_session_id: 'chat-1', node_id: 'node-1', node_status: 'running', health_status: 'healthy', last_heartbeat_at: recentHeartbeat }], + results: [ + { + workspace_status: 'running', + chat_session_id: 'chat-1', + node_id: 'node-1', + node_status: 'running', + health_status: 'healthy', + last_heartbeat_at: recentHeartbeat, + }, + ], }); projectDataMocks.listAcpSessions.mockRejectedValueOnce(new Error('ProjectData unavailable')); @@ -334,7 +355,7 @@ describe('recoverStuckTasks', () => { const responses = new Map(); // Query to find stuck tasks - responses.set('status IN (\'queued\', \'delegated\', \'in_progress\')', { + responses.set("status IN ('queued', 'delegated', 'in_progress')", { results: [ { id: 'task-1', @@ -358,7 +379,16 @@ describe('recoverStuckTasks', () => { results: [{ last_heartbeat_at: recentHeartbeat }], }); responses.set('w.chat_session_id', { - results: [{ workspace_status: 'running', chat_session_id: 'chat-1', node_id: 'node-1', node_status: 'running', health_status: 'healthy', last_heartbeat_at: recentHeartbeat }], + results: [ + { + workspace_status: 'running', + chat_session_id: 'chat-1', + node_id: 'node-1', + node_status: 'running', + health_status: 'healthy', + last_heartbeat_at: recentHeartbeat, + }, + ], }); const env = createMockEnv(responses); @@ -378,7 +408,7 @@ describe('recoverStuckTasks', () => { const recentHeartbeat = new Date(now - 30 * 1000).toISOString(); const responses = new Map(); - responses.set('status IN (\'queued\', \'delegated\', \'in_progress\')', { + responses.set("status IN ('queued', 'delegated', 'in_progress')", { results: [ { id: 'task-1', @@ -394,7 +424,16 @@ describe('recoverStuckTasks', () => { ], }); responses.set('w.chat_session_id', { - results: [{ workspace_status: 'running', chat_session_id: 'chat-1', node_id: 'node-1', node_status: 'running', health_status: 'healthy', last_heartbeat_at: recentHeartbeat }], + results: [ + { + workspace_status: 'running', + chat_session_id: 'chat-1', + node_id: 'node-1', + node_status: 'running', + health_status: 'healthy', + last_heartbeat_at: recentHeartbeat, + }, + ], }); const env = createMockEnv(responses); @@ -413,7 +452,7 @@ describe('recoverStuckTasks', () => { const staleHeartbeat = new Date(now - 10 * 60 * 1000).toISOString(); const responses = new Map(); - responses.set('status IN (\'queued\', \'delegated\', \'in_progress\')', { + responses.set("status IN ('queued', 'delegated', 'in_progress')", { results: [ { id: 'task-1', @@ -435,7 +474,16 @@ describe('recoverStuckTasks', () => { results: [{ last_heartbeat_at: staleHeartbeat }], }); responses.set('w.chat_session_id', { - results: [{ workspace_status: 'running', chat_session_id: 'chat-1', node_id: 'node-1', node_status: 'running', health_status: 'healthy', last_heartbeat_at: staleHeartbeat }], + results: [ + { + workspace_status: 'running', + chat_session_id: 'chat-1', + node_id: 'node-1', + node_status: 'running', + health_status: 'healthy', + last_heartbeat_at: staleHeartbeat, + }, + ], }); // Workspace status for diagnostics responses.set('node_id, status FROM workspaces', { @@ -446,7 +494,7 @@ describe('recoverStuckTasks', () => { results: [{ id: 'node-1', status: 'running', health_status: 'healthy' }], }); // Task update (mark as failed) - responses.set('UPDATE tasks SET status = \'failed\'', { + responses.set("UPDATE tasks SET status = 'failed'", { results: [], changes: 1, }); @@ -462,7 +510,7 @@ describe('recoverStuckTasks', () => { env.DATABASE, 'task-1', 'failed', - expect.stringContaining('runtime is no longer live'), + expect.stringContaining('runtime is no longer live') ); }); @@ -476,7 +524,7 @@ describe('recoverStuckTasks', () => { const recentHeartbeat = new Date(now - 30 * 1000).toISOString(); // node very much alive const responses = new Map(); - responses.set('status IN (\'queued\', \'delegated\', \'in_progress\')', { + responses.set("status IN ('queued', 'delegated', 'in_progress')", { results: [ { id: 'task-1', @@ -492,7 +540,16 @@ describe('recoverStuckTasks', () => { ], }); responses.set('w.chat_session_id', { - results: [{ workspace_status: 'running', chat_session_id: 'chat-1', node_id: 'node-1', node_status: 'running', health_status: 'healthy', last_heartbeat_at: recentHeartbeat }], + results: [ + { + workspace_status: 'running', + chat_session_id: 'chat-1', + node_id: 'node-1', + node_status: 'running', + health_status: 'healthy', + last_heartbeat_at: recentHeartbeat, + }, + ], }); responses.set('node_id, status FROM workspaces', { results: [{ id: 'ws-1', node_id: 'node-1', status: 'running' }], @@ -500,7 +557,7 @@ describe('recoverStuckTasks', () => { responses.set('status, health_status FROM nodes', { results: [{ id: 'node-1', status: 'running', health_status: 'healthy' }], }); - responses.set('UPDATE tasks SET status = \'failed\'', { results: [], changes: 1 }); + responses.set("UPDATE tasks SET status = 'failed'", { results: [], changes: 1 }); // No live task-scoped ACP session, despite the healthy node. projectDataMocks.listAcpSessions.mockResolvedValueOnce({ sessions: [], total: 0 }); @@ -515,7 +572,7 @@ describe('recoverStuckTasks', () => { env.DATABASE, 'task-1', 'failed', - expect.stringContaining('task_acp_session_not_live'), + expect.stringContaining('task_acp_session_not_live') ); }); @@ -525,7 +582,7 @@ describe('recoverStuckTasks', () => { const updatedAt = new Date(now - 5 * 60 * 60 * 1000).toISOString(); const responses = new Map(); - responses.set('status IN (\'queued\', \'delegated\', \'in_progress\')', { + responses.set("status IN ('queued', 'delegated', 'in_progress')", { results: [ { id: 'task-1', @@ -545,7 +602,7 @@ describe('recoverStuckTasks', () => { results: [], }); // Task update - responses.set('UPDATE tasks SET status = \'failed\'', { + responses.set("UPDATE tasks SET status = 'failed'", { results: [], changes: 1, }); @@ -698,7 +755,7 @@ describe('recoverStuckTasks', () => { const recentHeartbeat = new Date(now - 30 * 1000).toISOString(); const responses = new Map(); - responses.set('status IN (\'queued\', \'delegated\', \'in_progress\')', { + responses.set("status IN ('queued', 'delegated', 'in_progress')", { results: [ { id: 'task-hard', @@ -722,7 +779,16 @@ describe('recoverStuckTasks', () => { results: [{ last_heartbeat_at: recentHeartbeat }], }); responses.set('w.chat_session_id', { - results: [{ workspace_status: 'running', chat_session_id: 'chat-1', node_id: 'node-1', node_status: 'running', health_status: 'healthy', last_heartbeat_at: recentHeartbeat }], + results: [ + { + workspace_status: 'running', + chat_session_id: 'chat-1', + node_id: 'node-1', + node_status: 'running', + health_status: 'healthy', + last_heartbeat_at: recentHeartbeat, + }, + ], }); // Workspace status for diagnostics responses.set('node_id, status FROM workspaces', { @@ -733,7 +799,7 @@ describe('recoverStuckTasks', () => { results: [{ id: 'node-1', status: 'running', health_status: 'healthy' }], }); // Task update (mark as failed) - responses.set('UPDATE tasks SET status = \'failed\'', { + responses.set("UPDATE tasks SET status = 'failed'", { results: [], changes: 1, }); @@ -755,7 +821,7 @@ describe('recoverStuckTasks', () => { const recentHeartbeat = new Date(now - 30 * 1000).toISOString(); const responses = new Map(); - responses.set('status IN (\'queued\', \'delegated\', \'in_progress\')', { + responses.set("status IN ('queued', 'delegated', 'in_progress')", { results: [ { id: 'task-grace', @@ -777,7 +843,16 @@ describe('recoverStuckTasks', () => { results: [{ last_heartbeat_at: recentHeartbeat }], }); responses.set('w.chat_session_id', { - results: [{ workspace_status: 'running', chat_session_id: 'chat-1', node_id: 'node-1', node_status: 'running', health_status: 'healthy', last_heartbeat_at: recentHeartbeat }], + results: [ + { + workspace_status: 'running', + chat_session_id: 'chat-1', + node_id: 'node-1', + node_status: 'running', + health_status: 'healthy', + last_heartbeat_at: recentHeartbeat, + }, + ], }); const env = createMockEnv(responses); @@ -797,7 +872,7 @@ describe('recoverStuckTasks', () => { const staleHeartbeat = new Date(now - 10 * 60 * 1000).toISOString(); const responses = new Map(); - responses.set('status IN (\'queued\', \'delegated\', \'in_progress\')', { + responses.set("status IN ('queued', 'delegated', 'in_progress')", { results: [ { id: 'task-stale-grace', @@ -824,7 +899,7 @@ describe('recoverStuckTasks', () => { responses.set('status, health_status FROM nodes', { results: [{ id: 'node-1', status: 'running', health_status: 'healthy' }], }); - responses.set('UPDATE tasks SET status = \'failed\'', { + responses.set("UPDATE tasks SET status = 'failed'", { results: [], changes: 1, }); @@ -847,7 +922,7 @@ describe('recoverStuckTasks', () => { const updatedAt = new Date(now - 3 * 60 * 60 * 1000).toISOString(); const responses = new Map(); - responses.set('status IN (\'queued\', \'delegated\', \'in_progress\')', { + responses.set("status IN ('queued', 'delegated', 'in_progress')", { results: [ { id: 'task-custom', @@ -862,7 +937,7 @@ describe('recoverStuckTasks', () => { }, ], }); - responses.set('UPDATE tasks SET status = \'failed\'', { + responses.set("UPDATE tasks SET status = 'failed'", { results: [], changes: 1, }); @@ -882,7 +957,7 @@ describe('recoverStuckTasks', () => { env.DATABASE, 'task-custom', 'failed', - expect.stringContaining('120 minutes'), + expect.stringContaining('120 minutes') ); }); }); @@ -894,7 +969,7 @@ describe('recoverStuckTasks', () => { const updatedAt = new Date(now - 11 * 60 * 1000).toISOString(); const responses = new Map(); - responses.set('status IN (\'queued\', \'delegated\', \'in_progress\')', { + responses.set("status IN ('queued', 'delegated', 'in_progress')", { results: [ { id: 'task-q', @@ -909,7 +984,7 @@ describe('recoverStuckTasks', () => { }, ], }); - responses.set('UPDATE tasks SET status = \'failed\'', { results: [], changes: 1 }); + responses.set("UPDATE tasks SET status = 'failed'", { results: [], changes: 1 }); const env = createMockEnv(responses); const result = await recoverStuckTasks(env); @@ -923,7 +998,7 @@ describe('recoverStuckTasks', () => { const updatedAt = new Date(now - 32 * 60 * 1000).toISOString(); const responses = new Map(); - responses.set('status IN (\'queued\', \'delegated\', \'in_progress\')', { + responses.set("status IN ('queued', 'delegated', 'in_progress')", { results: [ { id: 'task-d', @@ -938,7 +1013,7 @@ describe('recoverStuckTasks', () => { }, ], }); - responses.set('UPDATE tasks SET status = \'failed\'', { results: [], changes: 1 }); + responses.set("UPDATE tasks SET status = 'failed'", { results: [], changes: 1 }); const env = createMockEnv(responses); const result = await recoverStuckTasks(env); @@ -947,11 +1022,149 @@ describe('recoverStuckTasks', () => { }); }); + describe('prompt dead-runtime reconciliation', () => { + function deadRuntimeResponses(taskId: string) { + const tenMinutesAgo = new Date(Date.now() - 10 * 60 * 1000).toISOString(); + return new Map([ + [ + "status IN ('queued', 'delegated', 'in_progress')", + { + results: [ + { + id: taskId, + project_id: 'proj-1', + user_id: 'user-1', + status: 'in_progress', + execution_step: 'running', + updated_at: tenMinutesAgo, + started_at: tenMinutesAgo, + workspace_id: 'ws-deleted', + auto_provisioned_node_id: 'node-deleted', + }, + ], + }, + ], + [ + 'w.chat_session_id', + { + results: [ + { + workspace_status: 'deleted', + chat_session_id: 'chat-1', + node_id: 'node-deleted', + node_status: 'deleted', + health_status: 'stale', + last_heartbeat_at: tenMinutesAgo, + }, + ], + }, + ], + [ + 'node_id, status FROM workspaces', + { + results: [{ id: 'ws-deleted', node_id: 'node-deleted', status: 'deleted' }], + }, + ], + [ + 'status, health_status FROM nodes', + { + results: [{ id: 'node-deleted', status: 'deleted', health_status: 'stale' }], + }, + ], + ["UPDATE tasks SET status = 'failed'", { results: [], changes: 1 }], + ]); + } + + it('fails a conclusively dead runtime even when TaskRunner state is missing', async () => { + const env = createMockEnv(deadRuntimeResponses('task-do-missing'), { + TASK_DO_MISMATCH_GRACE_MS: '60000', + }); + + const result = await recoverStuckTasks(env); + + expect(result.failedInProgress).toBe(1); + expect(result.deadRuntimeReconciled).toBe(1); + expect(result.doHealthMissing).toBe(1); + expect(syncTriggerExecutionMock).toHaveBeenCalledWith( + env.DATABASE, + 'task-do-missing', + 'failed', + expect.stringContaining('workspace_deleted') + ); + }); + + it('fails a conclusively dead runtime and records the TaskRunner RPC failure', async () => { + const env = createMockEnv( + deadRuntimeResponses('task-do-error'), + { TASK_DO_MISMATCH_GRACE_MS: '60000' }, + new Error('TaskRunner RPC unavailable') + ); + + const result = await recoverStuckTasks(env); + + expect(result.failedInProgress).toBe(1); + expect(result.deadRuntimeReconciled).toBe(1); + expect(result.doHealthErrors).toBe(1); + expect(syncTriggerExecutionMock).toHaveBeenCalledWith( + env.DATABASE, + 'task-do-error', + 'failed', + expect.stringContaining('workspace_deleted') + ); + }); + + it('classifies a bounded TaskRunner status timeout', async () => { + const neverResolves = new Promise(() => undefined); + const env = createMockEnv( + new Map(), + { TASK_LIVENESS_PROBE_TIMEOUT_MS: '1' }, + neverResolves, + ); + + const probe = await probeTaskRunnerStatus(env, 'task-do-timeout'); + + expect(probe).toMatchObject({ + outcome: 'timeout', + status: null, + error: expect.stringContaining('exceeded 1ms'), + }); + }); + + it('reports the exact read-only reconciliation decision for admins', async () => { + const tenMinutesAgo = new Date(Date.now() - 10 * 60 * 1000).toISOString(); + const responses = deadRuntimeResponses('task-admin-diagnostics'); + responses.set('FROM tasks\n WHERE id = ?', { + results: [ + { + id: 'task-admin-diagnostics', + project_id: 'proj-1', + status: 'in_progress', + execution_step: 'running', + started_at: tenMinutesAgo, + updated_at: tenMinutesAgo, + workspace_id: 'ws-deleted', + }, + ], + }); + const env = createMockEnv(responses, { TASK_DO_MISMATCH_GRACE_MS: '60000' }); + + const diagnostics = await getTaskReconciliationDiagnostics(env, 'task-admin-diagnostics'); + + expect(diagnostics).toMatchObject({ + taskId: 'task-admin-diagnostics', + eligible: true, + decision: 'reconcile_dead_runtime', + liveness: { conclusive: true, live: false, reason: 'workspace_deleted' }, + taskRunner: { outcome: 'missing', status: null }, + }); + }); + }); + describe('result structure', () => { it('returns all expected counters including heartbeatSkipped', async () => { - const env = createMockEnv(new Map([ - ['status IN (\'queued\', \'delegated\', \'in_progress\')', { results: [] }], - ])); + const env = createMockEnv( + new Map([["status IN ('queued', 'delegated', 'in_progress')", { results: [] }]]) + ); const result = await recoverStuckTasks(env); expect(result).toEqual({ @@ -961,6 +1174,9 @@ describe('recoverStuckTasks', () => { failedCompactionLoops: 0, heartbeatSkipped: 0, doHealthChecked: 0, + doHealthMissing: 0, + doHealthErrors: 0, + deadRuntimeReconciled: 0, errors: 0, }); }); diff --git a/apps/api/tests/workers/scheduled-stuck-tasks.test.ts b/apps/api/tests/workers/scheduled-stuck-tasks.test.ts index 0fda15597..090c9c682 100644 --- a/apps/api/tests/workers/scheduled-stuck-tasks.test.ts +++ b/apps/api/tests/workers/scheduled-stuck-tasks.test.ts @@ -42,30 +42,41 @@ async function getTaskStatus(taskId: string): Promise<{ execution_step: string | null; } | null> { return env.DATABASE.prepare( - 'SELECT status, error_message, completed_at, execution_step FROM tasks WHERE id = ?', + 'SELECT status, error_message, completed_at, execution_step FROM tasks WHERE id = ?' ) .bind(taskId) .first(); } -async function getTaskStatusEvents(taskId: string): Promise<{ - from_status: string | null; - to_status: string; - actor_type: string; - reason: string | null; -}[]> { +async function getTaskStatusEvents(taskId: string): Promise< + { + from_status: string | null; + to_status: string; + actor_type: string; + reason: string | null; + }[] +> { const result = await env.DATABASE.prepare( - 'SELECT from_status, to_status, actor_type, reason FROM task_status_events WHERE task_id = ? ORDER BY created_at', + 'SELECT from_status, to_status, actor_type, reason FROM task_status_events WHERE task_id = ? ORDER BY created_at' ) .bind(taskId) - .all<{ from_status: string | null; to_status: string; actor_type: string; reason: string | null }>(); + .all<{ + from_status: string | null; + to_status: string; + actor_type: string; + reason: string | null; + }>(); return result.results; } -async function getObservabilityEvents(taskId: string): Promise<{ message: string; context: string }[]> { +async function getObservabilityEvents( + taskId: string +): Promise<{ message: string; context: string }[]> { const result = await env.OBSERVABILITY_DATABASE.prepare( - `SELECT message, context FROM platform_errors WHERE context LIKE ? ORDER BY created_at DESC`, - ).bind(`%${taskId}%`).all<{ message: string; context: string }>(); + `SELECT message, context FROM platform_errors WHERE context LIKE ? ORDER BY created_at DESC` + ) + .bind(`%${taskId}%`) + .all<{ message: string; context: string }>(); return result.results; } @@ -93,7 +104,7 @@ describe('recoverStuckTasks — vertical slice', () => { }); const stub = env.TASK_RUNNER.get( - env.TASK_RUNNER.idFromName(taskId), + env.TASK_RUNNER.idFromName(taskId) ) as DurableObjectStub; await stub.start({ taskId, @@ -147,6 +158,55 @@ describe('recoverStuckTasks — vertical slice', () => { expect(task?.error_message).toContain('runtime is gone'); expect(task?.error_message).toContain('workspace_deleted'); }); + + it('reconciles a dead runtime when TaskRunner state is missing and stays terminal', async () => { + await seedBaseData(); + const taskId = 'task-st-do-missing-dead'; + const nodeId = 'node-st-do-missing-dead'; + const workspaceId = 'ws-st-do-missing-dead'; + const oldDate = new Date(Date.now() - 10 * 60 * 1000).toISOString(); + + await seedNode(nodeId, USER_ID, { status: 'deleted', healthStatus: 'stale' }); + await seedWorkspace(workspaceId, nodeId, USER_ID, { + projectId: PROJECT_ID, + status: 'deleted', + }); + await seedTask(taskId, PROJECT_ID, USER_ID, { + status: 'in_progress', + executionStep: 'running', + startedAt: oldDate, + updatedAt: oldDate, + workspaceId, + }); + + const testEnv = { + ...env, + TASK_DO_MISMATCH_GRACE_MS: '60000', + } as unknown as Env; + const firstSweep = await recoverStuckTasks(testEnv); + + expect(firstSweep).toMatchObject({ + failedInProgress: 1, + doHealthChecked: 1, + doHealthMissing: 1, + doHealthErrors: 0, + deadRuntimeReconciled: 1, + }); + const task = await getTaskStatus(taskId); + expect(task?.status).toBe('failed'); + expect(task?.error_message).toContain('workspace_deleted'); + + const eventsAfterFirstSweep = await getTaskStatusEvents(taskId); + expect(eventsAfterFirstSweep.filter((event) => event.to_status === 'failed')).toHaveLength(1); + + const secondSweep = await recoverStuckTasks(testEnv); + expect(secondSweep.deadRuntimeReconciled).toBe(0); + expect(secondSweep.failedInProgress).toBe(0); + const eventsAfterSecondSweep = await getTaskStatusEvents(taskId); + expect(eventsAfterSecondSweep.filter((event) => event.to_status === 'failed')).toHaveLength( + 1 + ); + }); }); describe('stuck queued task detection', () => { @@ -316,7 +376,7 @@ describe('recoverStuckTasks — vertical slice', () => { // Advance the task to 'delegated' just before recovery runs // This simulates the TaskRunner DO advancing the task between SELECT and UPDATE await env.DATABASE.prepare( - "UPDATE tasks SET status = 'delegated', updated_at = datetime('now') WHERE id = ?", + "UPDATE tasks SET status = 'delegated', updated_at = datetime('now') WHERE id = ?" ) .bind(taskId) .run(); @@ -424,7 +484,7 @@ describe('gatherDiagnostics', () => { auto_provisioned_node_id: null, }, 120000, - 'Test reason', + 'Test reason' ); expect(diagnostics.taskId).toBe('task-st-diag'); @@ -448,7 +508,7 @@ describe('gatherDiagnostics', () => { auto_provisioned_node_id: null, }, 60000, - 'Test reason', + 'Test reason' ); expect(diagnostics.workspaceStatus).toBeNull(); @@ -457,7 +517,10 @@ describe('gatherDiagnostics', () => { it('falls back to autoProvisionedNodeId when workspace has no node', async () => { await seedUser('user-st-diag2'); - await seedNode('node-st-auto', 'user-st-diag2', { status: 'creating', healthStatus: 'unknown' }); + await seedNode('node-st-auto', 'user-st-diag2', { + status: 'creating', + healthStatus: 'unknown', + }); const diagnostics = await gatherDiagnostics( env as unknown as Env, @@ -469,7 +532,7 @@ describe('gatherDiagnostics', () => { auto_provisioned_node_id: 'node-st-auto', }, 90000, - 'Test auto-prov', + 'Test auto-prov' ); expect(diagnostics.nodeId).toBe('node-st-auto'); diff --git a/apps/www/src/content/docs/docs/reference/configuration.md b/apps/www/src/content/docs/docs/reference/configuration.md index bc4f1c0ca..69ad965a9 100644 --- a/apps/www/src/content/docs/docs/reference/configuration.md +++ b/apps/www/src/content/docs/docs/reference/configuration.md @@ -201,7 +201,7 @@ SAM loads OpenCode Zen and OpenCode Go model choices through the authenticated m | Variable | Default | Description | | -------------------------------------------------- | ------------------ | ------------------------------------------------------------------------------------- | | `TASK_RUN_MAX_EXECUTION_MS` | `14400000` (4 hr) | Max task execution time | -| `TASK_STUCK_QUEUED_TIMEOUT_MS` | `600000` (10 min) | Timeout for tasks stuck in queued state | +| `TASK_STUCK_QUEUED_TIMEOUT_MS` | `1200000` (20 min) | Timeout for tasks stuck in queued state | | `TASK_STUCK_DELEGATED_TIMEOUT_MS` | `1860000` (31 min) | Timeout for tasks stuck in delegated state | | `TASK_DO_MISMATCH_GRACE_MS` | `300000` (5 min) | Minimum age before reconciling completed TaskRunner state with task-scoped liveness | | `STUCK_TASK_MAX_CANDIDATES_PER_SWEEP` | `100` | Maximum active tasks inspected by each recovery sweep | diff --git a/tasks/active/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md b/tasks/active/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md index 6a9560e92..9940c8225 100644 --- a/tasks/active/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md +++ b/tasks/active/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md @@ -64,10 +64,19 @@ the recovered work, all fixed: 5. Fixed lint (import sort). Documented the liveness-gated recovery semantic (live tasks are preserved past the hard ceiling) in `configuration.md`. +## Staging failure and follow-up (2026-07-12) + +- The original PR branch deployed successfully and passed browser/API regression checks, but a controlled one-row D1 mismatch remained `in_progress` across three five-minute cron boundaries while its workspace was `deleted`. +- `TaskRunner.completed` records successful orchestration handoff, not later agent completion. A missing/unreachable TaskRunner state must therefore remain diagnostic evidence; it cannot veto reconciliation when task-scoped runtime liveness is conclusively dead. +- Recovery now runs first in the five-minute operational sweep, before provisioning, migration, and node-cleanup phases can fail and suppress it. +- TaskRunner status probes are bounded and classified as `ok`, `missing`, `timeout`, or `error`; cron summaries expose missing/error/reconciled counters. +- A superadmin-only, read-only diagnostics endpoint reports the exact eligibility, liveness, TaskRunner probe, and decision for one task before any staging mutation. +- Unit coverage proves missing/error TaskRunner RPC outcomes still reconcile a deleted runtime. A Worker D1/DO vertical slice also asserts the second sweep is a no-op; local `workerd` currently exits with SIGSEGV before importing Worker tests, so that file remains for CI/staging verification. + ## Acceptance criteria - Demonstrably dead DO-completed+D1-active work reaches terminal D1 promptly with sanitized diagnostics. -- Concrete live workspace/agent evidence prevents early failure regardless of elapsed duration. +- Concrete live workspace/agent evidence prevents early failure below the configurable absolute runaway-cost ceiling. - Shared-node heartbeat and stale running session status cannot independently suppress reconciliation. - Recovery/callback interleavings converge on one terminal state with idempotent side effects. - Task/conversation awaiting-followup semantics remain intentional. From 2f5bf6acf382bc76f3c8f1003ecfdcd57ace85a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Sun, 12 Jul 2026 17:01:18 +0000 Subject: [PATCH 11/11] fix: rotate stuck task reconciliation candidates --- .claude/skills/api-reference/SKILL.md | 2 +- .claude/skills/env-reference/SKILL.md | 2 + apps/api/.env.example | 1 + apps/api/src/env.ts | 1 + apps/api/src/index.ts | 4 + apps/api/src/scheduled/stuck-tasks.ts | 234 ++++++++++++++++-- apps/api/tests/unit/stuck-tasks.test.ts | 128 +++++++++- .../workers/scheduled-stuck-tasks.test.ts | 57 +++++ .../docs/docs/reference/configuration.md | 45 ++-- packages/shared/src/constants/index.ts | 1 + .../shared/src/constants/task-execution.ts | 3 + ...-taskrunner-d1-lifecycle-reconciliation.md | 2 + 12 files changed, 428 insertions(+), 52 deletions(-) diff --git a/.claude/skills/api-reference/SKILL.md b/.claude/skills/api-reference/SKILL.md index 521cc5052..cc407d2e5 100644 --- a/.claude/skills/api-reference/SKILL.md +++ b/.claude/skills/api-reference/SKILL.md @@ -63,7 +63,7 @@ user-invocable: false ## Administration (Superadmin Only) - `GET /api/admin/tasks/stuck` — List tasks currently in transient states -- `GET /api/admin/tasks/:taskId/reconciliation-diagnostics` — Read the TaskRunner probe, task-scoped runtime liveness, eligibility threshold, and reconciliation decision without mutating task state +- `GET /api/admin/tasks/:taskId/reconciliation-diagnostics` — Read the TaskRunner probe, task-scoped runtime liveness, eligibility threshold, reconciliation decision, and whether/where the bounded cursor page selects the task, without mutating task state - `GET /api/admin/tasks/recent-failures` — List recent failed tasks with error details ## Agent Sessions diff --git a/.claude/skills/env-reference/SKILL.md b/.claude/skills/env-reference/SKILL.md index f0b7da378..10ac11aa5 100644 --- a/.claude/skills/env-reference/SKILL.md +++ b/.claude/skills/env-reference/SKILL.md @@ -88,6 +88,8 @@ See `apps/api/.env.example` for the full list. Key variables: - `MAX_PROJECTS_PER_USER` — Runtime project cap - `MAX_TASKS_PER_PROJECT` — Runtime task cap per project - `MAX_TASK_DEPENDENCIES_PER_TASK` — Runtime dependency-edge cap per task +- `STUCK_TASK_MAX_CANDIDATES_PER_SWEEP` — Maximum active task rows inspected by one five-minute recovery sweep (default: 100) +- `STUCK_TASK_SCAN_CURSOR_KV_KEY` — KV key used to resume the bounded recovery scan fairly across active rows (default: `scheduled:stuck-tasks:scan-cursor:v1`) - `PROJECT_INVITE_TOKEN_BYTES` — Random bytes used for generated project invite link tokens (default: 32) - `PROJECT_INVITE_DEFAULT_EXPIRY_DAYS` — Default lifetime for project invite links created without an explicit expiry (default: 7) - `PROJECT_INVITE_MAX_EXPIRY_DAYS` — Maximum allowed project invite link lifetime (default: 30) diff --git a/apps/api/.env.example b/apps/api/.env.example index 3a9718c9e..42996ee72 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -348,6 +348,7 @@ BASE_DOMAIN=workspaces.example.com # TASK_STUCK_DELEGATED_TIMEOUT_MS=1860000 # 31 minutes — max time in 'delegated' state (> workspace ready timeout) # TASK_DO_MISMATCH_GRACE_MS=300000 # 5 minutes — minimum age before DO/D1 liveness reconciliation # STUCK_TASK_MAX_CANDIDATES_PER_SWEEP=100 # Bound task rows inspected per cron invocation +# STUCK_TASK_SCAN_CURSOR_KV_KEY=scheduled:stuck-tasks:scan-cursor:v1 # KV cursor for fair bounded scans # TASK_LIVENESS_MAX_ACP_SESSIONS=5 # Bound task-scoped ACP sessions inspected per candidate # TASK_LIVENESS_PROBE_TIMEOUT_MS=5000 # Per-candidate timeout for the ACP liveness DO probe (inconclusive on timeout, never fatal) # TASK_RUN_ABSOLUTE_CEILING_MS=86400000 # 24h — absolute runaway-cost ceiling; fails even a live task diff --git a/apps/api/src/env.ts b/apps/api/src/env.ts index 5c46accc7..ee2130409 100644 --- a/apps/api/src/env.ts +++ b/apps/api/src/env.ts @@ -219,6 +219,7 @@ export interface Env { TASK_STUCK_DELEGATED_TIMEOUT_MS?: string; TASK_DO_MISMATCH_GRACE_MS?: string; STUCK_TASK_MAX_CANDIDATES_PER_SWEEP?: string; + STUCK_TASK_SCAN_CURSOR_KV_KEY?: string; TASK_LIVENESS_MAX_ACP_SESSIONS?: string; TASK_LIVENESS_PROBE_TIMEOUT_MS?: string; TASK_RUN_ABSOLUTE_CEILING_MS?: string; diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index af53d3ef9..5d3ae809e 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -995,6 +995,10 @@ export default { stuckTaskDoHealthMissing: stuckTasks.doHealthMissing, stuckTaskDoHealthErrors: stuckTasks.doHealthErrors, stuckTaskDeadRuntimeReconciled: stuckTasks.deadRuntimeReconciled, + stuckTaskCandidatesScanned: stuckTasks.candidatesScanned, + stuckTaskCandidateCursorLoaded: stuckTasks.candidateCursorLoaded, + stuckTaskCandidateCursorWrapped: stuckTasks.candidateCursorWrapped, + stuckTaskCandidateCursorErrors: stuckTasks.candidateCursorErrors, observabilityPurgedByAge: observabilityPurge.deletedByAge, observabilityPurgedByCount: observabilityPurge.deletedByCount, cronTriggersChecked: cronTriggers.checked, diff --git a/apps/api/src/scheduled/stuck-tasks.ts b/apps/api/src/scheduled/stuck-tasks.ts index d808ab769..cb805fe9e 100644 --- a/apps/api/src/scheduled/stuck-tasks.ts +++ b/apps/api/src/scheduled/stuck-tasks.ts @@ -25,6 +25,7 @@ import { DEFAULT_NODE_HEARTBEAT_STALE_SECONDS, DEFAULT_STUCK_TASK_MAX_CANDIDATES_PER_SWEEP, + DEFAULT_STUCK_TASK_SCAN_CURSOR_KV_KEY, DEFAULT_TASK_DO_MISMATCH_GRACE_MS, DEFAULT_TASK_LIVENESS_MAX_ACP_SESSIONS, DEFAULT_TASK_LIVENESS_PROBE_TIMEOUT_MS, @@ -84,9 +85,38 @@ export interface StuckTaskResult { doHealthMissing: number; doHealthErrors: number; deadRuntimeReconciled: number; + candidatesScanned: number; + candidateCursorLoaded: boolean; + candidateCursorWrapped: boolean; + candidateCursorErrors: number; errors: number; } +export interface StuckTaskCandidate { + id: string; + project_id: string; + user_id: string; + status: string; + execution_step: string | null; + updated_at: string; + started_at: string | null; + workspace_id: string | null; + auto_provisioned_node_id: string | null; +} + +export interface StuckTaskScanCursor { + updatedAt: string; + taskId: string; +} + +export interface StuckTaskCandidateSelection { + tasks: StuckTaskCandidate[]; + nextCursor: StuckTaskScanCursor | null; + cursorLoaded: boolean; + wrapped: boolean; + cursorErrors: number; +} + export interface TaskRunnerProbeResult { outcome: 'ok' | 'missing' | 'timeout' | 'error'; status: { @@ -155,11 +185,156 @@ export interface TaskReconciliationDiagnostics { decision: TaskReconciliationDecision; liveness: TaskRuntimeLiveness | null; taskRunner: TaskRunnerProbeResult; + candidateScan: { + limit: number; + selectedCount: number; + selected: boolean; + position: number | null; + cursorLoaded: boolean; + wrapped: boolean; + cursorErrors: number; + }; } const LIVE_WORKSPACE_STATUSES = new Set(['running', 'recovery']); const ACTIVE_ACP_STATUSES = new Set(['assigned', 'running']); +function stuckTaskScanCursorKey(env: Env): string { + return env.STUCK_TASK_SCAN_CURSOR_KV_KEY?.trim() || DEFAULT_STUCK_TASK_SCAN_CURSOR_KV_KEY; +} + +function parseStuckTaskScanCursor(raw: string | null): StuckTaskScanCursor | null { + if (!raw) return null; + try { + const parsed = JSON.parse(raw) as Partial; + if ( + typeof parsed.updatedAt !== 'string' || + !Number.isFinite(Date.parse(parsed.updatedAt)) || + typeof parsed.taskId !== 'string' || + parsed.taskId.length === 0 || + parsed.taskId.length > 128 + ) { + return null; + } + return { updatedAt: parsed.updatedAt, taskId: parsed.taskId }; + } catch { + return null; + } +} + +const STUCK_TASK_CANDIDATE_COLUMNS = `id, project_id, user_id, status, execution_step, updated_at, started_at, + workspace_id, auto_provisioned_node_id`; + +/** + * Select one bounded, fair page of active tasks. A KV cursor prevents old live + * or inconclusive rows from permanently starving later dead rows. The first + * cursorless page starts at the newest active rows, then subsequent sweeps + * continue in ascending order and wrap at the end. + */ +export async function selectStuckTaskCandidates( + env: Env, + maxCandidates: number +): Promise { + const key = stuckTaskScanCursorKey(env); + let cursor: StuckTaskScanCursor | null = null; + let cursorLoaded = false; + let cursorErrors = 0; + + try { + const rawCursor = await env.KV.get(key); + cursor = parseStuckTaskScanCursor(rawCursor); + cursorLoaded = cursor !== null; + if (rawCursor && !cursor) { + cursorErrors++; + log.warn('stuck_task.invalid_scan_cursor', { key }); + } + } catch (err) { + cursorErrors++; + log.warn('stuck_task.scan_cursor_read_failed', { + key, + error: err instanceof Error ? err.message : String(err), + }); + } + + let tasks: StuckTaskCandidate[]; + if (cursor) { + const forward = await env.DATABASE.prepare( + `SELECT ${STUCK_TASK_CANDIDATE_COLUMNS} + FROM tasks + WHERE status IN ('queued', 'delegated', 'in_progress') + AND (updated_at > ? OR (updated_at = ? AND id > ?)) + ORDER BY updated_at ASC, id ASC + LIMIT ?` + ) + .bind(cursor.updatedAt, cursor.updatedAt, cursor.taskId, maxCandidates) + .all(); + tasks = forward.results; + } else { + // On rollout (or after a deliberately cleared cursor), examine the newest + // bounded page first so a fresh dead-runtime mismatch is not delayed behind + // a historical backlog. The persisted cursor makes later pages fair. + const newest = await env.DATABASE.prepare( + `SELECT ${STUCK_TASK_CANDIDATE_COLUMNS} + FROM ( + SELECT ${STUCK_TASK_CANDIDATE_COLUMNS} + FROM tasks + WHERE status IN ('queued', 'delegated', 'in_progress') + ORDER BY updated_at DESC, id DESC + LIMIT ? + ) + ORDER BY updated_at ASC, id ASC` + ) + .bind(maxCandidates) + .all(); + tasks = newest.results; + } + + let wrapped = false; + if (cursor && tasks.length < maxCandidates) { + const remaining = maxCandidates - tasks.length; + const wrap = await env.DATABASE.prepare( + `SELECT ${STUCK_TASK_CANDIDATE_COLUMNS} + FROM tasks + WHERE status IN ('queued', 'delegated', 'in_progress') + AND (updated_at < ? OR (updated_at = ? AND id <= ?)) + ORDER BY updated_at ASC, id ASC + LIMIT ?` + ) + .bind(cursor.updatedAt, cursor.updatedAt, cursor.taskId, remaining) + .all(); + if (wrap.results.length > 0) { + wrapped = true; + tasks.push(...wrap.results); + } + } + + const last = tasks.at(-1); + return { + tasks, + nextCursor: last ? { updatedAt: last.updated_at, taskId: last.id } : null, + cursorLoaded, + wrapped, + cursorErrors, + }; +} + +export async function persistStuckTaskScanCursor( + env: Env, + cursor: StuckTaskScanCursor +): Promise { + const key = stuckTaskScanCursorKey(env); + try { + await env.KV.put(key, JSON.stringify(cursor)); + return true; + } catch (err) { + log.warn('stuck_task.scan_cursor_write_failed', { + key, + error: err instanceof Error ? err.message : String(err), + }); + return false; + } +} + /** Bound and classify the cross-DO status RPC used by reconciliation diagnostics. */ export async function probeTaskRunnerStatus( env: Env, @@ -408,6 +583,10 @@ export async function getTaskReconciliationDiagnostics( DEFAULT_TASK_STUCK_DELEGATED_TIMEOUT_MS ); const maxExecutionMs = parseMs(env.TASK_RUN_MAX_EXECUTION_MS, DEFAULT_TASK_RUN_MAX_EXECUTION_MS); + const maxCandidates = parseMs( + env.STUCK_TASK_MAX_CANDIDATES_PER_SWEEP, + DEFAULT_STUCK_TASK_MAX_CANDIDATES_PER_SWEEP + ); let timeForCheck = elapsedMs; let halfThreshold = mismatchGraceMs; @@ -424,10 +603,14 @@ export async function getTaskReconciliationDiagnostics( const eligibilityThresholdMs = Math.min(halfThreshold, mismatchGraceMs); const active = ['queued', 'delegated', 'in_progress'].includes(task.status); const eligible = active && timeForCheck > eligibilityThresholdMs; - const [taskRunner, liveness] = await Promise.all([ + const [taskRunner, liveness, candidateSelection] = await Promise.all([ probeTaskRunnerStatus(env, task.id), task.status === 'in_progress' ? getTaskRuntimeLiveness(env, task) : Promise.resolve(null), + selectStuckTaskCandidates(env, maxCandidates), ]); + const candidateIndex = candidateSelection.tasks.findIndex( + (candidate) => candidate.id === task.id + ); let decision: TaskReconciliationDecision; if (!active) { @@ -455,6 +638,15 @@ export async function getTaskReconciliationDiagnostics( decision, liveness, taskRunner, + candidateScan: { + limit: maxCandidates, + selectedCount: candidateSelection.tasks.length, + selected: candidateIndex >= 0, + position: candidateIndex >= 0 ? candidateIndex + 1 : null, + cursorLoaded: candidateSelection.cursorLoaded, + wrapped: candidateSelection.wrapped, + cursorErrors: candidateSelection.cursorErrors, + }, }; } @@ -554,6 +746,10 @@ export async function recoverStuckTasks(env: Env): Promise { doHealthMissing: 0, doHealthErrors: 0, deadRuntimeReconciled: 0, + candidatesScanned: 0, + candidateCursorLoaded: false, + candidateCursorWrapped: false, + candidateCursorErrors: 0, errors: 0, }; @@ -593,32 +789,15 @@ export async function recoverStuckTasks(env: Env): Promise { }); } - // Find stuck tasks via raw SQL — include workspace_id and auto_provisioned_node_id - // for diagnostic context capture. - const stuckTasks = await env.DATABASE.prepare( - `SELECT id, project_id, user_id, status, execution_step, updated_at, started_at, - workspace_id, auto_provisioned_node_id - FROM tasks - WHERE status IN ('queued', 'delegated', 'in_progress') - ORDER BY updated_at ASC - LIMIT ?` - ) - .bind(maxCandidates) - .all<{ - id: string; - project_id: string; - user_id: string; - status: string; - execution_step: string | null; - updated_at: string; - started_at: string | null; - workspace_id: string | null; - auto_provisioned_node_id: string | null; - }>(); + const candidateSelection = await selectStuckTaskCandidates(env, maxCandidates); + result.candidatesScanned = candidateSelection.tasks.length; + result.candidateCursorLoaded = candidateSelection.cursorLoaded; + result.candidateCursorWrapped = candidateSelection.wrapped; + result.candidateCursorErrors = candidateSelection.cursorErrors; const db = drizzle(env.DATABASE, { schema }); - for (const task of stuckTasks.results) { + for (const task of candidateSelection.tasks) { const updatedAt = new Date(task.updated_at).getTime(); const elapsedMs = now.getTime() - updatedAt; let isStuck = false; @@ -1018,5 +1197,12 @@ export async function recoverStuckTasks(env: Env): Promise { } } + if ( + candidateSelection.nextCursor && + !(await persistStuckTaskScanCursor(env, candidateSelection.nextCursor)) + ) { + result.candidateCursorErrors++; + } + return result; } diff --git a/apps/api/tests/unit/stuck-tasks.test.ts b/apps/api/tests/unit/stuck-tasks.test.ts index f112e6912..84d035dbd 100644 --- a/apps/api/tests/unit/stuck-tasks.test.ts +++ b/apps/api/tests/unit/stuck-tasks.test.ts @@ -12,8 +12,10 @@ import type { Env } from '../../src/env'; import { detectClaudeCodeCompactionLoop } from '../../src/scheduled/claude-code-compaction-loop'; import { getTaskReconciliationDiagnostics, + persistStuckTaskScanCursor, probeTaskRunnerStatus, recoverStuckTasks, + selectStuckTaskCandidates, } from '../../src/scheduled/stuck-tasks'; import { persistError } from '../../src/services/observability'; import { cleanupTaskRun } from '../../src/services/task-runner'; @@ -98,9 +100,16 @@ function createMockEnv( : vi.fn().mockResolvedValue(taskRunnerStatus), }), }; + const kvStore = new Map(); return { DATABASE: mockDb, + KV: { + get: vi.fn(async (key: string) => kvStore.get(key) ?? null), + put: vi.fn(async (key: string, value: string) => { + kvStore.set(key, value); + }), + } as unknown as KVNamespace, OBSERVABILITY_DATABASE: { prepare: vi.fn().mockReturnValue(mockPreparedStatement()), } as unknown as D1Database, @@ -1115,11 +1124,7 @@ describe('recoverStuckTasks', () => { it('classifies a bounded TaskRunner status timeout', async () => { const neverResolves = new Promise(() => undefined); - const env = createMockEnv( - new Map(), - { TASK_LIVENESS_PROBE_TIMEOUT_MS: '1' }, - neverResolves, - ); + const env = createMockEnv(new Map(), { TASK_LIVENESS_PROBE_TIMEOUT_MS: '1' }, neverResolves); const probe = await probeTaskRunnerStatus(env, 'task-do-timeout'); @@ -1156,7 +1161,116 @@ describe('recoverStuckTasks', () => { decision: 'reconcile_dead_runtime', liveness: { conclusive: true, live: false, reason: 'workspace_deleted' }, taskRunner: { outcome: 'missing', status: null }, + candidateScan: { + limit: 100, + selectedCount: 1, + selected: true, + position: 1, + cursorLoaded: false, + wrapped: false, + cursorErrors: 0, + }, + }); + }); + + it('persists a bounded scan cursor and resumes after it on the next sweep', async () => { + const candidateConfig = { + results: [ + { + id: 'task-oldest-page', + project_id: 'proj-1', + user_id: 'user-1', + status: 'in_progress', + execution_step: 'running', + updated_at: '2026-07-12T10:00:00.000Z', + started_at: '2026-07-12T09:00:00.000Z', + workspace_id: 'ws-live', + auto_provisioned_node_id: null, + }, + ], + }; + const responses = new Map([ + ["status IN ('queued', 'delegated', 'in_progress')", candidateConfig], + ]); + const env = createMockEnv(responses); + + const first = await selectStuckTaskCandidates(env, 1); + expect(first).toMatchObject({ + cursorLoaded: false, + wrapped: false, + tasks: [{ id: 'task-oldest-page' }], + nextCursor: { + updatedAt: '2026-07-12T10:00:00.000Z', + taskId: 'task-oldest-page', + }, + }); + expect(String(vi.mocked(env.DATABASE.prepare).mock.calls[0]?.[0])).toContain( + 'ORDER BY updated_at DESC, id DESC' + ); + expect(first.nextCursor).not.toBeNull(); + if (!first.nextCursor) throw new Error('Expected a scan cursor'); + expect(await persistStuckTaskScanCursor(env, first.nextCursor)).toBe(true); + + candidateConfig.results = [ + { + ...candidateConfig.results[0], + id: 'task-next-page', + updated_at: '2026-07-12T11:00:00.000Z', + }, + ]; + const second = await selectStuckTaskCandidates(env, 1); + + expect(second).toMatchObject({ + cursorLoaded: true, + tasks: [{ id: 'task-next-page' }], + }); + expect(String(vi.mocked(env.DATABASE.prepare).mock.calls[1]?.[0])).toContain( + '(updated_at > ? OR (updated_at = ? AND id > ?))' + ); + }); + + it('wraps to the oldest active row after reaching the cursor tail', async () => { + const cursor = { + updatedAt: '2026-07-12T12:00:00.000Z', + taskId: 'task-cursor-tail', + }; + const wrappedTask = { + id: 'task-oldest-active', + project_id: 'proj-1', + user_id: 'user-1', + status: 'in_progress', + execution_step: 'running', + updated_at: '2026-07-12T08:00:00.000Z', + started_at: '2026-07-12T07:00:00.000Z', + workspace_id: 'ws-oldest', + auto_provisioned_node_id: null, + }; + const prepare = vi.fn((sql: string) => + mockPreparedStatement(sql.includes('updated_at > ?') ? [] : [wrappedTask]) + ); + const env = { + DATABASE: { prepare } as unknown as D1Database, + KV: { + get: vi.fn().mockResolvedValue(JSON.stringify(cursor)), + put: vi.fn(), + } as unknown as KVNamespace, + } as Env; + + const selection = await selectStuckTaskCandidates(env, 1); + + expect(selection).toMatchObject({ + cursorLoaded: true, + wrapped: true, + tasks: [{ id: 'task-oldest-active' }], + nextCursor: { + updatedAt: wrappedTask.updated_at, + taskId: wrappedTask.id, + }, }); + expect(prepare).toHaveBeenCalledTimes(2); + expect(String(prepare.mock.calls[1]?.[0])).toContain( + '(updated_at < ? OR (updated_at = ? AND id <= ?))' + ); }); }); @@ -1177,6 +1291,10 @@ describe('recoverStuckTasks', () => { doHealthMissing: 0, doHealthErrors: 0, deadRuntimeReconciled: 0, + candidatesScanned: 0, + candidateCursorLoaded: false, + candidateCursorWrapped: false, + candidateCursorErrors: 0, errors: 0, }); }); diff --git a/apps/api/tests/workers/scheduled-stuck-tasks.test.ts b/apps/api/tests/workers/scheduled-stuck-tasks.test.ts index 090c9c682..c0ce2fa74 100644 --- a/apps/api/tests/workers/scheduled-stuck-tasks.test.ts +++ b/apps/api/tests/workers/scheduled-stuck-tasks.test.ts @@ -9,6 +9,7 @@ * - Optimistic locking (task advanced between SELECT and UPDATE) * - Heartbeat grace period for in_progress tasks * - Diagnostic gathering (workspace/node status from D1) + * - Bounded D1 discovery resumes and wraps through a real KV cursor */ import { env, runInDurableObject } from 'cloudflare:test'; import { describe, expect, it } from 'vitest'; @@ -207,6 +208,62 @@ describe('recoverStuckTasks — vertical slice', () => { 1 ); }); + + it('selects a fresh dead runtime first, then resumes and wraps to an older active row', async () => { + await seedBaseData(); + const oldQueuedTaskId = 'task-st-cursor-old-queued'; + const deadTaskId = 'task-st-cursor-new-dead'; + const nodeId = 'node-st-cursor-new-dead'; + const workspaceId = 'ws-st-cursor-new-dead'; + const twentyMinutesAgo = new Date(Date.now() - 20 * 60 * 1000).toISOString(); + const tenMinutesAgo = new Date(Date.now() - 10 * 60 * 1000).toISOString(); + const oneMinuteAgo = new Date(Date.now() - 60 * 1000).toISOString(); + + await seedTask(oldQueuedTaskId, PROJECT_ID, USER_ID, { + status: 'queued', + executionStep: 'node_selection', + updatedAt: twentyMinutesAgo, + }); + await seedNode(nodeId, USER_ID, { status: 'deleted', healthStatus: 'stale' }); + await seedWorkspace(workspaceId, nodeId, USER_ID, { + projectId: PROJECT_ID, + status: 'deleted', + }); + await seedTask(deadTaskId, PROJECT_ID, USER_ID, { + status: 'in_progress', + executionStep: 'running', + startedAt: tenMinutesAgo, + updatedAt: oneMinuteAgo, + workspaceId, + }); + + const testEnv = { + ...env, + STUCK_TASK_MAX_CANDIDATES_PER_SWEEP: '1', + STUCK_TASK_SCAN_CURSOR_KV_KEY: 'test:stuck-task-cursor:fresh-then-wrap', + TASK_DO_MISMATCH_GRACE_MS: '1000', + TASK_STUCK_QUEUED_TIMEOUT_MS: '3600000', + } as unknown as Env; + + const firstSweep = await recoverStuckTasks(testEnv); + expect(firstSweep).toMatchObject({ + candidatesScanned: 1, + candidateCursorLoaded: false, + failedInProgress: 1, + deadRuntimeReconciled: 1, + }); + expect((await getTaskStatus(deadTaskId))?.status).toBe('failed'); + expect((await getTaskStatus(oldQueuedTaskId))?.status).toBe('queued'); + + const secondSweep = await recoverStuckTasks(testEnv); + expect(secondSweep).toMatchObject({ + candidatesScanned: 1, + candidateCursorLoaded: true, + candidateCursorWrapped: true, + failedQueued: 0, + }); + expect((await getTaskStatus(oldQueuedTaskId))?.status).toBe('queued'); + }); }); describe('stuck queued task detection', () => { diff --git a/apps/www/src/content/docs/docs/reference/configuration.md b/apps/www/src/content/docs/docs/reference/configuration.md index 69ad965a9..6a1bd9e34 100644 --- a/apps/www/src/content/docs/docs/reference/configuration.md +++ b/apps/www/src/content/docs/docs/reference/configuration.md @@ -198,28 +198,29 @@ SAM loads OpenCode Zen and OpenCode Go model choices through the authenticated m ## Idea Execution Timeouts -| Variable | Default | Description | -| -------------------------------------------------- | ------------------ | ------------------------------------------------------------------------------------- | -| `TASK_RUN_MAX_EXECUTION_MS` | `14400000` (4 hr) | Max task execution time | -| `TASK_STUCK_QUEUED_TIMEOUT_MS` | `1200000` (20 min) | Timeout for tasks stuck in queued state | -| `TASK_STUCK_DELEGATED_TIMEOUT_MS` | `1860000` (31 min) | Timeout for tasks stuck in delegated state | -| `TASK_DO_MISMATCH_GRACE_MS` | `300000` (5 min) | Minimum age before reconciling completed TaskRunner state with task-scoped liveness | -| `STUCK_TASK_MAX_CANDIDATES_PER_SWEEP` | `100` | Maximum active tasks inspected by each recovery sweep | -| `TASK_LIVENESS_MAX_ACP_SESSIONS` | `5` | Maximum task-scoped ACP sessions inspected per liveness probe | -| `TASK_LIVENESS_PROBE_TIMEOUT_MS` | `5000` (5 sec) | Per-candidate timeout for the ACP liveness probe; a timeout is inconclusive (never fails a task) | -| `TASK_RUN_ABSOLUTE_CEILING_MS` | `86400000` (24 hr) | Absolute runaway-cost ceiling; fails even a task with a demonstrably live runtime | -| `CLAUDE_CODE_COMPACTION_LOOP_DETECTOR_ENABLED` | `true` | Enable Claude Code compaction-loop shutdown from recent message evidence | -| `CLAUDE_CODE_COMPACTION_LOOP_RECENT_MESSAGE_LIMIT` | `40` | Recent task-session messages to inspect for compaction-loop evidence | -| `CLAUDE_CODE_COMPACTION_LOOP_WINDOW_MESSAGES` | `20` | Rolling recent-message window used for compaction-loop detection | -| `CLAUDE_CODE_COMPACTION_LOOP_MIN_PAIRS` | `3` | Minimum `Compacting...` / `Compacting completed` marker pairs before failing a task | -| `TASK_CALLBACK_TIMEOUT_MS` | `10000` | Callback response timeout | -| `TASK_CALLBACK_RETRY_MAX_ATTEMPTS` | `3` | Max callback retry attempts | -| `TASK_RUN_CLEANUP_DELAY_MS` | `5000` | Delay before task cleanup | -| `TASK_RECONCILIATION_IDLE_MS` | `300000` (5 min) | Idle threshold before SAM sends a visible task check-in | -| `TASK_RECONCILIATION_RESPONSE_DEADLINE_MS` | `60000` (1 min) | Response deadline after a visible task check-in | -| `TASK_RECONCILIATION_PROMPT_SOFT_STALL_MS` | `1800000` (30 min) | In-flight prompt observation threshold before a non-interrupting reconciliation event | -| `TASK_RECONCILIATION_PROMPT_HARD_STALL_MS` | `7200000` (2 hr) | In-flight prompt hard-stall threshold before SAM requests prompt cancellation | -| `TASK_RECONCILIATION_MIN_ALARM_DELAY_MS` | `10000` (10 sec) | Minimum delay before the next reconciliation alarm can fire | +| Variable | Default | Description | +| -------------------------------------------------- | -------------------------------------- | ------------------------------------------------------------------------------------------------ | +| `TASK_RUN_MAX_EXECUTION_MS` | `14400000` (4 hr) | Max task execution time | +| `TASK_STUCK_QUEUED_TIMEOUT_MS` | `1200000` (20 min) | Timeout for tasks stuck in queued state | +| `TASK_STUCK_DELEGATED_TIMEOUT_MS` | `1860000` (31 min) | Timeout for tasks stuck in delegated state | +| `TASK_DO_MISMATCH_GRACE_MS` | `300000` (5 min) | Minimum age before reconciling completed TaskRunner state with task-scoped liveness | +| `STUCK_TASK_MAX_CANDIDATES_PER_SWEEP` | `100` | Maximum active tasks inspected by each recovery sweep | +| `STUCK_TASK_SCAN_CURSOR_KV_KEY` | `scheduled:stuck-tasks:scan-cursor:v1` | KV key used to resume bounded recovery scans fairly across active tasks | +| `TASK_LIVENESS_MAX_ACP_SESSIONS` | `5` | Maximum task-scoped ACP sessions inspected per liveness probe | +| `TASK_LIVENESS_PROBE_TIMEOUT_MS` | `5000` (5 sec) | Per-candidate timeout for the ACP liveness probe; a timeout is inconclusive (never fails a task) | +| `TASK_RUN_ABSOLUTE_CEILING_MS` | `86400000` (24 hr) | Absolute runaway-cost ceiling; fails even a task with a demonstrably live runtime | +| `CLAUDE_CODE_COMPACTION_LOOP_DETECTOR_ENABLED` | `true` | Enable Claude Code compaction-loop shutdown from recent message evidence | +| `CLAUDE_CODE_COMPACTION_LOOP_RECENT_MESSAGE_LIMIT` | `40` | Recent task-session messages to inspect for compaction-loop evidence | +| `CLAUDE_CODE_COMPACTION_LOOP_WINDOW_MESSAGES` | `20` | Rolling recent-message window used for compaction-loop detection | +| `CLAUDE_CODE_COMPACTION_LOOP_MIN_PAIRS` | `3` | Minimum `Compacting...` / `Compacting completed` marker pairs before failing a task | +| `TASK_CALLBACK_TIMEOUT_MS` | `10000` | Callback response timeout | +| `TASK_CALLBACK_RETRY_MAX_ATTEMPTS` | `3` | Max callback retry attempts | +| `TASK_RUN_CLEANUP_DELAY_MS` | `5000` | Delay before task cleanup | +| `TASK_RECONCILIATION_IDLE_MS` | `300000` (5 min) | Idle threshold before SAM sends a visible task check-in | +| `TASK_RECONCILIATION_RESPONSE_DEADLINE_MS` | `60000` (1 min) | Response deadline after a visible task check-in | +| `TASK_RECONCILIATION_PROMPT_SOFT_STALL_MS` | `1800000` (30 min) | In-flight prompt observation threshold before a non-interrupting reconciliation event | +| `TASK_RECONCILIATION_PROMPT_HARD_STALL_MS` | `7200000` (2 hr) | In-flight prompt hard-stall threshold before SAM requests prompt cancellation | +| `TASK_RECONCILIATION_MIN_ALARM_DELAY_MS` | `10000` (10 sec) | Minimum delay before the next reconciliation alarm can fire | > **Liveness-gated recovery.** Stuck-task recovery for `in_progress` tasks (including task-mode work paused at the `awaiting_followup` execution step) is gated on **task-scoped** liveness — a live workspace, a healthy node with a recent heartbeat, **and** an active task-scoped ACP session. A shared-node heartbeat alone is never sufficient. Consequently, `TASK_RUN_HARD_TIMEOUT_MS` and `TASK_RUN_MAX_EXECUTION_MS` bound the point at which a task with **no** proven live runtime is failed; a task with a demonstrably live runtime is preserved past those thresholds, but remains bounded by `TASK_RUN_ABSOLUTE_CEILING_MS` (24 hours by default) as a runaway-cost backstop. When liveness cannot be determined (probe timeout or error), the task is left untouched (fail-safe) until it reaches that absolute ceiling. diff --git a/packages/shared/src/constants/index.ts b/packages/shared/src/constants/index.ts index 2c893694e..a28a2d2f0 100644 --- a/packages/shared/src/constants/index.ts +++ b/packages/shared/src/constants/index.ts @@ -70,6 +70,7 @@ export { export { DEFAULT_MAX_WORKSPACES_PER_NODE, DEFAULT_STUCK_TASK_MAX_CANDIDATES_PER_SWEEP, + DEFAULT_STUCK_TASK_SCAN_CURSOR_KV_KEY, DEFAULT_TASK_DO_MISMATCH_GRACE_MS, DEFAULT_TASK_LIVENESS_MAX_ACP_SESSIONS, DEFAULT_TASK_LIVENESS_PROBE_TIMEOUT_MS, diff --git a/packages/shared/src/constants/task-execution.ts b/packages/shared/src/constants/task-execution.ts index b62b8910d..bb5e7229d 100644 --- a/packages/shared/src/constants/task-execution.ts +++ b/packages/shared/src/constants/task-execution.ts @@ -50,6 +50,9 @@ export const DEFAULT_TASK_DO_MISMATCH_GRACE_MS = 5 * 60 * 1000; /** Maximum active task rows inspected by one stuck-task cron invocation. */ export const DEFAULT_STUCK_TASK_MAX_CANDIDATES_PER_SWEEP = 100; +/** KV key used to resume the bounded stuck-task scan without starving later rows. */ +export const DEFAULT_STUCK_TASK_SCAN_CURSOR_KV_KEY = 'scheduled:stuck-tasks:scan-cursor:v1'; + /** Maximum ACP sessions read while proving task-scoped runtime liveness. */ export const DEFAULT_TASK_LIVENESS_MAX_ACP_SESSIONS = 5; diff --git a/tasks/active/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md b/tasks/active/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md index 9940c8225..9e0da1c07 100644 --- a/tasks/active/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md +++ b/tasks/active/2026-07-11-taskrunner-d1-lifecycle-reconciliation.md @@ -72,6 +72,8 @@ the recovered work, all fixed: - TaskRunner status probes are bounded and classified as `ok`, `missing`, `timeout`, or `error`; cron summaries expose missing/error/reconciled counters. - A superadmin-only, read-only diagnostics endpoint reports the exact eligibility, liveness, TaskRunner probe, and decision for one task before any staging mutation. - Unit coverage proves missing/error TaskRunner RPC outcomes still reconcile a deleted runtime. A Worker D1/DO vertical slice also asserts the second sweep is a no-op; local `workerd` currently exits with SIGSEGV before importing Worker tests, so that file remains for CI/staging verification. +- The follow-up staging mismatch was correctly classified as `reconcile_dead_runtime` but remained active across the 16:30 and 16:35 UTC sweeps. The bounded query always reread the same 100 oldest active rows, so live/inconclusive historical rows could permanently starve later dead rows. +- Recovery now persists a configurable KV scan cursor, starts a cursorless rollout at the newest bounded page, resumes after the prior page, and wraps fairly. Cron summaries expose scan count/cursor state/errors, and the read-only endpoint reports whether and where the next page selects the inspected task. ## Acceptance criteria