fix: cap live task runtime with absolute ceiling

Author: raphaeltm

.claude/skills/env-reference/SKILL.md

@@ -113,6 +113,7 @@ See `apps/api/.env.example` for the full list. Key variables:
 - `TASK_RECONCILIATION_PROMPT_SOFT_STALL_MS` — In-flight prompt observation threshold before SAM records a non-interrupting reconciliation event (default: 1800000)
 - `TASK_RECONCILIATION_PROMPT_HARD_STALL_MS` — In-flight prompt hard-stall threshold before SAM requests prompt cancellation and retries check-in later (default: 7200000)
 - `TASK_RECONCILIATION_MIN_ALARM_DELAY_MS` — Minimum delay before the next reconciliation alarm can fire (default: 10000)
+- `TASK_RUN_ABSOLUTE_CEILING_MS` — Absolute runaway-cost ceiling that fails even a demonstrably live task (default: 86400000 / 24h)
 - `SESSION_ACTIVITY_STALE_THRESHOLD_MS` — Evidence-based fallback threshold before stale working activity can be healed to idle (default: 300000)
 - `NODE_HEARTBEAT_STALE_SECONDS` — Staleness threshold for node health
 - `NODE_AGENT_READY_TIMEOUT_MS` — Max wait for freshly provisioned node-agent health

apps/api/.env.example

@@ -350,6 +350,7 @@ BASE_DOMAIN=workspaces.example.com
 # STUCK_TASK_MAX_CANDIDATES_PER_SWEEP=100   # Bound task rows inspected per cron invocation
 # TASK_LIVENESS_MAX_ACP_SESSIONS=5          # Bound task-scoped ACP sessions inspected per candidate
 # TASK_LIVENESS_PROBE_TIMEOUT_MS=5000       # Per-candidate timeout for the ACP liveness DO probe (inconclusive on timeout, never fatal)
+# TASK_RUN_ABSOLUTE_CEILING_MS=86400000 # 24h — absolute runaway-cost ceiling; fails even a live task
 # CLAUDE_CODE_COMPACTION_LOOP_DETECTOR_ENABLED=true
 # CLAUDE_CODE_COMPACTION_LOOP_RECENT_MESSAGE_LIMIT=40
 # CLAUDE_CODE_COMPACTION_LOOP_WINDOW_MESSAGES=20

apps/api/src/env.ts

@@ -221,6 +221,7 @@ export interface Env {
   STUCK_TASK_MAX_CANDIDATES_PER_SWEEP?: string;
   TASK_LIVENESS_MAX_ACP_SESSIONS?: string;
   TASK_LIVENESS_PROBE_TIMEOUT_MS?: string;
+  TASK_RUN_ABSOLUTE_CEILING_MS?: string;
   CLAUDE_CODE_COMPACTION_LOOP_DETECTOR_ENABLED?: string;
   CLAUDE_CODE_COMPACTION_LOOP_RECENT_MESSAGE_LIMIT?: string;
   CLAUDE_CODE_COMPACTION_LOOP_WINDOW_MESSAGES?: string;

apps/api/src/scheduled/stuck-tasks.ts

@@ -28,6 +28,7 @@ import {
   DEFAULT_TASK_DO_MISMATCH_GRACE_MS,
   DEFAULT_TASK_LIVENESS_MAX_ACP_SESSIONS,
   DEFAULT_TASK_LIVENESS_PROBE_TIMEOUT_MS,
+  DEFAULT_TASK_RUN_ABSOLUTE_CEILING_MS,
   DEFAULT_TASK_RUN_HARD_TIMEOUT_MS,
   DEFAULT_TASK_RUN_MAX_EXECUTION_MS,
   DEFAULT_TASK_STUCK_DELEGATED_TIMEOUT_MS,
@@ -304,6 +305,7 @@ export async function recoverStuckTasks(env: Env): Promise<StuckTaskResult> {
   const delegatedTimeoutMs = parseMs(env.TASK_STUCK_DELEGATED_TIMEOUT_MS, DEFAULT_TASK_STUCK_DELEGATED_TIMEOUT_MS);
   const maxExecutionMs = parseMs(env.TASK_RUN_MAX_EXECUTION_MS, DEFAULT_TASK_RUN_MAX_EXECUTION_MS);
   const hardTimeoutMs = parseMs(env.TASK_RUN_HARD_TIMEOUT_MS, DEFAULT_TASK_RUN_HARD_TIMEOUT_MS);
+  const absoluteCeilingMs = parseMs(env.TASK_RUN_ABSOLUTE_CEILING_MS, DEFAULT_TASK_RUN_ABSOLUTE_CEILING_MS);
   const mismatchGraceMs = parseMs(env.TASK_DO_MISMATCH_GRACE_MS, DEFAULT_TASK_DO_MISMATCH_GRACE_MS);
   const maxCandidates = parseMs(env.STUCK_TASK_MAX_CANDIDATES_PER_SWEEP, DEFAULT_STUCK_TASK_MAX_CANDIDATES_PER_SWEEP);
 
@@ -315,6 +317,13 @@ export async function recoverStuckTasks(env: Env): Promise<StuckTaskResult> {
     });
   }
 
+  if (absoluteCeilingMs <= hardTimeoutMs) {
+    log.warn('stuck_task.misconfigured_absolute_ceiling', {
+      absoluteCeilingMs,
+      hardTimeoutMs,
+    });
+  }
+
   // Find stuck tasks via raw SQL — include workspace_id and auto_provisioned_node_id
   // for diagnostic context capture.
   const stuckTasks = await env.DATABASE.prepare(
@@ -409,6 +418,11 @@ export async function recoverStuckTasks(env: Env): Promise<StuckTaskResult> {
           const startedAt = task.started_at ? new Date(task.started_at).getTime() : updatedAt;
           const executionMs = now.getTime() - startedAt;
           if (executionMs > maxExecutionMs) {
+            if (executionMs > absoluteCeilingMs) {
+              isStuck = true;
+              reason = `Task exceeded the absolute runaway-cost ceiling of ${Math.round(absoluteCeilingMs / 60000)} minutes; live-runtime tasks are bounded to prevent unbounded compute.${stepInfo}`;
+              break;
+            }
             const liveness = await probeLiveness();
             if (liveness.live || !liveness.conclusive) {
               if (liveness.live) {

apps/api/tests/unit/stuck-tasks.test.ts

@@ -558,6 +558,136 @@ describe('recoverStuckTasks', () => {
     });
   });
 
+  describe('absolute runaway-cost ceiling', () => {
+    it('fails a live task past the absolute runaway-cost ceiling without probing liveness', async () => {
+      const now = Date.now();
+      const startedAt = new Date(now - 25 * 60 * 60 * 1000).toISOString();
+      const recentHeartbeat = new Date(now - 30 * 1000).toISOString();
+      const responses = new Map<string, { results: unknown[]; changes?: number }>();
+      responses.set("status IN ('queued', 'delegated', 'in_progress')", {
+        results: [
+          {
+            id: 'task-absolute-ceiling',
+            project_id: 'proj-1',
+            user_id: 'user-1',
+            status: 'in_progress',
+            execution_step: 'running',
+            updated_at: startedAt,
+            started_at: startedAt,
+            workspace_id: 'ws-1',
+            auto_provisioned_node_id: 'node-1',
+          },
+        ],
+      });
+      responses.set('w.chat_session_id', {
+        results: [
+          {
+            workspace_status: 'running',
+            chat_session_id: 'chat-1',
+            node_id: 'node-1',
+            node_status: 'running',
+            health_status: 'healthy',
+            last_heartbeat_at: recentHeartbeat,
+          },
+        ],
+      });
+      responses.set('node_id, status FROM workspaces', {
+        results: [{ id: 'ws-1', node_id: 'node-1', status: 'running' }],
+      });
+      responses.set('status, health_status FROM nodes', {
+        results: [{ id: 'node-1', status: 'running', health_status: 'healthy' }],
+      });
+      responses.set("UPDATE tasks SET status = 'failed'", { results: [], changes: 1 });
+      const env = createMockEnv(responses);
+      const result = await recoverStuckTasks(env);
+      expect(result.failedInProgress).toBe(1);
+      expect(result.heartbeatSkipped).toBe(0);
+      expect(projectDataMocks.listAcpSessions).not.toHaveBeenCalled();
+      expect(syncTriggerExecutionMock).toHaveBeenCalledWith(
+        env.DATABASE,
+        'task-absolute-ceiling',
+        'failed',
+        expect.stringContaining('absolute runaway-cost ceiling')
+      );
+    });
+
+    it('preserves a live task below the absolute runaway-cost ceiling', async () => {
+      const now = Date.now();
+      const startedAt = new Date(now - 23 * 60 * 60 * 1000).toISOString();
+      const recentHeartbeat = new Date(now - 30 * 1000).toISOString();
+      const responses = new Map<string, { results: unknown[]; changes?: number }>();
+      responses.set("status IN ('queued', 'delegated', 'in_progress')", {
+        results: [
+          {
+            id: 'task-below-absolute-ceiling',
+            project_id: 'proj-1',
+            user_id: 'user-1',
+            status: 'in_progress',
+            execution_step: 'running',
+            updated_at: startedAt,
+            started_at: startedAt,
+            workspace_id: 'ws-1',
+            auto_provisioned_node_id: 'node-1',
+          },
+        ],
+      });
+      responses.set('w.chat_session_id', {
+        results: [
+          {
+            workspace_status: 'running',
+            chat_session_id: 'chat-1',
+            node_id: 'node-1',
+            node_status: 'running',
+            health_status: 'healthy',
+            last_heartbeat_at: recentHeartbeat,
+          },
+        ],
+      });
+      const result = await recoverStuckTasks(createMockEnv(responses));
+      expect(result.failedInProgress).toBe(0);
+      expect(result.heartbeatSkipped).toBe(1);
+    });
+
+    it('falls back to the default absolute ceiling for an invalid zero value', async () => {
+      const now = Date.now();
+      const startedAt = new Date(now - 23 * 60 * 60 * 1000).toISOString();
+      const recentHeartbeat = new Date(now - 30 * 1000).toISOString();
+      const responses = new Map<string, { results: unknown[]; changes?: number }>();
+      responses.set("status IN ('queued', 'delegated', 'in_progress')", {
+        results: [
+          {
+            id: 'task-invalid-absolute-ceiling',
+            project_id: 'proj-1',
+            user_id: 'user-1',
+            status: 'in_progress',
+            execution_step: 'running',
+            updated_at: startedAt,
+            started_at: startedAt,
+            workspace_id: 'ws-1',
+            auto_provisioned_node_id: 'node-1',
+          },
+        ],
+      });
+      responses.set('w.chat_session_id', {
+        results: [
+          {
+            workspace_status: 'running',
+            chat_session_id: 'chat-1',
+            node_id: 'node-1',
+            node_status: 'running',
+            health_status: 'healthy',
+            last_heartbeat_at: recentHeartbeat,
+          },
+        ],
+      });
+      const result = await recoverStuckTasks(
+        createMockEnv(responses, { TASK_RUN_ABSOLUTE_CEILING_MS: '0' })
+      );
+      expect(result.failedInProgress).toBe(0);
+      expect(result.heartbeatSkipped).toBe(1);
+    });
+  });
+
   describe('hard timeout enforcement', () => {
     it('preserves a genuinely live task past the hard timeout', async () => {
       const now = Date.now();

apps/www/src/content/docs/docs/reference/configuration.md

@@ -207,6 +207,7 @@ SAM loads OpenCode Zen and OpenCode Go model choices through the authenticated m
 | `STUCK_TASK_MAX_CANDIDATES_PER_SWEEP`             | `100`              | Maximum active tasks inspected by each recovery sweep                                 |
 | `TASK_LIVENESS_MAX_ACP_SESSIONS`                  | `5`                | Maximum task-scoped ACP sessions inspected per liveness probe                         |
 | `TASK_LIVENESS_PROBE_TIMEOUT_MS`                  | `5000` (5 sec)     | Per-candidate timeout for the ACP liveness probe; a timeout is inconclusive (never fails a task) |
+| `TASK_RUN_ABSOLUTE_CEILING_MS`                    | `86400000` (24 hr) | Absolute runaway-cost ceiling; fails even a task with a demonstrably live runtime     |
 | `CLAUDE_CODE_COMPACTION_LOOP_DETECTOR_ENABLED`     | `true`             | Enable Claude Code compaction-loop shutdown from recent message evidence              |
 | `CLAUDE_CODE_COMPACTION_LOOP_RECENT_MESSAGE_LIMIT` | `40`               | Recent task-session messages to inspect for compaction-loop evidence                  |
 | `CLAUDE_CODE_COMPACTION_LOOP_WINDOW_MESSAGES`      | `20`               | Rolling recent-message window used for compaction-loop detection                      |
@@ -220,7 +221,7 @@ SAM loads OpenCode Zen and OpenCode Go model choices through the authenticated m
 | `TASK_RECONCILIATION_PROMPT_HARD_STALL_MS`         | `7200000` (2 hr)   | In-flight prompt hard-stall threshold before SAM requests prompt cancellation         |
 | `TASK_RECONCILIATION_MIN_ALARM_DELAY_MS`           | `10000` (10 sec)   | Minimum delay before the next reconciliation alarm can fire                           |
 
-> **Liveness-gated recovery.** Stuck-task recovery for `in_progress` tasks (including task-mode work paused at the `awaiting_followup` execution step) is gated on **task-scoped** liveness — a live workspace, a healthy node with a recent heartbeat, **and** an active task-scoped ACP session. A shared-node heartbeat alone is never sufficient. Consequently, `TASK_RUN_HARD_TIMEOUT_MS` and `TASK_RUN_MAX_EXECUTION_MS` bound the point at which a task with **no** proven live runtime is failed; a task with a demonstrably live runtime is preserved past those thresholds rather than terminated on elapsed time alone. When liveness cannot be determined (probe timeout or error), the task is left untouched (fail-safe).
+> **Liveness-gated recovery.** Stuck-task recovery for `in_progress` tasks (including task-mode work paused at the `awaiting_followup` execution step) is gated on **task-scoped** liveness — a live workspace, a healthy node with a recent heartbeat, **and** an active task-scoped ACP session. A shared-node heartbeat alone is never sufficient. Consequently, `TASK_RUN_HARD_TIMEOUT_MS` and `TASK_RUN_MAX_EXECUTION_MS` bound the point at which a task with **no** proven live runtime is failed; a task with a demonstrably live runtime is preserved past those thresholds, but remains bounded by `TASK_RUN_ABSOLUTE_CEILING_MS` (24 hours by default) as a runaway-cost backstop. When liveness cannot be determined (probe timeout or error), the task is left untouched (fail-safe) until it reaches that absolute ceiling.
 
 ## Node & Workspace Readiness
 

packages/shared/src/constants/index.ts

@@ -73,6 +73,7 @@ export {
   DEFAULT_TASK_DO_MISMATCH_GRACE_MS,
   DEFAULT_TASK_LIVENESS_MAX_ACP_SESSIONS,
   DEFAULT_TASK_LIVENESS_PROBE_TIMEOUT_MS,
+  DEFAULT_TASK_RUN_ABSOLUTE_CEILING_MS,
   DEFAULT_TASK_RUN_CLEANUP_DELAY_MS,
   DEFAULT_TASK_RUN_HARD_TIMEOUT_MS,
   DEFAULT_TASK_RUN_MAX_EXECUTION_MS,

packages/shared/src/constants/task-execution.ts

@@ -30,6 +30,9 @@ export const DEFAULT_TASK_RUN_MAX_EXECUTION_MS = 4 * 60 * 60 * 1000; // 4 hours
  * Override via TASK_RUN_HARD_TIMEOUT_MS env var. */
 export const DEFAULT_TASK_RUN_HARD_TIMEOUT_MS = 8 * 60 * 60 * 1000; // 8 hours
 
+/** Absolute runaway-cost backstop (ms) that bounds even demonstrably live tasks. */
+export const DEFAULT_TASK_RUN_ABSOLUTE_CEILING_MS = 24 * 60 * 60 * 1000;
+
 /** Default threshold (ms) for a task stuck in 'queued' status. Override via TASK_STUCK_QUEUED_TIMEOUT_MS env var.
  * Must be > TASK_RUNNER_AGENT_READY_TIMEOUT_MS (15 min) to avoid the stuck-task cron killing tasks
  * that are legitimately waiting for cloud-init to finish. Cloud-init takes 8-12 min on Hetzner.