Skip to content

Commit c57e81f

Browse files
committed
fix(workflow): bound cancel/reclaim timeouts to prevent indefinite hangs
- Add RECLAIM_WORKTREE_TIMEOUT_MS (10s) to worktree.remove during reclaim - Add RECLAIM_ACTOR_TIMEOUT_MS (5s) to actor.cancel during reclaim - Add FIBER_INTERRUPT_TIMEOUT_MS (5s) to Fiber.interrupt in cancelEntry These timeouts prevent cancel from hanging indefinitely when: 1. git worktree remove --force blocks on processes using the worktree 2. A child actor is stuck in an uninterruptible operation 3. Fiber.interrupt stalls on a hung LLM fetch The specific fix addresses the 30s timeout in: 'WorkflowRuntime worktree isolation > cancel removes worktrees of in-flight isolated agents' Root cause: PR #1709 added Instance.disposeDirectory calls in worktree.remove which increased the time for cancel cleanup. Combined with git worktree remove --force potentially blocking, the cancel operation could exceed the test timeout.
1 parent b8ce1c6 commit c57e81f

1 file changed

Lines changed: 32 additions & 3 deletions

File tree

packages/opencode/src/workflow/runtime.ts

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -436,19 +436,39 @@ export const layer = Layer.effect(
436436
// worktree the run still owns, then clear the set. NEVER throws — a reclaim
437437
// failure must not mask the original terminal cause. NOT called on success:
438438
// kept (success+changed) worktrees are the deliverable and must survive.
439+
// Worktree removal is bounded by RECLAIM_WORKTREE_TIMEOUT_MS so a hung
440+
// git worktree remove (e.g. processes using the worktree) cannot block
441+
// cancel indefinitely. Actor cancel is bounded by RECLAIM_ACTOR_TIMEOUT_MS
442+
// so a hung child actor cannot block reclaim indefinitely.
443+
const RECLAIM_WORKTREE_TIMEOUT_MS = 10_000
444+
const RECLAIM_ACTOR_TIMEOUT_MS = 5_000
439445
const reclaim = (entry: RunEntry) =>
440446
Effect.gen(function* () {
441447
const actor = spawnRef.current
442448
if (actor) {
443449
yield* Effect.forEach(
444450
[...entry.childActorIDs],
445-
(childID) => actor.cancel(entry.sessionID, childID, "graceful").pipe(Effect.ignore),
451+
(childID) =>
452+
actor.cancel(entry.sessionID, childID, "graceful").pipe(
453+
Effect.timeout(RECLAIM_ACTOR_TIMEOUT_MS),
454+
Effect.catchTag("TimeoutError", () =>
455+
Effect.sync(() => log.warn("actor cancel timed out during reclaim", { childID })),
456+
),
457+
Effect.ignore,
458+
),
446459
{ concurrency: "unbounded", discard: true },
447460
)
448461
}
449462
yield* Effect.forEach(
450463
[...entry.worktrees],
451-
(directory) => worktree.remove({ directory }).pipe(Effect.ignore),
464+
(directory) =>
465+
worktree.remove({ directory }).pipe(
466+
Effect.timeout(RECLAIM_WORKTREE_TIMEOUT_MS),
467+
Effect.catchTag("TimeoutError", () =>
468+
Effect.sync(() => log.warn("worktree remove timed out during reclaim", { directory })),
469+
),
470+
Effect.ignore,
471+
),
452472
{ concurrency: "unbounded", discard: true },
453473
)
454474
entry.worktrees.clear()
@@ -472,13 +492,22 @@ export const layer = Layer.effect(
472492
)
473493
})
474494

495+
// Bounded interrupt: Fiber.interrupt can stall on a hung LLM fetch or an
496+
// uninterruptible operation. Bounding it so cancel completes in finite time.
497+
const FIBER_INTERRUPT_TIMEOUT_MS = 5_000
475498
const cancelEntry = (entry: RunEntry): Effect.Effect<void> =>
476499
Effect.gen(function* () {
477500
if (entry.status !== "running") return
478501
yield* reclaim(entry)
479502
yield* flushNow(entry)
480503
yield* WorkflowPersistence.recordTerminal({ runID: entry.runID, status: "cancelled" }).pipe(Effect.ignore)
481-
if (entry.fiber) yield* Fiber.interrupt(entry.fiber)
504+
if (entry.fiber)
505+
yield* Fiber.interrupt(entry.fiber).pipe(
506+
Effect.timeout(FIBER_INTERRUPT_TIMEOUT_MS),
507+
Effect.catchTag("TimeoutError", () =>
508+
Effect.sync(() => log.warn("fiber interrupt timed out during cancel", { runID: entry.runID })),
509+
),
510+
)
482511
entry.status = "cancelled"
483512
yield* Deferred.succeed(entry.deferred, { status: "cancelled" })
484513
yield* bus.publish(WorkflowFinished, { sessionID: entry.sessionID, runID: entry.runID, status: "cancelled" })

0 commit comments

Comments
 (0)