From 4c17153d5819dbc4c5febc7393e35ce20edd9792 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sat, 18 Jul 2026 21:45:08 +0200 Subject: [PATCH 01/15] docs(plan): multiple simultaneous approval requests in one turn Plan-only workspace for removing the one-approval-per-turn latch so each gated tool call in a turn emits its own approval card, pluralizing the parked-approval record and the warm/keep-alive resume path, and verifying the already-plural frontend and SDK behavior with tests. Plans #5373, assesses #5078 and #5097. Claude-Session: https://claude.ai/code/session_01DnWRxU3dCJ11hgDidm26vW --- .../projects/concurrent-approvals/PLAN.md | 432 ++++++++++++++++++ .../projects/concurrent-approvals/README.md | 52 +++ 2 files changed, 484 insertions(+) create mode 100644 docs/design/agent-workflows/projects/concurrent-approvals/PLAN.md create mode 100644 docs/design/agent-workflows/projects/concurrent-approvals/README.md diff --git a/docs/design/agent-workflows/projects/concurrent-approvals/PLAN.md b/docs/design/agent-workflows/projects/concurrent-approvals/PLAN.md new file mode 100644 index 0000000000..31da75e6a1 --- /dev/null +++ b/docs/design/agent-workflows/projects/concurrent-approvals/PLAN.md @@ -0,0 +1,432 @@ +# Plan: multiple simultaneous approval requests in one turn + +Read [README.md](README.md) first for the shared vocabulary. Every domain word used below +(runner, harness, gate, approval card, park, latch, cold path, warm path, force-settle) is +defined there. + +## Summary + +When an agent tries to call several gated tools at the same moment in one turn, for example +"read these three files" where each read needs approval, the runtime shows the human only +the first approval card. It cancels the other gated calls, marks them "not executed, paused", +and lets the model ask for them again on later turns. The user sees the agent stall, approve +one thing, then loop back and ask for the next, turn after turn. GitHub issue #5373 reports +exactly this. + +The cause is a deliberate one-approval-per-turn guard in the runner (the latch). Every layer +above the runner already handles more than one pending approval: the wire carries one event +per gate, the SDK emits one frame per event, and the frontend already waits for every pending +card to be answered before it resumes. Only the runner caps the count at one, and only the +warm resume path lacks a place to store more than one parked gate. + +This plan removes the cap and pluralizes the two singular runner data structures, so each +gate emits its own approval card and each card is settled by its own tool-call id. It then +verifies with tests that the frontend and SDK, which are already written for the plural case, +actually work on both the cold and warm resume paths. No wire contract changes shape. The +work is runner-only for behavior, plus new tests on the frontend and SDK. + +## What the user sees today + +An agent turn can call more than one tool at once. Claude does this routinely: asked to read +three files, it emits three `read_file` tool calls in a single assistant turn. If those tools +are gated, the harness raises three approval gates in that one turn. + +The runner handles the first gate normally: it shows the approval card and parks the turn. +For the second and third gates it does something different. It counts them, but it does not +show their cards. It force-settles them as "not executed, paused" so they do not hang as open +calls. The turn ends with one card visible. + +The human approves the one card. The run resumes. The model, seeing the first tool now done +and the other two never executed, asks for them again. The runtime shows the next card. The +human approves again. This repeats once per remaining gate. + +The observable result, quoting #5373: "the interaction takes additional turns and does not +behave correctly." It still finishes, but every extra gate costs a full extra round of model +call, pause, and human click. + +### Where each piece of that behavior lives + +All runner paths below are current on `origin/main` (verified 2026-07-18; line numbers may +drift slightly as the file changes). + +- **The one-per-turn cap (the latch).** `PendingApprovalLatch` + (`services/runner/src/permission-plan.ts:173-185`) returns `true` on its first + `tryAcquire()` call and `false` on every later call. The runner constructs one latch per + turn (`services/runner/src/engines/sandbox_agent/run-turn.ts:256`). +- **The card emit is guarded by the latch.** `pauseUserApproval` + (`services/runner/src/engines/sandbox_agent/acp-interactions.ts:152-183`) fires its + `onUserApprovalGate` signal first (line 161), then runs `if (!latch.tryAcquire()) return;` + (line 169) before emitting the `interaction_request` card (lines 173-183). So the second + gate's signal is recorded but its card is never emitted. +- **The siblings are force-settled.** When the turn pauses, open gated calls that did not get + a card are settled as `TOOL_NOT_EXECUTED_PAUSED` + (`run-turn.ts:231-236`, and a post-drain re-sweep at `run-turn.ts:487-492`). +- **Only the first gate is remembered for a live resume.** The parked-gate record is written + only when `env.approvalGateCount === 1` (`run-turn.ts:350-368`, the `env.parkedApproval = + {...}` at 353-366). The record type `ParkedApproval` holds a single `permissionId`, + `toolCallId`, and `toolName` (`runtime-contracts.ts:112-127`). +- **The warm resume path refuses multi-gate turns outright.** The keep-alive dispatch + (`server.ts:419-437`) declines to live-park when more than one gate is pending: + `if ((env.approvalGateCount ?? 0) > 1) { klog("multi-gate-no-park ..."); return false; }` + (`server.ts:428-431`). Multi-gate turns fall back to the cold path. + +## Why the layers above the runner are not the problem + +The research trace (Question 2 of the ground-truth document) confirmed, and this plan +re-verified against `origin/main`, that everything above the runner already handles more than +one pending approval: + +- **The wire already carries one event per gate.** There is no batched multi-approval frame + and none is needed. Each gate is its own `interaction_request` with a distinct `toolCallId`. +- **The SDK already emits one approval frame per event.** In the browser-facing egress, + `_interaction_parts` yields exactly one `tool-approval-request` chunk per `user_approval` + event (`sdks/python/agenta/sdk/agents/adapters/vercel/stream.py:708-712`). Feed it three + events and it emits three frames. +- **The SDK ingress already keys each answer by tool-call id.** Each `approval-responded` part + the browser sends back becomes a `tool_result` block keyed by `toolCallId` + (`sdks/python/agenta/sdk/agents/adapters/vercel/messages.py:176-206`). Many answers become + many blocks. +- **The runner's cold-path decision store is already plural.** `extractApprovalDecisions` + returns a `Map`, a list of decisions per key + (`services/runner/src/responder.ts:358-373`), and `ConversationDecisions.take` consumes one + per key in order (`responder.ts:256-266`). The map holds many; the cold path answers them + one gate per replayed turn. +- **The frontend already waits for every pending card.** `agentShouldResumeAfterApproval` + finds the last freshly-answered card and then requires + `toolParts.every(isSettledToolPart)` before it resumes + (`web/packages/agenta-playground/src/state/execution/agentApprovalResume.ts:131-165`, + the all-settled check at 163-164). A second still-pending card keeps the run paused until + the human answers it too. +- **The parallel-park pattern already exists in the runner.** Browser-fulfilled "client" + tools already park more than one widget in one turn. `buildClientToolRelay` deliberately + ignores the latch and emits one widget per pending call + (`services/runner/src/engines/sandbox_agent/client-tools.ts:176-266`). It marks each parked + call (`markPausedToolCall`) so the force-settle sweep skips it, and it pauses the turn once + through an idempotent `pause()`. This is the exact shape the approval path should copy. + +The conclusion: the approval path is the only place that caps the count, and the warm resume +path is the only place missing a plural data structure. The frontend and SDK need no behavior +change; they need tests that prove the plural case works end to end. + +## The design + +Four steps. Steps 1 to 3 change the runner. Step 4 adds tests to the frontend and SDK. The +approved direction from Mahmoud is exactly these; this section elaborates them. + +### Step 1: remove the one-approval-per-turn cap so each gate emits its own card + +Make the approval path behave like the existing client-tool path: no latch, one card per +gate, one idempotent pause for the turn, and each parked call marked so the force-settle +sweep leaves it alone. + +- In `pauseUserApproval` (`acp-interactions.ts`), delete the `if (!latch.tryAcquire()) + return;` guard (line 169). The `onPausedToolCall?.(toolCallId)` call that already runs + (line 172) is the equivalent of the client path's `markPausedToolCall`; it keeps this gate + out of the force-settle sweep. The pause that ends the turn is already idempotent, so N + gates end the turn once. +- Decide what to do with `pauseClientTool`'s own latch use + (`acp-interactions.ts:195`). This is a separate, less-used path: the ACP-gate variant of a + client tool. Two options, decided in step 1's implementation: (a) remove its latch too, so + it matches the primary client-tool path, or (b) leave it, because the primary client + delivery does not go through it. Recommendation: remove it for consistency, since after + step 1 the latch guards nothing that should be capped. See open question 1. +- The `PendingApprovalLatch` class itself (`permission-plan.ts:173-185`) becomes unused for + approvals. Keep the class only if `pauseClientTool` still uses it; otherwise delete the + class, its construction in `run-turn.ts:256`, and the `latch` field threaded through + `acp-interactions.ts` and the `LatchLike` surface in `client-tools.ts:164-181`. Removing + dead code is preferred once no caller remains. + +**Contract after step 1:** the runner emits one `interaction_request` (`kind: +"user_approval"`) per ask gate in a turn, each with a distinct `toolCallId`. The turn ends +once. No force-settle happens for a gate that got a card. + +### Step 2: store every parked gate, not just the first + +Turn the singular parked-gate record into a collection keyed by tool-call id, so a live +resume can answer each gate. + +- In `runtime-contracts.ts`, add a plural field to `SessionEnvironment`. The record type + `ParkedApproval` (lines 112-127) stays as the per-gate shape. Change the environment to + hold `parkedApprovals` as a `Map` keyed by `toolCallId` (or an + array; a map is preferred because resume answers arrive keyed by `toolCallId`). Keep a + derived singular accessor only if a caller genuinely needs "the first gate" for logging. +- In `run-turn.ts` (the `onUserApprovalGate` handler, lines 350-368), record every gate into + the map instead of only when `env.approvalGateCount === 1`. Keep `approvalGateCount` as the + count; it now equals `parkedApprovals.size`. +- In `run-turn.ts`, adjust the force-settle early-return + (`if (opts.approvalParkMode && env.parkedApproval) return;`, near line 178) to check "this + tool call is in `parkedApprovals`" rather than "any single park exists". + +**Contract after step 2:** the runner remembers every parked gate for the current turn, +addressable by `toolCallId`. Nothing is emitted differently; this is internal state that the +warm resume in step 3 reads. + +### Step 3: let the warm resume path park a multi-gate turn and answer every gate + +The warm path (keep-alive) keeps the harness process alive across the pause and delivers the +human's answer live. Today it refuses any multi-gate turn and drops to the cold path. Relax +that, and answer each parked gate by its id. + +- In `server.ts` (`approvalToPark`, lines 419-437), remove the hard `approvalGateCount > 1` + refusal (lines 428-431). Replace it with a check that the turn is fully parkable: every + pending gate has a recorded `ParkedApproval` (a `permissionId` the harness can answer). If + any pending gate is non-parkable, for example a client-tool MCP pause that carries no + `permissionId`, keep the whole turn on the cold path, because the cold path is the only one + that can multiplex a mixed set today. The existing `non-parkable-gate-no-park` branch + (lines 424-426) already handles "no park at all"; the new check generalizes it to "not all + gates parkable". +- The warm-park bookkeeping that reads `env.parkedApproval?.promptPromise` + (`server.ts:448,475,510`) needs a representative promise for the parked set. Use any one + gate's `promptPromise` for the watchdog, or gate the watchdog on "all parked promises + settled". Decide in implementation; the simplest correct choice is to watch the set and + evict when any parked prompt rejects. See open question 2. +- In `runtime-contracts.ts`, generalize the resume input. `ResumeApprovalInput` (lines + 129-138) answers one gate. Change the warm resume to carry a list, for example + `ResumeApprovalInput[]` or a `{ decisions: ResumeApprovalInput[] }` batch. The server builds + this list from the inbound responded parts, which the SDK already keyed by `toolCallId`, and + which `extractApprovalDecisions` already returns as a map. +- In `run-turn.ts` (the live resume, `env.session.respondPermission(...)` at lines 454-457), + iterate the parked gates. For each, look up the human's decision by `toolCallId`, and call + `respondPermission(permissionId, reply)` once per gate. This assumes the harness holds + several pending permission requests concurrently and answers each independently. Confirm + this against the Claude ACP adapter during implementation and in the live test. See open + question 3. + +**Contract after step 3:** a warm resume can carry more than one decision and settle each +parked gate on the live harness session by its `permissionId`. If any gate in the turn cannot +be parked, the whole turn uses the cold path, exactly as a single non-parkable gate does +today. + +### Step 4: prove the frontend and SDK handle the plural case + +The frontend and SDK already read as plural. This step adds tests, and a small check that two +cards render sensibly, so the plural case is pinned and cannot regress. + +- **Frontend unit test.** Feed `agentShouldResumeAfterApproval` a turn with two + `approval-requested` parts. Assert it does not resume until both are answered, then resumes + once both are settled. This exercises the all-settled check + (`agentApprovalResume.ts:163-164`) with more than one card. +- **Frontend rendering check.** In the playground, confirm two approval cards in one turn + render as two distinct, independently answerable widgets, correctly attached to their own + tool bubbles by `toolCallId`. This is a manual live check plus, if feasible, a component + test. It also checks the interaction with issue #5078 (see composition below). +- **SDK unit tests.** Egress: feed `_interaction_parts` two `user_approval` events, assert two + `tool-approval-request` frames. Ingress: feed `messages.py` two `approval-responded` parts, + assert two `tool_result` blocks keyed by their two `toolCallId`s. These pin + `stream.py:708-712` and `messages.py:176-206` for the plural case. + +## File-level change list + +Behavior changes (runner): + +| File | Change | Contract effect | +| --- | --- | --- | +| `services/runner/src/engines/sandbox_agent/acp-interactions.ts` | Remove the latch guard in `pauseUserApproval` (line 169); decide the same for `pauseClientTool` (line 195). | One approval card per gate; turn ends once. | +| `services/runner/src/permission-plan.ts` | Remove `PendingApprovalLatch` (lines 173-185) once no caller remains. | Dead code removed; no behavior of its own. | +| `services/runner/src/engines/sandbox_agent/run-turn.ts` | Record every gate into `parkedApprovals`; drop the `approvalGateCount === 1` condition (lines 350-368); adjust the force-settle early-return (near 178); iterate gates in the live resume (454-457). | Every parked gate remembered and answerable. | +| `services/runner/src/engines/sandbox_agent/runtime-contracts.ts` | Add `parkedApprovals: Map` to `SessionEnvironment`; make the warm resume input a list. | Internal runtime contract goes plural; wire unchanged. | +| `services/runner/src/engines/sandbox_agent/server.ts` | Replace the `approvalGateCount > 1` refusal (428-431) with an "all gates parkable" check; adapt the parked-prompt watchdog to the set. | Warm path can live-park a multi-gate turn; mixed sets still fall to cold. | + +Test-only changes: + +| File | Change | +| --- | --- | +| `services/runner/tests/unit/...` (new or extended) | Two-gates-one-turn on the cold path and the warm path; force-settle no longer fires for a second gate; multi-answer warm resume. | +| `web/packages/agenta-playground/src/state/execution/agentApprovalResume.test.ts` | Two pending cards: no resume until both settled. | +| `sdks/python/oss/tests/pytest/unit/agents/...` | Egress two frames; ingress two keyed blocks. | +| Release gate journey spec (see test plan) | New scenario: agent calls two gated tools in one turn. | + +Client-tool client-tool paths (`client-tools.ts`, `responder.ts` cold store, `transcript.ts` +resume frames) need no change. They are already plural; the plan reuses their pattern. + +## What changes on the wire and in the frontend, and what stays + +**Stays the same:** + +- The wire event shape. One `interaction_request` with `kind: "user_approval"` per gate, each + with its own `toolCallId`, exactly as today. We remove an artificial per-turn cap on how + many flow; we do not add a field or a batched frame. +- The SDK frame shape. One `tool-approval-request` per event; one `tool_result` block per + answer, keyed by `toolCallId`. +- The frontend resume rule. Wait until every pending card is answered, then resume once. +- The approve/deny envelope (`{ approved: boolean }` keyed by `toolCallId`) that the cold and + warm paths both consume. + +**Changes:** + +- Runner-internal contracts only: the parked-gate record goes from one to a map, and the warm + resume input goes from one decision to a list. These types are internal to the runner and + its keep-alive server; they never cross the browser boundary. +- Observable behavior: a turn can now show two or three cards at once, and the warm path can + answer them in one resume instead of forcing one gate per replayed turn. + +Because the wire shape is unchanged, an older frontend paired with the new runner still works: +it renders each card as it arrives and answers each, since it already did that for the +single-card case. The improvement is that the runner now sends all the cards at once instead +of trickling them across turns. + +## Test plan + +**Runner unit tests.** + +- Two gated tool calls in one turn: assert two `interaction_request` events emitted, the turn + ends once, and neither gate is force-settled as `TOOL_NOT_EXECUTED_PAUSED`. +- Cold resume of a two-gate turn: assert both decisions are read from the replayed history and + both tools run without a second re-ask. +- Warm resume of a two-gate turn: assert the dispatch live-parks (does not log + `multi-gate-no-park`), the resume input carries two decisions, and `respondPermission` is + called once per parked gate. +- Mixed set: one approval gate plus one non-parkable client-tool pause in one turn stays on + the cold path (the "all gates parkable" check fails), matching today's single non-parkable + behavior. + +**SDK unit tests.** Egress emits two frames for two events; ingress emits two keyed blocks +for two answers. Add these to the existing vercel adapter test files. + +**Frontend unit test.** `agentApprovalResume` does not resume with a second card pending; +resumes once both settle. + +**Live playground scenario (the headline manual test).** Configure an agent whose tools are +gated with an ask rule. Prompt it to do something that calls two gated tools at once, for +example "read file A and file B" where `read_file` is gated. Confirm: two approval cards +appear together in the one turn; each is independently approvable and denyable; approving both +runs both tools and the agent continues in one resume, with no extra re-ask turns. Run this on +the cold path first, then repeat with the warm (keep-alive) path enabled, since the warm path +is the least-tested corner. + +**Warm-path variant checks.** With keep-alive on: approve both cards, confirm one resume +settles both on the live session. Then a deny variant: deny one and approve the other in the +same turn, confirm the denied tool reports a decline and the approved one runs. Then a partial +answer: answer one card and leave the other pending, confirm the run stays paused (the +frontend all-settled rule holds). + +**Release-gate journey spec.** Add a scenario to the agent release gate harness (the +`agent-release-gate` skill's wire-level QA harness) that drives the product endpoint with an +agent that calls two gated tools in one turn, and asserts on the SSE frame stream: two +`tool-approval-request` frames in the turn, both tool results present after the resume, and no +duplicate or re-asked approval frames in a later turn. This asserts on frames and side +effects, not model prose, so it runs against any deployment. Consider pinning the run as an +`agent-replay-test` once green. + +## How this composes with in-flight work + +Three pull requests touch the same files or the same wire area. This plan is written to sit +cleanly next to all three. + +- **Deny-frame egress, PR #5381 (branch `feat/deny-frame-egress`).** It added a structural + `denied` marker on the runner `tool_result` event and a `tool-output-denied` egress frame, + so a decline renders differently from a tool breakage. It touches `acp-interactions.ts`, + `run-turn.ts`, `protocol.ts`, `tracing/otel.ts`, and `stream.py`. This plan touches + `acp-interactions.ts` and `run-turn.ts` too, but in different concerns: #5381 marks a + gate's deny outcome, this plan changes how many gates emit and park. The deny marker is + per-`toolCallId`, so it composes naturally with per-gate approval: each of two cards can be + approved or denied, and each denied gate gets its own `tool-output-denied` frame. The + implementer should land after #5381 or rebase onto it, and add a warm-path test that mixes + approve and deny across two gates in one turn (listed in the warm-path variant checks above). +- **Sessions turns/streams runner, PR #5376 (branch `sessions-rebase/runner`), JP's work.** + It rewrites the keep-alive machinery: `server.ts` (the alive watchdog becomes awaited and + gains an `onInterrupted` abort callback, plus `streamId` threading), `run-turn.ts` (turn + completion becomes `appendSessionTurn`), `runtime-contracts.ts`, and `protocol.ts`. This is + the warm/keep-alive path, which is exactly where this plan's step 3 lives. The two changes + overlap in `server.ts`, `run-turn.ts`, and `runtime-contracts.ts`. Because the warm resume + is built on the sessions machinery, this plan's warm-path step should stack on top of the + sessions PRs, not race them. See the rebase story below. +- **Client tools on Daytona (being planned in parallel, projects + [daytona-gate-delivery](../daytona-gate-delivery/) and + [mcp-client-tool-continuation](../mcp-client-tool-continuation/)).** That work adds a new + park/resume channel so browser-fulfilled client tools can round-trip on Claude-in-Daytona. + It shares this plan's interest in parking more than one interaction per turn, but on a + different channel (the MCP client-tool path, not the ACP approval path). The two do not + conflict in files. The shared invariant both must keep: the frontend all-settled rule and + the "every pending interaction is its own widget, keyed by tool-call id" model. State that + invariant in both plans so neither regresses it. + +## Rebase story + +The warm-path step (step 3) overlaps JP's sessions runner PR #5376 in `server.ts`, +`run-turn.ts`, and `runtime-contracts.ts`. Sequence the work to avoid a three-way tangle: + +1. Land the cold-path-safe part of this plan first if it helps: steps 1 and 2 (remove the + latch, pluralize the parked record) plus their tests are almost independent of the sessions + rewrite. They change `acp-interactions.ts`, `permission-plan.ts`, and the record type. This + can go on its own lane based on `main`, or stacked under the deny-frame lane if that lands + first. +2. Base step 3 (the warm resume) on JP's sessions runner once it merges, or stack the + concurrent-approvals warm lane on top of `sessions-rebase/runner` if the timing forces + parallel work. Do not edit JP's branch. Re-home this plan's `server.ts` and `run-turn.ts` + edits onto whatever those files look like after the sessions rewrite, since the sessions PR + renames and moves the very functions step 3 changes (`startAliveWatchdog`, the turn + completion path). +3. If the deny-frame PR #5381 has not merged when step 3 lands, rebase onto it too, because it + also edits `run-turn.ts` and `acp-interactions.ts`. The concerns are disjoint, so the + rebase is mechanical, but the two must be reconciled in one working tree before pushing. + +Order to prefer: deny-frame (#5381) and sessions (#5375/#5376) land, then concurrent-approvals +steps 1 to 2, then step 3 on top of the sessions machinery. If steps 1 to 2 are ready before +sessions lands, they can go first since they do not touch the keep-alive files, and step 3 +follows. + +## Rollout and rollback + +- **Rollout.** Steps 1 and 2 change cold-path and shared behavior and should ship together so + a multi-gate turn shows all its cards at once even on the cold path. Step 3 improves the warm + path and can ship in the same change or immediately after, gated behind the same keep-alive + configuration that already controls warm resume. No new flag is required for the card-count + change; the wire is unchanged and the frontend already handles it. +- **Rollback.** Reverting steps 1 to 2 restores the latch and the single-card-per-turn + behavior; the frontend and SDK tolerate this because it is today's behavior. Reverting step 3 + restores the `multi-gate-no-park` refusal, so multi-gate turns fall back to the cold path, + which still works, just with the extra re-ask turns. Each step is independently revertible. +- **Risk.** The warm resume answering several concurrent permission requests is the least + proven behavior. The live warm-path variant checks are the gate on shipping step 3. If the + Claude ACP adapter turns out to serialize gates rather than hold them concurrently (open + question 3), step 3's benefit shrinks to the cold path, which steps 1 and 2 already fix. + +## Non-goals + +- **No relay-executed ask parking.** Giving the runner-side relay loop a way to park a + resolved code or gateway tool that carries an ask rule is a separate, larger build tracked + as S5.2 in [../../scratch/open-issues.md](../../scratch/open-issues.md). This plan only + covers harness-raised ACP approval gates, which already have a park mechanism. +- **No cross-surface approval records.** Making an approval raised on one surface answerable + from another (for example a run started by an API client and approved in the playground) is + the multi-surface roadmap, not this plan. +- **No new client-tool channel.** Client tools on Claude-in-Daytona are the parallel + daytona-gate-delivery work. +- **No trace-join or usage-summing change.** Issue #5097's trace continuity work is assessed, + not owned, here (see composition and issue links). + +## Open questions for review + +1. **The `pauseClientTool` latch (`acp-interactions.ts:195`).** After step 1 the latch caps + nothing that should be capped. Remove it there too for consistency, or leave that ACP-gate + client-tool corner alone because the primary client delivery does not use it? Recommendation: + remove, and delete the now-dead `PendingApprovalLatch` class. +2. **The warm parked-prompt watchdog with a set of gates.** The keep-alive eviction watches + one gate's `promptPromise` today (`server.ts:448`). With several parked gates, watch any one, + or watch the whole set and evict when any parked prompt rejects? The set is safer; confirm it + does not over-evict. +3. **Does the Claude harness hold several permission requests concurrently in one turn?** Step + 3 assumes the ACP adapter raises multiple gates that each await their own `respondPermission`. + Issue #5373 (parallel file reads) strongly implies yes, but this must be confirmed against the + Claude ACP adapter and in the live test before step 3 is trusted. If Claude serializes gates + instead, the cold-path fix (steps 1 to 2) still resolves #5373, and step 3 becomes a smaller + optimization. +4. **Issue #5078 (approved tools appear multiple times).** Its root cause is a cold-replay + re-mint of the `toolCallId` on resume, addressed on the frontend in PR #5058 by deduplicating + on tool identity. Removing the multi-turn re-ask (this plan) reduces how often that resume + re-mint happens for the multi-gate case, so it should reduce #5078 occurrences, but it does + not replace the frontend dedup. Confirm during the live rendering check that two cards plus + the dedup do not interact badly. Treat #5078 as assessed and likely reduced, not owned. + +## Issue links + +- **Plans #5373** ("HITL breaks on multi-file approval flow"). This is exactly the multi-gate + breakage: several gated tool calls in one turn, capped to one card, re-asked across turns. The + implementation pull request that follows this plan will close it. +- **Assesses #5078** ("Approved tools appear multiple times in chat"). Related. Root cause is a + cold-replay `toolCallId` re-mint fixed on the frontend in #5058; this plan reduces the re-ask + turns that trigger it but does not own the fix. +- **Assesses #5097** ("Join a turn's approval-resume requests into one trace"). Related. Fewer + resume requests per turn touch the problem, but #5097's client-side trace replay is orthogonal + and still needed; this plan does not subsume it. diff --git a/docs/design/agent-workflows/projects/concurrent-approvals/README.md b/docs/design/agent-workflows/projects/concurrent-approvals/README.md new file mode 100644 index 0000000000..72f9f613c1 --- /dev/null +++ b/docs/design/agent-workflows/projects/concurrent-approvals/README.md @@ -0,0 +1,52 @@ +# Concurrent approvals: more than one approval request in a single turn + +This workspace plans one change to the agent runtime: let a single agent turn ask the +human to approve more than one tool call at the same time. Today the runtime shows exactly +one approval card per turn even when the agent tried to call several gated tools at once. +The extra calls are cancelled and re-requested on later turns, which the user sees as the +agent stalling and repeating itself. + +## Reading order + +- [PLAN.md](PLAN.md): the whole plan. Read it top to bottom. It opens with what the user + sees today, explains why, then gives the design, the file-level changes, the interface + effect, the test plan, how this composes with three in-flight pull requests, the rebase + story, rollout and rollback, non-goals, and open questions. + +There is only one plan document. This README exists to define the shared vocabulary once so +PLAN.md can lean on it. + +## Vocabulary (defined once here) + +- **Runner**: the Node/TypeScript service that runs one agent turn: it talks to the model + harness, streams events back to the browser, and executes or gates tool calls. Code lives + under `services/runner/`. +- **Harness**: the model driver the runner speaks to over a local protocol called ACP + (Agent Client Protocol). The two harnesses are Claude (Anthropic's agent SDK) and Pi. +- **Gate**: a point where a tool call needs a human yes or no before it runs. A gate that + needs a human is an "ask" gate. +- **Approval card / approval request**: the inline "Run this tool? Approve / Deny" widget + the playground shows for an ask gate. On the wire it is one `interaction_request` event + with `kind: "user_approval"`. +- **Park**: end the current turn while a gate waits for the human, then continue the same + logical turn once the human answers. Parking is how the run pauses without failing. +- **Latch**: a one-shot guard object (`PendingApprovalLatch`) that today allows only the + first approval card in a turn to be shown and silently blocks the rest. +- **Cold path (cold-replay)**: the resume style that rebuilds the whole conversation as + text and sends it to a fresh model turn. The human's answer is read back out of that + replayed history. +- **Warm path (keep-alive)**: the resume style that keeps the same harness process and ACP + session alive across the pause, so the human's answer is delivered live to the waiting + harness instead of being replayed as text. +- **Force-settle**: mark a gated tool call as "not executed, paused" so it does not hang as + an open call when its sibling gate parked the turn. +- **Frontend / FE**: the playground UI, in `web/packages/agenta-playground/` and + `web/oss/`. +- **SDK**: the Python agent SDK in `sdks/python/agenta/`. It converts between the browser's + message format and the runner's event stream. + +Related prior work this plan builds on: [../hitl-fix/](../hitl-fix/) (the original +approve/deny round-trip fix) and [../cold-replay-stopgaps/](../cold-replay-stopgaps/) (the +text-replay hardening). The ground-truth code trace this plan is built on lives at +[../../scratch/research-client-tools-and-concurrent-hitl.md](../../scratch/research-client-tools-and-concurrent-hitl.md), +Question 2. From 747024dbaa51de63dafbb92e9dba11dad1bb7111 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 19 Jul 2026 00:26:53 +0200 Subject: [PATCH 02/15] feat(runner): allow multiple simultaneous approval requests in one turn Remove the one-approval-per-turn latch (PendingApprovalLatch) so each gated tool call in a turn emits its own approval card; pluralize the parked-approval record (a Map keyed by tool-call id) and the warm keep-alive resume input (a list of decisions), settling each parked gate by its own tool-call id on a live resume; defer orphan-sibling settling to the post-drain sweep so a concurrent gate is not force-settled before its own permission request arrives. Add runner unit tests (two cards emit, neither force-settled; multi-gate warm park + resume; deny-one-approve-one; mixed/unresumable sets stay cold), SDK vercel adapter plural egress/ingress tests, and a playground all-settled resume test. Closes #5373. Claude-Session: https://claude.ai/code/session_01DnWRxU3dCJ11hgDidm26vW --- .../adapters/test_vercel_stream_park.py | 81 +++++ .../pytest/unit/agents/test_ui_messages.py | 41 +++ .../engines/sandbox_agent/acp-interactions.ts | 36 +- .../src/engines/sandbox_agent/client-tools.ts | 31 +- .../sandbox_agent/environment-setup.ts | 2 + .../sandbox_agent/runtime-contracts.ts | 36 +- .../engines/sandbox_agent/runtime-policy.ts | 19 +- services/runner/src/permission-plan.ts | 14 - services/runner/src/server.ts | 162 ++++++--- .../runner/tests/unit/client-tools.test.ts | 6 +- .../runner/tests/unit/permission-plan.test.ts | 12 - .../unit/sandbox-agent-orchestration.test.ts | 34 +- .../unit/session-keepalive-approval.test.ts | 337 +++++++++++++++--- .../unit/session-keepalive-dispatch.test.ts | 6 + .../tests/unit/agentApprovalResume.test.ts | 61 ++++ 15 files changed, 686 insertions(+), 192 deletions(-) diff --git a/sdks/python/oss/tests/pytest/unit/agents/adapters/test_vercel_stream_park.py b/sdks/python/oss/tests/pytest/unit/agents/adapters/test_vercel_stream_park.py index c9204c038e..eac1241c08 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/adapters/test_vercel_stream_park.py +++ b/sdks/python/oss/tests/pytest/unit/agents/adapters/test_vercel_stream_park.py @@ -84,6 +84,87 @@ async def test_parked_run_emits_approval_then_finish() -> None: assert finish["finishReason"] == "other" +def _two_concurrent_gates_run() -> AgentStream: + """One turn that raises TWO approval gates at once (concurrent approvals): two distinct tool + calls, each followed by its own ``user_approval`` request, then the terminal ``paused`` result. + The egress must surface one ``tool-approval-request`` per gate, each keyed to its own tool-call + id, and still drain to a single clean finish.""" + return AgentStream( + _records( + [ + { + "kind": "event", + "event": { + "type": "tool_call", + "id": "tool-1", + "name": "github__get_user", + "input": {}, + }, + }, + { + "kind": "event", + "event": { + "type": "tool_call", + "id": "tool-2", + "name": "github__list_repos", + "input": {}, + }, + }, + { + "kind": "event", + "event": { + "type": "interaction_request", + "id": "perm-1", + "kind": "user_approval", + "payload": {"toolCallId": "tool-1"}, + }, + }, + { + "kind": "event", + "event": { + "type": "interaction_request", + "id": "perm-2", + "kind": "user_approval", + "payload": {"toolCallId": "tool-2"}, + }, + }, + {"kind": "event", "event": {"type": "done", "stopReason": "paused"}}, + { + "kind": "result", + "result": { + "ok": True, + "output": "", + "stopReason": "paused", + "sessionId": "conv-1", + "traceId": "trace-1", + }, + }, + ] + ) + ) + + +@pytest.mark.asyncio +async def test_concurrent_gates_emit_one_approval_request_each() -> None: + """Two ``user_approval`` events in one turn yield TWO ``tool-approval-request`` frames — one + per event — each carrying its own approvalId and tool-call id, so the FE can prompt for both + gates independently. Pins the plural side of the single-gate contract above.""" + parts = [ + part async for part in agent_run_to_vercel_parts(_two_concurrent_gates_run()) + ] + + approvals = [p for p in parts if p["type"] == "tool-approval-request"] + assert len(approvals) == 2 + # One frame per event, each keyed to its own gate (perm/tool pair), order preserved. + assert [(a["approvalId"], a["toolCallId"]) for a in approvals] == [ + ("perm-1", "tool-1"), + ("perm-2", "tool-2"), + ] + # Both gates share the turn but the stream still drains to a single clean finish (F-040). + assert parts[-1]["type"] == "finish" + assert [p.get("type") for p in parts].count("finish") == 1 + + def _parked_run_with_real_args() -> AgentStream: """A parked turn where the runner first surfaced the tool call with EMPTY input, then the approval request carries the REAL args on ``payload.toolCall.rawInput`` (the cold-replay diff --git a/sdks/python/oss/tests/pytest/unit/agents/test_ui_messages.py b/sdks/python/oss/tests/pytest/unit/agents/test_ui_messages.py index b4e06b53fa..e9f90900b9 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/test_ui_messages.py +++ b/sdks/python/oss/tests/pytest/unit/agents/test_ui_messages.py @@ -240,6 +240,47 @@ def test_inline_approval_responded_approve_emits_approved_envelope(self): assert result.tool_call_id == "call_2" assert result.output == {"approved": True} + def test_concurrent_inline_approvals_each_emit_their_own_result(self): + # Concurrent approvals: the browser round-trips TWO tool parts in one turn, each with its + # own inline decision (`state: approval-responded`, `approval.approved`) — one approve, one + # deny. The ingress must emit one `{approved}` tool_result per part, keyed by its own + # toolCallId, preserving each decision independently (neither gate clobbers the other). + msgs = vercel_ui_messages_to_messages( + [ + { + "id": "m10", + "role": "assistant", + "parts": [ + { + "type": "tool-deleteFile", + "toolCallId": "call_1", + "state": "approval-responded", + "input": {"path": "/a"}, + "approval": {"id": "perm_1", "approved": True}, + }, + { + "type": "tool-writeFile", + "toolCallId": "call_2", + "state": "approval-responded", + "input": {"path": "/b"}, + "approval": {"id": "perm_2", "approved": False}, + }, + ], + } + ] + ) + results = [b for b in msgs[0].content if b.type == "tool_result"] + assert len(results) == 2 + # Each decision lands on its own toolCallId, in order, approve and deny preserved. + assert (results[0].tool_call_id, results[0].output) == ( + "call_1", + {"approved": True}, + ) + assert (results[1].tool_call_id, results[1].output) == ( + "call_2", + {"approved": False}, + ) + def test_inline_output_denied_state_emits_denied_envelope(self): # The AI SDK terminal deny state has no `approval.approved` flag; `output-denied` # itself means denied, so the ingress still emits `{approved: False}`. diff --git a/services/runner/src/engines/sandbox_agent/acp-interactions.ts b/services/runner/src/engines/sandbox_agent/acp-interactions.ts index 89cd808360..b70570693c 100644 --- a/services/runner/src/engines/sandbox_agent/acp-interactions.ts +++ b/services/runner/src/engines/sandbox_agent/acp-interactions.ts @@ -11,7 +11,6 @@ import { } from "../../responder.ts"; import { piBuiltinIdentity, - PendingApprovalLatch, type GateDescriptor, } from "../../permission-plan.ts"; import { @@ -44,7 +43,6 @@ export interface AttachPermissionResponderInput { markToolCallDenied?: (toolCallId: string | undefined) => void; }; responder: Responder; - latch: PendingApprovalLatch; serverPermissions?: ReadonlyMap; /** * Called when a gate pauses the turn. The orchestration loop uses this to end the turn @@ -54,6 +52,13 @@ export interface AttachPermissionResponderInput { log?: (msg: string) => void; /** Called with the ACP tool-call id when a gate pauses the turn. */ onPausedToolCall?: (id: string) => void; + /** + * Called when a NON-parkable pause happens this turn (a client-tool ACP gate, which cannot be + * answered across a turn boundary on the live session). The keep-alive dispatch reads this to + * keep a turn that mixes an approval gate with a client-tool pause on the cold path, since only + * the cold path can multiplex that mixed set. An approval gate does NOT fire it. + */ + onNonParkablePause?: () => void; /** Called on pause to record the pending gate as an interaction (fire-and-forget). */ onCreateInteraction?: ( token: string, @@ -65,10 +70,10 @@ export interface AttachPermissionResponderInput { onResolveInteraction?: (token: string) => void; /** * Fires for EVERY parkable permission gate (a Claude ACP gate or a Pi ACP gate) that - * resolves to pendingApproval, BEFORE the single-pause latch. Keep-alive uses it to record - * the parked permission id / tool-call id (for a live resume via `respondPermission`) and to - * count how many gates are pending this turn (a multi-gate pause does not park). It never - * fires for a client-tool gate (`pauseClientTool`), which stays on the cold path. + * resolves to pendingApproval. Keep-alive uses it to record every parked permission id / + * tool-call id (for a live resume via `respondPermission`, keyed by tool-call id) and to count + * how many gates are pending this turn. It never fires for a client-tool gate + * (`pauseClientTool`), which fires `onNonParkablePause` instead and stays on the cold path. */ onUserApprovalGate?: (info: { permissionId: string; @@ -109,11 +114,11 @@ export function attachPermissionResponder({ session, run, responder, - latch, serverPermissions = new Map(), onPause, log, onPausedToolCall, + onNonParkablePause, onCreateInteraction, onResolveInteraction, onUserApprovalGate, @@ -155,19 +160,18 @@ export function attachPermissionResponder({ gate: GateDescriptor, gateType: ParkedApprovalGateType, ): void => { - // Signal the parkable gate BEFORE the latch so a keep-alive resume can record the pending - // permission id and the multi-gate detector counts every pending gate (not just the first). - const gateToolCallId = stringValue(req?.toolCall?.toolCallId); + // Signal the parkable gate so a keep-alive resume can record the pending permission id and + // count every pending gate. Each gate emits its own card: there is no per-turn cap, so N + // gated calls in one turn all render and all park (the plural approval path). + const toolCallId = stringValue(req?.toolCall?.toolCallId); onUserApprovalGate?.({ permissionId: id, - toolCallId: gateToolCallId ?? "", + toolCallId: toolCallId ?? "", toolName: gate.toolName, args: gate.args, - interactionToken: interactionEventId(id, gateToolCallId), + interactionToken: interactionEventId(id, toolCallId), gateType, }); - if (!latch.tryAcquire()) return; - const toolCallId = stringValue(req?.toolCall?.toolCallId); const eventId = interactionEventId(id, toolCallId); if (toolCallId) onPausedToolCall?.(toolCallId); run.emitEvent({ @@ -192,7 +196,9 @@ export function attachPermissionResponder({ gate: GateDescriptor, spec: ResolvedToolSpec, ): void => { - if (!latch.tryAcquire()) return; + // A client-tool ACP pause cannot be answered on the live session across a turn boundary, so + // flag the turn non-parkable: a turn that mixes this with an approval gate stays cold. + onNonParkablePause?.(); const toolCallId = stringValue(req?.toolCall?.toolCallId); const eventId = interactionEventId(id, toolCallId); if (toolCallId) onPausedToolCall?.(toolCallId); diff --git a/services/runner/src/engines/sandbox_agent/client-tools.ts b/services/runner/src/engines/sandbox_agent/client-tools.ts index ee244aef59..492ead08de 100644 --- a/services/runner/src/engines/sandbox_agent/client-tools.ts +++ b/services/runner/src/engines/sandbox_agent/client-tools.ts @@ -11,7 +11,7 @@ * `tools/call` handler (`tools/tool-mcp-http.ts`). * * Both consume the `ClientToolRelay` built here, so the pause decision (the responder's verdict - * ladder), the single `client_tool` payload shape, the pause-latch bookkeeping, and the + * ladder), the single `client_tool` payload shape, the pause bookkeeping, and the * ACP-tool-call correlation live in ONE place instead of being re-derived per path. */ import type { AgentEvent, RenderHint } from "../../protocol.ts"; @@ -200,11 +200,6 @@ export function emitClientToolInteraction( }); } -/** The one-pause-per-turn latch surface the seam needs (see `PendingApprovalLatch`). */ -interface LatchLike { - tryAcquire(): boolean; -} - /** The pause-controller surface the seam needs (see `PendingApprovalPauseController`). */ interface PauseLike { markPausedToolCall(toolCallId: string): void; @@ -214,12 +209,14 @@ interface PauseLike { export interface BuildClientToolRelayInput { responder: Responder; run: EmitRun; - /** The approval-dialog latch. Client tools do NOT gate on it — each pending client tool parks - * its own widget, and only the approval path (`acp-interactions.ts`) enforces one dialog per - * turn. Kept in the shared input so the wiring stays symmetric; ignored here. */ - latch?: LatchLike; /** The turn-ender: `pause()` cancels the prompt; `markPausedToolCall` suppresses late frames. */ pause: PauseLike; + /** + * Flag the turn non-parkable when a browser-fulfilled client tool pauses: it cannot be answered + * on the live session across a turn boundary, so a turn that mixes it with an approval gate must + * stay on the cold path. Fires once per pending client tool (idempotent counter on the caller). + */ + onNonParkablePause?: () => void; /** Seeds the durable interactions plane for the pending call (fire-and-forget). */ recordPendingInteraction: ( token: string, @@ -247,6 +244,7 @@ export function buildClientToolRelay({ pause, recordPendingInteraction, toolCallIndex, + onNonParkablePause, log = () => {}, }: BuildClientToolRelayInput): ClientToolRelay { return { @@ -281,12 +279,15 @@ export function buildClientToolRelay({ const correlatedId = toolCallIndex?.lookup(request.toolName, request.input) ?? request.toolCallId; - // Every pending client tool parks its OWN widget. Unlike an approval DIALOG (one at a time, - // gated by the latch), browser-fulfilled client tools are independent surfaces — an agent may - // request several connections in one turn, and each must render. The turn still ends exactly - // once: `pause.pause()` (via `onPause`) is idempotent, so N emits pause the turn once, and - // `markPausedToolCall` keeps each parked call from being force-settled as a deferred sibling. + // Every pending client tool parks its OWN widget: browser-fulfilled client tools are + // independent surfaces — an agent may request several connections in one turn, and each must + // render. Approval gates now behave the same way (no per-turn cap). The turn still ends + // exactly once: `pause.pause()` (via `onPause`) is idempotent, so N emits pause the turn + // once, and `markPausedToolCall` keeps each parked call from being force-settled as a sibling. pause.markPausedToolCall(correlatedId); + // A client-tool pause keeps the whole turn on the cold path: the warm resume cannot answer + // a browser-fulfilled call, so a turn that also holds an approval gate must not park. + onNonParkablePause?.(); emitClientToolInteraction(run, { id: request.id, toolCallId: correlatedId, diff --git a/services/runner/src/engines/sandbox_agent/environment-setup.ts b/services/runner/src/engines/sandbox_agent/environment-setup.ts index e27d25fbb7..3e9d8ecfb7 100644 --- a/services/runner/src/engines/sandbox_agent/environment-setup.ts +++ b/services/runner/src/engines/sandbox_agent/environment-setup.ts @@ -347,8 +347,10 @@ export async function prepareEnvironmentSetup( closeToolMcp: undefined, currentTurn: undefined, lastTurnToolCallIds: [], + parkedApprovals: new Map(), parkedApproval: undefined, approvalGateCount: 0, + nonParkablePauseCount: 0, destroyed: false, destroy: async () => {}, clearTurn: () => {}, diff --git a/services/runner/src/engines/sandbox_agent/runtime-contracts.ts b/services/runner/src/engines/sandbox_agent/runtime-contracts.ts index 04479499a1..ee1db91430 100644 --- a/services/runner/src/engines/sandbox_agent/runtime-contracts.ts +++ b/services/runner/src/engines/sandbox_agent/runtime-contracts.ts @@ -156,8 +156,14 @@ export interface RunTurnOptions { * keep-alive turn. */ approvalParkMode?: boolean; - /** A live approval resume: answer the parked gate and stream the continued prompt's events. */ - resume?: ResumeApprovalInput; + /** + * A live approval resume: answer every parked gate of the turn (keyed by tool-call id) and + * stream the continued prompt's events. A turn can hold more than one parked gate, so the + * resume carries a LIST — the server builds it from the inbound approval replies, one decision + * per parked gate. All decisions share the one held prompt promise (there is one prompt per + * turn); `runTurn` answers each gate then awaits that single continuation. + */ + resume?: { decisions: ResumeApprovalInput[] }; } /** @@ -222,17 +228,31 @@ export interface SessionEnvironment { */ lastTurnToolCallIds: string[]; /** - * The Claude ACP permission gate the LAST turn paused on, or undefined. Set only for a harness - * ACP permission gate, reset at each turn start; the dispatch reads it after a paused turn to - * decide whether to park in `awaiting_approval` and, on the next request, how to resume. + * Every parkable ACP permission gate the LAST turn paused on, keyed by the gated tool-call id + * (reset at each turn start). This is the source of truth the warm resume iterates: a turn can + * hold more than one gate (parallel gated tool calls), and each is answered by its own + * `permissionId` on the live session. Empty when no parkable gate paused the turn. + */ + parkedApprovals: Map; + /** + * The FIRST parked gate this turn, a convenience for per-turn-uniform reads (logging, the + * gate-type check, the shared history/credential validation). Undefined when the map is empty. + * The multi-answer resume and the all-parkable park check read `parkedApprovals`, not this. */ parkedApproval?: ParkedApproval; /** - * How many Claude ACP permission gates resolved to pendingApproval THIS turn (reset at turn - * start). More than one means parallel gates the single-gate resume cannot answer, so the - * dispatch does not park (tears down cold as today). + * How many ACP permission gates resolved to pendingApproval THIS turn (reset at turn start). + * Equals `parkedApprovals.size` when every gate carried a resumable tool-call id; a larger + * count means a gate lacked an id and cannot be resumed live, so the dispatch stays cold. */ approvalGateCount: number; + /** + * How many NON-parkable pauses happened this turn (a client-tool ACP gate or a browser-fulfilled + * relay/MCP client tool), reset at turn start. Non-zero means the turn mixes an unanswerable + * client-tool pause into the set, so the whole turn stays on the cold path (only cold can + * multiplex a mixed set today). + */ + nonParkablePauseCount: number; destroyed: boolean; /** Complete, idempotent teardown selected from the typed teardown reason. */ destroy: (opts?: { reason?: TeardownReason }) => Promise; diff --git a/services/runner/src/engines/sandbox_agent/runtime-policy.ts b/services/runner/src/engines/sandbox_agent/runtime-policy.ts index 8df0af65be..bc3c96cc90 100644 --- a/services/runner/src/engines/sandbox_agent/runtime-policy.ts +++ b/services/runner/src/engines/sandbox_agent/runtime-policy.ts @@ -39,7 +39,24 @@ export function shouldSuppressPausedToolCallUpdate( if (kind !== "tool_call" && kind !== "tool_call_update") return false; const toolCallId = typeof frame?.toolCallId === "string" ? frame.toolCallId : undefined; - return pause.isPausedToolCall(toolCallId); + // F-024: a paused (gated) tool call's later frames are teardown artifacts and never reach the + // stream. + if (pause.isPausedToolCall(toolCallId)) return true; + // Once the turn is pausing, a FAILED update for any other open call is a managed-cancel + // artifact (a sibling that will not run this turn), so suppress it and let the deterministic + // post-drain sweep settle that call as TOOL_NOT_EXECUTED_PAUSED — the human sees "paused", not a + // spurious cancellation error. A `completed` update is NOT suppressed: an auto-allowed sibling + // that legitimately finishes while another gate holds the (kept-alive) warm session must show + // its real result, not be overwritten as paused. Announcements (`tool_call`) are never + // suppressed: the call must be recorded so the sweep knows which calls to settle. (Removing the + // eager first-pause settle is what lets concurrent gates each mark their own call paused before + // the sweep runs; this narrow suppression only masks the cancel artifact.) Residual: a genuine + // mid-pause tool error also arrives as `failed` and reads as paused here — acceptable, since the + // paused result invites a retry and the runner has no adapter provenance to tell the two apart. + if (kind === "tool_call_update" && pause.active && frame?.status === "failed") { + return true; + } + return false; } const CLAUDE_STRICT_DEPLOYMENTS = new Set([ diff --git a/services/runner/src/permission-plan.ts b/services/runner/src/permission-plan.ts index 012cda183e..3ebc43ce0b 100644 --- a/services/runner/src/permission-plan.ts +++ b/services/runner/src/permission-plan.ts @@ -170,20 +170,6 @@ export function storedDecisionKeyShape( }; } -export class PendingApprovalLatch { - private acquired = false; - - tryAcquire(): boolean { - if (this.acquired) return false; - this.acquired = true; - return true; - } - - get held(): boolean { - return this.acquired; - } -} - function normalizeRules(rawRules: unknown): PermissionPlan["rules"] { if (!Array.isArray(rawRules)) return []; const rules: PermissionPlan["rules"] = []; diff --git a/services/runner/src/server.ts b/services/runner/src/server.ts index 235f730559..9fb8c00fee 100644 --- a/services/runner/src/server.ts +++ b/services/runner/src/server.ts @@ -37,6 +37,7 @@ import { runSandboxAgent, runTurn, shouldPark, + type ResumeApprovalInput, type RunTurnOptions, type SessionEnvironment, } from "./engines/sandbox_agent.ts"; @@ -412,21 +413,34 @@ export async function runWithKeepalive( } }; - // Whether a paused turn holds a single, parkable permission gate (a Claude ACP gate or a Pi - // ACP gate). Only such a gate carries a `respondPermission`-answerable id; a client-tool MCP - // pause never records `parkedApproval`, and more than one pending gate cannot be answered by - // the single-gate resume — both stay on the cold path, logged. + // Whether a paused turn is fully parkable: every pending gate is a parkable ACP permission gate + // (Claude ACP or Pi ACP) that carries a `respondPermission`-answerable id, so the warm resume + // can answer them all. A turn parks when it has at least one parked gate AND every pending gate + // is parkable: no client-tool MCP pause (unanswerable across a turn boundary), and no approval + // gate that lacked a resumable id. A mixed or partly-unresumable set stays on the cold path, + // which is the only path that can multiplex it. Any count of parkable gates parks — a turn with + // several parallel gates is answered by several `respondPermission` calls on the resume. const approvalToPark = ( env: SessionEnvironment, result: AgentRunResult, ): boolean => { if (result.stopReason !== "paused") return false; - if (!env.parkedApproval) { + if (env.parkedApprovals.size === 0) { klog(`non-parkable-gate-no-park key=${key}`); return false; } - if ((env.approvalGateCount ?? 0) > 1) { - klog(`multi-gate-no-park key=${key} gates=${env.approvalGateCount}`); + if ((env.nonParkablePauseCount ?? 0) > 0) { + klog( + `mixed-gate-no-park key=${key} parkable=${env.parkedApprovals.size} ` + + `nonParkable=${env.nonParkablePauseCount}`, + ); + return false; + } + if (env.parkedApprovals.size !== (env.approvalGateCount ?? 0)) { + klog( + `unresumable-gate-no-park key=${key} parkable=${env.parkedApprovals.size} ` + + `gates=${env.approvalGateCount}`, + ); return false; } // An approval park waits for the HUMAN, who is still on the page even if the streaming client @@ -445,15 +459,22 @@ export async function runWithKeepalive( // one destroy, so no double-destroy is possible. The promise already carries runTurn's // swallowing catch, so no unhandled rejection is introduced. const watchParkedPrompt = (env: SessionEnvironment): void => { - const promptPromise = env.parkedApproval?.promptPromise; const entry = pool.get(key); - if (!promptPromise || !entry || entry.environment !== env) return; - promptPromise.catch(() => { - const current = pool.get(key); - if (current !== entry || current.state !== "awaiting_approval") return; - klog(`parked-prompt-rejected key=${key}; evict`); - void pool.evict(key, "parked-prompt-rejected", "failed-turn"); - }); + if (!entry || entry.environment !== env) return; + // Watch the WHOLE parked set: a turn is pending until every gate resolves, so a rejection of + // ANY parked prompt means the harness or sandbox died mid-park and the dead session must be + // evicted. Every parked gate shares the one turn prompt promise, so the catches are on the + // same promise; the eviction is identity-checked and idempotent, so repeated catches are safe. + for (const parked of env.parkedApprovals.values()) { + const promptPromise = parked.promptPromise; + if (!promptPromise) continue; + promptPromise.catch(() => { + const current = pool.get(key); + if (current !== entry || current.state !== "awaiting_approval") return; + klog(`parked-prompt-rejected key=${key}; evict`); + void pool.evict(key, "parked-prompt-rejected", "failed-turn"); + }); + } }; // Park a freshly cold-acquired environment (new pool slot) as approval / idle, or tear it down. @@ -657,29 +678,56 @@ export async function runWithKeepalive( // history fingerprint (an edited transcript must not continue wrongly), and a hard mount-expiry // bound — if the parked session's mount credentials are past expiry, its durable cwd can no // longer be written, so evict to cold. + // The turn may hold several parked gates (parallel gated tool calls). Build one resume + // decision per parked gate, keyed by tool-call id. The whole turn resumes live ONLY when the + // request answers EVERY parked gate — the frontend's all-settled rule guarantees a resume /run + // is sent only after the human answered all cards, so a partial set is a mismatch that + // degrades to cold (which re-raises and multiplexes whatever subset). `parked` (the first + // gate) is the representative for logging and the per-turn-uniform checks. const parked = existing.environment.parkedApproval; - const decision = parked - ? approvalDecisionForToolCall(request, parked.toolCallId) - : undefined; - const priorFp = historyFingerprint(priorConversation(request)); + const parkedList = [...existing.environment.parkedApprovals.values()]; + const resumeDecisions: ResumeApprovalInput[] = []; let mismatch: string | undefined; - if ( - !parked || - (parked.gateType !== "claude-acp-permission" && - parked.gateType !== "pi-acp-permission") - ) { - // Defensive: only a parkable gate type (Claude ACP or Pi ACP) ever parks here. Both - // resume via `respondPermission` on the live session; the daemon maps the reply by kind. - mismatch = "unrecognized-gate-type"; - } else if (!decision) { - mismatch = "no-matching-approval"; // fresh user text, or an approval for another id - } else if (priorFp !== existing.historyFingerprint) { - mismatch = "history"; - } else if (mountCredentialsExpired(existing.credentialEpoch)) { - mismatch = "credentials-expired"; + if (parkedList.length === 0) { + mismatch = "no-parked-gate"; + } else { + for (const gate of parkedList) { + if ( + gate.gateType !== "claude-acp-permission" && + gate.gateType !== "pi-acp-permission" + ) { + // Defensive: only a parkable gate type (Claude ACP or Pi ACP) ever parks here. Both + // resume via `respondPermission` on the live session; the daemon maps the reply by kind. + mismatch = "unrecognized-gate-type"; + break; + } + const decision = approvalDecisionForToolCall(request, gate.toolCallId); + if (!decision) { + // A gate with no matching reply: fresh user text, or the human answered only some cards. + mismatch = "no-matching-approval"; + break; + } + resumeDecisions.push({ + permissionId: gate.permissionId, + reply: decision === "allow" ? "once" : "reject", + toolCallId: gate.toolCallId, + toolName: gate.toolName, + args: gate.args, + interactionToken: gate.interactionToken, + promptPromise: gate.promptPromise, + }); + } + } + const priorFp = historyFingerprint(priorConversation(request)); + if (!mismatch) { + if (priorFp !== existing.historyFingerprint) { + mismatch = "history"; + } else if (mountCredentialsExpired(existing.credentialEpoch)) { + mismatch = "credentials-expired"; + } } - if (mismatch || !parked || !decision) { + if (mismatch || resumeDecisions.length === 0) { klog( `approval-mismatch (${mismatch ?? "unknown"}) key=${key}; evict + cold`, ); @@ -693,10 +741,13 @@ export async function runWithKeepalive( const live = pool.checkoutApproval(key); if (live) { - const reply = decision === "allow" ? "once" : "reject"; + const approveCount = resumeDecisions.filter( + (d) => d.reply === "once", + ).length; + const rejectCount = resumeDecisions.length - approveCount; klog( - `${reply === "once" ? "resume-approve" : "resume-reject"} key=${key} ` + - `tool=${parked.toolName ?? "?"}`, + `resume key=${key} gates=${resumeDecisions.length} ` + + `approve=${approveCount} reject=${rejectCount} tool=${parked?.toolName ?? "?"}`, ); let result: AgentRunResult; try { @@ -710,15 +761,7 @@ export async function runWithKeepalive( signal, { approvalParkMode: true, - resume: { - permissionId: parked.permissionId, - reply, - toolCallId: parked.toolCallId, - toolName: parked.toolName, - args: parked.args, - interactionToken: parked.interactionToken, - promptPromise: parked.promptPromise, - }, + resume: { decisions: resumeDecisions }, }, ); } catch (err) { @@ -803,18 +846,29 @@ const runAgent: RunAgent = (request, emit, signal, options) => { * resolves the token after consuming the decision. Swept first, the granted gate's record * lands as `cancelled` and the resolve 404s. */ -function inBandAnswerToken(request: AgentRunRequest): string | undefined { +function inBandAnswerTokens(request: AgentRunRequest): string[] | undefined { const sessionId = request.sessionId?.trim(); if (!sessionId) return undefined; const provider = resolveKeepaliveDispatch(request, keepaliveConfigs); if (!provider) return undefined; const parked = keepalivePools[provider].awaitingApproval(sessionId)?.environment - .parkedApproval; - if (!parked) return undefined; - return approvalDecisionForToolCall(request, parked.toolCallId) !== undefined - ? parked.interactionToken - : undefined; + .parkedApprovals; + if (!parked || parked.size === 0) return undefined; + // A turn can park several gates. Spare tokens from the stale sweep ONLY when this request + // answers EVERY parked gate — that is exactly when the dispatch resumes live and resolves them. + // If any gate is unanswered the turn degrades to cold (the all-or-cold resume rule), and sparing + // an answered subset would strand those interaction rows as pending forever (no live resume ever + // resolves them). In that case spare nothing: the cold path re-raises and the sweep correctly + // cancels the old rows. + const tokens: string[] = []; + for (const gate of parked.values()) { + if (approvalDecisionForToolCall(request, gate.toolCallId) === undefined) { + return undefined; + } + tokens.push(gate.interactionToken); + } + return tokens.length > 0 ? tokens : undefined; } /** @@ -926,11 +980,11 @@ async function runAndStreamWithApiBaseResolved( // A new turn supersedes any prior turn's unanswered gate: cancel stale pending // interactions (sparing this turn's own, plus a parked gate this turn answers in-band — // the resume resolves that one). Best-effort, never blocks the turn. - const answeredToken = inBandAnswerToken(request); + const answeredTokens = inBandAnswerTokens(request); void cancelStaleInteractions( sessionId, turnId, - answeredToken ? [answeredToken] : undefined, + answeredTokens, watchdog.credential, ); // Deny-set from THIS run's resolved provider keys + run credential (not process env, diff --git a/services/runner/tests/unit/client-tools.test.ts b/services/runner/tests/unit/client-tools.test.ts index 7902c47d88..b4af378b5f 100644 --- a/services/runner/tests/unit/client-tools.test.ts +++ b/services/runner/tests/unit/client-tools.test.ts @@ -18,7 +18,6 @@ import { extractClientToolOutputs, } from "../../src/responder.ts"; import type { ClientToolRelayRequest } from "../../src/tools/client-tool-relay.ts"; -import { PendingApprovalLatch } from "../../src/permission-plan.ts"; import { buildClientToolRelay, createToolCallCorrelationIndex, @@ -47,7 +46,7 @@ function responderReturning(verdict: ClientToolVerdict): Responder { }; } -/** A seam harness: fake pause controller + latch + captured events/interactions. */ +/** A seam harness: fake pause controller + captured events/interactions. */ function seam(verdict: ClientToolVerdict, opts: { index?: boolean } = {}) { const events: AgentEvent[] = []; const pausedToolCalls: string[] = []; @@ -57,7 +56,6 @@ function seam(verdict: ClientToolVerdict, opts: { index?: boolean } = {}) { const relay = buildClientToolRelay({ responder: responderReturning(verdict), run: { emitEvent: (e) => events.push(e) }, - latch: new PendingApprovalLatch(), pause: { markPausedToolCall: (id) => pausedToolCalls.push(id), pause: () => { @@ -327,7 +325,6 @@ describe("buildClientToolRelay", () => { }, }, run: { emitEvent: () => {} }, - latch: new PendingApprovalLatch(), pause: { markPausedToolCall: () => {}, pause: () => {} }, recordPendingInteraction: () => {}, }); @@ -393,7 +390,6 @@ describe("buildClientToolRelay with the real responder (history-driven)", () => const relay = buildClientToolRelay({ responder, run: { emitEvent: (e) => events.push(e) }, - latch: new PendingApprovalLatch(), pause: { markPausedToolCall: (id) => paused.push(id), pause: () => {} }, recordPendingInteraction: () => {}, }); diff --git a/services/runner/tests/unit/permission-plan.test.ts b/services/runner/tests/unit/permission-plan.test.ts index e327a606a6..4e193ac3aa 100644 --- a/services/runner/tests/unit/permission-plan.test.ts +++ b/services/runner/tests/unit/permission-plan.test.ts @@ -2,7 +2,6 @@ import { afterEach, beforeEach, describe, it } from "vitest"; import assert from "node:assert/strict"; import { - PendingApprovalLatch, decide, effectivePermission, permissionsFromRequest, @@ -291,17 +290,6 @@ describe("permissionsFromRequest", () => { }); }); -describe("PendingApprovalLatch", () => { - it("lets only the first caller acquire", () => { - const latch = new PendingApprovalLatch(); - assert.equal(latch.held, false); - assert.equal(latch.tryAcquire(), true); - assert.equal(latch.held, true); - assert.equal(latch.tryAcquire(), false); - assert.equal(latch.held, true); - }); -}); - describe("stored decisions", () => { it("consults stored decisions only under ask", () => { const stored: StoredPermissionDecisions = { diff --git a/services/runner/tests/unit/sandbox-agent-orchestration.test.ts b/services/runner/tests/unit/sandbox-agent-orchestration.test.ts index b7cce0d9ab..1873c4f5ee 100644 --- a/services/runner/tests/unit/sandbox-agent-orchestration.test.ts +++ b/services/runner/tests/unit/sandbox-agent-orchestration.test.ts @@ -1750,7 +1750,7 @@ describe("runSandboxAgent default ApprovalResponder wiring", () => { ); }); - it("settles a latch-loser sibling when read-only permission requests race", async () => { + it("emits a card for EACH of two racing ask gates and force-settles neither (cold-path plural approvals)", async () => { const readOnlySpec = (name: string) => ({ name, kind: "callback", @@ -1828,31 +1828,29 @@ describe("runSandboxAgent default ApprovalResponder wiring", () => { if (!result.ok) return; assert.equal(result.stopReason, "paused"); + // No latch: BOTH gated tool calls emit their own approval card in the one paused turn, each + // keyed by its own tool-call id. This is exactly issue #5373's fix — the human now sees every + // card at once instead of one, with the rest re-asked across later turns. const interactions = result.events?.filter( (event) => event.type === "interaction_request", ); - assert.equal(interactions?.length, 1); - assert.equal((interactions?.[0] as any).payload?.toolCallId, "tool-a"); + assert.equal(interactions?.length, 2); + assert.deepEqual( + interactions?.map((e) => (e as any).payload?.toolCallId).sort(), + ["tool-a", "tool-b"], + ); - const toolResults = result.events - ?.filter((event) => event.type === "tool_result") - .map((event) => ({ - id: (event as any).id, - output: (event as any).output, - isError: (event as any).isError, - })); - assert.deepEqual(toolResults, [ - { - id: "tool-b", - output: TOOL_NOT_EXECUTED_PAUSED, - isError: true, - }, - ]); + // Neither gate is force-settled as TOOL_NOT_EXECUTED_PAUSED: both are held open for their own + // approval, so no orphaned paused tool_result is emitted. + const toolResults = result.events?.filter( + (event) => event.type === "tool_result", + ); + assert.deepEqual(toolResults, []); }); it("settles a sibling whose announcement arrives AFTER the pause (teardown race)", async () => { // The live incident shape: the sibling's `tool_call` announcement rides the ACP event - // stream and can arrive at the runner AFTER the gate won the latch and the pause-time + // stream and can arrive at the runner AFTER the gate paused the turn and the pause-time // sweep already ran. The event-handler re-sweep must settle it deterministically. const { deps } = fakeHarness({ promptEvents: [ diff --git a/services/runner/tests/unit/session-keepalive-approval.test.ts b/services/runner/tests/unit/session-keepalive-approval.test.ts index d0db503bbc..f743e06739 100644 --- a/services/runner/tests/unit/session-keepalive-approval.test.ts +++ b/services/runner/tests/unit/session-keepalive-approval.test.ts @@ -50,13 +50,25 @@ const auth = { // ---------------------------------------------------------------------------- // interface TurnScript { - /** The turn pauses on a single parkable permission gate (records parkedApproval). */ + /** The turn pauses on one or more parkable permission gates (records parkedApprovals). */ approvalPause?: { permissionId: string; toolCallId: string; toolName?: string; - /** How many gates pended (>1 = multi-gate; still records the first). Default 1. */ + /** Additional parkable gates in the same turn (parallel gated tool calls). */ + extraGates?: Array<{ + permissionId: string; + toolCallId: string; + toolName?: string; + }>; + /** + * Override `approvalGateCount`. Defaults to the number of recorded gates. Set LARGER than the + * recorded gates to model a gate that lacked a resumable id (count > map size -> unresumable, + * no park). + */ gates?: number; + /** Also flag a non-parkable (client-tool) pause this turn, so the mixed set stays cold. */ + nonParkable?: boolean; /** The parked gate plane; default the Claude ACP gate. */ gateType?: ParkedApproval["gateType"]; }; @@ -75,8 +87,10 @@ interface DispatchFakeEnv { destroyed: number; turnsCleared: number; lastTurnToolCallIds: string[]; + parkedApprovals: Map; parkedApproval?: ParkedApproval; approvalGateCount: number; + nonParkablePauseCount: number; clearTurn: () => void; destroyImpl: () => Promise; destroy: () => Promise; @@ -112,8 +126,10 @@ function makeApprovalEngine( destroyed: 0, turnsCleared: 0, lastTurnToolCallIds: [], + parkedApprovals: new Map(), parkedApproval: undefined, approvalGateCount: 0, + nonParkablePauseCount: 0, clearTurn: () => { env.turnsCleared += 1; }, @@ -142,38 +158,61 @@ function makeApprovalEngine( const idx = calls.turns.length; const script = scripts[idx] ?? {}; // Mirror the real runTurn per-turn reset. + env.parkedApprovals = new Map(); env.parkedApproval = undefined; env.approvalGateCount = 0; + env.nonParkablePauseCount = 0; env.lastTurnToolCallIds = script.toolCallIds ?? []; calls.turns.push({ env, opts, idx }); if (opts?.resume) { - calls.resumes.push({ - permissionId: opts.resume.permissionId, - reply: opts.resume.reply, - toolCallId: opts.resume.toolCallId, - }); + // The warm resume carries a LIST of decisions (one per parked gate). + for (const decision of opts.resume.decisions) { + calls.resumes.push({ + permissionId: decision.permissionId, + reply: decision.reply, + toolCallId: decision.toolCallId, + }); + } } if (script.hold) { await new Promise((resolve) => holds.set(idx, resolve)); } if (script.approvalPause) { - env.approvalGateCount = script.approvalPause.gates ?? 1; + const gateType = + script.approvalPause.gateType ?? "claude-acp-permission"; + const parkableGates = [ + { + permissionId: script.approvalPause.permissionId, + toolCallId: script.approvalPause.toolCallId, + toolName: script.approvalPause.toolName, + }, + ...(script.approvalPause.extraGates ?? []), + ]; + // approvalGateCount defaults to the recorded-gate count; a larger override models a gate + // that lacked a resumable id (count > map size -> the dispatch treats the set as unresumable). + env.approvalGateCount = script.approvalPause.gates ?? parkableGates.length; + env.nonParkablePauseCount = script.approvalPause.nonParkable ? 1 : 0; // The held original prompt: pending until the test settles it (mirrors the real Claude - // prompt that never resolves on an unanswered gate). Carries the same swallowing catch - // runTurn attaches, so a test-driven rejection is never unhandled. + // prompt that never resolves on an unanswered gate). One prompt per turn, shared by every + // parked gate. Carries the same swallowing catch runTurn attaches, so a test-driven + // rejection is never unhandled. const promptPromise = new Promise((resolve, reject) => { calls.promptControls.push({ resolve, reject }); }); promptPromise.catch(() => {}); - env.parkedApproval = { - gateType: script.approvalPause.gateType ?? "claude-acp-permission", - permissionId: script.approvalPause.permissionId, - toolCallId: script.approvalPause.toolCallId, - toolName: script.approvalPause.toolName, - args: {}, - interactionToken: script.approvalPause.toolCallId, - promptPromise, - }; + for (const gate of parkableGates) { + const record: ParkedApproval = { + gateType, + permissionId: gate.permissionId, + toolCallId: gate.toolCallId, + toolName: gate.toolName, + args: {}, + interactionToken: gate.toolCallId, + promptPromise, + }; + env.parkedApprovals.set(gate.toolCallId, record); + env.parkedApproval ??= record; + } return { ok: true, stopReason: "paused" }; } if (script.nonClaudePause) return { ok: true, stopReason: "paused" }; @@ -268,6 +307,38 @@ function approveResume( }; } +/** A resume that answers several parked gates at once: one tool_call + one {approved} envelope + * per gate, exactly as the frontend sends once every card is answered. */ +function approveResumeMulti( + gates: Array<{ toolCallId: string; toolName?: string; approved: boolean }>, +): AgentRunRequest { + return { + harness: "claude", + model: "m1", + sessionId: "s1", + ...auth, + messages: [ + { role: "user", content: "do X" }, + { + role: "assistant", + content: gates.map((g) => ({ + type: "tool_call", + toolCallId: g.toolCallId, + toolName: g.toolName ?? "commit", + })), + }, + { + role: "user", + content: gates.map((g) => ({ + type: "tool_result", + toolCallId: g.toolCallId, + output: { approved: g.approved }, + })), + }, + ], + }; +} + function captureStderr() { const lines: string[] = []; const orig = process.stderr.write.bind(process.stderr); @@ -416,11 +487,126 @@ describe("runWithKeepalive: approval park + resume", () => { await runWithKeepalive(pauseTurn(), undefined, undefined, ctx); await runWithKeepalive(approveResume(true), undefined, undefined, ctx); assert.ok(cap.lines.some((l) => l.includes("park-approval"))); - assert.ok(cap.lines.some((l) => l.includes("resume-approve"))); + assert.ok( + cap.lines.some((l) => l.includes("resume ") && l.includes("approve=1")), + ); } finally { cap.restore(); } }); + + it("parks a two-gate turn and answers BOTH gates live on one resume", async () => { + const { engine, calls } = makeApprovalEngine([ + { + approvalPause: { + permissionId: "perm-1", + toolCallId: "tc-1", + toolName: "read_a", + extraGates: [ + { permissionId: "perm-2", toolCallId: "tc-2", toolName: "read_b" }, + ], + }, + toolCallIds: ["tc-1", "tc-2"], + }, + ]); + const ctx = makeCtx(engine); + + const r1 = await runWithKeepalive(pauseTurn(), undefined, undefined, ctx); + assert.equal(r1.stopReason, "paused"); + assert.equal( + ctx.pool.get(POOL_KEY)!.state, + "awaiting_approval", + "a two-gate turn parks (no longer forced cold)", + ); + + const r2 = await runWithKeepalive( + approveResumeMulti([ + { toolCallId: "tc-1", toolName: "read_a", approved: true }, + { toolCallId: "tc-2", toolName: "read_b", approved: true }, + ]), + undefined, + undefined, + ctx, + ); + assert.equal(r2.ok, true); + assert.equal(calls.acquire, 1, "the resume did NOT re-acquire cold"); + assert.equal( + calls.resumes.length, + 2, + "respondPermission is called once per parked gate", + ); + assert.deepEqual( + calls.resumes.map((r) => `${r.toolCallId}:${r.reply}`).sort(), + ["tc-1:once", "tc-2:once"], + ); + }); + + it("resumes a two-gate turn with deny-one-approve-one, each gate answered on its own id", async () => { + const { engine, calls } = makeApprovalEngine([ + { + approvalPause: { + permissionId: "perm-1", + toolCallId: "tc-1", + toolName: "read_a", + extraGates: [ + { permissionId: "perm-2", toolCallId: "tc-2", toolName: "read_b" }, + ], + }, + toolCallIds: ["tc-1", "tc-2"], + }, + ]); + const ctx = makeCtx(engine); + await runWithKeepalive(pauseTurn(), undefined, undefined, ctx); + + const r2 = await runWithKeepalive( + approveResumeMulti([ + { toolCallId: "tc-1", toolName: "read_a", approved: false }, + { toolCallId: "tc-2", toolName: "read_b", approved: true }, + ]), + undefined, + undefined, + ctx, + ); + assert.equal(r2.ok, true); + assert.equal(calls.resumes.length, 2); + assert.deepEqual( + calls.resumes.map((r) => `${r.toolCallId}:${r.reply}`).sort(), + ["tc-1:reject", "tc-2:once"], + "the denied gate rejects and the approved gate runs, each by its own id", + ); + }); + + it("keeps a partly-answered two-gate turn paused (only one card answered -> cold)", async () => { + const { engine, calls } = makeApprovalEngine([ + { + approvalPause: { + permissionId: "perm-1", + toolCallId: "tc-1", + toolName: "read_a", + extraGates: [ + { permissionId: "perm-2", toolCallId: "tc-2", toolName: "read_b" }, + ], + }, + toolCallIds: ["tc-1", "tc-2"], + }, + ]); + const ctx = makeCtx(engine); + await runWithKeepalive(pauseTurn(), undefined, undefined, ctx); + + // Only tc-1 answered; tc-2 still pending. The whole-set requirement is not met, so the resume + // does not answer any gate live and degrades to cold. + const r2 = await runWithKeepalive( + approveResumeMulti([ + { toolCallId: "tc-1", toolName: "read_a", approved: true }, + ]), + undefined, + undefined, + ctx, + ); + assert.equal(r2.ok, true); + assert.equal(calls.resumes.length, 0, "no gate answered live"); + assert.equal(calls.acquire, 2, "the partial answer degraded to cold (re-acquired)"); + }); }); describe("runWithKeepalive: never-park gate types stay cold", () => { @@ -443,9 +629,11 @@ describe("runWithKeepalive: never-park gate types stay cold", () => { } }); - it("a multi-gate pause (parallel gates) never parks, tears down cold", async () => { + it("a turn with an unresumable gate (pending count exceeds recorded gates) stays cold", async () => { const cap = captureStderr(); try { + // Two gates pended but only one carried a resumable id (map size 1, count 2): the set is not + // fully resumable, so the whole turn falls to the cold path. const { engine, calls } = makeApprovalEngine([ { approvalPause: { @@ -463,10 +651,37 @@ describe("runWithKeepalive: never-park gate types stay cold", () => { assert.equal( calls.acquiredEnvs[0].destroyed, 1, - "the single-gate resume cannot answer >1 gate -> cold", + "a gate that cannot be resumed live -> cold", ); assert.equal(ctx.pool.size(), 0); - assert.ok(cap.lines.some((l) => l.includes("multi-gate-no-park"))); + assert.ok(cap.lines.some((l) => l.includes("unresumable-gate-no-park"))); + } finally { + cap.restore(); + } + }); + + it("a mixed set (an approval gate plus a client-tool pause) stays cold", async () => { + const cap = captureStderr(); + try { + // One parkable approval gate AND one non-parkable client-tool pause in the same turn: only + // the cold path can multiplex the mixed set, so the whole turn stays cold. + const { engine, calls } = makeApprovalEngine([ + { + approvalPause: { + permissionId: "perm-1", + toolCallId: "tc-gate", + toolName: "commit", + nonParkable: true, + }, + toolCallIds: ["tc-gate"], + }, + ]); + const ctx = makeCtx(engine); + const r = await runWithKeepalive(pauseTurn(), undefined, undefined, ctx); + assert.equal(r.stopReason, "paused"); + assert.equal(calls.acquiredEnvs[0].destroyed, 1, "mixed set -> cold"); + assert.equal(ctx.pool.size(), 0); + assert.ok(cap.lines.some((l) => l.includes("mixed-gate-no-park"))); } finally { cap.restore(); } @@ -1098,13 +1313,17 @@ describe("runTurn: real approval park + respondPermission resume", () => { const p2 = runTurn(env, approveResume(true), undefined, undefined, { approvalParkMode: true, resume: { - permissionId: "perm-1", - reply: "once", - toolCallId: "tc-gate", - toolName: "commit", - args: { message: "hi" }, - interactionToken: "tc-gate", - promptPromise: held, + decisions: [ + { + permissionId: "perm-1", + reply: "once", + toolCallId: "tc-gate", + toolName: "commit", + args: { message: "hi" }, + interactionToken: "tc-gate", + promptPromise: held, + }, + ], }, }); await flush(); @@ -1191,13 +1410,17 @@ describe("runTurn: real approval park + respondPermission resume", () => { const p2 = runTurn(env, approveResume(false), undefined, undefined, { approvalParkMode: true, resume: { - permissionId: "perm-1", - reply: "reject", - toolCallId: "tc-gate", - toolName: "commit", - args: {}, - interactionToken: "tc-gate", - promptPromise: held, + decisions: [ + { + permissionId: "perm-1", + reply: "reject", + toolCallId: "tc-gate", + toolName: "commit", + args: {}, + interactionToken: "tc-gate", + promptPromise: held, + }, + ], }, }); await flush(); @@ -1263,8 +1486,9 @@ describe("runTurn: real approval park + respondPermission resume", () => { approvalParkMode: true, }); await flush(); - // A latch-loser sibling tool call announced BEFORE the winning gate: it can never execute - // this turn and must be settled with the deterministic paused result, park or no park. + // An announced-but-UNGATED sibling tool call (it never raises a permission request, so it gets + // no card): it can never execute once the turn pauses on the gate, so it must be settled with + // the deterministic paused result, park or no park. captured.onEvent!( updateEvent({ sessionUpdate: "tool_call", @@ -1310,7 +1534,7 @@ describe("runTurn: real approval park + respondPermission resume", () => { await env.destroy(); }); - it("the sibling settle also covers the multi-gate case the dispatch then destroys", async () => { + it("two parallel gates each emit a card and BOTH park (neither is force-settled)", async () => { const { calls, deps, captured } = pausableHarness(); const acquired = await acquireEnvironment(engineReq, deps); assert.equal(acquired.ok, true); @@ -1321,9 +1545,9 @@ describe("runTurn: real approval park + respondPermission resume", () => { approvalParkMode: true, }); await flush(); - // Two parallel gates: gate 1 wins the latch and pauses; at pause time gate 2's call is an - // open non-paused sibling, so the UNCONDITIONAL settle covers it even though parkedApproval - // is already set (the exact case an early-return-before-settle would orphan). + // Two parallel gated tool calls in one turn. With no latch, each raises its own permission + // request, emits its own card, and is marked paused — so NEITHER is force-settled, and BOTH + // are recorded in parkedApprovals for the live resume. captured.onEvent!( updateEvent({ sessionUpdate: "tool_call", @@ -1353,18 +1577,31 @@ describe("runTurn: real approval park + respondPermission resume", () => { assert.equal(r1.stopReason, "paused"); assert.equal(env.approvalGateCount, 2, "both pending gates were counted"); + assert.equal(env.parkedApprovals.size, 2, "both gates were parked"); + assert.deepEqual( + [...env.parkedApprovals.keys()].sort(), + ["tc-g1", "tc-g2"], + "each gate is keyed by its own tool-call id", + ); + assert.equal( + env.nonParkablePauseCount, + 0, + "no non-parkable pause -> the dispatch parks the whole set", + ); const run1 = calls.runs[0]; assert.deepEqual( run1.settled, - [{ id: "tc-g2", message: TOOL_NOT_EXECUTED_PAUSED }], - "the second gate's call settled as a sibling at pause time", + [], + "neither gate is force-settled: both got a card and are held for the resume", + ); + assert.ok(run1.open.has("tc-g1"), "the first gate's call stays open"); + assert.ok(run1.open.has("tc-g2"), "the second gate's call stays open"); + assert.equal( + calls.sessionDestroyed, + 0, + "the multi-gate park keeps the session alive", ); - assert.ok(run1.open.has("tc-g1"), "the winning gate's call stays open"); - // The dispatch refuses to park a multi-gate pause and destroys: the teardown the park - // skipped (destroySession, sandbox teardown) runs through env.destroy(). await env.destroy(); - assert.equal(calls.sessionDestroyed, 1, "destroy tore the session down"); - assert.equal(calls.sandboxDestroyed, 1); }); }); diff --git a/services/runner/tests/unit/session-keepalive-dispatch.test.ts b/services/runner/tests/unit/session-keepalive-dispatch.test.ts index 74c40f4f40..659b36f003 100644 --- a/services/runner/tests/unit/session-keepalive-dispatch.test.ts +++ b/services/runner/tests/unit/session-keepalive-dispatch.test.ts @@ -51,6 +51,9 @@ interface FakeEnv { destroyed: number; turnsCleared: number; lastTurnToolCallIds: string[]; + parkedApprovals: Map; + approvalGateCount: number; + nonParkablePauseCount: number; clearTurn: () => void; /** Replaceable body so a test can slow the teardown down; `destroy` stays a stable closure. */ destroyImpl: () => Promise; @@ -78,6 +81,9 @@ function makeEngine(options: EngineOptions = {}) { destroyed: 0, turnsCleared: 0, lastTurnToolCallIds: [], + parkedApprovals: new Map(), + approvalGateCount: 0, + nonParkablePauseCount: 0, clearTurn: () => { env.turnsCleared += 1; }, diff --git a/web/packages/agenta-playground/tests/unit/agentApprovalResume.test.ts b/web/packages/agenta-playground/tests/unit/agentApprovalResume.test.ts index 8d1455eeaf..4ccc115c73 100644 --- a/web/packages/agenta-playground/tests/unit/agentApprovalResume.test.ts +++ b/web/packages/agenta-playground/tests/unit/agentApprovalResume.test.ts @@ -128,6 +128,67 @@ describe("agentShouldResumeAfterApproval", () => { expect(agentShouldResumeAfterApproval({messages})).toBe(true) }) + it("does NOT resume while ONE of two concurrent approval cards is still pending", () => { + // Concurrent approvals: one turn shows TWO approval-requested gates (two distinct + // toolCallIds). Answering only the first must NOT resume — the second is still pending, + // so the run stays parked until EVERY card is settled. + const messages = [ + user("do two"), + { + id: "a1", + role: "assistant", + parts: [ + {type: "step-start"}, + { + type: "tool-deleteFile", + toolCallId: "call_1", + state: "approval-responded", + input: {path: "/x"}, + approval: {id: "perm_1", approved: true}, + }, + { + type: "tool-deleteFile", + toolCallId: "call_2", + state: "approval-requested", + input: {path: "/y"}, + approval: {id: "perm_2"}, + }, + ], + }, + ] + expect(agentShouldResumeAfterApproval({messages})).toBe(false) + }) + + it("RESUMES once BOTH concurrent approval cards are answered", () => { + // Same two-gate turn, now both answered (one approve, one deny). Every card is settled + // (`approval-responded`), so the run resumes and the runner gets both round-trips. + const messages = [ + user("do two"), + { + id: "a1", + role: "assistant", + parts: [ + {type: "step-start"}, + { + type: "tool-deleteFile", + toolCallId: "call_1", + state: "approval-responded", + input: {path: "/x"}, + approval: {id: "perm_1", approved: true}, + }, + { + type: "tool-deleteFile", + toolCallId: "call_2", + state: "approval-responded", + input: {path: "/y"}, + approval: {id: "perm_2", approved: false}, + }, + ], + }, + ] + expect(agentShouldResumeAfterApproval({messages})).toBe(true) + }) + it("does NOT resume when there are no messages", () => { expect(agentShouldResumeAfterApproval({messages: []})).toBe(false) }) From 4a9f91775a62b81ab98e493e0c4309e6a572468e Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 19 Jul 2026 00:34:21 +0200 Subject: [PATCH 03/15] feat(runner): record every parked gate and iterate the warm resume Complete the plural approval path in run-turn: record every parkable gate into env.parkedApprovals (keyed by tool-call id), iterate opts.resume.decisions to answer each parked gate by its own permissionId on the live session, defer the orphan-sibling settle to the post-drain sweep (so a concurrent gate is not force-settled before its own permission request arrives), and wire the non-parkable-pause signal. Update the acp-interactions unit test to assert both concurrent gates emit their own card. Part of the concurrent-approvals change (#5373); sits on the deny-frame egress lane whose markToolCallDenied the live-resume deny path reuses. Claude-Session: https://claude.ai/code/session_01DnWRxU3dCJ11hgDidm26vW --- .../src/engines/sandbox_agent/run-turn.ts | 186 +++++++++--------- .../sandbox-agent-acp-interactions.test.ts | 53 ++--- 2 files changed, 116 insertions(+), 123 deletions(-) diff --git a/services/runner/src/engines/sandbox_agent/run-turn.ts b/services/runner/src/engines/sandbox_agent/run-turn.ts index ef7e325a1d..842d85448d 100644 --- a/services/runner/src/engines/sandbox_agent/run-turn.ts +++ b/services/runner/src/engines/sandbox_agent/run-turn.ts @@ -1,7 +1,4 @@ -import { - PendingApprovalLatch, - permissionsFromRequest, -} from "../../permission-plan.ts"; +import { permissionsFromRequest } from "../../permission-plan.ts"; import { resolvePromptText, type AgentRunRequest, @@ -75,7 +72,7 @@ import { resolveRunUsage } from "./usage.ts"; /** * Run one turn against an acquired environment: start a fresh otel run, wire this turn's pause - * controller / latch / decisions / responder into `env.currentTurn`, restart the tool relay, + * controller / decisions / responder into `env.currentTurn`, restart the tool relay, * send the prompt, resolve usage, and finish + flush the trace. It does NOT tear down the * environment (the caller owns `env.destroy`). On a continuation the prompt is only the new user * text (`buildTurnText` does not run); on a cold turn it is `plan.turnText`, exactly as before. @@ -99,11 +96,13 @@ export async function runTurn( // Reset the per-turn tool-call id record (the park folds the completed turn's ids into the // expected next-history fingerprint). env.lastTurnToolCallIds = []; - // Reset the per-turn approval-park bookkeeping. A fresh turn starts with no parked gate; this - // turn re-records it only if it pauses on a Claude ACP permission gate. (The dispatch has - // already captured any prior park into `opts.resume` before calling us.) + // Reset the per-turn approval-park bookkeeping. A fresh turn starts with no parked gates; this + // turn re-records them only if it pauses on ACP permission gates. (The dispatch has already + // captured any prior park into `opts.resume` before calling us.) + env.parkedApprovals.clear(); env.parkedApproval = undefined; env.approvalGateCount = 0; + env.nonParkablePauseCount = 0; // Hoisted so the catch can flush a partial trace (mirroring the pre-split `otel?` handling — // a createOtel throw must still return `{ ok: false }`, not propagate raw) and the finally can // stop this turn's relay on EVERY exit path (a cleared sink must never orphan it). @@ -211,23 +210,22 @@ export async function runTurn( } const pause = new PendingApprovalPauseController(() => { - // The sibling settle runs UNCONDITIONALLY, park mode or not: latch-loser tool calls - // announced before the winning gate can never execute this turn, and skipping the settle - // here would leave them as orphaned open parts whenever the dispatch later refuses the park - // (multi-gate, pool full) — `env.destroy()` does not re-run it. The exclusion keeps the - // gated (paused) call itself open, so the live resume is untouched. - run.settleOpenToolCalls( - (id) => pause.isPausedToolCall(id), - TOOL_NOT_EXECUTED_PAUSED, - ); - // Park mode: a parkable permission gate (Claude ACP or Pi ACP) recorded - // `env.parkedApproval` BEFORE firing this pause (the onUserApprovalGate hook runs before - // the single-pause latch). Keep the live session — the gated tool runs on the resume — so - // skip ONLY the mcpAbort and the destroySession. The teardown is not lost: the dispatch - // either parks the session or, if it decides not to (multi-gate, pool full), calls - // `env.destroy()` which runs them. A non-parkable pause (keep-alive off, client tool) - // never records `parkedApproval`, so it still tears down here exactly as today. - if (opts.approvalParkMode && env.parkedApproval) return; + // Do NOT force-settle open tool calls here, at first pause. With concurrent approvals a + // second gated call may still be in flight (its permission request lands a tick after the + // first gate pauses the turn), and settling it here would orphan a call that is about to + // emit its own approval card. The orphan settle is deferred to the post-drain sweep below + // (which runs on every paused turn, after `waitForEventDrain` lets every pending gate mark + // its call paused) plus the in-band re-sweep in `handleUpdate` for a sibling announced after + // the pause. Both exclude paused (gated) calls, so every carded gate stays open for its + // resume and only a genuine orphan (announced, never gated) settles. + // Park mode: at least one parkable permission gate (Claude ACP or Pi ACP) recorded into + // `env.parkedApprovals` BEFORE firing this pause (the onUserApprovalGate hook runs as each + // gate resolves). Keep the live session — the gated tools run on the resume — so skip ONLY + // the mcpAbort and the destroySession. The teardown is not lost: the dispatch either parks + // the session or, if it decides not to (mixed non-parkable set, pool full), calls + // `env.destroy()` which runs them. A pause with no parkable gate (keep-alive off, client + // tool only) records nothing, so it still tears down here exactly as today. + if (opts.approvalParkMode && env.parkedApprovals.size > 0) return; // Abort any in-flight loopback `tools/call` (a paused Claude client tool) BEFORE the // session teardown, so its handler cannot write a result after the turn ends. env.mcpAbort.abort(); @@ -278,7 +276,7 @@ export async function runTurn( env.lastTurnToolCallIds.push(frame.toolCallId); } run.handleUpdate(update); - // A sibling announced AFTER the pause won the latch can never execute; settle it + // A non-gated sibling announced AFTER the pause can never execute this turn; settle it // immediately so the client never holds an orphaned part (idempotent re-sweep). if (pause.active) { run.settleOpenToolCalls( @@ -305,7 +303,6 @@ export async function runTurn( extractClientToolOutputs(request), ); const executionGrants = new ApprovedExecutionGrants(); - const latch = new PendingApprovalLatch(); const responder = deps.responderFactory?.(request) ?? new ApprovalResponder(permissionPlan, decisions, logger); @@ -359,11 +356,13 @@ export async function runTurn( }, run, responder, - latch, serverPermissions, log: logger, onPause: () => pause.pause(), onPausedToolCall: (id) => pause.markPausedToolCall(id), + onNonParkablePause: () => { + env.nonParkablePauseCount += 1; + }, onCreateInteraction: recordPendingInteraction, onResolveInteraction: resolveInteractionToken, toolSpecsByName: specsByName, @@ -390,27 +389,28 @@ export async function runTurn( // only a dialog-approved (or policy-allowed) call ever executes from the relay dir. onPiGateAllowed: (info) => executionGrants.grant(info.toolName, info.args), - // Record the parkable permission gate (only in keep-alive park mode) so the dispatch can - // resume it live. Fires per pending gate (before the latch) so a parallel gate is counted; - // the single-gate resume records only the FIRST gate's answer target. `info.gateType` names - // the plane (Claude ACP vs Pi ACP) so the resume answers on the right one. + // Record EVERY parkable permission gate (only in keep-alive park mode) so the dispatch can + // resume each one live. Fires per pending gate, so parallel gated tool calls in one turn + // all park, each keyed by its own tool-call id. `info.gateType` names the plane (Claude ACP + // vs Pi ACP) so the resume answers on the right one. `approvalGateCount` counts every gate; + // a gate that lacked a resumable id is counted but not recorded, so the dispatch can tell + // "every gate is resumable" (count === map size) from "a gate cannot be resumed live". onUserApprovalGate: opts.approvalParkMode ? (info) => { env.approvalGateCount += 1; - if ( - env.approvalGateCount === 1 && - info.permissionId && - info.toolCallId - ) { - env.parkedApproval = { - gateType: info.gateType, - permissionId: info.permissionId, - toolCallId: info.toolCallId, - toolName: info.toolName, - args: info.args, - interactionToken: info.interactionToken, - }; - } + if (!info.permissionId || !info.toolCallId) return; + const record: ParkedApproval = { + gateType: info.gateType, + permissionId: info.permissionId, + toolCallId: info.toolCallId, + toolName: info.toolName, + args: info.args, + interactionToken: info.interactionToken, + }; + env.parkedApprovals.set(info.toolCallId, record); + // The first recorded gate is the per-turn representative for logging and the + // per-turn-uniform validation reads (gate type, history, credentials). + env.parkedApproval ??= record; } : undefined, }); @@ -420,10 +420,12 @@ export async function runTurn( env.clientToolRelayRef.current = buildClientToolRelay({ responder, run, - latch, pause, recordPendingInteraction, toolCallIndex: plan.isPi ? undefined : env.toolCallIndex, + onNonParkablePause: () => { + env.nonParkablePauseCount += 1; + }, log: logger, }); @@ -480,43 +482,49 @@ export async function runTurn( let promptPromise: Promise; if (opts.resume) { // The new (resume) turn owns streaming + tracing; the environment is already wired to route - // continued events into this turn's sink (env.currentTurn was set above). Seed this run's - // trace with the parked tool call so the completing `tool_call_update` closes it and the FE - // approval part flips to output-available even if the adapter re-announces nothing. Then - // answer the gate on the live session — the original prompt continues from here. - run.handleUpdate({ - sessionUpdate: "tool_call", - toolCallId: opts.resume.toolCallId, - title: opts.resume.toolName, - kind: opts.resume.toolName, - rawInput: opts.resume.args, - }); - promptPromise = Promise.resolve(opts.resume.promptPromise); + // continued events into this turn's sink (env.currentTurn was set above). Every parked gate + // of the turn is answered here, one `respondPermission` per gate keyed by its tool-call id. + // All decisions share the ONE held prompt promise (one prompt per turn), so it is set once; + // answering the last gate lets the original prompt continue from here. + const decisions = opts.resume.decisions; + promptPromise = Promise.resolve(decisions[0]?.promptPromise); promptPromise.catch(() => {}); - // A parked Pi dialog gate resumes on a FRESH turn whose relay and grant ledger are new; - // grant the approved call here so the extension's execute record (written right after the - // confirm resolves) passes the relay guard. Claude resumes grant too — harmlessly, no - // guard consults it. - if (opts.resume.reply === "once") { - executionGrants.grant(opts.resume.toolName, opts.resume.args); - } - // A live-resume deny closes the seeded call as a failed tool call; flag it so the egress - // projects `tool-output-denied` (a decline), mirroring the cold decision-map deny path. - if (opts.resume.reply === "reject") { - run.markToolCallDenied(opts.resume.toolCallId); + for (const decision of decisions) { + // Seed this run's trace with the parked tool call so the completing `tool_call_update` + // closes it and the FE approval part flips to output-available even if the adapter + // re-announces nothing. + run.handleUpdate({ + sessionUpdate: "tool_call", + toolCallId: decision.toolCallId, + title: decision.toolName, + kind: decision.toolName, + rawInput: decision.args, + }); + // A parked Pi dialog gate resumes on a FRESH turn whose relay and grant ledger are new; + // grant the approved call here so the extension's execute record (written right after the + // confirm resolves) passes the relay guard. Claude resumes grant too — harmlessly, no + // guard consults it. + if (decision.reply === "once") { + executionGrants.grant(decision.toolName, decision.args); + } + // A live-resume deny closes the seeded call as a failed tool call; flag it so the egress + // projects `tool-output-denied` (a decline), mirroring the cold decision-map deny path. + if (decision.reply === "reject") { + run.markToolCallDenied(decision.toolCallId); + } + // Answer this gate on the live session. Each parked gate holds its OWN pending + // `respondPermission` on the harness, so answering them one by one settles each + // independently — an approve and a deny in the same turn each land on the right call. + await env.session.respondPermission(decision.permissionId, decision.reply); + // The gate is answered: resolve its durable interaction row (the parked pending row the + // cold path would otherwise resolve via its decision map). The fresh per-turn pause + // controller starts with an EMPTY pausedToolCallIds set, so the resumed calls' + // `tool_call_update` frames are no longer suppressed and stream through. + resolveInteractionToken(decision.interactionToken); + logger( + `[keepalive] resume answered gate reply=${decision.reply} tool=${decision.toolName ?? "?"}`, + ); } - await env.session.respondPermission( - opts.resume.permissionId, - opts.resume.reply, - ); - // The gate is answered: resolve the durable interaction row (the parked pending row the cold - // path would otherwise resolve via its decision map). The fresh per-turn pause controller - // starts with an EMPTY pausedToolCallIds set, so the resumed call's `tool_call_update` frames - // are no longer suppressed and stream through — the "clear pausedToolCallIds on resume" step. - resolveInteractionToken(opts.resume.interactionToken); - logger( - `[keepalive] resume answered gate reply=${opts.resume.reply} tool=${opts.resume.toolName ?? "?"}`, - ); } else { promptPromise = Promise.resolve( env.session.prompt([{ type: "text", text: turnText }]), @@ -546,13 +554,13 @@ export async function runTurn( ); } const result = raced === PAUSED ? undefined : raced; - // A parkable pause this turn: hand the still-pending prompt promise to the parked record so a - // later resume can await the same continuation. (Set after the race so `promptPromise` exists. - // The read is asserted because the onUserApprovalGate callback set the field via an async - // mutation TS's flow analysis cannot see, so it would otherwise narrow the reset to `never`.) - const parkedThisTurn = env.parkedApproval as ParkedApproval | undefined; - if (opts.approvalParkMode && pause.active && parkedThisTurn) { - parkedThisTurn.promptPromise = promptPromise; + // A parkable pause this turn: hand the still-pending prompt promise to EVERY parked record so a + // later resume can await the same continuation (there is one prompt per turn, so every gate + // shares it). Set after the race so `promptPromise` exists. + if (opts.approvalParkMode && pause.active && env.parkedApprovals.size > 0) { + for (const record of env.parkedApprovals.values()) { + record.promptPromise = promptPromise; + } } await turn.toolRelay?.stop(); logger(`prompt stopReason=${stopReason}`); diff --git a/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts b/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts index 07e1ae8e07..bdb1f7ed63 100644 --- a/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts +++ b/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts @@ -14,7 +14,6 @@ import { ConversationDecisions, } from "../../src/responder.ts"; import type { PermissionPlan, Verdict } from "../../src/permission-plan.ts"; -import { PendingApprovalLatch } from "../../src/permission-plan.ts"; import { attachPermissionResponder, type PiToolSpecMeta, @@ -96,7 +95,6 @@ describe("attachPermissionResponder", () => { session, run: { emitEvent: (event) => events.push(event) }, responder: fakeResponder({ kind: "allow" }, undefined, seen), - latch: new PendingApprovalLatch(), }); emit({ id: "perm-1", @@ -124,7 +122,6 @@ describe("attachPermissionResponder", () => { session, run: { emitEvent: (event) => events.push(event) }, responder: fakeResponder({ kind: "deny" }), - latch: new PendingApprovalLatch(), }); emit({ id: "perm-1", availableReplies: ["once", "reject"] }); await flushPromises(); @@ -148,7 +145,6 @@ describe("attachPermissionResponder", () => { markToolCallDenied: (id) => denied.push(id), }, responder: fakeResponder({ kind: "deny" }), - latch: new PendingApprovalLatch(), }); emit({ id: "perm-1", @@ -174,7 +170,6 @@ describe("attachPermissionResponder", () => { markToolCallDenied: (id) => denied.push(id), }, responder: fakeResponder({ kind: "allow" }), - latch: new PendingApprovalLatch(), }); emit({ id: "perm-1", @@ -205,7 +200,6 @@ describe("attachPermissionResponder", () => { session, run: { emitEvent: (event) => events.push(event) }, responder: fakeResponder({ kind: "pendingApproval" }), - latch: new PendingApprovalLatch(), onPause: () => { pauses += 1; }, @@ -256,27 +250,41 @@ describe("attachPermissionResponder", () => { ]); }); - it("two concurrent pending gates emit and pause only once through the latch", async () => { + it("two concurrent pending gates each emit their own card and pause the turn once", async () => { const { session, emit } = makeSession(); const events: AgentEvent[] = []; + const gates: string[] = []; let pauses = 0; attachPermissionResponder({ session, run: { emitEvent: (event) => events.push(event) }, responder: fakeResponder({ kind: "pendingApproval" }), - latch: new PendingApprovalLatch(), onPause: () => { pauses += 1; }, + onUserApprovalGate: (info) => { + gates.push(info.toolCallId); + }, }); emit({ id: "perm-1", toolCall: { toolCallId: "tool-1", name: "edit" } }); emit({ id: "perm-2", toolCall: { toolCallId: "tool-2", name: "bash" } }); await flushPromises(); - assert.equal(events.length, 1); - assert.equal((events[0] as any).id, "perm-1"); - assert.equal(pauses, 1); + // No latch: each gate emits its own interaction_request card, keyed by its own tool-call id. + assert.equal(events.length, 2); + assert.deepEqual( + events.map((e) => (e as any).id), + ["perm-1", "perm-2"], + ); + assert.deepEqual( + events.map((e) => (e as any).payload.toolCallId), + ["tool-1", "tool-2"], + ); + // onPause fires once per gate at this layer; the shared PendingApprovalPauseController dedupes + // to a single turn-end (asserted in pending-approval-pause.test.ts). Both gates signalled. + assert.equal(pauses, 2); + assert.deepEqual(gates, ["tool-1", "tool-2"]); }); it("reply rejection pauses and does not resolve the interaction", async () => { @@ -290,7 +298,6 @@ describe("attachPermissionResponder", () => { session, run: { emitEvent: () => {} }, responder: fakeResponder({ kind: "allow" }), - latch: new PendingApprovalLatch(), onPause: () => { pauses += 1; }, @@ -317,7 +324,6 @@ describe("attachPermissionResponder", () => { session, run: { emitEvent: (event) => events.push(event) }, responder: fakeResponder({ kind: "allow" }), - latch: new PendingApprovalLatch(), onPause: () => { pauses += 1; }, @@ -353,7 +359,6 @@ describe("attachPermissionResponder", () => { session, run: { emitEvent: (event) => events.push(event) }, responder: fakeResponder({ kind: "deny" }, { kind: "pendingApproval" }), - latch: new PendingApprovalLatch(), toolSpecsByName: specsByName([CLIENT_SPEC]), onPause: () => { pauses += 1; @@ -424,7 +429,6 @@ describe("attachPermissionResponder", () => { { kind: "deny" }, { kind: "fulfilled", output: { connected: true } }, ), - latch: new PendingApprovalLatch(), toolSpecsByName: specsByName([CLIENT_SPEC]), }); emit({ @@ -462,7 +466,6 @@ describe("attachPermissionResponder", () => { session, run: { emitEvent: () => {} }, responder: fakeResponder({ kind: "allow" }), - latch: new PendingApprovalLatch(), onResolveInteraction: (token) => { resolved.push(token); }, @@ -491,7 +494,6 @@ describe("attachPermissionResponder", () => { session, run: { emitEvent: () => {} }, responder: fakeResponder({ kind: "pendingApproval" }), - latch: new PendingApprovalLatch(), onCreateInteraction: (token) => { created.push(token); }, @@ -531,7 +533,6 @@ describe("attachPermissionResponder", () => { ], }, responder: fakeResponder({ kind: "allow" }, undefined, seen), - latch: new PendingApprovalLatch(), }); emit({ id: "perm-1", availableReplies: ["once", "reject"], toolCall }); await flushPromises(); @@ -557,7 +558,6 @@ describe("attachPermissionResponder", () => { session, run: { emitEvent: () => {} }, responder: fakeResponder({ kind: "pendingApproval" }, undefined, seen), - latch: new PendingApprovalLatch(), toolSpecsByName: specsByName([ { name: "commit_revision", permission: "ask", readOnly: false }, ]), @@ -592,7 +592,6 @@ describe("attachPermissionResponder", () => { session, run: { emitEvent: () => {} }, responder: fakeResponder({ kind: "pendingApproval" }, undefined, seen), - latch: new PendingApprovalLatch(), toolSpecsByName: specsByName([ { name: "delete_everything", permission: "deny", readOnly: false }, ]), @@ -627,7 +626,6 @@ describe("attachPermissionResponder", () => { session, run: { emitEvent: () => {} }, responder: fakeResponder({ kind: "pendingApproval" }, undefined, seen), - latch: new PendingApprovalLatch(), toolSpecsByName: specsByName([ { name: "commit_revision", permission: "ask" }, ]), @@ -657,7 +655,6 @@ describe("attachPermissionResponder", () => { session, run: { emitEvent: () => {} }, responder: fakeResponder({ kind: "pendingApproval" }, undefined, seen), - latch: new PendingApprovalLatch(), serverPermissions: new Map([["github", "deny"]]), }); emit({ @@ -738,7 +735,6 @@ describe("attachPermissionResponder: Pi dialog gate", () => { session, run: { emitEvent: (event) => events.push(event) }, responder: fakeResponder({ kind: "pendingApproval" }), - latch: new PendingApprovalLatch(), piToolSpecsByName: piSpecs(), onPausedToolCall: (id) => pausedToolCalls.push(id), onUserApprovalGate: (info) => gates.push(info), @@ -779,7 +775,6 @@ describe("attachPermissionResponder: Pi dialog gate", () => { run: { emitEvent: (event) => events.push(event) }, // A default-allow responder: if the malformed request fell through it would ALLOW. responder: fakeResponder({ kind: "allow" }), - latch: new PendingApprovalLatch(), piToolSpecsByName: piSpecs(), onPause: () => { pauses += 1; @@ -815,7 +810,6 @@ describe("attachPermissionResponder: Pi dialog gate", () => { session, run: { emitEvent: () => {} }, responder: fakeResponder({ kind: "pendingApproval" }, undefined, seen), - latch: new PendingApprovalLatch(), piToolSpecsByName: piSpecs(), }); emit( @@ -851,7 +845,6 @@ describe("attachPermissionResponder: Pi dialog gate", () => { session, run: { emitEvent: () => {} }, responder, - latch: new PendingApprovalLatch(), piToolSpecsByName, }); emit( @@ -892,7 +885,6 @@ describe("attachPermissionResponder: Pi dialog gate", () => { session, run: { emitEvent: () => {} }, responder, - latch: new PendingApprovalLatch(), piToolSpecsByName: piSpecs(), onPause: () => { pauses += 1; @@ -929,7 +921,6 @@ describe("attachPermissionResponder: Pi dialog gate", () => { session, run: { emitEvent: () => {} }, responder, - latch: new PendingApprovalLatch(), piToolSpecsByName: piSpecs(), onPause: () => { pauses += 1; @@ -965,7 +956,6 @@ describe("attachPermissionResponder: Pi dialog gate", () => { session, run: { emitEvent: (event) => events.push(event) }, responder, - latch: new PendingApprovalLatch(), piToolSpecsByName: piSpecs(), onPause: () => { pauses += 1; @@ -1000,7 +990,6 @@ describe("attachPermissionResponder: Pi dialog gate", () => { session, run: { emitEvent: (event) => events.push(event) }, responder, - latch: new PendingApprovalLatch(), piToolSpecsByName: piSpecs([["park_probe", {}]]), }); emit( @@ -1029,7 +1018,6 @@ describe("attachPermissionResponder: Pi dialog gate", () => { session, run: { emitEvent: (event) => events.push(event) }, responder: fakeResponder({ kind: "pendingApproval" }), - latch: new PendingApprovalLatch(), piToolSpecsByName: piSpecs([ [ "test_run", @@ -1073,7 +1061,6 @@ describe("attachPermissionResponder: Pi dialog gate", () => { session, run: { emitEvent: () => {} }, responder, - latch: new PendingApprovalLatch(), piToolSpecsByName: piSpecs([ [ "author_allow", @@ -1117,7 +1104,6 @@ describe("attachPermissionResponder: Pi dialog gate", () => { session, run: { emitEvent: () => {} }, responder, - latch: new PendingApprovalLatch(), piToolSpecsByName: piSpecs([["author_deny", { permission: "deny" }]]), onPiGateAllowed: (info) => allowed.push(info), }); @@ -1163,7 +1149,6 @@ describe("attachPermissionResponder: Pi dialog gate", () => { session, run: { emitEvent: (event) => events.push(event) }, responder: fakeResponder({ kind: "pendingApproval" }), - latch: new PendingApprovalLatch(), // piToolSpecsByName intentionally absent (a Claude run). onPause: () => { pauses += 1; From d0f9a101f01ffc8118d7da4b398a150a537d63f4 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 19 Jul 2026 15:50:01 +0200 Subject: [PATCH 04/15] =?UTF-8?q?docs(plan):=20approvals=20incident=20fixe?= =?UTF-8?q?s=20=E2=80=94=20root=20cause,=20Zed=20comparison,=20and=20execu?= =?UTF-8?q?tion=20plan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../approvals-incident-fixes/README.md | 50 ++ .../approvals-incident-fixes/context.md | 58 ++ .../projects/approvals-incident-fixes/plan.md | 547 ++++++++++++++++++ .../projects/approvals-incident-fixes/qa.md | 96 +++ .../approvals-incident-fixes/research.md | 484 ++++++++++++++++ .../approvals-incident-fixes/status.md | 17 + .../debug-concurrent-approvals-db58551b.md | 208 +++++++ .../scratch/debug-session-turns-append-500.md | 66 +++ .../scratch/zed-acp-approvals-comparison.md | 134 +++++ 9 files changed, 1660 insertions(+) create mode 100644 docs/design/agent-workflows/projects/approvals-incident-fixes/README.md create mode 100644 docs/design/agent-workflows/projects/approvals-incident-fixes/context.md create mode 100644 docs/design/agent-workflows/projects/approvals-incident-fixes/plan.md create mode 100644 docs/design/agent-workflows/projects/approvals-incident-fixes/qa.md create mode 100644 docs/design/agent-workflows/projects/approvals-incident-fixes/research.md create mode 100644 docs/design/agent-workflows/projects/approvals-incident-fixes/status.md create mode 100644 docs/design/agent-workflows/scratch/debug-concurrent-approvals-db58551b.md create mode 100644 docs/design/agent-workflows/scratch/debug-session-turns-append-500.md create mode 100644 docs/design/agent-workflows/scratch/zed-acp-approvals-comparison.md diff --git a/docs/design/agent-workflows/projects/approvals-incident-fixes/README.md b/docs/design/agent-workflows/projects/approvals-incident-fixes/README.md new file mode 100644 index 0000000000..ec2842ba9b --- /dev/null +++ b/docs/design/agent-workflows/projects/approvals-incident-fixes/README.md @@ -0,0 +1,50 @@ +# Concurrent approvals incident fixes + +This workspace plans the fixes for the concurrent human-approval failure observed in live +session `db58551b` on 2026-07-19, plus the session-turns counter bug found in the same logs. +The incident and its seven defects are documented and log-verified in +`docs/design/agent-workflows/scratch/debug-concurrent-approvals-db58551b.md`; the design +principles we adopt from Zed's handling of the same problem are in +`docs/design/agent-workflows/scratch/zed-acp-approvals-comparison.md`; the counter bug is in +`docs/design/agent-workflows/scratch/debug-session-turns-append-500.md`. + +## Reading order + +1. `context.md` explains why this work exists, what it must achieve, and what is + deliberately out of scope. +2. `research.md` holds the code-verified findings (R1 through R8) that the plan is built + on, with file and line evidence for every claim, and a prominent list of open risks. +3. `plan.md` is the implementation plan: five ordered steps, each with exact files, + behavioral contracts, acceptance criteria, and test commands. Each step is independently + landable. The implementer is expected to work from this file without having read the + incident conversation. +4. `qa.md` is the live QA script that reproduces the incident shape against the dev stack + and states the expected correct behavior at each point. +5. `status.md` tracks where the project stands. + +## Glossary + +Every domain term used in this workspace, one line each. + +- **Runner**: the Node/TypeScript sidecar under `services/runner/` that drives a coding-agent + harness inside a sandbox and streams events to the web UI. +- **Harness**: the agent program that implements the model-and-tool loop, such as Pi + (`pi_core`) or Claude Code (`claude`). The runner talks to it over ACP, the Agent Client + Protocol. +- **Gate**: a human-approval request. When the harness wants to run a permission-gated tool, + the runner shows the user an approval card and waits for allow or deny. +- **Park**: ending the browser-facing turn while an unanswered gate keeps the live harness + process waiting. The answer arrives in a later request. +- **Warm resume**: continuing a parked session in place. The runner checks the live process + out of the keepalive pool and answers the original permission request by its id, so the + approved tool runs with its original arguments. +- **Keepalive pool**: the runner's bounded collection of live parked sessions, each with a + time-to-live limit after which it is destroyed. +- **Records**: the durable per-session event stream. The runner posts every agent event to + the API's record-ingest endpoint; the frontend rebuilds a conversation from these rows + (that rebuild is called hydration). +- **Interactions**: the durable table of human-in-the-loop requests. Each gate creates one + interaction row with a lifecycle status (pending, responded, resolved, cancelled). +- **Turns**: the append-only `session_turns` table. One row per completed conversation turn, + carrying the harness's native session id so a restarted runner can resume the conversation + natively instead of replaying text. diff --git a/docs/design/agent-workflows/projects/approvals-incident-fixes/context.md b/docs/design/agent-workflows/projects/approvals-incident-fixes/context.md new file mode 100644 index 0000000000..521044bcca --- /dev/null +++ b/docs/design/agent-workflows/projects/approvals-incident-fixes/context.md @@ -0,0 +1,58 @@ +# Context: why this work exists + +## The incident in three sentences + +During live QA of two parallel permission-gated shell commands (session `db58551b`, +2026-07-19), both commands executed exactly once and only after their approvals, but the +system misreported both, flipped an already-approved card back to "waiting", and then went +silent because the user's final approval was never sent to the runner. The root causes are +that the durable session record stores every approval request but never any approval answer, +that the post-pause cleanup invents tool results (a false "not executed" error for an +approved running call, and a false success for a call that never started), and that the +frontend only dispatches approvals once every visible card looks settled, a condition that +can never hold after a state rebuild. A seventh, unrelated defect found in the same logs +makes the new `session_turns` ingestion fail with HTTP 500 on every warm turn, because the +turn index is computed once per environment acquire instead of once per turn. + +## Goals + +1. Fix the session-turns counter so `turn_index` is a true conversation-turn counter, and + make the API answer a duplicate append with 409 Conflict instead of 500. +2. Stop the pause cleanup from inventing tool results: an approved, executing call keeps its + real result, and a never-started call is recorded as deferred, never as success. +3. Persist the answer half of every gate (a new `interaction_response` record event plus the + allow/deny verdict in the interaction row's existing `resolution` field), and make + frontend hydration overlay answers onto requests, so a rebuilt conversation shows + answered cards as answered. +4. Dispatch each approval per card the moment the user answers it, and make the runner + accept a resume that answers only part of the parked gates while remaining parked on the + rest. +5. Add a regression test that reproduces the exact incident shape, then verify with live QA + on the dev stack. + +## Non-goals + +Two related fixes are deliberately out of scope here. + +- **Real turn cancellation** (the Zed-style cancel-then-restart when new user text arrives + while gates are parked, which fixes defect 6 of the incident report) goes to Arda after + JP's sessions PRs merge, because it depends on the session-streams control plane those PRs + introduce and is partly a product decision. +- **Audit-hardening of record ids** (scoping stable record ids by turn so contradictory + results append instead of overwriting, defect 5) is queued separately, because the answer + persistence in this project already fixes the user-facing rebuild problem and the id + rescope touches the record identity contract end to end. + +## Constraints already decided + +- Work lands in this order, on these lanes: the counter fix on the existing rebase lanes for + PR #5376 (runner, branch `sessions-rebase/runner`) and PR #5375 (backend, branch + `sessions-rebase/backend`), with a wipe of the wrongly numbered dev rows and an + explanatory note for JP; everything else on the PR #5382 lane (branch + `plan/concurrent-approvals`). +- `turn_index` is confirmed to be a true conversation-turn counter, not an acquire counter. + The implementation must carry a code comment stating this invariant, and the PR body must + explain what was done and what was assumed. +- The implementation will be done by an agent that has not read the incident conversation, + so every step in `plan.md` is specified with exact files, behavioral contracts, and + acceptance criteria. diff --git a/docs/design/agent-workflows/projects/approvals-incident-fixes/plan.md b/docs/design/agent-workflows/projects/approvals-incident-fixes/plan.md new file mode 100644 index 0000000000..66eab82aa2 --- /dev/null +++ b/docs/design/agent-workflows/projects/approvals-incident-fixes/plan.md @@ -0,0 +1,547 @@ +# Implementation plan + +Five ordered steps. Each step is independently landable and independently testable. Read +`context.md` for why, and `research.md` for the evidence behind every mechanic referenced +here. Line numbers refer to the current workspace working tree; on the individual branches +the same regions exist at nearby lines (verified in research.md R8). + +Conventions that apply to every step: + +- All version control goes through GitButler (`but`), never raw git branch/commit + commands. The repository root `AGENTS.md` documents the commands and the + one-lane-at-a-time isolation procedure. Because this project and JP's rebase lanes share + files (research.md R8), you MUST: assign exactly one lane's files at a time, commit with + `but commit --only`, then verify with `git show --stat --name-only ` AND + `git diff --name-only ..` (base is the branch below the lane) before + starting the next lane. If a file from another lane appears, stop and fix before + continuing. +- Code comments explain why and invariants in one or two sentences. No session narration. +- Runner checks before every push: `pnpm test` and `pnpm run typecheck` from + `services/runner`. API checks: `ruff format` then `ruff check --fix` from `api`, then + the API unit tests via `cd api && py-run-tests` where a step says so. Web checks: + `pnpm lint-fix` from `web`. +- Do not merge anything. Each step ends at green tests plus a pushed lane; merging is + Mahmoud's action. + +## Step 1: session-turns counter fix and 409 mapping + +Purpose: `session_turns` appends currently 500 on every warm turn because the turn index is +computed once per environment acquire instead of once per turn, and the API treats the +resulting unique-key violation as an unknown error. After this step the index is a true +conversation-turn counter and a duplicate append reads as a 409 Conflict. + +Lanes: the runner half lands on `sessions-rebase/runner` (PR #5376); the backend half lands +on `sessions-rebase/backend` (PR #5375). Both are amendments to JP's open rebase PRs, so +each half is committed to its lane and the PR bodies get a short addendum (text below). + +### 1a. Runner: compute the turn index per turn + +Files: `services/runner/src/engines/sandbox_agent/environment.ts`, +`services/runner/src/engines/sandbox_agent/run-turn.ts`, +`services/runner/tests/unit/session-continuity-durable.test.ts` (or a sibling unit file if +a dispatch-level test fits better there). + +Contract: + +- Delete the acquire-time assignment `environment.continuityTurnIndex = ...` at + `environment.ts:962-964`. The durable hydrate call just above it (lines 941-949) stays: + it seeds the shared store after a runner restart and must keep running at acquire. +- At the start of `runTurn` (`run-turn.ts`, alongside the per-turn resets at lines 88-95), + set `env.continuityTurnIndex` by calling `nextTurnIndex(env.sessionId, store)` where + `store` is `deps.sessionContinuityStore ?? sessionContinuityStore` (the same fallback the + record call at line 572 uses). When `env.sessionId` is empty, set it to `undefined`, + matching the old acquire-time behavior for sessionless runs. +- Add this code comment at the new computation, stating the invariant and the assumption + (required verbatim in substance, wording may be tightened): + "`turn_index` is a true conversation-turn counter for this session, not an acquire + counter: it must advance once per COMPLETED turn, shared across every environment that + serves the session. The store only advances on `record()` (a paused turn records + nothing), so a park-and-resume cycle spanning several runner turns consumes one index. + Computed at turn start, not at environment acquire, because a warm pooled environment + serves many turns." +- Everything downstream is unchanged: the completed-turn record and the durable append + (`run-turn.ts:566-597`) keep reading `env.continuityTurnIndex`. + +Behavior that must hold (and be pinned by tests): + +- A fresh session's first completed turn appends index 0; each later completed turn served + by the SAME warm environment appends 1, 2, 3 in order. +- Two environments serving one session (one approval-parked, one idle, the `poolSize=2` + shape) interleave completed turns with strictly increasing indexes, because both read + the shared process-wide store at turn start. +- A turn that ends paused appends nothing; the resume turn that completes the same + conversation turn appends the index the paused turn would have used. + +Tests: extend `tests/unit/session-continuity-durable.test.ts` (or the orchestration test +if the seam fits better) with the three behaviors above; the two-environment case is the +one named in the 500 report's open questions. Run `pnpm test` and `pnpm run typecheck` +from `services/runner`. + +### 1b. Backend: map the duplicate append to 409 + +Files: `api/oss/src/dbs/postgres/sessions/turns/dao.py`, plus a unit or integration test in +the API test tree if one covers the turns DAO (add a narrow one if none exists). + +Contract: + +- In `SessionTurnsDAO.append` (`dao.py:33-50`), wrap the add-and-commit in + `try/except IntegrityError`. On a violation of + `ix_session_turns_project_id_session_id_turn_index`, roll back and raise + `EntityCreationConflict` (from `api/oss/src/core/shared/exceptions.py`) with + `entity="Session turn"`, a message naming the session and turn index, and a `conflict` + dict carrying `session_id` and `turn_index`. Re-raise any other IntegrityError. Copy the + established pattern from `api/oss/src/dbs/postgres/sessions/streams/dao.py:51-61` (same + domain) or `api/oss/src/dbs/postgres/gateway/connections/dao.py:70-83`. +- No router change: `append_turn` is already wrapped in `@intercept_exceptions()` + (`api/oss/src/apis/fastapi/sessions/router.py:1070`), which converts + `EntityCreationConflict` to HTTP 409 (`api/oss/src/utils/exceptions.py:132-144`). + +Acceptance: POSTing the same `(session_id, turn_index)` twice returns 200 then 409, and +the API error log shows no traceback for the second call. The runner side needs no change +for this: its append helper already logs non-OK statuses as `append HTTP ` +(`session-continuity-durable.ts:190-192`), which becomes diagnosable on sight. + +Run `ruff format`, `ruff check --fix`, and the API tests from `api`. + +### 1c. Operational: wipe the wrongly numbered dev rows + +Every `session_turns` row written while the bug was live carries an acquire-counter index, +so the dev table is wiped, not repaired. On the dev stack's EE database (database +`agenta_ee_core`, credentials `username:password` per the root `AGENTS.md`): + +```sql +-- Inspect first: every row predating the fix is suspect. +SELECT count(*) FROM session_turns; +-- Wipe. +DELETE FROM session_turns; +``` + +Run this AFTER the runner fix is deployed to the dev stack, so no old runner re-inserts +wrong indexes. Effect: the next turn of any existing session cold-replays once (the +continuity lookup finds no row) and then rebuilds correct rows going forward. That is the +table's designed degradation, not data loss. + +### 1d. The note for JP + +Append to the PR #5376 description (and reference from #5375): + +> Two amendments landed on these lanes after live QA. First, the runner computed +> `turn_index` once per environment acquire, so every warm turn re-inserted the same index +> and the append 500ed (`ix_session_turns_project_id_session_id_turn_index`); the index is +> now computed at turn start from the shared continuity store, which also fixes the +> two-pooled-environments case. We confirmed the intended semantics: `turn_index` is a true +> conversation-turn counter, and the code now carries that invariant as a comment. Second, +> the turns DAO now maps the duplicate-key IntegrityError to `EntityCreationConflict`, so a +> duplicate append reads as 409 instead of an anonymous 500. The dev database's +> `session_turns` table was wiped because every row written while the bug was live carried +> acquire-counter indexes; sessions rebuild their rows on their next completed turn. + +## Step 2: stop the pause cleanup from inventing tool results + +Purpose: after this step, the post-pause sweep can only settle calls that never executed +and hold no approval; an approved, executing call keeps its real result; and a +cancellation-closure `completed` frame for a never-started call is recorded as deferred, +never as success. This removes incident defects 2 and 3 and, with them, the false +"retry the same call" invitation on commands that actually ran. + +Lane: `plan/concurrent-approvals` (PR #5382). + +Files: `services/runner/src/engines/sandbox_agent/run-turn.ts`, +`services/runner/src/engines/sandbox_agent/pause.ts`, +`services/runner/src/engines/sandbox_agent/runtime-policy.ts`, +`services/runner/src/engines/sandbox_agent/acp-interactions.ts`, +`services/runner/src/tracing/otel.ts`, `services/runner/src/responder.ts`, +`services/runner/tests/unit/sandbox-agent-orchestration.test.ts`, +`services/runner/tests/unit/session-keepalive-approval.test.ts`, +`services/runner/tests/unit/responder.test.ts`, +`services/runner/tests/unit/pending-approval-pause.test.ts`. + +### 2a. Track allowed executions by tool-call id + +Add a per-turn id set of calls whose execution this turn legitimately allowed. Suggested +home: the pause controller (`pause.ts`), next to `pausedToolCallIds`, as +`allowedExecutionToolCallIds` with `markAllowedExecution(toolCallId)` and +`isAllowedExecution(toolCallId)`. Populate it from both allow paths: + +- The warm-resume loop: for each `decision.reply === "once"`, mark + `decision.toolCallId` (`run-turn.ts:463-465`, where the execution grant is already + recorded). +- The in-turn allow path: `replyPermission` in `acp-interactions.ts:228-252` receives the + decision and the tool-call id; on `decision === "allow"`, invoke a new optional callback + (wired from `run-turn.ts` the same way `onPausedToolCall` is) that marks the id. This + covers auto-allowed and stored-decision-allowed calls, which can also be mid-execution + when a sibling gate pauses the turn. + +### 2b. Exclude allowed executions from both sweeps + +Both `settleOpenToolCalls` call sites currently exclude only paused gates. Change the +predicate at `run-turn.ts:232-237` (the in-band re-sweep) and `run-turn.ts:505-511` (the +post-drain sweep) to exclude a call when `pause.isPausedToolCall(id) || +pause.isAllowedExecution(id)`. + +### 2c. Let an allowed execution's real terminal frame land + +Two changes: + +- `shouldSuppressPausedToolCallUpdate` (`runtime-policy.ts:31-60`) currently suppresses + every `failed` frame while the pause is active. Exempt allowed-execution ids: their real + failure is genuine evidence and must stream through. (Their `completed` frames already + pass.) The function needs access to the allowed set; pass the pause controller as today + and read the new method. +- After the post-drain point (`run-turn.ts:505-511`), before the sweep runs, wait for every + id in the allowed set that is STILL OPEN in the tracer to reach its own terminal frame. + Expose the open-call ids from the tracer (a new `openToolCallIds(): string[]` on + `SandboxAgentOtel`, reading `toolSpans` keys, `otel.ts:1128-1131`) and await closure with + a bounded wait: per call, at most the configured per-tool-call limit from + `run-limits.ts` (read the same config value; do NOT re-arm the run-limits deadlines, + which `notePaused` retired at `run-turn.ts:188` because a human pause is legitimate). + The turn's sink is still active during this wait, so the arriving result flows through + `handleUpdate` and `maybeCloseTool` normally and is never dropped as a between-turns + event. +- If the bound expires for a call, settle THAT call with a new sentinel exported next to + the existing one in `otel.ts`: + `APPROVED_EXECUTION_RESULT_UNKNOWN: the approved call started but its result was not + observed before the pause ended the turn; do not assume it failed and do not retry a + side-effecting call.` Record it as `isError: true`. It must NOT begin with the + `DEFERRED_NOT_EXECUTED` prefix (that prefix means "never executed, retry is safe"). +- Exclude the new sentinel from the client-output store the same way the deferred one is + excluded (`responder.ts:398-401` and `491-496`): a sibling settled with either sentinel + must never fulfill a later identical call. + +### 2d. Record a never-started call as deferred, not success + +While the pause is active, a `completed` frame can be a cancellation-closure artifact for a +call that never ran (incident defect 3: a `"(no output)"` success for a command that was +never approved). Classify by fail-closed policy evidence: + +- Buffer `completed` frames that arrive while `pause.active` for calls that are not paused + gates and not allowed executions, instead of letting `maybeCloseTool` record them + immediately. Suggested seam: in `run-turn.ts`'s `handleUpdate` (lines 196-239), before + `run.handleUpdate(update)`. +- When `pause.waitForEventDrain()` resolves (the same point the sweep runs today), settle + each buffered frame: if its call became a paused gate during the drain, drop the frame + (the gate's card is the last word for that call this turn); if its call became an + allowed execution, deliver the frame (real result); otherwise, if the call's effective + permission is `ask` or `deny` (resolve it the same way the gate descriptor does, via the + turn's `permissionsFromRequest` plan and `toolSpecsByName`, both already in scope in + `runTurn`), record the deferred sentinel `TOOL_NOT_EXECUTED_PAUSED` for it, because a + fail-closed gate that was never answered allow cannot have executed; if the effective + permission is `allow`, deliver the frame (an auto-allowed sibling that legitimately + finished). +- Carry a code comment stating the invariant: "Execution of an ask-policy call requires an + answered allow; both harness gate paths fail closed. A completed frame during a pause + for an unanswered ask-policy call is therefore a cancellation-closure artifact, not + evidence of execution." + +### 2e. Model-transcript guarantee + +Add one assertion-level check (a debug log is enough, a throw is not wanted): at +terminalization of a paused turn, after the sweep, the tracer holds no open calls except +paused gates. Combined with 2c and 2d this preserves the requirement that every tool call +the model ever sees eventually carries some result on the replay-fallback path, and the +warm and cold-native paths never consume runner-side records (research.md R5). + +### Acceptance criteria for step 2 + +- Existing tests still pass, specifically `sandbox-agent-orchestration.test.ts:1753` (two + racing gates both card, neither settled), the non-gated-sibling settle test around + line 1690, and `responder.test.ts:679`. +- New unit tests pin: (1) an allowed-execution call is never stamped with + `TOOL_NOT_EXECUTED_PAUSED` by either sweep; (2) its real `completed` frame arriving after + the pause but before terminalization is recorded as its result; (3) its real `failed` + frame is not suppressed; (4) the bounded wait expiring records the + `APPROVED_EXECUTION_RESULT_UNKNOWN` sentinel and neither sentinel enters the + client-output store; (5) a `completed` frame during a pause for an unanswered ask-policy + call is recorded as `TOOL_NOT_EXECUTED_PAUSED`, while the same frame for an allow-policy + call keeps its real result. +- Test commands: `cd services/runner && pnpm test && pnpm run typecheck`. + +## Step 3: persist the answer half of every gate + +Purpose: after this step, every resolved gate leaves two durable traces: an +`interaction_response` event in the session records stream, and the allow/deny verdict in +the interaction row's `resolution` field. Frontend hydration overlays answers onto +requests, so a rebuilt conversation renders answered cards as answered. This removes +incident defect 4 and the rebuild half of defect 1. + +Lane: `plan/concurrent-approvals` (PR #5382). Note that `protocol.ts`, `persist.ts`, and +`api/oss/src/core/sessions/interactions/dtos.py` are also touched by JP's lanes in the +other stack; the regions edited here are identical on both stacks (research.md R8), but +follow the isolation commit procedure strictly. + +### 3a. Runner: the new event and its emission + +Files: `services/runner/src/protocol.ts`, `services/runner/src/sessions/persist.ts`, +`services/runner/src/engines/sandbox_agent/run-turn.ts`, +`services/runner/src/engines/sandbox_agent/acp-interactions.ts`, +`services/runner/src/sessions/interactions.ts`. + +Contract: + +- Add to the `AgentEvent` union (`protocol.ts`, next to `interaction_request` at + lines 362-367): + + ```ts + // The durable answer half of an interaction_request, emitted when the runner forwards + // a human decision to the harness. `id` equals the matching request's id. Hydration + // overlays it so a rebuilt conversation shows the gate as answered. + | { + type: "interaction_response"; + id: string; + kind: "user_approval"; + payload?: unknown; + } + ``` + + Payload for `user_approval`: `{toolCallId: string, approved: boolean}`. No actor field + (audit actor identity is queued work; the record row's credential-derived metadata is the + interim signal). Also add `interaction_response` to the documented known-types list in + `sdks/python/agenta/sdk/agents/wire_models.py:381` (a docstring, not enforcement; the + Python event model is an open union and needs no code change). +- Emit it at the ONE convergence point both answer paths share. In `run-turn.ts`, + `resolveInteractionToken` (lines 287-296) is called by the warm-resume loop (line 479) + and wired as `onResolveInteraction` into the gate handler (line 323), which fires it + from `replyPermission`/`replyClientTool` after a successful harness reply + (`acp-interactions.ts:223-226, 251, 271`). Extend the signature to carry the verdict and + the tool-call id: `resolveInteractionToken(token, {approved, toolCallId})`. Callers: + the resume loop derives `approved` from `decision.reply === "once"` and has + `decision.toolCallId`; `replyPermission` has `decision` and `toolCallId`; + `replyClientTool` resolves client-tool interactions, which have no allow/deny verdict, + so it passes no verdict and 3a emits nothing for it (client tools get their answer + recorded as the tool result itself). Inside the function, alongside the existing + `resolveInteraction` POST, call `run.emitEvent({type: "interaction_response", id: token, + kind: "user_approval", payload: {toolCallId, approved}})`. `run` is in scope in + `runTurn`; pass it in or close over it. +- Give the event a stable record id so a retried resume upserts one row: extend the + stable-id branch in `persist.ts` (lines 297-313) to include `interaction_response` + (keyed by the event's `id`, which is the interaction token, and the record type, via + the existing `stableRecordId`). +- The live stream also carries the event; the Python Vercel egress ignores unknown types + by construction (no `else` in its ladder) and the web ignores unknown record types + outside the hydration switch, so no egress change is needed. + +### 3b. Backend: the verdict on the interaction row + +Files: `api/oss/src/core/sessions/interactions/dtos.py`, +`api/oss/src/apis/fastapi/sessions/models.py`, +`api/oss/src/apis/fastapi/sessions/router.py`, +`api/oss/src/dbs/postgres/sessions/interactions/dao.py`, plus the interactions service and +interface files if they type the transition. + +Contract: + +- `SessionInteractionTransition` (`dtos.py:66-70`) gains + `resolution: Optional[Dict[str, Any]] = None`. Field classification: `resolution` is + data (the answer content), `status` stays lifecycle metadata, `token`/`session_id` stay + protocol context. The transition request model in `models.py` gains the same field, and + `transition_interaction` (`router.py:625-655`) passes it through. +- The DAO's transition UPDATE (`dao.py:91-119`) additionally writes the verdict into the + row's `data.resolution` when the transition carries one, without clobbering the rest of + `data` (use a JSONB set on the `resolution` key, or a read-modify-write inside the same + guard; the guard `status IN ('pending','responded')` is unchanged). A transition without + `resolution` behaves exactly as today. +- Resolution payload written by the runner: + `{"verdict": "approved" | "denied", "tool_call_id": }`. Full words, no + abbreviations. +- Runner side: `resolveInteraction` (`services/runner/src/sessions/interactions.ts:97-119`) + gains an optional `resolution` argument and includes it in the POST body when present. + `resolveInteractionToken` (3a) passes it. + +Acceptance: after an approval, `GET /sessions/interactions/{id}` returns the row with +`status: "resolved"` and `data.resolution == {"verdict": "approved", "tool_call_id": ...}`. +A transition without resolution leaves `data` untouched. Run the API formatters and tests. + +### 3c. Frontend: the hydration overlay + +Files: `web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.ts`, plus a unit +test colocated per the web testing conventions, and optionally +`web/oss/src/components/AgentChatSlice/components/Inspector/timeline.ts` (add +`interaction_response` to `TimelineEventType`, `EVENT_META`, and `KNOWN` so the Inspector +shows it as a first-class row instead of an "other" chip). + +Contract: + +- Add `case "interaction_response"` to the switch (line 87). Resolve the target tool part + with the same id-resolution the request case uses (lines 151-153): prefer + `payload.toolCallId`, fall back to a part whose `approval.id` equals the event's `id`. + When the part's `state` is `"approval-requested"`, set `state = "approval-responded"` + and `approval = {id: , approved: }`. When the part has + already advanced past the request state (for example the executed call's + `output-available` overwrote it), do nothing; execution states supersede answer states. +- The produced shape must be byte-identical in structure to what the live + `addToolApprovalResponse` produces, so the dock, the activity list, and the resume + predicate treat rebuilt and live states identically (research.md R2). +- The rebuilt answered card must not re-trigger any dispatch: dispatch fires only from live + clicks (the `liveGateInteractionRef` marker in `AgentConversation.tsx:1034-1040` and the + restored-tail guard at lines 1011-1016 stay authoritative). Add nothing that sends + network requests from hydration. + +Acceptance: a unit test feeds `transcriptToMessages` a record list containing a +`tool_call`, its `interaction_request`, and an `interaction_response` with +`approved: true`, and asserts the resulting part is `approval-responded` with +`approval.approved === true`; the negative case (no response record) stays +`approval-requested`; a response arriving after the call's `tool_result` leaves the +executed state untouched. Run `pnpm lint-fix` in `web` and the affected unit tests. + +## Step 4: per-card dispatch and partial answer sets + +Purpose: after this step, one click sends one answer. The frontend dispatches an approval +the moment the user answers a card, without waiting for sibling cards, and the runner +accepts a resume that answers a subset of the parked gates: it answers those, streams +their real results, and parks again on the rest. This removes incident defect 1's +dispatch half (the rebuild half fell to step 3). + +Lane: `plan/concurrent-approvals` (PR #5382). + +### 4a. Runner: accept a partial answer set + +Files: `services/runner/src/server.ts`, +`services/runner/src/engines/sandbox_agent/run-turn.ts`, +`services/runner/src/engines/sandbox_agent/runtime-contracts.ts`, +`services/runner/tests/unit/session-keepalive-approval.test.ts`. + +Contract (the mechanics are in research.md R4): + +- Dispatch (`server.ts:663-740`): building `resumeDecisions`, a gate without a matching + decision is no longer a mismatch. Split the parked set into `answered` (decisions built + as today) and `carriedForward` (the untouched `ParkedApproval` records). Resume live + when `answered` is non-empty; keep every other mismatch (unrecognized gate type, edited + history, expired mount) and the zero-answers case exactly as today (evict to cold). + Pass both sets to the engine: + `opts.resume = {decisions, carriedForward}` (extend `RunTurnOptions` in + `runtime-contracts.ts:141-167`). +- `inBandAnswerTokens` (`server.ts:849-872`): spare from the stale-interaction sweep the + tokens of the ANSWERED gates only. Carried-forward gates stay pending on the + interactions plane, and they now survive because the resume re-parks them (the comment + block there describing the all-or-cold rule must be rewritten to match the new + behavior). +- Resume turn (`run-turn.ts`): the turn-start clear (lines 88-95) currently empties + `env.parkedApprovals`. On a resume with carried-forward gates, re-seed the map with them + after the clear, restore `env.approvalGateCount` to the map's size, and call + `pause.markPausedToolCall(gate.toolCallId)` for each so their frames stay suppressed + (the harness will not re-raise these gates; they are still pending inside the live + process). After the answer loop (lines 448-483) finishes and carried-forward gates + remain, arm the pause (`pause.pause()`): the turn then ends parked once the answered + calls' results have landed (the step 2 bounded wait governs that), and the existing + re-park path (`reparkOrEvict` via `approvalToPark`, `server.ts:423-451, 517-559`) + re-parks in `awaiting_approval` with a fresh approval TTL, and `watchParkedPrompt` + re-attaches to the shared prompt promise (safe: identity-checked and idempotent). +- A new gate raised DURING the resume (the incident's exact shape: Pi serializes confirms, + so gate 2 surfaces only after gate 1 is answered) needs no new handling: it fires + `onUserApprovalGate` normally, joins `env.parkedApprovals` next to the carried-forward + records, and pauses the turn itself. +- Verify the history fingerprint is insensitive to which approval envelopes ride the + request (research.md open risk 1). Read `historyFingerprint` in + `session-identity.ts` first; if it hashes tool-result blocks, adjust the park-side + expected fingerprint so a partial answer still matches, and pin it in the dispatch + test. + +### 4b. Frontend: dispatch per card + +Files: `web/packages/agenta-playground/src/state/execution/agentApprovalResume.ts`, +`web/packages/agenta-playground/tests/unit/agentApprovalResume.test.ts`, and +`web/oss/src/components/AgentChatSlice/AgentConversation.tsx` only if the guard wiring +needs it. + +Contract: + +- `agentShouldResumeAfterApproval` (`agentApprovalResume.ts:131-165`) currently requires + EVERY tool part settled (`allSettled`, line 163). Replace that final condition: resume + when there is at least one freshly answered approval (the existing + "last freshly-resolved parked interaction" detection at lines 146-150) that no + `step-start` part follows (the existing already-resumed guard at lines 158-161), even + if sibling cards are still `approval-requested`. Client-tool results keep their + existing all-settled requirement if relaxing it is unsafe for them; this project's + scope is approval cards. +- The AI SDK evaluates this predicate on message updates and does not send while a stream + is in flight, so an answer clicked during a streaming resume dispatches when that + stream finishes. State this in a comment; it is the concurrency contract. +- The restored-tail guard stays: a rebuilt conversation whose answers came from + `interaction_response` records must not auto-fire (step 3c). Only a live click flips + `liveGateInteractionRef` and produces a "freshly" answered part. +- "Approve all" in the dock (`ApprovalDock.tsx:160-164`) loops the responses + synchronously; the predicate then fires once with every card answered, which the runner + handles as a full set. No dock change. + +### Acceptance criteria for step 4 + +- Dispatch-seam test (rewrite `session-keepalive-approval.test.ts:579`): a two-gate park + answered one card resumes live (never cold), answers exactly that gate's + `permissionId`, re-parks in `awaiting_approval` with the second gate carried forward, + and a second request answering the second card resumes live again and completes the + turn. A request answering zero gates still evicts to cold. +- Engine-seam test: a resume with one answered and one carried-forward gate marks the + carried-forward call paused (its frames suppressed), ends parked, and never settles it + with any sentinel. +- Frontend unit test: the predicate fires with one answered card and one pending card; + does not fire when a `step-start` follows the answer; does not fire on a rebuilt + conversation with no live click. +- Test commands: `cd services/runner && pnpm test && pnpm run typecheck`; web package + tests per `web/AGENTS.md` for `agenta-playground`; `pnpm lint-fix` in `web`. + +## Step 5: the incident regression test, then live QA + +Purpose: pin the exact incident shape end to end, then verify the deployed behavior by +hand. + +Lane: `plan/concurrent-approvals` (PR #5382) for the tests; QA is not a code change. + +### 5a. The incident regression test + +File: `services/runner/tests/unit/session-keepalive-approval.test.ts` (engine seam, the +pausable fake harness of the describe block at line 1260, extended), or a new sibling +file if it grows large. + +Script the exact db58551b shape: + +1. Turn 1: the fake harness announces two ask-policy calls `tool-a` and `tool-b` + (`sessionUpdate: "tool_call"` events), raises the permission request for `tool-a` + ONLY, and hangs the prompt. Expect: one `interaction_request` for `tool-a`; `tool-b` + is settled with `TOOL_NOT_EXECUTED_PAUSED` (announced, never gated, never started; it + holds no approval, so the sweep may settle it); the turn parks with one gate. +2. If the fake emits a cancellation-closure `completed` frame for `tool-b` during the + park (add this to the script), expect the deferred sentinel, NOT a success record. + This pins defect 3. +3. Resume 1 (approve `tool-a`): the fake answers the `respondPermission`, emits an + `in_progress` frame for `tool-a`, then raises the permission request for `tool-b` + (the gate that surfaces DURING the warm resume), then emits `tool-a`'s real + `completed` frame with distinctive output. Expect: `tool-a`'s real result recorded + exactly once, no sentinel ever attached to it (defect 2 pinned); an + `interaction_request` card for `tool-b`; an `interaction_response` event for `tool-a` + with `approved: true` (step 3 pinned); the turn re-parks on `tool-b`. +4. Resume 2 (approve `tool-b`): the fake answers, emits `tool-b`'s real `completed` + frame, and resolves the held prompt. Expect: `tool-b`'s real result exactly once, its + `interaction_response`, and a completed turn. +5. Global assertions: each permission id received exactly one `respondPermission` (the + "both side effects exactly once" proxy at this seam), and the final event log contains + exactly one real result per call and no success record for any never-started state. + +### 5b. The records and hydration regression test + +As specified in step 3c's acceptance criteria, plus one round-trip-shaped case: build the +record list in the exact order the runner persists during the incident shape (user +message, tool_call a, interaction_request a, tool_result b deferred, interaction_response +a, tool_result a real, interaction_request b) and assert the rebuilt messages show call a +executed with its real output, call b's card pending, and no card flipped back to +waiting. This is the state-rebuild half of the incident. + +### 5c. Live QA + +Run the script in `qa.md` against the dev stack after all lanes are deployed, record the +MP4, and post it as a PR comment on #5382. Then re-run the release-gate approval cells +listed there. + +## Landing order and dependencies + +The steps are ordered by user-facing severity and by dependency: + +1. Step 1 is independent of everything else and unblocks JP's PR stack; land first. +2. Step 2 is self-contained in the runner and makes results truthful; step 4's partial + resume DEPENDS on its bounded wait for the "answered call finishes before the re-park" + behavior, so land 2 before 4. +3. Step 3 is independent of 2 (different regions) and is what makes step 4's frontend + guard sound after a rebuild; land 3 before 4. +4. Step 4 last among the code steps, then step 5's tests can pin the full behavior (its + sub-assertions on sentinels and answer events need 2 and 3 in place). + +Each step is a separate commit (or small commit series) on its lane so review can bisect. diff --git a/docs/design/agent-workflows/projects/approvals-incident-fixes/qa.md b/docs/design/agent-workflows/projects/approvals-incident-fixes/qa.md new file mode 100644 index 0000000000..c6fd0f4ab3 --- /dev/null +++ b/docs/design/agent-workflows/projects/approvals-incident-fixes/qa.md @@ -0,0 +1,96 @@ +# Live QA script + +This script reproduces the shape of incident session db58551b on the dev stack and states +the correct behavior at every point. Run it after all five plan steps are deployed. The +run MUST produce a watchable MP4 screen recording (ffmpeg is available in `~/.local/bin` +on the dev box workflow), uploaded as a comment on PR #5382, and the recording is listed +first in the QA report. + +## Setup + +1. Deploy the branch stack to the local EE dev deployment (see the `debug-local-deployment` + skill for the box, ports, and login; see the root `AGENTS.md` for the + `load-env` + `run.sh --ee --dev` pairing). Confirm the runner container runs the + `plan/concurrent-approvals` code and the API runs the `sessions-rebase/backend` code. +2. If step 1c's wipe has not been run yet, run it now (plan.md step 1c), before any QA + turn, so turn indexes start clean. +3. Open the playground's agent chat with the Pi harness (`pi_core`), local sandbox, and a + permission policy that gates `Bash` with `ask` (the default ask policy used in the + incident). +4. Start the screen recording before the first prompt. + +## Scenario A: two parallel gated writes (the incident shape) + +Prompt: + +> Append the line "hello from QA" to agent-files/README.md and to agent-files/NOTES.md, +> as two separate Bash commands issued in parallel in the same turn. + +Walk through and check each point: + +1. **First card appears.** The model issues two Bash calls; Pi serializes confirms, so one + approval card appears first. Correct behavior: the second call shows as a pending tool + part or a deferred sibling, and NOTHING shows a successful "(no output)" result for a + command that has not run (defect 3). Open the Inspector's record timeline and confirm + the second call has no success `tool_result` row. +2. **Approve card 1.** Correct behavior: the approved command executes, its REAL output + (or a clean completed state) appears on the card, and the card stays in its approved + or executed state permanently. It must never flip back to "waiting for approval" + (defect 4/1), and it must never show the text "DEFERRED_NOT_EXECUTED" (defect 2). +3. **Second card appears** during or right after the first command's execution (it + surfaces on the warm resume). Correct behavior: card 1's state is unaffected. +4. **Approve card 2 and do nothing else.** This is the click that died in the incident. + Correct behavior: the approval dispatches on its own (watch the network tab for the + message request; no extra text message is needed), the second command executes, and + the assistant completes the turn. +5. **Verify the files.** In the session's workspace, each file contains the appended line + EXACTLY once. Two lines in one file or a missing line fails the run. +6. **Verify the records.** In the Inspector: two `interaction_request` rows and two + `interaction_response` rows (one per gate), and one truthful `tool_result` per call. +7. **Verify the turns.** On the dev database: + `SELECT turn_index, harness_kind, created_at FROM session_turns WHERE session_id = '' ORDER BY turn_index;` + Indexes are 0, 1, 2, ... with no gaps and no duplicates, and the API access log shows + no `POST /api/sessions/turns/ ... 500` lines for the session. + +## Scenario B: rebuild while a gate is pending + +1. Repeat the Scenario A prompt in a fresh session. Approve card 1, wait for card 2 to + appear, then RELOAD the page before answering it. +2. Correct behavior after reload: card 1 renders as answered/executed (hydrated from its + `interaction_response` record); card 2 renders as pending. No answered card is + resurrected as waiting. +3. Approve card 2 after the reload. Correct behavior: the approval dispatches and the turn + completes, exactly as without the reload. This was impossible before the fix (the + all-settled condition could never hold after a rebuild). +4. Open the same session in a second browser window and confirm the same rendering. + +## Scenario C: partial answers, one at a time + +1. To get two cards genuinely outstanding at once, use the Claude harness (`claude`) with + two ask-gated calls in one turn (Claude raises concurrent permission requests; Pi + serializes them, issue #5391). Prompt for two parallel gated writes as in Scenario A. +2. Answer ONLY the first card. Correct behavior: the answer dispatches immediately, the + first command executes and reports truthfully, and the second card remains pending + (the session re-parks; it does not degrade to a cold restart and does not cancel the + second gate). +3. Answer the second card. Correct behavior: the turn completes; both side effects exactly + once. + +## Known limitation to note in the report (not a gate) + +Sending a NEW text message while a card is pending still routes down the approval-resume +or eviction path and can consume the message into the stale task (incident defect 6). +Real cancel-then-restart on new user text is explicitly out of scope here (see +context.md); note the observed behavior in the report rather than failing the run on it. + +## Release-gate cells to re-run + +After the scenarios pass, run the `agent-release-gate` skill against the same deployment +and re-run at least: + +- the human-approval park and resume cells for `pi_core` on the local sandbox, +- the human-approval cells for `claude` on the local sandbox, +- the deny-path cell (a rejected gate renders as a decline, not an error), +- one full smoke cell per harness to catch regressions outside the approval path. + +Attach the gate output to the PR comment along with the MP4. diff --git a/docs/design/agent-workflows/projects/approvals-incident-fixes/research.md b/docs/design/agent-workflows/projects/approvals-incident-fixes/research.md new file mode 100644 index 0000000000..b1b5a5b0dc --- /dev/null +++ b/docs/design/agent-workflows/projects/approvals-incident-fixes/research.md @@ -0,0 +1,484 @@ +# Research findings (R1 through R8) + +Every claim below was verified against the working tree on 2026-07-19 unless marked +otherwise. Paths are repo-relative. Line numbers refer to the current workspace working +tree, which has both PR stacks applied; where a file differs on an individual branch, the +difference is called out. Open risks are collected at the end and also flagged inline. + +## R1. The records ingest path for a new event type + +Question: can a new `interaction_response` event flow from the runner into the durable +record store and back out to the frontend without any hop rejecting it? + +Answer: yes, with exactly two required code changes (one runner type, one frontend switch +case) and one recommended runner change (a stable record id). No API change is needed. + +The path, hop by hop: + +1. **Runner event type.** The runner's event union `AgentEvent` + (`services/runner/src/protocol.ts:325-383`) has an `interaction_request` member + (lines 362-367) and no answer member. A new member must be added here; TypeScript narrows + on `type`, so nothing else in the runner needs it. The cross-language wire contract does + NOT pin event union members: the golden fixtures pin request and result top-level keys + only (`services/runner/tests/unit/wire-contract.test.ts`), and the Python mirror's event + model is deliberately open ("keeps the whole event verbatim and drops a typeless event", + `sdks/python/agenta/sdk/agents/wire_models.py:370-384`). The docstring at + `wire_models.py:381` lists the known types for readers; add `interaction_response` to + that list when implementing (documentation only, not enforcement). +2. **Runner emission choke point.** Events emitted via `run.emitEvent` go through the single + `record()` choke point (`services/runner/src/tracing/otel.ts:1141-1150`), which appends to + the batch log and forwards to the live sink. The sink on session-owned runs is the + persisting emitter. +3. **Runner persistence.** `buildPersistingEmitter` + (`services/runner/src/sessions/persist.ts:160-352`) posts every event to + `POST /sessions/records/ingest` with `record_type: event.type` and the whole event as + `attributes` (`persist.ts:59-73`). An unknown type takes the generic persist branch at + `persist.ts:316-326` untouched. One change is recommended: the stable-id branch at + `persist.ts:297-313` gives `tool_result` and `interaction_request` a deterministic uuid5 + record id (`stableRecordId`, `services/runner/src/sessions/record-id.ts:40-47`, keyed on + session id, event id, and record type) so a re-sent event upserts one row instead of + appending duplicates. `interaction_response` should be added to that branch so a retried + resume cannot double-record an answer. Note the id is keyed by record type, so a request + and its answer land on two distinct rows even though they share the event id. +4. **Live stream egress (not on the persistence path, but the same event reaches it).** The + Python SDK's Vercel egress projects known event types through an `elif` ladder with no + `else` branch (`sdks/python/agenta/sdk/agents/adapters/vercel/stream.py:156-312` and + `436-592`); an unknown type is silently skipped. So emitting `interaction_response` on + the live stream is harmless: the live UI already knows the answer (the user just clicked + it) and needs no projection. +5. **API ingest.** `POST /sessions/records/ingest` + (`api/oss/src/apis/fastapi/sessions/router.py:506-536`) validates + `SessionRecordIngestRequest` (`api/oss/src/apis/fastapi/sessions/models.py:198-211`), + where `record_type` is `Optional[str]` and `attributes` is a free dict. It publishes to a + Redis stream (`api/oss/src/core/sessions/records/streaming.py:51-97`, XADD at line 89); + the records worker consumes it and calls `RecordsService.append_many` + (`api/oss/src/tasks/asyncio/sessions/records_worker.py:149`). The DAO upserts on the + composite primary key `(project_id, record_id)` + (`api/oss/src/dbs/postgres/sessions/records/dao.py:86-97`); a client-supplied stable id + is honored, otherwise a uuid4 is minted + (`api/oss/src/dbs/postgres/sessions/records/mappings.py:17-28`). There is no enum, + Literal, or DB constraint on the event type anywhere on this path: `record_type` is a + nullable String column (`api/oss/src/dbs/postgres/sessions/records/dbas.py:55-58`) and + `attributes` is unconstrained JSONB (`dbas.py:64-67`). The rows live in the `records` + table in the tracing database (`api/oss/src/dbs/postgres/sessions/records/dbes.py:15`). +6. **API query.** The frontend hydrates from `POST /sessions/records/query` + (`router.py:464-485`), which filters by project and session and orders by + `created_at ASC, record_index ASC` (`records/dao.py:99-116`). No type filter. +7. **Frontend fetch and validation.** `querySessionRecords` + (`web/packages/agenta-entities/src/session/api/api.ts:56-80`) calls the generated Fern + client and validates with `sessionRecordsQueryResponseSchema` + (`web/packages/agenta-entities/src/session/core/schema.ts:19-45`), where + `record_type: z.string().nullish()` and `attributes` is an open record. An unknown event + type passes validation unchanged. +8. **Frontend hydration.** `loadSession` + (`web/oss/src/components/AgentChatSlice/assets/loadSession.ts:29-35`) passes rows + straight to `transcriptToMessages` + (`web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.ts`), whose + `switch (type)` (line 87) has a silent `default: return` (lines 208-210). **This is the + one place an `interaction_response` record is dropped today and the one required frontend + change.** + +Sweep of every other frontend narrowing over event types (none blocks the new event): + +- `web/oss/src/components/AgentChatSlice/components/Inspector/timeline.ts:10-52`: a `KNOWN` + allowlist maps unknown types to an `"other"` chip. Cosmetic only; adding + `interaction_response` to `TimelineEventType`, `EVENT_META`, and `KNOWN` is optional. +- `Inspector/EventRow.tsx`, `Inspector/lenses/TimelineLens.tsx:26-27,114`, + `Inspector/lenses/ContextLens.tsx:50-56`: if-chains that ignore unmatched types. No crash. +- `web/oss/src/components/AgentChatSlice/assets/AgentChatTransport.ts:74`: normalizes batch + response blocks, not session records; unknown parts degrade to text. Not on this path. +- `web/packages/agenta-playground/`: zero matches for record event type literals. The + playground operates on assembled message parts, not on record events. + +No location was found where an unknown event type causes an error on any hop. + +## R2. The frontend hydration overlay design + +Question: where must answered state be injected so a rebuilt approval card renders as +answered, and how do live-session local state and rebuilt state converge? + +How the two states work today: + +- **Live state** is the AI SDK's `useChat` message array. `addToolApprovalResponse` + (destructured at `web/oss/src/components/AgentChatSlice/AgentConversation.tsx:575`, + wrapped at lines 1034-1040) mutates the matching tool part in memory: `state` goes from + `"approval-requested"` to `"approval-responded"` and `approval.approved` is set. Nothing + persists this. The wrapper also sets `liveGateInteractionRef.current = true`, the marker + that distinguishes a live click from restored state. +- **Rebuilt state** comes from `transcriptToMessages`, which constructs the card from + `interaction_request` only (`transcriptToMessages.ts:144-176`): it finds or synthesizes + the tool part for the gated call and stamps `state = "approval-requested"` and + `approval = {id}` (lines 171-174). There are no answer records, so every rebuilt card is + pending. This is the root of incident defect 4. + +The decided shape: hydration must produce the SAME part shape the live path produces, so the +two states converge on identical data and every downstream consumer (dock, activity list, +resume predicate) works unchanged. + +- The new event: `{type: "interaction_response", id: , kind: + "user_approval", payload: {toolCallId, approved: boolean}}`. The `id` equals the matching + `interaction_request`'s `id` (the interaction token minted at + `services/runner/src/engines/sandbox_agent/acp-interactions.ts:587-589`), which is the + linkage key. Field classification per the design-interfaces method: `approved` is the + data (the verdict itself); `toolCallId` and `id` are protocol context (correlation); + the record envelope's timestamp and credential-derived author are metadata and ride the + record row, not the payload. No actor field is added; recording who approved is part of + the queued audit-hardening work. +- The hydration change: a new `case "interaction_response"` in `transcriptToMessages.ts` + that resolves the same tool part (by `payload.toolCallId`, falling back to matching + `approval.id`) and, when the part is in `"approval-requested"`, sets + `state = "approval-responded"` and `approval = {id, approved}`. Exactly the shape + `addToolApprovalResponse` produces. +- Convergence rule: answered-by-record and answered-by-click are indistinguishable by shape. + The dispatch trigger is the live click (the `liveGateInteractionRef` marker and the + restored-tail guard at `AgentConversation.tsx:1011-1016`), never the state shape, so a + rebuilt already-answered turn cannot re-fire a resume. A card that was answered locally + but whose answer never reached the runner has no `interaction_response` record, so a + rebuild correctly shows it pending again and the user can answer again; that is the + correct recovery, since the decision existed only in lost browser memory. + +The card state literals are inline AI SDK `ToolUIPart["state"]` strings, not a local union: +`"approval-requested"` is set at `transcriptToMessages.ts:172` and consumed in +`ApprovalDock.tsx:33-45`, `ToolActivity.tsx`, and `agentApprovalResume.ts`; +`"approval-responded"` is set by the SDK and consumed in the same files. + +## R3. The interactions API verdict field + +Question: how does the allow/deny verdict get onto the interaction row? + +Today's state, verified: + +- `SessionInteractionData` (`api/oss/src/core/sessions/interactions/dtos.py:24-28`) already + has `resolution: Optional[Dict[str, Any]]` (line 28). A repo grep shows nothing ever + writes it. +- `SessionInteractionTransition` (`dtos.py:66-70`) carries only `project_id`, `session_id`, + `token`, `status`. The transition endpoint (`router.py:625-655`) passes it to the DAO, + whose UPDATE sets only `status` and `updated_at`, guarded on + `status IN ('pending','responded')` + (`api/oss/src/dbs/postgres/sessions/interactions/dao.py:91-119`, SET at 108-111). +- The status enum (`dtos.py:16-21`) is lifecycle only: `pending`, `responded`, `resolved`, + `cancelled`. The comment at line 17 states the verdict does not belong in it. That + matches the design-interfaces classification: `status` is lifecycle metadata, + `resolution` is data (the answer content), `token` and `session_id` are protocol context. +- The `data` column is JSONB and round-trips through + `SessionInteractionData.model_validate(dbe.data)` + (`.../interactions/mappings.py:34-36, 65`), so storing a verdict inside + `data.resolution` needs no migration. + +The specified addition (details in plan.md step 3): `SessionInteractionTransition` gains +`resolution: Optional[Dict[str, Any]]`; the transition endpoint and DAO write it into the +row's `data.resolution` when present; the runner's `resolveInteraction` +(`services/runner/src/sessions/interactions.ts:97-119`) gains the verdict payload +`{verdict: "approved" | "denied", tool_call_id}` and its callers pass the decision through +(the warm-resume loop at `services/runner/src/engines/sandbox_agent/run-turn.ts:479` and +the stored-decision path via `onResolveInteraction`, +`acp-interactions.ts:223-226` and `228-252`, where `replyPermission` already holds the +decision). + +## R4. The partial-resume change + +Question: what exactly must change so a resume request that answers only some parked gates +is accepted? + +Today's all-or-cold rule, verified: + +- The dispatch's approval branch (`services/runner/src/server.ts:663-740`) builds one resume + decision per parked gate from the request + (`approvalDecisionForToolCall`, + `services/runner/src/engines/sandbox_agent/session-identity.ts:274-291`, strict tool-call + id match on the `{approved}` envelope). Any gate without a matching decision sets + `mismatch = "no-matching-approval"` (line 707) and the whole parked session is evicted to + cold (lines 730-740). +- The stale-interaction sweep helper mirrors the same rule + (`inBandAnswerTokens`, `server.ts:849-872`). +- The warm-resume turn answers every decision in a loop + (`run-turn.ts:439-483`): seeds the trace with the parked call, grants execution for an + approve, marks a deny, calls `respondPermission` per gate, and resolves each interaction + row. All decisions share one held prompt promise (one prompt per turn). +- Per-turn park bookkeeping resets at every turn start (`run-turn.ts:88-95` clears + `env.parkedApprovals`), and the re-park only happens when the pause controller fires again + this turn (`run-turn.ts:516-520`, `server.ts:494-505` and `529-545` via `approvalToPark`, + `server.ts:423-451`). + +What "accept a partial answer set" must mean, given those mechanics: + +1. **Dispatch**: a request that answers a non-empty subset of the parked gates resumes live + with that subset. A request that answers none keeps today's behavior (evict to cold); + changing that is the out-of-scope defect 6 work. +2. **Carried-forward gates**: the unanswered gates' `ParkedApproval` records + (`runtime-contracts.ts:112-127`) must survive into the resume turn. The turn-start clear + must not drop them, because the harness will NOT re-raise those permission requests: they + are still pending inside the live process. The resume turn must re-mark their tool-call + ids as paused (`pause.markPausedToolCall`, + `services/runner/src/engines/sandbox_agent/pause.ts:54-57`) so their frames stay + suppressed, and must arm the pause so the turn ends parked again after the answered + calls' results land (the step 2 bounded wait governs when). +3. **Response stream**: the resume turn streams the answered calls' seeded `tool_call` + frames, their real execution results, and then ends with `stopReason: "paused"`; the + still-parked cards stay pending on the client. No eviction to cold happens. +4. **Re-park**: `reparkOrEvict` re-parks with state `awaiting_approval` because + `env.parkedApprovals` is non-empty and the pause is active; the approval TTL re-arms + fresh (default 5 minutes, `session-pool.ts` arms it in `park`/`repark`, + `session-pool.ts:147-232`), and `watchParkedPrompt` (`server.ts:461-478`) re-attaches to + the same held prompt promise; its catch-based eviction is identity-checked and idempotent, + so re-attachment is safe. +5. **Races**: `checkoutApproval` removes the entry from the local pool + (`session-pool.ts:126-134`), so a second answer arriving while a resume is in flight + misses the pool and runs cold; the cold decision-map path re-raises unanswered gates and + consumes carried envelopes, which is the safe degradation (pinned by the existing test + "a second identical approval while the first resume is in flight", + `tests/unit/session-keepalive-approval.test.ts:930`). The frontend dispatch (step 4) + only fires when the chat transport is idle, so this race needs a second browser to occur. + +One existing test pins the CURRENT all-or-cold behavior and must be rewritten by this +change: "keeps a partly-answered two-gate turn paused (only one card answered -> cold)" +(`tests/unit/session-keepalive-approval.test.ts:579`). + +**Open question resolved during research**: the history-fingerprint check +(`server.ts:721-728`) also gates the resume. Today's full-answer resumes pass it, and a +partial-answer request differs from a full-answer request only in which `{approved}` +tool-result envelopes ride the tail. The fingerprint folds emitted tool-call ids +(`session-identity.ts:240-251`); whether it also hashes tool-result blocks was NOT +conclusively verified from `historyFingerprint`'s implementation. This is flagged as an +open risk: the implementer must confirm (or make) the fingerprint insensitive to the +presence or absence of approval envelopes, or partial resumes will spuriously evict to +cold. The dispatch-seam unit test in step 4 pins this. + +## R5. The sweep replacement + +Question: which open tool calls may the post-pause sweep still settle, which must it never +touch, and how is a cancellation-closure frame for a never-started call recorded? + +The machinery, verified: + +- The sentinel: `TOOL_NOT_EXECUTED_PAUSED` (`services/runner/src/tracing/otel.ts:66`) is + `"DEFERRED_NOT_EXECUTED: paused for another approval; retry the same call if still + required."` The prefix is machine-read in two places: the responder keeps deferred + results out of the client-output store (`responder.ts:491-496`, consumed at + `responder.ts:398-401`), and the web renders deferred siblings distinctly. +- The sweep: `settleOpenToolCalls` (`otel.ts:1479-1492`) closes every tracked open call not + excluded by the predicate and records `tool_result {isError: true}` with the sentinel. + Two call sites, both excluding ONLY paused gates (`pause.isPausedToolCall`): the in-band + re-sweep on every non-suppressed frame while paused (`run-turn.ts:232-237`) and the + post-drain sweep (`run-turn.ts:505-511`). +- The result mapping: `maybeCloseTool` (`otel.ts:1446-1473`) records ANY + `completed`/`failed` status frame as the call's result, with + `isError: status === "failed"`. It has no notion of whether execution ever started. This + is how the incident's never-started call got a successful `"(no output)"` result. +- Frame suppression during a pause (`shouldSuppressPausedToolCallUpdate`, + `services/runner/src/engines/sandbox_agent/runtime-policy.ts:31-60`): paused-gate frames + are dropped (line 44), `failed` frames for any other call are dropped as managed-cancel + artifacts (lines 56-58), `completed` frames pass deliberately so a legitimately finishing + auto-allowed sibling keeps its real result (comment at lines 48-55). +- Late events: once the turn's sink is cleared, between-turn events are logged and dropped + (`services/runner/src/engines/sandbox_agent/session-events.ts:31-35`). This is what + destroyed the incident's approved call's real result: it landed after the park. +- The drain: `pause.waitForEventDrain` is one `setImmediate` after the destroy callback + settles (`pause.ts:27-47`), so "post-drain" is a single event-loop tick, not a bounded + wait for results. +- The per-turn approval ledger that exists today is name-and-args keyed + (`ApprovedExecutionGrants`, `responder.ts:85-105`), built for the relay execution guard, + not id-keyed sweep protection. The warm-resume loop knows each approved gate's tool-call + id (`run-turn.ts:448-483`), and the in-turn allow path knows it too + (`replyPermission`, `acp-interactions.ts:228-252`, which receives the `toolCallId`). + +The specified contract (mechanics in plan.md step 2): + +- **The sweep may still settle**: a call that was announced but never started and holds no + approval this turn; concretely, a call that is neither a paused gate nor in the new + approved-or-allowed id set. For such a call the deferred sentinel is correct: it never + executed, and inviting a retry is safe. +- **The sweep must never touch**: a call whose gate was answered allow this turn (warm + resume or in-turn), or whose policy auto-allowed it. Terminalization of a paused turn + must wait for those calls' own terminal frames, bounded per call by the existing + per-tool-call time limit (the run-limits deadlines are retired at pause by + `runLimits.notePaused()`, `run-turn.ts:188`, so this wait needs its own timer using the + same configured value from `run-limits.ts`). If the bound expires, the call is settled + with a NEW sentinel that must NOT invite a retry (execution started; a retry could double + a side effect), and that sentinel must also be excluded from the client-output store the + way the deferred one is (`responder.ts:398-401`). +- **Cancellation-closure detection**: while the pause is active, a `completed` frame for a + call that required a gate (effective permission `ask` or `deny`) that was never answered + allow CANNOT be a genuine completion, because both harness paths fail closed (Pi's + in-process confirm allows execution only on an explicit `true`, + `services/runner/src/extensions/agenta.ts`; Claude asks before executing). Such a frame + is a cancellation-closure artifact and must be recorded as the deferred sentinel, not as + success. A `completed` frame for an allow-policy call, or for a call in the + approved-or-allowed set, keeps its real result. Buffering `completed` frames for + unclassified calls until the drain settles (as the incident report sketches) makes the + classification race-free against a gate that arrives in the same tick. +- **The model-transcript requirement**: every tool call the model ever sees must eventually + carry SOME result. The design satisfies it on all three continuation paths. On a warm + resume and on a cold native continue, the harness owns its own transcript and the + runner's records never feed the model, so nothing is lost by holding the runner-side + record open briefly. On the replay fallback (the only path that rebuilds the prompt from + our records), every announced call ends the turn with exactly one of: its real result, + the deferred sentinel (never executed), or the executed-but-unreported sentinel (bounded + wait expired). The bounded wait plus the sweep guarantee the turn cannot end with an + open call. + +Tests that pin the current behavior and bound the change: + +- `tests/unit/responder.test.ts:679` pins that a deferred sibling result never enters the + client-output store (must keep passing; extend for the new sentinel). +- `tests/unit/sandbox-agent-orchestration.test.ts:1753` pins that two racing ask gates each + emit a card and neither is force-settled (must keep passing). +- The test around `sandbox-agent-orchestration.test.ts:1690` pins that a non-gated sibling + IS settled with the deferred sentinel (still correct under the new contract: that call + held no approval). + +## R6. The turn-counter fix + +Question: where does the per-turn computation live, how does it behave with two pooled +environments serving one session, and how is the API-side conflict mapped? + +The bug, verified end to end: + +- `acquireEnvironment` computes `environment.continuityTurnIndex = nextTurnIndex(...)` + exactly once, at acquire time + (`services/runner/src/engines/sandbox_agent/environment.ts:962-964`). +- `nextTurnIndex` reads the process-wide `SessionContinuityStore` singleton: latest + recorded turn plus one, or 0 for a fresh session + (`services/runner/src/engines/sandbox_agent/session-continuity.ts:107-112`, store at + 24-99, singleton at 99). +- Warm turns bypass acquire entirely: the `hit-continue` branch calls + `engine.runTurn(live.environment, ...)` directly (`server.ts:608-623`), and the approval + resume branch does the same (`server.ts:742-766`), so the frozen index is re-used. +- At the end of every completed turn, `runTurn` records that index into the store and + fires the durable append with it (`run-turn.ts:566-597`); `appendSessionTurn` POSTs + `POST /sessions/turns/` (`session-continuity-durable.ts:157-198`). +- The API's `SessionTurnsDAO.append` is a bare add-and-commit with no IntegrityError + handling (`api/oss/src/dbs/postgres/sessions/turns/dao.py:33-50`; the file does not even + import `IntegrityError`), so the unique index + `ix_session_turns_project_id_session_id_turn_index` + (`api/oss/src/dbs/postgres/sessions/turns/dbes.py:37-43`, created by migration + `oss000000014_add_session_turns.py:70-75`) surfaces as a 500 through + `intercept_exceptions`. + +Why moving the computation to turn start is correct, including the two-environment case: + +- The store advances only on a successful `record()` (`session-continuity.ts:47-59`), and a + paused turn deliberately never records (`run-turn.ts:567` guards on + `stopReason !== "paused"`). So with a per-turn read, a park-and-resume cycle that spans + several runner turns still consumes ONE conversation index, recorded once by the turn + that completes. That is exactly the "true conversation-turn counter" semantics. +- Two pooled environments for one session (the `poolSize=2` shape from the 500 report: one + approval-parked, one idle) both read the SAME process-wide store at their next turn + start, so indexes stay monotonic per session, not per environment. The read-then-record + window is not concurrent in practice: one Node process, and the local provider forbids a + second replica serving the same session (`LocalSandboxNotOwnerError`, + `session-continuity.ts:157-210`). A true cross-replica collision on a remote provider + would now surface as a 409 the runner logs and drops, which is the accepted answer to the + 500 report's open retry question (retry is over-engineering today). +- Runner restarts stay correct: `acquireEnvironment` hydrates the store from the durable + rows before the first turn (`environment.ts:941-949`, + `hydrateHarnessSessionFromDurable`, `session-continuity-durable.ts:104-148`), and warm + turns keep using the live in-process store. + +The API-side mapping has an established in-repo pattern to copy: catch +`sqlalchemy.exc.IntegrityError`, match the constraint name, raise `EntityCreationConflict` +(`api/oss/src/core/shared/exceptions.py:4-25`), which `intercept_exceptions` converts to a +409 `ConflictException` (`api/oss/src/utils/exceptions.py:132-144`, default code 409 at +246-248). Copy targets: `api/oss/src/dbs/postgres/gateway/connections/dao.py:70-83`, +`api/oss/src/dbs/postgres/triggers/dao.py:84-108`, and the sibling +`api/oss/src/dbs/postgres/sessions/streams/dao.py:51-61`. The route is already wrapped in +`@intercept_exceptions()` (`router.py:1070`). + +The dev-database wipe: every `session_turns` row written while the bug was live carries an +acquire-counter index, not a turn index, so the whole table on the dev stack is suspect and +is wiped rather than repaired. The exact operational step is in plan.md step 1. + +## R7. The regression test plan + +The existing orchestration harness (`tests/unit/sandbox-agent-orchestration.test.ts:35-303`) +fakes the whole stack: a scripted `session` whose `prompt()` replays `promptEvents`, raises +scripted permission requests through the registered handler, and can hang until +`destroySession` resolves it (the Claude pause shape); a fake `run` (otel) or the real +`createSandboxAgentOtel`; and a `deps` bag injected into `runSandboxAgent`/`runTurn`. The +keepalive tests add two more seams (`tests/unit/session-keepalive-approval.test.ts:1-134`): +a dispatch-level fake engine that scripts each turn's park state for `runWithKeepalive`, +and an engine-level pausable fake harness for real `runTurn` park-and-resume mechanics +(describe block at line 1260; the two-parallel-gates case at line 1537). + +What is missing, and what the new tests must pin, is the real Pi shape: gates that are NOT +all known before the first pause. The shapes are specified in plan.md step 5; in summary: + +- Engine seam: turn 1 announces two ask-policy calls, gates only the first, parks; the + resume answers gate 1, the fake then emits an in-progress frame for call 1, raises gate 2 + (a second permission request DURING the resume turn), and delivers call 1's real + `completed` frame; assertions: call 1's real result is recorded exactly once and no + deferred sentinel ever attaches to it, gate 2 emits its own card, the turn re-parks; a + second resume answers gate 2 and the held prompt completes; each gate received exactly + one `respondPermission`. +- Dispatch seam: a two-gate park answered one card at a time resumes live twice and never + degrades to cold (rewriting the all-or-cold test at + `session-keepalive-approval.test.ts:579`). +- Records and hydration: a unit test that feeds `transcriptToMessages` a record list + containing a `tool_call`, its `interaction_request`, and an `interaction_response`, and + asserts the rebuilt part is `approval-responded` with the verdict; plus the negative + case (request without response stays `approval-requested`). + +## R8. Lane and PR mechanics (documentation only) + +Branches, from `gh pr view`: + +- PR #5382 `plan/concurrent-approvals`, GitHub base `release/v0.105.6`. In the GitButler + workspace it is stacked on `feat/deny-frame-egress` (PR #5383's lane). +- PR #5375 `sessions-rebase/backend`, base `main`. +- PR #5376 `sessions-rebase/runner`, base `sessions-rebase/backend` (stacked on #5375's + lane in the workspace). + +File-overlap audit (from `gh pr diff --name-only` on all three PRs): + +- The #5382 diff and the #5376 diff ALREADY share three files today: + `services/runner/src/engines/sandbox_agent/run-turn.ts`, + `services/runner/src/engines/sandbox_agent/runtime-contracts.ts`, and + `services/runner/src/server.ts`. The two stacks are parallel (different bases), and the + working tree holds both applied; their hunks do not conflict today. +- The planned steps keep each step's hunks on one lane, but they add more shared FILES + across the two stacks: step 1 edits `run-turn.ts` (the turn-index region, lines 566-597) + on the #5376 lane while steps 2 and 4 edit `run-turn.ts` (the pause and resume regions, + lines 88-95, 163-260, 439-520) on the #5382 lane; step 3 edits + `services/runner/src/protocol.ts` and `services/runner/src/sessions/persist.ts` (both in + the #5376 diff) and `api/oss/src/core/sessions/interactions/dtos.py` (in the #5375 diff) + on the #5382 stack. +- Verified feasibility of that file sharing: the regions step 3 edits are textually + identical on both stacks (`git show plan/concurrent-approvals:...` confirms the + `AgentEvent` union, the persist stable-id branch, and `resolveInteraction` exist + unchanged there, at that branch's own line numbers), so the hunks route cleanly. The + turn-index region that step 1 edits exists ONLY on the #5376 lane, so those hunks cannot + land anywhere else. +- Because of the shared files, commits MUST follow the one-lane-at-a-time isolation + procedure from the root `AGENTS.md` (assign exactly one lane's files, commit with + `--only`, verify with `git show --stat` and per-lane `git diff --name-only .. + ` before touching the next lane). This is the top operational risk of the whole + project and is restated in plan.md. + +The note to JP (drafted in plan.md step 1) covers: what the two amendments on his lanes do, +why the dev `session_turns` rows were wiped, and the confirmed `turn_index` semantics. + +## Open risks + +1. **The history fingerprint under partial resumes (R4).** Not conclusively verified that + `historyFingerprint` ignores approval tool-result envelopes; if it does not, a + partial-answer resume would mismatch and evict to cold. The step 4 dispatch-seam test + pins the intended behavior; the implementer must read + `session-identity.ts`'s `historyFingerprint` before coding and adjust the park-side + expected fingerprint if needed. +2. **Cross-stack file sharing (R8).** `run-turn.ts`, `protocol.ts`, `persist.ts`, + `server.ts`, `runtime-contracts.ts`, and `interactions/dtos.py` are all touched by both + this project and JP's rebase lanes. Hunk mis-routing here scrambles two open PR stacks + at once. Mitigation is procedural (isolation commits plus per-lane diff verification), + not structural. +3. **The cancellation-closure emitter (R5).** The exact upstream emitter of the turn-1 + closure frame (pi-acp's turn-cancellation path closing unstarted calls) is the one link + the incident report could not pin to a line, and this research did not close it either. + The fix does not depend on the emitter's identity (it classifies frames by policy + evidence on the runner side), but a harness that legitimately completes an ask-policy + call without any gate would be misrecorded as deferred. No such harness path exists + today (both harnesses fail closed), so this is accepted and documented in the code + comment the step 2 spec requires. +4. **`in_progress` frame coverage (R5).** The started-evidence signal (`in_progress` + status frames) was not verified across both harnesses' real streams; the specified + classification therefore rests on permission policy (fail-closed), with startedness as + a reinforcing signal only. Live QA (qa.md) covers the real-stream behavior. diff --git a/docs/design/agent-workflows/projects/approvals-incident-fixes/status.md b/docs/design/agent-workflows/projects/approvals-incident-fixes/status.md new file mode 100644 index 0000000000..82b8e0aac3 --- /dev/null +++ b/docs/design/agent-workflows/projects/approvals-incident-fixes/status.md @@ -0,0 +1,17 @@ +# Status + +2026-07-19: Planning complete, awaiting review. + +- Research R1 through R8 closed with code evidence; findings in `research.md`. Four open + risks are listed at the end of that file; the history-fingerprint question (risk 1) and + the cross-stack file sharing (risk 2) are the two that need attention during + implementation, and both have prescribed mitigations in `plan.md`. +- Plan written as five independently landable steps in `plan.md`, targeted at an + implementer (GPT-5.6 Sol / Codex) who has not read the incident conversation. +- QA script written in `qa.md`, including the MP4 recording requirement and the + release-gate cells to re-run. +- Nothing has been implemented, committed, or pushed from this planning session. The dev + `session_turns` wipe (plan step 1c) has NOT been run. + +Next action: Mahmoud reviews this workspace; on approval, hand `plan.md` to the +implementation agent, step 1 first. diff --git a/docs/design/agent-workflows/scratch/debug-concurrent-approvals-db58551b.md b/docs/design/agent-workflows/scratch/debug-concurrent-approvals-db58551b.md new file mode 100644 index 0000000000..78cac39786 --- /dev/null +++ b/docs/design/agent-workflows/scratch/debug-concurrent-approvals-db58551b.md @@ -0,0 +1,208 @@ +# Root-cause report: concurrent human-approval failure (session db58551b) + +This report explains why Mahmoud's live QA of two parallel gated writes broke: the first +approved command was reported as "not executed", the conversation went dead after the second +approval, the first approval card flipped back to "waiting", and a follow-up message was +answered with "Done" instead of being read. Every claim below is verified against two +independent evidence sources: the persisted session dump +(`debug/session-db58551b.json`) and the runner container's timestamped logs +(`agenta-ee-dev-wp-b2-rendering-runner-1`, 09:51 to 09:56 UTC on 2026-07-19). Where the two +sources disagree, the logs win, because the session record turns out to rewrite itself (see +defect 5). + +## Orientation: the moving parts + +- **The runner** is the Node/TypeScript sidecar under `services/runner/`. It drives a + coding-agent harness (here Pi, whose tool calls carry OpenAI-style `call_...` ids) inside a + sandbox and streams events to the web UI. +- **A gate** is a human-approval request. When Pi wants to run a policy-gated builtin tool + (here `Bash`), our Pi extension's hook calls `ctx.ui.confirm` and waits + (`extensions/agenta.ts`, `piDialogAllows`). The hook is fail-closed: only an explicit + `true` lets the command run. The pi-acp bridge surfaces the confirm as an ACP + `session/request_permission`, which the runner classifies and shows to the user as an + approval card (`acp-interactions.ts`). +- **Park and warm resume.** When a gate has no answer, the runner ends the turn and "parks" + the live harness session in a keepalive pool (`server.ts`, `parkedApprovals`). When the + answer arrives, the runner checks the session back out and continues it in place; this is a + **warm resume** (`[keepalive] resume` in the logs). Every post-approval turn in this + session was a warm resume: tool-call ids are reused across turns, and the resumed turns + re-emit cached usage rather than making a new model call. +- **The cold path is native continuation, not replay.** On a keepalive miss the runner + builds a fresh environment, but the harness resumes its OWN session, located through the + harness session id the runner persists per turn + (`session-continuity-durable.ts`, `fetchLatestSessionTurn`). The model does not re-issue + tool calls. Replaying the conversation from our persisted event stream happens only as a + last-resort fallback when that continuity lookup fails. Neither cold tier was involved in + this incident. +- **The deferred-sibling sentinel.** `TOOL_NOT_EXECUTED_PAUSED` (`tracing/otel.ts:66`) is the + string `"DEFERRED_NOT_EXECUTED: paused for another approval; retry the same call if still + required."`. After a pause, `settleOpenToolCalls` (`tracing/otel.ts:1479`) stamps it onto + every still-open tool call not excluded by a predicate; the only exclusion today is "this + call is itself a paused gate" (`run-turn.ts:505`, predicate `pause.isPausedToolCall`). +- **Pi serializes confirms.** Throughout this session, Pi raised one `ctx.ui.confirm` at a + time. The second parallel call's confirm only surfaced after the first call's confirm was + answered, one park/resume cycle later. Two approval cards were therefore never truly + outstanding at the same moment on the runner; the user experienced them as one card per + cycle. This matches the adapter-serialization finding tracked in issue #5391. + +## The verified timeline + +The model issued two parallel `Bash` calls at 09:53:20: `IRll` (append a line to +`agent-files/README.md`) and `VIgq` (append the same line to `agent-files/NOTES.md`). + +1. **Turn 1 (09:53:20).** Pi raised the confirm for `IRll` only. The runner gated it (card + `f6f8384d`), parked, and the turn ended. `VIgq`'s confirm was never raised in this turn. + Twenty milliseconds after the park, a `tool_result` for `VIgq` was persisted with output + `"(no output)"` and `isError: false`: a successful-looking result for a command that had + not run and had never been approved. No gate, no execution, no bash activity for `VIgq` + appears anywhere in the logs for this turn. +2. **Turn 2 (09:53:22, warm resume after the user approved card 1).** The runner answered + `IRll`'s confirm with `once`. One millisecond later Pi raised the confirm for `VIgq`; the + runner gated it (card `b5f44eb8`) and parked again. The post-pause sweep then stamped + `TOOL_NOT_EXECUTED_PAUSED` onto `IRll`, the call the user had just approved, because the + sweep's exclusion list contains only paused calls and `IRll` was "approved and executing", + a state the sweep does not know about. `IRll`'s real completion arrived after the park and + was discarded (late events are dropped once the turn is cleared, + `session-events.ts:31`). The README append itself almost certainly executed on the + sandbox: the confirm resolved `true`, execution is never cancelled in park mode, and + nothing in the logs shows a failure. Only its report was destroyed. +3. **The dead hour of the session (09:53:22 to 09:54:17).** The user approved card 2 in the + UI. The logs show NO resume request arriving in this window. The click was recorded only + in the browser's local state. The frontend's auto-resume fires only when every approval + card on the last turn looks settled (`agentApprovalResume.ts:163`), and the re-rendered + state showed card 1 as pending again, so the auto-resume never fired. Card 1 looked + pending because nothing in the persisted record says a gate was ever answered (defect 4). +4. **Turn 3 (09:54:17, triggered by the user's text message).** The frontend bundled the + stored card-2 approval into the message request (`[keepalive] resume gates=1 approve=1`). + The runner used the request as an approval resume: it answered `VIgq`'s confirm, `VIgq` + executed (approved, at 09:54:17), and the model then continued the parked write task and + replied "Done - two separate write requests ran in parallel". The user's actual questions + were not answered; the turn was a resume of the stale task, not a reading of the new + message. + +Net effect on disk: both appends executed exactly once, each only after its approval. There +was NO unapproved execution in this session. What broke was reporting, state +reconstruction, and the resume trigger, plus a latent hazard described under defect 2. + +## Defects + +### Defect 1: the frontend never dispatches the resume for the last answered card + +The user's click on card 2 produced no network request. The decision sat in browser state +until the next message happened to carry it. Root cause: the auto-resume precondition +"every card settled" (`web/packages/agenta-playground/src/state/execution/agentApprovalResume.ts:163`) +can never hold after a state rebuild, because the persisted record contains the request half +of every gate and never the answer half (defect 4). This is what killed the conversation and +what made card 1 flip back to "waiting for approval". + +### Defect 2: the post-pause sweep clobbers an approved, executing call + +`run-turn.ts:505` sweeps every open call except currently-paused gates. A call that was just +approved on this resume and is mid-execution is not in that exclusion, so it gets stamped +`TOOL_NOT_EXECUTED_PAUSED` (`isError: true`) and its real result is dropped when it lands +late. Two consequences: the user sees their approved command reported as never executed, and +the sentinel's text invites the model to "retry the same call", which for a +side-effecting command that DID execute means a double execution. The model happened not to +retry in this session; nothing prevents it. The sweep predicate and the one-tick drain +(`pause.ts`, a single `setImmediate`) are pre-existing on `origin/main`; the #5382 lane's +warm re-park (commits `ab3c7819bb`, `0071f90090`) is what creates the state "approved and +executing while a new gate pauses the same turn" that exposes them. + +### Defect 3: a never-started sibling is recorded as a successful "(no output)" result + +In turn 1, `VIgq` had not started (its confirm was still queued inside Pi), yet a +`completed` frame closed it as a success. The frame-suppression policy deliberately lets +`completed` frames through during a pause so that a legitimately finishing auto-allowed +sibling keeps its real result (`runtime-policy.ts:45`); a cancellation-closure frame for a +never-started call takes the same path and is indistinguishable there. The result mapping +then records any `completed` frame as `isError: false` (`tracing/otel.ts:1445`). The exact +emitter of that closure frame (pi-acp's turn-cancellation path closing unstarted calls) is +the one link not pinned to a line; everything downstream of the frame is verified. The +correct record for that call at that moment is the deferred sentinel, not a success. + +### Defect 4: the answer half of a gate is never persisted + +The runner protocol has an `interaction_request` event and no answer event +(`protocol.ts:358`); a full-repo search finds no `interaction_response` producer. The +interactions table gets its row flipped to resolved (`interactions.ts:97`) but the payload +carries only lifecycle status, not the allow/deny verdict, even though the API schema has a +`resolution` field for it (`api/oss/src/core/sessions/interactions/dtos.py:24`). The live UI +tracks answers only in local memory (`AgentConversation.tsx:1031`); hydration reads only +session records (`loadSession.ts:20`) and reconstructs every persisted request as pending +(`transcriptToMessages.ts:144`). Every rebuild therefore resurrects answered gates. This is +the root that feeds defect 1, and it is also an audit gap: the record cannot say who +approved what. + +### Defect 5: the persisted session record rewrites itself + +Tool-result rows are stored under deterministic ids keyed by session, tool-call id, and +event type, with no turn scoping (`persist.ts:295`). A later result for the same call +overwrites the earlier row in place, preserving `created_at`. In this session, `VIgq`'s real +09:54:17 result silently replaced its turn-1 artifact row. The exported dump therefore +misrepresents history, which is how it initially supported a false "unapproved execution" +reading. Separately, an approval resume re-persists the recovered prior prompt as a fresh +user-message row (`server.ts:1004`), which is why "no i want to write requests in parallel" +appears twice in the record. The session record is currently not a trustworthy audit log. + +### Defect 6: a text message during a park is consumed as an approval resume + +A message request that arrives while gates are parked goes down the approval-resume branch +(`server.ts:663`). In this session it carried a bundled decision, matched, and warm-resumed +the stale task; the model completed the old work and never addressed the new text. Had it +carried no decision, the branch would have evicted the parked session and continued on the +cold path, with the same user-visible outcome (the stale task finishes, the question is +ignored). The +dispatch does not distinguish "this request answers the gates" from "this is new user text +that should supersede or queue behind the parked work". + +### Defect 7 (found in passing): session-turns append fails with HTTP 500 + +`[sandbox-agent] append HTTP 500 ... harness=pi_core turn=0` recurs throughout the logs on +this stack. The new append-only `session_turns` ingestion is failing for Pi sessions. This +is unrelated to approvals but means the new sessions plane is silently not recording these +turns. It needs its own investigation. + +## Fix plan, in order + +1. **Persist the answer half of every gate (fixes defects 4 and, through it, 1).** Emit an + answer event into the session record when a gate is resolved, and store the allow/deny + verdict in the interaction row's `resolution`. Hydration then overlays answers onto + requests, rebuilt state shows answered cards as answered, and the frontend's auto-resume + condition can become true. This is the smallest change that revives the dead conversation + and the flip-flopping cards. Add a frontend fallback so an answered card's decision is + dispatched even if other cards look unsettled, so one rendering bug can never strand a + decision in browser memory again. +2. **Protect approved, executing calls from the sweep, and let their real result land + (fixes defect 2).** Carry "approved this resume" ids into the pause controller's + exclusion set, and hold the turn's terminalization until those calls reach their own + terminal frame (bounded by the existing per-call time limit) so the real result replaces + nothing and the sentinel is never written onto them. The retry-inviting sentinel must + never be attached to a call whose execution actually started. +3. **Record a never-started sibling as deferred, not as success (fixes defect 3).** During a + pause, buffer `completed` frames for sibling calls until the drain settles which calls + gated; a closure frame for a call that never started becomes the deferred sentinel, a + genuine completion keeps its real result. +4. **Separate "answer the gates" from "new user text" in the parked dispatch (fixes defect + 6).** An incoming request with fresh text should either abandon the parked task cleanly or + queue the text as the next turn after the resume completes; it must never vanish into a + resume of stale work. This is partly a product decision and can land after 1 to 3. +5. **Make the session record append-only per turn (fixes defect 5).** Scope stable + record ids by turn so contradictory results append rather than overwrite, and stop + re-persisting the recovered prompt on approval resumes. Needed for trustworthy audits and + for debugging every future incident. +6. **Investigate the session-turns HTTP 500 (defect 7)** as its own thread against the new + sessions ingestion. +7. **Add the missing regression test.** The existing tests model gates that are all known + before the first pause. The real Pi shape is: two parallel gated calls, one gate raised, + approve, second gate surfaces during the warm resume while the first command is still + executing, approve, assert both side effects exactly once and both real results recorded. + +## Open questions + +- The precise emitter of the turn-1 closure frame for the unstarted call (pi-acp's + cancellation path is the strong candidate; confirming it pins defect 3's upstream half). +- Whether the frontend should also dispatch each answer immediately as it is clicked + (per-card dispatch) instead of batching until all cards settle; the Zed comparison report + (`zed-acp-approvals-comparison.md`, in progress) should inform this. +- Whether Pi can be configured or extended to raise multiple confirms concurrently, which is + the only path to two cards genuinely on screen at once (issue #5391). diff --git a/docs/design/agent-workflows/scratch/debug-session-turns-append-500.md b/docs/design/agent-workflows/scratch/debug-session-turns-append-500.md new file mode 100644 index 0000000000..9656916e2a --- /dev/null +++ b/docs/design/agent-workflows/scratch/debug-session-turns-append-500.md @@ -0,0 +1,66 @@ +# Debug report: session-turns append HTTP 500 + +Date: 2026-07-19. Stack: the local EE dev deployment (`agenta-ee-dev-wp-b2-rendering-*` containers). All code referenced here is the in-flight sessions redesign, which lives on JP's open PRs #5375 (backend) and #5376 (runner); none of it is on main yet. + +Two terms used throughout. A "turn" is one user-message-to-agent-reply exchange inside a chat session. The "turn-append" is the runner's durable write at the end of each completed turn: it INSERTs one row into the `session_turns` table so that a later runner restart can find the harness's native session id and resume the conversation natively instead of replaying the transcript as text. + +## 1. The verified request path and where the 500 is produced + +The append request goes exactly where it is supposed to go. The lead suggesting the request might be eaten by a redirect or land on the wrong service turned out to be wrong. + +- The runner container runs with `AGENTA_API_INTERNAL_URL=http://api:8000`, which resolves to the EE API container on the same compose network. +- `appendSessionTurn` POSTs to `${apiBase}/sessions/turns/` (services/runner/src/engines/sandbox_agent/session-continuity-durable.ts:168). +- The EE API access log records the failing requests at exactly the runner's timestamps. For the reported session `db58551b-f986-44ec-b939-d6b10b35717a`: + - Runner: `append OK ... turn=0` at 09:52:37.645, then `append HTTP 500 ... turn=0` at 09:53:00.765, 09:54:23.105, 09:54:50.008 (all UTC, 2026-07-19). + - EE API access log: `POST /api/sessions/turns/ HTTP/1.1" 500` at 09:53:00.765, 09:54:23.105, 09:54:50.008. + +The API error log at 09:53:00.764 shows the actual exception: + +``` +asyncpg.exceptions.UniqueViolationError: duplicate key value violates unique +constraint "ix_session_turns_project_id_session_id_turn_index" +DETAIL: Key (project_id, session_id, turn_index) = +(019f4d22-..., db58551b-f986-44ec-b939-d6b10b35717a, 0) already exists. +``` + +The traceback runs through `append_turn` (api/oss/src/apis/fastapi/sessions/router.py:1086) into `SessionTurnsService.append_turn` (api/oss/src/core/sessions/turns/service.py:37) into the DAO's bare `session.add` plus `commit` (api/oss/src/dbs/postgres/sessions/turns/dao.py, `append`). Nothing on that path handles `IntegrityError`, so the `@intercept_exceptions()` decorator (api/oss/src/utils/exceptions.py:119) converts it into a generic 500. The database confirms the state: `session_turns` holds exactly one row for this session, `(db58551b-..., turn_index 0, pi_core, created 09:52:37)`, matching the one append that succeeded. + +## 2. The root cause + +The runner computes the turn index once per sandbox environment, not once per turn, so a warm environment that serves several turns keeps re-INSERTing the same index. + +The mechanism, step by step: + +- `acquireEnvironment` sets `environment.continuityTurnIndex = nextTurnIndex(...)` exactly once, at environment acquire time (services/runner/src/engines/sandbox_agent/environment.ts:962). For a fresh session that value is 0. +- The runner keeps finished environments warm in a keep-alive pool. When the next user message arrives within the idle TTL, the server takes the pooled environment and calls `engine.runTurn(live.environment, ...)` directly (the `hit-continue` branch, services/runner/src/server.ts:610). `acquireEnvironment` never runs again, so `continuityTurnIndex` stays frozen at its acquire-time value. +- At the end of every completed turn, `runTurn` records that same frozen index into the in-memory store and fires the durable append with it (services/runner/src/engines/sandbox_agent/run-turn.ts:572 and :584). + +The first completed turn INSERTs `turn_index=0` and succeeds. Every later turn served by the same warm environment INSERTs `turn_index=0` again, which violates the unique index `(project_id, session_id, turn_index)` that migration oss000000014 created, and the API returns 500. That is precisely the db58551b log shape: one `append OK turn=0`, then `append HTTP 500 turn=0` on each following warm turn. + +The second log shape confirms the same mechanism from the other direction. Session `e744a157-...` on 2026-07-18 shows indexes incrementing (turn=0 through turn=10) because its turns arrived five or more minutes apart, past the 60-second idle TTL, so each turn triggered a fresh cold acquire that recomputed the index. Its 500s appear exactly when a turn DID reuse a warm environment (duplicate turn=10, duplicate turn=11), and its late `turn=2` and `turn=3` failures came from a second pooled environment (the pool logged `poolSize=2`; approval-parked environments live for 300 seconds) that still carried the frozen index from its own, much earlier acquire. + +A secondary defect sits on the API side: the write path treats a uniqueness conflict as an unknown error. The interceptor already maps `EntityCreationConflict` (api/oss/src/core/shared/exceptions.py) to a 409 Conflict response, but the turns DAO never raises it, so a duplicate INSERT surfaces as a 500 with a full traceback in the error log. + +## 3. Blast radius + +The failed append is fire-and-forget (`void ... .catch(() => {})` in run-turn.ts:584), so no turn fails and users see nothing at the moment of the error. The damage is deferred and silent: + +- Most turns never reach the durable turn log. For a busy session only the first completed turn per environment acquire gets a row. After a runner restart, `hydrateHarnessSessionFromDurable` and the sandbox reconnect ladder (`fetchLatestSessionTurn` in sandbox-reconnect.ts:35) read a stale latest row. In the common single-environment case the row still happens to carry the right `agent_session_id` and `sandbox_id`, because those do not change across warm turns. But whenever more than one environment served the session (the e744a157 case), the latest surviving row can point at a dead sandbox and an outdated native session, so cold starts lose native continuity and degrade to transcript replay. That is exactly the degradation this table was built to prevent. +- Per-turn metadata is lost. Each row was meant to carry that turn's `stream_id` and `trace_id`, linking the turn to its trace. For every turn whose append 500s, that linkage row simply does not exist, which undermines anything that joins turns to traces (the records/turn-span work in the same PR stack). +- `turn_index` is not a turn counter. Even the rows that do land carry indexes that count environment acquires, not conversation turns, so any consumer treating the index as "how many turns has this conversation had" gets wrong numbers. +- Operational noise: every duplicate produces a full traceback in the API error log and a misleading 500 in the access log, which is how this investigation started with a wrong lead. + +## 4. The minimal fix and where it should land + +The unique index is correct and should stay; it is the thing that caught the bug. The runner's turn-index accounting is the component to change, with a small API-side hardening alongside it. + +- Primary fix, on PR #5376's lane (the runner side of the sessions redesign): compute the turn index per turn instead of per acquire. Move the `nextTurnIndex` call from `acquireEnvironment` (environment.ts:962) to the start of `runTurn`, reading the process-wide `SessionContinuityStore` each time. The store already advances on every successful `record()` (session-continuity.ts:47), so a per-turn read yields 0, 1, 2, ... naturally, and it also fixes the two-pooled-environments case because both environments read the same shared store. `continuityTurnIndex` can stay as a field on the environment for the record-after-turn symmetry; it just needs to be refreshed at turn start. +- Secondary fix, on PR #5375's lane (the backend side): in `SessionTurnsDAO.append`, catch the SQLAlchemy `IntegrityError` for this unique constraint and raise `EntityCreationConflict`, which the existing interceptor already converts to a 409. A duplicate append then reads as what it is (a conflict on an idempotent-ish write) instead of an anonymous server error, and the runner's log line becomes diagnosable on sight. + +Both fixes belong on the existing PR lanes, not a new one: the bug lives entirely inside code those two open PRs introduce, and landing the fix anywhere else would leave the PRs shipping a known-broken write path. + +## 5. Open questions + +- Should the runner retry a 409 with a freshly computed index, or is dropping the row acceptable? Within one Node process the single-threaded event loop makes read-then-record effectively atomic, but two runner replicas serving the same session could still collide. Today the local provider forbids multi-replica sessions (`LocalSandboxNotOwnerError`), so a retry may be over-engineering for now. +- Is `turn_index` meant to be a true conversation turn counter for consumers beyond continuity (analytics, the UI, the turn-span join)? If yes, the per-turn fix gives correct values going forward, but rows written while the bug was live carry wrong indexes; the dev database may deserve a wipe of `session_turns` before review. +- The e744a157 anomaly showed an approval-parked environment appending an index from five turns earlier. After the per-turn fix its append would compute the current latest index instead; worth a unit test covering "two pooled environments, interleaved completed turns" in services/runner/tests/unit/session-continuity-durable.test.ts. diff --git a/docs/design/agent-workflows/scratch/zed-acp-approvals-comparison.md b/docs/design/agent-workflows/scratch/zed-acp-approvals-comparison.md new file mode 100644 index 0000000000..41562f328f --- /dev/null +++ b/docs/design/agent-workflows/scratch/zed-acp-approvals-comparison.md @@ -0,0 +1,134 @@ +# Comparison of human approval handling in Zed and Agenta + +## The incident and the design question + +In session `db58551b`, the model proposed two parallel shell commands. Each command required human approval. Both commands ran exactly once, and each ran only after the user approved it. No unapproved command executed. + +The system failed around that execution. It lost an approval after rebuilding the saved conversation, failed to send the final approval to the runner, misreported both commands, and later treated a new user message as the missing answer to the old task. The companion report reconstructs the incident and defines the seven defects used below (`docs/design/agent-workflows/scratch/debug-concurrent-approvals-db58551b.md:48`). + +Zed and Agenta use the same approval adapter and can both keep a live approval request waiting inside an agent process. The important difference appears after Agenta closes the browser response. Agenta needs another browser request to deliver the answer, but its durable record does not contain that answer. Its pause cleanup also assigns final results without reliable evidence from the executor. + +The Zed citations below use paths relative to the cloned `agent-client-protocol`, `claude-code-acp`, and `zed` repositories. The Agenta citations use paths relative to this repository. + +## ACP assigns execution to the agent and approval to the client + +ACP, or the Agent Client Protocol, defines how a coding agent and a user interface communicate. For example, ACP lets Claude Code ask Zed whether it may run `git status`. + +ACP uses JSON-RPC, which is a request and response format based on JSON. The sender gives each request an `id`. The receiver eventually returns a response with the same `id`. Several requests can remain unanswered at the same time because their ids keep them separate. + +ACP defines two roles: + +- The **agent** runs the model and executes its tools. In the example above, Claude Code is the agent. +- The **client** displays the conversation and answers the agent's questions. In the example above, Zed is the client. + +A **tool call** is the model's request to perform an action, such as reading a file or running a command. A **permission request** is the agent's request for the human to allow or reject one tool call. The client answers the permission request, but the agent executes the tool. ACP states this division directly (`agent-client-protocol/docs/protocol/v2/tool-calls.mdx:255`). + +A **tool-call status** records the call's current stage, such as pending, waiting for confirmation, running, completed, failed, rejected, or canceled. A **turn** contains one user message and the agent work that follows it. A **session** contains the agent's conversation state across turns. + +ACP also defines `requires_action`, a session state that means the agent cannot continue until the user acts. The agent should report `requires_action` while awaiting permission and `running` after receiving the answer (`agent-client-protocol/docs/protocol/v2/prompt-lifecycle.mdx:355`). + +In Agenta, a **harness** is the agent program that implements the model and tool loop, such as Pi or Claude Code. The runner acts as the ACP client toward that harness. It also streams events to the browser and persists an event stream, which is a chronological record that lets a reloaded page or a second browser rebuild the conversation. + +## Zed and Agenta use the same Claude adapter + +Zed's `claude-code-acp` repository publishes the same npm package that our runner uses: `@agentclientprotocol/claude-agent-acp`. Zed's repository contains version `0.59.0`, while our runner pins version `0.58.1` (`claude-code-acp/package.json:2`, `claude-code-acp/package.json:6`, `services/runner/package.json:23`). + +The adapter does not serialize approvals, which would mean forcing every approval to wait behind the previous one. Its `canUseTool` callback, the function that decides whether a tool may run, awaits `client.requestPermission` without an approval lock or queue (`claude-code-acp/src/acp-agent.ts:4112`, `claude-code-acp/src/acp-agent.ts:4334`). Its only queue, `turnQueue`, orders whole user turns (`claude-code-acp/src/acp-agent.ts:301`, `claude-code-acp/src/acp-agent.ts:1634`). + +ACP permits any number of outstanding permission requests, meaning requests that were sent but not yet answered. Each request has its own JSON-RPC id. The specification demonstrates two concurrent requests and later cancels each id separately (`agent-client-protocol/docs/protocol/v2/cancellation.mdx:52`). + +If Claude presents approvals one at a time, Claude's own tool dispatch creates that ordering. Neither ACP nor the shared adapter requires it. + +## Zed stores each approval on its tool call + +Zed processes one approval as follows. + +1. The adapter first emits the `tool_call` update. It does this deliberately so the client knows which tool call exists before receiving a permission request that refers to it (`claude-code-acp/src/acp-agent.ts:4112`, `claude-code-acp/src/acp-agent.ts:4153`). + +2. Zed creates a **one-shot channel**, which is a channel that carries one value and then closes. In this case, Zed stores the sending end on the tool call and waits on the receiving end until the human chooses allow or reject. The stored sender acts as that call's resolver, meaning it supplies the answer to the waiting request. + +3. Zed changes that call's status to `WaitingForConfirmation`. The status contains the approval options and its own one-shot resolver (`zed/crates/acp_thread/src/acp_thread.rs:1258`, `zed/crates/acp_thread/src/acp_thread.rs:3372`, `zed/crates/acp_thread/src/acp_thread.rs:3386`). + +4. When the human answers, Zed finds the tool call by id. It changes the status to `InProgress` or `Rejected`, then fires only that call's resolver (`zed/crates/acp_thread/src/acp_thread.rs:3425`, `zed/crates/acp_thread/src/acp_thread.rs:3463`). + +The answer is therefore the status transition itself. A redraw reads `InProgress` or `Rejected`; it cannot show the answered call as waiting. Several calls can wait together because each call has its own status and resolver. + +Zed normally leaves the permission request pending without a time limit. The live agent process and the current user turn remain open until the human answers or cancels (`zed/crates/acp_thread/src/acp_thread.rs:3372`). + +A new user message follows a different path. Zed first cancels the running turn, marks every pending or running tool call as `Canceled`, waits for cancellation to finish, and only then sends the new message to the same session (`zed/crates/acp_thread/src/acp_thread.rs:3724`, `zed/crates/acp_thread/src/acp_thread.rs:3733`, `zed/crates/acp_thread/src/acp_thread.rs:3911`). + +The adapter also gives each permission request an `AbortSignal`, which is a cancellation object tied to that request. Aborting it sends `$/cancel_request` for the corresponding JSON-RPC id (`claude-code-acp/src/acp-agent.ts:4105`). ACP requires the client to answer every pending permission request with `cancelled` and recommends marking all unfinished tool calls as canceled (`agent-client-protocol/docs/protocol/v2/prompt-lifecycle.mdx:495`, `agent-client-protocol/docs/protocol/v2/prompt-lifecycle.mdx:497`). + +## Agenta has three continuation paths + +Agenta does not routinely reconstruct and replay the conversation. It has three paths. + +### Warm park and warm resume + +To **park** an approval means ending the browser-facing turn while keeping the live harness process and its permission request waiting. A **keepalive pool** is the runner's bounded collection of these live processes. Each entry has a time-to-live limit, which is the maximum time it may remain parked before the runner destroys it (`services/runner/src/engines/sandbox_agent/session-pool.ts:222`). + +A **warm resume** continues that same live process. The next browser request checks the process out of the pool and answers the original permission request by its tool-call id. The original prompt then continues in place. The model does not issue the tool call again (`services/runner/src/engines/sandbox_agent/run-turn.ts:433`, `services/runner/src/engines/sandbox_agent/run-turn.ts:471`). + +Pi uses this mechanism for builtin tools. Its in-process confirmation waits indefinitely. It allows execution only when the answer is exactly `true`; an error or cancellation denies execution. This is fail-closed behavior (`services/runner/src/extensions/agenta.ts:94`, `services/runner/src/extensions/agenta.ts:112`). + +Warm park therefore uses the same pending-request mechanism as Zed. Three differences matter: + +1. Agenta bounds the wait with a time-to-live limit and evicts the process when that limit expires. Zed applies no normal approval timeout. +2. Agenta ends the browser-facing turn when it parks. The frontend must send a new resume request to deliver the answer. Zed keeps one turn open, so the answer returns through the still-open request. +3. Agenta's frontend resume trigger failed during the incident. The live permission request remained valid, but its answer never reached it. + +### Cold continue + +**Session continuity** is the ability to preserve the agent's conversation state when the runner must create a new process or environment. On a keepalive pool miss, Agenta creates a fresh environment and asks the harness to load its own session. The runner finds that session through the harness session id stored on the latest durable session turn (`services/runner/src/engines/sandbox_agent/session-continuity-durable.ts:1`, `services/runner/src/engines/sandbox_agent/session-continuity-durable.ts:58`). + +The fresh environment calls the harness's native session load, which means the harness restores its own saved state instead of receiving a reconstructed conversation from Agenta. If loading succeeds, the runner sends only the new user text (`services/runner/src/engines/sandbox_agent/environment.ts:929`, `services/runner/src/engines/sandbox_agent/environment.ts:982`, `services/runner/src/engines/sandbox_agent/runtime-contracts.ts:145`). The model does not reissue the earlier tool calls. + +### Failure fallback + +Only when the continuity lookup or native session load is unavailable does Agenta use replay as a failure fallback. **Replay** means rebuilding the prompt from Agenta's persisted event history and sending that reconstructed history to a new harness session (`services/runner/src/engines/sandbox_agent/run-turn.ts:124`). + +## Agenta persists the request but not the answer + +When the runner asks the human for approval, it writes an `interaction_request` event into the durable session record (`services/runner/src/engines/sandbox_agent/acp-interactions.ts:178`). It also creates a pending interaction row. + +When the human answers, the browser stores the choice in its local message state and sends it with the resume request (`web/oss/src/components/AgentChatSlice/AgentConversation.tsx:1032`). The runner later changes the interaction row to `resolved`, but that update contains only the lifecycle status. It does not store the allow or deny verdict (`services/runner/src/sessions/interactions.ts:94`, `services/runner/src/sessions/interactions.ts:104`). The protocol defines no `interaction_response` event (`services/runner/src/protocol.ts:363`). + +The answer therefore exists only in browser memory and inside the in-flight resume request. + +This produces two direct consequences. First, a page reload, a second browser, or a frontend re-render that rebuilds state from the database sees every `interaction_request` as `approval-requested` (`web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.ts:144`, `web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.ts:172`). Every answered gate appears to be waiting again. + +Second, the frontend sends the resume only when every visible approval card looks settled (`web/packages/agenta-playground/src/state/execution/agentApprovalResume.ts:163`). After a rebuild, at least one answered card looks pending. The condition can never become true. This is why the final approval remained in browser memory and the conversation stopped. + +## The seven defects differ from Zed at specific boundaries + +The hardest execution problem occurs when one call is awaiting approval, another is already executing, and a third has not started. A cleanup pass can see that all three calls remain open, but it cannot infer which actions occurred. That missing evidence caused defects 2 and 3. + +| Defect | Comparison with Zed | +|---|---| +| **1. The frontend never dispatches the final answer.** | Agenta waits for every card to look settled before sending a resume. Zed answers one tool-call id immediately and does not depend on sibling cards. | +| **2. The runner reports an approved, executing call as not executed.** | Agenta's post-pause sweep writes a synthetic `DEFERRED_NOT_EXECUTED` error onto open calls, including a call that is already running, then drops its later real result (`services/runner/src/engines/sandbox_agent/run-turn.ts:507`, `services/runner/src/tracing/otel.ts:1479`). Zed advances status only from the selected answer and agent updates. | +| **3. The runner reports a never-started call as a successful empty result.** | Agenta accepts a `completed` closure frame and maps it to success even when execution never began (`services/runner/src/engines/sandbox_agent/runtime-policy.ts:48`, `services/runner/src/tracing/otel.ts:1445`). Zed records unfinished calls as pending or canceled unless the agent reports a real result. | +| **4. The answer is not persisted.** | Agenta saves the permission request but not the verdict. Zed makes the answer a status transition on the tool call. | +| **5. The persisted record overwrites history.** | Agenta derives result-row ids from the session, tool-call id, and event type, without the turn id, so a later result can replace an earlier result (`services/runner/src/sessions/persist.ts:295`). Approval resumes also persist duplicate prompts. Per-call approval status fixes the rebuild problem, but full audit history still requires turn-scoped, append-only rows. | +| **6. A new message resumes the stale task.** | The incident's new message carried the stored approval, so Agenta consumed the request as a resume and completed the old task. Zed cancels the old turn before sending new text. | +| **7. Session-turn append returns HTTP 500.** | This failure affects durable session continuity but is unrelated to approval concurrency. It needs a separate investigation (`docs/design/agent-workflows/scratch/debug-concurrent-approvals-db58551b.md:158`). | + +No unapproved command ran in the incident. Pi's builtin confirmation and execution live in the same process, and the confirmation fails closed. This matches Zed's `canUseTool` shape. + +The **client-tool relay** is the separate path where the harness asks the runner to execute a product tool on its behalf. Only on that path do the approval gate and executor live in different processes. Runner-side cancellation remains necessary there as a safety backstop. + +## Recommended changes + +1. Persist each approval answer as a durable status transition on its tool call. Store the allow or deny verdict, actor, and time. A rebuilt conversation must read `approved`, `denied`, or a later execution status instead of reconstructing `waiting` from the request alone. This fixes defects 1 and 4 and the rebuild part of defect 5. + +2. Dispatch each approval independently. One click should persist and deliver one answer by tool-call id. The frontend must not wait for every visible card to settle, and one stale card must not block another answer. + +3. Cancel before accepting new user work. When new text arrives during pending approvals, the runner should send `session/cancel`, mark unfinished calls canceled, answer every pending permission request with `cancelled`, wait for the harness to report an idle canceled state, and then send the new text as a fresh prompt to the same session. This fixes defect 6 and implements ACP's cancellation contract. + +4. Accept only real terminal evidence. Only the executor may mark a call completed or failed. An explicit cancellation may mark it canceled. A pause sweep must not invent a success or a failure for an open call. This removes the false results behind defects 2 and 3 and prevents the retry instruction from repeating a command that already ran. + +5. Keep Agenta's durable event stream. Several browsers and page reloads are valid product requirements. The fix is to persist every state transition and scope result records by turn, not to replace durable history with Zed's in-memory state. + +6. Investigate defect 7 separately. A failed `session_turns` append weakens cold session continuity, but it did not cause the approval incident. + +Updating the shared Claude adapter alone will not fix these defects. The adapter already supports concurrent permission requests and per-request cancellation. The required changes belong in Agenta's browser dispatch, runner state transitions, cancellation path, and persistence model. From d467e86fe8e575005d89ad822925de0bac1f1206 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 19 Jul 2026 16:52:49 +0200 Subject: [PATCH 05/15] fix(runner): only real executor evidence settles a paused turn's tool calls --- .../engines/sandbox_agent/acp-interactions.ts | 8 +- .../runner/src/engines/sandbox_agent/pause.ts | 18 +- .../src/engines/sandbox_agent/run-turn.ts | 175 ++++++++++- .../engines/sandbox_agent/runtime-policy.ts | 20 +- services/runner/src/responder.ts | 22 +- services/runner/src/tracing/otel.ts | 6 + .../tests/unit/pending-approval-pause.test.ts | 14 + services/runner/tests/unit/responder.test.ts | 17 +- .../unit/sandbox-agent-orchestration.test.ts | 124 ++++++++ .../unit/session-keepalive-approval.test.ts | 286 +++++++++++++++++- 10 files changed, 651 insertions(+), 39 deletions(-) diff --git a/services/runner/src/engines/sandbox_agent/acp-interactions.ts b/services/runner/src/engines/sandbox_agent/acp-interactions.ts index b70570693c..6738e532e1 100644 --- a/services/runner/src/engines/sandbox_agent/acp-interactions.ts +++ b/services/runner/src/engines/sandbox_agent/acp-interactions.ts @@ -52,6 +52,8 @@ export interface AttachPermissionResponderInput { log?: (msg: string) => void; /** Called with the ACP tool-call id when a gate pauses the turn. */ onPausedToolCall?: (id: string) => void; + /** Called before an allow reply can release the harness to execute this tool call. */ + onAllowedExecution?: (id: string) => void; /** * Called when a NON-parkable pause happens this turn (a client-tool ACP gate, which cannot be * answered across a turn boundary on the live session). The keep-alive dispatch reads this to @@ -118,6 +120,7 @@ export function attachPermissionResponder({ onPause, log, onPausedToolCall, + onAllowedExecution, onNonParkablePause, onCreateInteraction, onResolveInteraction, @@ -236,6 +239,9 @@ export function attachPermissionResponder({ // breakage. (A malformed-envelope / unknown-builtin fail-closed reject goes through // `rejectRequest`, not here, so it stays a plain error — it is not a user/policy denial.) if (decision === "deny") run.markToolCallDenied?.(toolCallId); + // Mark before replying because an allow can release the harness synchronously and its first + // execution frame must already be protected from a concurrently active pause sweep. + if (decision === "allow" && toolCallId) onAllowedExecution?.(toolCallId); try { await session.respondPermission( id, @@ -518,7 +524,7 @@ function recordedToolName( * strip that prefix so the lookup hits). This is the ONLY source of a tool's true * `permission`/`readOnly`, so both the approval card and this descriptor come from one lookup. */ -function buildGateDescriptor( +export function buildGateDescriptor( toolCall: any, run: { events?: () => AgentEvent[] }, serverPermissions: ReadonlyMap, diff --git a/services/runner/src/engines/sandbox_agent/pause.ts b/services/runner/src/engines/sandbox_agent/pause.ts index 15bfb4d22b..1555790973 100644 --- a/services/runner/src/engines/sandbox_agent/pause.ts +++ b/services/runner/src/engines/sandbox_agent/pause.ts @@ -3,14 +3,15 @@ * * F-040: an unanswered approval must end the turn, destroy the session, and never reply to the * harness gate. Historical docs call this "park"; this module uses pause/pendingApproval names. - * The destroy callback also settles every announced-but-unresolved sibling tool call with a - * deterministic `tool_result` before teardown, so the client never holds an orphaned part. + * Terminalization classifies every announced-but-unresolved sibling after managed cancellation + * drains, so the client never holds an orphaned part or an invented execution result. */ export const PAUSED = Symbol("paused"); export class PendingApprovalPauseController { private pendingApproval = false; private readonly pausedToolCallIds = new Set(); + private readonly allowedExecutionToolCallIds = new Set(); private resolvePause: (() => void) | undefined; private eventDrain: Promise = Promise.resolve(); @@ -60,6 +61,19 @@ export class PendingApprovalPauseController { return toolCallId !== undefined && this.pausedToolCallIds.has(toolCallId); } + /** Record that this turn answered allow for the call, so pause cleanup preserves its result. */ + markAllowedExecution(toolCallId: string): void { + if (!toolCallId) return; + this.allowedExecutionToolCallIds.add(toolCallId); + } + + isAllowedExecution(toolCallId: string | undefined): boolean { + return ( + toolCallId !== undefined && + this.allowedExecutionToolCallIds.has(toolCallId) + ); + } + get active(): boolean { return this.pendingApproval; } diff --git a/services/runner/src/engines/sandbox_agent/run-turn.ts b/services/runner/src/engines/sandbox_agent/run-turn.ts index 842d85448d..6f955911cc 100644 --- a/services/runner/src/engines/sandbox_agent/run-turn.ts +++ b/services/runner/src/engines/sandbox_agent/run-turn.ts @@ -1,4 +1,7 @@ -import { permissionsFromRequest } from "../../permission-plan.ts"; +import { + effectivePermission, + permissionsFromRequest, +} from "../../permission-plan.ts"; import { resolvePromptText, type AgentRunRequest, @@ -27,10 +30,14 @@ import { type RelayExecutionGuard, } from "../../tools/relay.ts"; import { + APPROVED_EXECUTION_RESULT_UNKNOWN, createSandboxAgentOtel, TOOL_NOT_EXECUTED_PAUSED, } from "../../tracing/otel.ts"; -import { attachPermissionResponder } from "./acp-interactions.ts"; +import { + attachPermissionResponder, + buildGateDescriptor, +} from "./acp-interactions.ts"; import { buildClientToolRelay, relayWritesPausedAnswer, @@ -115,8 +122,9 @@ export async function runTurn( // caller's teardown (`runSandboxAgent`'s `finally`, or the keep-alive dispatch's evict-on-failure) // reclaims the sandbox exactly as any other error does. Disposed in the `finally` on every path. // A human pause retires the deadlines (`notePaused`): a HITL wait is legitimate, not a wedge. + const resolvedRunLimits = (deps.resolveRunLimits ?? resolveRunLimits)(logger); const runLimits = (deps.createRunLimits ?? createRunLimits)( - (deps.resolveRunLimits ?? resolveRunLimits)(logger), + resolvedRunLimits, { log: logger }, ); let runLimitTrip: (() => void) | undefined; @@ -216,8 +224,8 @@ export async function runTurn( // emit its own approval card. The orphan settle is deferred to the post-drain sweep below // (which runs on every paused turn, after `waitForEventDrain` lets every pending gate mark // its call paused) plus the in-band re-sweep in `handleUpdate` for a sibling announced after - // the pause. Both exclude paused (gated) calls, so every carded gate stays open for its - // resume and only a genuine orphan (announced, never gated) settles. + // the pause. Both exclude paused gates and allowed executions, so each keeps its own + // terminal outcome while only a genuine orphan settles. // Park mode: at least one parkable permission gate (Claude ACP or Pi ACP) recorded into // `env.parkedApprovals` BEFORE firing this pause (the onUserApprovalGate hook runs as each // gate resolves). Keep the live session — the gated tools run on the resume — so skip ONLY @@ -236,6 +244,43 @@ export async function runTurn( // place every pause path converges, so the one place to retire the run-limits deadlines for good. void pause.signal.then(() => runLimits.notePaused()); + const openToolCallIds = (): string[] => run.openToolCallIds?.() ?? []; + const bufferedPausedCompletedFrames = new Map(); + const toolCallClosureWaiters = new Map void>>(); + const notifyToolCallClosed = (toolCallId: string): void => { + if (openToolCallIds().includes(toolCallId)) return; + const waiters = toolCallClosureWaiters.get(toolCallId); + if (!waiters) return; + toolCallClosureWaiters.delete(toolCallId); + for (const waiter of waiters) waiter(); + }; + const waitForToolCallClosure = ( + toolCallId: string, + timeoutMs: number, + ): Promise => { + if (!openToolCallIds().includes(toolCallId)) { + return Promise.resolve(true); + } + return new Promise((resolve) => { + let timeout: NodeJS.Timeout | undefined; + let finished = false; + const finish = (closed: boolean): void => { + if (finished) return; + finished = true; + if (timeout) clearTimeout(timeout); + const waiters = toolCallClosureWaiters.get(toolCallId); + waiters?.delete(onClosed); + if (waiters?.size === 0) toolCallClosureWaiters.delete(toolCallId); + resolve(closed); + }; + const onClosed = (): void => finish(true); + const waiters = toolCallClosureWaiters.get(toolCallId) ?? new Set(); + waiters.add(onClosed); + toolCallClosureWaiters.set(toolCallId, waiters); + timeout = setTimeout(() => finish(false), timeoutMs); + }); + }; + // Publish this turn's sink so the session-lifetime listeners route into it. handleUpdate // reproduces the old per-event routing (suppress paused frames, handleUpdate, pause re-sweep). const turn: CurrentTurn = { @@ -275,12 +320,35 @@ export async function runTurn( ) { env.lastTurnToolCallIds.push(frame.toolCallId); } + const toolCallId = + typeof rawFrame.toolCallId === "string" + ? rawFrame.toolCallId + : undefined; + if ( + pause.active && + (rawFrame.sessionUpdate === "tool_call" || + rawFrame.sessionUpdate === "tool_call_update") && + rawFrame.status === "completed" && + toolCallId && + !pause.isPausedToolCall(toolCallId) && + !pause.isAllowedExecution(toolCallId) + ) { + bufferedPausedCompletedFrames.set(toolCallId, update); + return; + } run.handleUpdate(update); - // A non-gated sibling announced AFTER the pause can never execute this turn; settle it - // immediately so the client never holds an orphaned part (idempotent re-sweep). + if ( + toolCallId && + (rawFrame.status === "completed" || rawFrame.status === "failed") + ) { + notifyToolCallClosed(toolCallId); + } + // A sibling announced after the pause with neither a gate nor an allow cannot execute; + // the idempotent re-sweep closes it so the client never holds an orphaned part. if (pause.active) { run.settleOpenToolCalls( - (id) => pause.isPausedToolCall(id), + (id) => + pause.isPausedToolCall(id) || pause.isAllowedExecution(id), TOOL_NOT_EXECUTED_PAUSED, ); } @@ -342,6 +410,62 @@ export async function runTurn( // The SAME name->spec index the relay execute loop hands to the relay execution guard, so // the approval card and the guard cannot disagree about a tool's permission/readOnly. const specsByName = toolSpecsByName(plan.toolSpecs); + const settleBufferedPausedCompletions = (): void => { + for (const [toolCallId, update] of [ + ...bufferedPausedCompletedFrames.entries(), + ]) { + bufferedPausedCompletedFrames.delete(toolCallId); + if (pause.isPausedToolCall(toolCallId)) continue; + if (pause.isAllowedExecution(toolCallId)) { + run.handleUpdate(update); + notifyToolCallClosed(toolCallId); + continue; + } + const frame = update as { + sessionUpdate?: unknown; + name?: unknown; + title?: unknown; + kind?: unknown; + rawInput?: unknown; + input?: unknown; + }; + const { gate } = buildGateDescriptor( + { + toolCallId, + name: frame.name, + title: frame.title, + kind: frame.kind, + rawInput: frame.rawInput, + input: frame.input, + }, + run, + serverPermissions, + specsByName, + ); + const permission = effectivePermission(gate, permissionPlan); + if (permission === "allow") { + run.handleUpdate(update); + notifyToolCallClosed(toolCallId); + continue; + } + // Execution of an ask-policy call requires an answered allow; both harness gate paths fail + // closed. A completed frame during a pause for an unanswered ask-policy call is therefore + // a cancellation-closure artifact, not evidence of execution. + if ( + frame.sessionUpdate === "tool_call" && + !openToolCallIds().includes(toolCallId) + ) { + run.handleUpdate({ + ...(update as Record), + status: undefined, + }); + } + run.settleOpenToolCalls( + (id) => id !== toolCallId, + TOOL_NOT_EXECUTED_PAUSED, + ); + } + }; // Build the per-turn permission handler WITHOUT attaching to the live session: the // session-lifetime `onPermissionRequest` (in acquireEnvironment) routes into it via // `currentTurn`. A capturing shim reuses attachPermissionResponder unchanged; its @@ -360,6 +484,7 @@ export async function runTurn( log: logger, onPause: () => pause.pause(), onPausedToolCall: (id) => pause.markPausedToolCall(id), + onAllowedExecution: (id) => pause.markAllowedExecution(id), onNonParkablePause: () => { env.nonParkablePauseCount += 1; }, @@ -505,6 +630,7 @@ export async function runTurn( // confirm resolves) passes the relay guard. Claude resumes grant too — harmlessly, no // guard consults it. if (decision.reply === "once") { + pause.markAllowedExecution(decision.toolCallId); executionGrants.grant(decision.toolName, decision.args); } // A live-resume deny closes the seeded call as a failed tool call; flag it so the egress @@ -543,15 +669,40 @@ export async function runTurn( } const stopReason = raced === PAUSED || pause.active ? "paused" : (raced as any)?.stopReason; - // Pause notification is immediate, but terminalization must wait for managed cancellation - // and already-queued ACP updates. Re-sweep after the drain so a sibling announced during - // cancellation receives exactly one deterministic terminal result before `done`. + // Terminalization drains queued gates, classifies pause-time completions, and gives allowed + // executions their original per-call bound before the orphan sweep closes the turn. if (stopReason === "paused") { await pause.waitForEventDrain(); + settleBufferedPausedCompletions(); + const openAllowedExecutions = openToolCallIds() + .filter((id) => pause.isAllowedExecution(id)); + await Promise.all( + openAllowedExecutions.map(async (toolCallId) => { + const closed = await waitForToolCallClosure( + toolCallId, + resolvedRunLimits.toolCallMs, + ); + if (closed) return; + run.settleOpenToolCalls( + (id) => id !== toolCallId, + APPROVED_EXECUTION_RESULT_UNKNOWN, + ); + }), + ); + settleBufferedPausedCompletions(); run.settleOpenToolCalls( - (id) => pause.isPausedToolCall(id), + (id) => + pause.isPausedToolCall(id) || pause.isAllowedExecution(id), TOOL_NOT_EXECUTED_PAUSED, ); + const unexpectedOpenToolCallIds = openToolCallIds() + .filter((id) => !pause.isPausedToolCall(id)); + if (unexpectedOpenToolCallIds.length > 0) { + logger( + "[HITL] paused-turn transcript invariant left non-gated calls open: " + + unexpectedOpenToolCallIds.join(","), + ); + } } const result = raced === PAUSED ? undefined : raced; // A parkable pause this turn: hand the still-pending prompt promise to EVERY parked record so a diff --git a/services/runner/src/engines/sandbox_agent/runtime-policy.ts b/services/runner/src/engines/sandbox_agent/runtime-policy.ts index bc3c96cc90..80eb5104c1 100644 --- a/services/runner/src/engines/sandbox_agent/runtime-policy.ts +++ b/services/runner/src/engines/sandbox_agent/runtime-policy.ts @@ -42,18 +42,14 @@ export function shouldSuppressPausedToolCallUpdate( // F-024: a paused (gated) tool call's later frames are teardown artifacts and never reach the // stream. if (pause.isPausedToolCall(toolCallId)) return true; - // Once the turn is pausing, a FAILED update for any other open call is a managed-cancel - // artifact (a sibling that will not run this turn), so suppress it and let the deterministic - // post-drain sweep settle that call as TOOL_NOT_EXECUTED_PAUSED — the human sees "paused", not a - // spurious cancellation error. A `completed` update is NOT suppressed: an auto-allowed sibling - // that legitimately finishes while another gate holds the (kept-alive) warm session must show - // its real result, not be overwritten as paused. Announcements (`tool_call`) are never - // suppressed: the call must be recorded so the sweep knows which calls to settle. (Removing the - // eager first-pause settle is what lets concurrent gates each mark their own call paused before - // the sweep runs; this narrow suppression only masks the cancel artifact.) Residual: a genuine - // mid-pause tool error also arrives as `failed` and reads as paused here — acceptable, since the - // paused result invites a retry and the runner has no adapter provenance to tell the two apart. - if (kind === "tool_call_update" && pause.active && frame?.status === "failed") { + // While pausing, a failed frame without an allow is a managed-cancel artifact that the sweep + // records as deferred; an allowed call failure is genuine terminal evidence and must pass. + if ( + kind === "tool_call_update" && + pause.active && + frame?.status === "failed" && + !pause.isAllowedExecution(toolCallId) + ) { return true; } return false; diff --git a/services/runner/src/responder.ts b/services/runner/src/responder.ts index 21b5ec328a..f201597cfd 100644 --- a/services/runner/src/responder.ts +++ b/services/runner/src/responder.ts @@ -7,7 +7,10 @@ */ import type { AgentRunRequest, ContentBlock } from "./protocol.ts"; -import { DEFERRED_NOT_EXECUTED_PREFIX } from "./tracing/otel.ts"; +import { + APPROVED_EXECUTION_RESULT_UNKNOWN, + DEFERRED_NOT_EXECUTED_PREFIX, +} from "./tracing/otel.ts"; import { decide, effectivePermission, @@ -395,10 +398,9 @@ export function extractClientToolOutputs( const callShapeById = buildCallShapeIndex(request); for (const block of currentTurnToolResultBlocks(request)) { if (approvalDecisionOf(block) !== undefined) continue; // an approval, not a client output - // A sibling force-settled while another interaction paused this turn was NOT executed - // (`DEFERRED_NOT_EXECUTED`). It must not fulfill the model's retry of the same call, or the - // re-request resolves against the deferral and never re-parks (no new widget). - if (isDeferredNotExecuted(block)) continue; + // Pause terminalization sentinels are not client outputs: deferred work may re-park, while + // approved work with an unknown result must not be fulfilled or retried. + if (isPauseSyntheticResult(block)) continue; const argsKey = coldReplayKey(block, callShapeById); if (!argsKey) continue; const list = outputs.get(argsKey) ?? []; @@ -484,14 +486,14 @@ function isPermissionDecision(value: unknown): value is PermissionDecision { } /** - * A sibling force-settle result: the turn paused on another interaction, so this client tool was - * never executed and carries the `DEFERRED_NOT_EXECUTED` sentinel. It is not a browser output — the - * model is meant to re-issue the same call, which must re-park rather than resolve against this. + * Pause terminalization sentinels are runner bookkeeping, not browser outputs. A deferred call + * may safely re-park, while an approved call with an unobserved result must never be retried. */ -function isDeferredNotExecuted(block: ContentBlock): boolean { +function isPauseSyntheticResult(block: ContentBlock): boolean { return ( typeof block.output === "string" && - block.output.startsWith(DEFERRED_NOT_EXECUTED_PREFIX) + (block.output.startsWith(DEFERRED_NOT_EXECUTED_PREFIX) || + block.output === APPROVED_EXECUTION_RESULT_UNKNOWN) ); } diff --git a/services/runner/src/tracing/otel.ts b/services/runner/src/tracing/otel.ts index 5a78582b97..f446286671 100644 --- a/services/runner/src/tracing/otel.ts +++ b/services/runner/src/tracing/otel.ts @@ -65,6 +65,9 @@ export const DEFERRED_NOT_EXECUTED_PREFIX = "DEFERRED_NOT_EXECUTED"; export const TOOL_NOT_EXECUTED_PAUSED = `${DEFERRED_NOT_EXECUTED_PREFIX}: paused for another approval; retry the same call if still required.`; +export const APPROVED_EXECUTION_RESULT_UNKNOWN = + "APPROVED_EXECUTION_RESULT_UNKNOWN: the approved call started but its result was not observed before the pause ended the turn; do not assume it failed and do not retry a side-effecting call."; + // --------------------------------------------------------------------------- // Shared, process-wide tracing infrastructure // --------------------------------------------------------------------------- @@ -1086,6 +1089,8 @@ export interface SandboxAgentOtel { isExcluded: (id: string) => boolean, message: string, ): void; + /** Snapshot the ids whose tool calls have no terminal result yet. */ + openToolCallIds(): string[]; /** * Mark a tool-call id as DENIED (the runner replied `reject` to its gate). Its closing failed * result then carries `denied: true` so the egress projects `tool-output-denied` (a decline) @@ -1615,6 +1620,7 @@ export function createSandboxAgentOtel( output: () => accumulated, events: () => events, settleOpenToolCalls, + openToolCallIds: () => [...toolSpans.keys()], markToolCallDenied, usage: () => usage, }; diff --git a/services/runner/tests/unit/pending-approval-pause.test.ts b/services/runner/tests/unit/pending-approval-pause.test.ts index ca70f9a14b..302eceb0d4 100644 --- a/services/runner/tests/unit/pending-approval-pause.test.ts +++ b/services/runner/tests/unit/pending-approval-pause.test.ts @@ -53,4 +53,18 @@ describe("PendingApprovalPauseController", () => { await assert.doesNotReject(pause.signal); await assert.doesNotReject(pause.waitForEventDrain()); }); + + it("tracks allowed execution ids independently from paused gates", () => { + const pause = new PendingApprovalPauseController(() => {}); + + assert.equal(pause.isAllowedExecution(undefined), false); + assert.equal(pause.isAllowedExecution("tool-1"), false); + + pause.markAllowedExecution("tool-1"); + pause.markPausedToolCall("tool-2"); + + assert.equal(pause.isAllowedExecution("tool-1"), true); + assert.equal(pause.isPausedToolCall("tool-1"), false); + assert.equal(pause.isAllowedExecution("tool-2"), false); + }); }); diff --git a/services/runner/tests/unit/responder.test.ts b/services/runner/tests/unit/responder.test.ts index 83a75536e4..130aa69445 100644 --- a/services/runner/tests/unit/responder.test.ts +++ b/services/runner/tests/unit/responder.test.ts @@ -7,6 +7,7 @@ import { describe, it } from "vitest"; import assert from "node:assert/strict"; import { + APPROVED_EXECUTION_RESULT_UNKNOWN, createSandboxAgentOtel, TOOL_NOT_EXECUTED_PAUSED, } from "../../src/tracing/otel.ts"; @@ -701,6 +702,14 @@ describe("client-tool output store (separate from approvals)", () => { output: TOOL_NOT_EXECUTED_PAUSED, isError: true, }, + { + type: "tool_result", + toolCallId: "c-linear", + toolName: "request_connection", + input: { integration: "linear" }, + output: APPROVED_EXECUTION_RESULT_UNKNOWN, + isError: true, + }, ], }, ], @@ -719,7 +728,13 @@ describe("client-tool output store (separate from approvals)", () => { ), false, ); - // So the model's retry of slack re-parks (a fresh widget) instead of resolving as fulfilled. + assert.equal( + outputs.has( + approvedCallKey("request_connection", { integration: "linear" })!, + ), + false, + ); + // So the model retry of slack re-parks (a fresh widget) instead of resolving as fulfilled. const responder = new ApprovalResponder( plan("ask"), new ConversationDecisions(extractApprovalDecisions(request), outputs), diff --git a/services/runner/tests/unit/sandbox-agent-orchestration.test.ts b/services/runner/tests/unit/sandbox-agent-orchestration.test.ts index 1873c4f5ee..83ad9702b7 100644 --- a/services/runner/tests/unit/sandbox-agent-orchestration.test.ts +++ b/services/runner/tests/unit/sandbox-agent-orchestration.test.ts @@ -15,6 +15,7 @@ import { TOOL_NOT_EXECUTED_PAUSED, } from "../../src/tracing/otel.ts"; import { PendingApprovalPauseController } from "../../src/engines/sandbox_agent/pause.ts"; +import { shouldSuppressPausedToolCallUpdate } from "../../src/engines/sandbox_agent/runtime-policy.ts"; import { mountStorage } from "../../src/engines/sandbox_agent/mount.ts"; import { buildPiGateEnvelope } from "../../src/engines/sandbox_agent/pi-gate-envelope.ts"; import type { PermissionDecision } from "../../src/responder.ts"; @@ -219,6 +220,9 @@ function fakeHarness(options: FakeOptions = {}) { _isExcluded: (id: string) => boolean, _message: string, ) {}, + openToolCallIds() { + return []; + }, traceId() { return "trace-1"; }, @@ -314,6 +318,24 @@ describe("PendingApprovalPauseController", () => { assert.equal(pause.isPausedToolCall("tool-1"), true); assert.equal(pause.isPausedToolCall("tool-2"), false); }); + + it("passes through a failed frame for an allowed execution during a pause", () => { + const pause = new PendingApprovalPauseController(() => {}); + pause.markAllowedExecution("tool-allowed"); + pause.pause(); + + assert.equal( + shouldSuppressPausedToolCallUpdate( + { + sessionUpdate: "tool_call_update", + toolCallId: "tool-allowed", + status: "failed", + }, + pause, + ), + false, + ); + }); }); describe("runSandboxAgent orchestration", () => { @@ -1348,6 +1370,9 @@ describe("runSandboxAgent orchestration", () => { return []; }, settleOpenToolCalls() {}, + openToolCallIds() { + return []; + }, traceId() { return "trace-1"; }, @@ -2097,6 +2122,105 @@ describe("runSandboxAgent default ApprovalResponder wiring", () => { ); }); + it("classifies pause-time completed frames by effective permission", async () => { + const { deps } = fakeHarness({ + promptEvents: [ + { + payload: { + update: { + sessionUpdate: "tool_call", + toolCallId: "tool-ask", + title: "ask_sibling", + }, + }, + }, + { + payload: { + update: { + sessionUpdate: "tool_call", + toolCallId: "tool-allow", + title: "allow_sibling", + }, + }, + }, + { + payload: { + update: { + sessionUpdate: "tool_call", + toolCallId: "tool-gate", + title: "pause_gate", + }, + }, + }, + ], + emitPermission: true, + permissionToolCallId: "tool-gate", + permissionToolName: "pause_gate", + postPermissionEvents: [ + { + payload: { + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-ask", + status: "completed", + content: "cancellation closure", + }, + }, + }, + { + payload: { + update: { + sessionUpdate: "tool_call_update", + toolCallId: "tool-allow", + status: "completed", + content: "real allow result", + }, + }, + }, + ], + hangPrompt: true, + }); + delete deps.responderFactory; + deps.createOtel = createSandboxAgentOtel as any; + + const result = await runSandboxAgent( + { + harness: "claude", + permissions: { default: "ask" }, + customTools: [ + { name: "ask_sibling", permission: "ask" }, + { name: "allow_sibling", permission: "allow" }, + { name: "pause_gate", permission: "ask" }, + ], + messages: [{ role: "user", content: "run the siblings" }], + } as AgentRunRequest, + undefined, + undefined, + deps, + ); + + assert.equal(result.ok, true); + if (!result.ok) return; + assert.equal(result.stopReason, "paused"); + const toolResults = new Map( + result.events + ?.filter((event) => event.type === "tool_result") + .map((event) => [(event as any).id, event]), + ); + assert.deepEqual(toolResults.get("tool-ask"), { + type: "tool_result", + id: "tool-ask", + output: TOOL_NOT_EXECUTED_PAUSED, + isError: true, + }); + assert.deepEqual(toolResults.get("tool-allow"), { + type: "tool_result", + id: "tool-allow", + output: "real allow result", + isError: false, + }); + }); + it("park ENDS the turn even when the prompt hangs: terminal stopReason 'paused', no harness reply (F-040)", async () => { // The real regression: Claude does NOT end a turn on an unanswered gate, so without the // park->cancel path `session.prompt()` blocks forever and the run never returns. With diff --git a/services/runner/tests/unit/session-keepalive-approval.test.ts b/services/runner/tests/unit/session-keepalive-approval.test.ts index f743e06739..4d4832c6ef 100644 --- a/services/runner/tests/unit/session-keepalive-approval.test.ts +++ b/services/runner/tests/unit/session-keepalive-approval.test.ts @@ -33,7 +33,12 @@ import { type SandboxAgentDeps, type SessionEnvironment, } from "../../src/engines/sandbox_agent.ts"; -import { TOOL_NOT_EXECUTED_PAUSED } from "../../src/tracing/otel.ts"; +import { + APPROVED_EXECUTION_RESULT_UNKNOWN, + createSandboxAgentOtel, + DEFERRED_NOT_EXECUTED_PREFIX, + TOOL_NOT_EXECUTED_PAUSED, +} from "../../src/tracing/otel.ts"; function flush(): Promise { return new Promise((resolve) => setImmediate(resolve)); @@ -1088,6 +1093,10 @@ interface FakeRun { open: Set; /** What settleOpenToolCalls settled: the orphaned-sibling record (F-024). */ settled: Array<{ id: string; message: string }>; + sweeps: Array<{ + message: string; + isExcluded: (id: string) => boolean; + }>; } function pausableHarness(opts: { clientTool?: boolean } = {}) { @@ -1149,6 +1158,7 @@ function pausableHarness(opts: { clientTool?: boolean } = {}) { emitted: [], open: new Set(), settled: [], + sweeps: [], start() {}, // Track open tool calls the way the real otel run does, so settleOpenToolCalls is // meaningful: a tool_call opens an id, a completed/failed tool_call_update closes it. @@ -1187,12 +1197,16 @@ function pausableHarness(opts: { clientTool?: boolean } = {}) { isExcluded: (id: string) => boolean, message: string, ) { + run.sweeps.push({ message, isExcluded }); for (const id of [...run.open]) { if (isExcluded(id)) continue; run.open.delete(id); run.settled.push({ id, message }); } }, + openToolCallIds() { + return [...run.open]; + }, denied: [] as string[], markToolCallDenied(id: string | undefined) { if (id) run.denied.push(id); @@ -1386,6 +1400,276 @@ describe("runTurn: real approval park + respondPermission resume", () => { await env.destroy(); }); + it("excludes an allowed execution from both pause sweeps", async () => { + const { calls, deps, captured } = pausableHarness(); + const acquired = await acquireEnvironment(engineReq, deps); + assert.equal(acquired.ok, true); + if (!acquired.ok) return; + const env = acquired.env; + + const firstTurn = runTurn(env, engineReq, undefined, undefined, { + approvalParkMode: true, + }); + await flush(); + captured.onEvent!( + updateEvent({ + sessionUpdate: "tool_call", + toolCallId: "tc-approved", + title: "commit", + }), + ); + captured.onPermissionRequest!({ + id: "perm-approved", + availableReplies: ["once", "reject"], + toolCall: { toolCallId: "tc-approved", name: "commit" }, + }); + await firstTurn; + + const held = env.parkedApproval!.promptPromise!; + env.clearTurn(); + const resumeTurn = runTurn(env, approveResume(true), undefined, undefined, { + approvalParkMode: true, + resume: { + decisions: [ + { + permissionId: "perm-approved", + reply: "once", + toolCallId: "tc-approved", + toolName: "commit", + args: {}, + interactionToken: "tc-approved", + promptPromise: held, + }, + ], + }, + }); + await flush(); + captured.onEvent!( + updateEvent({ + sessionUpdate: "tool_call", + toolCallId: "tc-gate-2", + title: "deploy", + }), + ); + captured.onPermissionRequest!({ + id: "perm-2", + availableReplies: ["once", "reject"], + toolCall: { toolCallId: "tc-gate-2", name: "deploy" }, + }); + for (let i = 0; i < 5 && !env.currentTurn?.pause.active; i += 1) { + await Promise.resolve(); + } + assert.equal(env.currentTurn?.pause.active, true); + captured.onEvent!( + updateEvent({ + sessionUpdate: "tool_call_update", + toolCallId: "tc-approved", + status: "in_progress", + }), + ); + captured.onEvent!( + updateEvent({ + sessionUpdate: "tool_call_update", + toolCallId: "tc-approved", + status: "completed", + content: "real result", + }), + ); + await resumeTurn; + + const run = calls.runs[1]; + const deferredSweeps = run.sweeps.filter( + (sweep) => sweep.message === TOOL_NOT_EXECUTED_PAUSED, + ); + assert.ok(deferredSweeps.length >= 2); + assert.equal( + deferredSweeps.every((sweep) => sweep.isExcluded("tc-approved")), + true, + ); + assert.equal( + run.settled.some( + (entry) => + entry.id === "tc-approved" && + entry.message === TOOL_NOT_EXECUTED_PAUSED, + ), + false, + ); + + await env.destroy(); + }); + + it("records an approved completion that arrives after a sibling pause", async () => { + const { deps, captured } = pausableHarness(); + deps.createOtel = createSandboxAgentOtel as any; + const acquired = await acquireEnvironment(engineReq, deps); + assert.equal(acquired.ok, true); + if (!acquired.ok) return; + const env = acquired.env; + + const firstTurn = runTurn(env, engineReq, undefined, undefined, { + approvalParkMode: true, + }); + await flush(); + captured.onEvent!( + updateEvent({ + sessionUpdate: "tool_call", + toolCallId: "tc-approved", + title: "commit", + }), + ); + captured.onPermissionRequest!({ + id: "perm-approved", + availableReplies: ["once", "reject"], + toolCall: { toolCallId: "tc-approved", name: "commit" }, + }); + await firstTurn; + + const held = env.parkedApproval!.promptPromise!; + env.clearTurn(); + const resumeTurn = runTurn(env, approveResume(true), undefined, undefined, { + approvalParkMode: true, + resume: { + decisions: [ + { + permissionId: "perm-approved", + reply: "once", + toolCallId: "tc-approved", + toolName: "commit", + args: {}, + interactionToken: "tc-approved", + promptPromise: held, + }, + ], + }, + }); + await flush(); + captured.onEvent!( + updateEvent({ + sessionUpdate: "tool_call", + toolCallId: "tc-gate-2", + title: "deploy", + }), + ); + captured.onPermissionRequest!({ + id: "perm-2", + availableReplies: ["once", "reject"], + toolCall: { toolCallId: "tc-gate-2", name: "deploy" }, + }); + for (let i = 0; i < 5 && !env.currentTurn?.pause.active; i += 1) { + await Promise.resolve(); + } + assert.equal(env.currentTurn?.pause.active, true); + captured.onEvent!( + updateEvent({ + sessionUpdate: "tool_call_update", + toolCallId: "tc-approved", + status: "completed", + content: "approved call completed", + }), + ); + const result = await resumeTurn; + + assert.equal(result.ok, true); + if (!result.ok) return; + const approvedResults = result.events?.filter( + (event) => event.type === "tool_result" && event.id === "tc-approved", + ); + assert.deepEqual(approvedResults, [ + { + type: "tool_result", + id: "tc-approved", + output: "approved call completed", + isError: false, + }, + ]); + + await env.destroy(); + }); + + it("records the non-retry sentinel when an approved result misses the bound", async () => { + const { calls, deps, captured } = pausableHarness(); + deps.resolveRunLimits = () => ({ + totalMs: 1_000, + idleMs: 500, + ttfbMs: 500, + toolCallMs: 5, + }); + const acquired = await acquireEnvironment(engineReq, deps); + assert.equal(acquired.ok, true); + if (!acquired.ok) return; + const env = acquired.env; + + const firstTurn = runTurn(env, engineReq, undefined, undefined, { + approvalParkMode: true, + }); + await flush(); + captured.onEvent!( + updateEvent({ + sessionUpdate: "tool_call", + toolCallId: "tc-approved", + title: "commit", + }), + ); + captured.onPermissionRequest!({ + id: "perm-approved", + availableReplies: ["once", "reject"], + toolCall: { toolCallId: "tc-approved", name: "commit" }, + }); + await firstTurn; + + const held = env.parkedApproval!.promptPromise!; + env.clearTurn(); + const resumeTurn = runTurn(env, approveResume(true), undefined, undefined, { + approvalParkMode: true, + resume: { + decisions: [ + { + permissionId: "perm-approved", + reply: "once", + toolCallId: "tc-approved", + toolName: "commit", + args: {}, + interactionToken: "tc-approved", + promptPromise: held, + }, + ], + }, + }); + await flush(); + captured.onEvent!( + updateEvent({ + sessionUpdate: "tool_call", + toolCallId: "tc-gate-2", + title: "deploy", + }), + ); + captured.onPermissionRequest!({ + id: "perm-2", + availableReplies: ["once", "reject"], + toolCall: { toolCallId: "tc-gate-2", name: "deploy" }, + }); + const result = await resumeTurn; + + assert.equal(result.ok, true); + assert.deepEqual( + calls.runs[1].settled.filter((entry) => entry.id === "tc-approved"), + [ + { + id: "tc-approved", + message: APPROVED_EXECUTION_RESULT_UNKNOWN, + }, + ], + ); + assert.equal( + APPROVED_EXECUTION_RESULT_UNKNOWN.startsWith( + DEFERRED_NOT_EXECUTED_PREFIX, + ), + false, + ); + + await env.destroy(); + }); + it("forwards a reject on the resume when the decision is deny", async () => { const { calls, deps, captured } = pausableHarness(); const acquired = await acquireEnvironment(engineReq, deps); From 454a3aed4281250f7d5caafeb526305b826a4df3 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 19 Jul 2026 17:20:39 +0200 Subject: [PATCH 06/15] feat(sessions): persist the answer half of every approval gate --- api/oss/src/apis/fastapi/sessions/models.py | 1 + api/oss/src/apis/fastapi/sessions/router.py | 1 + .../src/core/sessions/interactions/dtos.py | 1 + .../dbs/postgres/sessions/interactions/dao.py | 26 ++++- .../test_interactions_transition_update.py | 80 ++++++++++++++ .../test_transition_interaction_resolution.py | 77 +++++++++++++ .../unit/sessions/test_wp5_dao_fanout.py | 71 ++++++++++++ sdks/python/agenta/sdk/agents/wire_models.py | 3 +- .../engines/sandbox_agent/acp-interactions.ts | 17 ++- .../src/engines/sandbox_agent/run-turn.ts | 30 +++++- services/runner/src/protocol.ts | 8 ++ services/runner/src/sessions/interactions.ts | 7 ++ services/runner/src/sessions/persist.ts | 8 +- .../sandbox-agent-acp-interactions.test.ts | 92 ++++++++++++++++ .../tests/unit/session-interactions.test.ts | 34 +++++- .../unit/session-keepalive-approval.test.ts | 25 +++++ .../runner/tests/unit/session-persist.test.ts | 32 ++++++ .../assets/transcriptToMessages.test.ts | 101 ++++++++++++++++++ .../assets/transcriptToMessages.ts | 23 ++++ 19 files changed, 620 insertions(+), 17 deletions(-) create mode 100644 api/oss/tests/pytest/unit/sessions/test_interactions_transition_update.py create mode 100644 api/oss/tests/pytest/unit/sessions/test_transition_interaction_resolution.py create mode 100644 web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.test.ts diff --git a/api/oss/src/apis/fastapi/sessions/models.py b/api/oss/src/apis/fastapi/sessions/models.py index 296478635d..9f969e5ca4 100644 --- a/api/oss/src/apis/fastapi/sessions/models.py +++ b/api/oss/src/apis/fastapi/sessions/models.py @@ -106,6 +106,7 @@ class SessionInteractionTransitionRequest(BaseModel): session_id: str token: str status: SessionInteractionStatus + resolution: Optional[Dict[str, Any]] = None class SessionInteractionCancelStaleRequest(BaseModel): diff --git a/api/oss/src/apis/fastapi/sessions/router.py b/api/oss/src/apis/fastapi/sessions/router.py index 50825eda8f..f8994eb76d 100644 --- a/api/oss/src/apis/fastapi/sessions/router.py +++ b/api/oss/src/apis/fastapi/sessions/router.py @@ -647,6 +647,7 @@ async def transition_interaction( session_id=body.session_id, token=body.token, status=body.status, + resolution=body.resolution, ), ) except InteractionNotFound: diff --git a/api/oss/src/core/sessions/interactions/dtos.py b/api/oss/src/core/sessions/interactions/dtos.py index 89e86ad64e..cd136a3a76 100644 --- a/api/oss/src/core/sessions/interactions/dtos.py +++ b/api/oss/src/core/sessions/interactions/dtos.py @@ -68,6 +68,7 @@ class SessionInteractionTransition(BaseModel): session_id: str token: str status: SessionInteractionStatus + resolution: Optional[Dict[str, Any]] = None class SessionInteractionQuery(BaseModel): diff --git a/api/oss/src/dbs/postgres/sessions/interactions/dao.py b/api/oss/src/dbs/postgres/sessions/interactions/dao.py index 7ec4a66b0d..33f8b0a159 100644 --- a/api/oss/src/dbs/postgres/sessions/interactions/dao.py +++ b/api/oss/src/dbs/postgres/sessions/interactions/dao.py @@ -2,7 +2,8 @@ from typing import List, Optional from uuid import UUID -from sqlalchemy import delete as sa_delete, func, select, update as sa_update +from sqlalchemy import cast, delete as sa_delete, func, select, update as sa_update +from sqlalchemy.dialects.postgresql import JSON, JSONB, array from sqlalchemy.exc import IntegrityError from oss.src.core.sessions.interactions.dtos import ( @@ -97,6 +98,24 @@ async def transition_interaction( # Only non-terminal interactions transition: pending (responded|resolved| # cancelled) and responded (resolved, when the runner consumes an API-plane # answer). resolved/cancelled are terminal. + transition_values = { + "status": transition.status.value, + "updated_at": datetime.now(timezone.utc), + } + if transition.resolution is not None: + # Preserve request/references while atomically adding the answer under the guard. + transition_values["data"] = cast( + func.jsonb_set( + func.coalesce( + cast(SessionInteractionDBE.data, JSONB), + cast({}, JSONB), + ), + array(["resolution"]), + cast(transition.resolution, JSONB), + True, + ), + JSON, + ) stmt = ( sa_update(SessionInteractionDBE) .where( @@ -105,10 +124,7 @@ async def transition_interaction( SessionInteractionDBE.token == transition.token, SessionInteractionDBE.status.in_(("pending", "responded")), ) - .values( - status=transition.status.value, - updated_at=datetime.now(timezone.utc), - ) + .values(**transition_values) .returning(SessionInteractionDBE) ) result = await session.execute(stmt) diff --git a/api/oss/tests/pytest/unit/sessions/test_interactions_transition_update.py b/api/oss/tests/pytest/unit/sessions/test_interactions_transition_update.py new file mode 100644 index 0000000000..47a96107a4 --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_interactions_transition_update.py @@ -0,0 +1,80 @@ +from uuid import uuid4 + +from sqlalchemy.dialects import postgresql + +from oss.src.core.sessions.interactions.dtos import ( + SessionInteractionStatus, + SessionInteractionTransition, +) +from oss.src.dbs.postgres.sessions.interactions.dao import SessionInteractionsDAO + + +class _Result: + def scalar_one_or_none(self): + return None + + +class _Session: + def __init__(self): + self.statement = None + + async def execute(self, statement): + self.statement = statement + return _Result() + + async def commit(self): + return None + + +class _SessionContext: + def __init__(self, session): + self.session = session + + async def __aenter__(self): + return self.session + + async def __aexit__(self, exc_type, exc, traceback): + return False + + +class _Engine: + def __init__(self, session): + self._session = session + + def session(self): + return _SessionContext(self._session) + + +async def _transition_statement(resolution=None): + session = _Session() + dao = SessionInteractionsDAO(engine=_Engine(session)) + await dao.transition_interaction( + transition=SessionInteractionTransition( + project_id=uuid4(), + session_id="session-1", + token="token-1", + status=SessionInteractionStatus.resolved, + resolution=resolution, + ) + ) + return session.statement + + +async def test_resolution_update_preserves_existing_data_with_jsonb_set(): + resolution = {"verdict": "approved", "tool_call_id": "tool-1"} + statement = await _transition_statement(resolution) + compiled = statement.compile(dialect=postgresql.dialect()) + set_clause = str(compiled).split(" WHERE ", maxsplit=1)[0] + + assert "jsonb_set" in set_clause + assert "CAST(session_interactions.data AS JSONB)" in set_clause + assert resolution in compiled.params.values() + + +async def test_transition_without_resolution_does_not_write_data(): + statement = await _transition_statement() + set_clause = str(statement.compile(dialect=postgresql.dialect())).split( + " WHERE ", maxsplit=1 + )[0] + + assert "data=" not in set_clause diff --git a/api/oss/tests/pytest/unit/sessions/test_transition_interaction_resolution.py b/api/oss/tests/pytest/unit/sessions/test_transition_interaction_resolution.py new file mode 100644 index 0000000000..0256fa010c --- /dev/null +++ b/api/oss/tests/pytest/unit/sessions/test_transition_interaction_resolution.py @@ -0,0 +1,77 @@ +from unittest.mock import AsyncMock, patch +from uuid import uuid4 + +from fastapi import FastAPI, Request + +from oss.src.apis.fastapi.sessions.models import SessionInteractionTransitionRequest +from oss.src.apis.fastapi.sessions.router import InteractionsRouter +from oss.src.core.sessions.interactions.dtos import ( + SessionInteraction, + SessionInteractionData, + SessionInteractionKind, + SessionInteractionStatus, +) + + +def _make_authed_request(app: FastAPI, project_id, user_id) -> Request: + request = Request( + { + "type": "http", + "method": "POST", + "path": "/sessions/interactions/transition", + "headers": [], + "app": app, + } + ) + request.state.project_id = project_id + request.state.user_id = user_id + return request + + +async def test_transition_route_passes_resolution_to_the_domain_transition(): + project_id = uuid4() + user_id = uuid4() + captured = [] + + class _InteractionsService: + async def transition_interaction(self, *, transition): + captured.append(transition) + return SessionInteraction( + project_id=transition.project_id, + session_id=transition.session_id, + token=transition.token, + kind=SessionInteractionKind.user_approval, + status=transition.status, + data=SessionInteractionData(resolution=transition.resolution), + ) + + router = InteractionsRouter( + interactions_service=_InteractionsService(), + workflows_service=AsyncMock(), + respond_task=AsyncMock(), + ) + body = SessionInteractionTransitionRequest( + session_id="session-1", + token="approval-token", + status=SessionInteractionStatus.resolved, + resolution={"verdict": "denied", "tool_call_id": "tool-1"}, + ) + + with patch( + "oss.src.apis.fastapi.sessions.router.check_action_access", + new_callable=AsyncMock, + return_value=True, + ): + response = await router.transition_interaction( + request=_make_authed_request(FastAPI(), project_id, user_id), + body=body, + ) + + assert len(captured) == 1 + assert captured[0].resolution == { + "verdict": "denied", + "tool_call_id": "tool-1", + } + assert response.interaction is not None + assert response.interaction.data is not None + assert response.interaction.data.resolution == captured[0].resolution diff --git a/api/oss/tests/pytest/unit/sessions/test_wp5_dao_fanout.py b/api/oss/tests/pytest/unit/sessions/test_wp5_dao_fanout.py index 810933bde4..d867d1eef2 100644 --- a/api/oss/tests/pytest/unit/sessions/test_wp5_dao_fanout.py +++ b/api/oss/tests/pytest/unit/sessions/test_wp5_dao_fanout.py @@ -22,8 +22,11 @@ from oss.src.core.sessions.interactions.dtos import ( SessionInteractionCreate, + SessionInteractionData, SessionInteractionKind, SessionInteractionQuery, + SessionInteractionStatus, + SessionInteractionTransition, ) from oss.src.core.mounts.dtos import MountCreate import oss.src.models.db_models # noqa: F401 @@ -143,6 +146,74 @@ def mounts_dao(): return MountsDAO(engine=get_transactions_engine()) +# --------------------------------------------------------------------------- +# SessionInteractionsDAO transition resolution +# --------------------------------------------------------------------------- + + +async def test_interaction_transition_preserves_data_and_optionally_adds_resolution( + interactions_dao, project +): + project_id = project["project_id"] + session_id = f"interaction-resolution-{uuid.uuid4().hex[:8]}" + request = {"tool": "bash", "args": {"command": "ls"}} + + await interactions_dao.create_interaction( + project_id=project_id, + user_id=None, + interaction=SessionInteractionCreate( + project_id=project_id, + session_id=session_id, + token="approval-token", + kind=SessionInteractionKind.user_approval, + data=SessionInteractionData(request=request), + ), + ) + transitioned = await interactions_dao.transition_interaction( + transition=SessionInteractionTransition( + project_id=project_id, + session_id=session_id, + token="approval-token", + status=SessionInteractionStatus.resolved, + resolution={"verdict": "approved", "tool_call_id": "tool-1"}, + ) + ) + + assert transitioned is not None + assert transitioned.status == SessionInteractionStatus.resolved + assert transitioned.data is not None + assert transitioned.data.request == request + assert transitioned.data.resolution == { + "verdict": "approved", + "tool_call_id": "tool-1", + } + + await interactions_dao.create_interaction( + project_id=project_id, + user_id=None, + interaction=SessionInteractionCreate( + project_id=project_id, + session_id=session_id, + token="client-token", + kind=SessionInteractionKind.client_tool, + data=SessionInteractionData(request=request), + ), + ) + transitioned_without_resolution = await interactions_dao.transition_interaction( + transition=SessionInteractionTransition( + project_id=project_id, + session_id=session_id, + token="client-token", + status=SessionInteractionStatus.resolved, + ) + ) + + assert transitioned_without_resolution is not None + assert transitioned_without_resolution.data is not None + assert transitioned_without_resolution.data.request == request + assert transitioned_without_resolution.data.resolution is None + + # --------------------------------------------------------------------------- # SessionInteractionsDAO.delete_by_session_id — new hard delete # --------------------------------------------------------------------------- diff --git a/sdks/python/agenta/sdk/agents/wire_models.py b/sdks/python/agenta/sdk/agents/wire_models.py index eb5366cc1d..759f227a7a 100644 --- a/sdks/python/agenta/sdk/agents/wire_models.py +++ b/sdks/python/agenta/sdk/agents/wire_models.py @@ -378,7 +378,8 @@ class WireEvent(_WireModel): the forward-compatible event types the runner may add, which contradicts the "drop unknown" guarantee. The known ``type`` values are documented for readers, not enforced: ``message``, ``thought``, the ``message_*`` / ``reasoning_*`` lifecycle trios, ``tool_call``, - ``tool_result``, ``interaction_request``, ``data``, ``file``, ``usage``, ``error``, ``done``. + ``tool_result``, ``interaction_request``, ``interaction_response``, ``data``, ``file``, + ``usage``, ``error``, ``done``. """ type: Optional[str] = None diff --git a/services/runner/src/engines/sandbox_agent/acp-interactions.ts b/services/runner/src/engines/sandbox_agent/acp-interactions.ts index 6738e532e1..321d6e6c29 100644 --- a/services/runner/src/engines/sandbox_agent/acp-interactions.ts +++ b/services/runner/src/engines/sandbox_agent/acp-interactions.ts @@ -69,7 +69,10 @@ export interface AttachPermissionResponderInput { kind: "user_approval" | "client_tool", ) => void; /** Called after a stored decision was successfully forwarded to the harness. */ - onResolveInteraction?: (token: string) => void; + onResolveInteraction?: ( + token: string, + verdict?: { approved: boolean; toolCallId: string }, + ) => void; /** * Fires for EVERY parkable permission gate (a Claude ACP gate or a Pi ACP gate) that * resolves to pendingApproval. Keep-alive uses it to record every parked permission id / @@ -223,9 +226,12 @@ export function attachPermissionResponder({ }; // Resolve the durable row this gate created, if it created one. - const resolveIfCreated = (id: string): void => { + const resolveIfCreated = ( + id: string, + verdict?: { approved: boolean; toolCallId: string }, + ): void => { if (!createdInteractionIds.delete(id)) return; - onResolveInteraction?.(id); + onResolveInteraction?.(id, verdict); }; const replyPermission = async ( @@ -254,7 +260,10 @@ export function attachPermissionResponder({ onPause?.(); return; } - resolveIfCreated(id); + resolveIfCreated(id, { + approved: decision === "allow", + toolCallId: toolCallId ?? id, + }); }; const replyClientTool = async ( diff --git a/services/runner/src/engines/sandbox_agent/run-turn.ts b/services/runner/src/engines/sandbox_agent/run-turn.ts index 6f955911cc..ba6fa66d88 100644 --- a/services/runner/src/engines/sandbox_agent/run-turn.ts +++ b/services/runner/src/engines/sandbox_agent/run-turn.ts @@ -401,10 +401,31 @@ export async function runTurn( // interactions-plane answer already transitioned it to responded, and an in-band answer is // detected at sweep time (`inBandAnswerToken`) and exempted via the sweep's `tokens` — the // row stays pending until this resolve lands it as resolved, never cancelled. - const resolveInteractionToken = (token: string): void => { + const resolveInteractionToken = ( + token: string, + verdict?: { approved: boolean; toolCallId: string }, + ): void => { + if (verdict) { + run.emitEvent({ + type: "interaction_response", + id: token, + kind: "user_approval", + payload: verdict, + }); + } const cred = runCredential(request); if (!cred) return; - void resolveInteraction(sessionId, token, () => cred); + void resolveInteraction( + sessionId, + token, + () => cred, + verdict + ? { + verdict: verdict.approved ? "approved" : "denied", + tool_call_id: verdict.toolCallId, + } + : undefined, + ); }; const serverPermissions = serverPermissionsFromRequest(request); // The SAME name->spec index the relay execute loop hands to the relay execution guard, so @@ -646,7 +667,10 @@ export async function runTurn( // cold path would otherwise resolve via its decision map). The fresh per-turn pause // controller starts with an EMPTY pausedToolCallIds set, so the resumed calls' // `tool_call_update` frames are no longer suppressed and stream through. - resolveInteractionToken(decision.interactionToken); + resolveInteractionToken(decision.interactionToken, { + approved: decision.reply === "once", + toolCallId: decision.toolCallId, + }); logger( `[keepalive] resume answered gate reply=${decision.reply} tool=${decision.toolName ?? "?"}`, ); diff --git a/services/runner/src/protocol.ts b/services/runner/src/protocol.ts index cdf95a6c08..168ce5c53e 100644 --- a/services/runner/src/protocol.ts +++ b/services/runner/src/protocol.ts @@ -365,6 +365,14 @@ export type AgentEvent = kind: "user_approval" | "user_input" | "client_tool"; payload?: unknown; } + // The durable answer half of an interaction_request, emitted when the runner forwards the + // human decision to the harness. Its id matches the request so hydration can overlay the answer. + | { + type: "interaction_response"; + id: string; + kind: "user_approval"; + payload?: unknown; + } // One-way generative-UI payloads (not tied to a tool result). `data` -> Vercel `data-`, // `file` -> Vercel `file`. | { type: "data"; name: string; data: unknown; transient?: boolean } diff --git a/services/runner/src/sessions/interactions.ts b/services/runner/src/sessions/interactions.ts index dcbaa67309..6fd2f44b11 100644 --- a/services/runner/src/sessions/interactions.ts +++ b/services/runner/src/sessions/interactions.ts @@ -8,6 +8,11 @@ import { apiBase } from "../apiBase.ts"; export type InteractionKind = "user_approval" | "user_input" | "client_tool"; +export type InteractionResolution = { + verdict: "approved" | "denied"; + tool_call_id: string; +}; + /** A platform entity reference (the API `Reference` shape). */ type Reference = { id?: string; slug?: string; version?: string }; @@ -98,6 +103,7 @@ export async function resolveInteraction( sessionId: string, token: string, auth: () => string, + resolution?: InteractionResolution, ): Promise { try { const res = await fetch(`${apiBase()}/sessions/interactions/transition`, { @@ -107,6 +113,7 @@ export async function resolveInteraction( session_id: sessionId, token, status: "resolved", + ...(resolution ? { resolution } : {}), }), }); if (!res.ok) throw new Error(`HTTP ${res.status}`); diff --git a/services/runner/src/sessions/persist.ts b/services/runner/src/sessions/persist.ts index b599c44d6c..1006de5227 100644 --- a/services/runner/src/sessions/persist.ts +++ b/services/runner/src/sessions/persist.ts @@ -292,10 +292,12 @@ export function buildPersistingEmitter( } } - // A tool_result / interaction_request carries the same tool-call id as its tool_call; - // give it its own stable id (keyed on the record type) so it lands on a distinct row. + // Related tool and interaction events share correlation ids but need distinct, retry-stable + // rows, so the record type remains part of the stable id. if ( - (event.type === "tool_result" || event.type === "interaction_request") && + (event.type === "tool_result" || + event.type === "interaction_request" || + event.type === "interaction_response") && event.id ) { persistEvent( diff --git a/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts b/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts index bdb1f7ed63..d3650d58b4 100644 --- a/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts +++ b/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts @@ -446,6 +446,98 @@ describe("attachPermissionResponder", () => { assert.deepEqual(events, []); }); + it("passes the approval verdict when resolving a previously created gate", async () => { + const { session, emit } = makeSession(); + let permissionCalls = 0; + const resolved: Array<{ + token: string; + verdict?: { approved: boolean; toolCallId: string }; + }> = []; + + attachPermissionResponder({ + session, + run: { emitEvent: () => {} }, + responder: { + async onPermission() { + permissionCalls += 1; + return permissionCalls === 1 + ? ({ kind: "pendingApproval" } as const) + : ({ kind: "deny" } as const); + }, + async onClientTool() { + return { kind: "deny" } as const; + }, + }, + onResolveInteraction: (token, verdict) => { + resolved.push({ token, verdict }); + }, + }); + const request = { + id: "approval-1", + availableReplies: ["once", "reject"], + toolCall: { toolCallId: "tool-1", name: "bash" }, + }; + emit(request); + await flushPromises(); + emit(request); + await flushPromises(); + + assert.deepEqual(resolved, [ + { + token: "approval-1", + verdict: { approved: false, toolCallId: "tool-1" }, + }, + ]); + }); + + it("resolves a client-tool row without an approval verdict", async () => { + const replies: Array<{ id: string; reply: string }> = []; + const { session, emit } = makeSession(async (id, reply) => { + replies.push({ id, reply }); + }); + let clientCalls = 0; + const resolved: Array<{ + token: string; + verdict?: { approved: boolean; toolCallId: string }; + }> = []; + + attachPermissionResponder({ + session, + run: { emitEvent: () => {} }, + responder: { + async onPermission() { + return { kind: "deny" } as const; + }, + async onClientTool() { + clientCalls += 1; + return clientCalls === 1 + ? ({ kind: "pendingApproval" } as const) + : ({ kind: "fulfilled", output: { connected: true } } as const); + }, + }, + toolSpecsByName: specsByName([CLIENT_SPEC]), + onResolveInteraction: (token, verdict) => { + resolved.push({ token, verdict }); + }, + }); + const request = { + id: "client-1", + availableReplies: ["once", "reject"], + toolCall: { + toolCallId: "tool-client", + name: "request_connection", + input: { integration: "slack" }, + }, + }; + emit(request); + await flushPromises(); + emit(request); + await flushPromises(); + + assert.deepEqual(replies, [{ id: "client-1", reply: "once" }]); + assert.deepEqual(resolved, [{ token: "client-1", verdict: undefined }]); + }); + it("onResolveInteraction never fires for an auto-allowed gate (no durable row exists)", async () => { // Only a PAUSED gate creates a durable interaction row. An auto-allowed gate replies to // the harness without ever creating one, so resolving its id would 404 against the diff --git a/services/runner/tests/unit/session-interactions.test.ts b/services/runner/tests/unit/session-interactions.test.ts index 584aca2358..546bb0641c 100644 --- a/services/runner/tests/unit/session-interactions.test.ts +++ b/services/runner/tests/unit/session-interactions.test.ts @@ -20,7 +20,8 @@ vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => { return new Response(JSON.stringify({ ok: true }), { status: 200 }); }); -const { createInteraction } = await import("../../src/sessions/interactions.ts"); +const { createInteraction, resolveInteraction } = + await import("../../src/sessions/interactions.ts"); beforeEach(() => { postedBodies.length = 0; @@ -72,3 +73,34 @@ describe("createInteraction", () => { ); }); }); + +describe("resolveInteraction", () => { + it("POSTs the approval resolution with full-word verdict fields", async () => { + await resolveInteraction( + "sess-5", + "tok-resolution", + () => "Secret t", + { verdict: "approved", tool_call_id: "tool-5" }, + ); + + assert.equal(postedBodies.length, 1); + const { url, body } = postedBodies[0]; + assert.ok(url.endsWith("/sessions/interactions/transition")); + assert.deepEqual(body, { + session_id: "sess-5", + token: "tok-resolution", + status: "resolved", + resolution: { verdict: "approved", tool_call_id: "tool-5" }, + }); + }); + + it("omits resolution when the transition has no verdict", async () => { + await resolveInteraction("sess-6", "tok-client", () => "Secret t"); + + assert.deepEqual(postedBodies[0].body, { + session_id: "sess-6", + token: "tok-client", + status: "resolved", + }); + }); +}); diff --git a/services/runner/tests/unit/session-keepalive-approval.test.ts b/services/runner/tests/unit/session-keepalive-approval.test.ts index 4d4832c6ef..6fe4ddd5e3 100644 --- a/services/runner/tests/unit/session-keepalive-approval.test.ts +++ b/services/runner/tests/unit/session-keepalive-approval.test.ts @@ -1397,6 +1397,18 @@ describe("runTurn: real approval park + respondPermission resume", () => { "the post-resume tool_call_update streamed (not suppressed) into the new run", ); + assert.deepEqual( + run2.emitted.filter((event) => event.type === "interaction_response"), + [ + { + type: "interaction_response", + id: "tc-gate", + kind: "user_approval", + payload: { toolCallId: "tc-gate", approved: true }, + }, + ], + ); + await env.destroy(); }); @@ -1714,6 +1726,19 @@ describe("runTurn: real approval park + respondPermission resume", () => { assert.deepEqual(calls.permissionReplies, [ { id: "perm-1", reply: "reject" }, ]); + assert.deepEqual( + calls.runs[1].emitted.filter( + (event) => event.type === "interaction_response", + ), + [ + { + type: "interaction_response", + id: "tc-gate", + kind: "user_approval", + payload: { toolCallId: "tc-gate", approved: false }, + }, + ], + ); await env.destroy(); }); diff --git a/services/runner/tests/unit/session-persist.test.ts b/services/runner/tests/unit/session-persist.test.ts index 6ab62e4722..449f021b90 100644 --- a/services/runner/tests/unit/session-persist.test.ts +++ b/services/runner/tests/unit/session-persist.test.ts @@ -153,6 +153,38 @@ describe("buildPersistingEmitter", () => { ); }); + it("gives interaction responses a retry-stable id distinct from the request", async () => { + const { emit, flush } = buildPersistingEmitter( + "sess-interaction", + () => "Secret t", + ); + + emit({ + type: "interaction_request", + id: "approval-1", + kind: "user_approval", + }); + emit({ + type: "interaction_response", + id: "approval-1", + kind: "user_approval", + payload: { toolCallId: "tool-1", approved: true }, + }); + emit({ + type: "interaction_response", + id: "approval-1", + kind: "user_approval", + payload: { toolCallId: "tool-1", approved: true }, + }); + await flush(); + + const bodies = postedBodies as Array>; + assert.ok(typeof bodies[0]["record_id"] === "string"); + assert.ok(typeof bodies[1]["record_id"] === "string"); + assert.notEqual(bodies[0]["record_id"], bodies[1]["record_id"]); + assert.equal(bodies[1]["record_id"], bodies[2]["record_id"]); + }); + it("record_index increments monotonically across events", async () => { const { emit, flush } = buildPersistingEmitter("sess-4", () => "Secret t"); diff --git a/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.test.ts b/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.test.ts new file mode 100644 index 0000000000..10deeefd22 --- /dev/null +++ b/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.test.ts @@ -0,0 +1,101 @@ +import type {SessionRecord} from "@agenta/entities/session" +import {describe, expect, it} from "vitest" + +import {transcriptToMessages} from "./transcriptToMessages" + +const record = (id: string, payload: Record): SessionRecord => ({ + id, + session_id: "session-1", + project_id: "project-1", + event_index: null, + sender: "agent", + session_update: String(payload.type), + payload, + created_at: null, +}) + +const approvalRecords = (): SessionRecord[] => [ + record("record-call", { + type: "tool_call", + id: "tool-1", + name: "bash", + input: {command: "ls"}, + }), + record("record-request", { + type: "interaction_request", + id: "approval-1", + kind: "user_approval", + payload: {toolCallId: "tool-1"}, + }), +] + +const firstPart = (records: SessionRecord[]): Record => { + const messages = transcriptToMessages(records) + expect(messages).not.toBeNull() + return messages?.[0].parts[0] as unknown as Record +} + +describe("transcriptToMessages approval hydration", () => { + it("overlays a persisted approval response with the live response shape", () => { + const part = firstPart([ + ...approvalRecords(), + record("record-response", { + type: "interaction_response", + id: "approval-1", + kind: "user_approval", + payload: {toolCallId: "tool-1", approved: true}, + }), + ]) + + expect(part).toEqual({ + type: "tool-bash", + toolCallId: "tool-1", + state: "approval-responded", + input: {command: "ls"}, + approval: {id: "approval-1", approved: true}, + }) + }) + + it("keeps an unanswered request pending", () => { + const part = firstPart(approvalRecords()) + + expect(part.state).toBe("approval-requested") + expect(part.approval).toEqual({id: "approval-1"}) + }) + + it("lets an executed tool result supersede a later approval response", () => { + const part = firstPart([ + ...approvalRecords(), + record("record-result", { + type: "tool_result", + id: "tool-1", + output: "done", + }), + record("record-response", { + type: "interaction_response", + id: "approval-1", + kind: "user_approval", + payload: {toolCallId: "tool-1", approved: true}, + }), + ]) + + expect(part.state).toBe("output-available") + expect(part.output).toBe("done") + expect(part.approval).toEqual({id: "approval-1"}) + }) + + it("falls back to the interaction id when the response omits the tool-call id", () => { + const part = firstPart([ + ...approvalRecords(), + record("record-response", { + type: "interaction_response", + id: "approval-1", + kind: "user_approval", + payload: {approved: false}, + }), + ]) + + expect(part.state).toBe("approval-responded") + expect(part.approval).toEqual({id: "approval-1", approved: false}) + }) +}) diff --git a/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.ts b/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.ts index b7c3aec300..5b2928b4b7 100644 --- a/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.ts +++ b/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.ts @@ -174,6 +174,29 @@ function applyEvent(draft: DraftMessage, payload: Record): void } return } + case "interaction_response": { + if (payload.kind !== "user_approval") return + const responsePayload = (payload.payload ?? {}) as Record + const responseId = str(payload.id) + const toolCallId = str(responsePayload.toolCallId) + let part = toolCallId ? draft.tools.get(toolCallId) : undefined + if (!part) { + part = draft.parts.find((candidate) => { + const approval = (candidate.approval ?? {}) as Record + return str(approval.id) === responseId + }) + } + if ( + !part || + part.state !== "approval-requested" || + typeof responsePayload.approved !== "boolean" + ) { + return + } + part.state = "approval-responded" + part.approval = {id: responseId, approved: responsePayload.approved} + return + } case "file": { draft.parts.push({ type: "file", From 3f04e3e4fec8d6472076823632a89928a0530ebd Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 19 Jul 2026 20:45:22 +0200 Subject: [PATCH 07/15] feat(runner): dispatch each approval independently and accept partial answer sets --- .../src/engines/sandbox_agent/run-turn.ts | 23 +- .../sandbox_agent/runtime-contracts.ts | 12 +- services/runner/src/server.ts | 37 ++- .../unit/session-keepalive-approval.test.ts | 216 ++++++++++++++++-- .../AgentChatSlice/AgentConversation.tsx | 15 +- .../state/execution/agentApprovalResume.ts | 35 ++- .../tests/unit/agentApprovalResume.test.ts | 44 +++- 7 files changed, 304 insertions(+), 78 deletions(-) diff --git a/services/runner/src/engines/sandbox_agent/run-turn.ts b/services/runner/src/engines/sandbox_agent/run-turn.ts index ba6fa66d88..d0762160e8 100644 --- a/services/runner/src/engines/sandbox_agent/run-turn.ts +++ b/services/runner/src/engines/sandbox_agent/run-turn.ts @@ -240,6 +240,14 @@ export async function runTurn( env.sessionDestroyRequested = true; return env.sandbox.destroySession?.(env.session.id); }); + if (opts.resume?.carriedForward.length) { + for (const gate of opts.resume.carriedForward) { + env.parkedApprovals.set(gate.toolCallId, gate); + env.parkedApproval ??= gate; + pause.markPausedToolCall(gate.toolCallId); + } + env.approvalGateCount = env.parkedApprovals.size; + } // A human pause resolves this signal exactly once, the moment the turn parks for input — the one // place every pause path converges, so the one place to retire the run-limits deadlines for good. void pause.signal.then(() => runLimits.notePaused()); @@ -627,11 +635,8 @@ export async function runTurn( // resolves, and the pause signal ends the turn. let promptPromise: Promise; if (opts.resume) { - // The new (resume) turn owns streaming + tracing; the environment is already wired to route - // continued events into this turn's sink (env.currentTurn was set above). Every parked gate - // of the turn is answered here, one `respondPermission` per gate keyed by its tool-call id. - // All decisions share the ONE held prompt promise (one prompt per turn), so it is set once; - // answering the last gate lets the original prompt continue from here. + // The resume turn owns continued events; each decision answers one parked gate by id. + // Carried gates keep the shared original prompt pending until a later answer. const decisions = opts.resume.decisions; promptPromise = Promise.resolve(decisions[0]?.promptPromise); promptPromise.catch(() => {}); @@ -664,9 +669,8 @@ export async function runTurn( // independently — an approve and a deny in the same turn each land on the right call. await env.session.respondPermission(decision.permissionId, decision.reply); // The gate is answered: resolve its durable interaction row (the parked pending row the - // cold path would otherwise resolve via its decision map). The fresh per-turn pause - // controller starts with an EMPTY pausedToolCallIds set, so the resumed calls' - // `tool_call_update` frames are no longer suppressed and stream through. + // cold path would otherwise resolve via its decision map). Only carried-forward ids were + // re-marked paused, so answered calls stream their terminal frames normally. resolveInteractionToken(decision.interactionToken, { approved: decision.reply === "once", toolCallId: decision.toolCallId, @@ -675,6 +679,9 @@ export async function runTurn( `[keepalive] resume answered gate reply=${decision.reply} tool=${decision.toolName ?? "?"}`, ); } + // The harness still holds carried gates inside the original prompt, so re-arm the pause after + // this answer batch and let the normal park path refresh their approval TTL. + if (opts.resume.carriedForward.length > 0) pause.pause(); } else { promptPromise = Promise.resolve( env.session.prompt([{ type: "text", text: turnText }]), diff --git a/services/runner/src/engines/sandbox_agent/runtime-contracts.ts b/services/runner/src/engines/sandbox_agent/runtime-contracts.ts index ee1db91430..9b4c4bf28c 100644 --- a/services/runner/src/engines/sandbox_agent/runtime-contracts.ts +++ b/services/runner/src/engines/sandbox_agent/runtime-contracts.ts @@ -157,13 +157,13 @@ export interface RunTurnOptions { */ approvalParkMode?: boolean; /** - * A live approval resume: answer every parked gate of the turn (keyed by tool-call id) and - * stream the continued prompt's events. A turn can hold more than one parked gate, so the - * resume carries a LIST — the server builds it from the inbound approval replies, one decision - * per parked gate. All decisions share the one held prompt promise (there is one prompt per - * turn); `runTurn` answers each gate then awaits that single continuation. + * A live approval resume: answer the matching parked gates and carry the untouched gates into + * the next park. All decisions share the one held prompt promise (there is one prompt per turn). */ - resume?: { decisions: ResumeApprovalInput[] }; + resume?: { + decisions: ResumeApprovalInput[]; + carriedForward: ParkedApproval[]; + }; } /** diff --git a/services/runner/src/server.ts b/services/runner/src/server.ts index 9fb8c00fee..8e1a326c25 100644 --- a/services/runner/src/server.ts +++ b/services/runner/src/server.ts @@ -37,6 +37,7 @@ import { runSandboxAgent, runTurn, shouldPark, + type ParkedApproval, type ResumeApprovalInput, type RunTurnOptions, type SessionEnvironment, @@ -678,15 +679,12 @@ export async function runWithKeepalive( // history fingerprint (an edited transcript must not continue wrongly), and a hard mount-expiry // bound — if the parked session's mount credentials are past expiry, its durable cwd can no // longer be written, so evict to cold. - // The turn may hold several parked gates (parallel gated tool calls). Build one resume - // decision per parked gate, keyed by tool-call id. The whole turn resumes live ONLY when the - // request answers EVERY parked gate — the frontend's all-settled rule guarantees a resume /run - // is sent only after the human answered all cards, so a partial set is a mismatch that - // degrades to cold (which re-raises and multiplexes whatever subset). `parked` (the first - // gate) is the representative for logging and the per-turn-uniform checks. + // Split parallel parked gates by tool-call id. Any non-empty answer set resumes live; untouched + // gates stay pending in the same process and are carried into the next approval park. const parked = existing.environment.parkedApproval; const parkedList = [...existing.environment.parkedApprovals.values()]; const resumeDecisions: ResumeApprovalInput[] = []; + const carriedForward: ParkedApproval[] = []; let mismatch: string | undefined; if (parkedList.length === 0) { mismatch = "no-parked-gate"; @@ -703,9 +701,8 @@ export async function runWithKeepalive( } const decision = approvalDecisionForToolCall(request, gate.toolCallId); if (!decision) { - // A gate with no matching reply: fresh user text, or the human answered only some cards. - mismatch = "no-matching-approval"; - break; + carriedForward.push(gate); + continue; } resumeDecisions.push({ permissionId: gate.permissionId, @@ -746,7 +743,8 @@ export async function runWithKeepalive( ).length; const rejectCount = resumeDecisions.length - approveCount; klog( - `resume key=${key} gates=${resumeDecisions.length} ` + + `resume key=${key} gates=${parkedList.length} answered=${resumeDecisions.length} ` + + `carried=${carriedForward.length} ` + `approve=${approveCount} reject=${rejectCount} tool=${parked?.toolName ?? "?"}`, ); let result: AgentRunResult; @@ -761,7 +759,7 @@ export async function runWithKeepalive( signal, { approvalParkMode: true, - resume: { decisions: resumeDecisions }, + resume: { decisions: resumeDecisions, carriedForward }, }, ); } catch (err) { @@ -839,8 +837,8 @@ const runAgent: RunAgent = (request, emit, signal, options) => { }; /** - * The durable interaction token of a parked approval gate this request answers in-band, if - * any. The turn-start cancel-stale sweep must spare it: the gate belongs to the PREVIOUS turn + * The durable interaction tokens of parked approval gates this request answers in-band, if + * any. The turn-start cancel-stale sweep must spare them: each gate belongs to the PREVIOUS turn * (so the sweep's own `turn_id` exemption misses it), an in-band answer never transitions the * row off `pending` (only the interactions-plane respond endpoint does), and the resume * resolves the token after consuming the decision. Swept first, the granted gate's record @@ -855,18 +853,13 @@ function inBandAnswerTokens(request: AgentRunRequest): string[] | undefined { keepalivePools[provider].awaitingApproval(sessionId)?.environment .parkedApprovals; if (!parked || parked.size === 0) return undefined; - // A turn can park several gates. Spare tokens from the stale sweep ONLY when this request - // answers EVERY parked gate — that is exactly when the dispatch resumes live and resolves them. - // If any gate is unanswered the turn degrades to cold (the all-or-cold resume rule), and sparing - // an answered subset would strand those interaction rows as pending forever (no live resume ever - // resolves them). In that case spare nothing: the cold path re-raises and the sweep correctly - // cancels the old rows. + // A partial resume resolves only the answered rows. Carried-forward rows stay pending and receive + // a fresh approval park, so only matching answer tokens are exempt from stale cancellation. const tokens: string[] = []; for (const gate of parked.values()) { - if (approvalDecisionForToolCall(request, gate.toolCallId) === undefined) { - return undefined; + if (approvalDecisionForToolCall(request, gate.toolCallId) !== undefined) { + tokens.push(gate.interactionToken); } - tokens.push(gate.interactionToken); } return tokens.length > 0 ? tokens : undefined; } diff --git a/services/runner/tests/unit/session-keepalive-approval.test.ts b/services/runner/tests/unit/session-keepalive-approval.test.ts index 6fe4ddd5e3..6fa948e702 100644 --- a/services/runner/tests/unit/session-keepalive-approval.test.ts +++ b/services/runner/tests/unit/session-keepalive-approval.test.ts @@ -162,11 +162,16 @@ function makeApprovalEngine( ): Promise => { const idx = calls.turns.length; const script = scripts[idx] ?? {}; - // Mirror the real runTurn per-turn reset. + // Mirror the real runTurn per-turn reset and carried-gate reseed. env.parkedApprovals = new Map(); env.parkedApproval = undefined; env.approvalGateCount = 0; env.nonParkablePauseCount = 0; + for (const gate of opts?.resume?.carriedForward ?? []) { + env.parkedApprovals.set(gate.toolCallId, gate); + env.parkedApproval ??= gate; + } + env.approvalGateCount = env.parkedApprovals.size; env.lastTurnToolCallIds = script.toolCallIds ?? []; calls.turns.push({ env, opts, idx }); if (opts?.resume) { @@ -221,6 +226,9 @@ function makeApprovalEngine( return { ok: true, stopReason: "paused" }; } if (script.nonClaudePause) return { ok: true, stopReason: "paused" }; + if (opts?.resume?.carriedForward?.length) { + return { ok: true, stopReason: "paused" }; + } return script.result ?? { ok: true, output: "ok", stopReason: "complete" }; }; @@ -312,10 +320,11 @@ function approveResume( }; } -/** A resume that answers several parked gates at once: one tool_call + one {approved} envelope - * per gate, exactly as the frontend sends once every card is answered. */ +/** A resume carrying every parked call plus the answer envelopes supplied in this request. */ function approveResumeMulti( - gates: Array<{ toolCallId: string; toolName?: string; approved: boolean }>, + answers: Array<{ toolCallId: string; toolName?: string; approved: boolean }>, + parkedCalls: Array<{ toolCallId: string; toolName?: string }> = answers.map(({ toolCallId, toolName }) => ({ toolCallId, toolName })), + priorAnswers: Array<{ toolCallId: string; approved: boolean }> = [], ): AgentRunRequest { return { harness: "claude", @@ -326,18 +335,28 @@ function approveResumeMulti( { role: "user", content: "do X" }, { role: "assistant", - content: gates.map((g) => ({ + content: parkedCalls.map((gate) => ({ type: "tool_call", - toolCallId: g.toolCallId, - toolName: g.toolName ?? "commit", + toolCallId: gate.toolCallId, + toolName: gate.toolName ?? "commit", })), }, + ...priorAnswers.map((answer) => ({ + role: "user" as const, + content: [ + { + type: "tool_result" as const, + toolCallId: answer.toolCallId, + output: { approved: answer.approved }, + }, + ], + })), { role: "user", - content: gates.map((g) => ({ + content: answers.map((answer) => ({ type: "tool_result", - toolCallId: g.toolCallId, - output: { approved: g.approved }, + toolCallId: answer.toolCallId, + output: { approved: answer.approved }, })), }, ], @@ -581,7 +600,7 @@ describe("runWithKeepalive: approval park + resume", () => { ); }); - it("keeps a partly-answered two-gate turn paused (only one card answered -> cold)", async () => { + it("resumes a partly answered two-gate turn live, re-parks, then completes live", async () => { const { engine, calls } = makeApprovalEngine([ { approvalPause: { @@ -596,21 +615,81 @@ describe("runWithKeepalive: approval park + resume", () => { }, ]); const ctx = makeCtx(engine); + const parkedCalls = [ + { toolCallId: "tc-1", toolName: "read_a" }, + { toolCallId: "tc-2", toolName: "read_b" }, + ]; await runWithKeepalive(pauseTurn(), undefined, undefined, ctx); - // Only tc-1 answered; tc-2 still pending. The whole-set requirement is not met, so the resume - // does not answer any gate live and degrades to cold. const r2 = await runWithKeepalive( - approveResumeMulti([ - { toolCallId: "tc-1", toolName: "read_a", approved: true }, - ]), + approveResumeMulti( + [{ toolCallId: "tc-1", toolName: "read_a", approved: true }], + parkedCalls, + ), undefined, undefined, ctx, ); - assert.equal(r2.ok, true); - assert.equal(calls.resumes.length, 0, "no gate answered live"); - assert.equal(calls.acquire, 2, "the partial answer degraded to cold (re-acquired)"); + assert.equal(r2.stopReason, "paused"); + assert.equal(calls.acquire, 1, "the partial answer stayed on the live environment"); + assert.deepEqual(calls.resumes, [ + { permissionId: "perm-1", reply: "once", toolCallId: "tc-1" }, + ]); + assert.equal(ctx.pool.get(POOL_KEY)?.state, "awaiting_approval"); + assert.deepEqual( + [...calls.acquiredEnvs[0].parkedApprovals.keys()], + ["tc-2"], + "the untouched gate re-parked with a fresh approval lease", + ); + assert.deepEqual( + calls.turns[1].opts.resume.carriedForward.map((gate: ParkedApproval) => + gate.permissionId, + ), + ["perm-2"], + ); + + const r3 = await runWithKeepalive( + approveResumeMulti( + [{ toolCallId: "tc-2", toolName: "read_b", approved: false }], + parkedCalls, + [{ toolCallId: "tc-1", approved: true }], + ), + undefined, + undefined, + ctx, + ); + assert.equal(r3.stopReason, "complete"); + assert.equal(calls.acquire, 1, "both partial requests matched the live history fingerprint"); + assert.deepEqual(calls.resumes, [ + { permissionId: "perm-1", reply: "once", toolCallId: "tc-1" }, + { permissionId: "perm-2", reply: "reject", toolCallId: "tc-2" }, + ]); + assert.equal(ctx.pool.get(POOL_KEY)?.state, "idle"); + }); + + it("evicts a two-gate park when the request answers zero gates", async () => { + const { engine, calls } = makeApprovalEngine([ + { + approvalPause: { + permissionId: "perm-1", + toolCallId: "tc-1", + extraGates: [{ permissionId: "perm-2", toolCallId: "tc-2" }], + }, + toolCallIds: ["tc-1", "tc-2"], + }, + ]); + const ctx = makeCtx(engine); + await runWithKeepalive(pauseTurn(), undefined, undefined, ctx); + + await runWithKeepalive( + approveResumeMulti([], [{ toolCallId: "tc-1" }, { toolCallId: "tc-2" }]), + undefined, + undefined, + ctx, + ); + + assert.equal(calls.resumes.length, 0); + assert.equal(calls.acquire, 2, "zero answers kept the existing cold fallback"); }); }); @@ -1338,6 +1417,7 @@ describe("runTurn: real approval park + respondPermission resume", () => { promptPromise: held, }, ], + carriedForward: [], }, }); await flush(); @@ -1412,6 +1492,100 @@ describe("runTurn: real approval park + respondPermission resume", () => { await env.destroy(); }); + it("carries an unanswered gate through a partial resume without settling it", async () => { + const { calls, deps, captured } = pausableHarness(); + const acquired = await acquireEnvironment(engineReq, deps); + assert.equal(acquired.ok, true); + if (!acquired.ok) return; + const env = acquired.env; + + const firstTurn = runTurn(env, engineReq, undefined, undefined, { + approvalParkMode: true, + }); + await flush(); + for (const [toolCallId, title] of [["tc-g1", "commit"], ["tc-g2", "deploy"]]) { + captured.onEvent!( + updateEvent({ sessionUpdate: "tool_call", toolCallId, title }), + ); + } + captured.onPermissionRequest!({ + id: "perm-1", + availableReplies: ["once", "reject"], + toolCall: { toolCallId: "tc-g1", name: "commit", rawInput: {} }, + }); + captured.onPermissionRequest!({ + id: "perm-2", + availableReplies: ["once", "reject"], + toolCall: { toolCallId: "tc-g2", name: "deploy", rawInput: {} }, + }); + await firstTurn; + + const answered = env.parkedApprovals.get("tc-g1")!; + const carried = env.parkedApprovals.get("tc-g2")!; + env.clearTurn(); + const resumeTurn = runTurn(env, approveResume(true), undefined, undefined, { + approvalParkMode: true, + resume: { + decisions: [ + { + permissionId: answered.permissionId, + reply: "once", + toolCallId: answered.toolCallId, + toolName: answered.toolName, + args: answered.args, + interactionToken: answered.interactionToken, + promptPromise: answered.promptPromise, + }, + ], + carriedForward: [carried], + }, + }); + for (let i = 0; i < 5 && !env.currentTurn?.pause.active; i += 1) { + await Promise.resolve(); + } + assert.equal(env.currentTurn?.pause.active, true); + captured.onEvent!( + updateEvent({ + sessionUpdate: "tool_call_update", + toolCallId: "tc-g2", + status: "completed", + content: "must stay suppressed", + }), + ); + captured.onEvent!( + updateEvent({ + sessionUpdate: "tool_call_update", + toolCallId: "tc-g1", + status: "completed", + content: "answered result", + }), + ); + const result = await resumeTurn; + + assert.equal(result.stopReason, "paused"); + assert.deepEqual(calls.permissionReplies, [{ id: "perm-1", reply: "once" }]); + assert.deepEqual([...env.parkedApprovals.keys()], ["tc-g2"]); + assert.equal(env.approvalGateCount, 1); + const run = calls.runs[1]; + assert.equal( + run.handled.some((update) => update.toolCallId === "tc-g2"), + false, + "carried gate frames stay suppressed", + ); + assert.equal( + run.sweeps.every((sweep) => sweep.isExcluded("tc-g2")), + true, + "every sentinel sweep spares the carried gate", + ); + assert.equal( + run.settled.some((entry) => entry.id === "tc-g2"), + false, + "the carried gate receives no sentinel", + ); + + await env.destroy(); + }); + it("excludes an allowed execution from both pause sweeps", async () => { const { calls, deps, captured } = pausableHarness(); const acquired = await acquireEnvironment(engineReq, deps); @@ -1453,6 +1627,7 @@ describe("runTurn: real approval park + respondPermission resume", () => { promptPromise: held, }, ], + carriedForward: [], }, }); await flush(); @@ -1552,6 +1727,7 @@ describe("runTurn: real approval park + respondPermission resume", () => { promptPromise: held, }, ], + carriedForward: [], }, }); await flush(); @@ -1645,6 +1821,7 @@ describe("runTurn: real approval park + respondPermission resume", () => { promptPromise: held, }, ], + carriedForward: [], }, }); await flush(); @@ -1717,6 +1894,7 @@ describe("runTurn: real approval park + respondPermission resume", () => { promptPromise: held, }, ], + carriedForward: [], }, }); await flush(); diff --git a/web/oss/src/components/AgentChatSlice/AgentConversation.tsx b/web/oss/src/components/AgentChatSlice/AgentConversation.tsx index bcce2e5f24..0888609fda 100644 --- a/web/oss/src/components/AgentChatSlice/AgentConversation.tsx +++ b/web/oss/src/components/AgentChatSlice/AgentConversation.tsx @@ -567,6 +567,8 @@ const AgentConversation = ({ const revalidateSessionMounts = useSetAtom(revalidateSessionMountsAtom) const revalidateSessionRecords = useSetAtom(revalidateSessionRecordsAtom) + // Only a gate settled in this mount may trigger an automatic resume; hydrated answers stay inert. + const liveGateInteractionRef = useRef(false) const { messages, @@ -587,7 +589,11 @@ const AgentConversation = ({ experimental_throttle: 50, // Approve AND deny both resume — a deny-only decision must re-send so the runner // gets the denial round-trip and the model continues (no `approval-responded` limbo). - sendAutomaticallyWhen: agentShouldResumeAfterApproval, + sendAutomaticallyWhen: ({messages}) => + agentShouldResumeAfterApproval({ + messages, + liveInteraction: liveGateInteractionRef.current, + }), // The turn's trace may not be ingested yet when the row asks for its summary — // marking it fresh lets the trace queries retry through the ingestion lag // (historical traces get no such grace; a 404 there means the trace is gone). @@ -671,13 +677,6 @@ const AgentConversation = ({ // Seed once per mounted session tab; `sessionId` is stable for this instance. }, [sessionId]) - // True once the user settles a gate (approval response / client-tool output) in THIS mount — - // i.e. the SDK's auto-resume genuinely is imminent, so the queue's pre-resume hold applies. - // Without a live interaction, a tail that READS as "resume imminent" is an orphan restored - // from storage (reload / remount killed the run mid-resume): nothing will ever fire that - // resume, and holding for it froze the composer forever (AGE-3937). - const liveGateInteractionRef = useRef(false) - // Settle a parked client tool (#4920). The dispatcher calls this from a widget (e.g. the connect // widget) with the structured reference; `addToolOutput` matches the part by `toolCallId` on the // last turn and the resume predicate auto-resends. `tool` is only the typed-tools key — matching diff --git a/web/packages/agenta-playground/src/state/execution/agentApprovalResume.ts b/web/packages/agenta-playground/src/state/execution/agentApprovalResume.ts index c097bd3e39..6eced25576 100644 --- a/web/packages/agenta-playground/src/state/execution/agentApprovalResume.ts +++ b/web/packages/agenta-playground/src/state/execution/agentApprovalResume.ts @@ -119,16 +119,20 @@ const isSettledToolPart = (part: ToolPartLike): boolean => part.state === "approval-responded") /** - * Resume when the last assistant turn carries at least one freshly-resolved parked interaction (an - * approval response OR a browser-fulfilled client-tool result) and EVERY non-provider-executed tool - * part on it is settled. Both paths share one rule so a single `sendAutomaticallyWhen` covers - * approvals and client tools alike: + * Resume when the last assistant turn carries a freshly-resolved parked interaction. Approval + * responses dispatch per card; browser-fulfilled client tools retain the all-settled rule: * - Approval (approve OR deny): a denied tool part is `approval-responded`, so a deny-only turn * still resumes and the runner gets the denial round-trip (the deny dead-end fix). * - Client tool: a `request_connection` the widget settled (success, cancel, failure, abandon) * reads as a client-tool result, so the run resumes and the runner re-resolves on cold-replay. */ -export function agentShouldResumeAfterApproval({messages}: {messages: MessageLike[]}): boolean { +export function agentShouldResumeAfterApproval({ + messages, + liveInteraction = true, +}: { + messages: MessageLike[] + liveInteraction?: boolean +}): boolean { const last = messages[messages.length - 1] if (!last || last.role !== "assistant") return false @@ -143,9 +147,15 @@ export function agentShouldResumeAfterApproval({messages}: {messages: MessageLik // browser-fulfilled client-tool result). Using the last one handles chained gates: a second // approval later in the turn is what should drive the (next) resume. let lastResolvedIdx = -1 + let lastResolvedIsApproval = false for (let i = 0; i < parts.length; i++) { - if (isRespondedToolPart(parts[i]) || isClientToolResult(parts[i], renderMap)) + if (isRespondedToolPart(parts[i])) { lastResolvedIdx = i + lastResolvedIsApproval = true + } else if (isClientToolResult(parts[i], renderMap)) { + lastResolvedIdx = i + lastResolvedIsApproval = false + } } if (lastResolvedIdx === -1) return false @@ -160,6 +170,15 @@ export function agentShouldResumeAfterApproval({messages}: {messages: MessageLik .some((part) => part.type === "step-start") if (resumedAlready) return false - const allSettled = toolParts.every(isSettledToolPart) - return allSettled + if (!liveInteraction) return false + + // The AI SDK re-evaluates after message updates and waits for an in-flight stream to finish, + // so an approval clicked during a resume dispatches when that stream finishes. + if (lastResolvedIsApproval) { + const pendingClientTool = toolParts.some((part) => + isPendingClientToolInteraction(part, renderMap), + ) + return !pendingClientTool + } + return toolParts.every(isSettledToolPart) } diff --git a/web/packages/agenta-playground/tests/unit/agentApprovalResume.test.ts b/web/packages/agenta-playground/tests/unit/agentApprovalResume.test.ts index 4ccc115c73..8b6f6e14a7 100644 --- a/web/packages/agenta-playground/tests/unit/agentApprovalResume.test.ts +++ b/web/packages/agenta-playground/tests/unit/agentApprovalResume.test.ts @@ -54,6 +54,11 @@ describe("agentShouldResumeAfterApproval", () => { expect(agentShouldResumeAfterApproval({messages})).toBe(true) }) + it("does NOT resume a rebuilt answered conversation without a live interaction", () => { + const messages = [user("do it"), assistantWithTool("approval-responded", true)] + expect(agentShouldResumeAfterApproval({messages, liveInteraction: false})).toBe(false) + }) + it("does NOT resume while a gate is still pending (approval-requested)", () => { const messages = [user("do it"), assistantWithTool("approval-requested")] expect(agentShouldResumeAfterApproval({messages})).toBe(false) @@ -72,7 +77,7 @@ describe("agentShouldResumeAfterApproval", () => { expect(agentShouldResumeAfterApproval({messages})).toBe(false) }) - it("does NOT resume when a sibling tool on the turn is unsettled", () => { + it("resumes when one approval is answered and a sibling approval remains pending", () => { const messages = [ user("do two"), { @@ -97,7 +102,7 @@ describe("agentShouldResumeAfterApproval", () => { ], }, ] - expect(agentShouldResumeAfterApproval({messages})).toBe(false) + expect(agentShouldResumeAfterApproval({messages})).toBe(true) }) it("resumes when a responded gate sits alongside an already-completed tool", () => { @@ -128,10 +133,8 @@ describe("agentShouldResumeAfterApproval", () => { expect(agentShouldResumeAfterApproval({messages})).toBe(true) }) - it("does NOT resume while ONE of two concurrent approval cards is still pending", () => { - // Concurrent approvals: one turn shows TWO approval-requested gates (two distinct - // toolCallIds). Answering only the first must NOT resume — the second is still pending, - // so the run stays parked until EVERY card is settled. + it("RESUMES per card while another concurrent approval is pending", () => { + // One live answer dispatches immediately; the runner carries the untouched gate forward. const messages = [ user("do two"), { @@ -156,7 +159,7 @@ describe("agentShouldResumeAfterApproval", () => { ], }, ] - expect(agentShouldResumeAfterApproval({messages})).toBe(false) + expect(agentShouldResumeAfterApproval({messages})).toBe(true) }) it("RESUMES once BOTH concurrent approval cards are answered", () => { @@ -224,6 +227,33 @@ describe("agentShouldResumeAfterApproval", () => { expect(agentShouldResumeAfterApproval({messages})).toBe(false) }) + it("keeps the all-settled rule when an approval sits beside a pending client tool", () => { + const messages = [ + user("connect then approve"), + { + id: "a1", + role: "assistant", + parts: [ + {type: "step-start"}, + { + type: "tool-request_connection", + toolCallId: "call_c", + state: "input-available", + input: {integration: "github"}, + }, + { + type: "tool-deleteFile", + toolCallId: "call_1", + state: "approval-responded", + input: {path: "/x"}, + approval: {id: "perm_1", approved: true}, + }, + ], + }, + ] + expect(agentShouldResumeAfterApproval({messages})).toBe(false) + }) + it("resumes when a fulfilled client tool sits alongside a responded approval", () => { const messages = [ user("do it then connect"), From 195c031ca1358814dafb84bc783a58c5ff3bdfc7 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 19 Jul 2026 21:03:45 +0200 Subject: [PATCH 08/15] test(runner): replay the concurrent-approval incident end to end --- .../unit/session-keepalive-approval.test.ts | 342 ++++++++++++++++++ .../assets/transcriptToMessages.test.ts | 74 +++- 2 files changed, 414 insertions(+), 2 deletions(-) diff --git a/services/runner/tests/unit/session-keepalive-approval.test.ts b/services/runner/tests/unit/session-keepalive-approval.test.ts index 6fa948e702..5f2c0eba36 100644 --- a/services/runner/tests/unit/session-keepalive-approval.test.ts +++ b/services/runner/tests/unit/session-keepalive-approval.test.ts @@ -1774,6 +1774,348 @@ describe("runTurn: real approval park + respondPermission resume", () => { await env.destroy(); }); + it("replays the concurrent-approval incident with deferred closure and exactly-once real results", async () => { + const { calls, deps, captured } = pausableHarness(); + deps.createOtel = createSandboxAgentOtel as any; + const incidentRequest: AgentRunRequest = { + ...engineReq, + permissions: { default: "ask" }, + customTools: [ + { name: "tool-a", permission: "ask" }, + { name: "tool-b", permission: "ask" }, + ], + messages: [{ role: "user", content: "run both writes in parallel" }], + }; + const acquired = await acquireEnvironment(incidentRequest, deps); + assert.equal(acquired.ok, true); + if (!acquired.ok) return; + const env = acquired.env; + + const firstTurn = runTurn( + env, + incidentRequest, + undefined, + undefined, + { approvalParkMode: true }, + ); + await flush(); + captured.onEvent!( + updateEvent({ + sessionUpdate: "tool_call", + toolCallId: "tool-a", + title: "tool-a", + rawInput: { target: "a" }, + }), + ); + captured.onEvent!( + updateEvent({ + sessionUpdate: "tool_call", + toolCallId: "tool-b", + title: "tool-b", + rawInput: { target: "b" }, + }), + ); + captured.onPermissionRequest!({ + id: "permission-a", + availableReplies: ["once", "reject"], + toolCall: { + toolCallId: "tool-a", + name: "tool-a", + rawInput: { target: "a" }, + }, + }); + for (let i = 0; i < 5 && !env.currentTurn?.pause.active; i += 1) { + await Promise.resolve(); + } + assert.equal(env.currentTurn?.pause.active, true); + captured.onEvent!( + updateEvent({ + sessionUpdate: "tool_call_update", + toolCallId: "tool-b", + status: "completed", + content: "tool-b cancellation closure", + }), + ); + const firstResult = await firstTurn; + + assert.equal(firstResult.ok, true); + assert.equal(firstResult.stopReason, "paused"); + assert.equal(env.parkedApprovals.size, 1); + assert.deepEqual([...env.parkedApprovals.keys()], ["tool-a"]); + const gateA = env.parkedApprovals.get("tool-a")!; + const firstEvents = firstResult.events ?? []; + assert.equal( + firstEvents.filter( + (event) => + event.type === "interaction_request" && + (event.payload as { toolCallId?: string }).toolCallId === "tool-a", + ).length, + 1, + ); + assert.deepEqual( + firstEvents.filter( + (event) => event.type === "tool_result" && event.id === "tool-b", + ), + [ + { + type: "tool_result", + id: "tool-b", + output: TOOL_NOT_EXECUTED_PAUSED, + isError: true, + }, + ], + ); + assert.equal( + firstEvents.some( + (event) => event.type === "tool_result" && event.isError === false, + ), + false, + "the never-started first-turn states have no success record", + ); + + env.clearTurn(); + const secondTurn = runTurn( + env, + incidentRequest, + undefined, + undefined, + { + approvalParkMode: true, + resume: { + decisions: [ + { + permissionId: gateA.permissionId, + reply: "once", + toolCallId: gateA.toolCallId, + toolName: gateA.toolName, + args: gateA.args, + interactionToken: gateA.interactionToken, + promptPromise: gateA.promptPromise, + }, + ], + carriedForward: [], + }, + }, + ); + for ( + let i = 0; + i < 5 && + !calls.permissionReplies.some((reply) => reply.id === gateA.permissionId); + i += 1 + ) { + await Promise.resolve(); + } + assert.equal( + calls.permissionReplies.filter((reply) => reply.id === gateA.permissionId) + .length, + 1, + ); + captured.onEvent!( + updateEvent({ + sessionUpdate: "tool_call_update", + toolCallId: "tool-a", + status: "in_progress", + }), + ); + captured.onPermissionRequest!({ + id: "permission-b", + availableReplies: ["once", "reject"], + toolCall: { + toolCallId: "tool-b", + name: "tool-b", + rawInput: { target: "b" }, + }, + }); + for (let i = 0; i < 5 && !env.currentTurn?.pause.active; i += 1) { + await Promise.resolve(); + } + assert.equal(env.currentTurn?.pause.active, true); + captured.onEvent!( + updateEvent({ + sessionUpdate: "tool_call_update", + toolCallId: "tool-a", + status: "completed", + content: "tool-a real output", + }), + ); + const secondResult = await secondTurn; + + assert.equal(secondResult.ok, true); + assert.equal(secondResult.stopReason, "paused"); + assert.equal(env.parkedApprovals.size, 1); + assert.deepEqual([...env.parkedApprovals.keys()], ["tool-b"]); + const gateB = env.parkedApprovals.get("tool-b")!; + const secondEvents = secondResult.events ?? []; + assert.deepEqual( + secondEvents.filter( + (event) => event.type === "tool_result" && event.id === "tool-a", + ), + [ + { + type: "tool_result", + id: "tool-a", + output: "tool-a real output", + isError: false, + }, + ], + ); + assert.equal( + secondEvents.filter( + (event) => + event.type === "interaction_request" && + (event.payload as { toolCallId?: string }).toolCallId === "tool-b", + ).length, + 1, + ); + assert.deepEqual( + secondEvents.filter( + (event) => + event.type === "interaction_response" && + (event.payload as { toolCallId?: string }).toolCallId === "tool-a", + ), + [ + { + type: "interaction_response", + id: gateA.interactionToken, + kind: "user_approval", + payload: { toolCallId: "tool-a", approved: true }, + }, + ], + ); + + env.clearTurn(); + const thirdTurn = runTurn( + env, + incidentRequest, + undefined, + undefined, + { + approvalParkMode: true, + resume: { + decisions: [ + { + permissionId: gateB.permissionId, + reply: "once", + toolCallId: gateB.toolCallId, + toolName: gateB.toolName, + args: gateB.args, + interactionToken: gateB.interactionToken, + promptPromise: gateB.promptPromise, + }, + ], + carriedForward: [], + }, + }, + ); + for ( + let i = 0; + i < 5 && + !calls.permissionReplies.some((reply) => reply.id === gateB.permissionId); + i += 1 + ) { + await Promise.resolve(); + } + assert.equal( + calls.permissionReplies.filter((reply) => reply.id === gateB.permissionId) + .length, + 1, + ); + captured.onEvent!( + updateEvent({ + sessionUpdate: "tool_call_update", + toolCallId: "tool-b", + status: "completed", + content: "tool-b real output", + }), + ); + calls.resolvePrompt!({ + stopReason: "complete", + usage: { inputTokens: 1, outputTokens: 1 }, + }); + const thirdResult = await thirdTurn; + + assert.equal(thirdResult.ok, true); + assert.equal(thirdResult.stopReason, "complete"); + const thirdEvents = thirdResult.events ?? []; + assert.deepEqual( + thirdEvents.filter( + (event) => event.type === "tool_result" && event.id === "tool-b", + ), + [ + { + type: "tool_result", + id: "tool-b", + output: "tool-b real output", + isError: false, + }, + ], + ); + assert.deepEqual( + thirdEvents.filter( + (event) => + event.type === "interaction_response" && + (event.payload as { toolCallId?: string }).toolCallId === "tool-b", + ), + [ + { + type: "interaction_response", + id: gateB.interactionToken, + kind: "user_approval", + payload: { toolCallId: "tool-b", approved: true }, + }, + ], + ); + + const allEvents = [...firstEvents, ...secondEvents, ...thirdEvents]; + const realResults = allEvents.filter( + ( + event, + ): event is Extract => + event.type === "tool_result" && event.isError === false, + ); + assert.deepEqual( + realResults.map((event) => event.id).sort(), + ["tool-a", "tool-b"], + "each call records exactly one real result", + ); + assert.equal( + allEvents.some( + (event) => + event.type === "tool_result" && + event.id === "tool-a" && + (event.output === TOOL_NOT_EXECUTED_PAUSED || + event.output === APPROVED_EXECUTION_RESULT_UNKNOWN), + ), + false, + "the approved tool-a result is never replaced by a sentinel", + ); + assert.equal( + allEvents.some( + (event) => + event.type === "tool_result" && + event.output === "tool-b cancellation closure", + ), + false, + "the cancellation closure is never recorded as success", + ); + const permissionReplyCounts = new Map(); + for (const reply of calls.permissionReplies) { + permissionReplyCounts.set( + reply.id, + (permissionReplyCounts.get(reply.id) ?? 0) + 1, + ); + } + assert.deepEqual( + permissionReplyCounts, + new Map([ + [gateA.permissionId, 1], + [gateB.permissionId, 1], + ]), + ); + + await env.destroy(); + }); + it("records the non-retry sentinel when an approved result misses the bound", async () => { const { calls, deps, captured } = pausableHarness(); deps.resolveRunLimits = () => ({ diff --git a/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.test.ts b/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.test.ts index 10deeefd22..e5c773f63d 100644 --- a/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.test.ts +++ b/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.test.ts @@ -3,12 +3,12 @@ import {describe, expect, it} from "vitest" import {transcriptToMessages} from "./transcriptToMessages" -const record = (id: string, payload: Record): SessionRecord => ({ +const record = (id: string, payload: Record, sender = "agent"): SessionRecord => ({ id, session_id: "session-1", project_id: "project-1", event_index: null, - sender: "agent", + sender, session_update: String(payload.type), payload, created_at: null, @@ -98,4 +98,74 @@ describe("transcriptToMessages approval hydration", () => { expect(part.state).toBe("approval-responded") expect(part.approval).toEqual({id: "approval-1", approved: false}) }) + + it("rebuilds the persisted incident order with call a executed and call b pending", () => { + const messages = transcriptToMessages([ + record("record-user", {type: "message", text: "run both writes"}, "user"), + record("record-call-a", { + type: "tool_call", + id: "tool-a", + name: "bash", + input: {command: "write a"}, + }), + record("record-request-a", { + type: "interaction_request", + id: "approval-a", + kind: "user_approval", + payload: {toolCallId: "tool-a"}, + }), + record("record-result-b-deferred", { + type: "tool_result", + id: "tool-b", + output: "DEFERRED_NOT_EXECUTED: paused for another approval; retry the same call if still required.", + isError: true, + }), + record("record-response-a", { + type: "interaction_response", + id: "approval-a", + kind: "user_approval", + payload: {toolCallId: "tool-a", approved: true}, + }), + record("record-result-a", { + type: "tool_result", + id: "tool-a", + output: "tool-a real output", + isError: false, + }), + record("record-request-b", { + type: "interaction_request", + id: "approval-b", + kind: "user_approval", + payload: { + toolCallId: "tool-b", + toolCall: { + toolCallId: "tool-b", + name: "bash", + rawInput: {command: "write b"}, + }, + }, + }), + ]) + + expect(messages).not.toBeNull() + expect(messages?.[0]).toMatchObject({ + role: "user", + parts: [{type: "text", text: "run both writes"}], + }) + const assistantParts = messages?.[1].parts as unknown as Record[] + const callA = assistantParts.find((part) => part.toolCallId === "tool-a") + const callB = assistantParts.find((part) => part.toolCallId === "tool-b") + + expect(callA).toMatchObject({ + state: "output-available", + output: "tool-a real output", + }) + expect(callB).toMatchObject({ + state: "approval-requested", + approval: {id: "approval-b"}, + }) + expect(assistantParts.filter((part) => part.state === "approval-requested")).toEqual([ + callB, + ]) + }) }) From 5a8305e5343cded89188ad2696ea3778e1d547ef Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 19 Jul 2026 22:07:20 +0200 Subject: [PATCH 09/15] fix(runner): close the adversarial-review findings on the approvals train --- api/oss/src/apis/fastapi/sessions/models.py | 27 +++- api/oss/src/apis/fastapi/sessions/router.py | 31 ++++- .../test_transition_interaction_resolution.py | 92 +++++++++++++- .../sdk/agents/adapters/vercel/messages.py | 17 ++- .../pytest/unit/agents/test_ui_messages.py | 19 ++- .../unit/test_workflow_control_running.py | 6 - .../engines/sandbox_agent/acp-interactions.ts | 42 +++++-- .../runner/src/engines/sandbox_agent/pause.ts | 38 +++++- .../src/engines/sandbox_agent/run-turn.ts | 2 + .../engines/sandbox_agent/runtime-policy.ts | 6 +- services/runner/src/permission-plan.ts | 26 +++- services/runner/src/responder.ts | 63 +++++++--- services/runner/src/server.ts | 36 +++--- .../tests/unit/pending-approval-pause.test.ts | 19 +++ services/runner/tests/unit/responder.test.ts | 19 +-- .../sandbox-agent-acp-interactions.test.ts | 77 +++++++++++- .../unit/session-keepalive-approval.test.ts | 119 +++++++++++++++++- .../AgentChatSlice/AgentConversation.tsx | 16 ++- .../assets/transcriptToMessages.test.ts | 6 +- .../assets/transcriptToMessages.ts | 37 +++--- web/packages/agenta-playground/src/index.ts | 2 +- .../state/execution/agentApprovalResume.ts | 41 +++--- .../src/state/execution/index.ts | 2 +- .../agenta-playground/src/state/index.ts | 2 +- .../tests/unit/agentApprovalResume.test.ts | 63 +++++++++- 25 files changed, 678 insertions(+), 130 deletions(-) diff --git a/api/oss/src/apis/fastapi/sessions/models.py b/api/oss/src/apis/fastapi/sessions/models.py index 9f969e5ca4..2a3c0fcd68 100644 --- a/api/oss/src/apis/fastapi/sessions/models.py +++ b/api/oss/src/apis/fastapi/sessions/models.py @@ -1,8 +1,8 @@ from datetime import datetime -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Literal, Optional from uuid import UUID -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator from oss.src.core.sessions.streams.dtos import ( SessionStream, @@ -101,17 +101,34 @@ class SessionInteractionCreateRequest(BaseModel): meta: Optional[Dict[str, Any]] = None +class SessionInteractionResolution(BaseModel): + model_config = ConfigDict(extra="forbid") + + verdict: Literal["approved", "denied"] + tool_call_id: str + + class SessionInteractionTransitionRequest(BaseModel): # No project_id: scope comes from the caller's credential (request.state). session_id: str token: str status: SessionInteractionStatus - resolution: Optional[Dict[str, Any]] = None + resolution: Optional[SessionInteractionResolution] = None + + @model_validator(mode="after") + def validate_resolution_status(self) -> "SessionInteractionTransitionRequest": + # Resolution is answer data, so it is valid only on the resolved lifecycle edge. + if ( + self.resolution is not None + and self.status != SessionInteractionStatus.resolved + ): + raise ValueError("resolution is only valid when status is resolved") + return self class SessionInteractionCancelStaleRequest(BaseModel): - # Cancels prior turns' pending gates, sparing this turn's own (`turn_id`) and any gates - # the current turn answers in-band (`tokens` — a resume must resolve them, not cancel). + # Cancels prior turns' pending gates, sparing this turn's own (`turn_id`) and every prior + # gate still owned by a live partial resume (`tokens`, including carried gates). session_id: str turn_id: str tokens: Optional[List[str]] = None diff --git a/api/oss/src/apis/fastapi/sessions/router.py b/api/oss/src/apis/fastapi/sessions/router.py index f8994eb76d..77d0f539f9 100644 --- a/api/oss/src/apis/fastapi/sessions/router.py +++ b/api/oss/src/apis/fastapi/sessions/router.py @@ -54,6 +54,8 @@ from oss.src.core.sessions.records.streaming import publish_record from oss.src.core.sessions.interactions.dtos import ( SessionInteractionCreate, + SessionInteractionKind, + SessionInteractionQuery, SessionInteractionStatus, SessionInteractionTransition, ) @@ -640,6 +642,33 @@ async def transition_interaction( ): raise FORBIDDEN_EXCEPTION + resolution = ( + body.resolution.model_dump() if body.resolution is not None else None + ) + if resolution is not None: + interactions = await self.interactions_service.query_interactions( + project_id=project_id, + query=SessionInteractionQuery(session_id=body.session_id), + ) + source = next( + ( + interaction + for interaction in interactions + if interaction.token == body.token + ), + None, + ) + if source is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Interaction not found or already terminal", + ) + if source.kind != SessionInteractionKind.user_approval: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="Resolution is only valid for user approval interactions", + ) + try: interaction = await self.interactions_service.transition_interaction( transition=SessionInteractionTransition( @@ -647,7 +676,7 @@ async def transition_interaction( session_id=body.session_id, token=body.token, status=body.status, - resolution=body.resolution, + resolution=resolution, ), ) except InteractionNotFound: diff --git a/api/oss/tests/pytest/unit/sessions/test_transition_interaction_resolution.py b/api/oss/tests/pytest/unit/sessions/test_transition_interaction_resolution.py index 0256fa010c..f345b07ffe 100644 --- a/api/oss/tests/pytest/unit/sessions/test_transition_interaction_resolution.py +++ b/api/oss/tests/pytest/unit/sessions/test_transition_interaction_resolution.py @@ -1,7 +1,9 @@ from unittest.mock import AsyncMock, patch from uuid import uuid4 -from fastapi import FastAPI, Request +from fastapi import FastAPI, HTTPException, Request +from fastapi.testclient import TestClient +import pytest from oss.src.apis.fastapi.sessions.models import SessionInteractionTransitionRequest from oss.src.apis.fastapi.sessions.router import InteractionsRouter @@ -34,6 +36,17 @@ async def test_transition_route_passes_resolution_to_the_domain_transition(): captured = [] class _InteractionsService: + async def query_interactions(self, *, project_id, query): + return [ + SessionInteraction( + project_id=project_id, + session_id=query.session_id, + token="approval-token", + kind=SessionInteractionKind.user_approval, + status=SessionInteractionStatus.pending, + ) + ] + async def transition_interaction(self, *, transition): captured.append(transition) return SessionInteraction( @@ -75,3 +88,80 @@ async def transition_interaction(self, *, transition): assert response.interaction is not None assert response.interaction.data is not None assert response.interaction.data.resolution == captured[0].resolution + + +@pytest.mark.parametrize( + "payload", + [ + { + "session_id": "session-1", + "token": "approval-token", + "status": "pending", + "resolution": {"verdict": "approved", "tool_call_id": "tool-1"}, + }, + { + "session_id": "session-1", + "token": "approval-token", + "status": "resolved", + "resolution": {"verdict": "maybe", "tool_call_id": "tool-1"}, + }, + { + "session_id": "session-1", + "token": "approval-token", + "status": "resolved", + "resolution": {"verdict": "approved"}, + }, + ], +) +def test_transition_route_rejects_invalid_approval_resolution_with_422(payload): + router = InteractionsRouter( + interactions_service=AsyncMock(), + workflows_service=AsyncMock(), + respond_task=AsyncMock(), + ) + app = FastAPI() + app.include_router(router.router) + + response = TestClient(app).post("/transition", json=payload) + + assert response.status_code == 422 + + +async def test_transition_route_rejects_resolution_for_client_tool_with_409(): + project_id = uuid4() + user_id = uuid4() + interactions_service = AsyncMock() + interactions_service.query_interactions.return_value = [ + SessionInteraction( + project_id=project_id, + session_id="session-1", + token="client-tool-token", + kind=SessionInteractionKind.client_tool, + status=SessionInteractionStatus.pending, + ) + ] + router = InteractionsRouter( + interactions_service=interactions_service, + workflows_service=AsyncMock(), + respond_task=AsyncMock(), + ) + body = SessionInteractionTransitionRequest( + session_id="session-1", + token="client-tool-token", + status=SessionInteractionStatus.resolved, + resolution={"verdict": "approved", "tool_call_id": "tool-1"}, + ) + + with patch( + "oss.src.apis.fastapi.sessions.router.check_action_access", + new_callable=AsyncMock, + return_value=True, + ): + with pytest.raises(HTTPException) as caught: + await router.transition_interaction( + request=_make_authed_request(FastAPI(), project_id, user_id), + body=body, + ) + + assert caught.value.status_code == 409 + interactions_service.transition_interaction.assert_not_awaited() diff --git a/sdks/python/agenta/sdk/agents/adapters/vercel/messages.py b/sdks/python/agenta/sdk/agents/adapters/vercel/messages.py index d66c4f5cfe..86c3668c7c 100644 --- a/sdks/python/agenta/sdk/agents/adapters/vercel/messages.py +++ b/sdks/python/agenta/sdk/agents/adapters/vercel/messages.py @@ -196,12 +196,19 @@ def _tool_part_blocks(part: Dict[str, Any], ptype: str) -> List[ContentBlock]: if isinstance(_input, dict) else type(_input).__name__, ) + approval_output: Dict[str, Any] = {"approved": approved} + approval = part.get("approval") + interaction_token = ( + approval.get("id") if isinstance(approval, dict) else None + ) + if isinstance(interaction_token, str) and interaction_token: + approval_output["interactionToken"] = interaction_token blocks.append( ContentBlock( type="tool_result", tool_call_id=tool_call_id, tool_name=tool_name, - output={"approved": approved}, + output=approval_output, ) ) return blocks @@ -229,7 +236,13 @@ def _approval_response_blocks(part: Dict[str, Any]) -> List[ContentBlock]: output = part.get("output") if output is None: approved = part.get("approved") - output = {"approved": approved} if approved is not None else part.get("reason") + if approved is not None: + output = {"approved": approved} + interaction_token = part.get("approvalId") + if isinstance(interaction_token, str) and interaction_token: + output["interactionToken"] = interaction_token + else: + output = part.get("reason") return [ContentBlock(type="tool_result", tool_call_id=tool_call_id, output=output)] diff --git a/sdks/python/oss/tests/pytest/unit/agents/test_ui_messages.py b/sdks/python/oss/tests/pytest/unit/agents/test_ui_messages.py index e9f90900b9..baa5453a52 100644 --- a/sdks/python/oss/tests/pytest/unit/agents/test_ui_messages.py +++ b/sdks/python/oss/tests/pytest/unit/agents/test_ui_messages.py @@ -212,7 +212,10 @@ def test_inline_approval_responded_deny_emits_approved_envelope(self): "type": "tool_result", "toolCallId": "call_1", "toolName": "deleteFile", - "output": {"approved": False}, + "output": { + "approved": False, + "interactionToken": "perm_1", + }, }, ] @@ -238,7 +241,10 @@ def test_inline_approval_responded_approve_emits_approved_envelope(self): result = msgs[0].content[1] assert result.type == "tool_result" assert result.tool_call_id == "call_2" - assert result.output == {"approved": True} + assert result.output == { + "approved": True, + "interactionToken": "perm_2", + } def test_concurrent_inline_approvals_each_emit_their_own_result(self): # Concurrent approvals: the browser round-trips TWO tool parts in one turn, each with its @@ -274,11 +280,11 @@ def test_concurrent_inline_approvals_each_emit_their_own_result(self): # Each decision lands on its own toolCallId, in order, approve and deny preserved. assert (results[0].tool_call_id, results[0].output) == ( "call_1", - {"approved": True}, + {"approved": True, "interactionToken": "perm_1"}, ) assert (results[1].tool_call_id, results[1].output) == ( "call_2", - {"approved": False}, + {"approved": False, "interactionToken": "perm_2"}, ) def test_inline_output_denied_state_emits_denied_envelope(self): @@ -304,7 +310,10 @@ def test_inline_output_denied_state_emits_denied_envelope(self): result = msgs[0].content[1] assert result.type == "tool_result" assert result.tool_call_id == "call_3" - assert result.output == {"approved": False} + assert result.output == { + "approved": False, + "interactionToken": "perm_3", + } def test_pending_approval_request_only_part_emits_no_decision(self): # A tool part still awaiting a decision (`approval-requested`, no `approval.approved`) diff --git a/sdks/python/oss/tests/pytest/unit/test_workflow_control_running.py b/sdks/python/oss/tests/pytest/unit/test_workflow_control_running.py index ab425f250e..17a7c64d43 100644 --- a/sdks/python/oss/tests/pytest/unit/test_workflow_control_running.py +++ b/sdks/python/oss/tests/pytest/unit/test_workflow_control_running.py @@ -117,7 +117,6 @@ def row(self, sid: str) -> dict: @pytest.mark.asyncio async def test_send_on_idle_session_marks_alive_and_increments_token(): - store = _FakeSessionStore() started = asyncio.Event() @@ -142,7 +141,6 @@ async def slow_run(): @pytest.mark.asyncio async def test_send_collision_when_alive_without_control_is_409(): - store = _FakeSessionStore() store.row("s1")["alive"] = True @@ -154,7 +152,6 @@ async def test_send_collision_when_alive_without_control_is_409(): @pytest.mark.asyncio async def test_steer_cancels_alive_run_and_restarts(): - store = _FakeSessionStore() row = store.row("s1") row["alive"] = True @@ -176,7 +173,6 @@ async def new_run(): @pytest.mark.asyncio async def test_cancel_marks_dead(): - store = _FakeSessionStore() row = store.row("s1") row["alive"] = True @@ -194,7 +190,6 @@ async def test_cancel_marks_dead(): @pytest.mark.asyncio async def test_attach_to_alive_detached_run(): - store = _FakeSessionStore() row = store.row("s1") row["alive"] = True @@ -211,7 +206,6 @@ async def test_attach_to_alive_detached_run(): @pytest.mark.asyncio async def test_detach_on_client_disconnect_keeps_run_alive(): - store = _FakeSessionStore() row = store.row("s1") row["alive"] = True diff --git a/services/runner/src/engines/sandbox_agent/acp-interactions.ts b/services/runner/src/engines/sandbox_agent/acp-interactions.ts index 321d6e6c29..3f04a0eecb 100644 --- a/services/runner/src/engines/sandbox_agent/acp-interactions.ts +++ b/services/runner/src/engines/sandbox_agent/acp-interactions.ts @@ -54,6 +54,8 @@ export interface AttachPermissionResponderInput { onPausedToolCall?: (id: string) => void; /** Called before an allow reply can release the harness to execute this tool call. */ onAllowedExecution?: (id: string) => void; + /** Called before a deny reply so its failed terminal frame remains authoritative. */ + onAnsweredDeny?: (id: string) => void; /** * Called when a NON-parkable pause happens this turn (a client-tool ACP gate, which cannot be * answered across a turn boundary on the live session). The keep-alive dispatch reads this to @@ -124,6 +126,7 @@ export function attachPermissionResponder({ log, onPausedToolCall, onAllowedExecution, + onAnsweredDeny, onNonParkablePause, onCreateInteraction, onResolveInteraction, @@ -225,13 +228,15 @@ export function attachPermissionResponder({ onPause?.(); }; - // Resolve the durable row this gate created, if it created one. - const resolveIfCreated = ( + // A cold responder has no local creation set, so its stored decision carries the original token. + const resolveAfterReply = ( id: string, verdict?: { approved: boolean; toolCallId: string }, + interactionToken?: string, ): void => { - if (!createdInteractionIds.delete(id)) return; - onResolveInteraction?.(id, verdict); + const token = createdInteractionIds.delete(id) ? id : interactionToken; + if (!token) return; + onResolveInteraction?.(token, verdict); }; const replyPermission = async ( @@ -239,12 +244,16 @@ export function attachPermissionResponder({ decision: PermissionDecision, availableReplies: string[], toolCallId?: string, + interactionToken?: string, ): Promise => { // A deny replies `reject`, which makes the harness close the call as a FAILED tool call. Flag // the id first so the closing result carries `denied` and the egress renders a decline, not a // breakage. (A malformed-envelope / unknown-builtin fail-closed reject goes through // `rejectRequest`, not here, so it stays a plain error — it is not a user/policy denial.) - if (decision === "deny") run.markToolCallDenied?.(toolCallId); + if (decision === "deny") { + run.markToolCallDenied?.(toolCallId); + if (toolCallId) onAnsweredDeny?.(toolCallId); + } // Mark before replying because an allow can release the harness synchronously and its first // execution frame must already be protected from a concurrently active pause sweep. if (decision === "allow" && toolCallId) onAllowedExecution?.(toolCallId); @@ -260,10 +269,14 @@ export function attachPermissionResponder({ onPause?.(); return; } - resolveIfCreated(id, { - approved: decision === "allow", - toolCallId: toolCallId ?? id, - }); + resolveAfterReply( + id, + { + approved: decision === "allow", + toolCallId: toolCallId ?? id, + }, + interactionToken, + ); }; const replyClientTool = async ( @@ -283,7 +296,7 @@ export function attachPermissionResponder({ onPause?.(); return; } - resolveIfCreated(id); + resolveAfterReply(id); }; // A bare reject that answers the harness WITHOUT touching the durable interactions plane (no @@ -369,7 +382,13 @@ export function attachPermissionResponder({ if (verdict.kind === "allow" && envelope.gate === "pi-custom-tool") { onPiGateAllowed?.({ toolName: gate.toolName!, args: gate.args }); } - await replyPermission(id, verdict.kind, availableReplies, envelope.toolCallId); + await replyPermission( + id, + verdict.kind, + availableReplies, + envelope.toolCallId, + verdict.interactionToken, + ); }; async function handleRequest(req: any): Promise { @@ -458,6 +477,7 @@ export function attachPermissionResponder({ verdict.kind, availableReplies, stringValue(toolCall?.toolCallId), + verdict.interactionToken, ); } } diff --git a/services/runner/src/engines/sandbox_agent/pause.ts b/services/runner/src/engines/sandbox_agent/pause.ts index 1555790973..fe7b2dc23e 100644 --- a/services/runner/src/engines/sandbox_agent/pause.ts +++ b/services/runner/src/engines/sandbox_agent/pause.ts @@ -8,11 +8,16 @@ */ export const PAUSED = Symbol("paused"); +const EVENT_DRAIN_QUIET_TICKS = 2; +const EVENT_DRAIN_MAX_TICKS = 6; + export class PendingApprovalPauseController { private pendingApproval = false; private readonly pausedToolCallIds = new Set(); private readonly allowedExecutionToolCallIds = new Set(); + private readonly answeredDenyToolCallIds = new Set(); private resolvePause: (() => void) | undefined; + private gateClassificationVersion = 0; private eventDrain: Promise = Promise.resolve(); readonly signal: Promise; @@ -36,12 +41,27 @@ export class PendingApprovalPauseController { } this.eventDrain = Promise.resolve(destroyResult) .catch(() => {}) - .then(() => new Promise((resolve) => setImmediate(resolve))); + .then(() => this.waitForGateClassificationQuietPeriod()); // The turn races this immediate signal; terminalization separately awaits eventDrain so the // permission callback never holds the human pause open while queued ACP updates still settle. this.resolvePause?.(); } + private async waitForGateClassificationQuietPeriod(): Promise { + let observedVersion = this.gateClassificationVersion; + let quietTicks = 0; + for (let tick = 0; tick < EVENT_DRAIN_MAX_TICKS; tick += 1) { + await new Promise((resolve) => setImmediate(resolve)); + if (observedVersion !== this.gateClassificationVersion) { + observedVersion = this.gateClassificationVersion; + quietTicks = 0; + continue; + } + quietTicks += 1; + if (quietTicks >= EVENT_DRAIN_QUIET_TICKS) return; + } + } + /** Wait until managed cancellation and already-queued ACP updates have drained. */ waitForEventDrain(): Promise { return this.eventDrain; @@ -54,7 +74,10 @@ export class PendingApprovalPauseController { */ markPausedToolCall(toolCallId: string): void { if (!toolCallId) return; + const added = !this.pausedToolCallIds.has(toolCallId); this.pausedToolCallIds.add(toolCallId); + // A late gate must extend terminalization long enough for its paused classification to land. + if (this.pendingApproval && added) this.gateClassificationVersion += 1; } isPausedToolCall(toolCallId: string | undefined): boolean { @@ -67,6 +90,19 @@ export class PendingApprovalPauseController { this.allowedExecutionToolCallIds.add(toolCallId); } + /** Record an answered deny so its authoritative failed frame survives a sibling pause. */ + markAnsweredDeny(toolCallId: string): void { + if (!toolCallId) return; + this.answeredDenyToolCallIds.add(toolCallId); + } + + isAnsweredDeny(toolCallId: string | undefined): boolean { + return ( + toolCallId !== undefined && + this.answeredDenyToolCallIds.has(toolCallId) + ); + } + isAllowedExecution(toolCallId: string | undefined): boolean { return ( toolCallId !== undefined && diff --git a/services/runner/src/engines/sandbox_agent/run-turn.ts b/services/runner/src/engines/sandbox_agent/run-turn.ts index d0762160e8..ab79ca98b1 100644 --- a/services/runner/src/engines/sandbox_agent/run-turn.ts +++ b/services/runner/src/engines/sandbox_agent/run-turn.ts @@ -514,6 +514,7 @@ export async function runTurn( onPause: () => pause.pause(), onPausedToolCall: (id) => pause.markPausedToolCall(id), onAllowedExecution: (id) => pause.markAllowedExecution(id), + onAnsweredDeny: (id) => pause.markAnsweredDeny(id), onNonParkablePause: () => { env.nonParkablePauseCount += 1; }, @@ -663,6 +664,7 @@ export async function runTurn( // projects `tool-output-denied` (a decline), mirroring the cold decision-map deny path. if (decision.reply === "reject") { run.markToolCallDenied(decision.toolCallId); + pause.markAnsweredDeny(decision.toolCallId); } // Answer this gate on the live session. Each parked gate holds its OWN pending // `respondPermission` on the harness, so answering them one by one settles each diff --git a/services/runner/src/engines/sandbox_agent/runtime-policy.ts b/services/runner/src/engines/sandbox_agent/runtime-policy.ts index 80eb5104c1..7d883bc2c4 100644 --- a/services/runner/src/engines/sandbox_agent/runtime-policy.ts +++ b/services/runner/src/engines/sandbox_agent/runtime-policy.ts @@ -42,13 +42,13 @@ export function shouldSuppressPausedToolCallUpdate( // F-024: a paused (gated) tool call's later frames are teardown artifacts and never reach the // stream. if (pause.isPausedToolCall(toolCallId)) return true; - // While pausing, a failed frame without an allow is a managed-cancel artifact that the sweep - // records as deferred; an allowed call failure is genuine terminal evidence and must pass. + // Answered allows and denies both carry authoritative terminal evidence through a sibling pause. if ( kind === "tool_call_update" && pause.active && frame?.status === "failed" && - !pause.isAllowedExecution(toolCallId) + !pause.isAllowedExecution(toolCallId) && + !pause.isAnsweredDeny(toolCallId) ) { return true; } diff --git a/services/runner/src/permission-plan.ts b/services/runner/src/permission-plan.ts index 3ebc43ce0b..b9ec102dcc 100644 --- a/services/runner/src/permission-plan.ts +++ b/services/runner/src/permission-plan.ts @@ -57,11 +57,19 @@ export interface PiBuiltinIdentity { readOnly: boolean; } +export interface StoredPermissionDecision { + decision: "allow" | "deny"; + interactionToken?: string; +} + export type Verdict = - { kind: "allow" } | { kind: "deny" } | { kind: "pendingApproval" }; + | { kind: "allow" | "deny"; interactionToken?: string } + | { kind: "pendingApproval" }; export interface StoredPermissionDecisions { - take(gate: GateDescriptor): "allow" | "deny" | undefined; + take( + gate: GateDescriptor, + ): "allow" | "deny" | StoredPermissionDecision | undefined; } const PERMISSION_MODES: readonly PermissionMode[] = [ @@ -145,8 +153,18 @@ export function decide( if (permission === "allow") return { kind: "allow" }; const storedDecision = stored.take(gate); - if (storedDecision === "allow") return { kind: "allow" }; - if (storedDecision === "deny") return { kind: "deny" }; + const decision = + typeof storedDecision === "string" + ? storedDecision + : storedDecision?.decision; + if (decision === "allow" || decision === "deny") { + return { + kind: decision, + ...(typeof storedDecision === "object" && storedDecision.interactionToken + ? { interactionToken: storedDecision.interactionToken } + : {}), + }; + } return { kind: "pendingApproval" }; } diff --git a/services/runner/src/responder.ts b/services/runner/src/responder.ts index f201597cfd..9a36ae8ddc 100644 --- a/services/runner/src/responder.ts +++ b/services/runner/src/responder.ts @@ -17,6 +17,7 @@ import { storedDecisionKeyShape, type GateDescriptor, type PermissionPlan, + type StoredPermissionDecision, type StoredPermissionDecisions, type Verdict, } from "./permission-plan.ts"; @@ -256,13 +257,15 @@ export class ConversationDecisions implements StoredPermissionDecisions { } /** allow|deny for this exact call (name + canonical args), consumed on first take. */ - take(gate: GateDescriptor): "allow" | "deny" | undefined { + take( + gate: GateDescriptor, + ): PermissionDecision | StoredPermissionDecision | undefined { const key = approvedCallKey(gate.toolName, gate.args); if (!key) return undefined; const queue = this.decisionQueues.get(key); if (!queue || queue.length === 0) return undefined; const value = queue[0]; - if (!isPermissionDecision(value)) return undefined; + if (!isStoredPermissionDecision(value)) return undefined; queue.shift(); if (queue.length === 0) this.decisionQueues.delete(key); return value; @@ -338,7 +341,11 @@ export class ApprovalResponder implements Responder { if (permission === "ask") { const storedDecision = this.decisions.take(request.gate); - if (storedDecision === "deny") return { kind: "deny" }; + const decision = + typeof storedDecision === "string" + ? storedDecision + : storedDecision?.decision; + if (decision === "deny") return { kind: "deny" }; } return { kind: "pendingApproval" }; } @@ -364,7 +371,7 @@ export function extractApprovalDecisions( const decisions = new Map(); const callShapeById = buildCallShapeIndex(request); for (const block of toolResultBlocks(request)) { - const decision = approvalDecisionOf(block); + const decision = storedApprovalDecisionOf(block); if (decision === undefined) continue; const argsKey = coldReplayKey(block, callShapeById); if (!argsKey) continue; @@ -485,6 +492,16 @@ function isPermissionDecision(value: unknown): value is PermissionDecision { return value === "allow" || value === "deny"; } +function isStoredPermissionDecision( + value: unknown, +): value is PermissionDecision | StoredPermissionDecision { + if (isPermissionDecision(value)) return true; + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + return isPermissionDecision( + (value as { decision?: unknown }).decision, + ); +} + /** * Pause terminalization sentinels are runner bookkeeping, not browser outputs. A deferred call * may safely re-park, while an approved call with an unobserved result must never be retried. @@ -498,25 +515,35 @@ function isPauseSyntheticResult(block: ContentBlock): boolean { } /** - * An approval reply uses an `{ approved: boolean }` envelope (the Vercel adapter's - * `_approval_response_blocks` shape), unique to a permission response. Returns - * `"allow"`/`"deny"` for one, or `undefined` for any other tool_result (a real browser/client - * output). + * Approval envelopes are the only tool results treated as permission decisions. Preserve their + * interaction token so a cold responder can resolve the original row after replying successfully; + * real browser/client outputs remain outside this path. */ -export function approvalDecisionOf( +function storedApprovalDecisionOf( block: ContentBlock, -): PermissionDecision | undefined { +): StoredPermissionDecision | undefined { if (!block || block.type !== "tool_result") return undefined; const output = block.output; - if ( - output && - typeof output === "object" && - !Array.isArray(output) && - typeof (output as { approved?: unknown }).approved === "boolean" - ) { - return (output as { approved: boolean }).approved ? "allow" : "deny"; + if (!output || typeof output !== "object" || Array.isArray(output)) { + return undefined; } - return undefined; + const envelope = output as { + approved?: unknown; + interactionToken?: unknown; + }; + if (typeof envelope.approved !== "boolean") return undefined; + return { + decision: envelope.approved ? "allow" : "deny", + ...(typeof envelope.interactionToken === "string" && envelope.interactionToken + ? { interactionToken: envelope.interactionToken } + : {}), + }; +} + +export function approvalDecisionOf( + block: ContentBlock, +): PermissionDecision | undefined { + return storedApprovalDecisionOf(block)?.decision; } /** diff --git a/services/runner/src/server.ts b/services/runner/src/server.ts index 8e1a326c25..c8e6d5ab8c 100644 --- a/services/runner/src/server.ts +++ b/services/runner/src/server.ts @@ -837,12 +837,26 @@ const runAgent: RunAgent = (request, emit, signal, options) => { }; /** - * The durable interaction tokens of parked approval gates this request answers in-band, if - * any. The turn-start cancel-stale sweep must spare them: each gate belongs to the PREVIOUS turn - * (so the sweep's own `turn_id` exemption misses it), an in-band answer never transitions the - * row off `pending` (only the interactions-plane respond endpoint does), and the resume - * resolves the token after consuming the decision. Swept first, the granted gate's record - * lands as `cancelled` and the resolve 404s. + * The stale-cancel exemptions for a parked approval set. A partial live resume owns both the + * answered and carried gates, so any answer spares every parked token; zero answers spare none. + */ +export function staleInteractionExemptTokens( + request: AgentRunRequest, + parked: ReadonlyMap, +): string[] | undefined { + const hasAnswer = [...parked.values()].some( + (gate) => + approvalDecisionForToolCall(request, gate.toolCallId) !== undefined, + ); + return hasAnswer + ? [...parked.values()].map((gate) => gate.interactionToken) + : undefined; +} + +/** + * A live partial resume owns the whole parked set, including unanswered gates it carries into the + * next pause. Once any parked gate is answered, the stale sweep must spare every parked token; a + * zero-answer request owns none and receives no exemptions. */ function inBandAnswerTokens(request: AgentRunRequest): string[] | undefined { const sessionId = request.sessionId?.trim(); @@ -853,15 +867,7 @@ function inBandAnswerTokens(request: AgentRunRequest): string[] | undefined { keepalivePools[provider].awaitingApproval(sessionId)?.environment .parkedApprovals; if (!parked || parked.size === 0) return undefined; - // A partial resume resolves only the answered rows. Carried-forward rows stay pending and receive - // a fresh approval park, so only matching answer tokens are exempt from stale cancellation. - const tokens: string[] = []; - for (const gate of parked.values()) { - if (approvalDecisionForToolCall(request, gate.toolCallId) !== undefined) { - tokens.push(gate.interactionToken); - } - } - return tokens.length > 0 ? tokens : undefined; + return staleInteractionExemptTokens(request, parked); } /** diff --git a/services/runner/tests/unit/pending-approval-pause.test.ts b/services/runner/tests/unit/pending-approval-pause.test.ts index 302eceb0d4..d0290286ce 100644 --- a/services/runner/tests/unit/pending-approval-pause.test.ts +++ b/services/runner/tests/unit/pending-approval-pause.test.ts @@ -32,6 +32,25 @@ describe("PendingApprovalPauseController", () => { assert.deepEqual(queuedUpdates, ["session update"]); }); + it("re-arms the bounded drain when a gate is classified after the first tick", async () => { + const pause = new PendingApprovalPauseController(() => {}); + pause.markPausedToolCall("tool-first"); + pause.pause(); + await Promise.resolve(); + await Promise.resolve(); + + let lateGateClassified = false; + setImmediate(() => { + pause.markPausedToolCall("tool-late"); + lateGateClassified = true; + }); + + await pause.waitForEventDrain(); + + assert.equal(lateGateClassified, true); + assert.equal(pause.isPausedToolCall("tool-late"), true); + }); + it("finishes the event drain when managed cancellation rejects", async () => { const pause = new PendingApprovalPauseController(() => Promise.reject(new Error("session already gone")), diff --git a/services/runner/tests/unit/responder.test.ts b/services/runner/tests/unit/responder.test.ts index 130aa69445..0d43735cc1 100644 --- a/services/runner/tests/unit/responder.test.ts +++ b/services/runner/tests/unit/responder.test.ts @@ -452,10 +452,10 @@ describe("extractApprovalDecisions", () => { const decisions = extractApprovalDecisions(request); assert.deepEqual( decisions.get(approvedCallKey("edit", { path: "a.txt" })!), - ["allow"], + [{ decision: "allow" }], ); assert.deepEqual(decisions.get(approvedCallKey("bash", { cmd: "ls" })!), [ - "deny", + { decision: "deny" }, ]); assert.equal(decisions.has("edit"), false); assert.equal(decisions.has("tc-1"), false); @@ -502,12 +502,15 @@ describe("extractApprovalDecisions", () => { const key = approvedCallKey("edit", { path: "a.txt" })!; const decisions = extractApprovalDecisions(request); - assert.deepEqual(decisions.get(key), ["allow", "allow"]); + assert.deepEqual(decisions.get(key), [ + { decision: "allow" }, + { decision: "allow" }, + ]); const stored = new ConversationDecisions(decisions); const duplicateGate = gate({ toolName: "edit", args: { path: "a.txt" } }); - assert.equal(stored.take(duplicateGate), "allow"); - assert.equal(stored.take(duplicateGate), "allow"); + assert.deepEqual(stored.take(duplicateGate), { decision: "allow" }); + assert.deepEqual(stored.take(duplicateGate), { decision: "allow" }); assert.equal(stored.take(duplicateGate), undefined); }); @@ -541,13 +544,13 @@ describe("extractApprovalDecisions", () => { }; const stored = new ConversationDecisions(extractApprovalDecisions(request)); - assert.equal( + assert.deepEqual( stored.take({ executor: "harness", toolName: "mcp__agenta-tools__commit_revision", args: { workflow_revision: JSON.stringify(workflowRevision) }, }), - "allow", + { decision: "allow" }, ); }); @@ -673,7 +676,7 @@ describe("client-tool output store (separate from approvals)", () => { // ...and lives only in the approval store. const decisions = extractApprovalDecisions(request); assert.deepEqual(decisions.get(approvedCallKey("edit", { path: "a" })!), [ - "allow", + { decision: "allow" }, ]); }); diff --git a/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts b/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts index d3650d58b4..663d3530d5 100644 --- a/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts +++ b/services/runner/tests/unit/sandbox-agent-acp-interactions.test.ts @@ -6,12 +6,17 @@ import { describe, it } from "vitest"; import assert from "node:assert/strict"; -import type { AgentEvent, ResolvedToolSpec } from "../../src/protocol.ts"; +import type { + AgentEvent, + AgentRunRequest, + ResolvedToolSpec, +} from "../../src/protocol.ts"; import { toolSpecsByName } from "../../src/tools/public-spec.ts"; import type { ClientToolVerdict, Responder } from "../../src/responder.ts"; import { ApprovalResponder, ConversationDecisions, + extractApprovalDecisions, } from "../../src/responder.ts"; import type { PermissionPlan, Verdict } from "../../src/permission-plan.ts"; import { @@ -446,6 +451,76 @@ describe("attachPermissionResponder", () => { assert.deepEqual(events, []); }); + it("resolves the original interaction token after a cold stored-decision reply", async () => { + const decisions = extractApprovalDecisions({ + messages: [ + { + role: "assistant", + content: [ + { + type: "tool_call", + toolCallId: "tool-original", + toolName: "edit", + input: { path: "a.txt" }, + }, + ], + }, + { + role: "tool", + content: [ + { + type: "tool_result", + toolCallId: "tool-original", + output: { + approved: true, + interactionToken: "interaction-original", + }, + }, + ], + }, + ], + } as AgentRunRequest); + const responder = new ApprovalResponder( + permissionPlan("ask"), + new ConversationDecisions(decisions), + ); + const replies: Array<{ id: string; reply: string }> = []; + const resolved: Array<{ + token: string; + verdict?: { approved: boolean; toolCallId: string }; + }> = []; + const { session, emit } = makeSession(async (id, reply) => { + replies.push({ id, reply }); + }); + + attachPermissionResponder({ + session, + run: { emitEvent: () => {} }, + responder, + onResolveInteraction: (token, verdict) => { + resolved.push({ token, verdict }); + }, + }); + emit({ + id: "permission-cold", + availableReplies: ["once", "reject"], + toolCall: { + toolCallId: "tool-cold", + name: "edit", + rawInput: { path: "a.txt" }, + }, + }); + await flushPromises(); + + assert.deepEqual(replies, [{ id: "permission-cold", reply: "once" }]); + assert.deepEqual(resolved, [ + { + token: "interaction-original", + verdict: { approved: true, toolCallId: "tool-cold" }, + }, + ]); + }); + it("passes the approval verdict when resolving a previously created gate", async () => { const { session, emit } = makeSession(); let permissionCalls = 0; diff --git a/services/runner/tests/unit/session-keepalive-approval.test.ts b/services/runner/tests/unit/session-keepalive-approval.test.ts index 5f2c0eba36..6df4834754 100644 --- a/services/runner/tests/unit/session-keepalive-approval.test.ts +++ b/services/runner/tests/unit/session-keepalive-approval.test.ts @@ -20,6 +20,7 @@ import type { } from "../../src/protocol.ts"; import { runWithKeepalive, + staleInteractionExemptTokens, type KeepaliveContext, type KeepaliveEngine, } from "../../src/server.ts"; @@ -600,7 +601,7 @@ describe("runWithKeepalive: approval park + resume", () => { ); }); - it("resumes a partly answered two-gate turn live, re-parks, then completes live", async () => { + it("spares every parked token while a partial answer re-parks and then completes live", async () => { const { engine, calls } = makeApprovalEngine([ { approvalPause: { @@ -621,11 +622,21 @@ describe("runWithKeepalive: approval park + resume", () => { ]; await runWithKeepalive(pauseTurn(), undefined, undefined, ctx); - const r2 = await runWithKeepalive( - approveResumeMulti( - [{ toolCallId: "tc-1", toolName: "read_a", approved: true }], - parkedCalls, + const partialRequest = approveResumeMulti( + [{ toolCallId: "tc-1", toolName: "read_a", approved: true }], + parkedCalls, + ); + assert.deepEqual( + staleInteractionExemptTokens( + partialRequest, + calls.acquiredEnvs[0].parkedApprovals, ), + ["tc-1", "tc-2"], + "the carried gate must survive the turn-start stale sweep", + ); + + const r2 = await runWithKeepalive( + partialRequest, undefined, undefined, ctx, @@ -681,8 +692,21 @@ describe("runWithKeepalive: approval park + resume", () => { const ctx = makeCtx(engine); await runWithKeepalive(pauseTurn(), undefined, undefined, ctx); + const zeroAnswerRequest = approveResumeMulti( + [], + [{ toolCallId: "tc-1" }, { toolCallId: "tc-2" }], + ); + assert.equal( + staleInteractionExemptTokens( + zeroAnswerRequest, + calls.acquiredEnvs[0].parkedApprovals, + ), + undefined, + "zero answers leave stale cancellation enabled", + ); + await runWithKeepalive( - approveResumeMulti([], [{ toolCallId: "tc-1" }, { toolCallId: "tc-2" }]), + zeroAnswerRequest, undefined, undefined, ctx, @@ -1172,6 +1196,7 @@ interface FakeRun { open: Set; /** What settleOpenToolCalls settled: the orphaned-sibling record (F-024). */ settled: Array<{ id: string; message: string }>; + denied: string[]; sweeps: Array<{ message: string; isExcluded: (id: string) => boolean; @@ -1492,6 +1517,88 @@ describe("runTurn: real approval park + respondPermission resume", () => { await env.destroy(); }); + it("preserves a denied call failed frame while a sibling gate is carried", async () => { + const { calls, deps, captured } = pausableHarness(); + const acquired = await acquireEnvironment(engineReq, deps); + assert.equal(acquired.ok, true); + if (!acquired.ok) return; + const env = acquired.env; + + const firstTurn = runTurn(env, engineReq, undefined, undefined, { + approvalParkMode: true, + }); + await flush(); + for (const [toolCallId, title] of [["tc-denied", "commit"], ["tc-carried", "deploy"]]) { + captured.onEvent!( + updateEvent({ sessionUpdate: "tool_call", toolCallId, title }), + ); + } + captured.onPermissionRequest!({ + id: "perm-denied", + availableReplies: ["once", "reject"], + toolCall: { toolCallId: "tc-denied", name: "commit", rawInput: {} }, + }); + captured.onPermissionRequest!({ + id: "perm-carried", + availableReplies: ["once", "reject"], + toolCall: { toolCallId: "tc-carried", name: "deploy", rawInput: {} }, + }); + await firstTurn; + + const answered = env.parkedApprovals.get("tc-denied")!; + const carried = env.parkedApprovals.get("tc-carried")!; + env.clearTurn(); + const resumeTurn = runTurn(env, approveResume(false), undefined, undefined, { + approvalParkMode: true, + resume: { + decisions: [ + { + permissionId: answered.permissionId, + reply: "reject", + toolCallId: answered.toolCallId, + toolName: answered.toolName, + args: answered.args, + interactionToken: answered.interactionToken, + promptPromise: answered.promptPromise, + }, + ], + carriedForward: [carried], + }, + }); + for (let i = 0; i < 5 && !env.currentTurn?.pause.active; i += 1) { + await Promise.resolve(); + } + assert.equal(env.currentTurn?.pause.active, true); + captured.onEvent!( + updateEvent({ + sessionUpdate: "tool_call_update", + toolCallId: "tc-denied", + status: "failed", + content: "user denied", + }), + ); + const result = await resumeTurn; + + assert.equal(result.stopReason, "paused"); + const run = calls.runs[1]; + assert.deepEqual(run.denied, ["tc-denied"]); + assert.equal( + run.handled.some( + (update) => + update.toolCallId === "tc-denied" && update.status === "failed", + ), + true, + "the authoritative denial frame must reach the tracer", + ); + assert.equal( + run.settled.some(({ id }) => id === "tc-denied"), + false, + "the deferred sentinel must not replace a denial", + ); + + await env.destroy(); + }); + it("carries an unanswered gate through a partial resume without settling it", async () => { const { calls, deps, captured } = pausableHarness(); const acquired = await acquireEnvironment(engineReq, deps); diff --git a/web/oss/src/components/AgentChatSlice/AgentConversation.tsx b/web/oss/src/components/AgentChatSlice/AgentConversation.tsx index 0888609fda..0fc3d7f26b 100644 --- a/web/oss/src/components/AgentChatSlice/AgentConversation.tsx +++ b/web/oss/src/components/AgentChatSlice/AgentConversation.tsx @@ -26,6 +26,7 @@ import { buildAgentRequest, buildTurnCapture, playgroundController, + type LiveAgentInteraction, } from "@agenta/playground" import {agentSelfCommitSignalAtom, simulatedAgentRunAtomFamily} from "@agenta/shared/state" import {generateId} from "@agenta/shared/utils" @@ -568,7 +569,7 @@ const AgentConversation = ({ const revalidateSessionMounts = useSetAtom(revalidateSessionMountsAtom) const revalidateSessionRecords = useSetAtom(revalidateSessionRecordsAtom) // Only a gate settled in this mount may trigger an automatic resume; hydrated answers stay inert. - const liveGateInteractionRef = useRef(false) + const liveGateInteractionRef = useRef(null) const { messages, @@ -589,11 +590,14 @@ const AgentConversation = ({ experimental_throttle: 50, // Approve AND deny both resume — a deny-only decision must re-send so the runner // gets the denial round-trip and the model continues (no `approval-responded` limbo). - sendAutomaticallyWhen: ({messages}) => - agentShouldResumeAfterApproval({ + sendAutomaticallyWhen: ({messages}) => { + const shouldDispatch = agentShouldResumeAfterApproval({ messages, liveInteraction: liveGateInteractionRef.current, - }), + }) + if (shouldDispatch) liveGateInteractionRef.current = null + return shouldDispatch + }, // The turn's trace may not be ingested yet when the row asks for its summary — // marking it fresh lets the trace queries retry through the ingestion lag // (historical traces get no such grace; a 404 there means the trace is gone). @@ -683,7 +687,7 @@ const AgentConversation = ({ // is by id — so a cast onto the untyped UIMessage tool map is safe. const handleClientToolOutput = useCallback( ({toolName, toolCallId, output, errorText}) => { - liveGateInteractionRef.current = true + liveGateInteractionRef.current = {kind: "client_tool", id: toolCallId} if (errorText !== undefined) { addToolOutput({ state: "output-error", @@ -1035,7 +1039,7 @@ const AgentConversation = ({ // after a reload genuinely auto-resumes, so the queue's pre-resume hold must apply to it. const handleApprovalResponse = useCallback( (args: {id: string; approved: boolean}) => { - liveGateInteractionRef.current = true + liveGateInteractionRef.current = {kind: "approval", id: args.id} addToolApprovalResponse(args) }, [addToolApprovalResponse], diff --git a/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.test.ts b/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.test.ts index e5c773f63d..0cbbd129b4 100644 --- a/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.test.ts +++ b/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.test.ts @@ -120,6 +120,7 @@ describe("transcriptToMessages approval hydration", () => { output: "DEFERRED_NOT_EXECUTED: paused for another approval; retry the same call if still required.", isError: true, }), + record("record-done-turn-1", {type: "done"}), record("record-response-a", { type: "interaction_response", id: "approval-a", @@ -145,6 +146,7 @@ describe("transcriptToMessages approval hydration", () => { }, }, }), + record("record-done-turn-2", {type: "done"}), ]) expect(messages).not.toBeNull() @@ -152,7 +154,9 @@ describe("transcriptToMessages approval hydration", () => { role: "user", parts: [{type: "text", text: "run both writes"}], }) - const assistantParts = messages?.[1].parts as unknown as Record[] + const assistantParts = messages + ?.filter((message) => message.role === "assistant") + .flatMap((message) => message.parts) as unknown as Record[] const callA = assistantParts.find((part) => part.toolCallId === "tool-a") const callB = assistantParts.find((part) => part.toolCallId === "tool-b") diff --git a/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.ts b/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.ts index 5b2928b4b7..203ed7a81b 100644 --- a/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.ts +++ b/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.ts @@ -25,14 +25,17 @@ interface DraftMessage { /** Open streamed text/reasoning parts keyed by event id, for delta accumulation. */ text: Map reasoning: Map - /** Tool parts keyed by toolCallId so results/approvals attach to the right call. */ - tools: Map /** The turn's observability trace id, if the durable record carries one (see below). */ traceId?: string /** Token/cost totals from the turn's persisted `usage` event, in the raw stream shape. */ usage?: {input?: number; output?: number; total?: number; cost?: number} } +interface TranscriptIndex { + tools: Map + approvals: Map +} + const roleOf = (sender?: string | null): "user" | "assistant" => sender === "user" ? "user" : "assistant" @@ -74,13 +77,16 @@ const newDraft = (id: string, role: "user" | "assistant"): DraftMessage => ({ parts: [], text: new Map(), reasoning: new Map(), - tools: new Map(), }) const toolPartType = (name?: string | null): string => (name ? `tool-${name}` : "dynamic-tool") /** Apply one transcript event's payload onto the current assistant/user draft message. */ -function applyEvent(draft: DraftMessage, payload: Record): void { +function applyEvent( + draft: DraftMessage, + payload: Record, + index: TranscriptIndex, +): void { const type = payload.type as string | undefined const str = (v: unknown): string => (typeof v === "string" ? v : v == null ? "" : String(v)) @@ -124,11 +130,11 @@ function applyEvent(draft: DraftMessage, payload: Record): void input: payload.input, } draft.parts.push(part) - draft.tools.set(toolCallId, part) + index.tools.set(toolCallId, part) return } case "tool_result": { - const part = draft.tools.get(str(payload.id)) + const part = index.tools.get(str(payload.id)) if (!part) return if (payload.denied) { part.state = "output-denied" @@ -151,7 +157,7 @@ function applyEvent(draft: DraftMessage, payload: Record): void const toolCallId = str( reqPayload.toolCallId ?? toolCall.id ?? toolCall.toolCallId ?? payload.id, ) - let part = draft.tools.get(toolCallId) + let part = index.tools.get(toolCallId) if (!part) { // The runner parked without first surfacing the tool call — synthesize one. part = { @@ -165,8 +171,9 @@ function applyEvent(draft: DraftMessage, payload: Record): void input: toolCall.rawInput ?? toolCall.input, } draft.parts.push(part) - draft.tools.set(toolCallId, part) + index.tools.set(toolCallId, part) } + index.approvals.set(str(payload.id), part) // Only park if still unsettled — a later `tool_result` overwrites this. if (part.state === "input-available") { part.state = "approval-requested" @@ -179,13 +186,9 @@ function applyEvent(draft: DraftMessage, payload: Record): void const responsePayload = (payload.payload ?? {}) as Record const responseId = str(payload.id) const toolCallId = str(responsePayload.toolCallId) - let part = toolCallId ? draft.tools.get(toolCallId) : undefined - if (!part) { - part = draft.parts.find((candidate) => { - const approval = (candidate.approval ?? {}) as Record - return str(approval.id) === responseId - }) - } + const part = + (toolCallId ? index.tools.get(toolCallId) : undefined) ?? + index.approvals.get(responseId) if ( !part || part.state !== "approval-requested" || @@ -241,6 +244,8 @@ function applyEvent(draft: DraftMessage, payload: Record): void export function transcriptToMessages(records: SessionRecord[]): UIMessage[] | null { const drafts: DraftMessage[] = [] let current: DraftMessage | null = null + // Paused resumes close the draft, but later answers and results still target its tool part. + const index: TranscriptIndex = {tools: new Map(), approvals: new Map()} for (const row of records) { const payload = row.payload @@ -263,7 +268,7 @@ export function transcriptToMessages(records: SessionRecord[]): UIMessage[] | nu drafts.push(current) } if (traceId && !current.traceId) current.traceId = traceId - applyEvent(current, p) + applyEvent(current, p, index) } const messages = drafts diff --git a/web/packages/agenta-playground/src/index.ts b/web/packages/agenta-playground/src/index.ts index e6501fef93..23faac7879 100644 --- a/web/packages/agenta-playground/src/index.ts +++ b/web/packages/agenta-playground/src/index.ts @@ -79,7 +79,7 @@ export { } from "./state" export type {AgentRequest, AgentChannelMode, NegotiatingFetch} from "./state" // HITL resume predicate for `useChat`'s `sendAutomaticallyWhen` (approve AND deny resume). -export {agentShouldResumeAfterApproval} from "./state" +export {agentShouldResumeAfterApproval, type LiveAgentInteraction} from "./state" // Render-hint map for interaction kinds (sibling `data-render` parts → toolCallId lookup). export {buildRenderMap, renderKindFor, type RenderHintLike} from "./state" // Queued-message release gate for the agent chat composer (HITL-safe, one-by-one). diff --git a/web/packages/agenta-playground/src/state/execution/agentApprovalResume.ts b/web/packages/agenta-playground/src/state/execution/agentApprovalResume.ts index 6eced25576..63d5128eed 100644 --- a/web/packages/agenta-playground/src/state/execution/agentApprovalResume.ts +++ b/web/packages/agenta-playground/src/state/execution/agentApprovalResume.ts @@ -24,11 +24,16 @@ */ interface ApprovalLike { + id?: string approved?: boolean } import {buildRenderMap, renderKindFor, type RenderHintLike} from "./renderMap" +export type LiveAgentInteraction = + | {kind: "approval"; id: string} + | {kind: "client_tool"; id: string} + interface ToolPartLike { type?: string state?: string @@ -128,10 +133,10 @@ const isSettledToolPart = (part: ToolPartLike): boolean => */ export function agentShouldResumeAfterApproval({ messages, - liveInteraction = true, + liveInteraction, }: { messages: MessageLike[] - liveInteraction?: boolean + liveInteraction?: LiveAgentInteraction | null }): boolean { const last = messages[messages.length - 1] if (!last || last.role !== "assistant") return false @@ -143,18 +148,26 @@ export function agentShouldResumeAfterApproval({ // Message-scoped render hints (sibling `data-render` parts) for client-tool detection. const renderMap = buildRenderMap(parts) - // Index of the LAST freshly-resolved parked interaction (an approval response or a - // browser-fulfilled client-tool result). Using the last one handles chained gates: a second - // approval later in the turn is what should drive the (next) resume. let lastResolvedIdx = -1 let lastResolvedIsApproval = false - for (let i = 0; i < parts.length; i++) { - if (isRespondedToolPart(parts[i])) { - lastResolvedIdx = i - lastResolvedIsApproval = true - } else if (isClientToolResult(parts[i], renderMap)) { - lastResolvedIdx = i - lastResolvedIsApproval = false + if (liveInteraction === null) return false + if (liveInteraction) { + lastResolvedIsApproval = liveInteraction.kind === "approval" + lastResolvedIdx = parts.findIndex((part) => + liveInteraction.kind === "approval" + ? isRespondedToolPart(part) && part.approval?.id === liveInteraction.id + : isClientToolResult(part, renderMap) && part.toolCallId === liveInteraction.id, + ) + } else { + // Queue/orphan checks omit a marker and inspect the latest resolved interaction shape. + for (let i = 0; i < parts.length; i++) { + if (isRespondedToolPart(parts[i])) { + lastResolvedIdx = i + lastResolvedIsApproval = true + } else if (isClientToolResult(parts[i], renderMap)) { + lastResolvedIdx = i + lastResolvedIsApproval = false + } } } if (lastResolvedIdx === -1) return false @@ -170,10 +183,10 @@ export function agentShouldResumeAfterApproval({ .some((part) => part.type === "step-start") if (resumedAlready) return false - if (!liveInteraction) return false - // The AI SDK re-evaluates after message updates and waits for an in-flight stream to finish, // so an approval clicked during a resume dispatches when that stream finishes. + if (!liveInteraction) return toolParts.every(isSettledToolPart) + if (lastResolvedIsApproval) { const pendingClientTool = toolParts.some((part) => isPendingClientToolInteraction(part, renderMap), diff --git a/web/packages/agenta-playground/src/state/execution/index.ts b/web/packages/agenta-playground/src/state/execution/index.ts index f6a0778ca5..8afdad87f6 100644 --- a/web/packages/agenta-playground/src/state/execution/index.ts +++ b/web/packages/agenta-playground/src/state/execution/index.ts @@ -363,7 +363,7 @@ export {agentChannelModeAtomFamily, type AgentChannelMode} from "./channelMode" // Transport negotiation: try stream, fall back to batch on 406, error gracefully otherwise. export {createNegotiatingFetch, type NegotiatingFetch} from "./agentNegotiation" // Agent-lane HITL resume predicate (approve AND deny both resume the conversation). -export {agentShouldResumeAfterApproval} from "./agentApprovalResume" +export {agentShouldResumeAfterApproval, type LiveAgentInteraction} from "./agentApprovalResume" // Render-hint map: sibling `data-render` parts → toolCallId lookup (interaction kinds). export {buildRenderMap, renderKindFor, type RenderHintLike} from "./renderMap" // Agent-lane queued-message release gate (never releases mid-HITL or pre-resume). diff --git a/web/packages/agenta-playground/src/state/index.ts b/web/packages/agenta-playground/src/state/index.ts index eee75406f1..d41a18f761 100644 --- a/web/packages/agenta-playground/src/state/index.ts +++ b/web/packages/agenta-playground/src/state/index.ts @@ -183,7 +183,7 @@ export { } from "./execution" export {agentChannelModeAtomFamily, type AgentChannelMode} from "./execution" export {createNegotiatingFetch, type NegotiatingFetch} from "./execution" -export {agentShouldResumeAfterApproval} from "./execution" +export {agentShouldResumeAfterApproval, type LiveAgentInteraction} from "./execution" export {buildRenderMap, renderKindFor, type RenderHintLike} from "./execution" export {canReleaseQueuedMessage, isHitlPending} from "./execution" export { diff --git a/web/packages/agenta-playground/tests/unit/agentApprovalResume.test.ts b/web/packages/agenta-playground/tests/unit/agentApprovalResume.test.ts index 8b6f6e14a7..231e8ea585 100644 --- a/web/packages/agenta-playground/tests/unit/agentApprovalResume.test.ts +++ b/web/packages/agenta-playground/tests/unit/agentApprovalResume.test.ts @@ -56,7 +56,7 @@ describe("agentShouldResumeAfterApproval", () => { it("does NOT resume a rebuilt answered conversation without a live interaction", () => { const messages = [user("do it"), assistantWithTool("approval-responded", true)] - expect(agentShouldResumeAfterApproval({messages, liveInteraction: false})).toBe(false) + expect(agentShouldResumeAfterApproval({messages, liveInteraction: null})).toBe(false) }) it("does NOT resume while a gate is still pending (approval-requested)", () => { @@ -102,7 +102,12 @@ describe("agentShouldResumeAfterApproval", () => { ], }, ] - expect(agentShouldResumeAfterApproval({messages})).toBe(true) + expect( + agentShouldResumeAfterApproval({ + messages, + liveInteraction: {kind: "approval", id: "perm_1"}, + }), + ).toBe(true) }) it("resumes when a responded gate sits alongside an already-completed tool", () => { @@ -159,7 +164,59 @@ describe("agentShouldResumeAfterApproval", () => { ], }, ] - expect(agentShouldResumeAfterApproval({messages})).toBe(true) + expect( + agentShouldResumeAfterApproval({ + messages, + liveInteraction: {kind: "approval", id: "perm_1"}, + }), + ).toBe(true) + }) + + it("matches the clicked approval when a later client-tool result exists", () => { + const messages = [ + user("approve one while connection result is present"), + { + id: "a1", + role: "assistant", + parts: [ + {type: "step-start"}, + { + type: "tool-deleteFile", + toolCallId: "call_clicked", + state: "approval-responded", + input: {path: "/x"}, + approval: {id: "perm_clicked", approved: true}, + }, + { + type: "tool-writeFile", + toolCallId: "call_pending", + state: "approval-requested", + input: {path: "/y"}, + approval: {id: "perm_pending"}, + }, + { + type: "tool-request_connection", + toolCallId: "call_client", + state: "output-available", + input: {integration: "github"}, + output: {connected: true}, + }, + ], + }, + ] + + expect( + agentShouldResumeAfterApproval({ + messages, + liveInteraction: {kind: "approval", id: "perm_clicked"}, + }), + ).toBe(true) + expect( + agentShouldResumeAfterApproval({ + messages, + liveInteraction: {kind: "approval", id: "perm_missing"}, + }), + ).toBe(false) }) it("RESUMES once BOTH concurrent approval cards are answered", () => { From e8b97ae33c7b7221332bb3b51cc30c6cfb380de9 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 19 Jul 2026 22:40:54 +0200 Subject: [PATCH 10/15] fix(runner): park immediately when Pi batching blocks an approved call --- .../src/engines/sandbox_agent/run-turn.ts | 92 ++++++- .../sandbox_agent/runtime-contracts.ts | 16 ++ .../unit/session-keepalive-approval.test.ts | 260 +++++++++++++++++- 3 files changed, 359 insertions(+), 9 deletions(-) diff --git a/services/runner/src/engines/sandbox_agent/run-turn.ts b/services/runner/src/engines/sandbox_agent/run-turn.ts index ab79ca98b1..9190d90854 100644 --- a/services/runner/src/engines/sandbox_agent/run-turn.ts +++ b/services/runner/src/engines/sandbox_agent/run-turn.ts @@ -59,6 +59,7 @@ import { sendLastMessageOnly, type CurrentTurn, type ParkedApproval, + type ParkedApprovedExecution, type RunTurnOptions, type SessionEnvironment, } from "./runtime-contracts.ts"; @@ -106,6 +107,11 @@ export async function runTurn( // Reset the per-turn approval-park bookkeeping. A fresh turn starts with no parked gates; this // turn re-records them only if it pauses on ACP permission gates. (The dispatch has already // captured any prior park into `opts.resume` before calling us.) + const carriedApprovedExecutions = opts.resume + ? [...(env.parkedApprovedExecutions?.values() ?? [])] + : []; + const parkedApprovedExecutions = new Map(); + env.parkedApprovedExecutions = parkedApprovedExecutions; env.parkedApprovals.clear(); env.parkedApproval = undefined; env.approvalGateCount = 0; @@ -253,6 +259,9 @@ export async function runTurn( void pause.signal.then(() => runLimits.notePaused()); const openToolCallIds = (): string[] => run.openToolCallIds?.() ?? []; + const approvedExecutionSeeds = new Map( + carriedApprovedExecutions.map((seed) => [seed.toolCallId, seed]), + ); const bufferedPausedCompletedFrames = new Map(); const toolCallClosureWaiters = new Map void>>(); const notifyToolCallClosed = (toolCallId: string): void => { @@ -328,6 +337,30 @@ export async function runTurn( ) { env.lastTurnToolCallIds.push(frame.toolCallId); } + if ( + frame?.sessionUpdate === "tool_call" && + typeof frame.toolCallId === "string" && + frame.toolCallId + ) { + const announced = update as { + name?: unknown; + title?: unknown; + kind?: unknown; + rawInput?: unknown; + input?: unknown; + }; + const existing = approvedExecutionSeeds.get(frame.toolCallId); + const toolName = [announced.name, announced.title, announced.kind] + .find( + (value): value is string => + typeof value === "string" && value.length > 0, + ) ?? existing?.toolName; + approvedExecutionSeeds.set(frame.toolCallId, { + toolCallId: frame.toolCallId, + toolName, + args: announced.rawInput ?? announced.input ?? existing?.args, + }); + } const toolCallId = typeof rawFrame.toolCallId === "string" ? rawFrame.toolCallId @@ -379,6 +412,18 @@ export async function runTurn( extractClientToolOutputs(request), ); const executionGrants = new ApprovedExecutionGrants(); + const seedApprovedExecution = (seed: ParkedApprovedExecution): void => { + approvedExecutionSeeds.set(seed.toolCallId, seed); + run.handleUpdate({ + sessionUpdate: "tool_call", + toolCallId: seed.toolCallId, + title: seed.toolName, + kind: seed.toolName, + rawInput: seed.args, + }); + pause.markAllowedExecution(seed.toolCallId); + executionGrants.grant(seed.toolName, seed.args); + }; const responder = deps.responderFactory?.(request) ?? new ApprovalResponder(permissionPlan, decisions, logger); @@ -641,6 +686,9 @@ export async function runTurn( const decisions = opts.resume.decisions; promptPromise = Promise.resolve(decisions[0]?.promptPromise); promptPromise.catch(() => {}); + for (const seed of carriedApprovedExecutions) { + seedApprovedExecution(seed); + } for (const decision of decisions) { // Seed this run's trace with the parked tool call so the completing `tool_call_update` // closes it and the FE approval part flips to output-available even if the adapter @@ -657,6 +705,11 @@ export async function runTurn( // confirm resolves) passes the relay guard. Claude resumes grant too — harmlessly, no // guard consults it. if (decision.reply === "once") { + approvedExecutionSeeds.set(decision.toolCallId, { + toolCallId: decision.toolCallId, + toolName: decision.toolName, + args: decision.args, + }); pause.markAllowedExecution(decision.toolCallId); executionGrants.grant(decision.toolName, decision.args); } @@ -709,19 +762,42 @@ export async function runTurn( settleBufferedPausedCompletions(); const openAllowedExecutions = openToolCallIds() .filter((id) => pause.isAllowedExecution(id)); - await Promise.all( - openAllowedExecutions.map(async (toolCallId) => { - const closed = await waitForToolCallClosure( + const piBatchBlockedByApproval = Boolean( + opts.resume && + plan.isPi && + opts.approvalParkMode && + env.parkedApprovals.size > 0, + ); + if (piBatchBlockedByApproval) { + // Pi prepares every call in a parallel batch before it executes any of them. While a + // sibling gate is pending, closure is impossible, so carry the approved spans and park. + for (const toolCallId of openAllowedExecutions) { + const seed = approvedExecutionSeeds.get(toolCallId) ?? { toolCallId, - resolvedRunLimits.toolCallMs, - ); - if (closed) return; + toolName: undefined, + args: undefined, + }; + parkedApprovedExecutions.set(toolCallId, seed); run.settleOpenToolCalls( (id) => id !== toolCallId, APPROVED_EXECUTION_RESULT_UNKNOWN, ); - }), - ); + } + } else { + await Promise.all( + openAllowedExecutions.map(async (toolCallId) => { + const closed = await waitForToolCallClosure( + toolCallId, + resolvedRunLimits.toolCallMs, + ); + if (closed) return; + run.settleOpenToolCalls( + (id) => id !== toolCallId, + APPROVED_EXECUTION_RESULT_UNKNOWN, + ); + }), + ); + } settleBufferedPausedCompletions(); run.settleOpenToolCalls( (id) => diff --git a/services/runner/src/engines/sandbox_agent/runtime-contracts.ts b/services/runner/src/engines/sandbox_agent/runtime-contracts.ts index 9b4c4bf28c..d32d446e2e 100644 --- a/services/runner/src/engines/sandbox_agent/runtime-contracts.ts +++ b/services/runner/src/engines/sandbox_agent/runtime-contracts.ts @@ -137,6 +137,17 @@ export interface ResumeApprovalInput { promptPromise?: Promise; } +/** + * An approved Pi call whose batched execution is still blocked by a sibling approval. The next + * resume seeds this context into its tracer and execution-grant ledger before Pi can emit the + * batch's real terminal frame. + */ +export interface ParkedApprovedExecution { + toolCallId: string; + toolName: string | undefined; + args: unknown; +} + /** Per-turn options for `runTurn`. Absent (flag off / cold) means today's byte-identical path. */ export interface RunTurnOptions { /** A live continuation: send only the new user text instead of the full cold transcript. */ @@ -240,6 +251,11 @@ export interface SessionEnvironment { * The multi-answer resume and the all-parkable park check read `parkedApprovals`, not this. */ parkedApproval?: ParkedApproval; + /** + * Approved Pi calls settled with the non-retry unknown-result sentinel while a sibling gate was + * parked. Consumed and re-seeded on the next live resume; empty outside that internal carry. + */ + parkedApprovedExecutions?: Map; /** * How many ACP permission gates resolved to pendingApproval THIS turn (reset at turn start). * Equals `parkedApprovals.size` when every gate carried a resumable tool-call id; a larger diff --git a/services/runner/tests/unit/session-keepalive-approval.test.ts b/services/runner/tests/unit/session-keepalive-approval.test.ts index 6df4834754..245c1ebfa9 100644 --- a/services/runner/tests/unit/session-keepalive-approval.test.ts +++ b/services/runner/tests/unit/session-keepalive-approval.test.ts @@ -1203,7 +1203,17 @@ interface FakeRun { }>; } -function pausableHarness(opts: { clientTool?: boolean } = {}) { +interface PiBatchCall { + permissionId: string; + toolCallId: string; + toolName: string; + args: unknown; + output: string; +} + +function pausableHarness( + opts: { clientTool?: boolean; piBatching?: PiBatchCall[] } = {}, +) { const calls = { permissionReplies: [] as Array<{ id: string; reply: string }>, runs: [] as FakeRun[], @@ -1218,6 +1228,49 @@ function pausableHarness(opts: { clientTool?: boolean } = {}) { onEvent: undefined as ((event: any) => void) | undefined, onPermissionRequest: undefined as ((req: any) => void) | undefined, }; + const answeredPiBatchPermissions = new Set(); + const emittedPiBatchPermissions = new Set(); + const emitPiBatchGate = (call: PiBatchCall): void => { + if (emittedPiBatchPermissions.has(call.permissionId)) return; + emittedPiBatchPermissions.add(call.permissionId); + captured.onEvent?.({ + payload: { + update: { + sessionUpdate: "tool_call", + toolCallId: call.toolCallId, + title: call.toolName, + rawInput: call.args, + }, + }, + }); + captured.onPermissionRequest?.({ + id: call.permissionId, + availableReplies: ["once", "reject"], + toolCall: { + toolCallId: call.toolCallId, + name: call.toolName, + rawInput: call.args, + }, + }); + }; + const emitPiBatchResults = (): void => { + for (const call of opts.piBatching ?? []) { + captured.onEvent?.({ + payload: { + update: { + sessionUpdate: "tool_call_update", + toolCallId: call.toolCallId, + status: "completed", + content: call.output, + }, + }, + }); + } + calls.resolvePrompt?.({ + stopReason: "complete", + usage: { inputTokens: 1, outputTokens: 1 }, + }); + }; const session = { id: "session-1", @@ -1229,6 +1282,17 @@ function pausableHarness(opts: { clientTool?: boolean } = {}) { }, async respondPermission(id: string, reply: string) { calls.permissionReplies.push({ id, reply }); + const batch = opts.piBatching; + if (!batch || reply !== "once") return; + const index = batch.findIndex((call) => call.permissionId === id); + if (index < 0) return; + answeredPiBatchPermissions.add(id); + // Pi resolves every approval hook before executing any call in the parallel batch. + if (index + 1 < batch.length) { + queueMicrotask(() => emitPiBatchGate(batch[index + 1])); + } else if (answeredPiBatchPermissions.size === batch.length) { + queueMicrotask(emitPiBatchResults); + } }, prompt(_blocks: any) { calls.promptCount += 1; @@ -1236,6 +1300,10 @@ function pausableHarness(opts: { clientTool?: boolean } = {}) { // it — modelling the ORIGINAL prompt continuing after the parked gate is answered. return new Promise((resolve) => { calls.resolvePrompt = resolve; + const firstPiBatchCall = opts.piBatching?.[0]; + if (firstPiBatchCall) { + queueMicrotask(() => emitPiBatchGate(firstPiBatchCall)); + } }); }, }; @@ -2223,6 +2291,196 @@ describe("runTurn: real approval park + respondPermission resume", () => { await env.destroy(); }); + it("re-parks a Pi batch without a closure wait and records real results after the final approval", async () => { + const batch: PiBatchCall[] = [ + { + permissionId: "permission-a", + toolCallId: "tool-a", + toolName: "tool-a", + args: { target: "a" }, + output: "tool-a real output", + }, + { + permissionId: "permission-b", + toolCallId: "tool-b", + toolName: "tool-b", + args: { target: "b" }, + output: "tool-b real output", + }, + ]; + const { deps } = pausableHarness({ piBatching: batch }); + deps.createOtel = createSandboxAgentOtel as any; + const closureWaitMs = 314_159; + deps.resolveRunLimits = () => ({ + totalMs: 1_000_000, + idleMs: 500_000, + ttfbMs: 500_000, + toolCallMs: closureWaitMs, + }); + deps.createRunLimits = () => ({ + onTrip() {}, + noteToolCallStart() {}, + noteToolCallEnd() {}, + wrapEmit: (emit: (event: any) => void) => emit, + notePaused() {}, + dispose() {}, + }); + const realSetTimeout = globalThis.setTimeout; + let closureWaitCount = 0; + const timeoutSpy = vi.spyOn(globalThis, "setTimeout").mockImplementation( + (( + handler: (...args: any[]) => void, + timeout?: number, + ...args: any[] + ) => { + if (timeout === closureWaitMs) { + closureWaitCount += 1; + return realSetTimeout(handler, 0, ...args); + } + return realSetTimeout(handler, timeout, ...args); + }) as typeof setTimeout, + ); + let env: SessionEnvironment | undefined; + + try { + const piRequest: AgentRunRequest = { + ...engineReq, + harness: "pi_agenta", + permissions: { default: "ask" }, + customTools: batch.map((call) => ({ + name: call.toolName, + permission: "ask", + })), + messages: [{ role: "user", content: "run both writes in parallel" }], + }; + const acquired = await acquireEnvironment(piRequest, deps); + assert.equal(acquired.ok, true); + if (!acquired.ok) return; + env = acquired.env; + + const firstResult = await runTurn( + env, + piRequest, + undefined, + undefined, + { approvalParkMode: true }, + ); + assert.equal(firstResult.stopReason, "paused"); + const gateA = env.parkedApprovals.get("tool-a")!; + + env.clearTurn(); + const secondResult = await runTurn( + env, + piRequest, + undefined, + undefined, + { + approvalParkMode: true, + resume: { + decisions: [ + { + permissionId: gateA.permissionId, + reply: "once", + toolCallId: gateA.toolCallId, + toolName: gateA.toolName, + args: gateA.args, + interactionToken: gateA.interactionToken, + promptPromise: gateA.promptPromise, + }, + ], + carriedForward: [], + }, + }, + ); + assert.equal(secondResult.stopReason, "paused"); + assert.equal( + closureWaitCount, + 0, + "the pending Pi sibling must bypass the bounded closure wait", + ); + assert.deepEqual( + (secondResult.events ?? []).filter( + (event) => event.type === "tool_result", + ), + [ + { + type: "tool_result", + id: "tool-a", + output: APPROVED_EXECUTION_RESULT_UNKNOWN, + isError: true, + }, + ], + ); + assert.deepEqual( + [...(env.parkedApprovedExecutions?.keys() ?? [])], + ["tool-a"], + ); + const gateB = env.parkedApprovals.get("tool-b")!; + + env.clearTurn(); + const thirdResult = await runTurn( + env, + piRequest, + undefined, + undefined, + { + approvalParkMode: true, + resume: { + decisions: [ + { + permissionId: gateB.permissionId, + reply: "once", + toolCallId: gateB.toolCallId, + toolName: gateB.toolName, + args: gateB.args, + interactionToken: gateB.interactionToken, + promptPromise: gateB.promptPromise, + }, + ], + carriedForward: [], + }, + }, + ); + assert.equal(thirdResult.stopReason, "complete"); + + const eventLog = [ + ...(firstResult.events ?? []), + ...(secondResult.events ?? []), + ...(thirdResult.events ?? []), + ]; + const realResults = eventLog.filter( + ( + event, + ): event is Extract => + event.type === "tool_result" && + (event.output === "tool-a real output" || + event.output === "tool-b real output"), + ); + assert.deepEqual( + realResults.map((event) => event.id).sort(), + ["tool-a", "tool-b"], + "both batched calls record one real result", + ); + const lastResultByCall = new Map< + string, + Extract + >(); + for (const event of eventLog) { + if (event.type === "tool_result" && event.id) { + lastResultByCall.set(event.id, event); + } + } + assert.equal(lastResultByCall.get("tool-a")?.output, "tool-a real output"); + assert.equal(lastResultByCall.get("tool-a")?.isError, false); + assert.equal(lastResultByCall.get("tool-b")?.output, "tool-b real output"); + assert.equal(lastResultByCall.get("tool-b")?.isError, false); + assert.equal(env.parkedApprovedExecutions?.size, 0); + } finally { + timeoutSpy.mockRestore(); + if (env) await env.destroy(); + } + }); + it("records the non-retry sentinel when an approved result misses the bound", async () => { const { calls, deps, captured } = pausableHarness(); deps.resolveRunLimits = () => ({ From 7ea9d6f5a250e76bc45e07d8171ccd7c73b7ce55 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Sun, 19 Jul 2026 23:31:07 +0200 Subject: [PATCH 11/15] fix(web): dispatch re-parked approvals and reopen sentinel-sealed cards on rebuild --- .../assets/transcriptToMessages.test.ts | 65 +++++++++++++++---- .../assets/transcriptToMessages.ts | 25 ++++++- .../components/ToolActivity.tsx | 31 +++++---- .../state/execution/agentApprovalResume.ts | 17 ++--- .../tests/unit/agentApprovalResume.test.ts | 41 ++++++++++++ 5 files changed, 145 insertions(+), 34 deletions(-) diff --git a/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.test.ts b/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.test.ts index 0cbbd129b4..840b0a2da5 100644 --- a/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.test.ts +++ b/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.test.ts @@ -1,7 +1,7 @@ import type {SessionRecord} from "@agenta/entities/session" import {describe, expect, it} from "vitest" -import {transcriptToMessages} from "./transcriptToMessages" +import {APPROVED_EXECUTION_RESULT_UNKNOWN, transcriptToMessages} from "./transcriptToMessages" const record = (id: string, payload: Record, sender = "agent"): SessionRecord => ({ id, @@ -99,7 +99,7 @@ describe("transcriptToMessages approval hydration", () => { expect(part.approval).toEqual({id: "approval-1", approved: false}) }) - it("rebuilds the persisted incident order with call a executed and call b pending", () => { + it("reopens deferred call b when its turn-2 approval request arrives", () => { const messages = transcriptToMessages([ record("record-user", {type: "message", text: "run both writes"}, "user"), record("record-call-a", { @@ -108,6 +108,12 @@ describe("transcriptToMessages approval hydration", () => { name: "bash", input: {command: "write a"}, }), + record("record-call-b", { + type: "tool_call", + id: "tool-b", + name: "bash", + input: {command: "write b"}, + }), record("record-request-a", { type: "interaction_request", id: "approval-a", @@ -121,18 +127,13 @@ describe("transcriptToMessages approval hydration", () => { isError: true, }), record("record-done-turn-1", {type: "done"}), + record("record-user-turn-2", {type: "message", text: "run both writes"}, "user"), record("record-response-a", { type: "interaction_response", id: "approval-a", kind: "user_approval", payload: {toolCallId: "tool-a", approved: true}, }), - record("record-result-a", { - type: "tool_result", - id: "tool-a", - output: "tool-a real output", - isError: false, - }), record("record-request-b", { type: "interaction_request", id: "approval-b", @@ -146,6 +147,12 @@ describe("transcriptToMessages approval hydration", () => { }, }, }), + record("record-result-a", { + type: "tool_result", + id: "tool-a", + output: APPROVED_EXECUTION_RESULT_UNKNOWN, + isError: true, + }), record("record-done-turn-2", {type: "done"}), ]) @@ -161,15 +168,51 @@ describe("transcriptToMessages approval hydration", () => { const callB = assistantParts.find((part) => part.toolCallId === "tool-b") expect(callA).toMatchObject({ - state: "output-available", - output: "tool-a real output", + state: "output-error", + errorText: APPROVED_EXECUTION_RESULT_UNKNOWN, + approval: {id: "approval-a", approved: true}, }) - expect(callB).toMatchObject({ + expect(callB).toEqual({ + type: "tool-bash", + toolCallId: "tool-b", state: "approval-requested", + input: {command: "write b"}, approval: {id: "approval-b"}, }) expect(assistantParts.filter((part) => part.state === "approval-requested")).toEqual([ callB, ]) }) + + it("keeps a real tool error closed when a late approval request arrives", () => { + const part = firstPart([ + record("record-call-b", { + type: "tool_call", + id: "tool-b", + name: "bash", + input: {command: "write b"}, + }), + record("record-result-b", { + type: "tool_result", + id: "tool-b", + output: "permission denied", + isError: true, + }), + record("record-done-turn-1", {type: "done"}), + record("record-request-b", { + type: "interaction_request", + id: "approval-b", + kind: "user_approval", + payload: {toolCallId: "tool-b"}, + }), + ]) + + expect(part).toEqual({ + type: "tool-bash", + toolCallId: "tool-b", + state: "output-error", + input: {command: "write b"}, + errorText: "permission denied", + }) + }) }) diff --git a/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.ts b/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.ts index 203ed7a81b..6483f57397 100644 --- a/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.ts +++ b/web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.ts @@ -18,6 +18,15 @@ import type {UIMessage} from "ai" type Part = Record +// Mirrors services/runner/src/tracing/otel.ts; park sentinels report skipped or unobserved work, not final results. +export const DEFERRED_NOT_EXECUTED_PREFIX = "DEFERRED_NOT_EXECUTED" +export const APPROVED_EXECUTION_RESULT_UNKNOWN = + "APPROVED_EXECUTION_RESULT_UNKNOWN: the approved call started but its result was not observed before the pause ended the turn; do not assume it failed and do not retry a side-effecting call." +export const APPROVED_EXECUTION_RESULT_UNKNOWN_PREFIX = APPROVED_EXECUTION_RESULT_UNKNOWN.slice( + 0, + APPROVED_EXECUTION_RESULT_UNKNOWN.indexOf(":"), +) + interface DraftMessage { id: string role: "user" | "assistant" @@ -81,6 +90,14 @@ const newDraft = (id: string, role: "user" | "assistant"): DraftMessage => ({ const toolPartType = (name?: string | null): string => (name ? `tool-${name}` : "dynamic-tool") +const isRunnerSentinelError = (part: Part): boolean => { + const errorText = typeof part.errorText === "string" ? part.errorText : "" + return ( + errorText.startsWith(DEFERRED_NOT_EXECUTED_PREFIX) || + errorText === APPROVED_EXECUTION_RESULT_UNKNOWN + ) +} + /** Apply one transcript event's payload onto the current assistant/user draft message. */ function applyEvent( draft: DraftMessage, @@ -174,8 +191,12 @@ function applyEvent( index.tools.set(toolCallId, part) } index.approvals.set(str(payload.id), part) - // Only park if still unsettled — a later `tool_result` overwrites this. - if (part.state === "input-available") { + const canRequestApproval = + part.state === "input-available" || + (part.state === "output-error" && isRunnerSentinelError(part)) + if (canRequestApproval) { + delete part.errorText + delete part.output part.state = "approval-requested" part.approval = {id: str(payload.id)} } diff --git a/web/oss/src/components/AgentChatSlice/components/ToolActivity.tsx b/web/oss/src/components/AgentChatSlice/components/ToolActivity.tsx index 057eb75712..d9675f88bc 100644 --- a/web/oss/src/components/AgentChatSlice/components/ToolActivity.tsx +++ b/web/oss/src/components/AgentChatSlice/components/ToolActivity.tsx @@ -21,6 +21,10 @@ import {DriveFileCard} from "@/oss/components/Drives/DriveFileCard" import {partToolName, resolveToolDisplay, type ToolDisplay} from "../assets/toolDisplay" import {formatToolValue, stripFence} from "../assets/toolFormat" +import { + APPROVED_EXECUTION_RESULT_UNKNOWN_PREFIX, + DEFERRED_NOT_EXECUTED_PREFIX, +} from "../assets/transcriptToMessages" import { expandedValueAtomFamily, setExpandedAtom, @@ -35,9 +39,12 @@ const {Text} = Typography const SETTLED = new Set(["output-available", "output-error", "output-denied"]) const isSettled = (state: string) => SETTLED.has(state) -const DEFERRED_PREFIX = "DEFERRED_NOT_EXECUTED:" const isDeferredError = (errorText: string | undefined): boolean => - !!errorText && errorText.startsWith(DEFERRED_PREFIX) + !!errorText && errorText.startsWith(DEFERRED_NOT_EXECUTED_PREFIX) +const isUnknownResultError = (errorText: string | undefined): boolean => + !!errorText && errorText.startsWith(APPROVED_EXECUTION_RESULT_UNKNOWN_PREFIX) +const isNonFinalRunnerError = (errorText: string | undefined): boolean => + isDeferredError(errorText) || isUnknownResultError(errorText) const isNotHandledOutput = (output: unknown): boolean => !!output && @@ -85,7 +92,9 @@ const rowSummary = (part: ToolUIPart, display?: ToolDisplay): string | null => { } if (part.state === "output-error") { const errorText = (part as {errorText?: string}).errorText - return isDeferredError(errorText) ? "waiting on another approval" : "failed" + if (isDeferredError(errorText)) return "waiting on another approval" + if (isUnknownResultError(errorText)) return "approved, result unknown" + return "failed" } if (part.state === "output-denied") return "denied" return null @@ -100,7 +109,7 @@ const StatusIcon = ({part}: {part: ToolUIPart}) => { return } if (state === "output-error") { - if (isDeferredError((part as {errorText?: string}).errorText)) + if (isNonFinalRunnerError((part as {errorText?: string}).errorText)) return return } @@ -153,7 +162,7 @@ const ToolRow = ({ const input = (part as {input?: unknown}).input const output = (part as {output?: unknown}).output const errorText = (part as {errorText?: string}).errorText - const deferred = state === "output-error" && isDeferredError(errorText) + const nonFinalError = state === "output-error" && isNonFinalRunnerError(errorText) const notHandled = state === "output-available" && isNotHandledOutput(output) // `approval-responded` is resolved (the user answered) — not "running". Its execution shows on // a sibling part, so this must not spin forever (the cold-replay lingering-gate spinner). @@ -169,8 +178,8 @@ const ToolRow = ({ : live && running ? "running…" : detailed - ? deferred - ? "waiting on another approval" + ? nonFinalError + ? rowSummary(part, display) : state === "output-error" ? "failed" : state === "output-denied" @@ -207,7 +216,7 @@ const ToolRow = ({ ) : null} {midText ? ( @@ -245,9 +254,9 @@ const ToolRow = ({ {hasInput ? : null} {hasError ? ( ) : hasOutput ? ( @@ -351,7 +360,7 @@ const ToolActivity = ({ const failed = parts.filter( (p) => (p.state as string) === "output-error" && - !isDeferredError((p as {errorText?: string}).errorText), + !isNonFinalRunnerError((p as {errorText?: string}).errorText), ).length const count = parts.length const single = count === 1 ? resolveToolDisplay(partToolName(parts[0])) : null diff --git a/web/packages/agenta-playground/src/state/execution/agentApprovalResume.ts b/web/packages/agenta-playground/src/state/execution/agentApprovalResume.ts index 63d5128eed..dd648a3830 100644 --- a/web/packages/agenta-playground/src/state/execution/agentApprovalResume.ts +++ b/web/packages/agenta-playground/src/state/execution/agentApprovalResume.ts @@ -172,16 +172,13 @@ export function agentShouldResumeAfterApproval({ } if (lastResolvedIdx === -1) return false - // ALREADY RESUMED guard (the post-resolve loop). The cold-replay runner re-issues the approved - // tool under a FRESH id, so its execution output attaches to a NEW part and the original - // `approval-responded` part LINGERS in this same assistant message forever. Once the model has - // continued past the approval, a new step begins — a `step-start` part appears AFTER it. Without - // this guard the predicate keeps seeing the lingering `approval-responded` and auto-resends after - // every completion, re-running the whole turn endlessly (the loop the HITL fix exposed). - const resumedAlready = parts - .slice(lastResolvedIdx + 1) - .some((part) => part.type === "step-start") - if (resumedAlready) return false + if (!liveInteraction) { + // Restored tails need this guard; exact live markers are single-use and may target a part before `step-start`. + const resumedAlready = parts + .slice(lastResolvedIdx + 1) + .some((part) => part.type === "step-start") + if (resumedAlready) return false + } // The AI SDK re-evaluates after message updates and waits for an in-flight stream to finish, // so an approval clicked during a resume dispatches when that stream finishes. diff --git a/web/packages/agenta-playground/tests/unit/agentApprovalResume.test.ts b/web/packages/agenta-playground/tests/unit/agentApprovalResume.test.ts index 231e8ea585..fd71915e3a 100644 --- a/web/packages/agenta-playground/tests/unit/agentApprovalResume.test.ts +++ b/web/packages/agenta-playground/tests/unit/agentApprovalResume.test.ts @@ -43,6 +43,30 @@ const assistantWithClientTool = (state: string, output?: unknown) => ({ ], }) +const warmResumedSecondApproval = () => ({ + id: "a1", + role: "assistant", + parts: [ + {type: "step-start"}, + { + type: "tool-bash", + toolCallId: "call_1", + state: "output-error", + input: {command: "command a"}, + errorText: "APPROVED_EXECUTION_RESULT_UNKNOWN: result was not observed", + approval: {id: "perm_1", approved: true}, + }, + { + type: "tool-bash", + toolCallId: "call_2", + state: "approval-responded", + input: {command: "command b"}, + approval: {id: "perm_2", approved: true}, + }, + {type: "step-start"}, + ], +}) + describe("agentShouldResumeAfterApproval", () => { it("RESUMES on a deny-only decision (the F-036 dead-end fix)", () => { const messages = [user("do it"), assistantWithTool("approval-responded", false)] @@ -434,6 +458,23 @@ describe("agentShouldResumeAfterApproval", () => { expect(agentShouldResumeAfterApproval({messages})).toBe(false) }) + it("dispatches a live second-card answer before a warm-resume step tail", () => { + const messages = [user("run two commands"), warmResumedSecondApproval()] + + expect( + agentShouldResumeAfterApproval({ + messages, + liveInteraction: {kind: "approval", id: "perm_2"}, + }), + ).toBe(true) + }) + + it("keeps the same warm-resume step tail inert without a live marker", () => { + const messages = [user("run two commands"), warmResumedSecondApproval()] + + expect(agentShouldResumeAfterApproval({messages})).toBe(false) + }) + it("STILL resumes on a chained second approval later in the turn", () => { // Two gates in one turn: the first was approved-and-resumed (step-start follows it), then a // SECOND gate was just approved and is the tail — its approval still needs the resume. From 427387b3bcc2c2cc3b893f323bb355550a7205bf Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Tue, 21 Jul 2026 15:32:55 +0200 Subject: [PATCH 12/15] docs(sessions): takeover architecture, reconciliation, and Arda handoff --- .../sessions-takeover/architecture.md | 1159 +++++++++++++++++ .../arda-branch-reconciliation.md | 362 +++++ .../sessions-takeover/arda-handoff.md | 261 ++++ 3 files changed, 1782 insertions(+) create mode 100644 docs/design/agent-workflows/projects/sessions-takeover/architecture.md create mode 100644 docs/design/agent-workflows/projects/sessions-takeover/arda-branch-reconciliation.md create mode 100644 docs/design/agent-workflows/projects/sessions-takeover/arda-handoff.md diff --git a/docs/design/agent-workflows/projects/sessions-takeover/architecture.md b/docs/design/agent-workflows/projects/sessions-takeover/architecture.md new file mode 100644 index 0000000000..f7fc2828bc --- /dev/null +++ b/docs/design/agent-workflows/projects/sessions-takeover/architecture.md @@ -0,0 +1,1159 @@ +# Session storage architecture + +This document describes every storage plane an agent session touches, at working depth, +verified against the code and against real rows in the EE dev database. It is written for +a senior engineer who does not live in this codebase. Every claim carries a file and line +reference. Where the in-flight work (PRs #5375 backend, #5376 runner, #5382 approvals) +differs from `origin/main`, the difference is called out explicitly; all file:line +references are to the current workspace tree, which has those lanes applied. + +Two wrong mental models circulated recently and are worth killing on page one: + +1. **"The `session_turns` table stores the conversation."** It does not. The conversation + content lives in two places: the harness's own native session file (a JSONL file on the + session's durable filesystem) and the `records` table (our own append-style event log, + in the tracing database). `session_turns` is a small ledger of turn metadata whose main + job is to remember which harness-native session id belongs to our session so the next + turn can resume natively. +2. **"The streams plane stores the streamed frames."** It does not. Nothing in + `session_streams` or Redis holds a single token of output. The runner persists every + event to the records plane promptly, whether or not anyone is watching. The streams + plane is liveness and ownership: who is running this session right now, on which + machine, and has anyone asked it to stop. + +## Vocabulary you need before anything else + +A **session** is one conversation between a user and an agent. It is identified by a +`session_id`, a caller-supplied string (in practice a UUID minted by the frontend), +validated against `^[a-zA-Z0-9_\-]{1,128}$` (`api/oss/src/dbs/redis/sessions/contract.py:122`). +It is deliberately not a foreign key anywhere: sessions may originate outside our database, +so every table stores it as a bare string correlator scoped by `project_id` +(`api/oss/src/dbs/postgres/sessions/turns/dbas.py:18-19`). + +A **turn** is one user-message-to-agent-reply exchange. Two different identifiers both get +called "turn" and confusing them causes real bugs: + +- The **turn id** (`turn_id`) is a UUID minted once per *execution*. It is a lock value + and correlator, not a database primary key. The runner mints it when a session-owned run + starts (`services/runner/src/server.ts:175-177`); the API mints its own in `_start_turn` + when a run is started through the coordination endpoint + (`api/oss/src/core/sessions/streams/service.py:543`). Approval pauses end an execution, + so one conversation turn that pauses twice for approvals spans three turn ids. This id + rides heartbeats, record rows, and interaction rows. +- The **turn index** (`turn_index`) is a per-session counter of *completed conversation + turns*: 0, 1, 2. It is the ordering key of the `session_turns` ledger. Paused executions + do not consume an index (`services/runner/src/engines/sandbox_agent/run-turn.ts:98-102`). + +A **harness** is the coding-agent program the runner drives: Pi or Claude Code. The +enum is `HarnessKind` with values `pi_core`, `pi_agenta`, and `claude` +(`sdks/python/agenta/sdk/agents/dtos.py:45-53`). The runner talks to the harness over +**ACP**, the Agent Client Protocol, a JSON-RPC protocol in which the harness owns its own +conversation state and exposes `session/new` and `session/load` verbs. + +The **agent session id** (`agent_session_id`) is the harness's *own* native id for the +conversation. Claude Code generates its ids internally; they cannot be derived from +anything we hold, which is the whole reason a durable mapping table has to exist. Pi's id +appears in its transcript filename (see the mount listing later in this document). + +The **runner** is the Node sidecar (`services/runner/`) that executes turns. A **replica** +is one runner container, identified by `REPLICA_ID`, a uuid minted once per process or +injected via `AGENTA_RUNNER_REPLICA_ID` (`services/runner/src/sessions/alive.ts:31-32`). + +A **sandbox** is the isolated environment (local daemon or Daytona) where the harness +actually runs. A **mount** is a durable object-store prefix attached into that sandbox via +geesefs, a FUSE filesystem over S3. + +With that vocabulary, the six planes this document covers are: + +| Plane | Where it lives | One-line job | +|---|---|---| +| Records | `records` table, **tracing** database | The human-readable event transcript the UI rebuilds from | +| Turns | `session_turns` table, core database | The completed-turn ledger: harness session id mapping and per-turn metadata | +| Streams + alive | `session_streams` table plus Redis keys | Liveness, ownership, and the control-signal path | +| Interactions | `session_interactions` table, core database | The stateful approval plane (pending, resolved, verdicts) | +| Mounts | `mounts` table plus the object store | The durable filesystem: workspace files and harness session files | +| Keepalive pool | Runner process memory | Parked live harness processes with TTLs; not storage at all | + +## 1. The life of one turn + +This walk uses the real QA session `6c23ff5f-79b7-45e3-82c3-52758d0f5fbe` (run on the EE +dev stack on 2026-07-19; the user asked the agent to append a line to two files via two +parallel Bash commands, each gated on human approval). Every step below left a row or log +line quoted later in this document. + +**Before the turn.** The user types a message in the agent chat. The frontend holds the +session list and the conversation locally (localStorage, +`web/oss/src/components/AgentChatSlice/state/sessions.ts:76-111`) and sends the message +through the normal workflow invoke path (`/invoke` via the Python agent service). The +invoke request carries `session_id`; it does not carry a turn id, so the runner will mint +one (`services/runner/src/server.ts:161-177`). + +**Turn start, coordination plane.** The runner sees the request is session-owned (it has a +`sessionId`) and starts the alive watchdog before running anything +(`services/runner/src/server.ts:956-978`). The first heartbeat, `POST +/sessions/streams/heartbeat`, does three things in one round trip +(`api/oss/src/core/sessions/streams/service.py:258-383`): + +- claims the `owner::session:` Redis key for this replica (affinity, never + stolen from a live owner), +- establishes the `alive` and `running` Redis locks under this turn id, +- creates or updates the durable `session_streams` row and returns its uuid, the + `stream_id`, which the runner threads onto the request for later use + (`services/runner/src/server.ts:976-978`). + +**Turn start, interactions and records.** The runner cancels any prior turn's still-pending +approval gates (`services/runner/src/sessions/interactions.ts:137-160`), then persists the +inbound user message as the first record of the turn, with `record_source: "user"` +(`services/runner/src/server.ts:1003-1007`). + +**Environment acquire, mounts.** The runner asks the API to sign short-lived, prefix-scoped +credentials for the session's durable mounts (`POST /sessions/mounts/sign`, +`services/runner/src/engines/sandbox_agent/mount.ts:68-131`) and geesefs-mounts them so the +agent's working directory survives sandbox teardown. Pi's native transcript directory is +placed *inside* that working directory, so it rides the same mount +(`services/runner/src/engines/sandbox_agent/pi-assets.ts:37-50`). + +**Continuation decision.** The runner decides how the harness gets its memory of the +conversation, in this order: warm resume from the keepalive pool if a compatible live +process is parked (`services/runner/src/server.ts:585-662`), otherwise a cold environment +that attempts a native `session/load` using the `agent_session_id` from the turns ledger +(`services/runner/src/engines/sandbox_agent/environment.ts:927-1004`), otherwise a plain +`session/new` with the prior conversation replayed as text +(`services/runner/src/engines/sandbox_agent/run-turn.ts:146-148`). + +**During the turn.** Every ACP event the harness emits is forwarded to the live stream +*and* enqueued for durable persistence in produced order, whether or not the browser is +still connected (`services/runner/src/sessions/persist.ts:95-122`). Delta families are +coalesced (one `message` record instead of fifty `message_delta`s), and tool-family records +get deterministic uuid5 ids so retries upsert rather than duplicate +(`services/runner/src/sessions/persist.ts:160-329`, +`services/runner/src/sessions/record-id.ts:41-47`). The watchdog heartbeats every 30 +seconds; each response can carry `is_current_turn: false`, which means a cancel, steer, or +kill took this turn's lock since the last beat, and the runner aborts the in-flight run +(`services/runner/src/sessions/alive.ts:165-207`). + +**If a tool needs approval.** The harness raises an ACP permission request. The runner +creates a durable interaction row (`services/runner/src/engines/sandbox_agent/run-turn.ts:390-409`), +persists an `interaction_request` record, parks the live process in the keepalive pool in +state `awaiting_approval`, and ends the execution. The user's approval arrives as a fresh +run request carrying the decision; the runner checks the parked process out, answers the +harness in place (a warm resume), transitions the interaction row to `resolved` with the +verdict, and persists an `interaction_response` record +(`services/runner/src/engines/sandbox_agent/run-turn.ts:417-447`, +`services/runner/src/server.ts:664-744`). + +**Turn end.** On a *completed* execution (not paused, not failed) the runner records the +harness's native session id in its in-memory continuity store and appends one row to the +`session_turns` ledger, fire-and-forget: session id, stream id, turn index, harness kind, +agent session id, sandbox id, workflow references, trace id +(`services/runner/src/engines/sandbox_agent/run-turn.ts:832-866`). A paused or failed +execution instead *invalidates* the harness's continuity record, because the harness may +have written a partial turn into its native file and resuming it would replay history the +canonical request never sent (`run-turn.ts:867-870, 887-892`; +`services/runner/src/engines/sandbox_agent/environment.ts:1056-1070`). The persist chain is +drained so the last record lands before teardown (`services/runner/src/server.ts:1021`), +the watchdog releases with a final `is_running: false` heartbeat +(`services/runner/src/sessions/alive.ts:215-219`), and the environment is either parked +warm in the keepalive pool or destroyed. + +```mermaid +sequenceDiagram + participant U as User (browser) + participant W as Web app + participant A as API + participant R as Runner + participant H as Harness (Pi/Claude) + participant S as Sandbox + mount + + U->>W: types message + W->>A: POST /invoke (session_id, messages) + A->>R: /run (sessionId, no turnId) + Note over R: mints turn_id (server.ts:175) + R->>A: POST /sessions/streams/heartbeat + Note over A: Redis: owner + alive + running locks
DB: session_streams row (streams plane) + A-->>R: stream_id, is_current_turn + R->>A: POST /sessions/interactions/cancel-stale + R->>A: POST /sessions/records/ingest (user message) + Note over A: records plane (async via Redis stream) + R->>A: POST /sessions/mounts/sign + A-->>R: scoped S3 credentials (mounts plane) + R->>S: geesefs mount cwd (+ harness session dir) + R->>H: session/load(agent_session_id) or session/new + Note over H,S: harness reads/writes its native
session file on the mount + loop every event + H-->>R: ACP event + R-->>W: live stream frame + R->>A: POST /sessions/records/ingest (records plane) + end + opt approval gate + H-->>R: session/request_permission + R->>A: POST /sessions/interactions/ (interactions plane) + Note over R: park in keepalive pool (memory) + U->>W: approve + W->>A: /invoke (decision) + A->>R: /run (same sessionId) + R->>H: respondPermission (warm resume) + R->>A: interactions/transition (resolved + verdict) + end + H-->>R: turn complete (agent_session_id) + R->>A: POST /sessions/turns/ (turns plane, fire-and-forget) + R->>A: heartbeat is_running=false + Note over R: park warm or destroy +``` + +## 2. Records: the durable transcript + +**One sentence.** The records plane is the append-style event log of everything said and +done in a session, stored per event in the tracing database, and it is what the UI replays +when you open a session in a browser that never ran it. + +**The question it answers.** "The user opens session 6c23ff5f from a deep link three days +later. What does the chat render?" Answer: `POST /sessions/records/query` returns the +ordered rows, and `transcriptToMessages` folds them into the same UI messages a live stream +would have produced (`web/oss/src/components/AgentChatSlice/assets/loadSession.ts:20-36`, +`web/oss/src/components/AgentChatSlice/assets/transcriptToMessages.ts:4-17`). + +### Schema + +Table `records`, in the **tracing** database (`agenta_ee_tracing` on the dev stack), not +the core database. The DAO binds to the analytics engine +(`api/oss/src/dbs/postgres/sessions/records/dao.py:21-25`); the table was created by +tracing migration `oss000000002_add_records.py` and extended with turn and span columns by +`oss000000004_add_records_turn_span.py` (unmerged, PR #5375). + +Columns (`api/oss/src/dbs/postgres/sessions/records/dbas.py`, +`api/oss/src/dbs/postgres/sessions/records/dbes.py`): + +- `project_id`, `record_id`: the composite primary key. `record_id` is either a + producer-supplied deterministic uuid5 (tool-family events) or a backend-minted uuid4 + fallback; it is **not** time-ordered (`dbas.py:24-33`). +- `session_id`: bare string correlator. +- `record_index`: producer-stamped per-turn ordinal. It restarts at 0 on each execution, so + reads order by ingest time first and index second + (`dao.py:112`, comment at `dbas.py:41-46`). +- `timestamp`: producer (runner) event time, distinct from `created_at` (ingest time). +- `record_type`: the event type (`message`, `thought`, `tool_call`, `tool_result`, + `interaction_request`, `interaction_response`, `usage`, `done`, `error`). +- `record_source`: who authored it, `"agent"` or `"user"` + (`services/runner/src/sessions/persist.ts:18-19`). +- `attributes`: the full event payload as JSONB, truncated at 64 KB at the producer + boundary (`api/oss/src/core/sessions/records/streaming.py:22, 64-77`). +- `turn_id`, `span_id`: forward-fill-only bridge columns, added by the unmerged migration. + `turn_id` is the execution turn id (the lock value); `span_id` is the OTel span id of the + invoke span, a 16-hex string that is deliberately **not** a UUID + (`api/oss/src/core/shared/dtos.py:53-57`, `dbas.py:7-21`). Old rows stay null; the + tracing database is never backfilled. + +### Who writes, who reads, when + +The **runner** is the only producer. `buildPersistingEmitter` wraps the live emitter so +every event is both streamed and persisted, decoupled from client connection state +(`services/runner/src/sessions/persist.ts:160-329`). Writes go to +`POST /sessions/records/ingest` authenticated as the invoke caller; the route does not +write the database directly. It publishes to the Redis stream `streams:records` +(`api/oss/src/apis/fastapi/sessions/router.py:508-538`, +`api/oss/src/core/sessions/records/streaming.py:51-97`), and a separate worker consumes +batches, applies EE ingest quotas, and upserts rows +(`api/oss/src/tasks/asyncio/sessions/records_worker.py:19-160`). The write is therefore +asynchronous: a record is durable in Redis quickly and in Postgres within the worker's +batch delay. On a local stack with the worker not running, records silently never land, +which is a documented frontend fallback case (`loadSession.ts:16-18`). + +Ordering within a session is guaranteed by a per-session promise chain in the runner +(`persist.ts:34-35, 95-122`), three retries with backoff per event, and a drain before +teardown so the last event survives the race (`persist.ts:129-137`, +`server.ts:1021`). A persist failure after retries is logged and dropped. + +Readers: the frontend replay path (`querySessionRecords` via the Fern client, +`web/packages/agenta-entities/src/session/api/api.ts:56-77`) and the two records read +endpoints (`router.py:439-506`). + +### Lifecycle and mutability + +This is the subtlety the word "append-only" hides: the write is an **upsert**, not an +insert. `ON CONFLICT (project_id, record_id) DO UPDATE` replaces the body, timestamp, and +turn/span columns of an existing row (`dao.py:84-97`). For randomly-minted ids that never +matters. For tool-family records the id is deterministic: +`uuid5(session_id:tool_call_id:record_type)` (`record-id.ts:41-47`), which is what makes a +streamed snapshot of one tool call, or a resume that re-announces it, settle onto one row +instead of duplicating. The cost is that the id is **not turn-scoped**: a later result for +the same tool call id silently overwrites the earlier row, preserving nothing. Incident +db58551b demonstrated this rewriting history (defect 5, +`docs/design/agent-workflows/scratch/debug-concurrent-approvals-db58551b.md`). The fix +plan (scope stable ids by turn) is agreed but not implemented. Until then, treat records as +a faithful *rendering* log and an unfaithful *audit* log. + +Deletion: never deleted by the session delete fan-out. The root delete explicitly leaves +records alone because they live in another database; tracing retention owns them +(`api/oss/src/core/sessions/service.py:83-91`). + +### Real rows + +From `agenta_ee_tracing.records`, session 6c23ff5f (the QA session): + +``` +record_id | a1aed416-34c4-40ae-a2e1-71299944e802 +session_id | 6c23ff5f-79b7-45e3-82c3-52758d0f5fbe +record_index | 0 +timestamp | 2026-07-19 21:33:32.372+00 +record_type | message +record_source | user +turn_id | 0cf4b750-7ddb-45d1-89b1-c2ef24802823 +span_id | cc57875a43d73a25 +created_at | 2026-07-19 21:33:32.661106+00 +attributes | {"text": "Append the line \"hello from QA\" to agent-files/README.md and + to agent-files/NOTES.md, as two separate Bash commands issued in parallel + in the same turn.", "type": "message"} +``` + +And the answer half of a gate, the `interaction_response` record type added by the +approvals work (note the deterministic uuid5 record id, and that its `turn_id` +`1eea4b3d...` differs from the request's `0cf4b750...`: the answer landed on the warm +resume execution, not the execution that asked): + +``` +record_id | 526453ac-b565-5dd1-8199-6d11504fb15d +record_type | interaction_response +record_index | 2 +turn_id | 1eea4b3d-b810-4125-a65c-faffc242551a +attributes | {"id": "56efc8f2-1830-44f5-87c3-62d8ab598997", "kind": "user_approval", + "type": "interaction_response", + "payload": {"approved": true, "toolCallId": "toolu_016ZdXAopbBhuFDqKCV5Pyvv"}} +``` + +The full type distribution for this session: 7 message, 4 done, 3 thought, 2 each of +interaction_request, interaction_response, tool_call, tool_result, usage. + +### Intended design and unmerged parts + +The `turn_id` and `span_id` columns (unmerged migration `oss000000004`) exist to group a +session's records by turn and to bridge each turn to observability. The +`interaction_response` record type is on the open #5382 lane; `origin/main`'s protocol has +no answer event at all, which is the root cause of the "answered cards resurrect as +pending on reload" bug (defect 4 of the db58551b report). + +### Sharp edges + +- `api/oss/src/core/sessions/records/utils.py` contains `strip_replay` and + `coalesce_events`, ported from the PoC, with **no callers** anywhere in the API (verified + by repo-wide grep) and no producer of the `__replay__:` marker in the runner or SDK. The + replay-stripping invariant its docstring describes is currently enforced by nothing. +- On an approval resume, the recovered prior prompt is re-persisted as a fresh user + message row, duplicating it in the transcript (db58551b defect 5, still open). +- `record_index` is only meaningful within one execution; cross-turn ordering rides on + ingest time (`dao.py:112`), so a clock or worker-batching anomaly can interleave turns. + +## 3. Turns: the completed-turn ledger + +**One sentence.** `session_turns` is one row per *completed* conversation turn, storing no +conversation content: it maps our session id to the harness's native session id, records +which harness and sandbox served the turn, and carries per-turn metadata (turn index, +workflow references, trace id, stream id). + +**The question it answers.** "The runner restarted overnight and its memory is empty. The +user sends the next message in session 6c23ff5f. Can the harness resume natively, and with +which of its own session ids?" Answer: read the latest turn row; if this harness authored +the latest completed turn, `session/load` its `agent_session_id`; otherwise fall back to +text replay (`services/runner/src/engines/sandbox_agent/session-continuity-durable.ts:104-148`). + +### Why a ledger, and why it replaced the states blob + +On `origin/main` this job is done by `session_states`, a mutable one-row-per-session JSON +blob that the runner syncs with a GET-then-PUT read-modify-write +(`git show origin/main:services/runner/src/engines/sandbox_agent/session-continuity-durable.ts`, +module docstring). That shape has two structural problems: the read-modify-write races +with itself across executions, and a blob folds away history, so you cannot ask "which +sandbox served turn 3". The unmerged redesign (#5375/#5376) replaces it with an +append-only table where a write is a plain INSERT, no read, no merge, no race +(`session-continuity-durable.ts:151-163` in the workspace tree), and the *current* resume +pointer is a query, not a stored fold: `ORDER BY turn_index DESC LIMIT 1`, which a late +lower-index write can never win (`api/oss/src/core/sessions/turns/service.py:1-6, 69-79`). +The states table is physically dropped by migration +`oss000000017_drop_session_states.py`. + +Separately, the ledger is deliberately **not** the conversation and **not** the harness +file. The harness file is the only faithful representation of what the model saw (including +its own native caching and tool-call framing); duplicating it would drift. What the +platform needs is a trust decision ("is the native file a faithful resume point?") plus +lookup keys, and that is exactly what the ledger stores. + +### Schema + +Table `session_turns`, core database, created by unmerged migration +`oss000000014_add_session_turns.py`. Columns +(`api/oss/src/dbs/postgres/sessions/turns/dbas.py`, +`.../turns/dbes.py`): + +- `project_id`, `id`: composite primary key; `id` is a uuid7, so insertion-ordered. +- `session_id`: bare string, NOT NULL, indexed with project. +- `stream_id`: NOT NULL foreign key to `session_streams(project_id, id)`. Every turn is + written with the stream row id in hand because the first heartbeat returned it + (`dbes.py:26-30`, `services/runner/src/server.ts:976-978`). +- `turn_index`: NOT NULL, with a **unique** index on `(project_id, session_id, + turn_index)` (`dbes.py:37-43`). This index is what caught the warm-environment + double-append bug (see sharp edges). +- `harness_kind`: `pi_core` / `pi_agenta` / `claude`, enum-validated at the DTO. +- `agent_session_id`: the harness-native session id. Nullable, because a harness might not + surface one. +- `sandbox_id`: which sandbox served the turn, e.g. `local/127.0.0.1:43699` or a Daytona + id. Read by the sandbox-reconnect ladder. +- `references`: JSONB list of `{id, slug, version}` entity references (the workflow, + variant, and revision this turn ran), GIN-indexed with `jsonb_path_ops` for containment + queries (`dbes.py:44-49`). This is what the root session list joins on. +- `trace_id`, `span_id`: observability bridge. `trace_id` is a 128-bit OTel trace id + stored as UUID; `span_id` would be the 16-hex span id (`dbas.py:36-38`). +- `start_time`, `end_time`: reserved timing columns. + +### Who writes, who reads, when + +One writer: the runner, at the end of a completed execution only. +`run-turn.ts:835-866` is the gate, and reading it precisely matters: + +```ts +if ( + stopReason !== "paused" && + env.continuityTurnIndex !== undefined && + sessionId && + env.session?.agentSessionId +) { + ...store.record(...); + void (deps.appendSessionTurn ?? appendSessionTurn)(...) +``` + +So a row is written only when the turn ran to completion AND the harness surfaced a native +session id. A pause (`stopReason === "paused"`) or any error path instead calls +`invalidateContinuity` (`run-turn.ts:867-870, 887-892`), which drops the in-memory record +so the harness can never be judged load-eligible from a possibly-partial native file. The +absence of a ledger row for the latest turn is therefore *information*: it means the +native file is not trusted and the next cold start must replay text. + +The append is fire-and-forget (`void ... .catch(() => {})`, `run-turn.ts:853-865`): a +failed append never fails the turn, it only degrades a future restart to replay. + +Readers: + +- `hydrateHarnessSessionFromDurable` at environment acquire, which restores the + cross-harness latest-turn counter first and then this harness's own record, never + clobbering a live in-process record with a stale read + (`session-continuity-durable.ts:104-148`, + `environment.ts:937-948`). +- The sandbox-reconnect ladder, which reads the latest row's `sandbox_id`. +- The root session list: `POST /sessions/query` with `references` filters resolves + matching sessions *through the turns' GIN index* rather than denormalizing references + onto the stream row (`api/oss/src/core/sessions/service.py:41-74`). +- The turns API surface: `POST /sessions/turns/` (append), `POST /sessions/turns/query`, + `GET /sessions/turns/{turn_id}` (`api/oss/src/apis/fastapi/sessions/router.py:1065-1178`). + +### The continuation decision tree + +Eligibility is strict: a harness may `session/load` if and only if it authored the +conversation's most recently completed turn (`session-continuity.ts:114-130`). A harness +that is behind (the user switched from Pi to Claude and back) is stale and must replay, +because its native file is missing the intervening turns. + +```mermaid +flowchart TD + A["New turn for session S, harness H"] --> B{"Keepalive pool has an idle + compatible live process? + (server.ts 585-609)"} + B -- "yes, fingerprints match" --> W["WARM RESUME + same harness process, native memory + prompt = new text only"] + B -- "no / mismatch" --> C["Acquire cold environment"] + C --> D["Hydrate continuity store from + session_turns latest row + (environment.ts 940-948)"] + D --> E{"H authored the LATEST completed + turn AND agent_session_id known? + (session-continuity.ts 121-130)"} + E -- yes --> F["COLD NATIVE LOAD + session/load with agent_session_id + harness reads its own file off the mount + (environment.ts 963-990)"] + F -- "load verified" --> G["prompt = new text only + (run-turn.ts 147-148)"] + F -- "load failed" --> H["TRANSCRIPT REPLAY"] + E -- "no row / stale / invalidated" --> H + H --> I["session/new; prompt = prior conversation + rendered as capped text + new message + (run-turn.ts 148, transcript.ts)"] +``` + +### Real rows + +Both completed turns of QA session 6c23ff5f, from `agenta_ee_core.session_turns`. Note +what is present (the resume keys, the references, the trace id) and what is absent (any +conversation content, and `span_id`/`start_time`/`end_time`, see sharp edges): + +``` +id | 019f7c4d-44e6-77f2-9f7b-2409615f9092 +project_id | 019e8df5-2a58-7501-8fe2-56f7b332bd00 +session_id | 6c23ff5f-79b7-45e3-82c3-52758d0f5fbe +stream_id | 019f7c4c-713f-7a23-a764-14552c3b9f6a +turn_index | 0 +harness_kind | pi_core +agent_session_id | 019f7c4c-7a8f-7c80-bd0f-05458ab62fab +sandbox_id | local/127.0.0.1:32905 +references | [{"id": "019f5b6c-bfa9-7073-a8aa-203b215cea22", "slug": "new-agent-mrj6ykqaa2"}, + {"id": "019f5b6c-bfef-7fa0-b0cc-14b52b3b6734", "slug": "new-agent-mrj6ykqaa2.default"}] +trace_id | f06ec815-62d9-cde0-0092-f99cff2e1b09 +span_id | (null) +start_time | (null) +end_time | (null) +created_at | 2026-07-19 21:34:26.534971+00 + +id | 019f7c4f-1e80-7861-9fb4-84d204967da4 +turn_index | 1 +agent_session_id | 019f7c4c-7a8f-7c80-bd0f-05458ab62fab +sandbox_id | local/127.0.0.1:43699 +trace_id | 981b03a3-0618-2355-c618-44bed54988ee +created_at | 2026-07-19 21:36:27.776117+00 +``` + +Both turns carry the *same* `agent_session_id` (one continuous Pi conversation) but +*different* `sandbox_id`s: turn 1 ran on a different local daemon port after the first +environment was torn down, and native continuity still held because Pi's session file +lives on the durable mount, not in the sandbox. + +### Intended design and unmerged parts + +The entire plane is unmerged (PRs #5375/#5376). The two bugs found during QA on 2026-07-19 +are fixed on those lanes: `turn_index` is now computed per turn from the shared store, not +frozen per environment acquire (commit `cae71e49ce`, `run-turn.ts:98-102`; the original +failure is documented in +`docs/design/agent-workflows/scratch/debug-session-turns-append-500.md`), and a duplicate +append now surfaces as HTTP 409 via `EntityCreationConflict` instead of an anonymous 500 +(commit `494cdd7996`, `api/oss/src/dbs/postgres/sessions/turns/dao.py:48-68`). + +Deletion is a hard delete by session id, part of the delete fan-out +(`turns/dao.py:182-196`). There is no soft delete for turns. + +### Sharp edges + +- `span_id`, `start_time`, and `end_time` are wired through the whole stack (wire model, + DTO, column) but the runner's append call passes only `streamId`, `agentSessionId`, + `sandboxId`, `references`, and `traceId` (`run-turn.ts:857-863`), so those three columns + are null in practice, as the real rows show. +- The ledger row does not carry the execution `turn_id` (the lock value that records and + interactions are stamped with). There is currently **no direct join** from a records row + to its ledger row; the only shared key is `stream_id` plus time. Anything that wants + "the records of turn 2" has to go through the trace id or timestamps. +- Because rows exist only for completed turns, a session whose last execution paused has a + ledger that is one turn behind reality, by design. Consumers must treat "latest row" as + "latest *trusted* resume point", not "latest activity". + +## 4. Streams and the alive lock: liveness and ownership + +**One sentence.** The streams plane is one durable row per session plus a nest of Redis +locks, telling the platform whether the session is claimed, whether a turn is executing +right now, who is watching, and which runner replica owns it; it stores no output. + +**The question it answers.** "Two browser tabs send a message to session S at the same +moment. Who wins, and how does the loser find out?" Answer: the first `send` acquires the +`alive` lock in `_start_turn`; the second gets `SessionTurnInUse` and an HTTP 409 whose +body carries the liveness snapshot (`api/oss/src/core/sessions/streams/service.py:103-119, +536-561`, `router.py:146-153`). + +### The Redis nest + +The contract file is the canonical source of truth and the TypeScript side mirrors it +exactly, pinned by a golden fixture test +(`api/oss/src/dbs/redis/sessions/contract.py:1-30`, +`services/runner/src/sessions/contract.ts:1-8`). Every key is project-scoped because +`session_id` is caller-supplied and only unique per project; omitting the project segment +would let a caller in project A cancel project B's runs (`contract.py:13-19`). + +| Key | Value | TTL (default) | Meaning | +|---|---|---|---| +| `alive::session:` | turn_id | 3600 s | Session claimed; at most one in-flight run | +| `running::session:` | turn_id | 3600 s | A turn is actively executing right now | +| `attached::session:` | watcher_id | 60 s | A client is watching the live view | +| `owner::session:` | replica_id | 120 s | Which runner container owns this session | +| `displaced::session:` | (pub/sub) | n/a | Attach-steal notifications | + +TTL defaults live in `api/oss/src/utils/env.py:1286-1317` (`AGENTA_SESSIONS_REDIS_*`). +The nest invariant is `alive ⊇ running ⊇ attached`: attached implies running implies +alive. `resumable` (alive and not running) and `reattachable` (running and not attached) +are derived client-side, never stored (`streams/dtos.py:12-22`). Lock operations are +owner-checked Lua scripts (release-if-owner, claim-or-read; +`contract.py:87-106`, implementations in `api/oss/src/dbs/redis/sessions/locks.py`). +`claim_owner` never steals from a live different owner; kill uses the unconditional +`force_clear_owner` twin precisely because a surviving owner key would lock the dead +session out of every other replica for the rest of its TTL (`locks.py:262-318`, +`streams/service.py:227-230`). + +### The durable row + +Table `session_streams`, core database, merged on main (migration `oss000000008`); the +header columns are the unmerged migration `oss000000015`. One row per `(project_id, +session_id)` (`api/oss/src/dbs/postgres/sessions/streams/dbes.py:22-74`): + +- `id`: uuid7; this is the `stream_id` the turns ledger references. +- `session_id`: bare string, unique with project. +- `flags`: JSONB `{is_alive, is_running, is_attached}`, the durable **mirror** of the + Redis nest. Redis is authoritative; the row mirrors for durability, orphan sweeps, and + observability (`dbes.py:33-41`). The read path overlays the live Redis snapshot onto the + row before returning it (`streams/service.py:385-409`). +- `turn_id`: the current execution's turn id, the Postgres mirror of the lock value; null + when idle (`dbes.py:48-50`). The heartbeat handler also uses it to disambiguate "first + heartbeat of a new turn" from "the lock was taken by a cancel" + (`streams/service.py:287-303`). +- `name`, `description`: the session header, written only by the rename edit, never by + the heartbeat path, so liveness writes cannot churn identity fields and vice versa + (`streams/dtos.py:54-59`, `streams/service.py:422-464`). +- `updated_at` doubles as the heartbeat timestamp (`dbes.py:37`). +- `sandbox_id` is deliberately NOT here; it lives on the latest turns row (`dbes.py:40`). + +### The command matrix + +`POST /sessions/streams/` is a state mutation over the lock nest, and it runs nothing +itself; the runner is the only component that executes +(`streams/dtos.py:76-89`, `streams/service.py:1-13`). The mode is derived from the +request shape: + +| inputs | force | Mode | Effect (`streams/service.py:81-177`) | +|---|---|---|---| +| yes | no | `send` | 409 if alive; else mint turn_id, acquire alive+running, stamp row | +| yes | yes | `steer` | force-drop holder's locks, mint new turn | +| no | no | `cancel` | force-drop locks, mark row ended, run nothing | +| no | yes | `attach` | steal the attached lock, return a watcher_id | + +`kill` (`DELETE /sessions/streams/`) is distinct from cancel: cancel ends the current +turn and the session can resume; kill ends the session. It collapses the whole nest +including owner affinity, displaces any watcher, calls the runner's own `/kill` directly +so the sandbox is actually torn down rather than left to idle-TTL eviction, marks the row +ended, soft-deletes it, and cancels every pending interaction +(`streams/service.py:202-256`, +`api/oss/src/core/sessions/streams/runner_client.py:30-62`, +`router.py:296-324`). + +### The heartbeat and the control-signal path + +The runner heartbeats every 30 seconds while a session-owned turn runs +(`services/runner/src/sessions/alive.ts:21-23, 165-223`). Each beat carries two distinct +ids: `replica_id` (the container, drives owner affinity) and `turn_id` (the execution, +proves lock ownership) (`streams/dtos.py:99-103`). The handler claims ownership without +stealing; a replica that lost the claim mutates nothing and is told the true owner +(`streams/service.py:266-284`), which the runner uses to refuse serving a local sandbox +session on the wrong host (`session-continuity.ts:157-210`). + +The response carries `is_current_turn`. It is false when this turn's lock was gone or +reassigned at the moment of the beat, disambiguated against the row's recorded `turn_id` +so a turn's very first heartbeat is not misread as an interruption +(`streams/service.py:287-336`, dtos docstring `streams/dtos.py:112-127`). The runner wires +this to `controller.abort()`, firing at most once: this is the plumbing that makes a +cancel, steer, or kill actually reach an in-flight run, with a worst-case latency of one +heartbeat interval (`alive.ts:147-207`, `server.ts:962-974`). Before W7.4 landed on this +lane, a cancel raced against the heartbeat's nx re-acquire would silently re-arm the same +lock and the interruption never surfaced. + +The heartbeat response also carries the `session_streams` row id, which is how the runner +gets the `stream_id` for the turn append without an extra round trip +(`alive.ts:52-59`, `server.ts:976-978`). + +```mermaid +sequenceDiagram + participant R as Runner (replica REPLICA_ID) + participant A as API + participant X as Redis + participant P as Postgres session_streams + + loop every 30 s while turn runs + R->>A: heartbeat {session_id, replica_id, turn_id, is_running:true} + A->>X: claim_owner(replica_id) - never steals + alt lost the claim + A-->>R: {replica_id: other, is_current_turn:false} + Note over R: refuse local session on wrong host + else owner + A->>X: refresh alive+running (owner-checked) + Note over A: refresh failed AND row.turn_id == this turn
=> a cancel/steer/kill took the lock + A->>P: mirror flags + turn_id, updated_at = now + A-->>R: {stream, replica_id, is_current_turn} + opt is_current_turn == false + R->>R: controller.abort() - at most once + end + end + end + R->>A: final heartbeat is_running=false + A->>X: clear running (alive outlives the turn) +``` + +### Real row + +From `agenta_ee_core.session_streams`, the QA session: + +``` +id | 019f7c4c-713f-7a23-a764-14552c3b9f6a +project_id | 019e8df5-2a58-7501-8fe2-56f7b332bd00 +session_id | 6c23ff5f-79b7-45e3-82c3-52758d0f5fbe +turn_id | 0cf4b750-7ddb-45d1-89b1-c2ef24802823 +flags | {"is_alive": true, "is_running": true, "is_attached": false} +created_at | 2026-07-19 21:33:32.351413+00 +updated_at | (null) +name | (null) +description | (null) +``` + +Read this row with the mirror caveat in mind: it still says alive and running two days +after the session went idle, and `updated_at` is null even though the deployed runner +logged successful `running=false` heartbeats. The dev deployment predates part of the +in-flight lane, and the row is exactly what the design says it is, a best-effort mirror. +The Redis keys have long expired, so `fetch` (which overlays the live snapshot, +`streams/service.py:385-409`) reports the truth while the raw row does not. Never read +`flags` off the table directly. + +### Intended design and unmerged parts + +The command matrix, kill's direct runner hop, the header edit, `is_current_turn`, the +hard-delete and archive verbs, and the concurrency cap +(`check_runner_concurrency_limit`, 429 above `AGENTA_SESSIONS_REDIS_CONCURRENCY_LIMIT`, +default 1000 running streams per project, `streams/service.py:530-534`, +`router.py:264`) are all on the unmerged #5375 lane. The frontend deliberately does not +surface steer, cancel, or attach yet, because without runner cooperation beyond W7.4 they +would be lock edits with no user-visible effect +(`web/packages/agenta-entities/src/session/api/api.ts:276-280`). + +### Sharp edges + +- `HEARTBEAT_WRITE_THRESHOLD_SECONDS` exists in both contracts (`contract.py:37`, + `contract.ts:19`) but nothing consumes it; every heartbeat currently writes the row. +- The heartbeat is fail-open on network error (`alive.ts:57-59, 99-104`): a transient API + blip neither aborts a healthy run nor fabricates a stream id, but it also means a + partitioned runner keeps executing while its locks expire, and a `send` on another + replica could then start a second run of the same session. The owner guard only + protects local sandboxes (`session-continuity.ts:189-210`). +- `count_active` counts rows whose mirrored flags say running (`streams/dao.py:274-292`); + a stale mirror (see the real row above) inflates the concurrency count until an orphan + sweep or the next heartbeat corrects it. + +## 5. Interactions: the stateful approval plane + +**One sentence.** `session_interactions` holds one row per human-in-the-loop gate (an +approval request, a user-input request, or a client tool call), with a lifecycle status +and, since the current approvals lane, the stored verdict; it is the plane that knows +whether anyone still needs to answer something. + +**The question it answers.** "Which approvals in this project are still waiting on a +human, and when the user clicks approve on a webhook-delivered card, how does the platform +resume the right workflow?" Answer: query rows with `status = 'pending'` (there is a +partial index for exactly this, +`api/oss/src/dbs/postgres/sessions/interactions/dbes.py:41-45`), and the row's stored +workflow `references` let the respond endpoint re-invoke the same workflow revision with +the answer as inputs (`router.py:764-858`). + +### Schema + +Table `session_interactions`, core database, merged on main (migration `oss000000009`). +Columns (`interactions/dbas.py`, `interactions/dbes.py`): + +- `project_id`, `id`: composite primary key. +- `session_id`, `turn_id`: correlators; `turn_id` is the execution id, used by the + stale-gate sweep to spare the current turn's own gates. +- `token`: the gate's identity, unique per `(project_id, session_id, token)` + (`dbes.py:19-24`), which is what makes the runner's fire-and-forget ingest retry-safe. +- `kind`: `user_approval` | `user_input` | `client_tool` + (`interactions/dtos.py:10-13`). +- `status`: lifecycle only, **not** the verdict: `pending` (awaiting a reaction), + `responded` (answered via the interactions API plane), `resolved` (answered via the + messages plane, or the runner consumed the answer), `cancelled` (runner abandoned the + gate; no one is waiting) (`dtos.py:16-22`). +- `data`: JSON with `request` (tool name and args), `references` (pointers to the + workflow/variant/revision this turn was running, stored so respond can re-resolve the + live revision at invoke time, not a copy of the revision itself, + `services/runner/src/sessions/interactions.ts:19-25`), `selector`, and `resolution` + (the verdict: `{"verdict": "approved"|"denied", "tool_call_id": ...}`). +- `flags`: delivery bookkeeping `{delivered_in_band, delivered_webhook}`. + +### Lifecycle + +Transitions are guarded in the DAO: only `pending` and `responded` rows can transition; +`resolved` and `cancelled` are terminal (`interactions/dao.py:92-135`). The respond +endpoint does a compare-and-set flip to `responded` first, so of two concurrent +responders exactly one enqueues the resume invoke (`router.py:802-829`). Resolution +payloads are validated: only a `user_approval` may carry one (`router.py:643-668`). + +```mermaid +stateDiagram-v2 + [*] --> pending: runner ingests gate (run-turn.ts 390-409) + pending --> responded: API-plane answer, CAS (router.py 802-819) + pending --> resolved: in-band answer consumed by runner (interactions.ts 102-126) + responded --> resolved: runner consumes the stored answer + pending --> cancelled: stale-gate sweep (dao.py 137-160) or session kill (router.py 319-323) + resolved --> [*] + cancelled --> [*] + note right of resolved + verdict stored in data.resolution + only on the number-5382 lane + end note +``` + +### Who writes, who reads, when + +The **runner** creates a row whenever an execution pauses on a gate +(`run-turn.ts:390-409` calling `interactions.ts:57-95`), idempotently (unique token). It +transitions the row to `resolved`, now with the verdict, when it forwards a stored or +in-band decision to the harness (`run-turn.ts:417-447`, `interactions.ts:102-126`). At +every turn start it sweeps *other* turns' still-pending gates to `cancelled`, sparing +gates the incoming request answers in-band (`server.ts:979-988`, +`interactions.ts:128-160`, `dao.py:137-160`). The **API** writes on the respond path +(the CAS flip) and on kill (cancel everything pending). + +Readers: the interactions query/fetch endpoints (`router.py:711-762`), and any future +inbox UI. The frontend today rebuilds approval cards from *records*, not from this table, +which is why persisting the answer half as an `interaction_response` record mattered so +much (db58551b defect 4). + +### Real rows + +A resolved gate carrying the verdict (the new shape, from a 2026-07-20 session on the dev +stack): + +``` +id | 019f7f14-d642-78a2-baf3-ecf7773759bc +session_id | 782ff288-d47e-40ac-b277-f29d79c833d1 +token | 2b8cdd00-27f1-4fd0-9994-1d2af50ca3a3 +kind | user_approval +status | resolved +resolution | {"verdict": "approved", "tool_call_id": "call_P70SW0Y8VEBrfhT78JZpnQZi|fc_039f2633..."} +created_at | 2026-07-20 10:31:39.843001+00 +updated_at | 2026-07-20 10:32:01.213554+00 +``` + +And the incident session db58551b's first gate, the pre-verdict shape (status flipped, +`turn_id` null, no resolution stored; note the full request and references in `data`): + +``` +id | 019f79cb-62e1-7b40-96fe-207ab3381e87 +session_id | db58551b-f986-44ec-b939-d6b10b35717a +token | f6f8384d-51f4-4bba-9b78-ca299f45465c +kind | user_approval +status | resolved +data | {"request": {"tool": "Bash", "args": {"command": "printf '\nParallel write + test completed.\n' >> agent-files/README.md"}}, + "references": {"workflow": {...}, "workflow_variant": {...}, + "workflow_revision": {"version": "1", ...}}} +flags | {"delivered_in_band": true, "delivered_webhook": false} +created_at | 2026-07-19 09:53:20.097197+00 +updated_at | 2026-07-19 09:53:22.469477+00 +``` + +### Intended design and unmerged parts + +The verdict (`resolution`) and the `interaction_response` record are on the open #5382 +lane ("reliable multi-gate approvals"); on `origin/main` the schema field exists but no +producer writes it, which is exactly what the db58551b incident report identified as the +root defect. The `respond_task` seam in the router lets the resume invoke run on a taskiq +worker when wired, with an inline blocking invoke as the fallback +(`router.py:821-857`). + +### Sharp edges + +- The runner refuses to create the row when the run context carries no + `workflow_revision` reference (`run-turn.ts:399-400`), because respond could not + re-invoke anything. This is why QA session 6c23ff5f has interaction *records* but zero + `session_interactions` rows (verified against the dev database): its run context carried + workflow and variant references but no revision. Gates in that state are answerable + in-band only, invisible to any interactions-plane inbox. +- `turn_id` is nullable and was null on the older rows above; the stale-gate sweep's + "spare the current turn" logic only works when the runner stamps it. +- The row's `data.request` stores raw tool args. These are redacted by the runner's + redactor on the records plane but go to interactions verbatim; treat this table as + sensitive as the transcript. + +## 6. Mounts: the durable filesystem + +**One sentence.** A mount is a per-session object-store prefix (bucket key prefix +`mounts//`) that geesefs attaches into the sandbox as a real +directory, carrying both the agent's workspace files and the harness's native session +files, which is what makes native continuation survive sandbox teardown. + +**The question it answers.** "The sandbox that served turn 0 is gone. Turn 1 starts on a +brand new sandbox. Where does Pi find the conversation it is supposed to resume?" Answer: +in `agents/sessions/pi/` inside the session's `cwd` mount, which the new sandbox mounts +before the harness starts; the file's name carries the `agent_session_id` from the turns +ledger. + +### Storage model + +The `mounts` table (core database, migration `oss000000006`, plus session-slug scoping in +`oss000000011` and `agent_id` in the unmerged `oss000000016`) stores mount *identity*: +`project_id`, `id`, optional `session_id`, optional `agent_id`, a slug, and a name +(`api/oss/src/core/mounts/dtos.py:25-34`). File contents live in the object store +(SeaweedFS on the dev stack, any S3 on prod), under +`[namespace/]mounts///` +(`api/oss/src/core/mounts/service.py:145-155`). A session mount's slug is deterministic: +`__ag__session____` (`mounts/service.py:72-78`), so re-signing +re-attaches the same files. + +The sessions router exposes a session-scoped view over the same table (no storage of its +own, `api/oss/src/core/sessions/mounts/service.py:1-6`): list, query, upload, download, +and the important one, `POST /sessions/mounts/sign?session_id=...&name=...` +(`router.py:861-1062`). Sign get-or-creates the named mount for the session and returns +short-lived credentials scoped to that prefix, minted from the store's STS endpoint; the +master key never leaves the API (`api/oss/src/core/mounts/dtos.py:97-113`). The `name` +parameter defaults to `cwd`; any other name creates an additional session-scoped mount +with its own prefix, which is how per-harness transcript mounts work +(`router.py:996-1005`, `mount.ts:59-67`). + +### Where the harness session files actually live + +This is the detail that keeps getting misremembered, so here it is per harness and per +sandbox type (`services/runner/src/engines/sandbox_agent/mount.ts:133-179` and +`pi-assets.ts:31-50`): + +- **Pi, everywhere:** Pi's transcript directory is pointed *into the workspace* via + `PI_CODING_AGENT_SESSION_DIR = /agents/sessions/pi`. Since cwd IS the durable + `cwd` mount on both local and Daytona runs, Pi's native files ride that one mount and + no separate transcript mount is needed. The separate `pi-sessions` mount name exists + for the case where Pi's default home-relative dir is in use + (`mount.ts:174-177`) but the runner always overrides via the env var. +- **Claude, remote (Daytona) sandboxes:** Claude Code keeps transcripts under + `~/.claude/projects`. That directory gets its own mount, name `claude-projects`, + signed and geesefs-mounted inside the sandbox (`mount.ts:171-172, 724-761`). + Deliberately NOT `~/.claude` whole, which would sweep OAuth credentials into the + durable store (`mount.ts:162-164`); credentials are re-injected per run instead + (`mount.ts:136-139`). +- **Claude, local sandboxes:** none of the harness-dir mounting runs. `~/.claude` is the + runner container's own disk (`mount.ts:716-719`), which persists across turns only as + long as that container lives. A local Claude session's native continuity does not + survive a runner container rebuild even though the turns ledger row does. + +### Who writes, who reads, when + +The runner signs and mounts at environment acquire; the agent reads and writes through +the FUSE mount for the whole turn; teardown unmounts. The API reads and writes the store +directly for the browser file endpoints (upload/download, +`router.py:1025-1062`) and deletes prefixes on the session delete fan-out +(`api/oss/src/core/sessions/service.py:100-103`). Archive and unarchive soft-toggle the +mount rows alongside the stream row (`service.py:109-146`). Sign failure is never fatal: +the run proceeds without the mount, degraded (`mount.ts:64-67, 84-90, 749-756`). + +### Real listing + +The QA session's single mount row, then the actual store listing showing the harness +session file sitting next to the workspace files. The Pi JSONL filename embeds the same +`agent_session_id` (`019f7c4c-7a8f...`) the turns ledger rows carry, which is the whole +mapping made visible: + +``` +id | 019f7c4c-7166-76f1-8477-37bcc3cede5d +project_id | 019e8df5-2a58-7501-8fe2-56f7b332bd00 +session_id | 6c23ff5f-79b7-45e3-82c3-52758d0f5fbe +agent_id | +slug | __ag__session__de851259-3492-5fab-a416-f249737f7d4d__cwd +name | cwd +created_at | 2026-07-19 21:33:32.390727+00 +``` + +``` +/buckets/agenta-store/mounts/019e8df5.../019f7c4c-7166-76f1-8477-37bcc3cede5d +├── AGENTS.md +├── agent-files/ (README.md, NOTES.md - the files the QA turn appended to) +└── agents + ├── sessions + │ └── pi + │ └── 2026-07-19T21-33-34-736Z_019f7c4c-7a8f-7c80-bd0f-05458ab62fab.jsonl + └── skills + └── 07b95dee.../ (.agenta-skill-set.json, build-an-agent/SKILL.md, references/...) +``` + +### Intended design and unmerged parts + +`agent_id` on the mount row (unmerged migration `oss000000016`) is the beginning of +agent-scoped (as opposed to session-scoped) durable directories, the agent-mount work +tracked in PR #5247. Direct geesefs write-through is the accepted default for harness +transcript dirs; copy-around-lifecycle is the documented fallback only if append-heavy +JSONL write amplification bites (`mount.ts:141-146`). + +### Sharp edges + +- The signed credentials expire in minutes; a parked keepalive session whose mount + credentials pass expiry is evicted to cold on resume rather than continued + (`server.ts:722-724`). +- Deleting a session deletes mount rows and prefixes, which includes the harness session + files, so native continuity is correctly destroyed with the session. But local Claude + files on the runner's own disk are outside that fan-out entirely. +- geesefs over an in-network store needs the tunnel endpoint for remote sandboxes + (`mount.ts:186-204`); a mis-detected endpoint degrades silently to no mount. + +## 7. The keepalive pool: memory, not storage + +**One sentence.** The keepalive pool is a per-replica in-process map of parked *live* +harness processes (sandbox, ACP connection, native model context all still warm), kept +for a short TTL so the next message in the same conversation continues in place; it +survives nothing and stores nothing durable. + +**The question it answers.** "The user replies within a minute. Does the platform pay a +cold sandbox start and a context re-feed?" Answer: no; the dispatch checks the pool key +`:`, validates fingerprints, checks the environment out, and runs +the turn against the same process (`services/runner/src/server.ts:585-624`). + +### Mechanics + +The pool (`services/runner/src/engines/sandbox_agent/session-pool.ts`) holds +`LiveSession` entries: an opaque environment handle, a config fingerprint, a history +fingerprint, a credential epoch, a state, an LRU stamp, and one idempotent teardown +closure (`session-pool.ts:30-43`). States are `busy`, `idle`, `awaiting_approval`, and +`destroyed` (`session-pool.ts:24`). Two park flavors exist with different TTLs, visible in +the QA session's own runner log: + +``` +[keepalive] park key=019e8df5...:6c23ff5f... ttl=300000ms state=awaiting_approval poolSize=2 +[keepalive] resume key=019e8df5...:6c23ff5f... gates=1 answered=1 carried=0 approve=1 reject=0 tool=Bash +[keepalive] park key=019e8df5...:6c23ff5f... ttl=60000ms state=idle (re-park) poolSize=2 +``` + +An idle park (60 s) awaits the next message; an approval park (300 s) awaits a human. On +checkout the dispatch validates: config fingerprint equality, history fingerprint +equality (an edited transcript must not continue a mismatched live memory), credential +epoch, and a fresh trailing user message; any mismatch evicts and degrades to cold +(`server.ts:585-607`). The approval-resume branch deliberately relaxes the config and +credential checks, because every approval reply carries freshly minted per-turn material +that can never equal the parked one, and requiring it would evict a perfectly good live +session on every approval (`server.ts:664-726`). Capacity is LRU with idle-only +eviction; parking is best-effort and a full pool simply tears the session down as before +(`session-pool.ts:186-232`). + +**Interaction with every durable plane.** The pool is why turn indexes must be computed +per turn from the shared store rather than per environment acquire (a warm environment +serves many turns; the frozen-index bug produced the duplicate `turn_index=0` appends in +the 500 report). A parked-then-resumed approval consumes one conversation turn index +across multiple executions and turn ids. A client disconnect during a session-owned run +does not abort the run but does veto parking, so a disconnected user's session is +destroyed at turn end (`server.ts:932-941`). + +**It is not storage.** A runner restart empties the map; the design degrades to cold +native load via the turns ledger, then to transcript replay, never to an error +(`session-continuity.ts:1-9`). Nothing in the pool is readable from outside the process; +the closest external signal is the streams plane's liveness flags. + +## 8. Cross-plane summary + +| Plane | Nature | Writer | Reader | Lifetime | Carries workflow refs? | Survives runner restart? | +|---|---|---|---|---|---|---| +| Records (`records`, tracing DB) | Upsert log (append-style, stable-id upserts) | Runner via async ingest worker | Web replay, records endpoints | Until tracing retention; untouched by session delete | No (only turn_id/span_id bridge) | Yes | +| Turns (`session_turns`, core DB) | Append-only INSERT, hard delete on session delete | Runner, at completed-turn end only | Runner hydrate/reconnect, root session query, turns endpoints | Until session delete | Yes (`references`, GIN) | Yes | +| Streams row (`session_streams`, core DB) | Mutable, 1 row per session | API only (heartbeat, commands, header edit) | Web session fetch, concurrency cap, turns FK | Soft-deleted by kill/archive, hard by delete | No | Yes (mirror may be stale) | +| Alive nest (Redis) | Ephemeral TTL locks | API only (runner drives via HTTP) | API (liveness, 409 bodies) | Seconds to an hour, TTL | No | Yes (Redis), but locks expire | +| Interactions (`session_interactions`, core DB) | Mutable rows, guarded transitions | Runner (create/resolve/sweep), API (respond CAS, kill cancel) | Interactions endpoints, future inbox | Until session delete (hard) or cancel (terminal state) | Yes (in `data.references`) | Yes | +| Mounts (rows + object store) | Filesystem (mutable), deterministic slugs | Agent through geesefs; API file endpoints | Agent, browser file panel, harness `session/load` | Until session delete (prefix wiped); archive is reversible | No (`agent_id` coming) | Yes | +| Keepalive pool | Process memory | Runner dispatch | Runner dispatch | 60 s idle / 300 s approval TTL | n/a | **No** | + +Authority rules worth memorizing: Redis beats the streams row for liveness; the harness's +native file beats everything for conversation content; the turns ledger beats the +in-memory continuity store after a restart but never beats a live in-process record +(`session-continuity-durable.ts:124-127`); records beat the browser's localStorage on +reload but localStorage is the fallback when records are absent. + +## 9. What is missing + +Six known gaps, each tied to the plane it lands on. + +**1. Agent references on records and streams.** The turns ledger carries workflow +references and mounts are growing `agent_id` (migration `oss000000016`), but neither the +records rows nor the `session_streams` row says which agent a session belongs to. The +session list can only be filtered per agent by joining through the turns' references +(`api/oss/src/core/sessions/service.py:57-67`), and a records row is agent-anonymous +forever. Lands on: records (new column, forward-fill only, tracing DB is never +backfilled) and streams (a reference or `agent_id` on the row, written by the first +heartbeat or the invoke path). + +**2. Trace id at turn start.** The turn append happens only at turn *end* +(`run-turn.ts:847-866`), so a turn that pauses, crashes, or is cancelled leaves no ledger +row and no turn-to-trace link, and even a healthy turn is invisible to trace-join queries +until it completes. `span_id`, `start_time`, and `end_time` are already in the schema but +never populated (the append passes only `traceId`, `run-turn.ts:857-863`). The fix shape +is a start-of-turn write (or enriching the heartbeat's row stamp) plus passing the span +and timing fields at completion. Lands on: turns, with the streams heartbeat as the +plausible carrier. + +**3. Session titles have no home.** The mutable `session_states` table is dropped +(migration `oss000000017`) and the streams row grew `name`/`description` with a dedicated +rename endpoint that cannot collide with liveness writes +(`PUT /sessions/streams/header`, `streams/service.py:411-464`, migration +`oss000000015`). But the frontend still keeps titles in localStorage +(`AgentChatSession.title`, +`web/oss/src/components/AgentChatSlice/state/sessions.ts:34-40`) and the rename UI in +`SessionTagBar` writes only local state. Until the chat writes and reads the header, a +title exists only in the browser that set it. Lands on: streams (server side is done on +the unmerged lane) and the frontend session state. + +**4. Cross-plane deletion.** `DELETE /sessions/` fans out to turns, interactions, mounts +(rows and prefixes), and the stream row (`api/oss/src/core/sessions/service.py:76-107`), +but: records are untouched by design (cross-database, tracing retention owns them, +`service.py:90-91`), live Redis keys are not cleared by delete (only kill collapses the +nest), local Claude native files on the runner's disk are outside any fan-out, and the +browser's localStorage copy of the conversation survives everything. A "delete this +conversation" product promise currently leaves the transcript readable in two places. +Lands on: every plane; records and the frontend cache are the real holes. + +**5. Cancel and steer.** The lock-side machinery is complete on the unmerged lane (the +command matrix, `force_cancel_alive`, W7.4's `is_current_turn` wired to +`controller.abort()`), but the signal reaches the run only at heartbeat granularity (up to +30 s), abort is the bluntest possible cancel (no graceful harness stop, no partial-turn +record), and the frontend deliberately does not expose steer or cancel because the +user-visible effect would be a stalled stream (`api.ts:276-280`). Steer additionally needs +the "new text supersedes parked work" dispatch semantics that db58551b defect 6 showed are +missing. Lands on: streams (signal), keepalive pool and runner dispatch (semantics), +records and turns (how an aborted turn is represented; today it is only an invalidated +continuity record and no ledger row). + +**6. Live mid-turn attach.** `attach` mode steals the attached lock and returns a +`watcher_id`, and the displaced channel exists for teardown notifications +(`streams/service.py:158-177`, `contract.py:60-73`), but there is no endpoint that +replays the in-flight turn's frames to a new watcher; the live stream is a property of the +original `/run` HTTP response. A reload today gets the records replay (complete only up to +the persist chain's lag) and then silence until the turn ends. Real mid-turn attach needs +a frame buffer or a Redis-stream relay per turn on the runner or API. Lands on: streams +(the lock half exists), records (the catch-up source), and a missing delivery path that +belongs to neither yet. + +## 10. Glossary + +- **ACP.** Agent Client Protocol; the JSON-RPC protocol between the runner and a harness, + with `session/new`, `session/load`, `session/prompt`, and permission-request verbs. +- **agent session id.** The harness's own native id for a conversation + (`session_turns.agent_session_id`). Claude's cannot be derived; Pi's appears in its + transcript filename. +- **alive lock.** Redis key asserting "at most one in-flight run per session"; value is + the owning turn id; cleared only by kill or TTL, so a session can be alive but idle. +- **attach / watcher.** Attach mode claims the `attached` lock for a `watcher_id`, + displacing any prior watcher via the `displaced` pub/sub channel. +- **cold native load.** Continuation tier two: a fresh environment whose harness resumes + its own session file via `session/load` with the ledger's `agent_session_id`. +- **gate.** A human-approval request raised by the harness mid-turn; stored as an + interaction row and an `interaction_request` record. +- **geesefs.** FUSE-over-S3 filesystem used to attach mount prefixes into sandboxes. +- **harness.** The coding-agent program the runner drives: Pi (`pi_core`, `pi_agenta`) or + Claude Code (`claude`). +- **heartbeat.** The runner's 30-second `POST /sessions/streams/heartbeat`, refreshing + locks, claiming ownership, mirroring the row, and carrying back `is_current_turn`. +- **keepalive pool.** Per-replica in-memory map of parked live environments; 60 s idle + TTL, 300 s approval TTL; not storage. +- **mount.** A durable object-store prefix (`mounts//`) attached into + the sandbox; identified by a row in the `mounts` table. +- **nest.** The containment invariant of the Redis locks: alive ⊇ running ⊇ attached. +- **owner / replica.** The `owner` Redis key maps a session to the runner container + (`replica_id`) currently serving it; claimed without stealing. +- **park / warm resume.** Parking keeps a live harness process in the keepalive pool at a + turn boundary; a warm resume checks it out and continues in place. +- **record.** One event row in the `records` table (tracing database): a message, + thought, tool call or result, interaction request or response, usage, done, or error. +- **replica id.** Stable per-process id of one runner container + (`AGENTA_RUNNER_REPLICA_ID` or a minted uuid). +- **session-owned run.** A `/run` request carrying a `sessionId`; it survives client + disconnect, heartbeats, and persists records; a non-session run aborts on disconnect. +- **states (historical).** The dropped `session_states` table: a mutable per-session JSON + blob previously used for continuity and superseded by the turns ledger and the streams + header. +- **steer.** Command-matrix mode: force-cancel the current holder and start a new turn + with new inputs; lock-level only today. +- **stream id.** The uuid7 primary key of the `session_streams` row; returned by the + heartbeat and stored on every turns row. Despite the name, no frames flow through it. +- **transcript replay.** Continuation tier three: `session/new` plus the prior + conversation rendered as capped text in the prompt. +- **turn id.** Per-execution uuid lock value and correlator; distinct from `turn_index`, + the per-session completed-turn counter. +- **verdict.** The approve/deny outcome of a gate, stored in + `session_interactions.data.resolution` and as an `interaction_response` record; distinct + from the row's lifecycle `status`. diff --git a/docs/design/agent-workflows/projects/sessions-takeover/arda-branch-reconciliation.md b/docs/design/agent-workflows/projects/sessions-takeover/arda-branch-reconciliation.md new file mode 100644 index 0000000000..6e43f6095d --- /dev/null +++ b/docs/design/agent-workflows/projects/sessions-takeover/arda-branch-reconciliation.md @@ -0,0 +1,362 @@ +# Reconciling Arda's approval-UI branch with the #5382 approvals fix train + +Written 2026-07-21. All claims below are verified against the git history in this +repository: `origin/fe-enhance/approval-ui-onbig` (Arda's branch, 11 commits on top of the +current `origin/main` at `dd19a601e9`, which is v0.105.7), its safety copy +`origin/backup/approval-ui-pre-stack`, and the open PR train `sessions-rebase/backend` +(PR #5375), `sessions-rebase/runner` (PR #5376), and `plan/concurrent-approvals` +(PR #5382). Conflict claims come from real `git merge-tree` trial merges, not from +eyeballing file lists. + +## Vocabulary used throughout + +- The **runner** is the Node/TypeScript sidecar under `services/runner/` that drives a + coding-agent harness inside a sandbox. A **harness** is the actual coding agent, either + Pi or Claude Code, speaking the ACP protocol. +- A **gate** is a human-approval request the harness raises before a policy-gated tool + runs. The playground shows a gate as an approval card. +- To **park** is to end the streamed turn while keeping the live harness session alive in + a keep-alive pool, waiting for the human's answer. A **warm resume** checks that session + back out and answers the gate in place. The **cold path** rebuilds the environment and + replays or continues the conversation instead. +- A **sentinel** is a bookkeeping string the runner writes as a tool result when the real + result is unavailable. Two matter here: `DEFERRED_NOT_EXECUTED` (the call never ran this + turn because the turn paused on another gate) and `APPROVED_EXECUTION_RESULT_UNKNOWN` + (the call was approved and started, but the pause ended the turn before its result was + observed; the second sentinel exists only on the #5382 side). +- **Hydration** is the frontend rebuilding the conversation from the persisted session + records on reload, in `transcriptToMessages.ts`. +- On `main` today the runner can park exactly **one** gate. A turn with more than one gate + bails out of parking entirely (`multi-gate-no-park` in `server.ts`) and falls to the + cold path. Both bodies of work below independently generalize this to many gates, and + that shared ambition is where they collide. + +## Part 1: what Arda built, told as a story + +Arda's branch is really three projects that happen to live on one branch. + +The first project is **approval throughput on the runner**. He observed that when the +model fires several tool calls in parallel, each needing approval, the runner parks on the +first gate and force-settles the siblings with the `DEFERRED_NOT_EXECUTED` sentinel, so a +two-file write costs the user one full park-and-resume round trip per file, and on cold +replay the sentinel rendered as `[tool error: ...]`, which the model read as a denial and +gave up. His answer has two halves. On the cold path, render the deferred sibling as a +"call it again with the same arguments" nudge so the model re-issues the call and its own +gate surfaces (commit `fcc7348156`). On the warm path, stop parking on the first gate: +collect the whole staggered batch behind a debounce window (a new +`AGENTA_RUNNER_APPROVAL_COLLECT_MS` environment variable, default 800 ms), park once with +every gate recorded in a new `env.parkedApprovals` array, and resume only when the +frontend has answered every gate in the batch (commit `23b9557fef`, plus the formatting +commit `a7524a835c`). + +While testing that flow he hit the same reload corruption our incident investigation hit, +and fixed it from his side: a paused turn persists a terminal `done` record and the resume +run re-persists the same user prompt, so a reloaded conversation splits the parked gate +from the resume that settles it and duplicates the user turn. His runner fix tags the +paused `done` with its stop reason and skips re-persisting the prompt on a resume +(commit `e658c1ec43`); his frontend fix keys tool parts transcript-wide during hydration +and keeps the draft message open across a paused `done` (commit `3e5c2d1c44`). + +The second project is **approval friction and config UX**, and it is the part with his +name on it in spirit: an "Always allow this tool for this agent" toggle on the approval +card that writes a real permission into the draft agent config (per-tool `permission` for +gateway and custom-function tools, a `harness.permissions.allow` rule for harness builtins +like bash), with a contained Undo notice in the config pane (commit `14e82e03c7`); an +"Approve all" action that resolves a whole batch of cards in one step instead of walking +1-of-N; and a set of config-pane primitives that surface "what changed" inline: a shared +`HeightCollapse` animation primitive (commit `491c593986`), `ChangedPathsContext` and +`FocusPathsContext` plus a section-change classifier (commit `0f1448f68a`), and the +drawer-backed "Model & harness" and "Advanced" sections showing a missing provider key or +uncommitted changes inline with revert affordances (commit `9ab4099cb8`). + +The third project is housekeeping: a one-line drawer width fix (commit `8bdd5c4ed4`) and +the Claude Code harness environment variables and volume mount for the EE dev compose +stack (commit `07c9153a62`). + +The branch name suffix "onbig" and the backup branch tell the provenance: he originally +built this on the big-agents workspace trunk interleaved with his Drive work (which has +since merged to main through PRs #5399 and #5400), then ported the eleven surviving +commits onto clean v0.105.7 main. Nothing in the branch touches the sessions backend, the +`interaction_response` event, or any API code: it is a pure runner-plus-frontend branch, +and it merges onto main today without help from any open PR. + +## Part 2: the commit map + +Classification key: (a) approval-machinery overlap with #5382, (b) approval UX net-new, +(c) config-section UI work unrelated to approvals, (d) infrastructure, (e) parked/resume +defect fixes overlapping #5382's defect fixes. + +| # | Commit | What it does | Main files | Class | Merges over #5382 stack? | +|---|--------|--------------|-----------|-------|--------------------------| +| 1 | `fcc7348156` fix(runner): render a deferred parallel-tool sibling as a retry nudge | Cold-replay transcript renders the `DEFERRED_NOT_EXECUTED` sentinel as a "call it again" nudge instead of an error the model reads as a denial. Exports `isDeferredNotExecuted`. | `transcript.ts`, `responder.ts`, tests | (e), mostly complementary | Near-clean; `transcript.ts` is untouched by #5382, one small conflict in `responder.ts` | +| 2 | `23b9557fef` feat(runner): hold parallel approval gates and resume them together | Generalizes the single parked gate to a `parkedApprovals` array; debounced collect-then-pause window (800 ms); resume answers the whole batch or goes cold. | `server.ts`, `run-turn.ts`, `pause.ts`, `acp-interactions.ts`, `runtime-contracts.ts`, `otel.ts`, tests | (a), direct machinery overlap | Conflicts across all six runner files | +| 3 | `491c593986` feat(ui): shared HeightCollapse + config-section animation primitives | One CSS collapse primitive; section shimmer and draft tones; adds the `motion` dependency to `@agenta/ui`. | `agenta-ui`, `AgentCommitNotice`, `RevealCollapse` | (c) | Clean | +| 4 | `0f1448f68a` feat(config): changed-path + focus primitives for config sections | `ChangedPathsContext`, `FocusPathsContext`, `RailField` opt-in, `sectionChanges` classifier, two new shared signal atoms. Includes unit tests. | `agenta-entity-ui`, `agenta-shared` | (c) | Clean | +| 5 | `9ab4099cb8` feat(config): context-driven config sections | Inline provider-key field, focus-filtered changed controls with revert, wired dot-paths for sandbox and harness permission controls. | `agenta-entity-ui` SchemaControls | (c) | Clean | +| 6 | `14e82e03c7` feat(agent-chat): always-allow + batch resolve | The always-allow toggle writing draft-config permissions via new `toolPermission` helpers (224 lines of tests); Approve-all; dock latch so a batch resolves in one visual step. | `ApprovalDock`, `useAlwaysAllowTool`, `toolPermission.ts`, notices | (b) | Clean | +| 7 | `8bdd5c4ed4` fix(ui): TriggerDeliveriesDrawer width | One line. | `TriggerDeliveriesDrawer.tsx` | (c) | Clean | +| 8 | `e658c1ec43` fix(runner): don't corrupt a parked+resumed turn's persisted transcript | Tags the paused turn's persisted `done` with `stopReason`; skips re-persisting the prompt on a resume (`tailIsFreshUserMessage`). | `server.ts`, `otel.ts`, `run-turn.ts`, tests | (e), overlaps #5382's deferred record-hygiene item | Conflicts in `server.ts` and `otel.ts` | +| 9 | `3e5c2d1c44` fix(agent-chat): restore a parked+resumed approval on reload | Hydration keys tool parts transcript-wide; dedupes the resume's re-emitted `tool_call`; a paused `done` no longer closes the draft message. | `transcriptToMessages.ts`, tests | (a)/(e), overlaps #5382's hydration work | Conflicts (content plus add/add on the test file) | +| 10 | `07c9153a62` chore(hosting): Claude Code harness config in EE dev compose | `CLAUDE_CONFIG_DIR`, `CLAUDE_CODE_OAUTH_TOKEN`, and a writable `~/.agenta-claude-config` mount on the dev runner. | `docker-compose.dev.yml` | (d) | Clean (auto-merges) | +| 11 | `a7524a835c` style(runner): prettier-format the rebased parallel-approval port | Formatting fallout of the port. | 3 runner files | (a) rider | Folds into the machinery decision | + +Trial-merge ground truth (`git merge-tree --write-tree`): merging Arda's branch over the +full #5382 stack conflicts in exactly ten files, all of them the runner machinery and the +hydration adapter (the six `services/runner/src` files, the keep-alive approval test, and +`transcriptToMessages.ts` plus its test). Every other file in the branch, meaning the +whole config-UX, always-allow, drawer, and compose set, auto-merges. Against the sessions +rebase alone (#5375 and #5376 without #5382) the conflict set shrinks to two files, +`run-turn.ts` and `runtime-contracts.ts`. + +## Part 3: the overlap analysis + +There are four overlapping pairs. For each: what defect it addresses, whether the two +sides conflict textually and semantically, and which is more complete, judged against the +incident root-cause reports +(`docs/design/agent-workflows/scratch/debug-concurrent-approvals-db58551b.md` and +`debug-frontend-approval-dispatch.md`) and each side's tests. + +### Pair 1: multi-gate parking machinery + +**His:** `23b9557fef` "hold parallel approval gates and resume them together". +**Ours:** the #5382 stack's park-and-resume train (`b831e753f3` multiple simultaneous +requests, `1171dd5beb` record every parked gate, `a05238576a` partial answer sets, +`24bc93b344` the Pi-batching park rule, `3204ad5517` review closure). + +Both sides generalize main's single `env.parkedApproval` to a collection, in the same six +files. The textual conflict is total. The semantic conflict is the important one, and it +is a genuine contract difference: + +- **Arda's resume is all-or-nothing.** His `server.ts` comment states the contract + plainly: "The FE only resumes once it has answered ALL pending gates ... a gate still + missing a decision means the request is not that resume ... treat as a mismatch, go + cold." A resume request that answers only some parked gates evicts the live session. + This is exactly the shape the incident report identifies as defect 1's enabling + condition: the frontend's "every card settled" precondition could never hold after a + state rebuild, so the last answer sat unsent in browser memory and the conversation + died. Arda's branch does not change the frontend dispatch predicate + (`agentApprovalResume.ts` is untouched), so the all-settled batching survives on his + branch, and his design leans on it. +- **#5382's resume is per-card with partial answer sets.** One click dispatches one + answer; the runner answers the subset it received, streams those results, and re-parks + on the remaining gates. This was adversarially reviewed, carries an end-to-end + regression replay of the incident (`b955011782`), and was verified in four live QA + cycles. +- **Arda's collect window is the genuinely new idea.** #5382 parks each gate as it + arrives and explicitly deferred "two cards genuinely on screen at once" to issue #5391 + because the harness adapters raise gates serially. Arda measured gates arriving roughly + half a second apart and debounces the pause so a staggered batch parks together and the + user sees N cards at once. One caution that decides the window's fate: the live incident + investigation established that BOTH harness adapters raise approval requests strictly + serially today, each blocking until answered (Pi's confirms, and Claude's adapter per the + evidence recorded on issue #5391). The second request does not exist until the first is + answered, so no window of any length has a burst to gather on either harness. The window + therefore only becomes useful when the upstream adapter work in #5391 makes requests + arrive together, and it belongs with that issue rather than in the port. + +What Arda's machinery does not cover, and #5382 does, mapping to the incident defects: +defect 2 (the post-pause sweep stamps the retry-inviting sentinel onto an approved, +mid-execution call; #5382 excludes approved-executing calls and introduced the +`APPROVED_EXECUTION_RESULT_UNKNOWN` sentinel; Arda's sweep exclusion is still only the +paused gates, and his branch has no second sentinel), defect 3 (a never-started sibling +recorded as a successful "(no output)" result; #5382 buffers completion frames during the +pause; Arda does not touch `runtime-policy.ts`), defect 4 (the answer half of a gate is +never persisted; #5382 emits a durable `interaction_response` and writes the verdict onto +the interaction row through the API; Arda's branch contains no `interaction_response` +producer at all), and the Pi-batching deadlock (#5382 parks immediately when Pi's +batching blocks an approved call; Arda's resume waits on the blocked prompt). + +There is also an interaction between defect 2 and Arda's own retry nudge that matters: +because his branch leaves the sweep unfixed, an approved call that actually executed can +still be stamped `DEFERRED_NOT_EXECUTED`, and his nudge then tells the model to run it +again with the same arguments. For a side-effecting command that is a double-execution +invitation, sharpened rather than softened. + +**Verdict: resolve in favor of #5382's machinery.** It fixes four root-caused defects his +does not, its contract survives state rebuilds, and it is regression-tested against the +literal incident. The one thing worth carrying forward from his commit is the +collect-then-pause window, which is small (a `schedulePause` debounce on the pause +controller plus wiring in `acp-interactions.ts`) and can be re-expressed on top of +#5382's parked-gates map as a follow-up. It is, in effect, the first half of issue #5391 +delivered without upstream adapter changes, for the Claude harness only. + +### Pair 2: reload hydration of a parked-and-resumed approval + +**His:** `3e5c2d1c44`. **Ours:** the hydration halves of `20dcb553ce` and `297d82ae1f`. + +Here the two sides converged on the same core insight independently: tool parts must be +keyed transcript-wide, not per-draft-message, because a gate parked in one run is settled +by a `tool_result` in the resume run, and per-draft keying drops that result and leaves +the card stuck at "Awaiting approval". Both implementations hoist the tool index out of +the draft. On top of that shared core: + +- #5382 additionally overlays the persisted `interaction_response` answers onto requests + (so an answered card rehydrates as answered even before any resume ran), reopens a + sentinel-only sealed card when a later approval request re-parks it (the defect-B fix + from the frontend report, with the sentinel prefixes mirrored as exported constants), + and keeps `done` as a message boundary, with a comment stating that choice: the index + settles across the boundary, and message-per-turn stays true. +- Arda additionally dedupes the resume's re-emitted `tool_call` record (update in place + rather than pushing a duplicate part) and treats a `stopReason: "paused"` `done` as a + non-boundary, keeping the paused turn and its resume in one assistant bubble. His + non-boundary reading requires his runner-side commit (pair 3) to have stamped the stop + reason; #5382's version needs no new persisted field. + +Textually they conflict (content conflict on the adapter, add/add on the test file). +Semantically they are two dialects of the same fix, but only #5382's covers the answered +half: without persisted answers, Arda's hydration still resurrects an answered-but-not- +yet-resumed gate as pending, which is the state that killed the incident conversation. + +**Verdict: #5382's version is the superset that matters; take Arda's `tool_call` dedupe +as a small follow-up.** The dedupe becomes genuinely necessary the moment record ids are +scoped per turn (the queued audit-hardening work makes re-emits append instead of +upserting in place), so it is worth keeping on the roadmap with his name on it. The +one-bubble-versus-two rendering choice is cosmetic and can be decided later. + +### Pair 3: persisted-transcript corruption on park and resume + +**His:** `e658c1ec43`. **Ours:** the truthful-terminalization work in `4c8c809984` plus +the deliberate deferral in #5382. + +These do not solve the same defect, despite the similar titles. #5382's runner commit is +about result truthfulness (an approved executing call keeps its real result; a +never-started call becomes deferred, never a fake success). Arda's commit is about record +hygiene: the duplicated user-message row on every resume, and the paused `done` masquerading +as a turn boundary. That is, almost exactly, item 5 of the incident report's fix plan +("stop re-persisting the recovered prompt on approval resumes", part of making the record +a trustworthy audit log), which #5382's PR description explicitly deferred. So Arda +implemented, in parallel, a piece of our own declared follow-up. + +The catch is purely textual: his edit sits in the same `server.ts` persist block and the +same `otel.ts` `finish()` that #5382 reworked, so the commit does not apply as-is. The +content, a `tailIsFreshUserMessage` guard before persisting the prompt and an optional +`stopReason` on the terminal `done` record, is small and re-expressible on the stack in +an afternoon. One design note before re-applying the guard: the frontend report shows the +#5382 hydration currently relies on the duplicated user row's existence in one place (the +server-transcript-adoption heuristic prefers the server copy "whenever it has MORE +messages"), so removing the duplicate row should land together with a re-check of that +heuristic, which is a reason to do it as a deliberate follow-up rather than a mechanical +cherry-pick. + +**Verdict: no contest to resolve; adopt the intent as the already-planned record-hygiene +follow-up on top of #5382, re-implemented against the stack's code.** + +### Pair 4: rendering the deferred sibling on cold replay + +**His:** `fcc7348156`. **Ours:** the sentinel taxonomy in `4c8c809984` and `3204ad5517`. + +This is the cleanest pair: they compose. #5382 never touched `transcript.ts`, the +cold-replay prompt builder, so his change (render a `DEFERRED_NOT_EXECUTED` block as a +neutral "this was skipped, not denied; call it again" nudge instead of an error string) +fills a real gap that exists on the #5382 stack too: any deferred sibling that does end +up on the cold path today still replays as `[tool error: ...]` and reads as a denial. The +only overlap is a trivial conflict in `responder.ts` where he exports the existing +`isDeferredNotExecuted` helper and #5382 edited neighboring lines. + +Two conditions for taking it: it must land on top of #5382's sweep fix, because only +there is the deferred sentinel guaranteed to mean "genuinely never started" (see the +double-execution note under pair 1); and it should extend the same courtesy to the +`APPROVED_EXECUTION_RESULT_UNKNOWN` sentinel, which his branch does not know about, with +the opposite instruction (do not retry a side-effecting call). + +**Verdict: take it, rebased onto the stack, with the UNKNOWN-sentinel case added.** + +## Part 4: the extraction plan + +The branch splits cleanly along the trial-merge conflict line. Everything below is +ordered so that no step waits on anything it does not truly need. + +**Now, independent of every open PR (target: main).** + +1. `8bdd5c4ed4` (drawer width) and `07c9153a62` (compose harness config). Trivial, + zero-risk, one small PR or folded into the next one. +2. The config-UX train as one stacked pair of PRs in this order: `491c593986` + (HeightCollapse and section primitives), then `0f1448f68a` (changed-path and focus + primitives), then `9ab4099cb8` (the context-driven sections). These three auto-merge + over both main and the #5382 stack and have their own unit tests + (`sectionChanges`, `formatCommitted`). Nothing approval-related blocks them. +3. `14e82e03c7` (always-allow plus batch resolve) can also go to main now on top of the + config train (it needs `HeightCollapse` and the shared signal atoms). It is frontend + plus config only and auto-merges over the stack. If it lands before #5382, give it one + QA pass again after #5382 merges: the dock's batch latch was written against the + all-settled dispatch, and under #5382's per-card dispatch an "Approve all" click fires + several responses that may resume-then-re-park in waves. The runner side accepts + partial answer sets by design, so this should compose, but it is the one seam nobody + has watched live. + +**After #5375, #5376, and #5382 merge (target: the merged main).** + +4. Drop, from Arda's branch, the four machinery commits as they stand: `23b9557fef`, + `e658c1ec43`, `3e5c2d1c44`, and `a7524a835c`. Their tests encode the all-or-nothing + resume contract and go with them. +5. Re-express the salvage as three small follow-ups, each cheap on top of the stack's + code, ideally by Arda so the authorship follows the ideas: + - the collect-then-pause window (`schedulePause` on the pause controller, the + `onScheduleApprovalPause` seam, the env var), now feeding #5382's parked-gates map, + positioned honestly as a Claude-harness batching feature and the first half of #5391; + - the record-hygiene pair (skip the duplicate user row via `tailIsFreshUserMessage`, + stamp `stopReason` on the paused `done`), landed together with a re-check of the + frontend's server-transcript-adoption heuristic, as the already-planned deferred + item 5; + - the cold-replay retry nudge in `transcript.ts`, extended to also treat the UNKNOWN + sentinel (as "do not retry"), plus the trivial `isDeferredNotExecuted` export. +6. The `tool_call` re-emit dedupe in hydration rides along whenever the per-turn + record-id scoping lands, since that change is what makes it necessary. + +## Part 5: the decision list for Mahmoud + +**Decision 1: which multi-gate machinery survives.** +Context: both bodies of work replace main's single-gate park with multi-gate machinery in +the same six runner files; the contracts are incompatible (all-or-nothing batch resume +versus per-card dispatch with partial answer sets and re-park). +Option 1: keep #5382's machinery and drop Arda's four machinery commits. +Option 2: keep Arda's and rework #5382 on top of it. +Consequence: option 2 would discard the fixes for incident defects 2, 3, and 4 (sweep +clobbering, phantom success, unpersisted answers), the Pi-batching deadlock fix, the +adversarial-review closures, and the end-to-end incident regression test, and would +reinstate the all-answers-required resume that caused the dead conversation. +Recommendation: option 1, without reservation. Arda's implementation is competent but it +was built without the incident evidence; nothing in it handles a case #5382 misses, while +#5382 handles four cases his misses. + +**Decision 2: whether the collect-then-pause window becomes a follow-up.** +Context: the window is Arda's genuinely novel contribution; it batches staggered gates so +the user sees N cards and answers once, which #5382 deferred to issue #5391. It helps the +Claude harness only, because Pi raises confirms strictly serially. +Option 1: have Arda re-implement it on top of the merged stack as a small, env-gated +follow-up, QA'd against the Claude harness, framed as the no-upstream-change half of #5391. +Option 2: drop it until #5391 resolves the adapter-serialization question for both +harnesses. +Consequence: option 2 leaves multi-file Claude approvals at one round trip per file for +however long #5391 takes. +Recommendation: option 1. It is small, additive on the surviving machinery, and it keeps +Arda's headline idea alive with his authorship. + +**Decision 3: whether Arda's UX and config commits land now or wait for the stack.** +Context: the always-allow toggle, batch resolve, config sections, drawer fix, and compose +config all auto-merge over both main and the #5382 stack; only the always-allow and +batch-resolve dock has any behavioral coupling to the approval flow. +Option 1: land them on main now (config train and housekeeping immediately; the approval +dock commit too, with a repeat QA pass after #5382 merges). +Option 2: hold everything until #5375/#5376/#5382 merge and rebase once. +Consequence: option 2 costs Arda one to two weeks of idle divergence for no correctness +gain; option 1 costs one extra QA pass on the dock. +Recommendation: option 1. This is what actually unblocks him this week. + +**Decision 4: who owns the record-hygiene and retry-nudge follow-ups.** +Context: Arda independently built two things #5382 explicitly deferred or missed (the +duplicate-user-row and paused-done tagging; the cold-replay nudge). Both need +re-implementation against the stack's code rather than cherry-picks. +Option 1: assign both to Arda as his first post-merge tasks on the approvals surface. +Option 2: fold them into the existing audit-hardening queue on our side. +Consequence: either works technically; option 1 turns the collision into shared ownership +of the surface and gives his parallel work a landing. +Recommendation: option 1, with the two guardrails named in part 3 (record-hygiene lands +with the frontend adoption-heuristic re-check; the nudge lands only on top of the sweep +fix and covers the UNKNOWN sentinel too). + +**The dependency answer, for completeness.** Arda's branch is standalone on v0.105.7 +main: it does not depend on #5375, #5376, or #5382, uses no sessions API, and merges to +main today. The dependency runs the other way: if the train merges first, ten files +conflict, all of them the machinery this document recommends dropping, and everything +else rebases clean. diff --git a/docs/design/agent-workflows/projects/sessions-takeover/arda-handoff.md b/docs/design/agent-workflows/projects/sessions-takeover/arda-handoff.md new file mode 100644 index 0000000000..abd8bfe43f --- /dev/null +++ b/docs/design/agent-workflows/projects/sessions-takeover/arda-handoff.md @@ -0,0 +1,261 @@ +# Handoff: the approvals and sessions work, and where your branch fits + +Arda, this is the context you need to pick up the approvals surface again after last +weekend. You wrote the approval user experience and the config-section work yourself, and +all of that ships. What changed underneath you is the approval machinery on the runner and +a restructure of how a session is stored. This document explains what happened, the +decisions Mahmoud has made about your branch, the branch you will work on, your task list, +and the two questions he wants your read on. Every decision below comes with the reason and +a place to verify it. You should be able to start within an hour, and you will not need a +meeting to do it. + +A note on vocabulary, because a few terms recur. The **runner** is the Node and TypeScript +sidecar under `services/runner/` that drives a coding agent inside a sandbox. The coding +agent itself is the **harness**, either Pi or Claude Code. A **gate** is a human-approval +request the harness raises before a policy-gated tool runs; the playground shows a gate as +an approval card. To **park** a gate is to end the streamed turn while keeping the live +harness process alive in a pool, waiting for the human's answer; a **warm resume** checks +that process back out and answers the gate in place. A **sentinel** is a bookkeeping string +the runner writes as a tool result when the real result is not available. **Hydration** is +the frontend rebuilding the conversation from the saved records when you reload a session. + +## 1. What happened while you were heads down + +During a live QA of two parallel approval-gated shell writes in one turn, the conversation +died: the first approved command was reported as not executed, the whole turn went silent +after the second approval, the first card flipped back to waiting, and a later question was +answered with a stale "Done" (the full reconstruction is +`docs/design/agent-workflows/scratch/debug-concurrent-approvals-db58551b.md`). A fix train, +now open as PR #5382, root-caused four defects behind that failure and rebuilt the +multi-gate approval machinery on the runner and the reload path on the frontend to fix them. +In parallel, the session storage layer was restructured: the mutable per-session state blob +became an append-only per-turn ledger plus a liveness row, now combined into one branch as +PR #5436. Because JP is away and you were heads-down on Drive work, Mahmoud took over both +lines and reconciled them against your branch (PR #5426), which had independently rebuilt +the same multi-gate machinery and the same reload fix without the incident evidence. Your +branch and the fix train collide because both generalize the runner's single-gate parking +into multi-gate parking in the same six runner files, and the two designs disagree on the +resume contract, which is exactly where the ruling below lands. + +## 2. The rulings, and the reason for each + +These are Mahmoud's decisions. Each one names its reason and where you can check it. None of +them is a comment on the quality of your work; where a decision goes against your branch, it +is because your branch was built before the incident produced its evidence. + +### 2.1 The multi-gate machinery from PR #5382 survives; your four machinery commits are not ported + +Your branch replaces the runner's single parked gate with a multi-gate collection, in +commits `23b9557fef` (hold parallel gates and resume together), `e658c1ec43` (parked-turn +transcript hygiene), `3e5c2d1c44` (reload restore of a parked-and-resumed approval), and +`a7524a835c` (the formatting rider). Those four are not carried onto the surviving branch. + +The reason is the resume contract, not the code. Your resume is all-or-nothing: the runner +only resumes once the frontend has answered every parked gate, and a resume that answers a +subset is treated as a mismatch and evicts the live session (your `server.ts` comment states +this contract directly). That is the exact shape whose failure caused the dead conversation. +After a state rebuild, the frontend's "every card settled" precondition can never hold, +because the saved record held every gate's request and never its answer, so the last answer +sat unsent in browser memory (incident report, defect 1 and defect 4). Your branch also +predates the four incident fixes: it does not exclude an approved, executing call from the +post-pause sweep, has no second sentinel for a call whose result is unknown, records a +never-started sibling as a fake success, and never persists the answer half of a gate at +all (incident report, defects 2 through 4, mapped commit-by-commit in +`docs/design/agent-workflows/projects/sessions-takeover/arda-branch-reconciliation.md`, +Pair 1 and Decision 1). PR #5382 fixes all four, carries an end-to-end regression replay of +the incident, and was verified across four live QA cycles (PR #5382 description, "How this +was verified"). Your implementation was competent; it was simply built without the evidence +that these four cases exist. + +### 2.2 The collect window is not ported now, and is recorded on issue #5391 with your authorship + +Your genuinely novel idea on the machinery side is the collect window: instead of parking on +the first gate, gather every gate raised inside a short debounce window (a new +`AGENTA_RUNNER_APPROVAL_COLLECT_MS`, default 800 milliseconds) so a staggered batch parks +together and the user sees several cards at once. It is not ported now. + +The reason is that under today's harness adapters there is no burst to gather. Both harnesses +raise their gates strictly one at a time and block: the second gate does not exist until the +first is answered. The incident timeline shows Pi raising confirms serially (incident report, +"Pi serializes confirms"), and the live QA on issue #5391 established the same for Claude: +two parallel gated calls in one turn produce exactly one `request_permission` from the +`claude-agent-acp` adapter, which blocks until answered (issue #5391, "the ACP adapter +serializes permission requests"; the Zed comparison reaches the same conclusion about the +shared adapter in `docs/design/agent-workflows/scratch/zed-acp-approvals-comparison.md`). +Until the upstream adapters can hold several permission requests open at once, a window of +any length batches nothing. So the idea is recorded on issue #5391 as the agreed client-side +half, with your authorship, to be built once the upstream half lands. + +### 2.3 Your user experience and config work ships + +The always-allow toggle, approve-all, the config sections, the drawer width fix, and the +compose harness config all ship. They are the parts with your name on them, and every one of +them merges cleanly over both the fix train and the storage rework (the trial-merge ground +truth is in the reconciliation document, Part 2: only the ten runner and hydration files +conflict, everything else auto-merges). Concretely these are commits `491c593986` (the +shared `HeightCollapse` and section primitives), `0f1448f68a` (the changed-path and focus +primitives), `9ab4099cb8` (the context-driven config sections), `14e82e03c7` (always-allow +and batch resolve), `8bdd5c4ed4` (drawer width), and `07c9153a62` (compose harness config). +They will be ported for you onto the new branch described in section 3. + +There is one caution you own. Your approve-all dock was written against the old dispatch, in +which the frontend sent the batch only once every card had settled. PR #5382 changes the +frontend to per-card dispatch, where one click sends one answer and the runner can resume on +a subset of the parked gates. The runner accepts partial answer sets by design, so an +approve-all click that fires several answers in waves should compose, but this is the one +seam nobody has watched live (reconciliation document, Part 4 item 4, and Decision 3). It +needs one repeat QA pass on the dock after #5382 merges. That pass is in your task list. + +### 2.4 Two of your ideas are adopted as your first tasks, rebuilt rather than cherry-picked + +Two things you built independently are the same things the fix train explicitly deferred or +had not reached. Both are adopted, and both are yours to build first. They are rebuilt +against the merged code rather than cherry-picked, because your original commits sit in the +same runner blocks the fix train reworked and do not apply as-is (reconciliation document, +Pair 3 and Pair 4). + +The first is the transcript hygiene pair: on a resume, do not save a second copy of the +user's message, and give a paused turn a distinct end-marker so a reload can tell a pause +from a real turn boundary. The guardrail is that one of the fix train's reload checks +currently relies on that duplicate row existing. Its server-transcript-adoption heuristic +prefers the server copy of the conversation whenever the server copy has more messages, and +the duplicate user row is part of what makes it have more (reconciliation document, Pair 3). +So when you remove the duplicate, re-check that frontend adoption heuristic in the same +change. + +The second is the deferred-command hint after replay: when a sibling tool call was skipped +because the turn paused for another approval, render it on the cold path as a neutral "this +was skipped, not denied, call it again" nudge instead of an error string the model reads as +a denial. Two guardrails. It lands only on top of the fixed post-pause cleanup, because only +there is the deferred marker trustworthy; without the sweep fix an approved call that +actually ran can still be stamped deferred, and your nudge would then invite the model to +run a side-effecting command twice (incident report, defect 2; reconciliation document, +Pair 4). And it must also cover the `APPROVED_EXECUTION_RESULT_UNKNOWN` sentinel, which your +branch does not know about, with the opposite instruction: do not retry. + +### 2.5 The rest of your roadmap comes from the known gaps + +The storage architecture document lists what the restructure does not yet do (architecture +document, section 9, "What is missing"). Mahmoud's rulings on those gaps set the rest of +your roadmap. + +Wire session titles to the new rename endpoint. The server side is done on PR #5436: the +liveness row grew `name` and `description` fields and a dedicated rename endpoint that cannot +collide with liveness writes. The frontend still keeps titles only in localStorage, so a +title exists only in the browser that set it (architecture document, gap 3). Your task is the +frontend half: write and read the header through that endpoint. + +Build cancel and steer on the signal plumbing that already exists. The lock-side machinery is +complete on the merged lane: the kill command tears the run down, and an interrupt flag rides +the heartbeat so a cancel reaches an in-flight run (the runner wires the heartbeat's +`is_current_turn` to an abort; architecture document, gap 5, and section 4). Use the protocol +shape from the Zed comparison: cancel the turn, answer every pending permission request as +cancelled, wait for the harness to settle into an idle cancelled state, then send the new +instruction as a fresh prompt (Zed comparison, "Recommended changes", item 3). This is what +db58551b defect 6 showed is missing today, where new text during a park is swallowed as a +resume of the stale task. + +Two things are explicitly out of scope for now: session deletion (gap 4) and live mid-turn +attach (gap 6). + +## 3. Your branch and how to work on it + +You do not need GitButler for any of this. You work on a plain git branch with normal git. + +The rest of us work on a stack of branches. `feat/sessions-storage-rework` (PR #5436) is the +storage rework at the bottom. `plan/concurrent-approvals` (PR #5382) sits on top of it and +carries the surviving multi-gate machinery. Your branch will be a new plain git branch, cut +from the head of `plan/concurrent-approvals`, with your keeper commits from section 2.3 +ported onto it. You work there. + +The one rule that protects you: while you are working, we do not rewrite the two branches +underneath you. Anything we need to change on them lands as new commits on top, so your base +never moves under you. If a rewrite of those two branches ever becomes unavoidable, we agree +the moment with you first, before it happens. That means you can treat your base as stable +and rebase on your own schedule, not on ours. + +Merge order is PR #5436 first, then PR #5382, then your branch's PR. Set your PR's base to +the branch directly below it so the diff shows only your work. + +Your old branch and PR #5426 will be closed with a comment linking this document, so the +reasoning is attached to the thread. Your original branch stays on origin untouched as the +archive of your version; nothing about your work is lost, and you can always diff against it. + +## 4. Your task list + +In order. Guardrails are inline so you do not have to hold them in your head. + +1. **Rebuild the transcript hygiene pair** (ruling 2.4, first idea). On resume, stop saving a + second copy of the user message, and stamp a distinct end-marker on a paused turn so + hydration can tell a pause from a turn boundary. Rebuild against the merged runner code, + not from your old commit. Guardrail: in the same change, re-check the frontend + server-transcript-adoption heuristic, which today leans on the duplicate row being present + (reconciliation document, Pair 3). + +2. **Rebuild the deferred-command hint after replay** (ruling 2.4, second idea). On the cold + path, render a deferred sibling as a "skipped, not denied, call it again" nudge. Guardrail + one: land it only on top of the fixed post-pause cleanup, so the deferred marker is + trustworthy (incident report, defect 2). Guardrail two: also handle the + `APPROVED_EXECUTION_RESULT_UNKNOWN` sentinel, with the opposite instruction, do not retry. + +3. **Repeat QA the approve-all dock against per-card dispatch** (ruling 2.3). After #5382 + merges, drive an approve-all click over several parked gates and confirm the waves of + partial answers resume and re-park cleanly. This is the one seam not yet watched live. + +4. **Wire session titles to the rename endpoint** (ruling 2.5). Frontend half only; the + server rename endpoint is on #5436. Replace the localStorage-only title with a write and + read through the header endpoint. + +5. **Build cancel and steer on the existing signal plumbing** (ruling 2.5). Use the Zed + protocol shape: cancel the turn, answer every pending permission as cancelled, wait for the + harness to settle, then send the new instruction. The kill command and the heartbeat + interrupt flag already exist; you are building the dispatch semantics and the surface on + top of them. + +Out of scope for now: session deletion and live mid-turn attach. + +## 5. What to read, and why each matters to you + +- `docs/design/agent-workflows/projects/sessions-takeover/arda-branch-reconciliation.md`. + The commit-by-commit map of your branch, which commits ship and which do not, the four + overlap verdicts, and the extraction plan. This is the primary evidence behind section 2; + read it first. +- `docs/design/agent-workflows/scratch/debug-concurrent-approvals-db58551b.md`. The incident + reconstruction and the four defects. This is why the machinery ruling went the way it did, + and it is the ground truth for the guardrails on your first two tasks. +- `docs/design/agent-workflows/scratch/zed-acp-approvals-comparison.md`. Where cancel and + steer come from. The "Recommended changes" section gives the exact cancel-then-restart + protocol shape you will build in task 5, and it explains why the adapter serializes gates. +- `docs/design/agent-workflows/projects/sessions-takeover/architecture.md`. The six storage + planes at working depth, verified against real database rows. You do not need to absorb it + all; the parts that touch your tasks are section 4 (the streams and liveness plane, for + cancel and steer), section 5 (the interactions plane, for open question 6.2), and section 9 + (the gaps that became your roadmap). Treat it as reference, not as required reading. +- PR #5382 and PR #5436 descriptions, for the exact shape of what merges beneath you. Your + own PR #5426 description, for the record of what you built and where each piece landed. + +## 6. Two questions Mahmoud wants your read on + +Neither is settled. Your view as the frontend owner is why he is asking. + +### 6.1 The streams-table rename + +The table named `session_streams` stores no streamed frames; the architecture document opens +by killing exactly that mental model (architecture document, page 1, mental model 2). It +holds liveness and ownership, and it now also carries the session identity, the `name` and +`description` header. The name misleads on both counts. The open question is whether to rename +the table so its name matches what it holds. You read and write this row from the frontend +(session fetch, and the rename UI you are about to wire in task 4), so the churn and the +clarity both land partly on you. Weigh in on whether the rename is worth it and what the name +should be. + +### 6.2 The interactions guard ruling + +The runner refuses to create an interaction row when the run context carries no +`workflow_revision` reference, because the respond path could not re-invoke anything without +it. The consequence is that gates in that state are answerable in-band only and are invisible +to any interactions-plane inbox; a real QA session had approval records but zero interaction +rows for exactly this reason (architecture document, section 5, "Sharp edges"). The open +question is whether to keep that guard as is, accepting that some gates never get a durable +interaction row, or to relax it. It matters to any future approval-inbox surface, which is +your territory. Weigh in on which way it should go. From 102aa5d43a9c5359d9245c2e8c648fa9a7d7b871 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Tue, 21 Jul 2026 15:43:42 +0200 Subject: [PATCH 13/15] docs(sessions): OpenCode comparison and steer-pattern pointer in the handoff --- .../sessions-takeover/arda-handoff.md | 15 +- .../sessions-takeover/opencode-comparison.md | 429 ++++++++++++++++++ 2 files changed, 439 insertions(+), 5 deletions(-) create mode 100644 docs/design/agent-workflows/projects/sessions-takeover/opencode-comparison.md diff --git a/docs/design/agent-workflows/projects/sessions-takeover/arda-handoff.md b/docs/design/agent-workflows/projects/sessions-takeover/arda-handoff.md index abd8bfe43f..5c16558867 100644 --- a/docs/design/agent-workflows/projects/sessions-takeover/arda-handoff.md +++ b/docs/design/agent-workflows/projects/sessions-takeover/arda-handoff.md @@ -206,11 +206,16 @@ In order. Guardrails are inline so you do not have to hold them in your head. server rename endpoint is on #5436. Replace the localStorage-only title with a write and read through the header endpoint. -5. **Build cancel and steer on the existing signal plumbing** (ruling 2.5). Use the Zed - protocol shape: cancel the turn, answer every pending permission as cancelled, wait for the - harness to settle, then send the new instruction. The kill command and the heartbeat - interrupt flag already exist; you are building the dispatch semantics and the surface on - top of them. +5. **Build cancel and steer on the existing signal plumbing** (ruling 2.5). For cancel, use + the Zed protocol shape: cancel the turn, answer every pending permission as cancelled, wait + for the harness to settle, then send the new instruction. For steer, read the OpenCode + comparison first (`opencode-comparison.md`, same folder): their design treats a + mid-turn message as durable data, a per-session inbox row marked "steer" or "queue" that + the loop picks up at safe boundaries, instead of a dispatch-time judgment call. That + pattern is the recommended shape for our steer semantics, because it is exactly what + makes "a new message must never be swallowed by parked work" structural rather than + heuristic. The kill command and the heartbeat interrupt flag already exist; you are + building the dispatch semantics and the surface on top of them. Out of scope for now: session deletion and live mid-turn attach. diff --git a/docs/design/agent-workflows/projects/sessions-takeover/opencode-comparison.md b/docs/design/agent-workflows/projects/sessions-takeover/opencode-comparison.md new file mode 100644 index 0000000000..b39177fcda --- /dev/null +++ b/docs/design/agent-workflows/projects/sessions-takeover/opencode-comparison.md @@ -0,0 +1,429 @@ +# Comparison of session storage and approvals in OpenCode and Agenta + +This report examines how OpenCode solves the problems our sessions-takeover week was about: +where conversations live, how a session continues after a restart, how a human approves a +tool call, what happens on cancel or a mid-turn message, and how several clients follow one +session. It is the sibling of the Zed study +(`docs/design/agent-workflows/scratch/zed-acp-approvals-comparison.md`) and reads against +our architecture document +(`docs/design/agent-workflows/projects/sessions-takeover/architecture.md`) and the +db58551b incident report +(`docs/design/agent-workflows/scratch/debug-concurrent-approvals-db58551b.md`). + +OpenCode citations are paths inside a clone of `https://github.com/sst/opencode` at commit +`cb562b2c6289` (2026-07-21). Agenta citations are paths in this repository. + +## 1. Orientation: what OpenCode is and how its division of labor differs from ours + +OpenCode is the open-source coding agent built by SST, written in TypeScript. It runs as a +local server process. The terminal UI, the web app, and the desktop app are all clients of +that server over HTTP plus one server-sent-events stream (SSE, a one-way HTTP stream of +JSON events). The server package is `packages/opencode`; the newer core is +`packages/core`; the wire schemas live in `packages/schema`; the SolidJS client that +powers web and desktop is `packages/app`. + +The architectural difference from Agenta that explains almost everything else in this +report: **OpenCode owns the whole agent loop in one process.** The server calls the model +provider itself through its own LLM layer, executes tools itself on the host filesystem, +and writes every step of the conversation into its own store. There is no harness. There +is no sandbox. There is no second copy of the conversation anywhere. + +Agenta drives external harnesses (Pi, Claude Code) over ACP inside sandboxes. The harness +owns the model-facing conversation in its own native session file; our platform keeps a +separate display transcript (records), a mapping ledger (turns), a liveness plane +(streams plus Redis), and a durable approval plane (interactions), spread across two +databases, Redis, and an object store. OpenCode has one SQLite file. + +One reading note: OpenCode currently contains two generations of internals. The shipped +path is the "v1" loop (`packages/opencode/src/session/prompt.ts`, the `SessionPrompt` +monolith). A "v2" rewrite lives in `packages/core/src/session/` with its own runner and +event family; its spec says the `session.next.*` schemas "remain experimental and +unshipped" (`specs/v2/session.md:173`). Both generations write through the same durable +event log described next, so the storage story below covers both; where behavior differs I +say which generation I am describing. + +## 2. Session storage + +### One SQLite file, written through an event log + +Everything durable lives in a single SQLite database at +`~/.local/share/opencode/opencode.db` (`packages/core/src/database/database.ts:46-54`, +data directory from `packages/core/src/global.ts:11`). An earlier generation stored +sessions as JSON files under a `storage/` directory with keys like +`session//.json` and `part//.json`; that module +still exists with the migrations that folded those files into the new layout +(`packages/opencode/src/storage/storage.ts:81-211`), but the tables are now the truth. + +The write path is event sourcing. An **aggregate** is the unit that owns an ordered event +history; here the aggregate is the session. Two tables implement it: `EventTable` holds +one row per durable event (id, aggregate id, per-aggregate sequence number, versioned +type, JSON payload) and `EventSequenceTable` holds the latest sequence per aggregate +(`packages/core/src/event/sql.ts`). Publishing a durable event runs one SQLite +transaction that: reads the latest sequence, assigns `seq = latest + 1`, runs every +registered **projector** (a function that updates a read-model table from the event) +inside that same transaction, and inserts the event row +(`packages/core/src/event.ts:236-353`). The projection can never disagree with the log +because they commit together. + +The same machinery gives them idempotent replay. Re-committing an event whose sequence +already exists succeeds silently if id, type, and payload match the stored row, and dies +with "Replay diverged" if they do not (`event.ts:262-290`). Sequences must be gapless +(`event.ts:294-302`), and an aggregate can carry an owner id so only one workspace may +extend a replicated history (`event.ts:254-260, 291-293`). This is the foundation of +their cross-machine sync: a client ships serialized events to another node, which replays +them into its own log (`packages/opencode/src/server/routes/instance/httpapi/handlers/sync.ts:35-60`). + +### What is durable and what is live-only + +The event family draws a deliberate line. Boundary events are durable and carry the full +value; streaming fragments are transient bus traffic. The schema says it in a comment: +"Stream fragments are live-only; Text.Ended is the replayable full-value boundary" +(`packages/schema/src/session-event.ts:208-218`). So `text.started`, `text.ended` (with +the complete text), `tool.called` (with full input), `tool.success`, `tool.failed`, +`step.started`, `step.ended`, `compaction.ended` are durable rows; `text.delta`, +`reasoning.delta`, `tool.input.delta` are published to live subscribers only and never +touch the log. The shipped v1 events follow the same split: `message.updated` and +`message.part.updated` are durable (`packages/schema/src/v1/session.ts:502-507`), while +`message.part.delta` is not (`v1/session.ts:632-641`). Our runner reaches the same shape +operationally by coalescing delta families before persisting +(`services/runner/src/sessions/persist.ts:160-329`); OpenCode makes it a schema-level +contract instead of a persistence-time optimization. + +### The projections + +The projectors maintain the read model (`packages/core/src/session/projector.ts`): + +- `session`: one row per session with title, slug, project id, parent id (subagent + sessions are child sessions), directory, agent, model, share URL, revert state, a + per-session permission ruleset override, archived timestamp, and running cost and token + totals that projectors increment as usage events land + (`packages/core/src/session/sql.ts:22-66`, `projector.ts:90-110`). +- `message` and `part` (v1): one row per message and per part, each a JSON blob keyed by + ULID-style ascending ids (`sql.ts:68-98`). +- `session_message` (v2): one row per message with a **unique `(session_id, seq)` + index**, so the projected conversation carries its durable event order + (`sql.ts:119-138`). +- `session_input` (v2): the prompt inbox, covered in section 4. + +There is no session-list JSON and no separate title store; the session row is the header, +and a `title` agent renames it asynchronously after the first turn +(`packages/opencode/src/session/prompt.ts:193-253`). + +### One store serves both the display and the model + +This is the answer to our "is the model-facing conversation a second store" question: no. +The same `message` and `part` rows the UI renders are translated into provider messages +right before each model call by `MessageV2.toModelMessages` +(`packages/opencode/src/session/message-v2.ts:290-374`). The translation handles the +awkward cases inline: a completed tool part becomes a tool result with truncated output; a +tool part still marked `pending` or `running` (which after a crash means the process died +mid-call) becomes a tool error reading "[Tool execution was interrupted]" so no dangling +`tool_use` block ever reaches Anthropic (`message-v2.ts:349-360`); reasoning survives only +while the model matches the one that produced it. Provider-native context caching is +handled by prompt-cache keys, not by preserving a native transcript. + +### Continuation after a restart + +Because the store is the conversation, restart continuation is trivial: there is nothing +to resume. Liveness is purely in-memory (`SessionStatus` is a Map, +`packages/opencode/src/session/status.ts:26-48`; the v2 coordinator's active set "is +runtime state and is empty after a process restart", `specs/v2/session.md:169`). The next +prompt reads projected history and runs. The only restart work is reconciliation of +half-open state: the v2 runner, before assembling a request, durably fails any tool still +projected as `running` from a previous process with "Tool execution interrupted" +(`packages/core/src/session/runner/llm.ts:119-139`, spec `specs/v2/session.md:50`), and +the v1 translation layer produces the interrupted-tool error shown above. Compare our +three-tier ladder (warm pool, cold native `session/load` via the turns ledger, transcript +replay, `architecture.md` section 3): their entire ladder collapses to "read your own +rows" because no external process holds a better copy. + +Two features fall out of owning the store that we do not have at all. **Fork** copies a +session's messages and parts up to a message id into a fresh session with new ids +(`packages/opencode/src/session/session.ts:693-734`). **Revert** stages a boundary +message, and committing it deletes every projected message and inbox row past that +boundary and resets the context epoch (`projector.ts:415-453`), giving "rewind the +conversation" as a first-class verb. + +## 3. The approval flow end to end + +### Rules first, humans second + +A **permission ruleset** is an ordered list of rules `{permission, pattern, action}` with +actions `allow`, `deny`, or `ask`; last matching rule wins and the default is `ask` +(`packages/opencode/src/permission/index.ts:28-38`). Rules come from the agent's config, +from a per-session override stored on the session row, and from rules approved earlier in +the instance. A tool that wants to act calls `ctx.ask` with the permission name, the +concrete patterns, and an `always` list of generalized patterns +(`packages/opencode/src/session/tools.ts:81-89`). The bash tool derives those generalized +patterns with a hand-built arity dictionary that knows `git checkout main` generalizes to +`git checkout *` but `npm run dev` generalizes to `npm run dev` and not `npm *` +(`packages/opencode/src/tool/shell.ts:270-286, 408-409`, +`packages/opencode/src/permission/arity.ts`). + +If every pattern evaluates to `allow`, the tool proceeds with no human involved. If any +evaluates to `deny`, the tool fails immediately with a denied error. Only `ask` reaches a +human (`permission/index.ts:67-107`). + +### The pending map and the one-shot answer + +An ask creates an in-memory entry: the request object plus an Effect `Deferred`, which is +a one-shot promise the asking tool fiber awaits (`permission/index.ts:18-25, 98-107`). +The entry goes into a Map keyed by request id, and a transient `permission.asked` event is +broadcast. The request carries `tool: {messageID, callID}`, so every client can anchor +the approval card on the exact tool call it gates (`tools.ts:86`). + +**Multiple simultaneous approvals are structurally free.** The pending store is a keyed +map; every concurrently executing tool call blocks its own fiber on its own deferred; the +UI holds a per-session sorted list of pending requests fed by asked and replied events +(`packages/app/src/context/global-sync/event-reducer.ts:330-357`). Nothing serializes +asks, and nothing anywhere waits for "all cards settled". + +The answer is one HTTP call naming one request id: +`POST /session/:sessionID/permissions/:permissionID` with reply `once`, `always`, or +`reject` (`packages/opencode/src/server/routes/instance/httpapi/handlers/session.ts:362-380`), +or the instance-level `POST /permission/:requestID` +(`handlers/permission.ts:16-38`). Reply semantics +(`permission/index.ts:109-167`): + +- `once` resolves that one deferred and the tool proceeds. +- `always` resolves it, appends the request's `always` patterns as allow rules to the + instance's approved set, and then walks the remaining pending map: any other pending + request in the same session now fully covered by the new rules is auto-approved and a + `permission.replied` event is emitted for it. Approving "always run git" settles the + three other git cards on screen in the same call. In v2 the saved rules become durable + per-project rows (`packages/core/src/permission/saved.ts`, + `packages/core/src/permission.ts:250-256`). +- `reject` fails that deferred and then **rejects every other pending request in the + session** (`permission/index.ts:129-139`; same in v2, `permission.ts:237-247`). Their + chosen policy is that one rejection means "stop what you are doing", and the loop halts + (a declined ask is treated as a user-initiated stop, not as model-visible tool output, + `packages/core/src/session/runner/llm.ts:144-150, 297-301`). +- `reject` with a message becomes a `CorrectedError` carrying the text, which flows back + to the model as feedback instead of a bare denial (`permission/index.ts:121-127`). + +### What is durable about an approval: nothing, on purpose + +The pending map is process memory. The `permission.asked` and `permission.replied` events +are defined without the `durable` option, so they never enter the event log +(`packages/schema/src/v1/permission.ts:61-70`, +`packages/schema/src/permission.ts:44-53`). A client that attaches mid-ask discovers +pending requests from `GET /permission` (`handlers/permission.ts:12-14`), which reads the +live map. If the server dies while an ask is pending, a finalizer fails every deferred +(`permission/index.ts:54-61`), the tool call errors, and the durable trace the next +reader sees is the tool part's terminal state, not an approval record. + +That is the deep contrast with our interactions plane. OpenCode can afford undurable +approvals because the asker, the executor, and the answer route all live in one process +and the turn simply stays open while the human thinks; the durable answer is the tool +call's own status transition, exactly the Zed shape. Agenta cannot: our gate and executor +are in different processes, the browser turn closes when we park, answers may arrive from +a webhook days later, and we owe tenants an audit trail of who approved what. Our +`session_interactions` rows with stored verdicts, and the `interaction_response` record +(#5382 lane), have no OpenCode equivalent because OpenCode does not have our problem. + +## 4. Cancel, steer, and multi-client attach + +### Cancel is a fiber interrupt with honest bookkeeping + +`POST /session/:id/abort` calls `SessionPrompt.cancel`, which interrupts the running +fiber (`handlers/session.ts:232-235`, `packages/opencode/src/session/run-state.ts:77-86`, +`packages/opencode/src/effect/runner.ts:171-201`). Because the loop is in-process, the +signal takes effect immediately; there is no heartbeat-granularity delay like our +30-second `is_current_turn` path (`architecture.md` gap 5). The cleanup then writes an +honest durable ending: every open tool call gets status `error`, error text "Tool +execution aborted", and `metadata.interrupted = true`, with its accumulated partial +output preserved in metadata (`packages/opencode/src/session/processor.ts:577-596`); the +assistant message is finalized with an aborted error +(`prompt.ts:1203-1211`). Notably, when an interrupted shell command produced output +before the abort, that partial output is replayed to the model as a tool result on the +next turn (`message-v2.ts:326-336`), so the model knows what actually happened. No +synthetic success is ever written, and no "retry the same call" sentinel exists. Their +cancel is everything our db58551b fix plan items 2 through 4 ask for. + +### Steer, shipped version: the loop re-reads the store + +In the shipped v1 path a new prompt during a running turn does not error and does not +queue in memory. `prompt()` writes the user message durably first +(`prompt.ts:1046-1047, 1052-1071`), then calls into the per-session runner, whose +`ensureRunning` joins the already-active run instead of starting a second one +(`effect/runner.ts:117-137`). The loop reloads the full projected history at the top of +every step (`prompt.ts:1092-1094`), and its exit condition is "the last assistant message +finished AND no user message is newer than it" (`prompt.ts:1111-1130`). A message that +landed mid-turn therefore steers the conversation at the next step boundary, and a +message that lands after the turn would have ended keeps the loop alive for another +round. There is no 409, no separate steer verb, and no way for new text to vanish into a +stale resume: the store is the queue. Our defect 6 (a text message during a park consumed +as an approval resume) cannot be expressed in this design. + +### Steer, v2 version: a durable prompt inbox + +The v2 slice makes the same idea explicit data. `session_input` is a durable inbox table; +every prompt is first **admitted** (durable `prompt.admitted` event, row with +`admitted_seq`) and later **promoted** (durable `prompted` event stamps `promoted_seq`, +and only then does the projector write the model-visible user message) +(`packages/core/src/session/sql.ts:140-166`, +`packages/core/src/session/input.ts:41-168`). Delivery is explicit per prompt +(`specs/v2/session.md:155-158`): + +- `steer` promotes at the next safe provider-turn boundary, including inside the current + drain; the runner computes a cutoff sequence and promotes all eligible steers in + admission order (`packages/core/src/session/runner/llm.ts:187-196`, + `input.ts:245-266`). +- `queue` waits until the session would otherwise go idle, then promotes exactly one + queued prompt FIFO (`input.ts:268-288`, `runner/llm.ts:383-406`). + +Interrupt stops execution but "preserves durable inbox rows for a later wake or resume" +(`specs/v2/session.md:22-27`), so cancel and steer compose: you can interrupt, and the +queued text is still there, admitted but unpromoted, visible to clients as queued input +(`specs/v2/session.md:35-37`). A per-process coordinator serializes execution per session +while letting sessions run concurrently, with joins and coalesced wakes +(`packages/core/src/session/run-coordinator.ts:5-15, 67-101`). + +### Attach: one event stream, snapshot endpoints, and a durable cursor + +Every client follows every session the same way: subscribe to `GET /event`, one SSE +stream carrying all bus events for the instance, then fetch snapshots (session list, +messages, pending permissions) through plain endpoints. The handler registers its +listener eagerly before the response body starts, with a comment explaining that events +published while the stream is starting cannot be lost +(`packages/opencode/src/server/routes/instance/httpapi/handlers/event.ts:28-31`). The +client applies events onto its stores with a reducer and refetches snapshots when the +stream reconnects (`packages/app/src/context/server-sync.tsx:388-393`, +`packages/app/src/context/global-sync/event-reducer.ts`). Attaching mid-turn needs no +special verb: the snapshot contains the partially-built assistant message (parts are +durably updated at every boundary), and the stream delivers deltas from now on. Several +clients attach concurrently; nobody holds a watcher lock; the TUI, the web app, and the +desktop render the same turn live. + +The v1 stream has no replay cursor (reconnect means refetch). The v2 contract closes that +gap precisely: `sessions.events({sessionID, after})` replays durable events after an +aggregate sequence and then tails live commits, and the implementation registers the +wake signal **before** the historical read so the replay-to-tail handoff cannot miss a +commit (`packages/core/src/event.ts:565-604`, `specs/v2/session.md:175-183`). Live-only +deltas are deliberately excluded from the replayable stream and can never advance the +cursor. A finite paged variant exists at `GET /api/session/:id/history`. + +Two cross-machine features sit on top. **Share** pushes session, message, and part +payloads to a cloud viewer keyed by a share id and secret, batched from bus events +(`packages/opencode/src/share/share-next.ts`). **Sync** replays serialized durable events +onto another node's log with owner claims, which is how a session moves between a laptop +and their cloud workspace (`handlers/sync.ts:35-91`). + +## 5. Side by side with our planes and our gaps + +| Agenta plane | OpenCode counterpart | Note | +|---|---|---| +| Records (tracing DB, upsert log) | `EventTable` plus projections in one SQLite file | Theirs is also the model conversation; ours is display-only because the harness owns the model copy | +| Turns ledger (`session_turns`) | None; nearest is the per-session `seq` in `EventSequenceTable` | The ledger exists to map to a harness-native session id; with no harness there is nothing to map | +| Streams row + Redis nest | In-memory runner map and status events | Single node, one writer per session; clustered ownership is on their future list (`specs/v2/session.md:109, 185`) | +| Interactions (durable gates, verdicts) | In-memory pending map + `permission.list` | No durable approval, no webhook path, no audit | +| Mounts (object store + geesefs) | The host filesystem | Their bash "is not sandboxed" by design (`specs/v2/session.md:204`) | +| Keepalive pool (parked processes) | Nothing to park; the server is the process | The open turn plays the role of our warm park | + +What they avoid structurally: the whole continuation ladder (no native session file to +trust or invalidate), heartbeat-latency cancel (in-process interrupt), the +answered-cards-resurrect bug class (pending truth is queryable and tool status is the +durable answer), and the split between transcript and model context (one store, one +translation). + +What they lack that we have and need: multi-tenant projects and authorization, remote +sandboxed execution with credential isolation, harness choice (their loop is their only +agent; we host Pi and Claude Code), durable approvals with webhook delivery and +re-invocation references, an audit trail, ingest quotas, and any notion of a runner fleet. +Their single-file, single-process shape is the reason their session code is small; it is +also the reason it cannot be our architecture. + +Against our two open gaps: gap 5 (cancel and steer) is where their design is most +instructive, because both halves are data problems in their system, not signaling +problems: cancel is an immediate interrupt followed by honest durable settlement, and +steer is a durable inbox with explicit delivery semantics. Gap 6 (live mid-turn attach) +is solved by the combination we lack: prompt boundary persistence plus a per-session +durable cursor plus a replay-then-tail read path with the wake registered before replay. + +## 6. Learnings + +**Adopt.** + +1. A per-session monotonic sequence and a replay-then-tail read contract, for gap 6. Our + records plane already persists promptly but orders by ingest time and restarts + `record_index` per execution (`architecture.md` section 2). Adding a per-session + sequence at ingest, an `after` cursor on the records query, and a tail wired to the + ingest path gives exactly their `sessions.events` contract + (`packages/core/src/event.ts:585-604`). Their two details worth copying verbatim: + register the live subscription before the historical read so the handoff cannot drop + an event, and keep live-only deltas out of the replayable stream so a cursor always + equals a persisted row. +2. Steer and queue as durable inbox rows with admitted and promoted states, for gap 5 and + for db58551b defect 6. "Is this request an answer to the gates or new work" stops + being a dispatch heuristic and becomes a field: a prompt with `delivery: steer` + promotes at the next safe boundary, a `queue` prompt waits for idle, and an approval + answer is neither. Our runner's parked dispatch would then consume answers from the + interactions plane and prompts from the inbox, and could never again consume new text + as a stale resume (`packages/core/src/session/sql.ts:140-166`, + `specs/v2/session.md:155-171`). +3. Terminal-evidence discipline on tool calls, confirming the db58551b fix plan. Their + only writers of terminal tool state are real settlement, explicit abort cleanup that + marks calls interrupted while preserving partial output + (`processor.ts:577-596`), and a drain-start reconciliation that durably fails calls + left `running` by a dead process (`runner/llm.ts:119-139`). Nothing ever fabricates a + success, and the interrupted marker even feeds the partial output back to the model + (`message-v2.ts:326-336`). Our post-pause sweep should converge on exactly this set of + writers. + +**Adapt.** + +4. The durable-boundary versus live-delta split as a schema contract. We already coalesce + deltas at persist time; declaring per record type whether it is replayable (with the + full value) or ephemeral would let the attach endpoint and the audit story reason + about the log instead of about persistence behavior + (`packages/schema/src/session-event.ts:208-218`). +5. Approval generalization at answer time. Their `always` reply saves wildcard rules + derived per tool (the bash arity dictionary) and immediately settles other pending + cards the new rule covers (`permission/index.ts:145-166`, `shell.ts:408-409`). For us + this maps to an "always allow this command shape for this session or agent" option on + the approval card, resolved in the runner's policy layer; it needs adaptation because + our rules must be tenant-scoped and durable rather than instance memory. +6. Reject with a message as model feedback. Their reject can carry text that reaches the + model as a correction instead of a bare denial (`permission/index.ts:121-127`). ACP + permission outcomes can carry equivalent context; our deny path currently sends only + the verdict. +7. Their reject-cascade policy (one rejection cancels every pending gate in the session, + `permission/index.ts:129-139`) is a defensible product answer to partial answer sets: + rejection means stop. If we adopt it, it belongs in the runner's gate bookkeeping, and + it would have turned the db58551b partial-answer deadlock into a clean stop. + +**Does not transfer.** + +8. In-memory pending approvals. Holding the turn open on a process-lifetime deferred is + only safe when the asker, executor, and answer route share a process and the client + connection model tolerates long-open turns. Our gates outlive browser requests, + arrive from webhooks, and must survive runner restarts; the durable interactions plane + stays. +9. Event-plus-projection in one transaction. Their consistency guarantee depends on one + SQLite file owned by one process (`event.ts:236-353`). We are cross-database and + cross-process by necessity; our nearest equivalent remains idempotent upserts plus the + sequence from learning 1. +10. No liveness plane. They need no Redis nest because exactly one process can run a + session and clients never contend for execution. Our owner, alive, and running locks + exist because many runner replicas serve many tenants; their own spec lists + "clustered Session execution ownership and stale-runtime fencing" as unsolved future + work (`specs/v2/session.md:109, 185`), which is a useful reminder that our streams + plane is us already paying a cost they have merely deferred. + +## Verdict on the choices we just made + +Per-card dispatch is vindicated: their reply names one request id, one click delivers one +answer, and no code path anywhere waits for sibling cards +(`handlers/session.ts:362-380`). Persisting the answer half of a gate is vindicated in an +indirect way: their durable answer is the tool call's status transition, and every rebuilt +client reads settled state rather than reconstructing "waiting" from requests, which is +the same invariant our `interaction_response` record restores. The turns ledger is +neither vindicated nor contradicted: OpenCode simply has no harness-native session id to +map, which confirms the ledger is an artifact of driving external harnesses, not a +conversation primitive; the conversation primitive their design argues for is the +per-session durable sequence. Park and warm resume is the one place their design pushes +back: they never park because the open in-process turn is their warm state, and their v2 +direction is to make every resumable thing a durable row rather than a held process. Our +keepalive pool remains justified by sandbox and model-context reuse economics, but the +inbox learning says the dispatch decisions around a parked process should be driven by +durable data, not by what happens to be in the pool. From ca104f0ab2c0e4e0ec0fbbeac3295c395d9dfe99 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Tue, 21 Jul 2026 17:13:06 +0200 Subject: [PATCH 14/15] feat(runner): every approval gate leaves a durable interaction row --- .../sessions/test_interactions_dispatcher.py | 45 ++++++ .../sessions-takeover/arda-handoff.md | 55 ++++--- services/runner/src/sessions/interactions.ts | 4 +- .../unit/session-keepalive-approval.test.ts | 142 ++++++++++++++++++ 4 files changed, 218 insertions(+), 28 deletions(-) diff --git a/api/oss/tests/pytest/unit/sessions/test_interactions_dispatcher.py b/api/oss/tests/pytest/unit/sessions/test_interactions_dispatcher.py index 6d2c4c7f7a..8088fb29ee 100644 --- a/api/oss/tests/pytest/unit/sessions/test_interactions_dispatcher.py +++ b/api/oss/tests/pytest/unit/sessions/test_interactions_dispatcher.py @@ -5,6 +5,9 @@ from unittest.mock import AsyncMock, MagicMock +from oss.src.apis.fastapi.sessions.models import SessionInteractionCreateRequest +from oss.src.core.sessions.interactions.dtos import SessionInteractionKind + from oss.src.tasks.asyncio.sessions.interactions_dispatcher import ( InteractionsDispatcher, ) @@ -31,6 +34,19 @@ def _make_interaction(*, with_refs=True): ) +def test_create_request_accepts_omitted_workflow_references(): + request = SessionInteractionCreateRequest( + session_id="sess-no-refs", + turn_id="turn-no-refs", + token="token-no-refs", + kind=SessionInteractionKind.user_approval, + data={"request": {"tool": "bash", "args": {"command": "pwd"}}}, + ) + + assert request.data is not None + assert request.data.references is None + + async def test_respond_fallback_calls_invoke_when_no_dispatch_fn(): interaction = _make_interaction() project_id = uuid4() @@ -60,6 +76,35 @@ async def test_respond_fallback_calls_invoke_when_no_dispatch_fn(): assert invoke_kwargs["user_id"] == user_id +async def test_respond_without_references_builds_a_safe_reference_less_request(): + interaction = _make_interaction(with_refs=False) + project_id = uuid4() + user_id = uuid4() + + interactions_service = MagicMock() + interactions_service.fetch_interaction = AsyncMock(return_value=interaction) + + workflows_service = MagicMock() + workflows_service.invoke_workflow = AsyncMock(return_value=SimpleNamespace()) + + worker = InteractionsDispatcher( + workflows_service=workflows_service, + interactions_service=interactions_service, + ) + + await worker.respond( + project_id=project_id, + user_id=user_id, + interaction_id=interaction.id, + answer={"approved": True}, + ) + + workflows_service.invoke_workflow.assert_awaited_once() + invoke_request = workflows_service.invoke_workflow.await_args.kwargs["request"] + assert invoke_request.references is None + assert invoke_request.session_id == interaction.session_id + + async def test_respond_detached_calls_dispatch_fn_not_invoke(): interaction = _make_interaction() project_id = uuid4() diff --git a/docs/design/agent-workflows/projects/sessions-takeover/arda-handoff.md b/docs/design/agent-workflows/projects/sessions-takeover/arda-handoff.md index 5c16558867..d8ee9682f4 100644 --- a/docs/design/agent-workflows/projects/sessions-takeover/arda-handoff.md +++ b/docs/design/agent-workflows/projects/sessions-takeover/arda-handoff.md @@ -239,28 +239,33 @@ Out of scope for now: session deletion and live mid-turn attach. - PR #5382 and PR #5436 descriptions, for the exact shape of what merges beneath you. Your own PR #5426 description, for the record of what you built and where each piece landed. -## 6. Two questions Mahmoud wants your read on - -Neither is settled. Your view as the frontend owner is why he is asking. - -### 6.1 The streams-table rename - -The table named `session_streams` stores no streamed frames; the architecture document opens -by killing exactly that mental model (architecture document, page 1, mental model 2). It -holds liveness and ownership, and it now also carries the session identity, the `name` and -`description` header. The name misleads on both counts. The open question is whether to rename -the table so its name matches what it holds. You read and write this row from the frontend -(session fetch, and the rename UI you are about to wire in task 4), so the churn and the -clarity both land partly on you. Weigh in on whether the rename is worth it and what the name -should be. - -### 6.2 The interactions guard ruling - -The runner refuses to create an interaction row when the run context carries no -`workflow_revision` reference, because the respond path could not re-invoke anything without -it. The consequence is that gates in that state are answerable in-band only and are invisible -to any interactions-plane inbox; a real QA session had approval records but zero interaction -rows for exactly this reason (architecture document, section 5, "Sharp edges"). The open -question is whether to keep that guard as is, accepting that some gates never get a durable -interaction row, or to relax it. It matters to any future approval-inbox surface, which is -your territory. Weigh in on which way it should go. +## 6. Where the open questions landed, and the one Mahmoud wants your proposal on + +Two questions this document originally left open were ruled by Mahmoud on 2026-07-21, so +you inherit the answers rather than the debates. + +**The streams table keeps its name.** `session_streams` stays as it is. Since the name +misleads (the table stores no streamed frames; it holds liveness, ownership, and the +session's name and description header), keep the architecture document nearby when working +against it: the row is the session header plus status, and only the service read that +overlays the live Redis state tells the truth about liveness. + +**The interactions guard is relaxed: every gate gets a durable row.** Today the runner +skips creating the interaction row when the run context carries no `workflow_revision` +reference, which leaves those gates invisible to any future inbox and without an audit +trail. The ruling: create the row anyway, with the reference left empty when absent. The +runner-side change is queued on our side; what it means for you is that the future inbox +must tolerate and sensibly display rows that cannot be attributed to an agent (group them +under the session). + +### The question that is yours: what should rejecting one card do to its siblings? + +Today rejecting an approval card rejects only that card; other pending cards stay open. +OpenCode ships the opposite policy: one rejection cancels every pending card in the +session, on the reasoning that a rejection means "stop what you are doing" rather than "no +to this one thing" (see the OpenCode comparison in this folder, the approval flow section). +That policy would also have turned the db58551b incident's dead conversation into a clean +stop. Mahmoud's ruling is that this is a product-feel call on your surface, and he wants +your proposal: keep per-card rejection, adopt the cascade, or something between (for +example, cascade only when the rejection carries no message). Bring the UX reasoning with +the proposal. diff --git a/services/runner/src/sessions/interactions.ts b/services/runner/src/sessions/interactions.ts index 6fd2f44b11..2cd9bb615d 100644 --- a/services/runner/src/sessions/interactions.ts +++ b/services/runner/src/sessions/interactions.ts @@ -18,9 +18,7 @@ type Reference = { id?: string; slug?: string; version?: string }; export type InteractionData = { request?: { tool: string; args: unknown }; - // The workflow references that identify which revision THIS turn is running, so the - // respond invoke re-resolves the SAME workflow. We store the references (pointers), not - // the revision data itself — respond resolves the live revision from them at invoke time. + // Optional attribution for out-of-band re-invocation; inbox/audit rows exist without it. references?: Record; }; diff --git a/services/runner/tests/unit/session-keepalive-approval.test.ts b/services/runner/tests/unit/session-keepalive-approval.test.ts index 245c1ebfa9..6035c7f232 100644 --- a/services/runner/tests/unit/session-keepalive-approval.test.ts +++ b/services/runner/tests/unit/session-keepalive-approval.test.ts @@ -1585,6 +1585,148 @@ describe("runTurn: real approval park + respondPermission resume", () => { await env.destroy(); }); + it("creates and resolves a durable gate row without workflow context", async () => { + const posted: Array<{ url: string; body: Record }> = []; + const fetchSpy = vi + .spyOn(globalThis, "fetch") + .mockImplementation(async (input, init) => { + posted.push({ + url: String(input), + body: JSON.parse(init?.body as string) as Record, + }); + return new Response(JSON.stringify({ ok: true }), { status: 200 }); + }); + + try { + const { calls, deps, captured } = pausableHarness(); + deps.hydrateHarnessSessionFromDurable = async () => {}; + const noWorkflowRequest: AgentRunRequest = { + ...engineReq, + ...auth, + sessionId: "sess-no-workflow", + turnId: "turn-no-workflow-start", + }; + const acquired = await acquireEnvironment(noWorkflowRequest, deps); + assert.equal(acquired.ok, true); + if (!acquired.ok) return; + const env = acquired.env; + + const firstTurn = runTurn( + env, + noWorkflowRequest, + undefined, + undefined, + { approvalParkMode: true }, + ); + await flush(); + captured.onEvent!( + updateEvent({ + sessionUpdate: "tool_call", + toolCallId: "tc-no-workflow", + title: "commit", + rawInput: { message: "hi" }, + }), + ); + captured.onPermissionRequest!({ + id: "perm-no-workflow", + availableReplies: ["once", "reject"], + toolCall: { + toolCallId: "tc-no-workflow", + name: "commit", + rawInput: { message: "hi" }, + }, + }); + const paused = await firstTurn; + assert.equal(paused.stopReason, "paused"); + await flush(); + + const createCall = posted.find(({ url }) => + url.endsWith("/sessions/interactions/"), + ); + assert.ok(createCall); + assert.equal(createCall.body["session_id"], "sess-no-workflow"); + assert.equal(createCall.body["turn_id"], "turn-no-workflow-start"); + const createData = createCall.body["data"] as Record; + assert.equal("references" in createData, false); + + const parked = env.parkedApprovals.get("tc-no-workflow"); + assert.ok(parked); + assert.ok(parked.promptPromise); + env.clearTurn(); + const resumedTurn = runTurn( + env, + approveResume(true, { + sessionId: "sess-no-workflow", + turnId: "turn-no-workflow-resume", + }), + undefined, + undefined, + { + approvalParkMode: true, + resume: { + decisions: [ + { + permissionId: parked.permissionId, + reply: "once", + toolCallId: parked.toolCallId, + toolName: parked.toolName, + args: parked.args, + interactionToken: parked.interactionToken, + promptPromise: parked.promptPromise, + }, + ], + carriedForward: [], + }, + }, + ); + for ( + let attempt = 0; + attempt < 10 && + !posted.some(({ url }) => + url.endsWith("/sessions/interactions/transition"), + ); + attempt += 1 + ) { + await flush(); + } + + const resolveCall = posted.find(({ url }) => + url.endsWith("/sessions/interactions/transition"), + ); + assert.ok( + resolveCall, + JSON.stringify({ posted, permissionReplies: calls.permissionReplies }), + ); + assert.deepEqual(resolveCall.body, { + session_id: "sess-no-workflow", + token: parked.interactionToken, + status: "resolved", + resolution: { + verdict: "approved", + tool_call_id: "tc-no-workflow", + }, + }); + + captured.onEvent!( + updateEvent({ + sessionUpdate: "tool_call_update", + toolCallId: "tc-no-workflow", + status: "completed", + content: "committed", + }), + ); + calls.resolvePrompt!({ + stopReason: "complete", + usage: { inputTokens: 1, outputTokens: 1 }, + }); + const resumed = await resumedTurn; + assert.equal(resumed.ok, true); + await env.destroy(); + } finally { + fetchSpy.mockRestore(); + } + }); + it("preserves a denied call failed frame while a sibling gate is carried", async () => { const { calls, deps, captured } = pausableHarness(); const acquired = await acquireEnvironment(engineReq, deps); From d5155fda561accc2b9786d3c7c418cab1f1c9e53 Mon Sep 17 00:00:00 2001 From: Mahmoud Mabrouk Date: Tue, 21 Jul 2026 17:35:21 +0200 Subject: [PATCH 15/15] fix(runner): thread the resolved execution id into every request consumer --- services/runner/src/server.ts | 6 ++++++ .../unit/session-keepalive-dispatch.test.ts | 17 +++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/services/runner/src/server.ts b/services/runner/src/server.ts index c8e6d5ab8c..93beeb2591 100644 --- a/services/runner/src/server.ts +++ b/services/runner/src/server.ts @@ -334,6 +334,9 @@ export async function runWithKeepalive( ): Promise { const { engine, pool, config, clientGone } = ctx; const sessionId = request.sessionId?.trim(); + // Every execution carries an id: callers that omit `turnId` get one minted here, so the + // turns-ledger append and interaction rows are never written without their execution id. + request.turnId = request.turnId?.trim() || randomUUID(); // Track whether anything reached the client on this streaming edge. A live continuation/resume // that fails AFTER emitting (a partial answer or an error event) must NOT retry cold: the client @@ -912,6 +915,9 @@ async function runAndStreamWithApiBaseResolved( const sessionOwned = isSessionOwned(request); const sessionId = request.sessionId!; const turnId = resolveTurnId(request); + // Write the resolved id back: every downstream reader of `request.turnId` (the turns-ledger + // append, interaction rows) must see the SAME execution id the alive-lock and records use. + request.turnId = turnId; // Diagnostic: surface whether the session-owned persist/alive path is entered and // whether the invoke credential arrived. Empty cred => heartbeat/persist would 401. diff --git a/services/runner/tests/unit/session-keepalive-dispatch.test.ts b/services/runner/tests/unit/session-keepalive-dispatch.test.ts index 659b36f003..04129c0bc4 100644 --- a/services/runner/tests/unit/session-keepalive-dispatch.test.ts +++ b/services/runner/tests/unit/session-keepalive-dispatch.test.ts @@ -216,6 +216,23 @@ function turn2( } describe("runWithKeepalive: park + hit", () => { + it("mints an execution turnId when the request omits one, before the engine runs", async () => { + const made = makeEngine({}); + const seen: Array = []; + const inner = made.engine.runTurn; + made.engine.runTurn = async (env, req, emit, signal, opts) => { + seen.push(req.turnId); + return inner(env, req, emit, signal, opts); + }; + const req = turn1("turnid-session"); + delete (req as Record)["turnId"]; + await runWithKeepalive(req, undefined, undefined, makeCtx(made.engine)); + assert.equal(seen.length, 1); + // The ledger append and interaction rows read request.turnId; a run without one would + // write NULL execution ids (the live-verification defect this pins). + assert.ok(typeof seen[0] === "string" && seen[0].length > 0); + }); + it("calls the live-park hook once for Daytona, never for local", async () => { const daytona = makeEngine({ onParkedLive: true }); const daytonaContext = makeCtx(daytona.engine);