From f44abedc0f3a2de3570dcdf9e1a479970f433eff Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sat, 1 Aug 2026 08:19:44 +0900 Subject: [PATCH 01/30] docs(plan): map the zero-leak state-store unit before touching the stores MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exhaustive 36-store inventory (continuation, blobs, rings, caches, registries, translator-layer accumulators) with per-store bounds, eviction contracts, worst-case math, and blast-radius notes; audited implementation roadmap across five review rounds (ownership matrix, spill-failure tombstone ladder, blob provenance admission, three-way eviction mechanism lock, ACL trust-boundary contract); decade docs 010-070 written to diff level, one PABCD cycle each. The design goal: the strongest theoretical no-leak posture among production TS LLM proxies — held as a hypothesis until the 060 benchmark proves it with a source ledger. --- .../000_state_store_inventory.md | 378 ++++++++++++++++++ .../005_impl_roadmap.md | 171 ++++++++ .../006_roadmap_audit_synthesis.md | 50 +++ .../010_continuation_hard_cap.md | 342 ++++++++++++++++ .../020_blob_and_replay_caps.md | 266 ++++++++++++ .../030_eviction_mechanisms.md | 188 +++++++++ .../035_registry_admission_caps.md | 334 ++++++++++++++++ .../040_app_bytes_observability.md | 254 ++++++++++++ .../050_translator_stream_bounds.md | 255 ++++++++++++ .../055_background_shell_lifecycle.md | 201 ++++++++++ .../060_proxy_benchmark.md | 159 ++++++++ .../070_close_and_push.md | 232 +++++++++++ 12 files changed, 2830 insertions(+) create mode 100644 devlog/_plan/260801_zero_leak_state_stores/000_state_store_inventory.md create mode 100644 devlog/_plan/260801_zero_leak_state_stores/005_impl_roadmap.md create mode 100644 devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md create mode 100644 devlog/_plan/260801_zero_leak_state_stores/010_continuation_hard_cap.md create mode 100644 devlog/_plan/260801_zero_leak_state_stores/020_blob_and_replay_caps.md create mode 100644 devlog/_plan/260801_zero_leak_state_stores/030_eviction_mechanisms.md create mode 100644 devlog/_plan/260801_zero_leak_state_stores/035_registry_admission_caps.md create mode 100644 devlog/_plan/260801_zero_leak_state_stores/040_app_bytes_observability.md create mode 100644 devlog/_plan/260801_zero_leak_state_stores/050_translator_stream_bounds.md create mode 100644 devlog/_plan/260801_zero_leak_state_stores/055_background_shell_lifecycle.md create mode 100644 devlog/_plan/260801_zero_leak_state_stores/060_proxy_benchmark.md create mode 100644 devlog/_plan/260801_zero_leak_state_stores/070_close_and_push.md diff --git a/devlog/_plan/260801_zero_leak_state_stores/000_state_store_inventory.md b/devlog/_plan/260801_zero_leak_state_stores/000_state_store_inventory.md new file mode 100644 index 000000000..e85d73273 --- /dev/null +++ b/devlog/_plan/260801_zero_leak_state_stores/000_state_store_inventory.md @@ -0,0 +1,378 @@ +# Zero-leak state-store inventory + +Audited 2026-08-01 against `dev@a9926d01da5dde052dba2ac6d804a3336edb2f99`. +This read-only roadmap input inherits +`devlog/_plan/260731_macos_rss_retention/062_external_handoff_audit.md`. +The prior unit's stream-path work is out of scope. + +## Scope and verdicts + +This covers app-owned process state that can survive a request/stream, plus +request-scoped state that exists because OpenCodex is a **bidirectional protocol +translator**. Immutable lookup tables, non-escaping locals, weak collections +that do not keep keys alive, and disk files not retained in RAM are excluded. + +- **BOUNDED**: finite production count/bytes, fixed slot, or current-config snapshot. +- **CONDITIONALLY-UNBOUNDED**: one dimension is uncapped, cleanup is lazy, historical + key churn accumulates, or growth is limited only by active concurrency/config. +- **UNBOUNDED**: reachable additions have no count, byte, TTL, or replacement bound. +- **Translation duty**: state required to assemble/correlate/replay protocols. It + must stay but be bounded; deleting it like a pure relay would break the product. + +`S(x)` below means the retained size of variable value `x`. + +## Summary table + +| # | Store | Current bound | Verdict | Translation duty | +|---:|---|---|---|:---:| +| 1 | Responses continuation | 1,000; lazy 1 h TTL; nominal 64 MiB but last entry exempt | **CONDITIONALLY-UNBOUNDED** | yes | +| 2 | Responses replay provenance | weak body-key map | **BOUNDED** | yes | +| 3 | Cursor shared blobs | 4,096; lazy 15 min TTL; no bytes | **CONDITIONALLY-UNBOUNDED** | yes | +| 4 | Cursor context-usage carry | 200 numeric rows; lazy 1 h TTL | **BOUNDED** | yes | +| 5 | Cursor thread overrides | 2,048; lazy 1 h TTL/LRU | **BOUNDED** | yes | +| 6 | Antigravity signature replay | 10,240 sessions; unbounded inner call maps | **CONDITIONALLY-UNBOUNDED** | yes | +| 7 | Request-log ring | 2,000 entries; no bytes | **CONDITIONALLY-UNBOUNDED** | no | +| 8 | Memory watchdog | 360 fixed samples | **BOUNDED** | no | +| 9 | Debug/injection/Claude-debug rings | 2,000 / 2,000 / 20; variable entries | **CONDITIONALLY-UNBOUNDED** | no | +| 10 | Debug subscribers | active listener set; no count | **CONDITIONALLY-UNBOUNDED** | no | +| 11 | Crash fetch ring + inspector counters | 12 traces (URL/stack/rejection strings uncapped — audit R1-4) + five numbers | **CONDITIONALLY-UNBOUNDED** | no | +| 12 | Image normalization cache | 64 MiB payload; zero-weight sentinels uncapped | **UNBOUNDED** | yes | +| 13 | Model/discovery caches | bounded response/provider; historical provider keys uncapped | **CONDITIONALLY-UNBOUNDED** | no | +| 14 | Catalog/warning state | current config plus historical warning keys | **CONDITIONALLY-UNBOUNDED** | no | +| 15 | Usage summary cache | 12 keys; summary model cardinality uncapped | **CONDITIONALLY-UNBOUNDED** | no | +| 16 | API-key rollup | one 60 s current-config snapshot | **BOUNDED** | no | +| 17 | Provider/Codex quota caches | account keyed; historical keys uncapped | **CONDITIONALLY-UNBOUNDED** | no | +| 18 | Codex/Anthropic routing health | account keyed; health cleanup lazy | **CONDITIONALLY-UNBOUNDED** | no | +| 19 | Codex/Anthropic affinity | 2,048 / 2,000; lazy 24 h TTL/LRU | **BOUNDED** | no | +| 20 | Subagent quota-failure records | expired keys never removed | **UNBOUNDED** | no | +| 21 | Pool/combo rotation + cooldown | config keyed; historical/expired keys lazy or permanent | **CONDITIONALLY-UNBOUNDED** | no | +| 22 | OAuth/Codex login and refresh state | provider/flow/active-flight keyed | **CONDITIONALLY-UNBOUNDED** | no | +| 23 | Guardian/reauth/GCP token state | account/source keyed; historical keys possible | **CONDITIONALLY-UNBOUNDED** | no | +| 24 | GUI management sessions | 128; lazy 5 min TTL | **BOUNDED** | no | +| 25 | Current-config registries | whole-map replacement, current catalog/config | **BOUNDED** | partly | +| 26 | Config ownership/PID/warning memos | root/PID/path/history keyed; no general cap | **CONDITIONALLY-UNBOUNDED** | no | +| 27 | Windows ACL memo sets | unique atomic-temp success paths never clear | **UNBOUNDED on Windows** | no | +| 28 | Single-slot diagnostics/jobs | one value/job/outcome per owner | **BOUNDED** | no | +| 29 | Active turns/sockets/workers/slots | cleanup-bound; no registry admission cap | **CONDITIONALLY-UNBOUNDED** | no | +| 30 | Cursor background shells | live child until close; no count/age cap | **UNBOUNDED when enabled** | yes | +| 31 | Cursor per-stream MCP manager | config-sized; disposed on close; no payload/count cap | **CONDITIONALLY-UNBOUNDED** | yes | +| 32 | Vision-description LRU | 256 entries; cached text has no byte cap | **CONDITIONALLY-UNBOUNDED** | yes | +| 33 | Serialized promise tails | one tail reference; pending closure chain has no admission cap | **CONDITIONALLY-UNBOUNDED** | partly | +| 34 | Codex credential-refresh flights | one active promise/grant fingerprint; no admission cap | **CONDITIONALLY-UNBOUNDED** | no | +| 35 | MiMo JWT/client-id cache | fixed slots; JWT value has no byte cap | **CONDITIONALLY-UNBOUNDED** | no | +| 36 | Usage-log management read flight | one active full-log parse/result | **CONDITIONALLY-UNBOUNDED** | no | + +## 1. Responses continuation state + +- **Owner/purpose:** `src/responses/state.ts:6-35,319-334,415-455` stores + expanded input plus authoritative output by response id so + `previous_response_id` works against stateless/routed providers. +- **Growth driver:** one row per remembered turn. Each row is + `inputItems(request.input) + response.output` (`state.ts:444-447`). Arrays are + new; history objects/strings may be shared, while persistence serializes each + full prefix. +- **Bounds:** 1,000 rows, `createdAt` TTL 1 h, nominal 64 MiB + (`state.ts:6-16`). TTL pruning is access-triggered, not periodic. The sole + measurement is `JSON.stringify(entry.items).length` in JS code units; it omits + provider/map/object overhead and counts stringify failure as zero + (`state.ts:52-60`). +- **Eviction:** TTL, then count, then oldest-bytes. The byte loop is + `while (storedResponseBytes > byteCap() && states.size > 1)`, so it never + removes the newest/only row (`state.ts:319-333`). +- **Worst case:** measured retention is `max(64 MiB, S(newest row))`. The newest + row can include a near-256 MiB accepted request plus unbounded streamed output + and base64 items. Idle expired rows remain past 1 h until another access. + Verdict: **CONDITIONALLY-UNBOUNDED**. +- **Disk contract:** persistence is 2 s debounced/single-flight. A serialized + pair over 2 MiB is skipped; newest-first snapshot selection stops at 24 MiB + (`state.ts:252-299`). The timer admits one scheduled write; `persistGate` + serializes overlapping explicit flushes, whose active call stacks can still + grow with unconstrained concurrent `flushResponseState()` callers. Thus + oversized rows remain in RAM but disappear after restart. + `tests/responses-state.test.ts:856-883` pins this exact behavior. +- **Naive-cap blast radius:** evicting/rejecting the newest row makes the next + chained turn a naked delta. Cursor/Kiro lose conversation identity; tool + results lose prior tool metadata; stateless upstreams cannot resolve history. + Use explicit admission/miss semantics and bounded blob/spill references, not + silent replay truncation. + +### Exact continuation contract + +`tests/responses-state.test.ts` was read fully (1,016 lines): + +1. Replay prepends prior input and output, including tool-call metadata needed to + parse later `function_call_output` (`:67-179`). +2. Completed and only `incomplete/max_output_tokens` partial output are replayable; + failed/content-filtered partials are not (`:90-155`). +3. Ordinary `store:false` skips storage, but forced storage is required for + Cursor, Kiro, and stateless passthrough continuation (`:181-217,495-533`). +4. Empty terminal output may be reconstructed from `output_item.done`. Non-empty + terminal output wins; malformed items are ignored; duplicate indices are + last-write-wins; cap-tainted reconstruction is never stored (`:219-493`). +5. Weak replay provenance distinguishes restored prefix from new suffix, avoiding + duplicate developer guidance and false compaction resets (`:335-397,913-957`; + `state.ts:79-82`). +6. Provider-keyed Cursor/Kiro state survives restart. Cursor conversation id + survives client-tool suspension, while `checkpointUsable` becomes false for a + function-call output (`state.ts:436-452`; tests `:782-804,885-911`). +7. Replacement, count eviction, TTL eviction, and restart load must keep byte + accounting exact (`:535-658`). +8. Snapshot recovery is best-effort; v1 migrates; stale/corrupt snapshots are + ignored; valid small chains survive restart (`:660-883`). +9. Metrics are observe-only and may not load, prune, evict, or reserialize + (`:959-1015`; `state.ts:371-408`). + +The “newest link survives” assertion (`tests/responses-state.test.ts:535-555`) is +the policy conflicting with a hard ceiling. Redesign must replace it explicitly. + +## 2. Cursor and translation replay stores + +| Store | Growth and bounds | Worst case / verdict | Naive-cap blast radius | +|---|---|---|---| +| Shared blobs (`src/adapters/cursor/native-exec.ts:75-136,202-227`) | Content-addressed roots/turn/KV bytes shared across streams. Re-store refreshes order. 4,096 rows; lazy 15 min TTL/count eviction; no per-blob/aggregate bytes. | `sum(S(blob_i)), i<=4096`. Connect allows payload length `2^32-1` (`cursor/framing.ts:1-5`), nearly 16 TiB theoretical at 4,096 rows; no finite app byte bound. **CONDITIONALLY-UNBOUNDED**. | Missing blobs break Cursor hydration. Preserve hash lookup/live refresh; add per-blob+aggregate admission and explicit miss. | +| Context usage (`cursor/protobuf-events.ts:17-123`; singleton `live-transport.ts:73`) | Absolute token total by conversation; 200 rows, lazy 1 h TTL/oldest prune. | `200 × (id + numbers)`. **BOUNDED**. | Eviction loses absolute carry-forward and can report tiny output as context; retain compaction reset/rekey. | +| Thread override (`cursor/thread-continuity.ts:9-62`) | Recovered thread/scope→conversation; 2,048, lazy 1 h TTL/LRU. | `2048 × two ids`. **BOUNDED**. | Eviction can revert to stale deterministic conversation id. | +| Antigravity replay (`src/adapters/google-antigravity-replay.ts:13-24,60-130`) | Outer model/session rows; inner map per distinct function-call canonical args. Outer lazy 1 h TTL/max 10,240; no inner count/bytes. | `<=10240 × unbounded calls/session × (canonical args + signature)`. **CONDITIONALLY-UNBOUNDED**. | Clearing active identities causes upstream invalid-signature 400. Bound bytes/calls while retaining recent referenced calls and clear-on-invalid. | + +The Cursor external selected-root budget is 192 roots/512 KiB +(`cursor/protobuf-request.ts:54-60,233-305`), but all candidate roots are stored +before selection (`:195-230`). Native step blobs (`:340-404`) and KV +`setBlobArgs` do not inherit that budget. + +## 3. Logs, metrics, and image cache + +| Store | Bounds/eviction and worst case | Verdict | Blast radius | +|---|---|---|---| +| Request log (`src/server/request-log.ts:150-154,218-246`) | Latest 2,000, hydrated/shifted. No byte cap: `2000 × S(RequestLogEntry)`; nested attempts and several strings have no aggregate accounting. | **CONDITIONALLY-UNBOUNDED** | Preserve retry/failover attribution; normalize per-entry bytes rather than silently dropping attempts. | +| Watchdog (`src/server/memory-watchdog.ts:53-60,102-155`) | Default 360 fixed samples; splice on tick; singleton replacement stops prior timer. | **BOUNDED** | Lower count only weakens trend diagnosis. | +| Provider debug (`src/lib/debug-log-buffer.ts:10-35`) | 2,000 unbounded-length lines; active listener set has explicit unsubscribe but no count cap. | **CONDITIONALLY-UNBOUNDED** | Preserve safe diagnostic endings and GUI tailing; cap line bytes/admission. | +| Injection debug (`src/lib/injection-debug-log.ts:10-27`) | 2,000 unbounded-length lines. | **CONDITIONALLY-UNBOUNDED** | Same; retain capture semantics. | +| Claude debug (`src/claude/inbound-debug.ts:40-106`) | 20 rows; metadata key arrays/model/beta strings locally uncapped, globally body-capped. | **CONDITIONALLY-UNBOUNDED** | Preserve privacy tags and flag-off clearing. | +| Crash ring (`src/lib/crash-guard.ts:206-245`) / inspector counters (`src/server/relay.ts:18-49`) | 12 traces, but URL/stack-origin/rejection strings per trace are uncapped (audit R1-4); counters are a fixed five-number object. | **CONDITIONALLY-UNBOUNDED** (value bytes) | Truncate retained strings (035); too-small caps lose Bun-failure provenance. | +| Image normalize (`src/adapters/anthropic-image-normalize.ts:96-143`) | Encoded data has 64 MiB LRU. `\"pass\"`/`\"miss\"` are assigned zero size, so unique sentinel keys never trigger eviction; no count/TTL. | **UNBOUNDED** Map metadata | Deleting cache repeats decode/encode for accumulated screenshots. Add count/key-overhead cap including sentinels; retain encoded LRU. | + +## 4. Catalog, usage, and quota caches + +| Store | Growth/bounds/eviction | Verdict | Blast radius | +|---|---|---|---| +| Model cache/status (`src/codex/model-cache.ts:42-56,114-147`) | Last-good arrays + status per provider. Fresh 5 min/failure cooldown 30 s, but stale arrays and historical providers persist until explicit clear. Generic discovery is capped at 4 MiB, 2,000 models, 1,024-char ids (`providers/model-discovery.ts:12-16,175-230`). Cursor's custom RPC buffers every response chunk before applying a 500-id result cap (`adapters/cursor/live-models.ts:98-125`), so that active value has no byte cap. | **CONDITIONALLY-UNBOUNDED** provider churn/active Cursor response | Last-good is availability behavior; prune only providers absent from current config and cap Cursor RPC bytes before decode. | +| Gather in-flight (`catalog/provider-fetch.ts:64-129,645-665`) | One promise/result per distinct concurrent config fingerprint, deleted in `finally`; no concurrency cap. A Cursor flight may retain the uncapped RPC body above. | **CONDITIONALLY-UNBOUNDED by active concurrency/value bytes** | Keep same-key dedup; reject/coalesce churn and abort oversized discovery bodies. | +| Warning/omission state (`provider-fetch.ts:219-245`; `catalog/aggregation.ts:37-69,235-320`; `router.ts:152-180`; `combos/request.ts:4-38`; `config.ts:435,2052-2053`) | Catalog keys clear on sync (`catalog/sync.ts:313-317`); router/config/combo warning keys do not. Latest omissions are replacement/config-sized. | **CONDITIONALLY-UNBOUNDED** historical signatures | Clearing every request floods logs; use config generation/bounded signature LRU. | +| Usage summary (`management/logs-usage-routes.ts:81-115,187-205`) | Exactly 3 ranges × 4 surfaces. A summary can retain every distinct model in append-only usage history. | **CONDITIONALLY-UNBOUNDED** value bytes | Aggregate excess models into “other”; never drop totals. | +| API-key rollup (`management/api-key-usage.ts:105-161`) | One replacement snapshot, 60 s TTL, current configured keys. | **BOUNDED by config** | Preserve duplicate-id ambiguity and revision/config keying. | +| Provider quota (`src/providers/quota.ts:52-61,195-239,278-407`) | One report, active flights, 10 min per-account rows. Historical account rows require explicit clear; flights delete in `finally`. | **CONDITIONALLY-UNBOUNDED** | Preserve last-good negative caching; prune removed credential generations. | +| Codex quota (`src/codex/quota.ts:14-27,51,261-326`) | Fixed numeric row/account persisted. Disk load admits rows <=6 h old, but loaded rows do not age out; no count/file-size cap. | **CONDITIONALLY-UNBOUNDED** | Retain live account quota bars/selection; reconcile with live store plus age. | +| Desktop registry (`src/claude/desktop-3p.ts:106-107,151-205,246-280`) | Whole-map replacement, current models/profile. | **BOUNDED by config**, translation-required | Removal breaks inbound alias decoding. | + +## 5. Routing, cooldown, and auth stores + +| Store | Current contract | Worst case / verdict | Blast radius | +|---|---|---|---| +| Codex health (`src/codex/routing.ts:85-138,209-243`) | Account-wide + fixed quota-scope maps; semantic cooldowns, manual/outcome clears, no historical-account cap. | Historical accounts × fixed rows. **CONDITIONALLY-UNBOUNDED**. | Never erase live Retry-After/probe generation. | +| Codex affinity (`routing.ts:105-109,624-688`) | Hard 2,048 total thread/scope bindings, lazy 24 h TTL/LRU. | **BOUNDED**. | Too small causes account flapping/cache misses. | +| Anthropic health/affinity (`oauth/anthropic-routing.ts:34-40,62-63,96-116,234-241`) | Health max 15 min semantically but no global expired sweep; affinity hard 2,000/lazy 24 h TTL/LRU. | **CONDITIONALLY-UNBOUNDED** historical health; affinity **BOUNDED**. | Preserve live Retry-After/session affinity. | +| Subagent failure (`codex/subagent-model-fallback.ts:40-42,160-171,229-253`) | `unavailableUntil` is checked but expired rows are never deleted; no count cap. Prime map has one `global` key. | Arbitrary failed model/account keys forever. **UNBOUNDED**. | Delete on expiry/cap history without disabling active fallback suppression. | +| Pool/combo rotation (`codex/pool-rotation.ts:6-12,44-80,180-185`; `combos/resolve.ts:13-20,85-105,161-167`) | Outer per pool/combo, inner weight per account/target; explicit clear only. | Historical config keys. **CONDITIONALLY-UNBOUNDED**. | Preserve deterministic sticky/round-robin state for current config. | +| Combo/key cooldown (`combos/failover.ts:9-76`; `providers/key-failover.ts:21-53,89-130,174-190`) | Duration <=10 min, but expiration deletes only when that exact key is checked. | Historical expired keys. **CONDITIONALLY-UNBOUNDED**. | Global-prune expired rows; retain live 429 suppression. | +| OAuth (`oauth/index.ts:54-61,823-904,939-1020`) | Refresh flights delete in `finally`. XAI verdict TTL 30 s is exact-key lazy. Login/abort/manual maps are static-provider keyed; pending pasted code is variable size. | **CONDITIONALLY-UNBOUNDED** verdict churn/value bytes. | Preserve one flow/provider, early-paste/CSRF handoff, refresh dedup. | +| Codex auth (`codex/auth-api.ts:91,246-249,437,579-605,1193-1414`) | Random flow rows have 5 min timers (some setup errors 30 s); quota flights delete when empty; no flow admission cap. | Request rate × 5 min + active probes. **CONDITIONALLY-UNBOUNDED**. | Return explicit busy; never evict in-progress generation owner. | +| Guardian/reauth (`oauth/token-guardian.ts:54-99,118-225`; `codex/account-runtime-state.ts:1-13`) | Account-keyed rows delete on success/manual reauth; deleted historical accounts may remain. | **CONDITIONALLY-UNBOUNDED**. | Do not aggressively retry revoked tokens. | +| GCP ADC (`lib/gcp-adc.ts:61-66,83-127,274-302`) | Source-fingerprint token/flight maps; expired tokens sweep on resolve, flights delete in `finally`; source churn uncapped. | Source churn × token. **CONDITIONALLY-UNBOUNDED**. | Keep source freshness and refresh dedup. | +| GUI sessions (`server/management-auth.ts:27-45,150-184`) | Hard 128, lazy 5 min TTL, oldest eviction before issue; fixed token/CSRF plus origin. | **BOUNDED**. | Preserve same-origin/CSRF checks. | + +## 6. Filesystem memos, fixed singletons, and active resources + +| Store | Growth/eviction | Verdict | Blast radius | +|---|---|---|---| +| Ownership (`lib/config-ownership.ts:79-87,233-258`) | Manifest snapshot/config root; root row deleted only when that root is removed. Manifest paths are capped, roots are not. | **CONDITIONALLY-UNBOUNDED** home churn | Ownership is deletion safety; prune only proven inactive roots. | +| Windows ACL (`lib/windows-secret-acl.ts:37-40,375-448`) | Successful file/dir paths and timeout keys; no production clear. Atomic writes harden unique `*.ocx...tmp` paths (`config.ts:96-113,174-197`), adding dead success paths each write. | **UNBOUNDED on Windows**, about one path/write | Keep timeout protection/fail-closed ACL semantics. Remedy (audit R1-2, supersedes earlier draft): DELETE the ephemeral-temp success entry after rename — never re-key success by destination/generation, because a destination-keyed success memo could skip hardening future secret-bearing temps (windows-secret-acl.ts:47 doctrine: only the timeout memo may use the destination key). | +| Config/PID/warnings (`config.ts:413-435,2111-2121`; `router.ts:152-180`) | Config dir is one slot; PID results and warning keys have no age/count cleanup. | Slot **BOUNDED**; PID/warnings **CONDITIONALLY-UNBOUNDED** | Keep PID identity safety and warning dedup; prune by liveness/config generation. | +| Fixed caches (`server/startup-health-cache.ts:10-14,77-99`; `codex/main-account-cache.ts:13-24`; `github/star-state.ts:28-30,85-94`; `codex/project-config-warnings.ts:33,302-315`) | One cached value and at most one flight/owner; current warning array replacement. | **BOUNDED** | Retain stale-while-revalidate/last-known status. | +| Job/install slots (`storage/policy-job.ts:72-83`; `storage/restore-job.ts:37,66-68`; `server/startup-action-control.ts:43-57`) | One state/outcome, worker, cancellation hook and flight/owner; worker timeout 10 min. | **BOUNDED** | Never evict active destructive-operation locks. | +| Active turns (`server/lifecycle.ts:13-23,37-67,70-95`) | One controller/live turn; remove on finish/cancel, force clear at shutdown deadline; no admission cap. | **CONDITIONALLY-UNBOUNDED by concurrency/stalls** | Cap request admission, never untrack live work. | +| Sockets (`codex/websocket-registry.ts:4-35,47-73`) | One active pool-auth socket; remove on close/invalidate. | **CONDITIONALLY-UNBOUNDED by connections** | Needed for credential invalidation. | +| Storage workers/slots (`storage/worker-lifecycle.ts:25-80`; `storage/storage-mutation-coordinator.ts:20-64`) | Live workers explicitly terminate/drain; one mutation slot/distinct home, deleted in `finally`. | **CONDITIONALLY-UNBOUNDED by workers/homes** | Cap creation, never stop tracking live worker/lock. | +| Cursor background shells (`cursor/native-exec-shell.ts:23-24,215-265`) | Live child + output-length counter; deleted only on child close. No count/TTL/idle/transport ownership. App does not retain output text here. | **UNBOUNDED** when unsafe native exec enabled | Blind eviction kills side effects/data. Add session ownership, max-live, idle/absolute lifetime and controlled termination. | + +### Additional process caches and queues + +| Store | Growth/eviction and worst case | Verdict | Blast radius | +|---|---|---|---| +| Vision descriptions (`vision/index.ts:18-24,32-68,215-233,303-343`) | Process LRU, one row per persistent data-image + backend/model/detail/context identity; 256 rows, no TTL. Display is clamped to 2,000 chars only after lookup, while the cache stores the upstream `outcome.text.trim()` before that clamp. Worst `256 x S(upstream description)` plus keys; no byte bound. | **CONDITIONALLY-UNBOUNDED**, translation duty | Clearing loses cross-turn image-to-text reuse and repeats paid sidecars. Clamp before insert and add aggregate bytes while retaining identity/LRU. | +| Image retention tail (`images/fulfill.ts:12-24,89-106`) | One promise points to a FIFO chain of every concurrent write-prune-filter closure. Settled work collapses to a resolved tail, but arrivals have no admission/backlog cap. Each queued closure retains up to four artifact paths. Worst `pending fulfillments x path bytes`. | **CONDITIONALLY-UNBOUNDED while backlog grows**, translation duty | Parallel pruning can delete another response's just-written artifact. Keep serialization; bound image-fulfillment admission/backlog rather than dropping queued prune steps. | +| OAuth mutation tail (`oauth/store.ts:292-305`) | One promise points to an uncapped FIFO of load-modify-persist closures. No timeout or queue-depth admission; settled work collapses. Worst is every pending mutation closure and captured credential/config values. | **CONDITIONALLY-UNBOUNDED while disk mutation stalls** | Removing serialization reintroduces lost updates between refresh and account switching. Add admission/timeout without reordering accepted writes. | +| Grok apply tail (`server/management/agent-settings-routes.ts:62-70,545`) | One promise points to an uncapped FIFO of management apply closures; settled work collapses. Worst pending applies times captured apply arguments/config references. | **CONDITIONALLY-UNBOUNDED while apply stalls** | Concurrent read-modify-write cycles can overwrite each other. Coalesce superseded settings or return busy, preserving accepted order. | +| Codex credential refresh (`codex/account-store.ts:20-22,268-270,349-360,383-458`) | Active promise per distinct refresh-grant fingerprint; same grant deduplicates. Rows delete in `finally`; fetch is 30 s, file-lock wait is 65 s, but distinct admissions have no cap. Worst concurrent expired grants times credential/fetch state. | **CONDITIONALLY-UNBOUNDED by active account concurrency** | Evicting a live flight duplicates token rotation and can cause generation conflicts. Bound admission; never detach an accepted grant owner. | +| MiMo bootstrap (`adapters/mimo-free.ts:27-35,47-74,92-133`) | One JWT, expiry number, one in-flight bootstrap, and one 36-char client id. Bootstrap has a 15 s timeout and flight clears in `finally`; JWT refresh is expiry-triggered. Count is fixed, but upstream `data.jwt` has no response/value byte cap. | **CONDITIONALLY-UNBOUNDED value bytes** | Keep same-flight bootstrap, expiry skew, and stable anonymous client id; reject an oversized bootstrap response/JWT before caching. | +| Management usage read (`usage/log.ts:389-403,470-511`) | One active revision-keyed promise; same revision deduplicates and it clears in `finally`. The promise retains the fully parsed append-only usage log and byte/text intermediates until completion; no file-size cap. | **CONDITIONALLY-UNBOUNDED active value bytes** | Dedup prevents duplicate full parses. Bound/read incrementally or aggregate, but do not cache parsed rows after completion. | + +Other fixed process slots are finite by replacement: Codex runtime resolution +(`codex/runtime.ts:362-385`, one 15 s result), Kiro throttle probe/cooldown +(`adapters/kiro-retry.ts:31-52`, one probe and timestamp), sidecar/activity +breadcrumbs (`lib/sidecar-tracker.ts:10-48`, two latest strings/counters), storage +scheduler timers (`storage/policy-scheduler.ts:10-11`), restart acceptance +(`server/management/system-restart.ts:51-53`), current CORS origin +(`server/auth-cors.ts:20`), main-account plan/cache (`codex/main-account.ts:12`; +`codex/main-account-cache.ts:13-24`), latest shim-discovery error +(`codex/shim.ts:34`), Claude Desktop health counters +(`claude/desktop-health.ts:9`), finite debug-flag override keys +(`lib/debug-settings.ts:33`), and MiMo's client id above. Their string-bearing +slots should still receive local length normalization, but key/count growth is +**BOUNDED**. Replacing them indiscriminately would weaken runtime discovery, +throttle single-flight, crash attribution, scheduling, restart idempotency, or +current health/config reporting respectively. + +## Translator-layer state + +OpenCodex cannot be reduced to a byte relay. The structures below exist because +of translation duty. Most die with the turn/stream, but multiply under concurrency. + +### Tool-call assembly (20+ parallel calls is normal) + +| Path | State / bounds / worst case | Verdict | Naive-cap blast radius | +|---|---|---|---| +| OpenAI Chat (`src/adapters/openai-chat.ts:697-712,792-818,854-861`) | `pendingToolCalls[]` per interleaved call; `args += fragment`. No call-count/aggregate bytes. Retained O(calls + final args); concatenation may copy O(args²). | **CONDITIONALLY-UNBOUNDED per stream** | A single-digit cap breaks normal 20+ calls. Bound aggregate bytes/pathological calls, preserve index/id interleaving, fail coherently. | +| Generic bridge (`src/bridge.ts:317-324,667-718`) | One current atomic call and accumulating args; completed items retained later. No arg bytes. | **CONDITIONALLY-UNBOUNDED per stream** | Truncation emits invalid JSON. Overflow must fail/cancel, not complete truncated args. | +| Cursor (`cursor/protobuf-events.ts:152-163,355-410,441-470`; `live-transport.ts:506-529`) | `openToolCalls` cumulative args, `completedToolCalls`, schema/name maps. No count/bytes; atomic completion serializes parallel calls for bridge. | **CONDITIONALLY-UNBOUNDED per turn** | Preserve atomic start→delta→end, dedup, and 20+ normal calls. | +| Responses→Chat (`chat/outbound.ts:153-157,304-365,545-620`) | Per-index id/name/argument maps; collector also accumulates content/reasoning. No aggregate output cap. | **CONDITIONALLY-UNBOUNDED per response** | Eviction cross-wires deltas or duplicates authoritative done snapshot. | +| Anthropic (`adapters/anthropic.ts:832-885`) | One current block, `currentToolCallJson += partial_json`, forwards fragments while retaining validation copy. | **CONDITIONALLY-UNBOUNDED per block** | Cannot truncate after forwarding; overflow must fail turn before completion. | +| Responses→Claude (`claude/outbound.ts:162-250,600-629`) | One open block/WebSearch args; non-stream fold retains text/thinking/tool JSON/content. | **CONDITIONALLY-UNBOUNDED per response** | Preserve block order, one signature, tool JSON integrity. | +| Kiro (`adapters/kiro.ts:857-975,978-1111`) | One open tool at a time with `chunks[]`; `assistantText`, `outputChars`, required-mode `deferred[]`, and text-fallback events also accumulate. Each Smithy message is capped at 16 MiB, but aggregate stream/tool bytes are not. | **CONDITIONALLY-UNBOUNDED per stream** | Kiro rejects parallel calls, but truncating the one open call corrupts JSON/private completion; dropping deferred events breaks required/text-fallback terminal classification. | +| Cursor queue/frame (`cursor/live-transport.ts:457-493,727-756`; `cursor/framing.ts:1-5,68-123`) | Producer queue uncapped; partial frame repeatedly concatenated; announced frame up to `2^32-1`. | **CONDITIONALLY-UNBOUNDED per stream** | Add backpressure and reject announced size before buffering; never reorder terminal/tool messages. | + +Kiro's eventstream decoder also caps headers at 128 KiB +(`src/lib/eventstream-decoder.ts:26-27,189-203`). Google function calls arrive +atomically. Neither adapter has a process-level pending-call store. + +### MCP namespaces and manager state + +- `buildToolBridgeMaps()` creates request-local `toolNsMap`, + `freeformToolNames`, and `toolSearchToolNames`, one row/tool + (`server/responses/collaboration.ts:102-115`). No local count/bytes; accepted + body max 256 MiB; freed with request. Worst O(tool count + names) per turn; + **CONDITIONALLY-UNBOUNDED per request**. Preserve exact + `namespacedToolName()` round-trip or calls/results route incorrectly. +- One Cursor MCP manager/transport holds configured server/tool maps + (`cursor/mcp-manager.ts:54-70,80-139`), with 15 s connect/120 s call timeouts, + and clears/closes in `dispose()` (`:198-223`). Transport calls dispose without + awaiting (`live-transport.ts:579-599`). Counts/schema/resource bytes have no + local cap: **CONDITIONALLY-UNBOUNDED per stream/config**, with possible short + post-stream close lifetime. Bound catalog/resource payloads; keep tool/resource + execution and explicit ownership. +- Per-turn Cursor KV clones every value and has no count/bytes + (`cursor.ts:74-82`; `cursor/kv-store.ts:10-24`). It dies with the turn: + **CONDITIONALLY-UNBOUNDED per turn**. It is distinct from shared blobs. + +### Images + +- `imageGenCallAliases` builds at most two exact aliases/relevant tool and lives + for one response rewrite (`server/responses/core.ts:1427-1429,1638,1768-1792`; + `server/responses-image-gen-repair.ts:16-47,100-117`). It is tool-catalog + sized with no independent cap: + **CONDITIONALLY-UNBOUNDED per request**, translation-required. +- Anthropic normalization allows four concurrent decodes, rejects one input + base64 over 64 MiB/100M pixels, then reduces final image share to 20 MiB + (`anthropic-image-normalize.ts:50-64,286-401`; + `anthropic-image-guard.ts:19-45`). Original strings coexist with normalized + copies/native bitmaps; worst accepted peak is multiple body copies plus four + decoded images. +- Image responses buffer under 100 MiB total/50 MiB decoded per image + (`server/images.ts:46-49,211-231,291-304,443-449`; + `images/artifacts.ts:11-25,112-129`): **BOUNDED per request**, but + concurrency-multiplicative. +- Continuation retains raw expanded input/output (`responses/state.ts:444-447`) + before provider-specific image reduction makes it cheap. This is how one + screenshot-heavy row exceeds 64 MiB, remains exempt, and is skipped on disk. + +### Reasoning/thinking carry + +- Anthropic keeps current block/tool JSON and emits opaque redacted/signature + events (`adapters/anthropic.ts:832-870`). The bridge accumulates a coherent + reasoning item and pending redacted blocks (`bridge.ts:251-324`). No bytes: + **CONDITIONALLY-UNBOUNDED per stream**. +- Responses parsing preserves real signature/redacted envelopes + (`responses/parser.ts:411-425`); remembered envelopes inherit continuation's + cap hole. +- Responses→Claude emits one synthetic signature/close and non-stream fold keeps + real signature (`claude/outbound.ts:214-248,600-629`). Removing state breaks + block validity/order. +- Antigravity cross-turn replay is process state, covered in section 2. + +### Responses item-ID repair + +Each rewritten stream owns provider-config placeholder sets and two +`output_index → id` maps (`server/responses-item-id-repair.ts:7-12,51-84,187-205`). +Distinct message/reasoning indices have no count/bytes; closure dies with stream. +Worst O(output count × id bytes): **CONDITIONALLY-UNBOUNDED per stream**. Eviction +would make ids inconsistent across item/delta/terminal snapshot. Any cap must +fail coherently; function-call ids stay untouched and raw continuation keeps +upstream ids (`:169-185`). + +### `tool_search` lifecycle + +`toolSearchToolNames` is request-local (`collaboration.ts:107-115`). The bridge +classifies matching calls, buffers/parses current args, retains pending web +sources, and emits `tool_search_call`/results (`bridge.ts:122-143,199-215,317-324,667-718`). +No cross-turn discovery registry exists; completed output may enter ordinary +continuation. Tool-catalog and current argument/source bytes are uncapped: +**CONDITIONALLY-UNBOUNDED per stream**. A cap must not recast the call as a +normal function or lose source attribution. + +### Request-direction conversion + +- Responses, Chat, and Claude JSON use `readJsonRequestBody()`: + `await req.arrayBuffer()` → optional decompression → full text → `JSON.parse` + (`server/request-decompress.ts:15-21,52-84`). Accepted decoded/raw identity body + cap is 256 MiB, but identity raw bytes are fully materialized before assertion; + decoded bytes, text, parsed object, translated object, and serialized internal + request can overlap. +- Chat converts and serializes a new Responses request + (`server/chat-completions.ts:40-58,120-150`; `chat/inbound.ts:209-294`). + Claude does likewise (`server/claude-messages.ts:65-70,510-589`; + `claude/inbound.ts:407-508`). Responses retains parsed raw body through routing + (`server/responses/core.ts:1070-1116`). +- Copies normally die with request: **BOUNDED per accepted body but + high-water-multiplicative**. Exception: `rememberResponseState()` stores the + translated Responses input/output. Lowering the body cap naively breaks normal + screenshot-heavy sessions; reduce copies/stream where semantics permit and + surface coherent 413 on hard overflow. + +### SSE inspection + +The inspector retains at most a 4 MiB partial frame and 256 completed items / +8 MiB source bytes; overflow taints reconstruction; callback/dispose clears all +(`server/relay.ts:18-21,555-648,680-724`): **BOUNDED per stream**. Eager relay +backpressures above 8 MiB and bounds post-cancel drain to 15 s/32 MiB +(`server/relay-eager.ts:45-75,123-188`). Preserve terminal/log/continuation +callbacks and never persist a tainted partial reconstruction. + +## Exclusions and roadmap implications + +- Weak response provenance (`responses/state.ts:82`) and relay response tags + (`server/relay.ts:15-16`) do not keep keys alive. +- Immutable provider/model/header/effort sets and generated descriptors do not grow. +- Durable JSONL/snapshot/config/OAuth/update files are disk stores; their in-RAM + snapshots are listed above. +- Other per-request search/image/video/compact/non-stream collectors are not + process stores; their surface caps are high-water controls. + +Minimum coherent implementation order: + +1. Define aggregate-byte accounting/admission/metrics for continuation, Cursor + blobs, Antigravity replay, and image-cache metadata. +2. Preserve continuation via bounded blobs/spill and explicit misses; intentionally + replace “always keep newest.” +3. Add per-blob + aggregate Cursor caps while retaining hash lookup/re-store refresh. +4. Bound Antigravity inner calls and image sentinel count/metadata. +5. Globally prune expired cooldown/health rows and reconcile config/account/catalog + maps by generation. +6. Windows ACL: delete the ephemeral-temp success entry after rename (no + destination/generation re-keying — see the ACL row above; audit R1-2), + preserving timeout/fail-closed behavior. +7. Add translator-wide per-turn budgets for tool args/calls, item-id maps, Cursor + frame/queue, and body-copy high water. Acceptance must explicitly include 20+ + parallel tool calls; overflow must be coherent protocol failure, never truncated + JSON or cross-wired calls. + +No source/test behavior should change until these contracts and regressions are +split into implementation phases. diff --git a/devlog/_plan/260801_zero_leak_state_stores/005_impl_roadmap.md b/devlog/_plan/260801_zero_leak_state_stores/005_impl_roadmap.md new file mode 100644 index 000000000..8fc6d7931 --- /dev/null +++ b/devlog/_plan/260801_zero_leak_state_stores/005_impl_roadmap.md @@ -0,0 +1,171 @@ +# Zero-leak state stores — implementation roadmap + +Opened 2026-08-01. Consumes `000_state_store_inventory.md` (36 stores, verdicts) +and the prior unit's stream-path work (260731_macos_rss_retention, landed). +Push to origin/dev is user-authorized for this unit. + +## Design goal + +Among production TS-based LLM proxies on GitHub, opencodex should have the +strongest theoretical no-leak posture — while carrying translation duty +(bidirectional tool conversion, MCP namespace flatten/restore, image +normalization, reasoning-signature carry) that pure relays do not have. +The competitive evidence (060) defines "strongest" per category; the phases +below close every UNBOUNDED and the material CONDITIONALLY-UNBOUNDED verdicts. + +## External evidence (Luna lanes, source-opened 2026-08-01) + +- Portkey Gateway: no post-stream replay state, but NO upstream abort on client + disconnect and unbounded local transform buffers (streamHandler.ts, + streamHandlerUtils.ts). No process-wide byte budget. +- Claude Code Router: per-request AbortController + upstream cancel (good); + context-archive store enforces TTL+count+aggregate-body-bytes (the only + competitor with a 3-dimension eviction contract). +- mcp-proxy (punkpeye): replay event store FIFO 1,000 events, no TTL/bytes; + session map cleanup only on transport close. +- LiteLLM: added per-entry max cache item size (v1.63.14) after OOMs; + streaming retention issue #6404 open. one-api: externalizes state + (Redis/DB), memory cache default off. +- Eviction-contract precedent: lru-cache maxSize+maxEntrySize+TTL+LRU; + cacache for content-addressed disk spill (eviction separate). + +Conclusion: no surveyed TS proxy has a process-wide app-owned byte budget. +Landing one (wp5) plus per-store byte caps positions opencodex to be stronger +than every surveyed competitor — a claim that stays PROVISIONAL until 060 +supplies the full source ledger (audit round 1). + +## Phase map (dependency-ordered; one decade doc per work-phase) + +| Doc | WP | Content | Depends | +|---|---|---|---| +| `010_continuation_hard_cap.md` | wp2 | continuation last-entry cap hole → true hard cap; oversized entries demote to disk stub ONLY after atomic per-entry spill (fsync+rename); missing/corrupt spill = explicit continuation failure, never silent naked-delta (R1-2); contract redefinition | — | +| `020_blob_and_replay_caps.md` | wp3 | Cursor blob byte cap split by PROVENANCE (local-regenerated evictable, remote-origin pinned within TTL — R1-3), Antigravity inner-call bounds, vision-LRU clamp-before-insert, image zero-weight sentinel accounting (R1-1) | — | +| `030_eviction_mechanisms.md` | wp4 | store-by-store cleanup matrix with three mechanisms: TTL sweep / config-generation reconciliation / admission caps (R1-4). Covers: subagent quota-failure records, key/combo cooldowns, provider+codex quota history, routing health, warning memos, model-cache provider history, pool/combo history, XAI verdicts, guardian backoff, GCP source tokens, ownership/PID memos, OAuth/reauth reconciliation. Windows ACL memo: eviction-after-rename keyed by actual temp path, NO destination-level reuse (R1-5); explicit security review required | — | +| `035_registry_admission_caps.md` | wp4b | remaining operational registries/flights: debug-subscriber count cap, active turns/sockets/workers hard admission CAP (reject beyond cap with a coherent busy error — S3-3) + leak metric, credential-refresh flights bounded distinct-grant admission (cap on concurrent grant fingerprints) + staleness replacement (S3-3), usage-read flight staleness guard PLUS bounded parse (stream/limit the full-log read so the in-flight value itself is byte-capped — S2-1), Cursor model-discovery response byte cap + gather-flight concurrency cap, OAuth pending-code value byte cap + auth flow/probe admission caps, MiMo JWT value byte cap, Cursor MCP manager payload caps, crash-ring + fixed-slot diagnostic + AFFINITY value-byte truncation (S2-2) | — | +| `040_app_bytes_observability.md` | wp5 | appOwnedBytes payload on /api/system/memory (continuation, blobs, rings, caches) + process-wide budget with oldest-first demotion contract | 010,020,035 (their accounting hooks; 030 provides sweeping only) | +| `050_translator_stream_bounds.md` | wp6 | per-stream translator caps for ALL inventory §Translator-layer accumulators: tool-arg aggregate bytes (adapters keep interleaving), Responses→Chat/Claude output collectors, Kiro deferred/text state, reasoning carry, item-ID maps, tool-search sources, request-direction copy high-water, cursor producer queue backpressure + frame-size admission, image-retention/oauth/grok tail admission bounds | — (parallel-safe) | +| `055_background_shell_lifecycle.md` | wp6b | Cursor background-shell store (UNBOUNDED when enabled, R1-1): session ownership, max-live admission, idle/absolute lifetime, controlled termination | — (parallel-safe) | +| `060_proxy_benchmark.md` | wp7 | competitive comparison table with source URLs; per-category verdicts; gap list must be empty or converted to work-phases; superiority claims live ONLY here | 010–055 landed | +| `070_close_and_push.md` | wp8 | final gates, docs sync, push, HEAD parity | all | + +### Budget scope split (second-audit R1-6) + +The 040 process-wide budget governs EVICTABLE RETAINED stores only +(continuation, blobs, rings, caches). Translator per-stream buffers and +serialized-tail backlogs are OBSERVED (high-water fields in appOwnedBytes) +but never budget-evicted — in-flight conversion state cannot be evicted +coherently; it is bounded by 050's per-stream caps instead. With this split +050 is genuinely parallel to 040, and 040's dependency is on the accounting +hooks of 010/020/035 (030 provides sweeping only, no accounting interface). + +### Store-by-store mechanism lock for 030 (S2-5) + +| Store | Mechanism | Why | +|---|---|---| +| subagent quota-failure records | (a) TTL sweep | getters already treat expired rows as absent | +| key/combo cooldowns | (a) TTL sweep | same | +| Anthropic routing health | (a) TTL sweep | same | +| XAI verdicts | (a) TTL sweep | 30 s TTL with exact-key lazy expiry today (S3-2); reconciliation would leave expired churn until config changes | +| warning-key memos (all) | (b) reconciliation ONLY | no expiry semantics; TTL would resurrect suppressed warnings | +| codex quota rows | (b) reconciliation ONLY | 6 h is disk-hydration admission, not a live TTL; live getters serve rows indefinitely (quota.ts:261/298) | +| provider quota history, model-cache provider history, pool/combo history, guardian backoff, GCP source tokens, ownership/PID memos, OAuth/reauth keys | (b) reconciliation | dead keys unreachable from current config/accounts | +| Windows ACL success memos | (c) delete-after-rename | R1-2 contract; destination re-keying forbidden | + +Scope guards: provider adapter LOGIC unchanged (bounds are added at the +adapter-local accumulation sites without altering event semantics — the +"translation duty" stores are bounded, never deleted); #820 architecture +(turn lease/session lanes/scheduler) stays out; stream-path work from the +prior unit is not reopened. + +## Locked design decisions + +1. **Continuation (010):** keep the store authoritative for replay; single + oversized entries spill to a DEDICATED per-entry file (not the debounced + monolithic snapshot, which is async/best-effort/2 MiB-skipping — R1-2). + Demotion ordering is locked: RAM row becomes a stub ONLY after an atomic + successful spill (write-fsync-rename). Spill FAILURE (disk permissions, + exhaustion, I/O) evicts the row from RAM and records a small + `spill-failed` tombstone for the response id: later continuation + against it returns the same explicit structured not-found contract as a + corrupt spill, and the client falls back to full-context resend — the + same recovery path TTL expiry produces today. Replay through a missing/ + corrupt stub likewise returns an EXPLICIT structured continuation error, + never silent forwarding of the naked delta. The last-entry exemption is + deleted; `storedResponseBytes > byteCap()` always spills or + tombstone-evicts within a bounded in-flight window — the RAM cap is + unconditionally hard. + UNSAFE boundary: if spill-through cannot preserve replay, stop and report + rather than truncating conversations. +2. **Blobs (020):** per-blob admission cap + aggregate byte cap with LRU; + eviction split by provenance (R1-3): locally-regenerated blobs (root/turn, + rebuilt each request per protobuf-request.ts) are LRU-evictable; REMOTE + `setBlobArgs` blobs are pinned within TTL — evicted only at TTL expiry + unless protocol-level re-send is proven by regression. The aggregate cap + is enforced AT INSERT for every provenance: if admitting a remote + blob would exceed the cap after evicting all evictable entries, the + insert is REJECTED and the hash takes the explicit-miss path (identical + protocol surface to an evicted-hash miss) — pinning orders eviction + preference, it never overrides the cap. Explicit miss on + evicted hash. Antigravity inner maps get count+byte bounds with + clear-on-invalid preserved. Image zero-weight sentinels get key-byte + accounting and a count cap (R1-1). +3. **Sweeper (030):** ONE shared `sweepExpired()` utility with per-store + registration, run on a coarse timer (60 s, unref) and on store-write — + but with THREE mechanism classes per the 030 matrix (R1-4): clock-TTL + predicates, config-generation reconciliation for generation-owned keys, + and admission caps for active resources. No behavior change for live keys. + Windows ACL success memo: evict the temp-path row after rename completes; + never let one temp's success vouch for another (R1-5, security review). +4. **Budget (040):** observability FIRST (appOwnedBytes payload), then a + documented demotion contract: when the global budget is exceeded, evict + oldest unpinned entries store-by-store in a fixed priority order + (logs/rings → caches → blobs → continuation spill). Never reject new work + admission as the first lever. +5. **Translator bounds (050):** aggregate-byte caps sized for 20+ parallel + tool calls as NORMAL (per prior handoff): per-call args cap 2 MiB, per-turn + aggregate 32 MiB, overflow FAILS the turn coherently (never truncated JSON). + Cursor frame admission rejects announced sizes above cap before buffering. + +## Open questions carried into phase docs + +- 010: spill file naming/GC — per-entry content-addressed vs id-keyed files; + orphan cleanup on startup. Decide in the phase doc after reading the disk + path (the debounced snapshot is NOT reused for spill — locked, R1-2). +- 040: budget default size and config surface (fixed vs configurable). +- 050: which tails (image/oauth/grok) get admission caps vs busy-rejection. + +## Regression-class requirements per phase (audit round 1) + +- 010: spill durability-before-demotion; missing/ + corrupt spill → explicit error; spill-failure tombstone eviction + counter; + restart replay; provider metadata + tool- + result replay through stubs; replacement/TTL cleanup of spill files; orphan + cleanup; concurrent flush/demotion; disk-permission failure. +- 020: local vs remote blob provenance; exact boundary admission; aggregate + accounting; replacement/re-store LRU; explicit misses; Antigravity live-call + preservation; image sentinel metadata accounting; pinned-saturation + (remote blobs collectively at cap → new inserts rejected with explicit miss). +- 030: fake-clock global expiry; live-key preservation; generation + reconciliation; timer singleton/unref/stop; write-trigger sweep; Windows + repeated-temp ACL proof (second temp for same destination is re-hardened). +- 035: subscriber admission cap (rejects beyond cap, existing listeners + unaffected); active-registry admission counter + leak metric monotonicity; + active-registry HARD admission cap (beyond-cap request receives a coherent + busy error, existing work unaffected — S3-3); flight staleness (a stuck + flight is replaced, not joined, after the guard) plus bounded + distinct-grant admission (concurrent grant fingerprints capped — S3-3); + usage-read bounded parse (oversized log yields a capped result, + not an unbounded in-flight value); discovery response byte cap + + concurrent-gather cap; OAuth pending-code value cap + flow/probe admission; + MiMo JWT byte cap; MCP payload caps; value-byte truncation for crash-ring, + fixed-slot diagnostics, and affinity entries (truncated string still + useful, marker appended). +- 040: observe-only metrics; exact replacement/eviction accounting; pinned/ + no-evictable fallback; single-entry-over-budget; privacy-safe payloads. +- 050: 20+ interleaved calls normal; per-call and aggregate boundaries; + coherent overflow (no truncated JSON, turn fails); upstream cancellation; + Cursor announced-frame rejection before allocation; queue ordering/ + backpressure; stalled-tail busy/admission. +- 055: max-live admission; idle + absolute lifetime termination; session + ownership (no cross-session kill); controlled termination drains output; + disabled-by-default posture unchanged. diff --git a/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md b/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md new file mode 100644 index 000000000..673428d16 --- /dev/null +++ b/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md @@ -0,0 +1,50 @@ +# 006 — roadmap audit synthesis + +## Round 1 (reviewer Maxwell, FAIL 7) — adjudication + +| # | Blocker | Decision | Action | +|---|---|---|---| +| R1-1 | Coverage gap: phases don't own every leak verdict (cursor shells, debug subscribers, pool/oauth/guardian history, ownership memos, active registries, MCP payloads, refresh flights, MiMo JWT, usage-read; translator accumulators beyond tool-args/cursor framing) | ACCEPT | 005 gains a full ownership matrix: every one of the 36 stores mapped to a phase OR given an explicit NON-MATERIAL (with why) / DEFERRED-#820 (with why) verdict. New phase 035 (registry & flight admission caps) absorbs the orphaned operational stores; 050 scope widened to ALL translator accumulators. | +| R1-2 | Windows ACL "keying fix" unsafe: success memo by destination could skip hardening future temps | ACCEPT | Remedy redefined: success on an ephemeral temp is NOT memoized persistently (removed after rename); destination-key memoization stays timeout-only, matching windows-secret-acl.ts:47 doctrine. Moved out of the sweeper phase into its own 030 sub-item with this exact contract. | +| R1-3 | Continuation disk path not reusable as spill-through (2 MiB skip / 24 MiB stop / debounced best-effort whole-snapshot / sync APIs) | ACCEPT | 010 redesigned: spill is a NEW per-entry content-addressed file write with durable-success-before-stub-swap transaction, defined read-miss contract (replay returns not-found → client falls back to full-context resend, same as TTL expiry today), and the legacy snapshot path untouched for small entries. UNSAFE boundary retained. | +| R1-4 | Crash ring BOUNDED verdict hides uncapped value strings; fixed-slot/affinity value bytes unaudited | ACCEPT | Inventory corrected (crash ring → CONDITIONALLY-UNBOUNDED, value-byte audit noted for fixed-slot diagnostics and affinity); 030/035 include value-byte truncation for retained diagnostic strings. | +| R1-5 | Universal 60 s sweeper unsafe for warning memos (no expiry, intentional suppress), codex quota rows (6 h is hydration-admission only; live getters return rows indefinitely), ACL memos | ACCEPT | 030 split into three mechanisms: (a) exact-expiry sweep only for stores whose getters already treat expired as absent; (b) config/account-generation reconciliation for keyed stores (drop keys no longer in current config/accounts — behavior-preserving because dead keys are unreachable); (c) ACL-specific contract per R1-2. Warning memos: reconciliation-only, never TTL. Codex quota rows: reconciliation-only, never TTL. | +| R1-6 | Dependency graph inconsistent: 040 claims a 030 accounting dependency that 030 doesn't provide; 050 not parallel-safe if the global budget includes translator buffers | ACCEPT | 040 depends on 010/020/035 accounting hooks (030 provides none); budget scope split: BUDGET covers evictable RETAINED stores only (continuation, blobs, rings, caches); translator per-stream buffers are OBSERVED (high-water in appOwnedBytes) but never budget-evicted (in-flight state can't be evicted coherently). With that split 050 is genuinely parallel. | +| R1-7 | "Strictly stronger in every category" premature before 060's table exists | ACCEPT | 005 conclusion reworded to a HYPOTHESIS the 060 gate must prove (empty-gap-list required); superiority claim removed until then. | + +All seven accepted — no rebuttals. Re-audit with the same reviewer after amendments. + +## Round 2 (reviewer Singer, FAIL 5) — adjudication (IDs S2-*) + +| # | Blocker | Decision | Action | +|---|---|---|---| +| S2-1 | Ownership orphans: cursor discovery bytes/gather flights, OAuth pending-code values + auth flow/probe admission, usage-read value bytes (staleness guard ≠ byte bound) | ACCEPT | 035 scope extended to own all three explicitly (incl. bounded parse for usage-read so the in-flight value is byte-capped). | +| S2-2 | Crash-ring detail row still BOUNDED; affinity value-byte coverage promised but unrecorded | ACCEPT | Inventory detail row corrected to CONDITIONALLY-UNBOUNDED; 035 adds affinity value-byte truncation explicitly. | +| S2-3 | 040 dependency row still said 010,020,030 | ACCEPT | Phase-map row corrected to 010,020,035. | +| S2-4 | 035 had no regression-class requirements | ACCEPT | Regression classes added for every 035 item. | +| S2-5 | Inventory still recommended destination/generation ACL re-keying (twice); 030 mechanism assignment not locked store-by-store | ACCEPT | Both inventory sites rewritten to delete-after-rename with the doctrine citation; 005 gains the store-by-store mechanism lock table. | + +All five accepted. Re-audit round 3. + +## Round 3 (reviewer Singer, FAIL 4) — adjudication (IDs S3-*) + +| # | Blocker | Decision | Action | +|---|---|---|---| +| S3-1 | 040 (wp5) sequenced before its dependency 035 (wp5b) | ACCEPT | 035 renumbered wp4b and moved above 040 in the phase map; execution order now satisfies the accounting-hook dependency. | +| S3-2 | XAI verdicts (30 s TTL, exact-key lazy expiry) wrongly under reconciliation | ACCEPT | Moved to mechanism (a) TTL sweep in the lock table. | +| S3-3 | Active registries "admission counter" and refresh flights "staleness guard" do not impose finite bounds | ACCEPT | 035 remedies upgraded: active turns/sockets/workers get a hard admission CAP (reject beyond cap with a coherent busy error) + leak metric; refresh flights get bounded distinct-grant admission (cap on concurrent distinct grant fingerprints) + staleness replacement. | +| S3-4 | Duplicate "Round 2" headings / colliding R2-* IDs in this file | ACCEPT | Rounds relabeled: implementation-review rounds from the PREVIOUS unit keep R*-; this unit's roadmap-audit rounds use S2-*/S3-*; roadmap references updated. | + +## Round 2 (parallel reviewers Raman FAIL 5 / Godel FAIL 2) — adjudication + +Raman's five findings substantially overlap Maxwell's round 1 (UNBOUNDED +coverage, spill safety, blob provenance, sweeper matrix, ACL keying) and are +already folded above. The two NET-NEW blockers from Godel: + +| # | Blocker | Decision | Fix | +|---|---|---|---| +| R2-1 | 010: "spill failure keeps the row hot" contradicts the hard cap — a failing disk leaves oversized rows resident indefinitely | ACCEPT | Failure ladder locked: (1) spill success → stub swap; (2) spill FAILURE → the row is EVICTED from RAM and the response id records a small `spill-failed` tombstone; later continuation against that id returns the same explicit structured not-found/error contract as a corrupt spill (client falls back to full-context resend). Continuity is sacrificed only on real disk failure, surfaced via warning + counter — never silently. The RAM cap is unconditionally hard: every over-cap row spills or is tombstone-evicted within the bounded in-flight window. | +| R2-2 | 020: pinned remote blobs can collectively exceed the aggregate cap; pre-insert admission undefined; deferring to 040 invalid (040 depends on 020 accounting) | ACCEPT | Admission ladder locked in 020: the aggregate cap is enforced AT INSERT for every provenance. If inserting a remote blob would exceed the cap after evicting all evictable (local/TTL-expired) entries, the insert is REJECTED and the hash takes the explicit-miss path (identical protocol surface to an evicted-hash miss). Pinning orders eviction preference only — it never overrides the aggregate cap. Pinned-saturation regression added to 020's class list. | + +Both preserve the structured-error-over-silent-corruption principle. 005 +amended accordingly (failure ladder + admission ladder). diff --git a/devlog/_plan/260801_zero_leak_state_stores/010_continuation_hard_cap.md b/devlog/_plan/260801_zero_leak_state_stores/010_continuation_hard_cap.md new file mode 100644 index 000000000..9ee1db8a2 --- /dev/null +++ b/devlog/_plan/260801_zero_leak_state_stores/010_continuation_hard_cap.md @@ -0,0 +1,342 @@ +# 010 — continuation hard cap with durable per-entry spill + +Date: 2026-08-01 +Work phase: wp2 +Depends on: none +Binding inputs: `000_state_store_inventory.md` §1, `005_impl_roadmap.md` locked decision 1, `006_roadmap_audit_synthesis.md` R1-3/R2-1. + +## Outcome + +Make the Responses continuation store authoritative but unconditionally bounded in RAM. +An entry that would leave resident bytes above 64 MiB is synchronously written to a +dedicated id-keyed spill file, fsynced, atomically renamed, and only then +replaced by a small RAM stub. Any write failure removes the resident entry and records +a bounded tombstone. A stub whose file is missing or corrupt produces the same explicit +structured continuation-not-found response; it never forwards a naked delta. + +The legacy debounced `responses-state.json` path remains the persistence path for small +resident entries. Its 2 MiB per-entry and 24 MiB snapshot selection rules are not reused +as the spill mechanism and are not relaxed. + +## Current contract and verified anchors + +- `src/responses/state.ts:6-20` owns count, TTL, 64 MiB nominal RAM cap, legacy snapshot + limits, and stale snapshot-temp cleanup. +- `src/responses/state.ts:22-78` has one resident shape, one `states` map, one byte + counter, and centralized `setEntry()` / `deleteEntry()` accounting. +- `src/responses/state.ts:202-250` lazily loads v1/v2 monolithic snapshots and treats + corrupt snapshots as an empty cache. +- `src/responses/state.ts:252-299` performs debounced best-effort snapshot writes. It + skips a serialized pair above 2 MiB and stops newest-first selection at 24 MiB. +- `src/responses/state.ts:319-334` prunes TTL/count and then bytes only while + `states.size > 1`. This is the last-entry exemption to delete. +- `src/responses/state.ts:336-350` returns the original request on every miss; the + caller cannot distinguish ordinary absence from a known broken spill. +- `src/responses/state.ts:363-408` reads provider metadata and exposes observe-only + metrics without loading, pruning, or serializing. +- `src/responses/state.ts:415-455` synchronously stores expanded input plus output and + immediately prunes/schedules persistence. +- `src/server/responses/core.ts:1092-1115` expands before parsing and detects expansion + only by object identity. +- `src/server/responses/core.ts:1228-1239,1338-1343` already returns structured 400s for + canonical-forward and Kiro misses, while `:1445-1449` still warns and forwards a + passthrough naked delta. +- `tests/responses-state.test.ts:535-555` asserts the obsolete “newest survives even + when over cap” policy. +- `tests/responses-state.test.ts:660-883` covers restart, stale/corrupt snapshots, and + the obsolete “oversized stays in RAM but is skipped on disk” contract at `:856-883`. + +Inventory blast-radius constraint: “evicting/rejecting the newest row makes the next +chained turn a naked delta.” This phase therefore spills or emits an explicit miss; it +does not truncate replay history. + +## File changes + +### NEW `src/responses/spill-store.ts` + +Own all spill filesystem I/O. Do not put spill code in the generic config writer: the +legacy writer is best-effort and does not fsync. + +```ts +export const RESPONSE_SPILL_VERSION = 1; +export const RESPONSE_SPILL_DIR_NAME = "responses-state-spill"; +export const RESPONSE_SPILL_ORPHAN_GRACE_MS = 15 * 60_000; +export const RESPONSE_SPILL_SCAN_MAX = 4_096; +export const RESPONSE_SPILL_CLEANUP_MAX = 512; + +export interface ResponseSpillPayload { + version: 1; + responseId: string; + createdAt: number; + items: unknown[]; + providers?: OcxProviderContinuationState; +} + +export interface ResponseSpillRef { + version: 1; + fileName: string; // sanitized response id + id digest + payload size + digest: string; // lowercase SHA-256 of the exact UTF-8 file bytes + payloadBytes: number; +} + +export type ResponseSpillReadResult = + | { ok: true; payload: ResponseSpillPayload } + | { ok: false; reason: "missing" | "corrupt" }; + +export interface ResponseSpillCleanupResult { + scanned: number; + removed: number; + failed: number; + bytesRemoved: number; +} + +export function writeResponseSpillDurably( + responseId: string, + state: Omit, +): ResponseSpillRef; +export function readResponseSpill(ref: ResponseSpillRef): ResponseSpillReadResult; +export function deleteResponseSpill(ref: ResponseSpillRef): void; +export function recoverOrphanedResponseSpills( + referencedFileNames: ReadonlySet, + dir?: string, +): ResponseSpillCleanupResult; +``` + +`writeResponseSpillDurably()` transaction, in this exact order: + +1. Serialize `{version:1,responseId,createdAt,items,providers}` once and compute UTF-8 + bytes and SHA-256 from those exact bytes. +2. Build an id-keyed basename + `...spill.json`. Normalize the id to NFC, + replace every character outside `[A-Za-z0-9._-]` with `_`, collapse repeated `_`, + trim the visible portion to 80 characters, and use `response` if it becomes empty. + `id-digest-12` is the first 12 lowercase hex characters of SHA-256(responseId), so + ids that sanitize/truncate to the same visible text cannot collide. Require the full + owned-file regex before joining it to the spill directory. +3. Resolve `/responses-state-spill/`; create/harden the directory + to user-only permissions. +4. Open a same-directory unique temp with `wx` and mode `0600`. +5. Write all bytes, call `fsyncSync(fd)`, close the descriptor, and harden the temp. +6. Rename temp to the id-keyed destination. If the basename already exists, accept it + only after regular-file, exact-size, response-id, and full-digest verification; + otherwise replace it through another same-directory temp. Replacement of a response + id deletes the old basename only after the new rename and stub swap succeed. +7. Return the ref only after rename succeeds. On every failure, close/unlink the temp + best-effort and rethrow a sanitized error containing no response id or path. + +The id-keyed layout is selected over content-addressing because lifecycle, replacement, +TTL, and tombstone ownership are all by response id. A sanitized visible component makes +orphan diagnosis practical, the size component permits bounded startup accounting +without opening every file, and the short id digest prevents collisions after +sanitization/truncation. The full content digest remains in the stub and is verified on +read; neither visible id nor filename size is trusted as integrity proof. + +This phase deliberately uses a synchronous transaction. `rememberResponseState()` is +called from synchronous bridge completion callbacks (`src/bridge.ts:143,811-817` and +`src/server/relay.ts:498,680-718`); an un-awaited asynchronous spill would permit an +unbounded pending-write closure chain and a post-response stub race. The synchronous +path is the bounded in-flight window: at most one transaction exists on the JS thread, +with no queue. Spills are exceptional, not the small-entry hot path. + +`readResponseSpill()` must reject symlinks/non-regular files, require the basename to +match the owned regex, require stat size to match both the filename size and +`ref.payloadBytes`, enforce ref/content digest equality, validate the exact schema and +matching `responseId`, and return `corrupt` on any parse, +shape, or digest failure. Never return partial items. + +### MODIFY `src/responses/state.ts` + +Replace the monomorphic row with this union: + +```ts +interface ResidentResponseState { + kind: "resident"; + createdAt: number; + items: unknown[]; + providers?: OcxProviderContinuationState; + sizeBytes: number; +} +interface SpilledResponseState { + kind: "spill"; + createdAt: number; + providers?: OcxProviderContinuationState; + spill: ResponseSpillRef; + sizeBytes: number; // cached serialized stub bytes, not payloadBytes +} +interface SpillFailedResponseState { + kind: "spill-failed"; + createdAt: number; + sizeBytes: number; +} +type StoredResponseState = ResidentResponseState | SpilledResponseState | SpillFailedResponseState; + +export type PreviousResponseReplayFailure = { + code: "previous_response_not_found"; + reason: "spill_missing" | "spill_corrupt" | "spill_failed"; +}; +``` + +Add a private `WeakMap` beside +`replayedInputPrefixLengths` and an observe-only accessor: + +```ts +export function previousResponseReplayFailure(body: unknown): PreviousResponseReplayFailure | undefined; +``` + +Centralize transitions; no caller mutates `states`, counters, or spill files directly: + +```ts +function setResidentEntry(id: string, entry: ResidentInput): void; +function swapResidentForSpill(id: string, expected: ResidentResponseState, ref: ResponseSpillRef): void; +function replaceWithSpillFailure(id: string, expected?: StoredResponseState): void; +function deleteEntry(id: string, options?: { deleteSpill?: boolean }): void; +function materializeEntry(id: string, entry: StoredResponseState): + | { ok: true; state: ResidentResponseState } + | { ok: false; failure: PreviousResponseReplayFailure }; +``` + +`deleteEntry()` subtracts the row's RAM `sizeBytes`; for a spill stub it also unlinks +the dedicated file unless `deleteSpill:false` is used during a verified load/swap. +Replacement and TTL/count eviction therefore remove old spill files. + +Replace the byte loop with: + +```ts +while (storedResponseBytes > byteCap() && states.size > 0) { + const oldestId = states.keys().next().value as string | undefined; + if (!oldestId) break; + const entry = states.get(oldestId)!; + if (entry.kind !== "resident") { deleteEntry(oldestId); continue; } + try { + const ref = writeResponseSpillDurably(oldestId, entry); + swapResidentForSpill(oldestId, entry, ref); // only after durable success + } catch { + replaceWithSpillFailure(oldestId, entry); // R2-1: no hot oversized row + spillCounters.writeFailures++; + } +} +``` + +After each swap/tombstone, re-check the condition. A stub/tombstone that alone exceeds +the test override is deleted, so the invariant remains `storedResponseBytes <= byteCap()`. +Delete the `states.size > 1` exemption and update the old test-only cap comment. + +`expandPreviousResponseInput()` behavior: + +- resident: current expansion, unchanged; +- spill: read and fully validate, expand from payload, set replay provenance; +- missing/corrupt spill: leave body unchanged, set the failure WeakMap, increment the + matching counter, delete the stub and bad/missing file best-effort, then insert a + `spill-failed` tombstone for deterministic later calls; +- spill-failed: leave body unchanged and set `{code, reason:"spill_failed"}`; +- ordinary absent/TTL-expired id: preserve the current generic miss behavior. + +`previousResponseProviderState()` reads the provider copy on a spill stub without loading +the payload. Tombstones return undefined. + +Extend metrics without filesystem reads: + +```ts +export interface ResponseStateMetrics { + count: number; + residentCount: number; + spillStubCount: number; + tombstoneCount: number; + totalBytes: number; + spillPayloadBytes: number; + largestBytes: number; + oldestAgeMs: number; + spillWrites: number; + spillWriteFailures: number; + spillReadFailures: number; +} +``` + +`spillPayloadBytes` is the sum of refs, not file stat calls. `responseStateMetrics()` +remains observe-only. + +Snapshot compatibility: + +- Keep `version:2` and the exact current serialization/selection for resident rows. +- Persist spill stubs and tombstones because both are small; never inline spill payload. +- Continue accepting v1/v2 resident rows and recomputing resident bytes locally. +- On first load, collect referenced basenames, run orphan cleanup, then prune. Startup + cleanup removes unreferenced valid spill/temp names only after the 15-minute grace; + it never follows symlinks and obeys scan/cleanup caps. +- `clearResponseStateMemoryForTests()` clears memory only. `clearResponseStateForTests()` + also removes this test home's spill directory after deleting known refs. + +### MODIFY `src/server/responses/core.ts` + +Immediately after `body = expandPreviousResponseInput(body)` and before parsing, inspect +`previousResponseReplayFailure(body)`. Return one canonical response for all three known +failure reasons: + +```ts +formatErrorResponse( + 400, + "previous_response_not_found", + "Continuation state is unavailable or corrupt; resend the full conversation without previous_response_id.", +) +``` + +Add a `classifyError()` branch in `src/lib/errors.ts` mapping that explicit type to +`{type:"invalid_request_error", code:"previous_response_not_found"}`. Do not expose +the spill reason, response id, digest, path, or OS error. Remove the `:1445-1449` +warn-and-forward path for known spill failures; ordinary upstream-capable misses retain +their existing routing behavior. + +## Regression tests + +Extend `tests/responses-state.test.ts` with these exact tests/fixtures: + +- `spills the only oversized continuation and leaves resident bytes at or below cap` +- `does not swap a resident row to a stub before fsync and rename succeed` +- `replays provider metadata and function_call_output history through a spill stub` +- `replays a durable spill after simulated process restart` +- `returns previous_response_not_found for a missing spill file without forwarding delta` +- `returns previous_response_not_found for a corrupt or digest-mismatched spill` +- `spill write failure evicts resident bytes and records one bounded tombstone` +- `disk permission failure increments spillWriteFailures without retaining payload` +- `replacing a response id deletes its previous dedicated spill file` +- `TTL and count eviction delete dedicated spill files and release stub bytes` +- `startup orphan cleanup removes only old unreferenced regular spill files` +- `startup orphan cleanup preserves referenced young live and unrelated files` +- `concurrent flush and synchronous demotion cannot inline or lose the spill stub` +- `small entries retain the legacy v2 debounced snapshot representation` +- redefine `tests/responses-state.test.ts:535` as + `byte cap spills the newest-only chain instead of exempting it`; +- redefine `tests/responses-state.test.ts:856` as + `oversized entries replay from dedicated spill across restart while small entries use snapshot`. + +Add endpoint coverage in `tests/server-combo-failover-e2e.test.ts` (or the nearest +Responses endpoint test) named: + +- `known continuation spill failure returns structured previous_response_not_found before upstream I/O`. + +Fixtures must use an injected spill I/O seam, not chmod-only assertions that are unreliable +on Windows. The durability-order test records `write`, `fsync`, `close`, `harden`, `rename`, +`stub-swap` and asserts that order exactly. + +Verification: + +```bash +bun test tests/responses-state.test.ts tests/server-combo-failover-e2e.test.ts +bun run typecheck +bun run test +bun run privacy:scan +``` + +## Commit + +`feat(responses): hard-cap continuation state with durable spill` + +## Explicitly not changed + +- No replay truncation, history compaction, provider-metadata dropping, or naked-delta fallback. +- No use of the 2 MiB/24 MiB monolithic snapshot as spill storage. +- No change to small-entry debounce timing or newest-first snapshot selection. +- No change to `store:false` force policy, partial-output eligibility, replay provenance, + Cursor checkpoint semantics, or Kiro conversation identity. +- No provider adapter logic, request body cap, stream relay, or `#820` scheduler/lease work. +- No user-visible spill paths/digests and no security notes outside this closed implementation unit. diff --git a/devlog/_plan/260801_zero_leak_state_stores/020_blob_and_replay_caps.md b/devlog/_plan/260801_zero_leak_state_stores/020_blob_and_replay_caps.md new file mode 100644 index 000000000..063c18ed5 --- /dev/null +++ b/devlog/_plan/260801_zero_leak_state_stores/020_blob_and_replay_caps.md @@ -0,0 +1,266 @@ +# 020 — Cursor blob and replay-cache byte caps + +Date: 2026-08-01 +Work phase: wp3 +Depends on: none +Binding inputs: `000_state_store_inventory.md` §§2–3 and “Additional process caches”, `005_impl_roadmap.md` locked decision 2, `006_roadmap_audit_synthesis.md` R1-1/R1-3/R2-2. + +## Outcome + +Bound all four translation-duty stores owned by this phase at insertion time: + +1. Cursor shared blobs: per-blob + aggregate bytes, provenance-aware eviction, and + pinned-saturation rejection through the existing get-blob miss surface. +2. Antigravity replay: bounded calls and bytes per session while retaining recent + live identities and clear-on-invalid behavior. +3. Vision descriptions: clamp before insertion and byte-weight the existing LRU. +4. Anthropic image normalization: count key/sentinel metadata and prevent a single + cache value from bypassing the aggregate cap. + +## Current code and verified anchors + +- `src/adapters/cursor/native-exec.ts:75-136` stores 4,096 shared blobs with 15-minute + lazy TTL/count eviction but no byte or provenance field. `setBlob()` returns void. +- `src/adapters/cursor/native-exec.ts:202-227` maps missing `getBlobArgs` to an empty + `GetBlobResult`; remote `setBlobArgs` always acknowledges after insertion. +- `src/adapters/cursor/protobuf-request.ts:54-60,195-305` limits selected external roots + to 192/512 KiB only after candidates have been stored. Inventory warning: all + candidates are stored before selection. +- `src/adapters/google-antigravity-replay.ts:13-24` has a bounded outer map and an + unbounded inner `Map`. +- `src/adapters/google-antigravity-replay.ts:79-98` accumulates every call identity for + a session; `:105-125` replays live rows; `:128-135` clears on invalid/reset. +- `src/vision/index.ts:18-24,32-68` has a 256-entry LRU with no value-byte accounting. +- `src/vision/index.ts:197-204` clamps only when rendering, while `:341-343` caches the + unclamped `outcome.text.trim()`. +- `src/adapters/anthropic-image-normalize.ts:96-143` gives `pass`/`miss` zero weight, + has no entry cap, and inserts a single encoded value even if it exceeds 64 MiB. + +Blast-radius constraints from the inventory: missing Cursor blobs break hydration; +clearing Antigravity identities can cause invalid-signature 400s; clearing vision/image +caches repeats paid or expensive work. The remedy is bounded admission/LRU, not deletion +of the translation feature. + +## Cursor blob-store diff + +Modify `src/adapters/cursor/native-exec.ts`: + +```ts +const BLOB_TTL_MS = 15 * 60_000; +const BLOB_MAX_ENTRIES = 4_096; +const BLOB_MAX_ENTRY_BYTES = 16 * 1024 * 1024; +const BLOB_MAX_TOTAL_BYTES = 64 * 1024 * 1024; + +type CursorBlobProvenance = "local-regenerated" | "remote-setBlobArgs"; +interface CursorBlobEntry { + data: Uint8Array; + storedAt: number; + sizeBytes: number; + provenance: CursorBlobProvenance; +} +type CursorBlobAdmission = + | { admitted: true; replaced: boolean } + | { admitted: false; reason: "entry_too_large" | "pinned_saturation" }; + +let blobBytes = 0; +function setBlob(k: string, data: Uint8Array, provenance: CursorBlobProvenance): CursorBlobAdmission; +function deleteBlob(k: string): void; +function evictExpiredBlobs(at: number): void; +function evictOldestLocalBlob(): boolean; +``` + +Admission ladder for every insert, including replacement: + +1. Reject `data.byteLength > BLOB_MAX_ENTRY_BYTES`; do not remove an existing same-key + row until admission is known to succeed. +2. Sweep all TTL-expired rows, regardless of insertion order/provenance. +3. Compute projected bytes after subtracting a same-key replacement. +4. While projected bytes exceed the aggregate cap, evict the oldest + `local-regenerated` row. A remote row is pinned only while its TTL is live. +5. If still over cap, reject with `pinned_saturation`; the store and byte counter remain + unchanged. +6. On success, delete the old row through `deleteBlob()`, insert the immutable byte view, + add exact `byteLength`, and refresh Map recency. +7. Apply the 4,096 count cap using the same policy: expired, then oldest local. If only + live remote rows remain, reject rather than evict a pin or exceed the cap. + +`storeCursorBlob(data)` remains `Uint8Array -> Uint8Array`: it computes and returns the +SHA-256 id even if admission rejects. A later `getBlobArgs` receives the existing empty +result—the explicit protocol miss surface. It passes `local-regenerated`. + +`setBlobArgs` passes `remote-setBlobArgs`. `SetBlobResult` has no typed rejection field, +so it still acknowledges transport receipt; a rejected hash is intentionally absent and +subsequent `getBlobArgs` returns the explicit miss. Add one privacy-safe diagnostic +counter, not the hash or blob bytes. + +Expose accounting for 040: + +```ts +export interface CursorBlobMetrics { + count: number; + totalBytes: number; + localBytes: number; + pinnedBytes: number; + rejectedEntryTooLarge: number; + rejectedPinnedSaturation: number; + oldestAt: number | null; +} +export function cursorBlobMetrics(): CursorBlobMetrics; +export function evictOldestCursorBlobForBudget(): number; // local only; bytes released +``` + +Metrics read cached fields only. Add test-only reset/cap overrides; production constants +remain fixed. + +## Antigravity replay diff + +Modify `src/adapters/google-antigravity-replay.ts`: + +```ts +const REPLAY_MAX_CALLS_PER_SESSION = 256; +const REPLAY_MAX_BYTES_PER_SESSION = 2 * 1024 * 1024; +const REPLAY_MAX_SIGNATURE_BYTES = 64 * 1024; + +interface ReplayCall { signature: string; sizeBytes: number; touchedAtMs: number } +interface ReplayEntry { + byCall: Map; + bytes: number; + expiresAtMs: number; +} +``` + +Use `TextEncoder` byte lengths for canonical call key + signature. On observation: + +- ignore an individual call whose signature or combined row exceeds its cap; +- replace through a centralized delete/subtract helper; +- insert/refresh the observed call as newest; +- evict oldest inner calls until both count and bytes fit; +- refresh the outer session TTL only when at least one valid call was inserted; +- delete expired outer entries during observe/apply and retain the existing outer cap. + +`applyAntigravityReplay()` refreshes inner recency when a signature is actually matched; +it does not refresh session TTL. `clearAntigravityReplay()` still deletes the entire +session immediately after an upstream invalid-signature response (`src/adapters/google.ts:451-455`). + +Expose scalar accounting/test seams: + +```ts +export function antigravityReplayMetrics(): { + sessions: number; calls: number; totalBytes: number; largestSessionBytes: number; +}; +``` + +## Vision description-cache diff + +Modify `src/vision/index.ts`: + +```ts +const DESCRIPTION_CACHE_MAX_ENTRIES = 256; +const DESCRIPTION_CACHE_MAX_BYTES = 1024 * 1024; + +interface VisionDescriptionCache { + get(key: string): string | undefined; + set(key: string, value: string): void; + clear(): void; + metrics?(): { count: number; totalBytes: number; oldestAt: number | null }; +} +``` + +`BoundedLruDescriptionCache` stores `{value,sizeBytes,storedAt}` and tracks key UTF-8 +bytes plus value UTF-8 bytes. Replacement subtracts first; eviction happens before +insert until count and projected bytes fit. A single value that cannot fit is not cached. + +At `src/vision/index.ts:341-343`, change the insertion value to: + +```ts +const successfulText = outcome.error ? "" : clamp(outcome.text.trim(), DESC_MAX_CHARS); +if (identity.persistent && successfulText) descriptionCache.set(identity.key, successfulText); +``` + +Return the same clamped text in `outcome` so first use and cache hit are byte-identical. +Do not clamp error markers or change paid-sidecar admission/concurrency. + +## Image-normalization cache diff + +Modify `src/adapters/anthropic-image-normalize.ts:96-143`: + +```ts +const CACHE_BYTE_CAP = 64 * MiB; +const CACHE_MAX_ENTRIES = 4_096; +const CACHE_MAX_ENTRY_BYTES = 20 * MiB; +type CacheValue = { data: string; mediaType: string } | "pass" | "miss"; +interface CacheEntry { value: CacheValue; sizeBytes: number; storedAt: number } +``` + +`sizeBytes` includes UTF-8 key bytes for every row, sentinel marker bytes, media type, +and encoded data. `cachePut()` returns `boolean`; it skips an individually oversized +entry and evicts before insertion until both count and aggregate byte caps fit. There is +no zero-weight path. `cacheGet()` preserves true LRU and returns `entry.value`. + +Extend `getNormalizeStatsForTests()` and the 040 hook with `sentinelEntries`, +`metadataBytes`, and `oldestAt`. Budget eviction removes the oldest row through the same +centralized subtract helper. + +## Regression tests + +`tests/cursor-blob.test.ts`: + +- `admits a local blob exactly at the per-blob byte boundary` +- `rejects a local blob one byte above the per-blob boundary and getBlob returns miss` +- `replacement subtracts old bytes and refreshes local LRU` +- `aggregate admission evicts oldest local-regenerated blobs first` +- `remote setBlobArgs remains pinned within TTL while local blobs are evicted` +- `expired remote setBlobArgs becomes evictable before aggregate admission` +- `pinned saturation rejects a new remote blob without exceeding aggregate bytes` +- `rejected same-key replacement preserves the previously admitted blob` +- `blob metrics remain observe-only and exact after reset replacement and eviction`. + +`tests/google-antigravity-replay.test.ts`: + +- `preserves 20+ live signed calls below count and byte caps` +- `evicts oldest inner call at the exact per-session count boundary` +- `evicts oldest inner calls to satisfy aggregate session bytes` +- `does not cache one oversized signature` +- `apply refreshes matched call recency without extending session TTL` +- `clear-on-invalid drops the bounded session and all byte accounting`. + +`tests/vision-cache.test.ts`: + +- `clamps a successful description before cache insertion and first render` +- `cache hit returns the same clamped description without a sidecar call` +- `vision LRU evicts before insert at the aggregate byte boundary` +- `one oversized cache value is observed but not retained`. + +`tests/anthropic-image-normalize.test.ts`: + +- `unique pass and miss sentinels consume metadata bytes and hit the count cap` +- `encoded replacement keeps aggregate accounting exact` +- `one encoded value above maxEntrySize is returned but not cached` +- `cache eviction occurs before insertion and never exceeds 64 MiB`. + +Verification: + +```bash +bun test tests/cursor-blob.test.ts tests/google-antigravity-replay.test.ts \ + tests/vision-cache.test.ts tests/anthropic-image-normalize.test.ts +bun run typecheck +bun run test +``` + +## Commit + +`fix(state): bound Cursor blobs and translation replay caches` + +## Explicitly not changed + +- No Cursor blob-id/hash format, protobuf schema, selected-root 192/512 KiB policy, + hydration lookup, or remote TTL change. +- No eviction of live remote blobs merely to admit another pinned blob. +- No Antigravity identity algorithm, canonical JSON format, signature validity threshold, + Claude-on-Antigravity behavior, or clear-on-invalid behavior. +- No vision sidecar backend/model selection, paid-call concurrency, cache identity, or + image-description wording beyond using the existing clamp earlier. +- No image tier ladder, decode validation, wire mutation, demotion order, or 20 MiB + request-level image budget. +- No process-wide budget; 040 only consumes the accounting/demotion hooks defined here. + diff --git a/devlog/_plan/260801_zero_leak_state_stores/030_eviction_mechanisms.md b/devlog/_plan/260801_zero_leak_state_stores/030_eviction_mechanisms.md new file mode 100644 index 000000000..1c9088afc --- /dev/null +++ b/devlog/_plan/260801_zero_leak_state_stores/030_eviction_mechanisms.md @@ -0,0 +1,188 @@ +# 030 — state-store eviction mechanisms + +Date: 2026-08-01 +Work phase: wp4 +Depends on: none +Binding inputs: `000_state_store_inventory.md` §§4–6, `005_impl_roadmap.md` locked decision 3 and store-by-store lock, `006_roadmap_audit_synthesis.md` R1-2/R1-5/S2-5/S3-2. + +## Outcome + +Apply exactly three mechanisms, selected per store rather than by convenience: + +1. exact clock-TTL sweep where current getters already treat an expired row as absent; +2. config/account-generation reconciliation where a key is owned by the current configuration; +3. admission or owner-specific release where a live resource cannot be evicted safely. + +Live keys keep their current behavior. Warning memos and Codex quota rows never gain a +TTL. Windows ACL successes are removed only for the actual renamed temp path. Retained +diagnostic values are byte-normalized at admission in 035, not swept by time. + +## Shared lifecycle utility + +### NEW `src/lib/state-store-sweeper.ts` + +```ts +export const STATE_SWEEP_INTERVAL_MS = 60_000; + +export interface GenerationContext { + generation: number; + providerNames: ReadonlySet; + comboIds: ReadonlySet; + codexAccountIds: ReadonlySet; + oauthAccountKeys: ReadonlySet; + configRoots: ReadonlySet; +} +export interface StateStoreRegistration { + name: string; + sweepExpired?: (now: number) => number; + reconcileGeneration?: (context: GenerationContext) => number; +} +export interface StateSweepResult { storesVisited: number; rowsRemoved: number } + +export function registerStateStore(registration: StateStoreRegistration): () => void; +export function sweepExpired(now?: number): StateSweepResult; +export function sweepExpiredOnWrite(now?: number): StateSweepResult; +export function reconcileStateGeneration(context: GenerationContext): StateSweepResult; +export function startStateStoreSweeper(options?: StateStoreSweeperOptions): { stop(): void }; +export function stopStateStoreSweeper(): void; +``` + +Registration names are static and unique; replacement does not duplicate callbacks. +`startStateStoreSweeper()` replaces the prior singleton, creates one 60-second interval, +and invokes `unref?.()`. One callback failure logs only the static registration name and +does not stop later callbacks. `sweepExpiredOnWrite()` runs synchronously after a +successful owner write; it creates no promise tail. Reconciliation never runs from the +clock timer. + +Start the singleton beside the watchdog in `src/server/index.ts:300-316`; stop it in +`drainAndShutdown()` at `src/server/lifecycle.ts:70-95`. If side-effect registration +would create an import cycle, add explicit wiring in NEW +`src/lib/state-store-registrations.ts`; do not add a generic helper module. + +## Locked store-by-store table + +The inventory shorthand `catalog/*` resolves to `src/codex/catalog/*` in this checkout. + +| Store and current anchor | Locked mechanism | Diff-level owner change | +|---|---|---| +| Subagent `modelHealth`, `src/codex/subagent-model-fallback.ts:35-42,160-171,229-248` | TTL sweep | Export `sweepExpiredSubagentModelHealth(now)`; delete `unavailableUntil <= now`; invoke write sweep after `modelHealth.set`; never touch `quotaPrimedAt.global`. | +| Combo cooldown, `src/combos/failover.ts:5-19,40-77` | TTL sweep | Export/register `sweepExpiredComboTargetCooldowns(now)` and preserve every live Retry-After row. | +| API-key cooldown, `src/providers/key-failover.ts:17-53,89-130,174-190` | TTL sweep | Delete only `cooldownUntil <= now`; keep key ordering and current exact-key lazy cleanup. | +| Anthropic health, `src/oauth/anthropic-routing.ts:34-40,96-123` | TTL sweep | Export a health sweep using the same semantic deadline as `isCooled`; affinity at `:234-241` remains its existing 2,000/24 h LRU. | +| XAI permanent-failure verdicts, `src/oauth/index.ts:54-61,326-327` | TTL sweep | Globally delete the same 30-second-expired verdicts that `cached()` already treats as absent; S3-2 forbids reconciliation-only handling. | +| Warning memos, `src/codex/catalog/provider-fetch.ts:219`, `src/codex/catalog/aggregation.ts:37-39,281-283`, `src/router.ts:154-180`, `src/combos/request.ts:4-38`, `src/config.ts:435,2052-2053` | Reconciliation only | Clear/rebuild only after a complete new config generation. No TTL: time expiry would re-emit intentionally suppressed warnings. | +| Codex quota, `src/codex/quota.ts:15-27,51,261-326` | Reconciliation only | Remove accounts absent from current account generation and persist the reduced map. The 6 h rule is hydration admission, not live expiry. | +| Provider quota history, `src/providers/quota.ts:52-61,195-239,278-342` | Reconciliation plus admission | Remove dead provider/account keys; cap distinct live flights in 035; never detach an accepted flight. | +| Codex routing health, `src/codex/routing.ts:85-138,209-243` | Reconciliation | Remove account-wide and quota-scope rows only for deleted accounts; preserve live Retry-After and probe generation. | +| Model-cache history, `src/codex/model-cache.ts:42-56,114-147` | Reconciliation | Add `reconcileModelCacheProviders(validProviders)` covering cache, failure, status, and live-count maps atomically. | +| Pool rotation, `src/codex/pool-rotation.ts:6-12,44-80,180-185` | Reconciliation | Remove deleted pool/account rows while preserving current sticky/RR weights. | +| Combo rotation, `src/combos/resolve.ts:13-20,85-105,161-167` | Reconciliation | Remove deleted combo ids and target weights; preserve current target order. | +| Guardian backoff, `src/oauth/token-guardian.ts:54-99,118-225` | Reconciliation | Remove deleted provider/account rows; keep a configured revoked account at its current backoff. | +| Reauth state, `src/codex/account-runtime-state.ts:1-13` plus OAuth account maps | Reconciliation | Remove only keys absent after a successful persisted account mutation. | +| GCP ADC, `src/lib/gcp-adc.ts:61-66,83-127,274-302` | Reconciliation plus admission | Remove source fingerprints absent from the authoritative ADC source set; cap active source flights in 035; retain current expiry-on-resolve. | +| Config ownership, `src/lib/config-ownership.ts:79-87,233-258` | Reconciliation | Remove a root only when absent from manifest/current roots and no owned path references it. Never infer inactivity from age. | +| PID and config warnings, `src/config.ts:413-435,2052-2053,2112-2121` | Reconciliation | Delete PID rows only after failed identity/liveness proof; warning rows follow config generation. | +| OAuth login/abort/manual maps, `src/oauth/index.ts:823-904,939-1020` | Reconciliation plus admission | Remove dead provider/account generations; 035 caps flows/probes and pending-code bytes; never evict an in-progress owner. | +| Active turns/sockets/workers/slots | Admission only | Implement the hard caps and coherent busy responses in 035; do not register an accepted owner and later sweep it away. | + +Each owner exports a semantic sweep/reconcile function. The coordinator passes complete +sets; it must not recreate owner key strings. Add a monotonic config generation beside +the canonical loaded-config owner, and call reconciliation only after successful initial +load, successful disk commit, account add/delete/reauth commit, or catalog sync with the +complete provider/combo set. Failed parse/save and speculative routing do not advance it. + +## Windows ACL delete-after-rename contract + +Current anchors: + +- `src/lib/windows-secret-acl.ts:37-40` retains directory/file success paths and timeout keys. +- `src/lib/windows-secret-acl.ts:363-367` permits a destination key only for timeout memoization. +- `src/lib/windows-secret-acl.ts:375-455` adds the actual `targetPath` after successful `icacls`. +- `src/config.ts:96-113,174-197` hardens unique `*.ocx...tmp` files, then renames them. + +Add: + +```ts +export function forgetHardenedSecretPath(targetPath: string): void { + hardenedPaths.delete(targetPath); +} +export function hardenedSecretPathCountForTests(): number; +``` + +For both atomic writers the order is fixed: + +```ts +io.harden(tmp); // memo belongs to this exact temp +io.rename(tmp, path); // durable destination replacement +forgetHardenedSecretPath(tmp); // only after successful rename +``` + +After a failed transaction, forget the success memo only after that exact temp has been +unlinked. If a hardened residual remains, keep its memo until removal. Never add the +destination to `hardenedPaths`, never reuse one temp's success for the next temp, and +never clear `timedOutPaths[destination]` on rename. + +This code protects credential/config files and requires explicit security review under +`AGENTS.md`/`MAINTAINERS.md`. The security reviewer must specifically attest that the +second temp for one destination executes `icacls` again and failure remains fail-closed. + +## Diagnostic values: classification, not a clock sweep + +Crash trace strings at `src/lib/crash-guard.ts:206-245`, debug lines/subscribers at +`src/lib/debug-log-buffer.ts:3-35`, injection lines at +`src/lib/injection-debug-log.ts:10-27`, Claude metadata at +`src/claude/inbound-debug.ts:40-106`, fixed breadcrumbs at +`src/lib/sidecar-tracker.ts:10-48`, and affinity components at +`src/codex/routing.ts:105-109,624-688` / `src/oauth/anthropic-routing.ts:34-40,234-241` +have no expiry semantics. They therefore use mechanism 3: 035's UTF-8 byte admission +and truncation marker. Do not register them with `sweepExpired()` and do not truncate +tool JSON, credential values, or route keys into ambiguous identities. + +## Regression cases + +Add `tests/state-store-sweeper.test.ts`: + +- `global fake-clock sweep visits every registered TTL owner once` +- `expiry boundary removes expired rows and preserves live rows` +- `one throwing owner does not block later sweep owners` +- `write-trigger uses the same callbacks without creating a queue` +- `timer start is singleton unrefed and stop is idempotent` +- `reconciliation runs only for a newer complete generation` +- `stale or duplicate generation cannot delete current keys` + +Extend nearest owner suites with: + +- `subagent health sweep removes expired untouched model keys` +- `combo and API-key sweeps preserve live Retry-After rows` +- `Anthropic health and XAI verdict sweeps use their exact semantic deadlines` +- `warning reconciliation never expires a current-generation warning by time` +- `Codex quota reconciliation ignores row age and removes only deleted accounts` +- `model-cache reconciliation removes all four maps for one deleted provider` +- `pool combo guardian GCP ownership and reauth reconciliation preserve current keys` + +Extend `tests/windows-secret-acl.test.ts` and the atomic-writer cases in +`tests/config.test.ts`: + +- `second atomic temp for the same destination is hardened again` +- `rename success forgets only the actual temp success memo` +- `destination timeout memo never vouches for a new temp` +- `failed unlink retains the residual temp success memo` +- `later successful residual cleanup forgets that exact memo`. + +Verification: + +```bash +bun test tests/state-store-sweeper.test.ts tests/windows-secret-acl.test.ts tests/config.test.ts +bun run typecheck +bun run test +bun run privacy:scan +``` + +## Explicitly not changed + +- No TTL for warning memos or live Codex quota rows. +- No deletion of live Retry-After, accepted resources/flights, current sticky state, + last-good models, or configured guardian backoff. +- No process-wide accounting interface; 030 provides sweeping only. +- No provider event semantics, `#820` scheduler architecture, destination-level ACL + success memo, or fail-open hardening behavior. diff --git a/devlog/_plan/260801_zero_leak_state_stores/035_registry_admission_caps.md b/devlog/_plan/260801_zero_leak_state_stores/035_registry_admission_caps.md new file mode 100644 index 000000000..174ab4e61 --- /dev/null +++ b/devlog/_plan/260801_zero_leak_state_stores/035_registry_admission_caps.md @@ -0,0 +1,334 @@ +# 035 — registry, flight, discovery, and diagnostic admission caps + +Date: 2026-08-01 +Work phase: wp4b (must land before 040) +Depends on: none +Binding inputs: `000_state_store_inventory.md` §§3–6, `005_impl_roadmap.md` 035 regression classes, `006_roadmap_audit_synthesis.md` S2-1/S2-2/S3-1/S3-3. + +## Outcome + +Close the operational stores omitted by the initial roadmap audit. Every active registry +gets a finite hard admission cap and coherent busy result; every single-flight has a +finite distinct-key cap and stale-owner policy; discovery/usage/MCP payloads are bounded +before full materialization; retained diagnostic and affinity strings are truncated at +insertion with a visible marker. Existing accepted owners are never silently untracked. + +This phase provides retained-byte accounting hooks for 040. It does not implement the +process-wide eviction policy. + +## Shared primitives + +### NEW `src/lib/admission.ts` + +```ts +export const RETAINED_TRUNCATION_MARKER = "\n…[truncated by opencodex]"; + +export class ResourceAdmissionError extends Error { + readonly code = "server_busy"; + constructor(readonly resource: string, readonly limit: number); +} +export interface AdmissionMetrics { + active: number; + peak: number; + admitted: number; + rejected: number; + releaseMisses: number; +} +export interface AdmissionLease { release(): void } +export function createAdmissionGate(name: string, limit: number): { + tryAcquire(): AdmissionLease | null; + metrics(): Readonly; +}; +export function truncateRetainedUtf8(value: string, maxBytes: number): string; +export function retainedUtf8Bytes(value: string): number; +``` + +`truncateRetainedUtf8()` cuts on a valid UTF-8 boundary and reserves marker bytes. It +never returns an invalid surrogate fragment. Metrics are scalar-only and monotonic except +`active`; they retain no ids, keys, paths, URLs, commands, request bodies, or errors. + +## Debug subscribers and diagnostic rings + +Current anchors: + +- `src/lib/debug-log-buffer.ts:3-35` retains 2,000 unbounded lines and an unbounded + listener set. +- `src/lib/injection-debug-log.ts:10-27` retains 2,000 unbounded lines. +- `src/claude/inbound-debug.ts:40-106` retains 20 variable metadata rows. +- `src/lib/crash-guard.ts:206-245` retains 12 traces with unbounded URL/origin/rejection. +- fixed string slots include `src/lib/sidecar-tracker.ts:10-48`, startup health, + main-account cache, star state, shim error, and project-config warnings listed in + inventory §6. +- Codex/Anthropic affinities are count-bounded at + `src/codex/routing.ts:105-109,624-688` and + `src/oauth/anthropic-routing.ts:34-40,62-63,234-241`, but ids are not byte-bounded. + +Constants and changes: + +```ts +const MAX_DEBUG_SUBSCRIBERS = 64; +const MAX_DEBUG_LINE_BYTES = 16 * 1024; +const MAX_DIAGNOSTIC_VALUE_BYTES = 8 * 1024; +const MAX_AFFINITY_COMPONENT_BYTES = 512; +``` + +- `subscribeDebugLogEntries()` throws `ResourceAdmissionError("debug_subscribers",64)` + before insertion. Existing subscribers remain active; unsubscribe is idempotent and + updates release-miss metrics. +- The management SSE/tail route catches that error and returns structured 503 with + `Retry-After: 1`; no listener is added on rejection. +- Debug/injection lines are truncated before ring insertion and before listener fanout; + console logging may retain its existing safe line, but no ring stores the original. +- Crash trace URL/origin/rejected fields and fixed-slot strings use the 8 KiB cap at + assignment. Preserve redaction first, then truncate. +- Affinity key components are normalized at admission. Oversized thread/scope/account ids + are rejected from affinity caching, not truncated into collision-prone keys; displayed + diagnostic aliases may be truncated. Existing request routing continues without affinity. + +Expose hooks: + +```ts +export function debugBufferMetrics(): { entries: number; bytes: number; subscribers: AdmissionMetrics; oldestAt: number | null }; +export function injectionBufferMetrics(): { entries: number; bytes: number; oldestAt: number | null }; +export function crashRingMetrics(): { entries: number; bytes: number; oldestAt: number | null }; +export function evictOldestDebugEntryForBudget(): number; +export function evictOldestInjectionEntryForBudget(): number; +export function evictOldestCrashTraceForBudget(): number; +``` + +All ring replacements/evictions use centralized subtract helpers. + +## Active turns, WebSockets, workers, and slots + +Current anchors: + +- `src/server/lifecycle.ts:13-23,37-67` registers every live turn with no admission cap. +- `src/codex/websocket-registry.ts:4-35,47-73` tracks sockets by account until close. +- `src/storage/worker-lifecycle.ts:25-80` tracks workers until deterministic termination. +- `src/storage/storage-mutation-coordinator.ts:20-64` has one slot per distinct home but + no total-home cap. + +Production defaults: + +```ts +export const MAX_ACTIVE_TURNS = 256; +export const MAX_TRACKED_CODEX_WEBSOCKETS = 128; +export const MAX_LIVE_STORAGE_WORKERS = 16; +export const MAX_ACTIVE_STORAGE_HOME_SLOTS = 32; +``` + +Change signatures: + +```ts +export function tryRegisterTurn(ac: AbortController): AdmissionLease | null; +export function tryRegisterCodexWebSocket(ws: ServerWebSocket): AdmissionLease | null; +export function tryRegisterStorageWorker(worker: Worker): AdmissionLease | null; +export function tryBeginStorageMutation(...): + | { acquired: true; lease: AdmissionLease } + | { acquired: false; error: "storage_mutation_busy" }; +``` + +Admission must occur before response stream/upgrade/worker creation. HTTP and WS rejects +use structured 503 `server_busy`; storage retains `storage_mutation_busy`. Accepted work +holds one idempotent lease released from every current finish/cancel/error/close/finally +path. Do not throw after a Worker or socket has been created but before it is tracked. + +Add `activeRegistryMetrics()` returning per-registry `AdmissionMetrics`. `releaseMisses` +is the leak signal: increment only when an unregister/finish path attempts to release an +unknown owner; never remove another owner to hide it. + +## Codex credential refresh flights + +Current `src/codex/account-store.ts:268-270,349-465` deduplicates by grant fingerprint +and deletes in `finally`, but distinct fingerprints have no cap and a stuck Promise is +joined forever. + +```ts +const MAX_CODEX_REFRESH_FLIGHTS = 32; +const CODEX_REFRESH_FLIGHT_STALE_MS = 120_000; +interface RefreshFlight { + promise: Promise; + startedAt: number; + abort: AbortController; +} +const refreshLocks = new Map(); +``` + +Admission order: + +1. Same fingerprint and age <=120 s: join it. +2. Same fingerprint older than 120 s: abort with a typed stale reason, remove only if + still the same owner, then create a replacement. +3. New fingerprint with 32 live rows: throw + `CodexCredentialRefreshBusyError` before file lock/fetch. +4. Thread `AbortSignal.any([flight.abort.signal, timeout])` through lock wait and fetch. +5. `finally` deletes only if `refreshLocks.get(key) === flight`. + +An aborted stale owner remains awaited by its original callers and settles with a typed +retryable error; it is not silently detached. The replacement is the only mapped owner. + +## Management usage-read bound + +Current `src/usage/log.ts:389-403,470-511` reads `stat.size` into one Buffer, converts all +text, splits every line, and retains the parsed array in one revision-keyed Promise. + +```ts +const MANAGEMENT_USAGE_MAX_READ_BYTES = 64 * 1024 * 1024; +const MANAGEMENT_USAGE_MAX_ENTRIES = 200_000; +const MANAGEMENT_USAGE_FLIGHT_STALE_MS = 30_000; +interface ManagementUsageSnapshot { + entries: PersistedUsageEntry[]; + revision: UsageLogRevision; + truncatedPrefixBytes: number; +} +``` + +- Read at most the newest 64 MiB through bounded 1 MiB chunks; when starting mid-file, + discard the first partial line. +- Parse batches cooperatively and retain at most the newest 200,000 valid rows. +- Return `truncatedPrefixBytes` so callers/GUI cannot present capped history as complete. +- A 30-second-stale same-revision flight is aborted through a local controller and + replaced. At most one management usage-read flight exists. +- Never allocate `Buffer.allocUnsafe(stat.size)` for a file over the cap. + +Update usage summary/routes to preserve totals for the returned window and expose an +additive `historyTruncated` boolean; do not fabricate lifetime totals. + +## Cursor model discovery and gather flights + +Inventory anchors: + +- `src/adapters/cursor/live-models.ts:98-125` buffers all RPC chunks before applying the + 500-id result cap. +- `src/codex/catalog/provider-fetch.ts:64-129,645-665` keeps one gather Promise per + distinct config fingerprint without a concurrency cap. + +```ts +const CURSOR_MODEL_DISCOVERY_MAX_BYTES = 4 * 1024 * 1024; +const MAX_CONCURRENT_CATALOG_GATHERS = 8; +``` + +Reject an announced `content-length` above 4 MiB before reading and cancel the body once +streamed chunks exceed 4 MiB. Decode only after admission. Keep the existing 500 model-id +result cap. Add a gate around distinct gather fingerprints; same-fingerprint callers +still join, while the ninth distinct gather receives a typed busy result and starts no +provider requests. Release in `finally` and expose scalar peak/reject counters. + +## OAuth flow/probe and pending-code bounds + +Current `src/oauth/index.ts:54-61,823-904,939-1020` owns provider/flow maps, XAI probes, +pending pasted code, and refresh flights. The management boundary already rejects pasted +input above 4,096 chars at `src/server/management/oauth-account-routes.ts:183-190`, but +internal callers can bypass that route. + +```ts +const OAUTH_PENDING_CODE_MAX_BYTES = 4 * 1024; +const MAX_ACTIVE_OAUTH_FLOWS = 32; +const MAX_ACTIVE_OAUTH_PROBES = 16; +``` + +Enforce pending-code UTF-8 bytes again in the owner before assignment. Flow/probe +admission happens before timer, listener, browser/device request, or Promise creation; +reject with the existing 409 busy surface. Existing generation owners remain until their +normal finish/abort timer. Reconciliation of dead provider/account keys remains in 030. + +## MiMo bootstrap value cap + +Current `src/adapters/mimo-free.ts:27-35,92-133` caches one unbounded JWT and parses the +entire bootstrap JSON after a 15-second fetch timeout. + +```ts +const MIMO_BOOTSTRAP_MAX_BYTES = 128 * 1024; +const MIMO_JWT_MAX_BYTES = 64 * 1024; +``` + +Use bounded response-body reading, reject content-length/chunks above 128 KiB before +`JSON.parse`, require `jwt` UTF-8 bytes <=64 KiB, then cache and parse expiry. Oversized +values throw `MiMo bootstrap response too large`, never enter `cachedJwt`, and preserve +single-flight cleanup. + +## Cursor MCP manager payload caps + +Current `src/adapters/cursor/mcp-manager.ts:54-70,80-139,152-195,198-223` retains +configured connections/tool schemas and materializes tool/resource payloads without +local count/byte caps. + +```ts +const CURSOR_MCP_MAX_SERVERS = 32; +const CURSOR_MCP_MAX_TOOLS = 512; +const CURSOR_MCP_MAX_RESOURCES = 1_024; +const CURSOR_MCP_MAX_SCHEMA_BYTES = 256 * 1024; +const CURSOR_MCP_MAX_CATALOG_BYTES = 4 * 1024 * 1024; +const CURSOR_MCP_MAX_RESULT_BYTES = 8 * 1024 * 1024; +``` + +Validate configured server count in the constructor. During `indexTools`, measure +advertised name + description + canonical schema before adding; stop with a typed +catalog-too-large error instead of retaining a partial catalog. Resource listings and +call/read results are measured before normalization/copy into manager-owned objects; +cancel/reject over cap. `dispose()` remains authoritative and clears accounting before +awaiting client closes. Do not truncate tool schemas or resource payloads into invalid data. + +## Regression tests + +Concrete names/fixtures. Put cross-registry lease/accounting cases in NEW +`tests/active-registry-admission.test.ts`; extend existing owner suites for the rest: + +- `debug subscriber 65 is rejected while the first 64 still receive entries` +- `subscriber unsubscribe is idempotent and leak metric records unknown release` +- `debug injection crash and fixed-slot strings truncate on UTF-8 boundary with marker` +- `affinity rejects an oversized key component without colliding or changing routing` +- `active turn 257 returns structured server_busy before handler work` +- `websocket 129 rejects upgrade without entering account registry` +- `storage worker 17 is not spawned and accepted workers drain normally` +- `storage home slot 33 returns storage_mutation_busy without dropping active slots` +- `active registry peak rejected and release-miss metrics are monotonic` +- `same refresh grant joins a live flight` +- `33rd distinct refresh grant is rejected before file lock and fetch` +- `stale refresh flight is aborted and replaced without deleting the replacement` +- `usage reader never requests more than 64 MiB from an oversized log` +- `usage reader returns newest complete capped rows and historyTruncated` +- `stale usage-read flight is replaced and old completion cannot clear new owner` +- `Cursor model discovery rejects announced and streamed 4 MiB overflow before decode` +- `ninth distinct catalog gather is busy while same-fingerprint caller still joins` +- `OAuth pending code rejects 4097 UTF-8 bytes in the owner` +- `OAuth flow 33 and probe 17 reject before creating timers or requests` +- `MiMo accepts exact JWT boundary and rejects one byte over without caching` +- `MCP exact catalog boundary admits and one byte over disposes partial state` +- `MCP oversized tool result and resource are rejected without truncated payload`. + +Verification: + +```bash +bun test tests/debug.test.ts tests/active-registry-admission.test.ts \ + tests/codex-websocket-registry.test.ts tests/storage-worker-lifecycle.test.ts \ + tests/xai-refresh-lock.test.ts tests/usage-log.test.ts tests/cursor-hardening.test.ts \ + tests/gather-routed-models-single-flight.test.ts tests/oauth-manual-code.test.ts \ + tests/codex-auth-api.test.ts tests/mimo-free-provider.test.ts tests/cursor-mcp-manager.test.ts +bun run typecheck +bun run test +bun run privacy:scan +``` + +File ownership is fixed as follows: debug subscriber/value cases extend +`tests/debug.test.ts`; turns/sockets/workers/slots use the new cross-registry file plus +the existing websocket/worker suites; refresh flights extend `tests/xai-refresh-lock.test.ts`; +usage extends `tests/usage-log.test.ts`; Cursor discovery and gather admission extend +`tests/cursor-hardening.test.ts` and `tests/gather-routed-models-single-flight.test.ts`; +OAuth owner/code and Codex flow/probe cases extend `tests/oauth-manual-code.test.ts` and +`tests/codex-auth-api.test.ts`; MiMo extends `tests/mimo-free-provider.test.ts`; MCP +extends `tests/cursor-mcp-manager.test.ts`. Do not invent parallel test harness names. + +## Commit + +`fix(runtime): cap registries flights and retained diagnostics` + +## Explicitly not changed + +- No forced eviction/untracking of accepted turns, sockets, workers, mutation slots, + refresh grants, OAuth flows, or probes. +- No `#820` scheduler/session-lane architecture. +- No credential, token, account id, URL, path, command, or body in metrics. +- No truncation of JSON tool schemas/results into syntactically valid-looking partials. +- No change to provider retry/rotation, MCP tool execution semantics, or usage-log disk format. +- No process-wide demotion; 040 consumes only the retained-ring accounting hooks. diff --git a/devlog/_plan/260801_zero_leak_state_stores/040_app_bytes_observability.md b/devlog/_plan/260801_zero_leak_state_stores/040_app_bytes_observability.md new file mode 100644 index 000000000..dbdeaaeab --- /dev/null +++ b/devlog/_plan/260801_zero_leak_state_stores/040_app_bytes_observability.md @@ -0,0 +1,254 @@ +# 040 — app-owned byte observability and retained-store budget + +Date: 2026-08-01 +Work phase: wp5 +Depends on: 010, 020, 035 (030 provides no accounting hook) +Binding inputs: `000_state_store_inventory.md`, `005_impl_roadmap.md` budget scope split and locked decision 4, `006_roadmap_audit_synthesis.md` R1-6/S2-3/S3-1. + +## Outcome + +Expose one privacy-safe `appOwnedBytes` block on authenticated +`GET /api/system/memory`, then enforce a configurable 256 MiB process-wide budget over +evictable retained stores only. Enforcement always demotes oldest unpinned retained +entries in fixed category order: + +`logs/rings -> caches -> blobs -> continuation spill`. + +In-flight translator buffers and serialized-tail backlogs are observed as current/high- +water counters but never evicted by this budget. Their hard admission is owned by 050. + +## Current code and anchors + +- `src/server/management/system-routes.ts:1-31` documents scalar/privacy constraints. +- `src/server/management/system-routes.ts:33-95` assembles process memory, + `responseState`, inspector counters, watchdog, and active-turn scalars. +- `src/responses/state.ts:371-408` is an existing observe-only retained-store seam. +- `src/server/memory-watchdog.ts:53-60,102-155` owns a separate warn-only RSS/native + watchdog with a 360-sample bounded ring; it does not manage app-owned state. +- `src/types.ts:531-725` contains the top-level `OcxConfig` runtime fields. +- `src/config.ts:703-733` begins the Zod config schema and reaches adjacent scalar + fields; write-time validation follows + the existing positive-integer helpers at `:530-550`. +- `src/server/index.ts:300-316` starts process-wide memory/scheduler singletons. +- `tests/memory-watchdog.test.ts:162-233` pins the current endpoint scalar shape. + +## Config decision + +Choose a user-configurable top-level MiB field, not an environment-only or fixed knob: + +```ts +export interface OcxConfig { + /** Evictable retained app-state budget in MiB. Default 256; valid 64..4096. */ + appOwnedMemoryBudgetMb?: number; +} + +export const DEFAULT_APP_OWNED_MEMORY_BUDGET_BYTES = 256 * 1024 * 1024; +export const MIN_APP_OWNED_MEMORY_BUDGET_MB = 64; +export const MAX_APP_OWNED_MEMORY_BUDGET_MB = 4_096; +export function resolveAppOwnedMemoryBudgetBytes(value: unknown): number; +``` + +Rationale: 256 MiB leaves room for the existing 64 MiB continuation, 64 MiB Cursor +blob, and 64 MiB image-normalization ceilings plus bounded logs/caches, while still +forcing cross-store demotion before retained state becomes multi-GiB. MiB is readable in +`config.json`; all metrics remain bytes. Reject non-integer/out-of-range values at the +management write boundary. On load, malformed legacy hand edits degrade to the default +without resetting unrelated config, following existing schema doctrine. + +Update English and translated configuration tables. This is a user-facing operational +control, so docs cannot be English-only or imply it caps RSS/native memory. + +## NEW `src/lib/app-owned-memory.ts` + +```ts +export type AppOwnedRetainedCategory = "logs" | "caches" | "blobs" | "continuation"; +export type AppOwnedObservedCategory = "translator" | "serialized_tails"; + +export interface RetainedStoreSnapshot { + count: number; + bytes: number; + evictableBytes: number; + pinnedBytes: number; + oldestAt: number | null; +} +export interface RetainedStoreRegistration { + id: string; + category: AppOwnedRetainedCategory; + snapshot(): RetainedStoreSnapshot; // observe-only, no sweep/load/serialization + evictOldest(): number; // bytes released; 0 means no candidate +} +export interface ObservedBufferRegistration { + id: string; + category: AppOwnedObservedCategory; + snapshot(): { currentBytes: number; highWaterBytes: number; active: number }; +} +export interface AppOwnedBytesSnapshot { + budgetBytes: number; + retainedBytes: number; + evictableBytes: number; + pinnedBytes: number; + overBudgetBytes: number; + stores: Record; + observedInFlight: Record; + enforcement: { + runs: number; + entriesDemoted: number; + bytesReleased: number; + noEvictableCandidate: number; + }; +} + +export function registerRetainedStore(registration: RetainedStoreRegistration): () => void; +export function registerObservedBuffer(registration: ObservedBufferRegistration): () => void; +export function configureAppOwnedMemoryBudget(bytes: number): void; +export function appOwnedBytesSnapshot(): AppOwnedBytesSnapshot; +export function enforceAppOwnedMemoryBudget(): AppOwnedBytesSnapshot; +export function resetAppOwnedMemoryForTests(): void; +``` + +Registrations are unique by static id. Duplicate registration replaces callbacks and +does not duplicate bytes. Snapshot collection catches one owner failure and reports its +store as zeros; it never invokes `evictOldest()`. + +## Retained-store registrations + +Register hooks delivered by 010/020/035 and existing owners: + +| Category | Store ids | Demotion rule | +|---|---|---| +| logs | `request_log`, `provider_debug`, `injection_debug`, `claude_debug`, `crash_ring` | Remove oldest complete diagnostic row; never truncate an attempt array during budget enforcement. | +| caches | `image_normalize`, `vision_descriptions`, `antigravity_replay`, `model_cache`, `usage_summary` | Remove oldest LRU/session/provider value through owner accounting. Preserve “other” usage totals. | +| blobs | `cursor_blobs` | Remove oldest local-regenerated blob only. Live remote blobs report as pinned. | +| continuation | `responses_continuation` | Demote oldest resident row through 010 durable spill. Spill stubs/tombstones are not repeatedly demoted. | + +The request-log owner (`src/server/request-log.ts:150-154,218-246`) must add per-entry +UTF-8 byte accounting and a centralized oldest delete. Normalize individual retained +diagnostic strings per 035, but preserve retry/failover attempt structure. + +Model cache and usage summary values receive owner-local byte accounting before they can +register. Usage summary overflow aggregates excess model cardinality into an `other` +bucket without dropping token/cost totals; it is not permissible to delete totals. + +## Enforcement algorithm + +```ts +const CATEGORY_ORDER: readonly AppOwnedRetainedCategory[] = [ + "logs", "caches", "blobs", "continuation", +]; + +while (retainedBytes() > budgetBytes) { + const candidate = oldestEvictableStoreInFirstNonemptyCategory(CATEGORY_ORDER); + if (!candidate) { counters.noEvictableCandidate++; break; } + const released = candidate.evictOldest(); + if (released <= 0) markCandidateIneligibleForThisRun(candidate); + else recordAndContinue(released); +} +``` + +Within the first category that has any evictable bytes, choose the store whose +`oldestAt` is earliest. Re-snapshot after every demotion; never trust a stale projected +counter across a spill or replacement. A per-run visited/no-progress set prevents loops. + +Call enforcement after successful retained-store insertion/replacement, after startup +registrations, and after live config budget changes. Observation happens first: update +owner bytes, then enforce. Never reject new request admission as the first lever. + +Edge contracts: + +- A single entry over global budget is demoted/evicted even when it is the only entry. +- Pinned-only saturation may leave `overBudgetBytes > 0`; record + `noEvictableCandidate`, warn once per 60 seconds, and do not violate per-store pinning. +- Continuation spill failure follows 010 tombstone eviction and counts released RAM; + budget enforcement does not keep the row hot. +- If a callback throws or reports zero release, move to the next candidate without + spinning and retain honest over-budget metrics. +- Budget decrease is applied synchronously through the same demotion order. + +## `/api/system/memory` payload + +At `src/server/management/system-routes.ts:74-95`, add: + +```ts +appOwnedBytes: appOwnedBytesSnapshot(), +``` + +Retain `responseState` for compatibility during this unit; it may duplicate a scalar +subset. `appOwnedBytes` contains only static store ids, categories, counts, byte totals, +timestamps/ages, and counters. It must never include keys, ids, model/provider names, +paths, hashes, errors, commands, tool arguments, prompts, URLs, or account data. + +Recommended wire example: + +```json +{ + "budgetBytes": 268435456, + "retainedBytes": 0, + "evictableBytes": 0, + "pinnedBytes": 0, + "overBudgetBytes": 0, + "stores": {}, + "observedInFlight": {}, + "enforcement": { + "runs": 0, + "entriesDemoted": 0, + "bytesReleased": 0, + "noEvictableCandidate": 0 + } +} +``` + +## Regression tests + +Add `tests/app-owned-memory.test.ts`: + +- `snapshot is observe-only and never calls an eviction callback` +- `replacement registration cannot double-count one store id` +- `exact budget boundary performs no demotion` +- `one byte over budget demotes oldest log before newer log` +- `category order beats cross-category timestamp order` +- `cache demotion starts only after logs and rings have no candidates` +- `local blobs demote before continuation and pinned remote bytes remain` +- `continuation is the final demotion category and uses durable spill callback` +- `single retained entry over budget is demoted even when it is the only entry` +- `pinned-only saturation reports honest overBudgetBytes and noEvictableCandidate` +- `zero-release and throwing callbacks cannot spin or hide over-budget bytes` +- `replacement and eviction byte accounting remains exact across all hooks` +- `translator and serialized-tail observations never invoke budget eviction` +- `budget decrease enforces synchronously in the documented order`. + +Extend `tests/memory-watchdog.test.ts`: + +- `GET system memory includes privacy-safe appOwnedBytes scalars` +- `GET system memory does not load prune serialize or evict retained stores` +- `payload contains no dynamic store keys paths ids or diagnostic text`. + +Config tests: + +- `appOwnedMemoryBudgetMb defaults to 256 MiB` +- `accepts integer bounds 64 and 4096` +- `rejects management writes below above fractional or nonnumeric values` +- `malformed persisted value degrades to default without dropping providers`. + +Run: + +```bash +bun test tests/app-owned-memory.test.ts tests/memory-watchdog.test.ts tests/config.test.ts +bun run typecheck +bun run test +bun run privacy:scan +``` + +## Commit + +`feat(memory): enforce an app-owned retained-state byte budget` + +## Explicitly not changed + +- No RSS/native-memory restart threshold or watchdog behavior. +- No eviction of in-flight translator buffers, promise-tail closures, active turns, + sockets, workers, refreshes, OAuth flows, or MCP calls. +- No first-line request rejection while an evictable retained candidate exists. +- No pin override for live remote Cursor blobs. +- No path/id/account/provider/model/prompt/tool/error content in observability. +- No 030 dependency or accounting hook; expiration sweeping stays independent. +- No GUI redesign beyond consuming the additive payload if desired in docs sync. diff --git a/devlog/_plan/260801_zero_leak_state_stores/050_translator_stream_bounds.md b/devlog/_plan/260801_zero_leak_state_stores/050_translator_stream_bounds.md new file mode 100644 index 000000000..a526f50b6 --- /dev/null +++ b/devlog/_plan/260801_zero_leak_state_stores/050_translator_stream_bounds.md @@ -0,0 +1,255 @@ +# 050 — translator stream and serialized-tail bounds + +Date: 2026-08-01 +Work phase: wp6 +Depends on: none; parallel-safe with 040 +Binding inputs: `000_state_store_inventory.md` §Translator-layer, `005_impl_roadmap.md` locked decision 5 and budget-scope split. + +## Outcome + +Bound every translator-owned accumulator without deleting translation duty. A normal +turn may contain 20 or more interleaved tool calls. The limits are therefore byte-based: + +```ts +export const TRANSLATOR_MAX_CALL_ARGUMENT_BYTES = 2 * 1024 * 1024; +export const TRANSLATOR_MAX_TURN_BYTES = 32 * 1024 * 1024; +``` + +The per-call boundary covers one assembled argument stream. The per-turn boundary covers +all charged translator copies for that turn. Overflow cancels upstream and fails the +turn coherently; no completed frame contains truncated JSON or cross-wired ids. + +## Shared budget and signature changes + +### NEW `src/lib/translator-budget.ts` + +```ts +export type TranslatorBufferKind = + | "tool_args" | "output_collectors" | "reasoning" | "item_ids" + | "tool_search_sources" | "cursor_queue" | "request_copies" + | "serialized_tail"; + +export class TranslatorBudgetExceededError extends Error { + readonly code = "translation_buffer_limit"; + constructor(readonly kind: TranslatorBufferKind, readonly limitBytes: number); +} +export interface TranslatorBudgetSnapshot { + currentBytes: number; + highWaterBytes: number; + activeCalls: number; + overflows: number; +} +export interface TranslatorBudget { + openCall(id: string): void; + appendCallBytes(id: string, bytes: number): void; + closeCall(id: string): void; + charge(kind: TranslatorBufferKind, bytes: number): void; + release(kind: TranslatorBufferKind, bytes: number): void; + snapshot(): TranslatorBudgetSnapshot; + dispose(): void; +} +export function createTranslatorBudget(options?: TranslatorBudgetOptions): TranslatorBudget; +``` + +Use UTF-8 bytes from `TextEncoder`, not JS code units. `charge` happens before append, +and failed charge leaves both buffer and counters unchanged. Call ids in this tracker +are request-local only and never appear in metrics. + +Modify `src/adapters/base.ts:3-39`: + +```ts +export interface IncomingMeta { + // existing fields unchanged + translatorBudget?: TranslatorBudget; +} +export interface ProviderAdapter { + parseStream(response: Response, budget?: TranslatorBudget): AsyncGenerator; +} +``` + +At `src/server/responses/core.ts:1070-1116`, create one budget after body admission and +attach it to `IncomingMeta`; pass it to `adapter.parseStream(response, budget)` at current +`src/server/responses/core.ts:2479`. Cursor `runTurn` receives it through `IncomingMeta`. +Dispose in every terminal/cancel/error branch after 040 records the final high-water. + +## Coherent overflow boundary + +Adapter-local overflow is converted to one event: + +```ts +{ + type: "error", + status: 502, + errorType: "upstream_error", + code: "translation_buffer_limit", + message: "upstream translation buffer exceeded the safe limit" +} +``` + +The bridge already has the required atomic failure path: + +- `src/bridge.ts:437-468` (`failCurrentToolCall`) emits an incomplete item with no + argument-done frame; +- `src/bridge.ts:709-734` uses that path for malformed assembled arguments; +- `src/bridge.ts:851-865` handles adapter `error`, fails any open call, and emits + `response.failed`. + +Route every overflow through the same `case "error"` branch and invoke `onCancel` once, +which reaches the request AbortController/upstream reader. Never call +`closeCurrentToolCall()` for overflow. Request-direction overflow is client input and +returns structured 413 `request_too_large` before upstream creation. + +## Accumulator-by-accumulator diff + +### Tool-call assembly + +| Owner and current anchor | Exact charge point | +|---|---| +| OpenAI Chat `pendingToolCalls`, `src/adapters/openai-chat.ts:697-712,792-818,854-861` | `openCall` on first index/id; charge each `function.arguments` fragment before `call.args +=`; close after atomic emission. Keep index/id fallback and 20+ call interleaving. | +| Generic bridge current call, `src/bridge.ts:317,395-435,661-707` | Charge each `tool_call_delta` before line 684 append. Use the same budget for freeform and tool-search; release when the item moves to `finishedItems`. | +| Cursor calls, `src/adapters/cursor/protobuf-events.ts:152-183,348-410,436-478` | Charge streamed args per call id; count completed-call ids/name/schema maps in turn aggregate; preserve deferred atomic start/delta/end. | +| Anthropic, `src/adapters/anthropic.ts:793-795,837-890` | Charge `partial_json` before `currentToolCallJson +=`; on overflow cancel and never emit `tool_call_end`. | +| Kiro, `src/adapters/kiro.ts:769-782,857-975,1015-1108` | Charge each `ev.input` before `open.chunks.push`; private completion tool obeys the same 2 MiB rule; protocol terminal remains authoritative. | + +Twenty-four calls of 1 MiB each must pass. One 2 MiB call passes exactly. One byte over +either per-call or 32 MiB aggregate fails the turn. + +### Output collectors and reasoning carry + +- Responses→Chat streaming maps/content/reasoning at + `src/chat/outbound.ts:153-157,220,303-365`; non-stream fold at `:481-527`; terminal + fold maps at `:553-636`: charge text, reasoning, ids, names, and argument fragments. +- Responses→Claude block/WebSearch state at `src/claude/outbound.ts:162-250` and + non-stream content fold at `:600-629`: charge text/thinking/tool JSON/signatures before + retaining; preserve block order and the real signature. +- Bridge message/reasoning/raw carry at `src/bridge.ts:300-324,334-393,650-658`: + charge `hiddenRawReasoningText`, `compactionText`, current message/reasoning text, and + pending redacted envelopes. Release a current buffer when its immutable finished item + replaces it; finished items remain charged under `output_collectors` until terminal. +- Kiro `assistantText`, `outputChars`, `deferred`, `fallbackEvents`, and thinking parser + state at `src/adapters/kiro.ts:725-782,792-922`: count only retained copies and release + spliced queues. Do not double-charge `outputChars` if it aliases a fragment already + charged as assistant/tool text; document one canonical owner for each copy. + +### Item-id and tool-search maps + +- `src/server/responses-item-id-repair.ts:7-12,51-84,169-205`: charge placeholder sets + and both `output_index -> id` maps. Cap is aggregate only; overflow fails the stream + before emitting a replacement id so one index never changes identity mid-stream. +- `buildToolBridgeMaps()` at `src/server/responses/collaboration.ts:102-115`: charge + namespace/name strings and set entries while walking accepted tools. If the 32 MiB + turn cap is exceeded, return 413 before adapter construction. +- Bridge `pendingWebSources` at `src/bridge.ts:321-331` and tool-search args at + `:413-418,661-707`: charge URL/title source attribution and current args. Deduped + sources release only after binding to the next assistant item. +- Cursor request-local KV in `src/adapters/cursor.ts:74-82` and + `src/adapters/cursor/kv-store.ts:10-24`: charge cloned key/value bytes on set, + subtract replacement, and fail the turn on overflow. Shared blobs remain 020-owned. +- Cursor MCP catalogs/results remain under 035's local payload caps; also report their + per-stream current/high-water to this budget without making 040 evict them. + +### Request-direction copies and images + +`readJsonRequestBody()` at `src/server/request-decompress.ts:15-21,52-84` currently +overlaps raw/decoded `ArrayBuffer`, text, parsed object, translated object, and serialized +internal body. Add a request-copy high-water tracker with these insertion points: + +- Responses `src/server/responses/core.ts:1079-1116`; +- Chat read/translation `src/server/chat-completions.ts:40-58,120-150` and + `src/chat/inbound.ts:209-294`; +- Claude read/translation `src/server/claude-messages.ts:65-70,510-589,672` and + `src/claude/inbound.ts:407-508`. + +Retain the existing 256 MiB accepted-body compatibility cap. The 32 MiB translator +budget applies to newly retained translator structures, not the original accepted body; +the request-copy tracker is observe/high-water plus copy-release accounting. If a newly +materialized translation alone exceeds 32 MiB, return 413 and release original buffers +as soon as parsing/serialization no longer needs them. Never persist a partial converted +request. Image aliases at `src/server/responses/core.ts:1427-1429,1638,1768-1792` are +charged as request-local map entries; actual image normalization limits remain unchanged. + +### Cursor producer queue and frame admission + +Current queue/pending anchors are `src/adapters/cursor/live-transport.ts:457-493` and +`:727-761`; announced length is read in `src/adapters/cursor/framing.ts:68-80` but the +protocol maximum at `:4` is 4 GiB. + +```ts +export const CURSOR_MAX_CONNECT_FRAME_BYTES = 32 * 1024 * 1024; +export const CURSOR_PRODUCER_QUEUE_MAX_BYTES = 32 * 1024 * 1024; +export const CURSOR_PRODUCER_QUEUE_MAX_MESSAGES = 4_096; +``` + +Change `tryDecodeConnectFrame(input, offset, maxPayloadBytes = +MAX_CONNECT_FRAME_PAYLOAD_BYTES)` to throw `payload_too_large` immediately after reading +the five-byte header and before waiting for/allocating payload. Live transport passes +32 MiB. Track queued protobuf message bytes/count; pause the HTTP/2 stream at high-water +and resume below 16 MiB. If a peer continues beyond the hard cap, close/cancel and emit +the coherent adapter error. Preserve FIFO terminal/tool ordering. + +## Serialized promise tails + +Resolve the open question as follows: + +| Tail | Current anchor | Bound and rejection contract | +|---|---|---| +| Image retention | `src/images/fulfill.ts:12-24,89-106` | `MAX_PENDING_IMAGE_FULFILLMENTS = 64`; reserve before provider/artifact work; the 65th call returns an ordinary image tool error `image_fulfillment_busy`. Accepted write→prune→filter order is unchanged. | +| OAuth mutation | `src/oauth/store.ts:292-305` | `MAX_PENDING_OAUTH_MUTATIONS = 128`, 30 s queue-wait guard; overflow throws typed `OAuthMutationBusyError`; no accepted mutation is reordered or dropped. | +| Grok apply | `src/server/management/agent-settings-routes.ts:62-70,534-559` | No-body applies are equivalent and read persisted state at execution, so concurrent callers join one active `grokApplyFlight`; no FIFO chain. A stuck flight older than 120 s is not joined and the caller receives 409 `grok_apply_busy`. | + +All three expose scalar current/peak/rejected/high-water counters to 040 as observed +in-flight fields. They are never global-budget eviction candidates. + +## Regression cases + +Add/extend the nearest existing suites with these exact cases: + +- `24 interleaved OpenAI Chat tool calls complete without reordering` +- `one tool call admits exactly 2 MiB and rejects one byte over` +- `aggregate tool arguments admit exactly 32 MiB and fail one byte over` +- `overflow emits no arguments.done or completed tool item` +- `overflow cancels upstream once and bridge emits response.failed` +- `Responses to Chat and Claude collectors stay exact at the aggregate boundary` +- `Kiro deferred reasoning text and private completion share the turn cap` +- `reasoning and redacted carry fail coherently without a partial envelope` +- `item id repair overflow never changes an already emitted index id` +- `tool search source overflow preserves prior source attribution and fails turn` +- `request-local Cursor KV replacement releases charged bytes` +- `translated request over 32 MiB returns 413 before upstream creation` +- `Cursor announced 32 MiB overflow rejects after header before payload allocation` +- `Cursor queue pauses and resumes without reordering terminal or tool messages` +- `image fulfillment 65 returns busy before provider or artifact work` +- `OAuth mutation 129 returns busy while 128 accepted writes preserve order` +- `concurrent no-body Grok applies share one flight and a stale flight returns busy`. + +Primary test files: `tests/openai-chat-parallel-stream.test.ts`, `tests/bridge.test.ts`, +`tests/cursor-protobuf-events.test.ts`, `tests/cursor-framing.test.ts`, +`tests/cursor-adapter.test.ts`, `tests/chat-completions-endpoint.test.ts`, +`tests/claude-outbound.test.ts`, `tests/kiro-stream.test.ts`, +`tests/responses-item-id-repair.test.ts`, `tests/request-decompress.test.ts`, +`tests/images/z-fulfill.test.ts`, `tests/oauth-store-multi.test.ts`, and +`tests/grok-management-api.test.ts`. + +Verification: + +```bash +bun test tests/openai-chat-parallel-stream.test.ts tests/bridge.test.ts \ + tests/cursor-protobuf-events.test.ts tests/cursor-framing.test.ts \ + tests/chat-completions-endpoint.test.ts tests/claude-outbound.test.ts tests/kiro-stream.test.ts \ + tests/images/z-fulfill.test.ts tests/oauth-store-multi.test.ts tests/grok-management-api.test.ts +bun run typecheck +bun run test +bun run privacy:scan +``` + +## Commit + +`fix(translators): bound stream accumulation and serialized tails` + +## Explicitly not changed + +- No single-digit tool-call count cap; 20+ parallel calls are a normal acceptance case. +- No truncation of JSON, reasoning envelopes, ids, source attribution, or MCP payloads. +- No global-budget eviction of in-flight translator state. +- No request body compatibility-cap reduction, provider event semantic change, or + reopening of the prior stream-path work. diff --git a/devlog/_plan/260801_zero_leak_state_stores/055_background_shell_lifecycle.md b/devlog/_plan/260801_zero_leak_state_stores/055_background_shell_lifecycle.md new file mode 100644 index 000000000..0d82cdafb --- /dev/null +++ b/devlog/_plan/260801_zero_leak_state_stores/055_background_shell_lifecycle.md @@ -0,0 +1,201 @@ +# 055 — Cursor background-shell lifecycle + +Date: 2026-08-01 +Work phase: wp6b +Depends on: none; parallel-safe with 040/050 +Binding inputs: inventory store 30, `005_impl_roadmap.md` phase 055 and regression classes. + +## Outcome + +Keep Cursor native local execution disabled by default. When a trusted operator opts into +`nativeLocalExec: "on"`, every background shell belongs to one transport session, live +admission is capped, idle and absolute lifetimes are finite, and termination drains pipe +events before removing registry ownership. + +Defaults: + +```ts +export const CURSOR_BACKGROUND_SHELL_MAX_LIVE = 8; +export const CURSOR_BACKGROUND_SHELL_IDLE_MS = 5 * 60_000; +export const CURSOR_BACKGROUND_SHELL_ABSOLUTE_MS = 30 * 60_000; +export const CURSOR_BACKGROUND_SHELL_TERM_GRACE_MS = 2_000; +``` + +Eight allows several concurrent development servers/commands without turning a remote +Cursor session into an unbounded process supervisor. Five idle minutes covers interactive +stdin pauses; 30 minutes is an escape hatch for forgotten long-lived children. These are +fixed safety constants, not new user configuration. + +## Current anchors + +- `src/adapters/cursor/native-exec-shell.ts:23-24` owns one process-wide map containing + only child and output-length counter. +- `src/adapters/cursor/native-exec-shell.ts:215-243` spawns and registers without owner, + count cap, timeout, or error cleanup. +- `src/adapters/cursor/native-exec-shell.ts:252-265` writes stdin by global numeric id, + so a different transport can currently address another session's shell. +- `src/adapters/cursor/native-exec.ts:62-69,139-168` carries native exec context and + dispatches background spawn/stdin. +- `src/adapters/cursor/live-transport.ts:385-425` already owns stable private + `sessionId` at line 407 and builds `execContext`. +- `src/adapters/cursor/live-transport.ts:579-600` closes transport/MCP state but not + background children. +- `src/types.ts:1126-1144` and English configuration docs keep native local exec opt-in; + `src/adapters/cursor/native-exec.ts:147-157` rejects it by default. + +## Registry diff + +Modify `src/adapters/cursor/native-exec-shell.ts`: + +```ts +interface BackgroundShellEntry { + shellId: number; + sessionId: string; + child: ChildProcessWithoutNullStreams; + outputLength: number; + startedAt: number; + lastActivityAt: number; + idleTimer: ReturnType; + absoluteTimer: ReturnType; + terminating: Promise | null; +} +const backgroundShells = new Map(); + +export class CursorBackgroundShellBusyError extends Error { + readonly code = "background_shell_busy"; +} +export function backgroundShellSpawnExec( + execMsg: ExecServerMessage, + sessionId: string, +): Uint8Array; +export function writeShellStdinExec( + execMsg: ExecServerMessage, + sessionId: string, +): Uint8Array; +export function terminateBackgroundShellsForSession(sessionId: string): Promise; +export function backgroundShellMetrics(): { + live: number; peak: number; rejected: number; idleKills: number; absoluteKills: number; +}; +``` + +Admission at `backgroundShellSpawnExec()` is exact: + +1. require non-empty session id; +2. if `backgroundShells.size >= 8`, return typed + `BackgroundShellSpawnError{error:"background shell limit reached"}` before `spawn()`; +3. spawn child, attach `error`, `close`, stdout and stderr listeners, then insert one + entry and arm both unrefed timers; +4. if insertion/listener setup fails, terminate the just-created child and return error; +5. numeric ids remain process-monotonic, but authorization always checks session id. + +`writeShellStdinExec()` returns the existing typed unknown-shell error when absent and a +new typed `shell belongs to another session` error on owner mismatch. It never reveals +the owner id. A successful stdin write updates `lastActivityAt` and re-arms only the idle +timer. Any stdout/stderr data also increments `outputLength`, updates activity, and +re-arms idle. + +## Controlled termination + +Add one idempotent owner: + +```ts +async function terminateBackgroundShell( + entry: BackgroundShellEntry, + reason: "session_close" | "idle" | "absolute", +): Promise; +``` + +The sequence is fixed: + +1. set/reuse `entry.terminating`; clear both timers; +2. call `stdin.end()` best-effort; +3. keep stdout/stderr listeners attached and call `resume()` so kernel pipes drain; +4. send ordinary termination (`child.kill()` / SIGTERM) and await `close`; +5. after 2 seconds, send SIGKILL best-effort and still await close for one bounded + additional 2-second window; +6. delete only when `backgroundShells.get(shellId) === entry`; settle even if the child + never emits close, and record a privacy-safe counter. + +No stdout/stderr text is retained. “Drain output” means continue consuming pipe bytes +until close so the child cannot block on a full pipe; only the existing byte count is +kept. `close` and `error` both clear timers and compare/delete the exact owner. + +Idle timeout calls the helper with `idle`; absolute timeout is never refreshed. Session +close snapshots only entries with matching `sessionId` and awaits all controlled +terminations. It cannot kill a different session's process. + +## Transport ownership wiring + +Modify `CursorNativeExecContext` at `src/adapters/cursor/native-exec.ts:62-69`: + +```ts +sessionId?: string; +``` + +At `src/adapters/cursor/live-transport.ts:409-425`, include +`sessionId: this.sessionId` in the initial context. Preserve it in the spread-based +reassignments at `:439-444` and `:501-505`. Dispatch spawn/stdin with that owner at +`src/adapters/cursor/native-exec.ts:166-167`; missing owner returns a typed error and +never spawns/writes. + +Change `LiveCursorTransport.close()` and `cancelCursorRun()` at +`src/adapters/cursor/live-transport.ts:579-600` to start +`terminateBackgroundShellsForSession(this.sessionId)`. Because the public `close()` +signature is synchronous, retain one `shellCleanup` promise and let the transport's +existing async close/dispose path await it where available; repeated close calls share +the same cleanup. Process shutdown adds a global drain only after all transport-owned +cleanup has been requested. + +## Disabled-by-default contract + +Do not change `cursorUnsafeNativeLocalExecEnabled()` at +`src/adapters/cursor/native-exec.ts:71-73` or the rejection dispatch at `:147-157`. +No new config field enables shells. `nativeLocalExec` remains `"off"` by default and +`"codex-sandbox"` remains fail-closed. The legacy unsafe boolean remains compatibility +only; docs keep warning that the opt-in bypasses Codex approvals/sandboxing. + +## Regression cases + +Extend `tests/cursor-native-exec-shell.test.ts` with injected spawn/timer seams: + +- `eight live background shells are admitted and the ninth is rejected before spawn` +- `close removes the exact child only after its output pipes drain` +- `idle lifetime terminates after five minutes and stdin activity rearms idle` +- `absolute lifetime terminates after thirty minutes despite activity` +- `controlled termination sends graceful kill before forced kill` +- `a child that never closes cannot retain the registry forever` +- `session cleanup terminates only shells owned by that session` +- `cross-session stdin write is rejected without revealing owner identity` +- `close and error races delete only the current map owner` +- `metrics contain counts only and reset deterministically`. + +Extend `tests/cursor-native-exec.test.ts:354-375`: + +- `background shell spawn and stdin share the transport session owner` +- `disabled native exec rejects background spawn before lifecycle admission`. + +Extend `tests/cursor-native-exec-policy.test.ts:250-285`: + +- `default off and codex-sandbox never create a background process` +- `only explicit nativeLocalExec on reaches bounded shell admission`. + +Verification: + +```bash +bun test tests/cursor-native-exec-shell.test.ts tests/cursor-native-exec.test.ts \ + tests/cursor-native-exec-policy.test.ts +bun run typecheck +bun run test +bun run privacy:scan +``` + +## Commit + +`fix(cursor): bound background shell ownership and lifetime` + +## Explicitly not changed + +- No default enablement, daemonization, output-text retention, shell-id persistence, or + cross-session control. +- No blind LRU eviction of a live child and no process kill before owner resolution. +- No change to foreground shell semantics, MCP/desktop executors, or Cursor protobufs. diff --git a/devlog/_plan/260801_zero_leak_state_stores/060_proxy_benchmark.md b/devlog/_plan/260801_zero_leak_state_stores/060_proxy_benchmark.md new file mode 100644 index 000000000..8b5ef7385 --- /dev/null +++ b/devlog/_plan/260801_zero_leak_state_stores/060_proxy_benchmark.md @@ -0,0 +1,159 @@ +# 060 — proxy no-leak benchmark skeleton + +Date: 2026-08-01 +Work phase: wp7 +Depends on: 010–055 landed and re-audited +Status: SKELETON ONLY; wp7 owns evidence population + +## Purpose + +This is the only document allowed to make a comparative superiority claim for the unit. +It must compare production TypeScript-based LLM proxies from opened source, separate +translation-duty products from pure relays, and distinguish a finite theoretical bound +from operational cleanup that merely “usually happens.” + +All result cells below remain `TBD` until wp7 opens a commit-pinned source URL, records +the observed implementation, and rechecks OpenCodex after 010–055 land. + +## Evidence rules + +- Prefer commit-pinned GitHub source URLs, then release notes/issues for historical + context. Repository home pages are discovery pointers, not sufficient proof. +- Record source-open date, commit/tag, exact file/function, bound dimensions, and cleanup + trigger. Search snippets and README claims are not implementation evidence. +- Classify count, per-entry bytes, aggregate bytes, TTL, admission, active-resource + ownership, and upstream cancellation independently. +- A process restart is not an eviction policy. External Redis/DB is not app-owned RAM, + but client/request buffers and local registries still count. +- “No continuation store” is not automatically stronger when the product is a pure relay; + translation duty and replay semantics must be called out. +- Any ambiguous cell is `UNKNOWN`, not `PASS`. + +## Evidence URL ledger + +The current `005_impl_roadmap.md:16-30` contains Luna findings but, at HEAD +`86a82246b827524d074ef0cfed37241b96722000`, no literal URLs. Seed repository/issue +roots here; wp7 must replace or supplement each with commit-pinned file URLs. + +| Subject | Discovery URL | Exact source permalink | Opened at wp7 | Notes | +|---|---|---|---|---| +| Portkey Gateway | https://github.com/Portkey-AI/gateway | TBD | TBD | Luna named `streamHandler.ts` / `streamHandlerUtils.ts`. | +| Claude Code Router | https://github.com/musistudio/claude-code-router | TBD | TBD | Luna found AbortController/upstream cancel and context archive TTL+count+bytes. | +| punkpeye mcp-proxy | https://github.com/punkpeye/mcp-proxy | TBD | TBD | Luna found 1,000-event FIFO and close-owned sessions. | +| LiteLLM | https://github.com/BerriAI/litellm | TBD | TBD | Verify current cache and streaming retention, not only release history. | +| LiteLLM issue 6404 | https://github.com/BerriAI/litellm/issues/6404 | n/a | TBD | Historical issue evidence only. | +| one-api | https://github.com/songquanpeng/one-api | TBD | TBD | Verify default memory-cache posture and external state boundaries. | +| lru-cache precedent | https://github.com/isaacs/node-lru-cache | TBD | TBD | Precedent, not a competitor row. | +| cacache precedent | https://github.com/npm/cacache | TBD | TBD | Spill precedent, not a competitor row. | +| OpenCodex | repository under this unit | commit-pinned local/GitHub URLs TBD | TBD | Must point at landed 010–055 code and tests. | + +## Comparison categories + +Score each category as `PASS`, `PARTIAL`, `FAIL`, `N/A`, or `UNKNOWN`, with one evidence +URL and a one-sentence reason. “PASS” means a finite contract exists in production code +and the relevant negative/boundary test exists. + +1. request/body admission before large allocation; +2. upstream abort on client disconnect/cancel; +3. per-stream frame and producer-queue bounds; +4. tool/reasoning/output translator aggregate bounds; +5. continuation/replay count + TTL + per-entry + aggregate bytes; +6. durable spill ordering and explicit missing/corrupt replay semantics; +7. content/blob cache provenance, per-entry bytes, aggregate bytes, TTL/LRU; +8. diagnostic rings: count and value-byte bounds; +9. stale-key TTL sweep and config-generation reconciliation; +10. active turns/sockets/workers/flights admission caps; +11. serialized-tail/backlog admission; +12. background process/session lifecycle; +13. process-wide retained-store byte budget and deterministic demotion order; +14. observe-only, privacy-safe app-owned byte metrics; +15. security boundary for secret-file atomic writes/ACL memo lifecycle; +16. normal 20+ parallel tool-call acceptance without truncation. + +## Competitor matrix — wp7 data cells + +| Category | OpenCodex | Portkey | Claude Code Router | mcp-proxy | LiteLLM | one-api | +|---|---|---|---|---|---|---| +| 1. Request admission | TBD | TBD | TBD | TBD | TBD | TBD | +| 2. Disconnect abort | TBD | TBD | TBD | TBD | TBD | TBD | +| 3. Stream/frame queue | TBD | TBD | TBD | TBD | TBD | TBD | +| 4. Translator aggregate | TBD | TBD | TBD | TBD | TBD | TBD | +| 5. Continuation dimensions | TBD | TBD | TBD | TBD | TBD | TBD | +| 6. Spill + explicit replay miss | TBD | TBD | TBD | TBD | TBD | TBD | +| 7. Blob/cache dimensions | TBD | TBD | TBD | TBD | TBD | TBD | +| 8. Diagnostic value bytes | TBD | TBD | TBD | TBD | TBD | TBD | +| 9. Expiry/reconciliation | TBD | TBD | TBD | TBD | TBD | TBD | +| 10. Active admission | TBD | TBD | TBD | TBD | TBD | TBD | +| 11. Tail admission | TBD | TBD | TBD | TBD | TBD | TBD | +| 12. Background lifecycle | TBD | TBD | TBD | TBD | TBD | TBD | +| 13. Global retained budget | TBD | TBD | TBD | TBD | TBD | TBD | +| 14. App-byte observability | TBD | TBD | TBD | TBD | TBD | TBD | +| 15. Secret-file hardening | TBD | TBD | TBD | TBD | TBD | TBD | +| 16. 20+ call acceptance | TBD | TBD | TBD | TBD | TBD | TBD | + +## Per-competitor evidence rows + +Populate at least one row per material category; add rows rather than packing multiple +claims into one citation. + +| Competitor | Category | Verdict | Commit/tag | File/function | Source URL | Exact bound/behavior | Test URL | Caveat | +|---|---|---|---|---|---|---|---|---| +| OpenCodex | TBD | TBD | TBD | TBD | TBD | TBD | TBD | Translation duty. | +| Portkey | TBD | TBD | TBD | TBD | TBD | TBD | TBD | TBD | +| Claude Code Router | TBD | TBD | TBD | TBD | TBD | TBD | TBD | TBD | +| mcp-proxy | TBD | TBD | TBD | TBD | TBD | TBD | TBD | MCP/session focus. | +| LiteLLM | TBD | TBD | TBD | TBD | TBD | TBD | TBD | Python runtime; include only if wp7 keeps the broader production-proxy cohort. | +| one-api | TBD | TBD | TBD | TBD | TBD | TBD | TBD | Go runtime; external state distinction required. | + +## Gap-list gate + +Create a final gap list from every OpenCodex `PARTIAL`, `FAIL`, or `UNKNOWN` cell and +every category where a competitor has a materially stronger finite contract. + +The wp7 gate passes only when one of these is true for every gap: + +- the gap is closed by landed, tested code in 010–055 and the matrix is updated; or +- the gap is converted into a numbered work phase with owner, exact files, acceptance + tests, and dependency placement, and this unit is not closed until that phase lands. + +An unexplained `N/A`, a planned-but-unlanded phase, or “low practical risk” does not +empty the gap list. The final section must literally record `Open gap count: 0` before +070 may proceed. + +## Superiority-claim rule + +No claim may appear in `005`, release notes, README, social copy, or PR description +before this table is complete. After the empty-gap gate: + +- category claims may say “stronger than the surveyed proxies in category X” only when + every competitor has opened evidence and OpenCodex is strictly stronger there; +- the broad claim “strongest theoretical no-leak posture among surveyed production + TypeScript LLM proxies” requires no `UNKNOWN` cells in that cohort and no category + where a competitor is stronger; +- never claim “leak-free,” zero RSS growth, or stronger than projects outside the frozen + cohort; +- name survey date, cohort, and theoretical/app-owned scope in the sentence; +- if evidence is mixed, publish the matrix and omit the superiority sentence. + +## wp7 completion checklist + +- [ ] Freeze competitor repository SHAs/tags and source-open date. +- [ ] Replace every source `TBD` with commit-pinned URLs or mark `UNKNOWN` with reason. +- [ ] Re-open landed OpenCodex source/tests; do not score from roadmap prose. +- [ ] Complete all 16 category rows and per-competitor evidence rows. +- [ ] Resolve cohort consistency for LiteLLM/one-api versus TypeScript-only headline. +- [ ] Record and close/phase every gap. +- [ ] Record `Open gap count: 0`. +- [ ] Add the one permitted scoped superiority statement, or explicitly omit it. + +## Commit + +`docs(devlog): record zero-leak proxy benchmark evidence` + +## Explicitly not changed + +- This skeleton does not assign final verdicts, fill benchmark measurements, or claim + superiority; wp7 owns those evidence-backed conclusions. +- No runtime, test, config, dependency, benchmark harness, release, or provider behavior + changes in this document. +- No popularity, star count, or throughput result is treated as a retention guarantee. diff --git a/devlog/_plan/260801_zero_leak_state_stores/070_close_and_push.md b/devlog/_plan/260801_zero_leak_state_stores/070_close_and_push.md new file mode 100644 index 000000000..5d0e0f844 --- /dev/null +++ b/devlog/_plan/260801_zero_leak_state_stores/070_close_and_push.md @@ -0,0 +1,232 @@ +# 070 — close, selectively stage, and push + +Date: 2026-08-01 +Work phase: wp8 +Depends on: 010–060 complete; 060 records `Open gap count: 0` +Execution status: procedure only; this docs-only B-phase performs no commit or push. + +## Terminal outcome + +Close the unit only after the exact implementation HEAD passes typecheck, the full Bun +test suite, and privacy scan; the diff contains no unrelated work; the local commit is +pushed without force; and local HEAD, `origin/dev`, and remote `refs/heads/dev` resolve +to the same SHA. + +The unit targets `dev`. `main` and `preview` are not moved by this phase. + +## Preflight: freeze the checkout + +Run from `/Users/jun/Developer/new/700_projects/opencodex`: + +```bash +pwd +git rev-parse --show-toplevel +git branch --show-current +git rev-parse HEAD +git remote -v +git status --short --branch --untracked-files=all +``` + +Required assertions: + +- repository root is exactly the path above; +- branch is `dev` unless the parent explicitly records an approved integration branch; +- remote `origin` is the intended OpenCodex repository; +- no merge/rebase/cherry-pick is in progress; +- every dirty path is classified as this unit, a named exclusion below, or unrelated + user work that remains untouched. + +Fetch read-only remote state immediately before integration: + +```bash +git fetch origin dev +git rev-parse HEAD +git rev-parse origin/dev +git merge-base --is-ancestor origin/dev HEAD +git merge-base --is-ancestor HEAD origin/dev +``` + +If `origin/dev` advanced, do not push a stale direct update. Reconcile using the parent's +approved integration method, rerun every gate on the new exact HEAD, and refresh 060 +source anchors if the rebase/merge changed them. Never force-push `dev`. + +## Required change audit + +Review both tracked and untracked scope: + +```bash +git status --short --untracked-files=all +git diff --stat +git diff --check +git diff -- src tests docs-site gui structure scripts .github +git diff -- devlog/_plan/260801_zero_leak_state_stores +``` + +Acceptance: + +- 010 implements durable spill, explicit miss, cleanup, accounting, and tests; +- 020 implements provenance-split blob admission plus replay/image/vision bounds; +- 030 implements only the locked TTL/reconciliation/admission assignments and receives + the required Windows ACL security review; +- 035 covers every registry/flight/value item and returns coherent busy errors; +- 040 exposes privacy-safe `appOwnedBytes` and deterministic retained-store demotion; +- 050 bounds every translator accumulator and preserves normal 20+ interleaved calls; +- 055 keeps native shell execution disabled by default and owns every enabled child; +- 060 has commit-pinned evidence, no unexplained unknowns for the claimed cohort, and + records `Open gap count: 0`; +- user-facing configuration/docs are synchronized, including translated configuration + pages where an existing English field table has locale peers; +- no source/test behavior differs from the decade docs without the docs being amended + first. + +## Gate sequence + +Run on one frozen HEAD with no edits between commands: + +```bash +bun run typecheck +bun run test +bun run privacy:scan +``` + +The full suite is mandatory because this unit crosses continuation, adapters, auth, +management API, filesystem hardening, workers, and stream conversion. Focused phase +tests are useful during implementation but do not replace `bun run test` here. + +Record for each gate: + +- command and UTC/local timestamp; +- exact `git rev-parse HEAD` before the command; +- exit code and concise final output/counts; +- machine/OS/Bun version; +- zero failures, crashes, timeouts, or cancellations. + +A cancellation is not a pass. A Bun crash is not a test assertion failure. Diagnose and +repair the concrete class, then restart the entire terminal gate sequence from +typecheck. Do not raise a timeout merely to turn an unexplained wait green. + +Also run `git diff --check` after the gates. Documentation-only warnings or pre-existing +unrelated failures must be reported and resolved by the parent; they cannot be silently +declared out of scope when they block a required command. + +## Selective staging contract + +These paths are explicit exclusions and must never enter this unit's index/commit: + +```text +devlog/_plan/260731_client_config_export/ +devlog/_chase/DSCodex/ +devlog/_plan/260801_pr611_volcengine_evidence/ +``` + +Do not use broad staging commands such as `git add .`, `git add -A`, `git add devlog`, or +`git commit -a`. Build an explicit path list from the audited change manifest, then add +only those files. The list will include the implemented `src/`, focused/full-regression +`tests/`, synchronized `docs-site/` pages, and this unit's own decade documents. + +Before commit, prove exclusions and scope: + +```bash +git diff --cached --name-only +git diff --cached --stat +git diff --cached --check +git diff --cached +``` + +Fail staging if any cached path begins with an exclusion prefix, if any unrelated dirty +path is staged, or if any expected unit file is absent. Unstaged/untracked excluded work +is preserved exactly as found. + +Suggested commit subject after the parent approves the final staged diff: + +```text +fix(state): hard-bound retained and translator stores +``` + +No commit is created by this roadmap-writing task. + +## Push procedure + +The unit-level roadmap records authorization to push to `origin/dev`, but the executor +must still re-read current parent instructions at execution time. If authorization was +revoked or branch policy changed, stop before external mutation. + +Immediately before push: + +```bash +git status --short --branch +git rev-parse HEAD +git fetch origin dev +git rev-parse origin/dev +git merge-base --is-ancestor origin/dev HEAD +``` + +Required state is a clean index/worktree except preserved named exclusions/unrelated +user files, and `origin/dev` is an ancestor of the tested local HEAD. Push without force: + +```bash +git push origin HEAD:dev +``` + +If the push is rejected because remote advanced, fetch/reconcile and rerun all terminal +gates. Do not use `--force`, `--force-with-lease`, or push a merge that was not gated. + +## HEAD parity proof + +After successful push, collect three independent values: + +```bash +local_head=$(git rev-parse HEAD) +tracking_head=$(git rev-parse origin/dev) +remote_head=$(git ls-remote origin refs/heads/dev | awk '{print $1}') +test "$local_head" = "$tracking_head" +test "$local_head" = "$remote_head" +``` + +Then report the literal SHA and the successful equality checks. Do not report “pushed” +from command exit alone. Re-run `git status --short --branch` and list preserved dirty +paths so the user can see that exclusions survived. + +If CI is part of the parent's completion gate, verify checks on this exact SHA rather +than a branch-name snapshot. Classify test failure, timeout, runner crash, and +cancellation separately; do not rerun or alter remote CI without current authorization. + +## Unit closure record + +Append the final evidence ledger/attestation to the unit's owning plan document, then +move the unit from `_plan` to `_fin` only after: + +- all gates pass on pushed parity SHA; +- 060 has zero open gaps and only evidence-supported comparative wording; +- security review for the Windows ACL change is recorded; +- no required source/docs/test changes remain; +- named exclusions remain outside the commit. + +The move itself must be selectively staged with the same exclusion audit. Record the old +and new path, final SHA, gate evidence, review verdict, push parity, and any explicitly +preserved unrelated work. + +## Stop conditions + +Stop and report instead of improvising when: + +- checkout, branch, remote, or target SHA differs from the frozen scope; +- a locked decision cannot be preserved without truncation/silent continuation loss; +- Windows ACL review does not approve repeated-temp hardening; +- a required gate fails, crashes, times out, or is cancelled; +- remote `dev` advances after the final gates; +- selective staging cannot separate this unit from overlapping user work; +- 060 has any unconverted gap or unsupported superiority claim. + +## Commit + +`docs(devlog): close zero-leak state-store hardening` + +## Explicitly not changed by closure + +- No release promotion to `main` or `preview`, version bump, npm publish, tag, or + GitHub Release. +- No force-push, unrelated PR/issue mutation, CI rerun/cancel, or branch-protection + change. +- No cleanup, staging, reset, or overwrite of unrelated dirty/untracked work. +- No superiority claim unless 060's immutable-source and zero-gap gates pass. From dd5af60bba67a02e278b497f4843c1545594bbf1 Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sat, 1 Aug 2026 08:22:59 +0900 Subject: [PATCH 02/30] docs(plan): final refinements to the continuation, blob, and benchmark phase docs --- .../010_continuation_hard_cap.md | 5 +++++ .../020_blob_and_replay_caps.md | 21 ++++++++++++++++++- .../060_proxy_benchmark.md | 9 +++++--- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/devlog/_plan/260801_zero_leak_state_stores/010_continuation_hard_cap.md b/devlog/_plan/260801_zero_leak_state_stores/010_continuation_hard_cap.md index 9ee1db8a2..fbdd11550 100644 --- a/devlog/_plan/260801_zero_leak_state_stores/010_continuation_hard_cap.md +++ b/devlog/_plan/260801_zero_leak_state_stores/010_continuation_hard_cap.md @@ -18,6 +18,11 @@ The legacy debounced `responses-state.json` path remains the persistence path fo resident entries. Its 2 MiB per-entry and 24 MiB snapshot selection rules are not reused as the spill mechanism and are not relaxed. +No new 010 configuration surface is added. Keep the owner-local RAM ceiling fixed at +64 MiB and retain only the existing test override; 040 owns the configurable process-wide +retained-state budget. A separate continuation knob would create two user settings with +overlapping demotion authority and could raise the local store above the global budget. + ## Current contract and verified anchors - `src/responses/state.ts:6-20` owns count, TTL, 64 MiB nominal RAM cap, legacy snapshot diff --git a/devlog/_plan/260801_zero_leak_state_stores/020_blob_and_replay_caps.md b/devlog/_plan/260801_zero_leak_state_stores/020_blob_and_replay_caps.md index 063c18ed5..ec6d3658e 100644 --- a/devlog/_plan/260801_zero_leak_state_stores/020_blob_and_replay_caps.md +++ b/devlog/_plan/260801_zero_leak_state_stores/020_blob_and_replay_caps.md @@ -201,6 +201,26 @@ Extend `getNormalizeStatsForTests()` and the 040 hook with `sentinelEntries`, `metadataBytes`, and `oldestAt`. Budget eviction removes the oldest row through the same centralized subtract helper. +## Cap rationale + +- Cursor 16 MiB per blob is 32 times the external selected-root budget and still admits + native conversation-step/KV payloads, while rejecting protocol-scale allocations long + before the 32 MiB translator frame ceiling. The 64 MiB aggregate matches the existing + continuation/image-cache order of magnitude and gives four maximum entries or many + ordinary roots without allowing the 4,096 count cap to imply TiB retention. +- Antigravity 256 calls is more than ten times the normal 20+ parallel-call acceptance; + 2 MiB/session permits roughly 8 KiB per identity on average. A 64 KiB signature ceiling + is far above observed opaque signatures but prevents one value from consuming the + entire session budget. +- Vision keeps the established 256 identities. The 1 MiB aggregate holds hundreds of + ordinary short descriptions; clamp-before-insert guarantees every retained value is + at most the existing 2,000-character presentation contract, so paid-call reuse remains + useful while pathological upstream prose cannot dominate the process. +- Image normalization keeps a generous 4,096-row metadata ceiling so pass/miss reuse is + not destroyed by screenshot churn. The 20 MiB per-entry ceiling matches Anthropic's + final aggregate image-share contract and is above every normal ladder output; the + existing 64 MiB aggregate remains the stronger ordinary constraint. + ## Regression tests `tests/cursor-blob.test.ts`: @@ -263,4 +283,3 @@ bun run test - No image tier ladder, decode validation, wire mutation, demotion order, or 20 MiB request-level image budget. - No process-wide budget; 040 only consumes the accounting/demotion hooks defined here. - diff --git a/devlog/_plan/260801_zero_leak_state_stores/060_proxy_benchmark.md b/devlog/_plan/260801_zero_leak_state_stores/060_proxy_benchmark.md index 8b5ef7385..bfe867127 100644 --- a/devlog/_plan/260801_zero_leak_state_stores/060_proxy_benchmark.md +++ b/devlog/_plan/260801_zero_leak_state_stores/060_proxy_benchmark.md @@ -31,9 +31,12 @@ the observed implementation, and rechecks OpenCodex after 010–055 land. ## Evidence URL ledger -The current `005_impl_roadmap.md:16-30` contains Luna findings but, at HEAD -`86a82246b827524d074ef0cfed37241b96722000`, no literal URLs. Seed repository/issue -roots here; wp7 must replace or supplement each with commit-pinned file URLs. +The current `005_impl_roadmap.md:16-30` contains Luna findings but no literal URLs. +Source anchors were verified against `86a82246b827524d074ef0cfed37241b96722000`; +the shared checkout later advanced to docs-only commit +`860ec897bbc23e04a7b07e93bfcd1dac20e7609f` with `src/` and `tests/` byte-unchanged. +Seed repository/issue roots here; wp7 must replace or supplement each with commit-pinned +file URLs. | Subject | Discovery URL | Exact source permalink | Opened at wp7 | Notes | |---|---|---|---|---| From 4bf0844a0ca21886f099fabe7cd94023ebd430f4 Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sat, 1 Aug 2026 08:38:21 +0900 Subject: [PATCH 03/30] docs(plan): harden the continuation spill transaction through eight review rounds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generation-unique spill basenames (content digest + monotonic counter), atomic no-replace publication (link/exclusive-copy, EEXIST retry — plain rename replaces and is unsuitable), and exactly-once supersededSpill ownership across stub-to-resident and resident-to-resident replacements. The 005 locked decision and phase map now match 010 end to end. --- .../005_impl_roadmap.md | 12 +- .../006_roadmap_audit_synthesis.md | 6 + .../010_continuation_hard_cap.md | 233 +++++++++++++++--- 3 files changed, 205 insertions(+), 46 deletions(-) diff --git a/devlog/_plan/260801_zero_leak_state_stores/005_impl_roadmap.md b/devlog/_plan/260801_zero_leak_state_stores/005_impl_roadmap.md index 8fc6d7931..a935aaa65 100644 --- a/devlog/_plan/260801_zero_leak_state_stores/005_impl_roadmap.md +++ b/devlog/_plan/260801_zero_leak_state_stores/005_impl_roadmap.md @@ -38,7 +38,7 @@ supplies the full source ledger (audit round 1). | Doc | WP | Content | Depends | |---|---|---|---| -| `010_continuation_hard_cap.md` | wp2 | continuation last-entry cap hole → true hard cap; oversized entries demote to disk stub ONLY after atomic per-entry spill (fsync+rename); missing/corrupt spill = explicit continuation failure, never silent naked-delta (R1-2); contract redefinition | — | +| `010_continuation_hard_cap.md` | wp2 | continuation last-entry cap hole → true hard cap; oversized entries demote to disk stub ONLY after atomic per-entry spill (fsync + no-replace publication); missing/corrupt spill = explicit continuation failure, never silent naked-delta (R1-2); contract redefinition | — | | `020_blob_and_replay_caps.md` | wp3 | Cursor blob byte cap split by PROVENANCE (local-regenerated evictable, remote-origin pinned within TTL — R1-3), Antigravity inner-call bounds, vision-LRU clamp-before-insert, image zero-weight sentinel accounting (R1-1) | — | | `030_eviction_mechanisms.md` | wp4 | store-by-store cleanup matrix with three mechanisms: TTL sweep / config-generation reconciliation / admission caps (R1-4). Covers: subagent quota-failure records, key/combo cooldowns, provider+codex quota history, routing health, warning memos, model-cache provider history, pool/combo history, XAI verdicts, guardian backoff, GCP source tokens, ownership/PID memos, OAuth/reauth reconciliation. Windows ACL memo: eviction-after-rename keyed by actual temp path, NO destination-level reuse (R1-5); explicit security review required | — | | `035_registry_admission_caps.md` | wp4b | remaining operational registries/flights: debug-subscriber count cap, active turns/sockets/workers hard admission CAP (reject beyond cap with a coherent busy error — S3-3) + leak metric, credential-refresh flights bounded distinct-grant admission (cap on concurrent grant fingerprints) + staleness replacement (S3-3), usage-read flight staleness guard PLUS bounded parse (stream/limit the full-log read so the in-flight value itself is byte-capped — S2-1), Cursor model-discovery response byte cap + gather-flight concurrency cap, OAuth pending-code value byte cap + auth flow/probe admission caps, MiMo JWT value byte cap, Cursor MCP manager payload caps, crash-ring + fixed-slot diagnostic + AFFINITY value-byte truncation (S2-2) | — | @@ -83,12 +83,14 @@ prior unit is not reopened. oversized entries spill to a DEDICATED per-entry file (not the debounced monolithic snapshot, which is async/best-effort/2 MiB-skipping — R1-2). Demotion ordering is locked: RAM row becomes a stub ONLY after an atomic - successful spill (write-fsync-rename). Spill FAILURE (disk permissions, + successful spill (write-fsync + atomic NO-REPLACE publication — link or + exclusive copy; plain rename replaces existing destinations and is + unsuitable, 010 round-6). Spill FAILURE (disk permissions, exhaustion, I/O) evicts the row from RAM and records a small `spill-failed` tombstone for the response id: later continuation - against it returns the same explicit structured not-found contract as a - corrupt spill, and the client falls back to full-context resend — the - same recovery path TTL expiry produces today. Replay through a missing/ + against it returns a terminal structured 400 telling the caller to resend + the full context without `previous_response_id` — caller-driven recovery, + not an automatic client fallback. Replay through a missing/ corrupt stub likewise returns an EXPLICIT structured continuation error, never silent forwarding of the naked delta. The last-entry exemption is deleted; `storedResponseBytes > byteCap()` always spills or diff --git a/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md b/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md index 673428d16..d6c043479 100644 --- a/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md +++ b/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md @@ -48,3 +48,9 @@ already folded above. The two NET-NEW blockers from Godel: Both preserve the structured-error-over-silent-corruption principle. 005 amended accordingly (failure ladder + admission ladder). + +## wp2 A-gate (reviewer Banach, FAIL 5) — adjudication + +| # | Blockers | Decision | Action | +|---|---|---|---| +| B1-B5 | Spill read lacked a trusted expected id; same-id/equal-size replacement collided and relied on non-portable rename-over-existing; automatic client fallback was unproven; resident measurement was incomplete/weightless on serialization failure; the 3 MiB restart fixture did not cross the 64 MiB spill threshold | ACCEPT | 010 now passes `responseId` into spill reads; uses payload-digest plus generation-distinct basenames and post-swap old-file unlink; defines caller-driven terminal structured 400 recovery; measures the complete retained payload in UTF-8 and tombstones serialization failures; lowers the test cap beneath spill fixtures; records all five current contract-test redefinitions, consumer seams, expanded regression classes, and process-crash-only durability. 005 receives the matching one-sentence recovery correction. | diff --git a/devlog/_plan/260801_zero_leak_state_stores/010_continuation_hard_cap.md b/devlog/_plan/260801_zero_leak_state_stores/010_continuation_hard_cap.md index fbdd11550..cd504519d 100644 --- a/devlog/_plan/260801_zero_leak_state_stores/010_continuation_hard_cap.md +++ b/devlog/_plan/260801_zero_leak_state_stores/010_continuation_hard_cap.md @@ -9,10 +9,14 @@ Binding inputs: `000_state_store_inventory.md` §1, `005_impl_roadmap.md` locked Make the Responses continuation store authoritative but unconditionally bounded in RAM. An entry that would leave resident bytes above 64 MiB is synchronously written to a -dedicated id-keyed spill file, fsynced, atomically renamed, and only then -replaced by a small RAM stub. Any write failure removes the resident entry and records -a bounded tombstone. A stub whose file is missing or corrupt produces the same explicit -structured continuation-not-found response; it never forwards a naked delta. +dedicated id-keyed, generation-distinct spill file, file-fsynced, atomically published +(no-replace link/exclusive-copy), +and only then replaced by a small RAM stub. This is process-crash durability, not a +power-loss guarantee: this phase does not claim parent-directory fsync portability. +Any write failure removes the resident entry and records a bounded tombstone. A stub +whose file is missing or corrupt produces the same terminal structured +continuation-not-found response telling the caller to resend the full context; it never +forwards a naked delta and does not claim an automatic client retry. The legacy debounced `responses-state.json` path remains the persistence path for small resident entries. Its 2 MiB per-entry and 24 MiB snapshot selection rules are not reused @@ -37,8 +41,8 @@ overlapping demotion authority and could raise the local store above the global `states.size > 1`. This is the last-entry exemption to delete. - `src/responses/state.ts:336-350` returns the original request on every miss; the caller cannot distinguish ordinary absence from a known broken spill. -- `src/responses/state.ts:363-408` reads provider metadata and exposes observe-only - metrics without loading, pruning, or serializing. +- `src/responses/state.ts:363-369` reads provider metadata and does lazy-load/prune; + `:371-408` exposes observe-only metrics without loading, pruning, or serializing. - `src/responses/state.ts:415-455` synchronously stores expanded input plus output and immediately prunes/schedules persistence. - `src/server/responses/core.ts:1092-1115` expands before parsing and detects expansion @@ -55,6 +59,24 @@ Inventory blast-radius constraint: “evicting/rejecting the newest row makes th chained turn a naked delta.” This phase therefore spills or emits an explicit miss; it does not truncate replay history. +### A-gate corrections (Banach) — consumer contract ledger + +The implementation and regression pass must account for every current consumer of the +changed state contract: + +- `src/responses/parser.ts:269` consumes replay-prefix provenance; spill materialization + must set the provenance WeakMap on the exact expanded body just like a resident hit. +- `src/server/lifecycle.ts:89` awaits `flushResponseState()` during shutdown; a concurrent + legacy snapshot flush must serialize the already-committed stub/tombstone representation + and must never inline, lose, or race a spill payload. +- `src/server/management/system-routes.ts:88` publishes `responseStateMetrics()`; + `gui/src/components/MemoryObservabilityCard.tsx:27` owns its GUI type and + `tests/memory-watchdog.test.ts:181` owns the scalar/privacy contract. All added fields + remain finite numbers and expose no response ids, paths, digests, or payload content. +- `tests/issue-702-expired-replay-state.test.ts:250-283` remains the fail-closed reference: + known broken local state must stop before upstream I/O, while an ordinary miss on an + upstream-capable API-key Responses provider retains its current native-continuation path. + ## File changes ### NEW `src/responses/spill-store.ts` @@ -79,7 +101,7 @@ export interface ResponseSpillPayload { export interface ResponseSpillRef { version: 1; - fileName: string; // sanitized response id + id digest + payload size + fileName: string; // id slug + id digest + payload digest prefix + generation + size digest: string; // lowercase SHA-256 of the exact UTF-8 file bytes payloadBytes: number; } @@ -99,7 +121,10 @@ export function writeResponseSpillDurably( responseId: string, state: Omit, ): ResponseSpillRef; -export function readResponseSpill(ref: ResponseSpillRef): ResponseSpillReadResult; +export function readResponseSpill( + responseId: string, + ref: ResponseSpillRef, +): ResponseSpillReadResult; export function deleteResponseSpill(ref: ResponseSpillRef): void; export function recoverOrphanedResponseSpills( referencedFileNames: ReadonlySet, @@ -107,35 +132,67 @@ export function recoverOrphanedResponseSpills( ): ResponseSpillCleanupResult; ``` +#### A-gate corrections (Banach) — spill identity and process-crash durability + `writeResponseSpillDurably()` transaction, in this exact order: 1. Serialize `{version:1,responseId,createdAt,items,providers}` once and compute UTF-8 bytes and SHA-256 from those exact bytes. -2. Build an id-keyed basename - `...spill.json`. Normalize the id to NFC, - replace every character outside `[A-Za-z0-9._-]` with `_`, collapse repeated `_`, - trim the visible portion to 80 characters, and use `response` if it becomes empty. - `id-digest-12` is the first 12 lowercase hex characters of SHA-256(responseId), so - ids that sanitize/truncate to the same visible text cannot collide. Require the full - owned-file regex before joining it to the spill directory. +2. Allocate a generation-unique basename + `.....spill.json`. + Normalize the id to NFC, replace every character outside `[A-Za-z0-9._-]` with + `_`, collapse repeated `_`, trim the visible portion to 80 characters, and use + `response` if it becomes empty. `id-digest-12` is the first 12 lowercase hex + characters of SHA-256(responseId); `content-digest-24` is the first 24 lowercase + hex characters of the payload SHA-256 from step 1 (the prefix MUST be at least + 16 hex characters). Without a + content component, re-spilling the same id with different content but equal + byte length would rename ONTO the live old file and destroy it before the stub + swap — a crash in that window leaves a persisted stub pointing at bytes with + the wrong digest. `generation` is a process-local monotonic counter; advance it + until the destination basename is absent. Old and new generations therefore + coexist on disk until the swap completes. Require the full owned-file regex + before joining it to the spill directory. 3. Resolve `/responses-state-spill/`; create/harden the directory to user-only permissions. 4. Open a same-directory unique temp with `wx` and mode `0600`. 5. Write all bytes, call `fsyncSync(fd)`, close the descriptor, and harden the temp. -6. Rename temp to the id-keyed destination. If the basename already exists, accept it - only after regular-file, exact-size, response-id, and full-digest verification; - otherwise replace it through another same-directory temp. Replacement of a response - id deletes the old basename only after the new rename and stub swap succeed. -7. Return the ref only after rename succeeds. On every failure, close/unlink the temp +6. Link the temp to the newly allocated destination with an atomic NO-REPLACE + primitive (round-5 blocker 2: plain `renameSync` replaces an existing + destination on POSIX): use `fs.linkSync(temp, dest)` — which fails with + `EEXIST` when the destination exists — then `unlinkSync(temp)`; on Windows, + where `link` support varies by filesystem, fall back to `copyFileSync(temp, + dest, COPYFILE_EXCL)` + fsync of the copy. An `EEXIST` loss means another + writer allocated the same generation basename: advance the generation + counter and retry allocation (bounded retries), then treat persistent + failure as a sanitized write failure. The normal path never replaces an + existing destination: same-id/same-size and even same-content replacements + receive a distinct generation basename. + A different generation for the same id never shares a destination: the old generation's + file stays untouched through the publication and is unlinked only AFTER the new + publication and in-memory stub swap have both succeeded. A crash between publication + and swap leaves both + files; the stub still references the old valid generation and startup GC removes the + newer unreferenced file as an orphan. +7. Return the ref only after publication succeeds. On every failure, close/unlink the temp best-effort and rethrow a sanitized error containing no response id or path. -The id-keyed layout is selected over content-addressing because lifecycle, replacement, -TTL, and tombstone ownership are all by response id. A sanitized visible component makes -orphan diagnosis practical, the size component permits bounded startup accounting -without opening every file, and the short id digest prevents collisions after +The id+content+generation-keyed layout keeps lifecycle/TTL/tombstone ownership by response +id while the content-digest and generation components provide replacement uniqueness. +A sanitized visible component makes orphan diagnosis practical, the size component permits +bounded startup accounting without opening every file, and the short id digest prevents collisions after sanitization/truncation. The full content digest remains in the stub and is verified on read; neither visible id nor filename size is trusted as integrity proof. +Windows `renameSync(temp, existingDestination)` is not a portable atomic-replace +primitive. Generation-distinct destination basenames make that caveat moot: 010 never +intentionally publishes over the old file. The state owner preserves the old row and +spill file while writing the candidate, swaps to the new stub only after the new +publication succeeds, and only then unlinks the old file. File fsync before publication +plus same-directory atomic no-replace publication (link / exclusive copy) is the +process-crash durability boundary claimed here. Parent-directory fsync is +not required by 010, so sudden-power-loss namespace durability remains outside the claim. + This phase deliberately uses a synchronous transaction. `rememberResponseState()` is called from synchronous bridge completion callbacks (`src/bridge.ts:143,811-817` and `src/server/relay.ts:498,680-718`); an un-awaited asynchronous spill would permit an @@ -143,14 +200,32 @@ unbounded pending-write closure chain and a post-response stub race. The synchro path is the bounded in-flight window: at most one transaction exists on the JS thread, with no queue. Spills are exceptional, not the small-entry hot path. -`readResponseSpill()` must reject symlinks/non-regular files, require the basename to +`readResponseSpill(responseId, ref)` receives the trusted expected id from the in-memory +map lookup. It must reject symlinks/non-regular files, require the basename to match the owned regex, require stat size to match both the filename size and `ref.payloadBytes`, enforce ref/content digest equality, validate the exact schema and -matching `responseId`, and return `corrupt` on any parse, +require `payload.responseId === responseId`, and return `corrupt` on any parse, shape, or digest failure. Never return partial items. ### MODIFY `src/responses/state.ts` +### A-gate corrections (Banach) — failure-safe measurement and replacement + +Replace the current `JSON.stringify(entry.items).length` / weightless-on-error behavior at +`src/responses/state.ts:52-60`. The one resident measurement serializes the complete retained +payload `{responseId,createdAt,items,providers}` once and measures its UTF-8 bytes with +`TextEncoder` or `Buffer.byteLength`; provider metadata and the map key are therefore part of +admission accounting. Cached `sizeBytes` is this measured payload weight. A serialization +failure rejects the candidate from resident storage and atomically replaces the old row, if +any, with a bounded `spill-failed` tombstone before unlinking an old spill file; it must never +retain a zero-weight or partially measured row. + +`setResidentEntry()` must not preserve the current delete-first behavior at +`src/responses/state.ts:64-68`. Measure the complete candidate before mutating `states`. For +an ordinary insert or resident-row replacement, commit the measured resident row and then +run the normal synchronous prune/demotion loop. Replacement of an existing spill stub uses +the separate literal-B2 transaction below and never installs an intermediary resident row. + Replace the monomorphic row with this union: ```ts @@ -191,7 +266,13 @@ export function previousResponseReplayFailure(body: unknown): PreviousResponseRe Centralize transitions; no caller mutates `states`, counters, or spill files directly: ```ts +function measureResidentEntry(id: string, entry: ResidentInput): ResidentResponseState | null; function setResidentEntry(id: string, entry: ResidentInput): void; +function replaceSpillEntryAtomically( + id: string, + expected: SpilledResponseState, + candidate: ResidentResponseState, +): void; function swapResidentForSpill(id: string, expected: ResidentResponseState, ref: ResponseSpillRef): void; function replaceWithSpillFailure(id: string, expected?: StoredResponseState): void; function deleteEntry(id: string, options?: { deleteSpill?: boolean }): void; @@ -200,9 +281,27 @@ function materializeEntry(id: string, entry: StoredResponseState): | { ok: false; failure: PreviousResponseReplayFailure }; ``` -`deleteEntry()` subtracts the row's RAM `sizeBytes`; for a spill stub it also unlinks -the dedicated file unless `deleteSpill:false` is used during a verified load/swap. -Replacement and TTL/count eviction therefore remove old spill files. +**Literal-B2 atomic replacement:** replacing an existing spill stub follows one transaction +and never routes through `setResidentEntry()` or an intermediary resident row: + +1. Read `expected = states.get(id)` and measure the complete candidate while `expected` + remains installed and replay-addressable. +2. Serialize the candidate and synchronously write, file-fsync, and publish the + distinct-generation spill file. Until publication succeeds, every lookup still sees and + can replay `expected`; the old file remains untouched. +3. Build the new stub from the candidate's `createdAt`, provider metadata, measured stub + bytes, and returned `ResponseSpillRef`. Verify `states.get(id) === expected`, then perform + one accounting/map transition from the old stub directly to the new stub. +4. Only after the new stub is installed, unlink `expected.spill` best-effort. A crash before + the swap leaves the old row/file authoritative and the new file orphan-recoverable; a + crash after the swap leaves the new row authoritative and the old file orphan-recoverable. +5. Serialization or publication failure atomically replaces `expected` with the bounded + tombstone, then unlinks the old file; it never exposes a resident candidate or naked delta. + +`deleteEntry()` subtracts the row's RAM `sizeBytes`; for a spill stub it also unlinks the +dedicated file unless `deleteSpill:false` is used during a verified atomic swap. TTL/count +eviction therefore removes spill files. No `supersededSpill` field or secondary ownership +state exists. Replace the byte loop with: @@ -213,7 +312,11 @@ while (storedResponseBytes > byteCap() && states.size > 0) { const entry = states.get(oldestId)!; if (entry.kind !== "resident") { deleteEntry(oldestId); continue; } try { - const ref = writeResponseSpillDurably(oldestId, entry); + const ref = writeResponseSpillDurably(oldestId, { + createdAt: entry.createdAt, + items: entry.items, + ...(entry.providers ? { providers: entry.providers } : {}), + }); swapResidentForSpill(oldestId, entry, ref); // only after durable success } catch { replaceWithSpillFailure(oldestId, entry); // R2-1: no hot oversized row @@ -229,7 +332,8 @@ Delete the `states.size > 1` exemption and update the old test-only cap comment. `expandPreviousResponseInput()` behavior: - resident: current expansion, unchanged; -- spill: read and fully validate, expand from payload, set replay provenance; +- spill: call `readResponseSpill(previousId, entry.spill)`, fully validate, expand from + payload, and set replay provenance on the exact expanded body; - missing/corrupt spill: leave body unchanged, set the failure WeakMap, increment the matching counter, delete the stub and bad/missing file best-effort, then insert a `spill-failed` tombstone for deterministic later calls; @@ -258,7 +362,9 @@ export interface ResponseStateMetrics { ``` `spillPayloadBytes` is the sum of refs, not file stat calls. `responseStateMetrics()` -remains observe-only. +remains observe-only. Update `src/server/management/system-routes.ts:88`, the GUI +`ResponseState` type, and `tests/memory-watchdog.test.ts:181` so every new field is +finite, scalar-only, and privacy-safe without making the metrics probe load/prune/read files. Snapshot compatibility: @@ -273,6 +379,8 @@ Snapshot compatibility: ### MODIFY `src/server/responses/core.ts` +### A-gate corrections (Banach) — caller-driven recovery contract + Immediately after `body = expandPreviousResponseInput(body)` and before parsing, inspect `previousResponseReplayFailure(body)`. Return one canonical response for all three known failure reasons: @@ -289,14 +397,26 @@ Add a `classifyError()` branch in `src/lib/errors.ts` mapping that explicit type `{type:"invalid_request_error", code:"previous_response_not_found"}`. Do not expose the spill reason, response id, digest, path, or OS error. Remove the `:1445-1449` warn-and-forward path for known spill failures; ordinary upstream-capable misses retain -their existing routing behavior. +their existing routing behavior. The guaranteed recovery contract is this terminal 400 +telling the caller to resend full context without `previous_response_id`; no automatic +Codex/client retry is claimed. + +Endpoint acceptance coverage exercises the route classes around +`src/server/responses/core.ts:1228-1239,1338-1343,1445-1449`: for a known spill failure, +each affected path returns status 400 with +`error.type === "invalid_request_error"`, +`error.code === "previous_response_not_found"`, and the canonical resend message before +any auth/upstream I/O. Existing ordinary-miss behavior remains separately asserted, +including the API-key provider case in `tests/issue-702-expired-replay-state.test.ts`. ## Regression tests +### A-gate corrections (Banach) — contract redefinitions and added classes + Extend `tests/responses-state.test.ts` with these exact tests/fixtures: - `spills the only oversized continuation and leaves resident bytes at or below cap` -- `does not swap a resident row to a stub before fsync and rename succeed` +- `does not swap a resident row to a stub before fsync and no-replace publication succeed` - `replays provider metadata and function_call_output history through a spill stub` - `replays a durable spill after simulated process restart` - `returns previous_response_not_found for a missing spill file without forwarding delta` @@ -309,24 +429,55 @@ Extend `tests/responses-state.test.ts` with these exact tests/fixtures: - `startup orphan cleanup preserves referenced young live and unrelated files` - `concurrent flush and synchronous demotion cannot inline or lose the spill stub` - `small entries retain the legacy v2 debounced snapshot representation` +- `re-spilling an id with changed content keeps the old generation until stub swap` + (A-gate blocker regression: crash simulated between new-generation publication and + stub swap — the stub must still replay the OLD generation intact, and startup + GC must remove the newer unreferenced generation file) +- `same response id and equal-size replacement use distinct basenames and unlink old only after stub swap` +- `spill-failed tombstone survives simulated process restart and returns the canonical failure` +- `spill replay preserves replay-prefix provenance and compaction-marker acknowledgement` +- `multibyte resident payload accounting uses complete UTF-8 bytes including provider metadata` +- `serialization failure rejects the candidate and records a bounded tombstone` +- `write fsync or publication failure cleans its temp and preserves no unmeasured resident payload` +- `EEXIST publication loss advances the generation and retries within the bound` +- `orphan cleanup obeys scan and cleanup caps rejects symlinks and counts failed unlink` +- `response-state management metrics keep every added field finite scalar and privacy-safe` - redefine `tests/responses-state.test.ts:535` as - `byte cap spills the newest-only chain instead of exempting it`; + `byte cap spills over-cap rows instead of evicting prior continuation ids`; +- redefine `tests/responses-state.test.ts:557` as + `byte accounting across restart preserves spilled ids and recomputes resident and stub bytes`; - redefine `tests/responses-state.test.ts:856` as - `oversized entries replay from dedicated spill across restart while small entries use snapshot`. + `oversized entries replay from dedicated spill across restart while small entries use snapshot`; +- redefine `tests/responses-state.test.ts:960` as + `empty store reports every resident spill tombstone and counter metric as zero`; +- redefine `tests/responses-state.test.ts:999` (including the exact object at `:1007`) as + `metrics remain side-effect free with the complete additive metric shape`. + +Every test intended to exercise spill sets `setResponseStateByteCapForTests(...)` below +the fixture's measured UTF-8 payload size and restores it in `finally`. In particular, the +current 3 MiB fixture at `tests/responses-state.test.ts:856` must run under a lower override; +the 2 MiB legacy snapshot-entry cap alone does not trigger spill. -Add endpoint coverage in `tests/server-combo-failover-e2e.test.ts` (or the nearest -Responses endpoint test) named: +Add endpoint coverage in `tests/issue-702-expired-replay-state.test.ts` (the +nearest existing expired-replay endpoint owner, per A-gate note; not the combo +failover e2e) named: -- `known continuation spill failure returns structured previous_response_not_found before upstream I/O`. +- `known continuation spill failure returns terminal structured previous_response_not_found before upstream I/O`. Fixtures must use an injected spill I/O seam, not chmod-only assertions that are unreliable -on Windows. The durability-order test records `write`, `fsync`, `close`, `harden`, `rename`, +on Windows. The durability-order test records `write`, `fsync`, `close`, `harden`, `publish`, `stub-swap` and asserts that order exactly. +The seam is a module-level injectable in `spill-store.ts` +(`setSpillIoForTest({write,fsync,link,copyFileExcl,unlink}| null)` — `publish` covers +both the link path and the exclusive-copy fallback), and the `stub-swap` +event is recorded by `state.ts` calling a `noteStubSwapForTest()` hook exported +from the same seam — the order assertion therefore spans the +`spill-store.ts`/`state.ts` boundary through one recorder. Verification: ```bash -bun test tests/responses-state.test.ts tests/server-combo-failover-e2e.test.ts +bun test tests/responses-state.test.ts tests/issue-702-expired-replay-state.test.ts bun run typecheck bun run test bun run privacy:scan From 29bcb0ee273156fef4cb0fe6b0346f97d1f91524 Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sat, 1 Aug 2026 08:39:49 +0900 Subject: [PATCH 04/30] docs(plan): lock the spill replacement to one atomic stub swap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A-gate rounds (Banach): trusted-ID spill reads, digest+generation basenames, caller-driven 400 recovery, failure-safe UTF-8 measurement with reject+tombstone, cap-override test triggers, consumer ledger, and the final residual — the replacement transaction is now a single old-stub→new-stub swap with the old file unlinked last. --- .../010_continuation_hard_cap.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/devlog/_plan/260801_zero_leak_state_stores/010_continuation_hard_cap.md b/devlog/_plan/260801_zero_leak_state_stores/010_continuation_hard_cap.md index cd504519d..3ab41ea4d 100644 --- a/devlog/_plan/260801_zero_leak_state_stores/010_continuation_hard_cap.md +++ b/devlog/_plan/260801_zero_leak_state_stores/010_continuation_hard_cap.md @@ -300,8 +300,7 @@ and never routes through `setResidentEntry()` or an intermediary resident row: `deleteEntry()` subtracts the row's RAM `sizeBytes`; for a spill stub it also unlinks the dedicated file unless `deleteSpill:false` is used during a verified atomic swap. TTL/count -eviction therefore removes spill files. No `supersededSpill` field or secondary ownership -state exists. +eviction therefore removes spill files. No secondary ownership state exists. Replace the byte loop with: @@ -429,10 +428,10 @@ Extend `tests/responses-state.test.ts` with these exact tests/fixtures: - `startup orphan cleanup preserves referenced young live and unrelated files` - `concurrent flush and synchronous demotion cannot inline or lose the spill stub` - `small entries retain the legacy v2 debounced snapshot representation` -- `re-spilling an id with changed content keeps the old generation until stub swap` - (A-gate blocker regression: crash simulated between new-generation publication and - stub swap — the stub must still replay the OLD generation intact, and startup - GC must remove the newer unreferenced generation file) +- `same-id spill replacement keeps the old row replay-addressable until atomic stub swap` + (the injected publication hook replays the id before swap and gets the old payload; + after swap it gets the new payload; the old file exists through swap and is unlinked after) +- `crash between new-generation publication and stub swap preserves the old row and reclaims the new orphan` - `same response id and equal-size replacement use distinct basenames and unlink old only after stub swap` - `spill-failed tombstone survives simulated process restart and returns the canonical failure` - `spill replay preserves replay-prefix provenance and compaction-marker acknowledgement` From d1408b92fc6a4823fae0be91890319cc2f0328eb Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sat, 1 Aug 2026 09:02:50 +0900 Subject: [PATCH 05/30] feat(responses): hard-cap continuation state with durable spill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The continuation store kept its newest entry exempt from the 64 MiB cap — one oversized response (images, giant tool results) could pin hundreds of MiB for an hour. The cap is now unconditional: oversized entries demote to a disk stub only after an atomic no-replace publication (write-fsync-link, EEXIST advances a generation counter), replay reads through the stub with digest verification, and a spill failure leaves a tombstone that fails continuation explicitly instead of silently dropping context. Startup GC reclaims orphaned generations; the legacy debounced snapshot still serves small entries. /api/system/memory responseState now reports resident/stub/tombstone counts, spill payload bytes, and write/read failure counters. Plan: devlog/_plan/260801_zero_leak_state_stores/010 (8 audit rounds). --- .../010_continuation_hard_cap.md | 25 +- .../components/MemoryObservabilityCard.tsx | 7 + src/lib/errors.ts | 3 + src/responses/spill-store.ts | 351 +++++++++++ src/responses/state.ts | 468 +++++++++++--- src/server/management/system-routes.ts | 4 +- src/server/responses/core.ts | 14 +- tests/issue-702-expired-replay-state.test.ts | 98 ++- tests/memory-watchdog.test.ts | 13 +- tests/responses-state.test.ts | 577 +++++++++++++++++- 10 files changed, 1444 insertions(+), 116 deletions(-) create mode 100644 src/responses/spill-store.ts diff --git a/devlog/_plan/260801_zero_leak_state_stores/010_continuation_hard_cap.md b/devlog/_plan/260801_zero_leak_state_stores/010_continuation_hard_cap.md index 3ab41ea4d..dbaa73f00 100644 --- a/devlog/_plan/260801_zero_leak_state_stores/010_continuation_hard_cap.md +++ b/devlog/_plan/260801_zero_leak_state_stores/010_continuation_hard_cap.md @@ -292,15 +292,32 @@ and never routes through `setResidentEntry()` or an intermediary resident row: 3. Build the new stub from the candidate's `createdAt`, provider metadata, measured stub bytes, and returned `ResponseSpillRef`. Verify `states.get(id) === expected`, then perform one accounting/map transition from the old stub directly to the new stub. -4. Only after the new stub is installed, unlink `expected.spill` best-effort. A crash before - the swap leaves the old row/file authoritative and the new file orphan-recoverable; a - crash after the swap leaves the new row authoritative and the old file orphan-recoverable. +4. Only after the new stub is installed, queue `expected.spill` on the bounded + `pendingSpillUnlinks` deferral (NOT an immediate unlink — see the deferred-deletion + contract below): the old generation must survive until the debounced snapshot that + references the new stub is durable, because a crash before that flush reloads the OLD + stub, which must still find its file. A crash before the swap leaves the old row/file + authoritative and the new file orphan-recoverable; a crash after the swap but before + the snapshot flush replays through the old snapshot stub against the still-present old + file; a crash after the flush leaves the new row authoritative and the queued old file + orphan-recoverable. 5. Serialization or publication failure atomically replaces `expected` with the bounded tombstone, then unlinks the old file; it never exposes a resident candidate or naked delta. `deleteEntry()` subtracts the row's RAM `sizeBytes`; for a spill stub it also unlinks the dedicated file unless `deleteSpill:false` is used during a verified atomic swap. TTL/count -eviction therefore removes spill files. No secondary ownership state exists. +eviction therefore removes spill files. No secondary OWNERSHIP state exists — but one +bounded piece of deferred-DELETION bookkeeping does (review C1-1/C2-1): a superseded +generation cannot be unlinked at swap time, because the new stub only becomes durable +when the debounced snapshot flushes; a crash before the flush reloads the OLD stub, +which must still find its file. The swap therefore queues the old ref in +`pendingSpillUnlinks` (hard cap 128 entries — small refs, not payloads; overflow +unlinks oldest-first immediately, accepting the narrow crash-window regression only +under pathological replacement churn), and `persistNow()` drains the queue strictly +AFTER the snapshot write succeeds. The queue holds no payload bytes, never counts +toward `storedResponseBytes`, and is not ownership: files it references are already +unreferenced by the post-flush snapshot and would be reclaimed by orphan GC anyway — +the queue only accelerates that reclamation. Replace the byte loop with: diff --git a/gui/src/components/MemoryObservabilityCard.tsx b/gui/src/components/MemoryObservabilityCard.tsx index 16146c629..da8709c29 100644 --- a/gui/src/components/MemoryObservabilityCard.tsx +++ b/gui/src/components/MemoryObservabilityCard.tsx @@ -26,9 +26,16 @@ type MemoryMetric = "rss" | "external" | "arrayBuffers"; interface ResponseState { count: number; + residentCount: number; + spillStubCount: number; + tombstoneCount: number; totalBytes: number; + spillPayloadBytes: number; largestBytes: number; oldestAgeMs: number; + spillWrites: number; + spillWriteFailures: number; + spillReadFailures: number; } interface SystemMemory { diff --git a/src/lib/errors.ts b/src/lib/errors.ts index b47b4529a..43db6cf4c 100644 --- a/src/lib/errors.ts +++ b/src/lib/errors.ts @@ -104,6 +104,9 @@ export function isClientClosedMessage(text: string): boolean { export function classifyError(status: number, type: string, message: string): OcxErrorPayload { const text = message.toLowerCase(); + if (type === "previous_response_not_found") { + return { message, type: "invalid_request_error", code: "previous_response_not_found" }; + } // Preserve explicit cancel types used by compact/combo JSON errors; unify message-inferred // client closes (web-search abort text) onto client_closed_request for /api/logs. if (type === "client_cancelled") { diff --git a/src/responses/spill-store.ts b/src/responses/spill-store.ts new file mode 100644 index 000000000..669ea22a3 --- /dev/null +++ b/src/responses/spill-store.ts @@ -0,0 +1,351 @@ +import { + chmodSync, + closeSync, + constants, + copyFileSync, + existsSync, + fsyncSync, + linkSync, + lstatSync, + mkdirSync, + openSync, + opendirSync, + readFileSync, + unlinkSync, + writeSync, +} from "node:fs"; +import { createHash, randomBytes } from "node:crypto"; +import { join } from "node:path"; +import { getConfigDir } from "../config"; +import { hardenSecretDir, hardenSecretPath } from "../lib/windows-secret-acl"; +import type { OcxProviderContinuationState } from "../types"; + +export const RESPONSE_SPILL_VERSION = 1; +export const RESPONSE_SPILL_DIR_NAME = "responses-state-spill"; +export const RESPONSE_SPILL_ORPHAN_GRACE_MS = 15 * 60_000; +export const RESPONSE_SPILL_SCAN_MAX = 4_096; +export const RESPONSE_SPILL_CLEANUP_MAX = 512; + +const RESPONSE_SPILL_PUBLISH_RETRIES = 64; +const OWNED_SPILL_NAME = /^([A-Za-z0-9._-]{1,80})\.([0-9a-f]{12})\.([0-9a-f]{24})\.(\d+)\.(\d+)\.spill\.json$/; +const OWNED_SPILL_TEMP_NAME = /^\.response-spill\.[0-9]+\.[0-9a-f]{16}\.tmp$/; + +export interface ResponseSpillPayload { + version: 1; + responseId: string; + createdAt: number; + items: unknown[]; + providers?: OcxProviderContinuationState; +} + +export interface ResponseSpillRef { + version: 1; + fileName: string; + digest: string; + payloadBytes: number; +} + +export type ResponseSpillReadResult = + | { ok: true; payload: ResponseSpillPayload } + | { ok: false; reason: "missing" | "corrupt" }; + +export interface ResponseSpillCleanupResult { + scanned: number; + removed: number; + failed: number; + bytesRemoved: number; +} + +export interface ResponseSpillIoForTest { + write?: (fd: number, bytes: Uint8Array) => void; + fsync?: (fd: number) => void; + link?: (tempPath: string, destinationPath: string) => void; + copyFileExcl?: (tempPath: string, destinationPath: string) => void; + unlink?: (path: string) => void; + /** Orphan-GC enumeration seam: returns the next directory entry NAME or + * null for end-of-directory. Lets tests prove the scan cap binds without + * materializing thousands of real files. */ + readdirEntry?: () => string | null; + record?: (event: "write" | "fsync" | "close" | "harden" | "publish" | "stub-swap") => void; +} + +let spillIoForTest: ResponseSpillIoForTest | null = null; +let spillGeneration = 0; + +export function setSpillIoForTest(io: ResponseSpillIoForTest | null): void { + spillIoForTest = io; +} + +function record(event: "write" | "fsync" | "close" | "harden" | "publish" | "stub-swap"): void { + spillIoForTest?.record?.(event); +} + +export function noteStubSwapForTest(): void { + record("stub-swap"); +} + +export function responseSpillDirectory(dir = getConfigDir()): string { + return join(dir, RESPONSE_SPILL_DIR_NAME); +} + +function sha256(value: string | Uint8Array): string { + return createHash("sha256").update(value).digest("hex"); +} + +function sanitizeResponseId(responseId: string): string { + const visible = responseId + .normalize("NFC") + .replace(/[^A-Za-z0-9._-]/g, "_") + .replace(/_+/g, "_") + .slice(0, 80); + return visible || "response"; +} + +function isErrno(error: unknown, code: string): boolean { + return !!error && typeof error === "object" && (error as NodeJS.ErrnoException).code === code; +} + +function canUseExclusiveCopyFallback(error: unknown): boolean { + return process.platform === "win32" || ["EPERM", "EACCES", "ENOSYS", "ENOTSUP", "EOPNOTSUPP", "EXDEV"] + .some(code => isErrno(error, code)); +} + +function harden(path: string, mode: number): void { + try { + chmodSync(path, mode); + } catch { + if (process.platform !== "win32") throw new Error("Response spill permission hardening failed"); + } + if (process.platform === "win32") { + const result = mode === 0o700 + ? hardenSecretDir(path, { required: true }) + : hardenSecretPath(path, { required: true }); + if (!result.ok) throw new Error("Response spill permission hardening failed"); + } +} + +function writeAll(fd: number, bytes: Uint8Array): void { + if (spillIoForTest?.write) spillIoForTest.write(fd, bytes); + else { + let offset = 0; + while (offset < bytes.byteLength) offset += writeSync(fd, bytes, offset, bytes.byteLength - offset); + } + record("write"); +} + +function fsyncFile(fd: number): void { + if (spillIoForTest?.fsync) spillIoForTest.fsync(fd); + else fsyncSync(fd); + record("fsync"); +} + +function closeFile(fd: number): void { + closeSync(fd); + record("close"); +} + +function unlink(path: string): void { + if (spillIoForTest?.unlink) spillIoForTest.unlink(path); + else unlinkSync(path); +} + +function publishNoReplace(tempPath: string, destinationPath: string): void { + try { + if (spillIoForTest?.link) spillIoForTest.link(tempPath, destinationPath); + else linkSync(tempPath, destinationPath); + } catch (error) { + if (isErrno(error, "EEXIST")) throw error; + if (!canUseExclusiveCopyFallback(error)) throw error; + let copied = false; + try { + if (spillIoForTest?.copyFileExcl) spillIoForTest.copyFileExcl(tempPath, destinationPath); + else copyFileSync(tempPath, destinationPath, constants.COPYFILE_EXCL); + copied = true; + harden(destinationPath, 0o600); + const copyFd = openSync(destinationPath, "r"); + try { + if (spillIoForTest?.fsync) spillIoForTest.fsync(copyFd); + else fsyncSync(copyFd); + } finally { + closeSync(copyFd); + } + } catch (copyError) { + if (copied) { + try { unlink(destinationPath); } catch { /* startup GC reclaims an incomplete publication */ } + } + throw copyError; + } + } + record("publish"); +} + +function validSpillRef(ref: ResponseSpillRef): boolean { + return ref.version === 1 + && OWNED_SPILL_NAME.test(ref.fileName) + && /^[0-9a-f]{64}$/.test(ref.digest) + && Number.isSafeInteger(ref.payloadBytes) + && ref.payloadBytes >= 0; +} + +function validPayload(value: unknown, responseId: string): value is ResponseSpillPayload { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const payload = value as Record; + const keys = Object.keys(payload); + if (keys.some(key => !["version", "responseId", "createdAt", "items", "providers"].includes(key))) return false; + if (payload.version !== 1 || payload.responseId !== responseId) return false; + if (typeof payload.createdAt !== "number" || !Number.isFinite(payload.createdAt)) return false; + if (!Array.isArray(payload.items)) return false; + if (payload.providers !== undefined) { + if (!payload.providers || typeof payload.providers !== "object" || Array.isArray(payload.providers)) return false; + for (const providerState of Object.values(payload.providers)) { + if (!providerState || typeof providerState !== "object" || Array.isArray(providerState)) return false; + } + } + return true; +} + +export function writeResponseSpillDurably( + responseId: string, + state: Omit, +): ResponseSpillRef { + let tempPath: string | null = null; + let fd: number | null = null; + try { + const payload: ResponseSpillPayload = { + version: 1, + responseId, + createdAt: state.createdAt, + items: state.items, + ...(state.providers ? { providers: state.providers } : {}), + }; + const serialized = JSON.stringify(payload); + if (serialized === undefined) throw new Error("Response spill serialization failed"); + const bytes = Buffer.from(serialized, "utf8"); + const digest = sha256(bytes); + const idDigest = sha256(responseId).slice(0, 12); + const contentDigest = digest.slice(0, 24); + const dir = responseSpillDirectory(); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + harden(dir, 0o700); + + tempPath = join(dir, `.response-spill.${process.pid}.${randomBytes(8).toString("hex")}.tmp`); + fd = openSync(tempPath, "wx", 0o600); + writeAll(fd, bytes); + fsyncFile(fd); + closeFile(fd); + fd = null; + harden(tempPath, 0o600); + record("harden"); + const publishTempPath = tempPath; + + for (let attempt = 0; attempt < RESPONSE_SPILL_PUBLISH_RETRIES; attempt++) { + spillGeneration += 1; + const fileName = `${sanitizeResponseId(responseId)}.${idDigest}.${contentDigest}.${spillGeneration}.${bytes.byteLength}.spill.json`; + if (!OWNED_SPILL_NAME.test(fileName)) throw new Error("Response spill name allocation failed"); + const destinationPath = join(dir, fileName); + try { + publishNoReplace(publishTempPath, destinationPath); + unlink(publishTempPath); + tempPath = null; + return { version: 1, fileName, digest, payloadBytes: bytes.byteLength }; + } catch (error) { + if (isErrno(error, "EEXIST")) continue; + throw error; + } + } + throw new Error("Response spill publication retries exhausted"); + } catch { + if (fd !== null) { + try { closeSync(fd); } catch { /* best effort */ } + } + if (tempPath) { + try { unlink(tempPath); } catch { /* best effort */ } + } + throw new Error("Response spill write failed"); + } +} + +export function readResponseSpill(responseId: string, ref: ResponseSpillRef): ResponseSpillReadResult { + if (!validSpillRef(ref)) return { ok: false, reason: "corrupt" }; + const match = OWNED_SPILL_NAME.exec(ref.fileName); + if (!match + || match[2] !== sha256(responseId).slice(0, 12) + || match[3] !== ref.digest.slice(0, 24) + || !Number.isSafeInteger(Number(match[4])) + || Number(match[4]) <= 0 + || Number(match[5]) !== ref.payloadBytes) return { ok: false, reason: "corrupt" }; + const path = join(responseSpillDirectory(), ref.fileName); + let bytes: Buffer; + try { + const stat = lstatSync(path); + if (!stat.isFile() || stat.isSymbolicLink()) return { ok: false, reason: "corrupt" }; + if (stat.size !== ref.payloadBytes) return { ok: false, reason: "corrupt" }; + bytes = readFileSync(path); + } catch (error) { + return { ok: false, reason: isErrno(error, "ENOENT") ? "missing" : "corrupt" }; + } + if (bytes.byteLength !== ref.payloadBytes || sha256(bytes) !== ref.digest) return { ok: false, reason: "corrupt" }; + try { + const payload = JSON.parse(bytes.toString("utf8")) as unknown; + return validPayload(payload, responseId) ? { ok: true, payload } : { ok: false, reason: "corrupt" }; + } catch { + return { ok: false, reason: "corrupt" }; + } +} + +export function deleteResponseSpill(ref: ResponseSpillRef): void { + if (!validSpillRef(ref)) return; + try { unlink(join(responseSpillDirectory(), ref.fileName)); } catch { /* best effort */ } +} + +export function recoverOrphanedResponseSpills( + referencedFileNames: ReadonlySet, + dir = responseSpillDirectory(), + opts?: { graceMs?: number }, +): ResponseSpillCleanupResult { + const result: ResponseSpillCleanupResult = { scanned: 0, removed: 0, failed: 0, bytesRemoved: 0 }; + const graceMs = opts?.graceMs ?? RESPONSE_SPILL_ORPHAN_GRACE_MS; + // ONE loop serves both the real directory handle and the injected test seam + // (review C2-2: two duplicated loops let the test prove only its own copy). + // The reader is called strictly AFTER the scan-cap check, so entry + // SCAN_MAX+1 is never requested from either source. + let handle: ReturnType | null = null; + const injected = spillIoForTest?.readdirEntry; + if (!injected) { + try { handle = opendirSync(dir); } catch { return result; } + } + const nextName = (): string | null => { + if (injected) return injected(); + const entry = handle!.readSync(); + return entry ? entry.name : null; + }; + try { + while (result.scanned < RESPONSE_SPILL_SCAN_MAX) { + const name = nextName(); + if (name === null) break; + result.scanned += 1; + if (result.removed + result.failed >= RESPONSE_SPILL_CLEANUP_MAX) break; + const spillMatch = OWNED_SPILL_NAME.exec(name); + const isOwnedTemp = OWNED_SPILL_TEMP_NAME.test(name); + if ((!spillMatch && !isOwnedTemp) || referencedFileNames.has(name)) continue; + const path = join(dir, name); + let stat: ReturnType; + try { stat = lstatSync(path); } catch { continue; } + if (!stat.isFile() || stat.isSymbolicLink() || Date.now() - stat.mtimeMs < graceMs) continue; + try { + unlink(path); + result.removed += 1; + result.bytesRemoved += stat.size; + } catch { + result.failed += 1; + } + } + } finally { + try { handle?.closeSync(); } catch { /* best effort */ } + } + return result; +} + +export function responseSpillExistsForTests(ref: ResponseSpillRef): boolean { + return validSpillRef(ref) && existsSync(join(responseSpillDirectory(), ref.fileName)); +} diff --git a/src/responses/state.ts b/src/responses/state.ts index 3345e4626..164a252d3 100644 --- a/src/responses/state.ts +++ b/src/responses/state.ts @@ -1,7 +1,16 @@ -import { chmodSync, existsSync, lstatSync, mkdirSync, opendirSync, readFileSync, unlinkSync } from "node:fs"; +import { chmodSync, existsSync, lstatSync, mkdirSync, opendirSync, readFileSync, rmSync, unlinkSync } from "node:fs"; import { dirname, join } from "node:path"; import { atomicWriteFileAsync, getConfigDir } from "../config"; import type { OcxProviderContinuationState } from "../types"; +import { + deleteResponseSpill, + noteStubSwapForTest, + readResponseSpill, + recoverOrphanedResponseSpills, + responseSpillDirectory, + type ResponseSpillRef, + writeResponseSpillDurably, +} from "./spill-store"; const MAX_STORED_RESPONSES = 1_000; const RESPONSE_TTL_MS = 60 * 60 * 1_000; @@ -10,8 +19,7 @@ const SNAPSHOT_DEBOUNCE_MS = 2_000; * continuation chains) stores the full expanded input each turn — ~quadratic bytes per chain — * so a count cap alone cannot bound memory. Oldest-first eviction applies past this mark. */ const MAX_STORED_RESPONSE_BYTES = 64 * 1024 * 1024; -/** Entries whose serialized size exceeds this are kept in memory but skipped on disk: inputs can - * carry base64 `input_image` data URLs, and one screenshot-heavy thread must not balloon the file. */ +/** Legacy snapshot selection only. Spill demotion is governed solely by the RAM cap above. */ const SNAPSHOT_ENTRY_MAX_BYTES = 2 * 1024 * 1024; const SNAPSHOT_TOTAL_MAX_BYTES = 24 * 1024 * 1024; const STALE_TEMP_GRACE_MS = 15 * 60 * 1_000; @@ -19,21 +27,52 @@ const STALE_TEMP_MAX_ENTRIES = 4_096; const STALE_TEMP_MAX_CLEANUPS = 512; const RESPONSE_STATE_TEMP_NAME = /^responses-state\.json\.ocx\.(\d+)\.(\d+)\.tmp$/; -interface StoredResponseState { +interface ResidentResponseState { + kind: "resident"; createdAt: number; items: unknown[]; - /** v2 provider-keyed continuation metadata. */ providers?: OcxProviderContinuationState; - /** v1 Cursor-only metadata, accepted only while loading old snapshots. */ - conversationId?: string; - cursorCheckpointUsable?: boolean; - /** Approximate in-memory size, computed locally at insert time (never trusted from disk). */ - sizeBytes?: number; + sizeBytes: number; } +interface SpilledResponseState { + kind: "spill"; + createdAt: number; + providers?: OcxProviderContinuationState; + spill: ResponseSpillRef; + sizeBytes: number; +} + +interface SpillFailedResponseState { + kind: "spill-failed"; + createdAt: number; + sizeBytes: number; +} + +type StoredResponseState = ResidentResponseState | SpilledResponseState | SpillFailedResponseState; +type ResidentInput = Omit; + +export type PreviousResponseReplayFailure = { + code: "previous_response_not_found"; + reason: "spill_missing" | "spill_corrupt" | "spill_failed"; +}; + const states = new Map(); let storedResponseBytes = 0; let byteCapOverride: number | null = null; +let stateRevision = 0; +const spillCounters = { writes: 0, writeFailures: 0, readFailures: 0 }; +// Superseded spill generations awaiting a durable snapshot before unlink +// (review C1-1: unlinking at swap time races a crash against the debounced +// snapshot — the reloaded OLD stub would point at a deleted file). +const pendingSpillUnlinks: ResponseSpillRef[] = []; +// The queue itself must stay bounded (review C2-2: repeated replacements with +// a persistently failing snapshot write would otherwise grow it without +// limit). Beyond the cap the OLDEST superseded generation is unlinked +// immediately: the accepted worst case is that a crash inside that window +// reloads a stub whose file is gone, which fails replay with the explicit +// structured 400 — bounded-loss, never silent corruption or unbounded disk. +const PENDING_SPILL_UNLINKS_MAX = 128; function byteCap(): number { return byteCapOverride ?? MAX_STORED_RESPONSE_BYTES; @@ -49,37 +88,149 @@ export function getStoredResponseBytesForTests(): number { return storedResponseBytes; } -/** The ONLY size computation: approximate entry weight from its items payload. */ -function measuredEntry(entry: Omit): StoredResponseState { - let sizeBytes = 0; +function serializedBytes(value: unknown): number | null { try { - sizeBytes = JSON.stringify(entry.items).length; + const serialized = JSON.stringify(value); + return serialized === undefined ? null : Buffer.byteLength(serialized, "utf8"); } catch { - /* unserializable items: weightless rather than fatal */ + return null; } - return { ...entry, sizeBytes }; } -/** The ONLY insertion point: keeps the byte counter consistent on replacement. */ -function setEntry(id: string, entry: Omit): void { - deleteEntry(id); - const measured = measuredEntry(entry); - storedResponseBytes += measured.sizeBytes ?? 0; - states.set(id, measured); +function measureResidentEntry(id: string, entry: ResidentInput): ResidentResponseState | null { + const sizeBytes = serializedBytes({ + responseId: id, + createdAt: entry.createdAt, + items: entry.items, + ...(entry.providers ? { providers: entry.providers } : {}), + }); + return sizeBytes === null ? null : { kind: "resident", ...entry, sizeBytes }; +} + +function replaceMapEntry(id: string, next: StoredResponseState, expected?: StoredResponseState): boolean { + const existing = states.get(id); + if (expected && existing !== expected) return false; + storedResponseBytes -= existing?.sizeBytes ?? 0; + storedResponseBytes += next.sizeBytes; + if (storedResponseBytes < 0) storedResponseBytes = 0; + if (existing) states.delete(id); + states.set(id, next); + stateRevision += 1; + return true; +} + +function stubSize(id: string, entry: Omit): number { + return serializedBytes({ responseId: id, ...entry }) ?? 0; +} + +function tombstone(id: string, createdAt: number): SpillFailedResponseState { + const base = { kind: "spill-failed" as const, createdAt }; + return { ...base, sizeBytes: serializedBytes({ responseId: id, ...base }) ?? 0 }; +} + +function deleteOwnedSpills(entry: StoredResponseState): void { + if (entry.kind === "spill") deleteResponseSpill(entry.spill); } /** The ONLY deletion point: TTL, count, byte, and explicit deletes all route here. */ -function deleteEntry(id: string): void { +function deleteEntry(id: string, options: { deleteSpill?: boolean } = {}): void { const existing = states.get(id); if (!existing) return; - storedResponseBytes -= existing.sizeBytes ?? 0; + storedResponseBytes -= existing.sizeBytes; if (storedResponseBytes < 0) storedResponseBytes = 0; states.delete(id); + stateRevision += 1; + if (options.deleteSpill !== false) deleteOwnedSpills(existing); +} + +function replaceWithSpillFailure(id: string, expected?: StoredResponseState): void { + const existing = states.get(id); + if (expected && existing !== expected) return; + const failed = tombstone(id, expected?.createdAt ?? existing?.createdAt ?? now()); + if (replaceMapEntry(id, failed, expected)) { + if (existing) deleteOwnedSpills(existing); + } +} + +function swapResidentForSpill(id: string, expected: ResidentResponseState, ref: ResponseSpillRef): boolean { + const base: Omit = { + kind: "spill", + createdAt: expected.createdAt, + ...(expected.providers ? { providers: expected.providers } : {}), + spill: ref, + }; + const next: SpilledResponseState = { ...base, sizeBytes: stubSize(id, base) }; + if (!replaceMapEntry(id, next, expected)) { + deleteResponseSpill(ref); + return false; + } + noteStubSwapForTest(); + return true; +} + +function replaceSpillEntryAtomically( + id: string, + expected: SpilledResponseState, + candidate: ResidentResponseState, +): void { + try { + const ref = writeResponseSpillDurably(id, { + createdAt: candidate.createdAt, + items: candidate.items, + ...(candidate.providers ? { providers: candidate.providers } : {}), + }); + const base: Omit = { + kind: "spill", + createdAt: candidate.createdAt, + ...(candidate.providers ? { providers: candidate.providers } : {}), + spill: ref, + }; + const next: SpilledResponseState = { ...base, sizeBytes: stubSize(id, base) }; + if (!replaceMapEntry(id, next, expected)) { + deleteResponseSpill(ref); + return; + } + spillCounters.writes += 1; + noteStubSwapForTest(); + // The old generation is NOT unlinked here (review C1-1): the new stub is + // only durable once the debounced snapshot flushes — a crash before that + // reloads the OLD stub, which must still find its file. Queue the unlink; + // persistNow() drains the queue only after the snapshot write succeeds. + pendingSpillUnlinks.push(expected.spill); + while (pendingSpillUnlinks.length > PENDING_SPILL_UNLINKS_MAX) { + deleteResponseSpill(pendingSpillUnlinks.shift()!); + } + } catch { + spillCounters.writeFailures += 1; + replaceWithSpillFailure(id, expected); + } +} + +function setResidentEntry(id: string, entry: ResidentInput): void { + const expected = states.get(id); + const candidate = measureResidentEntry(id, entry); + if (!candidate) { + replaceWithSpillFailure(id, expected); + // A tombstone is tiny but still resident state: the hard-cap invariant + // must hold on EVERY mutation path (review C2-1 — with a test cap below + // tombstone size, skipping the prune leaves the store over cap). + pruneResponses(); + return; + } + if (expected?.kind === "spill") { + replaceSpillEntryAtomically(id, expected, candidate); + pruneResponses(); + return; + } + if (!replaceMapEntry(id, candidate, expected)) return; + pruneResponses(); } + // Expansion provenance must stay proxy-private: a WeakMap distinguishes replayed history from the // newly appended input suffix without adding an unknown field that native passthrough could send // upstream. The parser uses this boundary to acknowledge historical compaction markers exactly once. const replayedInputPrefixLengths = new WeakMap(); +const replayFailures = new WeakMap(); let loaded = false; let persistTimer: ReturnType | null = null; let pendingPersistPath: string | null = null; @@ -94,6 +245,64 @@ function snapshotPath(): string { return join(getConfigDir(), "responses-state.json"); } +interface LegacySnapshotState { + createdAt?: unknown; + items?: unknown; + providers?: OcxProviderContinuationState; + conversationId?: unknown; + cursorCheckpointUsable?: unknown; +} + +function isSpillRef(value: unknown): value is ResponseSpillRef { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const ref = value as ResponseSpillRef; + return ref.version === 1 + && typeof ref.fileName === "string" + && /^[0-9a-f]{64}$/.test(ref.digest) + && Number.isSafeInteger(ref.payloadBytes) + && ref.payloadBytes >= 0; +} + +function loadSnapshotEntry(id: string, value: unknown): void { + if (!value || typeof value !== "object" || Array.isArray(value)) return; + const rec = value as LegacySnapshotState & { kind?: unknown; spill?: unknown }; + if (typeof rec.createdAt !== "number" || !Number.isFinite(rec.createdAt)) return; + if (rec.kind === "spill") { + if (!isSpillRef(rec.spill)) return; + const base: Omit = { + kind: "spill", + createdAt: rec.createdAt, + ...(rec.providers ? { providers: rec.providers } : {}), + spill: rec.spill, + }; + replaceMapEntry(id, { ...base, sizeBytes: stubSize(id, base) }); + return; + } + if (rec.kind === "spill-failed") { + replaceMapEntry(id, tombstone(id, rec.createdAt)); + return; + } + if (rec.kind !== undefined && rec.kind !== "resident") return; + if (!Array.isArray(rec.items)) return; + const providers = rec.providers ?? (typeof rec.conversationId === "string" + ? { + cursor: { + conversationId: rec.conversationId, + ...(typeof rec.cursorCheckpointUsable === "boolean" + ? { checkpointUsable: rec.cursorCheckpointUsable } + : {}), + }, + } + : undefined); + const resident = measureResidentEntry(id, { + createdAt: rec.createdAt, + items: rec.items, + ...(providers ? { providers } : {}), + }); + if (resident) replaceMapEntry(id, resident); + else replaceMapEntry(id, tombstone(id, rec.createdAt)); +} + export interface ResponseStateTempRecoveryResult { matched: number; removed: number; @@ -216,37 +425,24 @@ function ensureLoaded(): void { /* best-effort cleanup only; snapshot loading must remain independent */ } try { - if (!existsSync(path)) return; - const raw = JSON.parse(readFileSync(path, "utf-8")) as { version?: unknown; states?: unknown }; - if ((raw.version !== 1 && raw.version !== 2) || !Array.isArray(raw.states)) return; - for (const entry of raw.states) { - if (!Array.isArray(entry) || entry.length !== 2) continue; - const [id, state] = entry as [unknown, unknown]; - if (typeof id !== "string" || !state || typeof state !== "object") continue; - const rec = state as StoredResponseState; - if (typeof rec.createdAt !== "number" || !Array.isArray(rec.items)) continue; - const providers = rec.providers ?? (rec.conversationId - ? { - cursor: { - conversationId: rec.conversationId, - ...(rec.cursorCheckpointUsable !== undefined - ? { checkpointUsable: rec.cursorCheckpointUsable } - : {}), - }, - } - : undefined); - // Recompute sizes locally while loading — persisted sizeBytes (absent in v1/v2 - // snapshots anyway) is never trusted. - setEntry(id, { - createdAt: rec.createdAt, - items: rec.items, - ...(providers ? { providers } : {}), - }); + if (existsSync(path)) { + const raw = JSON.parse(readFileSync(path, "utf-8")) as { version?: unknown; states?: unknown }; + if ((raw.version === 1 || raw.version === 2) && Array.isArray(raw.states)) { + for (const entry of raw.states) { + if (!Array.isArray(entry) || entry.length !== 2 || typeof entry[0] !== "string") continue; + loadSnapshotEntry(entry[0], entry[1]); + } + } } - pruneResponses(); } catch { /* missing/corrupt snapshot: start empty */ } + const referenced = new Set(); + for (const state of states.values()) { + if (state.kind === "spill") referenced.add(state.spill.fileName); + } + try { recoverOrphanedResponseSpills(referenced); } catch { /* best effort */ } + pruneResponses(); } async function persistNow(path: string): Promise { @@ -262,26 +458,39 @@ async function persistNow(path: string): Promise { persistGate = new Promise(resolve => { release = resolve; }); await previous; try { - const entries: [string, StoredResponseState][] = []; - let total = 0; - // Newest-first so the most recent chains survive both caps. - for (const entry of [...states].reverse()) { - // sizeBytes is in-memory accounting only; keep it out of the disk snapshot. - const [id, state] = entry; - const { sizeBytes: _sizeBytes, ...persistable } = state; - const persistEntry: [string, StoredResponseState] = [id, persistable]; - const size = JSON.stringify(persistEntry).length; - if (size > SNAPSHOT_ENTRY_MAX_BYTES) continue; - if (total + size > SNAPSHOT_TOTAL_MAX_BYTES) break; - total += size; - entries.push(persistEntry); + for (;;) { + const revision = stateRevision; + const entries: Array<[string, unknown]> = []; + let total = 0; + // Newest-first so the most recent chains survive both legacy snapshot caps. + for (const [id, state] of [...states].reverse()) { + let persistable: unknown; + if (state.kind === "resident") { + const { sizeBytes: _sizeBytes, kind: _kind, ...resident } = state; + persistable = resident; + } else { + const { sizeBytes: _sizeBytes, ...smallState } = state; + persistable = smallState; + } + const persistEntry: [string, unknown] = [id, persistable]; + const size = JSON.stringify(persistEntry).length; + if (state.kind === "resident" && size > SNAPSHOT_ENTRY_MAX_BYTES) continue; + if (total + size > SNAPSHOT_TOTAL_MAX_BYTES) break; + total += size; + entries.push(persistEntry); + } + entries.reverse(); + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); + try { chmodSync(dirname(path), 0o700); } catch { /* best-effort (e.g. Windows) */ } + await atomicWriteFileAsync(path, JSON.stringify({ version: 2, states: entries })); + if (revision === stateRevision) break; + } + // Snapshot is durable: every stub it references is the NEW generation, so + // superseded generations can no longer be reloaded — unlink them now. + while (pendingSpillUnlinks.length > 0) { + const ref = pendingSpillUnlinks.shift()!; + deleteResponseSpill(ref); } - entries.reverse(); - mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); - // mkdirSync's mode only applies on creation — re-harden an existing config dir so the - // conversation-content snapshot never lands in a group/world-readable directory. - try { chmodSync(dirname(path), 0o700); } catch { /* best-effort (e.g. Windows) */ } - await atomicWriteFileAsync(path, JSON.stringify({ version: 2, states: entries })); } catch { /* best-effort: disk trouble must never affect request handling */ } finally { @@ -325,14 +534,63 @@ function pruneResponses(at = now()): void { if (!oldest) break; deleteEntry(oldest); } - // Byte high-water eviction, oldest-first (Map preserves insertion order). - while (storedResponseBytes > byteCap() && states.size > 1) { - const oldest = states.keys().next().value; - if (!oldest) break; - deleteEntry(oldest); + // Unconditional RAM cap. Resident payloads demote durably; stubs/tombstones are + // deleted only when even their bounded metadata cannot fit the override. + while (storedResponseBytes > byteCap() && states.size > 0) { + const oldestId = states.keys().next().value as string | undefined; + if (!oldestId) break; + const entry = states.get(oldestId)!; + if (entry.kind !== "resident") { + deleteEntry(oldestId); + continue; + } + try { + const ref = writeResponseSpillDurably(oldestId, { + createdAt: entry.createdAt, + items: entry.items, + ...(entry.providers ? { providers: entry.providers } : {}), + }); + if (swapResidentForSpill(oldestId, entry, ref)) spillCounters.writes += 1; + } catch { + spillCounters.writeFailures += 1; + replaceWithSpillFailure(oldestId, entry); + } } } +function materializeEntry( + id: string, + entry: StoredResponseState, +): { ok: true; state: ResidentResponseState } | { ok: false; failure: PreviousResponseReplayFailure } { + if (entry.kind === "resident") return { ok: true, state: entry }; + if (entry.kind === "spill-failed") { + return { ok: false, failure: { code: "previous_response_not_found", reason: "spill_failed" } }; + } + const result = readResponseSpill(id, entry.spill); + if (!result.ok) { + spillCounters.readFailures += 1; + const failure: PreviousResponseReplayFailure = { + code: "previous_response_not_found", + reason: result.reason === "missing" ? "spill_missing" : "spill_corrupt", + }; + replaceWithSpillFailure(id, entry); + schedulePersist(); + return { ok: false, failure }; + } + const state = measureResidentEntry(id, { + createdAt: result.payload.createdAt, + items: result.payload.items, + ...(result.payload.providers ? { providers: result.payload.providers } : {}), + }); + if (!state) { + spillCounters.readFailures += 1; + replaceWithSpillFailure(id, entry); + schedulePersist(); + return { ok: false, failure: { code: "previous_response_not_found", reason: "spill_corrupt" } }; + } + return { ok: true, state }; +} + export function expandPreviousResponseInput(body: unknown): unknown { if (!body || typeof body !== "object" || Array.isArray(body)) return body; const request = body as Record; @@ -342,14 +600,24 @@ export function expandPreviousResponseInput(body: unknown): unknown { pruneResponses(); const previous = states.get(previousId); if (!previous) return body; + const materialized = materializeEntry(previousId, previous); + if (!materialized.ok) { + replayFailures.set(request, materialized.failure); + return body; + } const expanded = { ...request, - input: [...previous.items, ...inputItems(request.input)], + input: [...materialized.state.items, ...inputItems(request.input)], }; - replayedInputPrefixLengths.set(expanded, previous.items.length); + replayedInputPrefixLengths.set(expanded, materialized.state.items.length); return expanded; } +export function previousResponseReplayFailure(body: unknown): PreviousResponseReplayFailure | undefined { + if (!body || typeof body !== "object" || Array.isArray(body)) return undefined; + return replayFailures.get(body); +} + /** Number of leading input items restored from previous_response_id state for this exact body. */ export function previousResponseReplayPrefixLength(body: unknown): number { if (!body || typeof body !== "object" || Array.isArray(body)) return 0; @@ -364,22 +632,23 @@ export function previousResponseProviderState(responseId: string | undefined): O if (!responseId) return undefined; ensureLoaded(); pruneResponses(); - const providers = states.get(responseId)?.providers; + const state = states.get(responseId); + const providers = state?.kind === "spill-failed" ? undefined : state?.providers; return providers ? structuredClone(providers) : undefined; } export interface ResponseStateMetrics { - /** Number of retained continuation entries currently in RAM. */ count: number; - /** Sum of serialized item bytes across all entries (the running in-memory byte accounting). - * Because expanded previous_response_id chains share prefix item references, this is an UPPER - * bound on true heap (it re-counts shared history), never an under-count — safe as a - * memory-pressure signal. */ + residentCount: number; + spillStubCount: number; + tombstoneCount: number; totalBytes: number; - /** Serialized item bytes of the single largest entry (a long chain's head, or an image-heavy turn). */ + spillPayloadBytes: number; largestBytes: number; - /** Age of the oldest retained entry, in ms (0 when the store is empty). */ oldestAgeMs: number; + spillWrites: number; + spillWriteFailures: number; + spillReadFailures: number; } /** @@ -395,16 +664,33 @@ export function responseStateMetrics(): ResponseStateMetrics { const at = now(); let largestBytes = 0; let oldestCreatedAt = at; + let residentCount = 0; + let spillStubCount = 0; + let tombstoneCount = 0; + let spillPayloadBytes = 0; for (const state of states.values()) { - const bytes = state.sizeBytes ?? 0; + const bytes = state.sizeBytes; if (bytes > largestBytes) largestBytes = bytes; if (state.createdAt < oldestCreatedAt) oldestCreatedAt = state.createdAt; + if (state.kind === "resident") { + residentCount += 1; + } else if (state.kind === "spill") { + spillStubCount += 1; + spillPayloadBytes += state.spill.payloadBytes; + } else tombstoneCount += 1; } return { count: states.size, + residentCount, + spillStubCount, + tombstoneCount, totalBytes: storedResponseBytes, + spillPayloadBytes, largestBytes, oldestAgeMs: states.size > 0 ? at - oldestCreatedAt : 0, + spillWrites: spillCounters.writes, + spillWriteFailures: spillCounters.writeFailures, + spillReadFailures: spillCounters.readFailures, }; } @@ -441,7 +727,7 @@ export function rememberResponseState( return !!item && typeof item === "object" && (item as { type?: unknown }).type === "function_call"; }); } - setEntry(response.id, { + setResidentEntry(response.id, { createdAt: now(), items: [...inputItems(request.input), ...response.output], // Always preserve the Cursor conversation id so the next tool-result turn can continue the SAME @@ -451,7 +737,6 @@ export function rememberResponseState( // checkpoint must not be reused — but the conversation id string itself is still valid. ...(Object.keys(normalizedProviderState).length > 0 ? { providers: normalizedProviderState } : {}), }); - pruneResponses(); schedulePersist(); } @@ -464,14 +749,21 @@ export function clearResponseStateMemoryForTests(): void { pendingPersistPath = null; states.clear(); storedResponseBytes = 0; + stateRevision = 0; + pendingSpillUnlinks.length = 0; + spillCounters.writes = 0; + spillCounters.writeFailures = 0; + spillCounters.readFailures = 0; loaded = false; } export function clearResponseStateForTests(): void { + for (const entry of states.values()) deleteOwnedSpills(entry); clearResponseStateMemoryForTests(); try { unlinkSync(snapshotPath()); } catch { /* no snapshot on disk */ } + try { rmSync(responseSpillDirectory(), { recursive: true, force: true }); } catch { /* no spill directory */ } } diff --git a/src/server/management/system-routes.ts b/src/server/management/system-routes.ts index ff04a3b0f..9d4acd022 100644 --- a/src/server/management/system-routes.ts +++ b/src/server/management/system-routes.ts @@ -14,7 +14,9 @@ * it is not a standalone leak discriminator. `responseState` attributes growth * further: it is the proxy's previous_response_id continuation store, so a * growing responseState.totalBytes under rising observed memory points at - * conversation retention rather than the runtime allocator. + * conversation retention rather than the runtime allocator. Spill counts, + * payload-byte totals, tombstones, and failure counters remain finite scalars; + * response ids, filenames, digests, paths, and payload content never leave the owner. * * `activeTurnCount` / `isDraining` are scalar lifecycle counters for the * dashboard drain-and-restart confirm UX — never request bodies or IDs. diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index a5c5f7d1d..7ddd1a007 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -10,7 +10,12 @@ import { import { parseRequest } from "../../responses/parser"; import { buildCompactV1Output, COMPACT_PROMPT, decodeCompactionSummary, extractCompactUserMessages } from "../../responses/compaction"; import { FORWARD_HEADERS, sanitizeReasoningInputContent } from "../../adapters/openai-responses"; -import { expandPreviousResponseInput, previousResponseProviderState, rememberResponseState } from "../../responses/state"; +import { + expandPreviousResponseInput, + previousResponseProviderState, + previousResponseReplayFailure, + rememberResponseState, +} from "../../responses/state"; import { routeModel, type RouteResult } from "../../router"; import { advanceComboAfterFailure, @@ -1091,6 +1096,13 @@ export async function handleResponses( ); const originalBody = body; body = expandPreviousResponseInput(body); + if (previousResponseReplayFailure(body)) { + return formatErrorResponse( + 400, + "previous_response_not_found", + "Continuation state is unavailable or corrupt; resend the full conversation without previous_response_id.", + ); + } const previousResponseInputExpanded = body !== originalBody; // Spawn-message compatibility (both directions): agent_message task payloads ride in diff --git a/tests/issue-702-expired-replay-state.test.ts b/tests/issue-702-expired-replay-state.test.ts index 6e5ff413e..8deabef5e 100644 --- a/tests/issue-702-expired-replay-state.test.ts +++ b/tests/issue-702-expired-replay-state.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { mkdtempSync, rmSync } from "node:fs"; +import { mkdtempSync, readdirSync, rmSync, unlinkSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { saveConfig } from "../src/config"; @@ -10,9 +10,12 @@ import { import { clearResponseStateForTests, clearResponseStateMemoryForTests, + rememberResponseState, responseStateMetrics, + setResponseStateByteCapForTests, type ResponseStateMetrics, } from "../src/responses/state"; +import { responseSpillDirectory } from "../src/responses/spill-store"; import { startServer } from "../src/server"; import type { OcxConfig } from "../src/types"; import { fakeChatGptJwt } from "./helpers/fake-chatgpt-jwt"; @@ -228,6 +231,7 @@ beforeEach(() => { afterEach(() => { globalThis.fetch = originalFetch; clearResponseStateForTests(); + setResponseStateByteCapForTests(null); resetSubagentModelFallbackStateForTests(); isolatedCodexHome?.restore(); isolatedCodexHome = null; @@ -240,6 +244,98 @@ afterEach(() => { }); describe("Issue #702 expired forward replay state", () => { + test("known continuation spill failure returns terminal structured previous_response_not_found before upstream I/O", async () => { + const responseId = "resp_issue_702_missing_spill"; + setResponseStateByteCapForTests(1_024); + rememberResponseState( + { model: "openai/gpt-5.6-sol", input: "x".repeat(8_000), store: false }, + { id: responseId, status: "completed", output: [{ role: "assistant", content: "done" }] }, + undefined, + { force: true }, + ); + const spillDir = responseSpillDirectory(testHome); + const spill = readdirSync(spillDir).find(name => name.endsWith(".spill.json")); + expect(spill).toBeDefined(); + unlinkSync(join(spillDir, spill!)); + + let upstreamCalls = 0; + globalThis.fetch = (async () => { + upstreamCalls += 1; + throw new Error("upstream must not be called"); + }) as typeof fetch; + const routeClasses: Array<{ config: OcxConfig; model: string }> = [ + { config: forwardConfig(), model: "gpt-5.6-sol" }, + { + config: { + port: 0, + hostname: "127.0.0.1", + openaiProviderTierVersion: 2, + defaultProvider: "kiro-test", + providers: { + "kiro-test": { + adapter: "kiro", + baseUrl: "https://runtime.us-east-1.kiro.dev", + authMode: "key", + apiKey: "synthetic-token", + models: ["gpt-5.6-sol"], + }, + }, + } as OcxConfig, + model: "kiro-test/gpt-5.6-sol", + }, + { + config: { + port: 0, + hostname: "127.0.0.1", + openaiProviderTierVersion: 2, + defaultProvider: "test-openai", + providers: { + "test-openai": { + adapter: "openai-responses", + baseUrl: "https://api.openai.com/v1", + authMode: "key", + apiKey: "provider-key", + models: ["gpt-5.6-sol"], + }, + }, + } as OcxConfig, + model: "test-openai/gpt-5.6-sol", + }, + ]; + + try { + for (const routeClass of routeClasses) { + saveConfig(routeClass.config); + const server = startServer(0); + try { + const response = await originalFetch(new URL("/v1/responses", server.url), { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: routeClass.model, + previous_response_id: responseId, + input: [inputMessage(CURRENT_USER_SENTINEL)], + stream: true, + }), + }); + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ + error: { + message: "Continuation state is unavailable or corrupt; resend the full conversation without previous_response_id.", + type: "invalid_request_error", + code: "previous_response_not_found", + }, + }); + } finally { + await server.stop(true); + } + } + expect(upstreamCalls).toBe(0); + } finally { + globalThis.fetch = originalFetch; + } + }); + test("forward mode fails closed when previous response replay state has expired", async () => { let quotaPrimeCalls = 0; setSubagentQuotaPrimeForTests(async () => { diff --git a/tests/memory-watchdog.test.ts b/tests/memory-watchdog.test.ts index 4cde5d229..294084375 100644 --- a/tests/memory-watchdog.test.ts +++ b/tests/memory-watchdog.test.ts @@ -178,7 +178,11 @@ describe("GET /api/system/memory", () => { pid: number; bunVersion: string; platform: string; rss: number; heapUsed: number; external: number; arrayBuffers: number; observedBytes: number; observedMetric: string; jscHeap: { heapSize: number } | null; - responseState: { count: number; totalBytes: number; largestBytes: number; oldestAgeMs: number }; + responseState: { + count: number; residentCount: number; spillStubCount: number; tombstoneCount: number; + totalBytes: number; spillPayloadBytes: number; largestBytes: number; oldestAgeMs: number; + spillWrites: number; spillWriteFailures: number; spillReadFailures: number; + }; inspectionCounters: { frameBufferHighWaterBytes: number; completedItemsMaxCount: number; frameCapOverflows: number; itemCapEvictions: number; postCancelDrainStops: number; @@ -198,10 +202,9 @@ describe("GET /api/system/memory", () => { expect(body.jscHeap?.heapSize).toBeGreaterThan(0); // responseState is a scalar-only continuation-store attribution block: every field is a // finite number (no paths, tokens, or account identifiers), so it is safe on this surface. - expect(typeof body.responseState.count).toBe("number"); - expect(typeof body.responseState.totalBytes).toBe("number"); - expect(typeof body.responseState.largestBytes).toBe("number"); - expect(typeof body.responseState.oldestAgeMs).toBe("number"); + const responseStateValues = Object.values(body.responseState); + expect(responseStateValues).toHaveLength(11); + expect(responseStateValues.every(value => typeof value === "number" && Number.isFinite(value))).toBe(true); expect(body.responseState.count).toBeGreaterThanOrEqual(0); expect(body.inspectionCounters).toEqual({ frameBufferHighWaterBytes: expect.any(Number), diff --git a/tests/responses-state.test.ts b/tests/responses-state.test.ts index 1e18efda2..cb582798d 100644 --- a/tests/responses-state.test.ts +++ b/tests/responses-state.test.ts @@ -1,5 +1,17 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, utimesSync, writeFileSync } from "node:fs"; +import { + existsSync, + linkSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + symlinkSync, + unlinkSync, + utimesSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { buildResponseJSON } from "../src/bridge"; @@ -14,12 +26,21 @@ import { flushResponseState, previousResponseConversationId, previousResponseProviderState, + previousResponseReplayFailure, + previousResponseReplayPrefixLength, recoverStaleResponseStateTemps, rememberResponseState, responseStateMetrics, setResponseStateByteCapForTests, getStoredResponseBytesForTests, } from "../src/responses/state"; +import { + readResponseSpill, + recoverOrphanedResponseSpills, + responseSpillDirectory, + setSpillIoForTest, + writeResponseSpillDurably, +} from "../src/responses/spill-store"; import { adapterNeedsForcedContinuation, injectDeveloperMessage } from "../src/server/responses"; function feedInspector( @@ -45,6 +66,24 @@ function isExactGuidanceItem(item: unknown, text: string): boolean { && (part as Record).text === text; } +function fixedResponse(id: string, output: unknown[]): { id: string; output: unknown[]; status: string } { + return { id, output, status: "completed" }; +} + +function spillFileNames(home: string): string[] { + const dir = responseSpillDirectory(home); + return existsSync(dir) ? readdirSync(dir).filter(name => name.endsWith(".spill.json")) : []; +} + +function rememberLarge(id: string, text: string, providers?: Parameters[2]): void { + rememberResponseState( + { model: "test/model", input: text, store: false }, + fixedResponse(id, [{ type: "message", role: "assistant", content: text }]), + providers, + { force: true }, + ); +} + describe("Responses previous_response_id state", () => { // Sandbox OPENCODEX_HOME: the state store now snapshots to disk, and these tests must never // touch the real ~/.opencodex. @@ -58,6 +97,8 @@ describe("Responses previous_response_id state", () => { }); afterEach(() => { + setSpillIoForTest(null); + setResponseStateByteCapForTests(null); clearResponseStateForTests(); rmSync(home, { recursive: true, force: true }); if (priorHome === undefined) delete process.env["OPENCODEX_HOME"]; @@ -532,7 +573,474 @@ describe("Responses previous_response_id state", () => { expect(adapterNeedsForcedContinuation("")).toBe(false); }); - test("byte cap evicts oldest entries while the newest chain link survives", () => { + test("spills the only oversized continuation and leaves resident bytes at or below cap", () => { + setResponseStateByteCapForTests(1_024); + rememberLarge("resp_only_oversized", "x".repeat(8_000)); + const metrics = responseStateMetrics(); + expect(metrics).toMatchObject({ count: 1, residentCount: 0, spillStubCount: 1 }); + expect(metrics.totalBytes).toBeLessThanOrEqual(1_024); + expect((expandPreviousResponseInput({ + previous_response_id: "resp_only_oversized", input: "next", + }) as { input: unknown[] }).input).toHaveLength(3); + }); + + test("does not swap a resident row to a stub before fsync and no-replace publication succeed", () => { + const events: string[] = []; + setSpillIoForTest({ record: event => events.push(event) }); + setResponseStateByteCapForTests(1_024); + rememberLarge("resp_durable_order", "x".repeat(8_000)); + expect(events).toEqual(["write", "fsync", "close", "harden", "publish", "stub-swap"]); + }); + + test("replays provider metadata and function_call_output history through a spill stub", () => { + setResponseStateByteCapForTests(1_024); + rememberResponseState( + { input: "x".repeat(8_000), store: false }, + fixedResponse("resp_spill_tool", [{ type: "function_call", call_id: "call_1", name: "ping", arguments: "{}" }]), + { cursor: { conversationId: "cursor-spill" } }, + { force: true }, + ); + const expanded = expandPreviousResponseInput({ + previous_response_id: "resp_spill_tool", + input: [{ type: "function_call_output", call_id: "call_1", output: "ok" }], + }) as { input: unknown[] }; + expect(previousResponseProviderState("resp_spill_tool")?.cursor?.conversationId).toBe("cursor-spill"); + expect(expanded.input.at(-1)).toMatchObject({ type: "function_call_output", call_id: "call_1" }); + }); + + test("replays a durable spill after simulated process restart", async () => { + setResponseStateByteCapForTests(1_024); + rememberLarge("resp_spill_restart", "r".repeat(8_000)); + await flushResponseState(); + clearResponseStateMemoryForTests(); + setResponseStateByteCapForTests(1_024); + const expanded = expandPreviousResponseInput({ previous_response_id: "resp_spill_restart", input: "next" }); + expect((expanded as { input: unknown[] }).input).toHaveLength(3); + expect(responseStateMetrics().spillStubCount).toBe(1); + }); + + test("spill references bind the expected response id and use the locked digest basename", () => { + const ref = writeResponseSpillDurably("resp_identity", { + createdAt: Date.now(), + items: ["payload"], + }); + expect(ref.fileName).toMatch(/\.[0-9a-f]{12}\.[0-9a-f]{24}\.\d+\.\d+\.spill\.json$/); + expect(readResponseSpill("resp_identity", ref).ok).toBe(true); + expect(readResponseSpill("resp_other", ref)).toEqual({ ok: false, reason: "corrupt" }); + }); + + test("returns previous_response_not_found for a missing spill file without forwarding delta", () => { + setResponseStateByteCapForTests(1_024); + rememberLarge("resp_missing_spill", "m".repeat(8_000)); + const file = spillFileNames(home)[0]!; + unlinkSync(join(responseSpillDirectory(home), file)); + const body = { previous_response_id: "resp_missing_spill", input: "delta" }; + expect(expandPreviousResponseInput(body)).toBe(body); + expect(previousResponseReplayFailure(body)).toEqual({ + code: "previous_response_not_found", reason: "spill_missing", + }); + expect(responseStateMetrics()).toMatchObject({ spillStubCount: 0, tombstoneCount: 1, spillReadFailures: 1 }); + }); + + test("returns previous_response_not_found for a corrupt or digest-mismatched spill", () => { + setResponseStateByteCapForTests(1_024); + rememberLarge("resp_corrupt_spill", "c".repeat(8_000)); + const path = join(responseSpillDirectory(home), spillFileNames(home)[0]!); + const size = readFileSync(path).byteLength; + writeFileSync(path, "z".repeat(size)); + const body = { previous_response_id: "resp_corrupt_spill", input: "delta" }; + expect(expandPreviousResponseInput(body)).toBe(body); + expect(previousResponseReplayFailure(body)?.reason).toBe("spill_corrupt"); + }); + + test("spill write failure evicts resident bytes and records one bounded tombstone", () => { + setSpillIoForTest({ write: () => { throw new Error("injected write failure"); } }); + setResponseStateByteCapForTests(1_024); + rememberLarge("resp_write_failure", "w".repeat(8_000)); + const metrics = responseStateMetrics(); + expect(metrics).toMatchObject({ count: 1, residentCount: 0, tombstoneCount: 1, spillWriteFailures: 1 }); + expect(metrics.totalBytes).toBeLessThanOrEqual(1_024); + }); + + test("disk permission failure increments spillWriteFailures without retaining payload", () => { + const denied = Object.assign(new Error("denied"), { code: "EACCES" }); + setSpillIoForTest({ write: () => { throw denied; } }); + setResponseStateByteCapForTests(1_024); + rememberLarge("resp_permission_failure", "p".repeat(8_000)); + expect(responseStateMetrics()).toMatchObject({ residentCount: 0, tombstoneCount: 1, spillWriteFailures: 1 }); + }); + + test("replacing a response id deletes its previous dedicated spill file after the snapshot flush", async () => { + setResponseStateByteCapForTests(1_024); + rememberLarge("resp_replace_spill", "a".repeat(8_000)); + const old = spillFileNames(home)[0]!; + rememberLarge("resp_replace_spill", "b".repeat(8_000)); + // The superseded generation survives until the debounced snapshot is + // durable (crash safety: a pre-flush reload must find the OLD file). + expect(existsSync(join(responseSpillDirectory(home), old))).toBe(true); + await flushResponseState(); + expect(existsSync(join(responseSpillDirectory(home), old))).toBe(false); + expect(spillFileNames(home)).toHaveLength(1); + }); + + test("TTL and count eviction delete dedicated spill files and release stub bytes", () => { + const realNow = Date.now; + setResponseStateByteCapForTests(1_024); + try { + Date.now = () => realNow() - 2 * 60 * 60 * 1_000; + rememberLarge("resp_ttl_spill", "t".repeat(8_000)); + const ttlFile = spillFileNames(home)[0]!; + Date.now = realNow; + rememberLarge("resp_after_ttl", "u".repeat(8_000)); + expect(existsSync(join(responseSpillDirectory(home), ttlFile))).toBe(false); + + clearResponseStateForTests(); + setResponseStateByteCapForTests(1_024); + rememberLarge("resp_count_spill", "v".repeat(8_000)); + const countFile = spillFileNames(home)[0]!; + setResponseStateByteCapForTests(1_000_000_000); + for (let i = 0; i < 1_000; i++) rememberLarge(`resp_count_${i}`, "x"); + expect(existsSync(join(responseSpillDirectory(home), countFile))).toBe(false); + expect(responseStateMetrics().count).toBe(1_000); + } finally { + Date.now = realNow; + } + }); + + test("startup orphan cleanup removes only old unreferenced regular spill files", async () => { + setResponseStateByteCapForTests(1_024); + rememberLarge("resp_live_orphan_gc", "l".repeat(8_000)); + await flushResponseState(); + const live = spillFileNames(home)[0]!; + const orphanRef = writeResponseSpillDurably("resp_orphan_gc", { + createdAt: Date.now(), items: ["orphan"], + }); + const old = new Date(Date.now() - 20 * 60_000); + utimesSync(join(responseSpillDirectory(home), orphanRef.fileName), old, old); + clearResponseStateMemoryForTests(); + expandPreviousResponseInput({ previous_response_id: "resp_live_orphan_gc", input: "next" }); + expect(existsSync(join(responseSpillDirectory(home), live))).toBe(true); + expect(existsSync(join(responseSpillDirectory(home), orphanRef.fileName))).toBe(false); + }); + + test("startup orphan cleanup preserves referenced young live and unrelated files", async () => { + setResponseStateByteCapForTests(1_024); + rememberLarge("resp_gc_preserve", "g".repeat(8_000)); + await flushResponseState(); + const live = spillFileNames(home)[0]!; + const young = writeResponseSpillDurably("resp_young", { createdAt: Date.now(), items: ["young"] }); + const unrelated = join(responseSpillDirectory(home), "unrelated.txt"); + writeFileSync(unrelated, "keep"); + clearResponseStateMemoryForTests(); + expandPreviousResponseInput({ previous_response_id: "resp_gc_preserve", input: "next" }); + expect(existsSync(join(responseSpillDirectory(home), live))).toBe(true); + expect(existsSync(join(responseSpillDirectory(home), young.fileName))).toBe(true); + expect(existsSync(unrelated)).toBe(true); + }); + + test("concurrent flush and synchronous demotion cannot inline or lose the spill stub", async () => { + rememberLarge("resp_before_flush", "small"); + const flushing = flushResponseState(); + setResponseStateByteCapForTests(1_024); + rememberLarge("resp_during_flush", "d".repeat(8_000)); + await flushing; + await flushResponseState(); + const snapshot = JSON.parse(readFileSync(join(home, "responses-state.json"), "utf8")) as { states: [string, Record][] }; + const row = snapshot.states.find(([id]) => id === "resp_during_flush")?.[1]; + expect(row).toMatchObject({ kind: "spill" }); + expect(row?.items).toBeUndefined(); + }); + + test("small entries retain the legacy v2 debounced snapshot representation", async () => { + rememberLarge("resp_legacy_small", "small"); + await flushResponseState(); + const snapshot = JSON.parse(readFileSync(join(home, "responses-state.json"), "utf8")) as { version: number; states: [string, Record][] }; + expect(snapshot.version).toBe(2); + const row = snapshot.states.find(([id]) => id === "resp_legacy_small")?.[1]; + expect(row).toMatchObject({ items: expect.any(Array) }); + expect(row?.kind).toBeUndefined(); + expect(row?.sizeBytes).toBeUndefined(); + }); + + test("same-id spill replacement keeps the old row replay-addressable until atomic stub swap", async () => { + setResponseStateByteCapForTests(1_024); + rememberLarge("resp_atomic_replace", "old".repeat(3_000)); + const oldFile = spillFileNames(home)[0]!; + let observedOld = false; + setSpillIoForTest({ + record(event) { + if (event !== "publish") return; + const replay = expandPreviousResponseInput({ previous_response_id: "resp_atomic_replace", input: "delta" }); + observedOld = JSON.stringify(replay).includes("oldoldold"); + expect(existsSync(join(responseSpillDirectory(home), oldFile))).toBe(true); + }, + }); + rememberLarge("resp_atomic_replace", "new".repeat(3_000)); + expect(observedOld).toBe(true); + expect(JSON.stringify(expandPreviousResponseInput({ previous_response_id: "resp_atomic_replace", input: "delta" }))) + .toContain("newnewnew"); + await flushResponseState(); + expect(existsSync(join(responseSpillDirectory(home), oldFile))).toBe(false); + }); + + test("existing spill replacement stays a direct stub-to-stub swap even when the candidate fits RAM", async () => { + setResponseStateByteCapForTests(1_024); + rememberLarge("resp_direct_b2", "old".repeat(3_000)); + const oldFile = spillFileNames(home)[0]!; + const events: string[] = []; + setSpillIoForTest({ record: event => events.push(event) }); + setResponseStateByteCapForTests(1_000_000); + + rememberLarge("resp_direct_b2", "small replacement"); + + expect(events).toContain("publish"); + expect(events).toContain("stub-swap"); + expect(responseStateMetrics()).toMatchObject({ residentCount: 0, spillStubCount: 1 }); + await flushResponseState(); + expect(existsSync(join(responseSpillDirectory(home), oldFile))).toBe(false); + expect(JSON.stringify(expandPreviousResponseInput({ previous_response_id: "resp_direct_b2", input: "delta" }))) + .toContain("small replacement"); + }); + + test("crash between new-generation publication and stub swap preserves the old row and reclaims the new orphan", async () => { + setResponseStateByteCapForTests(1_024); + rememberLarge("resp_crash_window", "old".repeat(3_000)); + await flushResponseState(); + const oldFile = spillFileNames(home)[0]!; + const orphan = writeResponseSpillDurably("resp_crash_window", { + createdAt: Date.now(), items: ["new".repeat(3_000)], + }); + const oldTime = new Date(Date.now() - 20 * 60_000); + utimesSync(join(responseSpillDirectory(home), orphan.fileName), oldTime, oldTime); + clearResponseStateMemoryForTests(); + setResponseStateByteCapForTests(1_024); + expect(JSON.stringify(expandPreviousResponseInput({ previous_response_id: "resp_crash_window", input: "delta" }))) + .toContain("oldoldold"); + expect(existsSync(join(responseSpillDirectory(home), oldFile))).toBe(true); + expect(existsSync(join(responseSpillDirectory(home), orphan.fileName))).toBe(false); + }); + + test("same response id and equal-size replacement use distinct basenames and unlink old only after stub swap", async () => { + setResponseStateByteCapForTests(1_024); + rememberLarge("resp_equal_size", "a".repeat(8_000)); + const old = spillFileNames(home)[0]!; + let published: string[] = []; + setSpillIoForTest({ record: event => { if (event === "publish") published = spillFileNames(home); } }); + rememberLarge("resp_equal_size", "b".repeat(8_000)); + expect(published).toHaveLength(2); + expect(new Set(published).size).toBe(2); + await flushResponseState(); + expect(spillFileNames(home)).toHaveLength(1); + expect(existsSync(join(responseSpillDirectory(home), old))).toBe(false); + }); + + test("identical-content re-spill still allocates a generation-unique basename", () => { + setResponseStateByteCapForTests(1_024); + rememberLarge("resp_identical", "i".repeat(8_000)); + const old = spillFileNames(home)[0]!; + let duringPublish: string[] = []; + setSpillIoForTest({ record: event => { if (event === "publish") duringPublish = spillFileNames(home); } }); + rememberLarge("resp_identical", "i".repeat(8_000)); + expect(duringPublish).toHaveLength(2); + expect(duringPublish.some(name => name !== old)).toBe(true); + }); + + test("spill-failed tombstone survives simulated process restart and returns the canonical failure", async () => { + setSpillIoForTest({ write: () => { throw new Error("fail"); } }); + setResponseStateByteCapForTests(1_024); + rememberLarge("resp_failed_restart", "f".repeat(8_000)); + setSpillIoForTest(null); + await flushResponseState(); + clearResponseStateMemoryForTests(); + const body = { previous_response_id: "resp_failed_restart", input: "delta" }; + expect(expandPreviousResponseInput(body)).toBe(body); + expect(previousResponseReplayFailure(body)).toEqual({ + code: "previous_response_not_found", reason: "spill_failed", + }); + }); + + test("spill replay preserves replay-prefix provenance and compaction-marker acknowledgement", () => { + setResponseStateByteCapForTests(1_024); + rememberResponseState( + { input: [{ type: "context_compaction" }, { role: "user", content: "x".repeat(8_000) }] }, + fixedResponse("resp_spill_provenance", [{ role: "assistant", content: "done" }]), + ); + const expanded = expandPreviousResponseInput({ + model: "test/model", previous_response_id: "resp_spill_provenance", input: "next", + }); + expect(previousResponseReplayPrefixLength(expanded)).toBe(3); + expect(parseRequest(expanded)._contextCompactionBoundary).toBeUndefined(); + }); + + test("multibyte resident payload accounting uses complete UTF-8 bytes including provider metadata", () => { + const realNow = Date.now; + const at = 1_900_000_000_000; + Date.now = () => at; + try { + const output = [{ role: "assistant", content: "🙂" }]; + const providers = { kiro: { conversationId: "대화🙂" } }; + rememberResponseState( + { input: "한글🙂" }, + fixedResponse("resp_다국어", output), + providers, + ); + const items = [{ role: "user", content: "한글🙂" }, ...output]; + const expected = Buffer.byteLength(JSON.stringify({ + responseId: "resp_다국어", createdAt: at, items, providers, + }), "utf8"); + expect(getStoredResponseBytesForTests()).toBe(expected); + } finally { + Date.now = realNow; + } + }); + + test("serialization failure rejects the candidate and records a bounded tombstone", () => { + const circular: Record = {}; + circular.self = circular; + rememberResponseState({ input: "safe" }, fixedResponse("resp_circular", [circular])); + expect(responseStateMetrics()).toMatchObject({ residentCount: 0, tombstoneCount: 1 }); + const body = { previous_response_id: "resp_circular", input: "delta" }; + expandPreviousResponseInput(body); + expect(previousResponseReplayFailure(body)?.reason).toBe("spill_failed"); + + clearResponseStateForTests(); + setResponseStateByteCapForTests(1_024); + rememberLarge("resp_circular_replace", "old".repeat(3_000)); + const oldFile = spillFileNames(home)[0]!; + rememberResponseState({ input: "replacement" }, fixedResponse("resp_circular_replace", [circular])); + expect(responseStateMetrics()).toMatchObject({ residentCount: 0, spillStubCount: 0, tombstoneCount: 1 }); + expect(existsSync(join(responseSpillDirectory(home), oldFile))).toBe(false); + }); + + test("serialization-failure tombstone still submits to the hard cap (C2-1)", () => { + // Cap below tombstone size: the tombstone must be pruned away, not stay + // resident above the cap because the failure path skipped pruning. + setResponseStateByteCapForTests(1); + const circular: Record = {}; + circular.self = circular; + rememberResponseState({ input: "safe" }, fixedResponse("resp_tiny_cap_circular", [circular])); + const metrics = responseStateMetrics(); + expect(metrics.totalBytes).toBeLessThanOrEqual(1); + expect(metrics.tombstoneCount).toBe(0); + }); + + test("pending spill-unlink deferral is bounded at 128 superseded generations (C2-2)", () => { + // Persistently failing snapshot writes mean persistNow() never drains the + // deferral queue. 140 same-id replacements must leave at most current + + // 128 pending files on disk — the overflow unlinks oldest-first instead + // of growing without limit. + setSpillIoForTest({ write: undefined }); // default write; only block snapshots below + setResponseStateByteCapForTests(1_024); + const dir = responseSpillDirectory(home); + for (let i = 0; i < 140; i++) { + rememberLarge("resp_unlink_bound", `payload-${i}-${"x".repeat(4_000)}`); + } + const files = spillFileNames(home); + // 1 current generation + at most 128 deferred superseded generations. + expect(files.length).toBeLessThanOrEqual(129); + expect(files.length).toBeGreaterThan(1); // deferral is real, not immediate unlink + void dir; + }); + + test("write fsync or publication failure cleans its temp and preserves no unmeasured resident payload", () => { + const failures = [ + { write: () => { throw new Error("write"); } }, + { fsync: () => { throw new Error("fsync"); } }, + { link: () => { throw Object.assign(new Error("publish"), { code: "EIO" }); } }, + ]; + for (const [index, io] of failures.entries()) { + clearResponseStateForTests(); + setSpillIoForTest(io); + setResponseStateByteCapForTests(1_024); + rememberLarge(`resp_io_failure_${index}`, "x".repeat(8_000)); + expect(responseStateMetrics()).toMatchObject({ residentCount: 0, tombstoneCount: 1, spillWriteFailures: 1 }); + const dir = responseSpillDirectory(home); + expect(existsSync(dir) ? readdirSync(dir) : []).toEqual([]); + } + }); + + test("EEXIST publication loss advances the generation and retries within the bound", () => { + let links = 0; + setSpillIoForTest({ + link(temp, destination) { + links += 1; + if (links === 1) throw Object.assign(new Error("lost generation"), { code: "EEXIST" }); + linkSync(temp, destination); + }, + }); + setResponseStateByteCapForTests(1_024); + rememberLarge("resp_eexist", "e".repeat(8_000)); + expect(links).toBe(2); + expect(responseStateMetrics()).toMatchObject({ spillStubCount: 1, spillWriteFailures: 0 }); + }); + + test("orphan cleanup obeys scan and cleanup caps rejects symlinks and counts failed unlink", () => { + const dir = responseSpillDirectory(home); + mkdirSync(dir, { recursive: true }); + const old = new Date(Date.now() - 20 * 60_000); + const target = join(dir, "target.txt"); + writeFileSync(target, "target"); + const symlinkRef = writeResponseSpillDurably("symlink", { createdAt: Date.now(), items: ["target"] }); + const symlinkName = symlinkRef.fileName; + unlinkSync(join(dir, symlinkName)); + symlinkSync(target, join(dir, symlinkName)); + for (let i = 0; i < 520; i++) { + const ref = writeResponseSpillDurably(`orphan-${i}`, { createdAt: Date.now(), items: [i] }); + const path = join(dir, ref.fileName); + utimesSync(path, old, old); + } + let failedOnce = false; + setSpillIoForTest({ unlink(path) { + if (!failedOnce) { failedOnce = true; throw new Error("locked"); } + unlinkSync(path); + } }); + const result = recoverOrphanedResponseSpills(new Set(), dir); + // 521 orphan candidates exist (520 aged + 1 symlink); the CLEANUP cap + // (512) must bind strictly below the candidate count, proving the cap is + // live rather than vacuously larger than the workload (review C1-3). + expect(result.removed + result.failed).toBe(512); + expect(result.scanned).toBeLessThanOrEqual(4_096); + expect(result.failed).toBe(1); + expect(existsSync(join(dir, symlinkName))).toBe(true); + }); + + test("orphan scan cap stops enumeration at the limit with an injected directory", () => { + // Prove RESPONSE_SPILL_SCAN_MAX itself binds: enumerate more entries than + // the cap via the injected readdir seam and assert the scan stops exactly + // at the limit without touching the excess (review C1-3 — a sub-cap + // workload cannot distinguish capped from uncapped scanning). + const dir = responseSpillDirectory(home); + mkdirSync(dir, { recursive: true }); + const old = new Date(Date.now() - 20 * 60_000); + const seed = writeResponseSpillDurably("scan-seed", { createdAt: Date.now(), items: ["s"] }); + utimesSync(join(dir, seed.fileName), old, old); + let served = 0; + setSpillIoForTest({ + readdirEntry() { + served += 1; + // Serve the one real file first, then virtual entries that are + // filtered by the owned-file regex (name mismatch) — enumeration + // cost still counts toward the scan cap. + return served === 1 ? seed.fileName : `not-owned-${served}.txt`; + }, + }); + const result = recoverOrphanedResponseSpills(new Set(), dir); + expect(result.scanned).toBe(4_096); + expect(served).toBe(4_096); + }); + + test("response-state management metrics keep every added field finite scalar and privacy-safe", () => { + setResponseStateByteCapForTests(1_024); + rememberLarge("resp_private_metric_id", "secret-content".repeat(1_000)); + const metrics = responseStateMetrics(); + expect(Object.values(metrics).every(value => typeof value === "number" && Number.isFinite(value))).toBe(true); + const serialized = JSON.stringify(metrics); + expect(serialized).not.toContain("resp_private_metric_id"); + expect(serialized).not.toContain("secret-content"); + expect(serialized).not.toContain(responseSpillDirectory(home)); + }); + + test("byte cap spills over-cap rows instead of evicting prior continuation ids", () => { setResponseStateByteCapForTests(4_000); try { const bulk = "x".repeat(1_500); @@ -543,23 +1051,26 @@ describe("Responses previous_response_id state", () => { { type: "text_delta", text: "ok" }, { type: "done" }, ], "cursor/grok-4.5"); + json.id = `resp_cap_${i}`; rememberResponseState(body, json, { cursor: { conversationId: `conv_${i}` } }, { force: true }); ids.push(json.id as string); } - // Oldest entries were evicted by the byte high-water; the newest survives. - expect(previousResponseProviderState(ids[0])).toBeUndefined(); + expect(previousResponseProviderState(ids[0])?.cursor?.conversationId).toBe("conv_0"); expect(previousResponseProviderState(ids[3])?.cursor?.conversationId).toBe("conv_3"); + expect(responseStateMetrics().spillStubCount).toBeGreaterThan(0); + expect(getStoredResponseBytesForTests()).toBeLessThanOrEqual(4_000); } finally { setResponseStateByteCapForTests(null); } }); - test("byte accounting survives restart (sizes recomputed on load)", async () => { + test("byte accounting across restart preserves spilled ids and recomputes resident and stub bytes", async () => { setResponseStateByteCapForTests(4_000); try { const bulk = "y".repeat(1_500); const bodyA = { model: "cursor/grok-4.5", input: `${bulk}-a`, store: false }; const jsonA = buildResponseJSON([{ type: "text_delta", text: "ok" }, { type: "done" }], "cursor/grok-4.5"); + jsonA.id = "resp_restart_a"; rememberResponseState(bodyA, jsonA, { cursor: { conversationId: "conv_a" } }, { force: true }); await flushResponseState(); @@ -570,13 +1081,16 @@ describe("Responses previous_response_id state", () => { // Post-restart stores still enforce the cap against recomputed sizes. const bodyB = { model: "cursor/grok-4.5", input: `${bulk}-b1`, store: false }; const jsonB = buildResponseJSON([{ type: "text_delta", text: "ok" }, { type: "done" }], "cursor/grok-4.5"); + jsonB.id = "resp_restart_b"; rememberResponseState(bodyB, jsonB, { cursor: { conversationId: "conv_b1" } }, { force: true }); const bodyC = { model: "cursor/grok-4.5", input: `${bulk}-b2`, store: false }; const jsonC = buildResponseJSON([{ type: "text_delta", text: "ok" }, { type: "done" }], "cursor/grok-4.5"); + jsonC.id = "resp_restart_c"; rememberResponseState(bodyC, jsonC, { cursor: { conversationId: "conv_b2" } }, { force: true }); - expect(previousResponseProviderState(jsonA.id as string)).toBeUndefined(); + expect(previousResponseProviderState(jsonA.id as string)?.cursor?.conversationId).toBe("conv_a"); expect(previousResponseProviderState(jsonC.id as string)?.cursor?.conversationId).toBe("conv_b2"); + expect(getStoredResponseBytesForTests()).toBeLessThanOrEqual(4_000); } finally { setResponseStateByteCapForTests(null); } @@ -619,7 +1133,8 @@ describe("Responses previous_response_id state", () => { // entry the same size, so leaked decrements would show up as >1000x. const total = getStoredResponseBytesForTests(); expect(perEntryBytes).toBeGreaterThan(2_000); - expect(total).toBeLessThanOrEqual(perEntryBytes * 1_000); + expect(responseStateMetrics().count).toBe(1_000); + expect(total).toBeLessThanOrEqual((perEntryBytes + 4) * 1_000); expect(total).toBeGreaterThanOrEqual(perEntryBytes * 999); } finally { setResponseStateByteCapForTests(null); @@ -853,7 +1368,9 @@ describe("Responses previous_response_id state", () => { expect(expanded.input).toHaveLength(3); }); - test("oversized entries stay in memory but are skipped on disk", async () => { + test("oversized entries replay from dedicated spill across restart while small entries use snapshot", async () => { + setResponseStateByteCapForTests(128 * 1024); + try { const big = "x".repeat(3 * 1024 * 1024); // > 2MiB per-entry cap const first = buildResponseJSON([ { type: "text_delta", text: big }, @@ -868,18 +1385,22 @@ describe("Responses previous_response_id state", () => { rememberResponseState({ model: "gpt-5.5", input: "small turn" }, small); await flushResponseState(); - // In-memory: both expand. + // In-memory: both expand, with the oversized entry represented by a stub. expect((expandPreviousResponseInput({ model: "gpt-5.5", previous_response_id: first.id, input: "n", }) as { input: unknown[] }).input).toHaveLength(3); - // After restart: only the small entry survived on disk. + expect(responseStateMetrics().spillStubCount).toBe(1); clearResponseStateMemoryForTests(); - const bigMiss = { model: "gpt-5.5", previous_response_id: first.id, input: "n" }; - expect(expandPreviousResponseInput(bigMiss)).toEqual(bigMiss); + expect((expandPreviousResponseInput({ + model: "gpt-5.5", previous_response_id: first.id, input: "n", + }) as { input: unknown[] }).input).toHaveLength(3); expect((expandPreviousResponseInput({ model: "gpt-5.5", previous_response_id: small.id, input: "n", }) as { input: unknown[] }).input).toHaveLength(3); + } finally { + setResponseStateByteCapForTests(null); + } }); test("stores provider conversation id alongside Responses output state", () => { @@ -957,9 +1478,21 @@ describe("Responses previous_response_id state", () => { }); describe("responseStateMetrics (observe-only snapshot)", () => { - test("empty store reports all-zeroed metrics", () => { + test("empty store reports every resident spill tombstone and counter metric as zero", () => { const metrics = responseStateMetrics(); - expect(metrics).toEqual({ count: 0, totalBytes: 0, largestBytes: 0, oldestAgeMs: 0 }); + expect(metrics).toEqual({ + count: 0, + residentCount: 0, + spillStubCount: 0, + tombstoneCount: 0, + totalBytes: 0, + spillPayloadBytes: 0, + largestBytes: 0, + oldestAgeMs: 0, + spillWrites: 0, + spillWriteFailures: 0, + spillReadFailures: 0, + }); }); test("largest entry over 200KB is reflected, and total is >= largest", () => { @@ -996,7 +1529,7 @@ describe("Responses previous_response_id state", () => { } }); - test("is side-effect free: it never lazy-loads the disk snapshot, prunes, or evicts", async () => { + test("metrics remain side-effect free with the complete additive metric shape", async () => { const first = buildResponseJSON([{ type: "text_delta", text: "persisted" }, { type: "done" }], "gpt-5.5"); rememberResponseState({ model: "gpt-5.5", input: "persisted turn" }, first); await flushResponseState(); @@ -1004,7 +1537,19 @@ describe("Responses previous_response_id state", () => { clearResponseStateMemoryForTests(); // A probe must NOT trigger the lazy disk load — the store still reads empty. - expect(responseStateMetrics()).toEqual({ count: 0, totalBytes: 0, largestBytes: 0, oldestAgeMs: 0 }); + expect(responseStateMetrics()).toEqual({ + count: 0, + residentCount: 0, + spillStubCount: 0, + tombstoneCount: 0, + totalBytes: 0, + spillPayloadBytes: 0, + largestBytes: 0, + oldestAgeMs: 0, + spillWrites: 0, + spillWriteFailures: 0, + spillReadFailures: 0, + }); // The real request path DOES load; the probe then reflects the loaded entry. expandPreviousResponseInput({ model: "gpt-5.5", previous_response_id: first.id, input: "next" }); From 3bffd92d143bb8e376798d90d8d3f418f5eb7794 Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sat, 1 Aug 2026 09:39:53 +0900 Subject: [PATCH 06/30] docs(plan): pin blob liveness to the request and make admission atomic wp3 A-gate: request-scope pins own in-flight blob liveness (provenance only orders post-request eviction), the admission ladder precomputes victims against both limits and commits once, the SetBlobResult typed error replaces a false schema premise, all four stores get exact 040 snapshot/eviction exports, and the unreachable vision oversized test becomes a configurable-limit seam. --- .../006_roadmap_audit_synthesis.md | 19 ++ .../020_blob_and_replay_caps.md | 247 +++++++++++++++--- 2 files changed, 226 insertions(+), 40 deletions(-) diff --git a/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md b/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md index d6c043479..e8ba3a0a0 100644 --- a/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md +++ b/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md @@ -54,3 +54,22 @@ amended accordingly (failure ladder + admission ladder). | # | Blockers | Decision | Action | |---|---|---|---| | B1-B5 | Spill read lacked a trusted expected id; same-id/equal-size replacement collided and relied on non-portable rename-over-existing; automatic client fallback was unproven; resident measurement was incomplete/weightless on serialization failure; the 3 MiB restart fixture did not cross the 64 MiB spill threshold | ACCEPT | 010 now passes `responseId` into spill reads; uses payload-digest plus generation-distinct basenames and post-swap old-file unlink; defines caller-driven terminal structured 400 recovery; measures the complete retained payload in UTF-8 and tombstones serialization failures; lowers the test cap beneath spill fixtures; records all five current contract-test redefinitions, consumer seams, expanded regression classes, and process-crash-only durability. 005 receives the matching one-sentence recovery correction. | + +## wp3 A-gate (Carson) + +| # | Blocker | Decision | Action | +|---|---|---|---| +| B1 | Local provenance identified origin but not request-lifetime eviction safety; one request could advertise blobs evicted during its own construction | ACCEPT | 020 adds sealed per-request scope tokens carried into each live stream. Only selected external candidates enter the store; every advertised local blob is pinned through construction and hydration. Stream-scoped successful gets release only that request's key pin, and terminal cleanup releases leftovers exactly once. Provenance applies only after request pins clear. Infeasible construction throws a structured capacity error before any request or unstored hash reaches the wire. | +| B2 | The admission ladder mutated TTL/local victims before a later pinned-saturation rejection that claimed the store was unchanged | ACCEPT | 020 now requires two-phase admission: logically precompute eligible TTL removals, replacement, and oldest-local victims against byte and count limits; prove feasibility; then commit once. Any rejection preserves the complete map, bytes, recency, pins, counters, unrelated victims, and same-key predecessor byte-for-byte. | +| B3 | 020 incorrectly claimed `SetBlobResult` had no typed rejection field and underspecified the get-miss wire shape | ACCEPT | The plan records generated `SetBlobResult.error`/`Error.message`, returns that typed error for rejected remote sets, and locks get miss to the current request-id-preserving `KvClientMessage`/`getBlobResult` with optional `blobData` omitted. Hit, miss, and rejected-set-followed-by-miss wire tests are explicit. | +| B4 | 020 did not deliver executable 040 snapshot/eviction registrations for Antigravity, vision, or image normalization | ACCEPT | Exact owner exports are now named for all four stores. Every snapshot returns count, bytes, evictable bytes, pinned bytes, and oldest timestamp; every budget eviction returns exact released bytes and removes one complete owner-defined oldest row. | +| B5 | The vision single-entry-over-1-MiB test was unreachable after the 2,000-character clamp | ACCEPT | 020 adds a production-safe test-only description-cache limit override, restores production limits on reset, and replaces the fixture with small-limit single-entry non-retention plus exact multi-entry aggregate-boundary/oldest-eviction coverage. Existing Cursor, Antigravity, vision, and image-cache behavior tests and reset consumers are recorded for preservation or explicit redefinition. | + +## wp3 A-gate round 2 (Avicenna, FAIL 4) — adjudication + +| # | Blocker | Decision | Action | +|---|---|---|---| +| AV-1 | 040 snapshot/eviction exports defined only for Cursor; Antigravity/vision/image had metrics-only or unnamed hooks; oldestAt ambiguous | ACCEPT | 020 now names owner exports for ALL FOUR stores (cursorBlobRetainedStoreSnapshot/evictOldestCursorBlobForBudget, antigravityReplayRetainedStoreSnapshot/evictOldest…, vision + image equivalents) each returning count/bytes/evictableBytes/pinnedBytes/oldestAt with exact released-bytes eviction; Cursor oldestAt = oldest EVICTABLE local timestamp. | +| AV-2 | Same-key cross-provenance replacement could downgrade a live remote pin to evictable local | ACCEPT | Admission step 1 now merges provenance to the STRONGER class (remote wins; TTL clock kept on remote→local refresh, upgrade on local→remote); remote→local and local→remote regression tests named. | +| AV-3 | Oversized-vision-entry test unreachable (2000-char clamp < 1 MiB cap) | ACCEPT | setVisionDescriptionCacheLimitsForTests() override (production limits restored on undefined) + small-limit fixtures replace the unreachable case. | +| AV-4 | Vision clamp snippet left the first-use outcome unclamped, contradicting the byte-identical contract | ACCEPT | Snippet extended: successful outcome is replaced with the clamped text BEFORE resolveOutcome; regression asserts cache value and first-use outcome identical, error outcomes untouched. | diff --git a/devlog/_plan/260801_zero_leak_state_stores/020_blob_and_replay_caps.md b/devlog/_plan/260801_zero_leak_state_stores/020_blob_and_replay_caps.md index ec6d3658e..5257856a1 100644 --- a/devlog/_plan/260801_zero_leak_state_stores/020_blob_and_replay_caps.md +++ b/devlog/_plan/260801_zero_leak_state_stores/020_blob_and_replay_caps.md @@ -9,8 +9,9 @@ Binding inputs: `000_state_store_inventory.md` §§2–3 and “Additional proce Bound all four translation-duty stores owned by this phase at insertion time: -1. Cursor shared blobs: per-blob + aggregate bytes, provenance-aware eviction, and - pinned-saturation rejection through the existing get-blob miss surface. +1. Cursor shared blobs: per-blob + aggregate bytes, request-lifetime liveness pins, + provenance-aware post-request eviction, and coherent capacity rejection before any + request containing an unstored hash reaches the wire. 2. Antigravity replay: bounded calls and bytes per session while retaining recent live identities and clear-on-invalid behavior. 3. Vision descriptions: clamp before insertion and byte-weight the existing LRU. @@ -23,9 +24,17 @@ Bound all four translation-duty stores owned by this phase at insertion time: lazy TTL/count eviction but no byte or provenance field. `setBlob()` returns void. - `src/adapters/cursor/native-exec.ts:202-227` maps missing `getBlobArgs` to an empty `GetBlobResult`; remote `setBlobArgs` always acknowledges after insertion. +- `src/adapters/cursor/gen/agent_pb.ts:7866-7880` defines the exact get-miss wire shape: + `GetBlobResult.blobData` is optional. `:7903-7917` defines + `SetBlobResult.error`, and `:13245-13258` defines its typed `Error.message`; rejection + is not limited to an empty acknowledgement. - `src/adapters/cursor/protobuf-request.ts:54-60,195-305` limits selected external roots to 192/512 KiB only after candidates have been stored. Inventory warning: all candidates are stored before selection. +- `src/adapters/cursor/protobuf-request.ts:107,352,391,399,430,449,471,484,495` calls + `storeCursorBlob()` repeatedly while constructing one request. Origin provenance is + knowable (`storeCursorBlob` versus remote `setBlobArgs`), but origin alone does not + prove that a blob is safe to evict before that request has hydrated it. - `src/adapters/google-antigravity-replay.ts:13-24` has a bounded outer map and an unbounded inner `Map`. - `src/adapters/google-antigravity-replay.ts:79-98` accumulates every call identity for @@ -43,7 +52,8 @@ of the translation feature. ## Cursor blob-store diff -Modify `src/adapters/cursor/native-exec.ts`: +Modify `src/adapters/cursor/native-exec.ts`, `src/adapters/cursor/protobuf-request.ts`, +and `src/adapters/cursor/live-transport.ts`: ```ts const BLOB_TTL_MS = 15 * 60_000; @@ -52,46 +62,112 @@ const BLOB_MAX_ENTRY_BYTES = 16 * 1024 * 1024; const BLOB_MAX_TOTAL_BYTES = 64 * 1024 * 1024; type CursorBlobProvenance = "local-regenerated" | "remote-setBlobArgs"; +export type CursorBlobRequestScopeToken = symbol; interface CursorBlobEntry { data: Uint8Array; storedAt: number; sizeBytes: number; provenance: CursorBlobProvenance; + requestPins: Set; } type CursorBlobAdmission = | { admitted: true; replaced: boolean } - | { admitted: false; reason: "entry_too_large" | "pinned_saturation" }; + | { admitted: false; reason: "entry_too_large" | "pinned_saturation" | "request_pinned_conflict" }; let blobBytes = 0; -function setBlob(k: string, data: Uint8Array, provenance: CursorBlobProvenance): CursorBlobAdmission; +function setBlob( + k: string, + data: Uint8Array, + provenance: CursorBlobProvenance, + requestScope?: CursorBlobRequestScopeToken, +): CursorBlobAdmission; function deleteBlob(k: string): void; -function evictExpiredBlobs(at: number): void; -function evictOldestLocalBlob(): boolean; +export function createCursorBlobRequestScope(): CursorBlobRequestScopeToken; +export function sealCursorBlobRequestScope(scope: CursorBlobRequestScopeToken): void; +export function releaseCursorBlobRequestScope(scope: CursorBlobRequestScopeToken): void; +export function handleCursorNativeKv( + kvMsg: KvServerMessage, + requestScope?: CursorBlobRequestScopeToken, +): Uint8Array; ``` -Admission ladder for every insert, including replacement: - -1. Reject `data.byteLength > BLOB_MAX_ENTRY_BYTES`; do not remove an existing same-key - row until admission is known to succeed. -2. Sweep all TTL-expired rows, regardless of insertion order/provenance. -3. Compute projected bytes after subtracting a same-key replacement. -4. While projected bytes exceed the aggregate cap, evict the oldest - `local-regenerated` row. A remote row is pinned only while its TTL is live. -5. If still over cap, reject with `pinned_saturation`; the store and byte counter remain - unchanged. -6. On success, delete the old row through `deleteBlob()`, insert the immutable byte view, - add exact `byteLength`, and refresh Map recency. -7. Apply the 4,096 count cap using the same policy: expired, then oldest local. If only - live remote rows remain, reject rather than evict a pin or exceed the cap. - -`storeCursorBlob(data)` remains `Uint8Array -> Uint8Array`: it computes and returns the -SHA-256 id even if admission rejects. A later `getBlobArgs` receives the existing empty -result—the explicit protocol miss surface. It passes `local-regenerated`. - -`setBlobArgs` passes `remote-setBlobArgs`. `SetBlobResult` has no typed rejection field, -so it still acknowledges transport receipt; a rejected hash is intentionally absent and -subsequent `getBlobArgs` returns the explicit miss. Add one privacy-safe diagnostic -counter, not the hash or blob bytes. +`prepareCursorRunRequest()` owns one request-scope token and passes it through every +root/turn/step helper into `storeCursorBlob(data, scope)`. Each stored or reused blob adds +that scope to `requestPins`; sealing after construction fixes the complete set of distinct +blob keys advertised by the request. `PreparedCursorRunRequest` carries the sealed token +beside `bytes`, and `CursorLiveRun` passes that stream's token into +`handleCursorNativeKv()`. This is necessary because blob id alone cannot distinguish two +concurrent requests that reference identical content. A successful `getBlobArgs` removes +only that stream token's pin for the hydrated key. When all sealed keys have hydrated, +the scope releases; the stream's single terminal cleanup releases any remainder exactly +once on close/error/abort. A request pin +always wins over provenance and TTL: an in-flight request's blob is not an eviction +candidate. Provenance controls eviction only after request pins are gone: + +- unpinned `local-regenerated` rows are LRU-evictable; +- unpinned live `remote-setBlobArgs` rows stay provenance-pinned until TTL expiry; +- expired remote rows become removal candidates once no request scope pins them. + +External-root selection moves before global insertion. Candidate construction retains +serialized bytes locally; after the existing 192-root/512-KiB policy selects the final +roots, only those selected bytes call `storeCursorBlob(data, scope)`. Unadvertised +candidates never consume store capacity or acquire pins. Native turns/steps are all +advertised and store directly with the same scope. This changes construction order, not +the selected-root policy or blob-id hash format. + +Admission is a synchronous two-phase transaction for every insert or replacement: + +1. Validate the per-entry cap and same-key predecessor. A differing replacement cannot + displace a request-pinned predecessor; reject `request_pinned_conflict` without mutation. + Same-key CROSS-PROVENANCE refresh never downgrades protection (A-gate blocker: + naive replacement would turn a live remote pin into an evictable local row): + the surviving row's provenance is the STRONGER of the two — a live + `remote-setBlobArgs` row refreshed by a local `storeCursorBlob()` for the same + content-addressed key keeps `remote-setBlobArgs` provenance and its TTL clock; + a local row refreshed by a remote set upgrades to `remote-setBlobArgs`. + (Content-addressed keys make the bytes identical by construction, so only the + protection class is merged.) Regression tests: remote→local refresh stays + pinned within TTL; local→remote refresh upgrades and survives local LRU. +2. Build a logical candidate view containing all TTL-expired, unpinned rows. Do not delete + them yet. Request-pinned rows remain live regardless of age. +3. Compute projected bytes and count after the eligible same-key replacement and logical + TTL removals. De-duplicate the same key across replacement/TTL/victim sets so its bytes + and count are subtracted exactly once. +4. Select additional oldest unpinned `local-regenerated` victims until **both** + `BLOB_MAX_TOTAL_BYTES` and `BLOB_MAX_ENTRIES` would fit. Live remote rows and all + request-pinned rows are ineligible. +5. If either limit still cannot fit, reject with `pinned_saturation`. The map, byte + counter, recency, request pins, and same-key predecessor remain byte-for-byte unchanged; + logically selected TTL/local victims are not committed. +6. Once feasibility is proven, commit the complete removal set, eligible replacement, + immutable-byte insertion/reuse, request-pin attachment, recency update, and exact + counters in one non-throwing synchronous section. There is no observable intermediate + over-cap state and no post-insert rejection path. + +`storeCursorBlob(data, scope)` computes the SHA-256 id and passes +`local-regenerated`, but returns the id only after the blob is stored/reused and pinned by +that scope. Infeasible admission throws a typed, privacy-safe +`CursorBlobAdmissionError` with stable code `cursor_blob_capacity`; request preparation +releases the partial scope and fails before `live-transport.ts` writes an +`AgentRunRequest`. The adapter surfaces the ordinary structured provider error. It never +returns an unstored hash and never sends a partially constructed request. + +`PreparedCursorRunRequest` adds `blobRequestScope: CursorBlobRequestScopeToken` so the +production transport owns terminal cleanup. The byte-only `encodeCursorRunRequest()` +compatibility helper remains test-only in current consumers: its scope self-releases after +all advertised distinct keys are hydrated, and deterministic test reset releases any +scope left by a test that intentionally does not hydrate every id. + +`setBlobArgs` passes `remote-setBlobArgs`. On rejection it returns a `KvClientMessage` +with the original request id, `message.case = "setBlobResult"`, and the optional typed +`SetBlobResult.error` populated with a bounded privacy-safe capacity message. It does not +acknowledge success for an absent blob and does not log the hash or bytes. + +The existing get-miss contract remains distinct: `getBlobArgs` always returns a +`KvClientMessage` preserving the request id with `message.case = "getBlobResult"`; when +the key is absent, optional `GetBlobResult.blobData` is omitted. Evicted/expired hashes +use that exact shape. A successful get includes `blobData` and advances request-scope +hydration accounting. Expose accounting for 040: @@ -106,11 +182,17 @@ export interface CursorBlobMetrics { oldestAt: number | null; } export function cursorBlobMetrics(): CursorBlobMetrics; +export function cursorBlobRetainedStoreSnapshot(): { + count: number; bytes: number; evictableBytes: number; pinnedBytes: number; oldestAt: number | null; +}; export function evictOldestCursorBlobForBudget(): number; // local only; bytes released ``` -Metrics read cached fields only. Add test-only reset/cap overrides; production constants -remain fixed. +Snapshot/metrics read cached fields only. For the 040 registration, `pinnedBytes` includes +live remote-provenance rows and every request-pinned row (without double counting), while +`evictableBytes` includes only currently unpinned local rows. Budget eviction removes the +oldest unpinned local row and returns exact released bytes. Add test-only reset/cap +overrides; production constants remain fixed. ## Antigravity replay diff @@ -148,8 +230,19 @@ Expose scalar accounting/test seams: export function antigravityReplayMetrics(): { sessions: number; calls: number; totalBytes: number; largestSessionBytes: number; }; +export function antigravityReplayRetainedStoreSnapshot(): { + count: number; bytes: number; evictableBytes: number; pinnedBytes: number; oldestAt: number | null; +}; +export function evictOldestAntigravityReplayForBudget(): number; ``` +For 040, one replay session is one evictable retained row: the snapshot reports all replay +bytes as evictable and zero pinned bytes, `oldestAt` is the oldest session/call touch, and +budget eviction removes the oldest complete session through the same centralized +subtract helper. It never partially clears a session. The existing invalid-signature path +at `src/adapters/google.ts:451-455` still calls `clearAntigravityReplay()` and immediately +removes the complete bounded session with exact byte subtraction. + ## Vision description-cache diff Modify `src/vision/index.ts`: @@ -162,23 +255,51 @@ interface VisionDescriptionCache { get(key: string): string | undefined; set(key: string, value: string): void; clear(): void; - metrics?(): { count: number; totalBytes: number; oldestAt: number | null }; + snapshot?(): { count: number; bytes: number; oldestAt: number | null }; + evictOldest?(): number; } + +export function visionDescriptionRetainedStoreSnapshot(): { + count: number; bytes: number; evictableBytes: number; pinnedBytes: number; oldestAt: number | null; +}; +export function evictOldestVisionDescriptionForBudget(): number; +export function setVisionDescriptionCacheLimitsForTests( + limits?: { maxEntries?: number; maxBytes?: number }, +): void; ``` `BoundedLruDescriptionCache` stores `{value,sizeBytes,storedAt}` and tracks key UTF-8 bytes plus value UTF-8 bytes. Replacement subtracts first; eviction happens before insert until count and projected bytes fit. A single value that cannot fit is not cached. +The owner-level 040 snapshot delegates to the production cache's cached accounting and +reports all bytes evictable and zero pinned bytes; owner-level eviction removes its oldest +complete LRU row and returns exact released bytes. An injected custom test cache that +does not provide the optional methods reports an empty owner snapshot and zero release, +so observation never mutates or guesses external state. + +`setVisionDescriptionCacheLimitsForTests()` rebuilds an empty default cache with bounded +test limits; `undefined` restores the production 256-entry/1 MiB limits. It follows the +continuation store's cap-override pattern and permits boundary tests without allocating +MiB-scale fixtures. + At `src/vision/index.ts:341-343`, change the insertion value to: ```ts const successfulText = outcome.error ? "" : clamp(outcome.text.trim(), DESC_MAX_CHARS); if (identity.persistent && successfulText) descriptionCache.set(identity.key, successfulText); +// The outcome handed to resolveOutcome must carry the SAME clamped text — +// building successfulText alone leaves the first-use outcome unclamped +// (A-gate blocker 4). Replace the successful outcome before resolution: +const resolvedOutcome = outcome.error ? outcome : { ...outcome, text: successfulText }; +resolveOutcome(resolvedOutcome); ``` Return the same clamped text in `outcome` so first use and cache hit are byte-identical. Do not clamp error markers or change paid-sidecar admission/concurrency. +The regression `clamps a successful description before cache insertion and first render` +must assert BOTH surfaces: the cached value and the first-use resolved outcome are the +identical clamped string; an error outcome passes through resolveOutcome unmodified. ## Image-normalization cache diff @@ -201,6 +322,20 @@ Extend `getNormalizeStatsForTests()` and the 040 hook with `sentinelEntries`, `metadataBytes`, and `oldestAt`. Budget eviction removes the oldest row through the same centralized subtract helper. +Expose the 040 owner contract explicitly: + +```ts +export function anthropicImageNormalizeRetainedStoreSnapshot(): { + count: number; bytes: number; evictableBytes: number; pinnedBytes: number; oldestAt: number | null; +}; +export function evictOldestAnthropicImageNormalizeForBudget(): number; +``` + +All image-normalization rows, including `pass`/`miss`, are evictable and none are pinned; +the snapshot therefore reports `evictableBytes === bytes` and `pinnedBytes === 0`. +Eviction removes the oldest complete LRU row and returns its full key/value/metadata byte +weight. + ## Cap rationale - Cursor 16 MiB per blob is 32 times the external selected-root budget and still admits @@ -226,15 +361,29 @@ centralized subtract helper. `tests/cursor-blob.test.ts`: - `admits a local blob exactly at the per-blob byte boundary` -- `rejects a local blob one byte above the per-blob boundary and getBlob returns miss` +- `request construction one byte above the per-blob boundary fails before writing a request and returns no unstored hash` +- `request-scope pins preserve every advertised root turn and step until each distinct getBlob hydration completes` +- `two concurrent streams sharing one blob id release only their own request-scope pin on getBlob` +- `stream close error and abort release every remaining request-scope pin` +- `external root pruning stores and pins only selected candidates and cannot fail from discarded history bytes` - `replacement subtracts old bytes and refreshes local LRU` - `aggregate admission evicts oldest local-regenerated blobs first` - `remote setBlobArgs remains pinned within TTL while local blobs are evicted` - `expired remote setBlobArgs becomes evictable before aggregate admission` -- `pinned saturation rejects a new remote blob without exceeding aggregate bytes` +- `pinned saturation returns typed SetBlobResult.error without exceeding aggregate bytes` +- `getBlob miss preserves the request id and emits getBlobResult with blobData omitted` +- `getBlob hit preserves the request id includes blobData and releases that key's request pin` +- `pinned-saturation get after rejected set uses the same omitted-blobData miss shape` - `rejected same-key replacement preserves the previously admitted blob` +- `atomic pinned-saturation rejection preserves unrelated TTL candidates local victims recency pins counters and same-key predecessor byte-for-byte` +- `one request whose construction crosses the aggregate cap fails coherently instead of emitting IDs evicted earlier in that request` - `blob metrics remain observe-only and exact after reset replacement and eviction`. +Preserve/redefine the existing `tests/cursor-blob.test.ts` contract that every blob id +emitted by `encodeCursorRunRequest()` is immediately hydratable by the current +`blobData()` helper. Add deterministic reset/cap setup so the process-global store cannot +leak state between old handshake/root/turn tests and the new saturation tests. + `tests/google-antigravity-replay.test.ts`: - `preserves 20+ live signed calls below count and byte caps` @@ -242,21 +391,37 @@ centralized subtract helper. - `evicts oldest inner calls to satisfy aggregate session bytes` - `does not cache one oversized signature` - `apply refreshes matched call recency without extending session TTL` -- `clear-on-invalid drops the bounded session and all byte accounting`. +- `clear-on-invalid drops the bounded session and all byte accounting` +- `040 snapshot is observe-only and oldest-session eviction returns exact released bytes`. + +Retain the existing canonical nested-argument identity, order independence, nested +signature alias, outgoing-signature no-clobber, Claude bypass, sequential multi-call +retention, and direct clear regressions in `tests/google-antigravity-replay.test.ts`. `tests/vision-cache.test.ts`: - `clamps a successful description before cache insertion and first render` - `cache hit returns the same clamped description without a sidecar call` -- `vision LRU evicts before insert at the aggregate byte boundary` -- `one oversized cache value is observed but not retained`. +- `test-only limits make a successful clamped value larger than maxBytes observable but not retained` +- `multiple entries fit exactly at the aggregate byte boundary and the next byte evicts the oldest before insert` +- `040 snapshot is observe-only and oldest-entry eviction returns exact released bytes`. + +Retain the existing single-flight/later-turn cache hit, failed/empty non-caching, +hit/miss/over-cap message ordering, cache-key backend/model/detail/context partitioning, +and explicit-zero per-turn-cap tests in `tests/vision-cache.test.ts`. `tests/anthropic-image-normalize.test.ts`: - `unique pass and miss sentinels consume metadata bytes and hit the count cap` - `encoded replacement keeps aggregate accounting exact` - `one encoded value above maxEntrySize is returned but not cached` -- `cache eviction occurs before insertion and never exceeds 64 MiB`. +- `cache eviction occurs before insertion and never exceeds 64 MiB` +- `040 snapshot is observe-only and oldest-row eviction returns full metadata-inclusive released bytes`. + +Retain the existing zero-additional-encode cache-hit and media-type-partition regressions +in `tests/anthropic-image-normalize.test.ts`, plus the reset-dependent retry, Kiro image, +Claude native passthrough, and retry-E2E suites. Sentinel/accounting changes must not +alter image wire bytes, ladder/demotion order, terminal behavior, or retry tightening. Verification: @@ -274,8 +439,10 @@ bun run test ## Explicitly not changed - No Cursor blob-id/hash format, protobuf schema, selected-root 192/512 KiB policy, - hydration lookup, or remote TTL change. + get-miss wire shape, or remote TTL change. Hydration lookup only adds request-pin release + accounting after a successful get. - No eviction of live remote blobs merely to admit another pinned blob. +- No eviction of any blob pinned by an in-flight request, regardless of provenance or TTL. - No Antigravity identity algorithm, canonical JSON format, signature validity threshold, Claude-on-Antigravity behavior, or clear-on-invalid behavior. - No vision sidecar backend/model selection, paid-call concurrency, cache identity, or From 319542c6dd744adc989743f3e6ceb553a8a2009d Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sat, 1 Aug 2026 09:44:25 +0900 Subject: [PATCH 07/30] docs(plan): settle the blob-cap accounting classes across three audit rounds Cross-provenance refresh merges to the stronger class, expired unpinned remote rows join the evictable set, oldestAt names the exact next eviction victim, and the vision clamp now covers the first-use outcome. --- .../006_roadmap_audit_synthesis.md | 2 +- .../020_blob_and_replay_caps.md | 21 +++++++++++++------ .../040_app_bytes_observability.md | 2 +- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md b/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md index e8ba3a0a0..c9cf9a0a5 100644 --- a/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md +++ b/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md @@ -69,7 +69,7 @@ amended accordingly (failure ladder + admission ladder). | # | Blocker | Decision | Action | |---|---|---|---| -| AV-1 | 040 snapshot/eviction exports defined only for Cursor; Antigravity/vision/image had metrics-only or unnamed hooks; oldestAt ambiguous | ACCEPT | 020 now names owner exports for ALL FOUR stores (cursorBlobRetainedStoreSnapshot/evictOldestCursorBlobForBudget, antigravityReplayRetainedStoreSnapshot/evictOldest…, vision + image equivalents) each returning count/bytes/evictableBytes/pinnedBytes/oldestAt with exact released-bytes eviction; Cursor oldestAt = oldest EVICTABLE local timestamp. | +| AV-1 | 040 snapshot/eviction exports defined only for Cursor; Antigravity/vision/image had metrics-only or unnamed hooks; oldestAt ambiguous | ACCEPT | 020 now names owner exports for ALL FOUR stores (cursorBlobRetainedStoreSnapshot/evictOldestCursorBlobForBudget, antigravityReplayRetainedStoreSnapshot/evictOldest…, vision + image equivalents) each returning count/bytes/evictableBytes/pinnedBytes/oldestAt with exact released-bytes eviction; Cursor oldestAt = storedAt of the exact next budget-eviction victim (oldest member of the evictable class incl. expired unpinned remote rows — refined round 4). | | AV-2 | Same-key cross-provenance replacement could downgrade a live remote pin to evictable local | ACCEPT | Admission step 1 now merges provenance to the STRONGER class (remote wins; TTL clock kept on remote→local refresh, upgrade on local→remote); remote→local and local→remote regression tests named. | | AV-3 | Oversized-vision-entry test unreachable (2000-char clamp < 1 MiB cap) | ACCEPT | setVisionDescriptionCacheLimitsForTests() override (production limits restored on undefined) + small-limit fixtures replace the unreachable case. | | AV-4 | Vision clamp snippet left the first-use outcome unclamped, contradicting the byte-identical contract | ACCEPT | Snippet extended: successful outcome is replaced with the clamped text BEFORE resolveOutcome; regression asserts cache value and first-use outcome identical, error outcomes untouched. | diff --git a/devlog/_plan/260801_zero_leak_state_stores/020_blob_and_replay_caps.md b/devlog/_plan/260801_zero_leak_state_stores/020_blob_and_replay_caps.md index 5257856a1..7dfcd69fa 100644 --- a/devlog/_plan/260801_zero_leak_state_stores/020_blob_and_replay_caps.md +++ b/devlog/_plan/260801_zero_leak_state_stores/020_blob_and_replay_caps.md @@ -185,14 +185,23 @@ export function cursorBlobMetrics(): CursorBlobMetrics; export function cursorBlobRetainedStoreSnapshot(): { count: number; bytes: number; evictableBytes: number; pinnedBytes: number; oldestAt: number | null; }; -export function evictOldestCursorBlobForBudget(): number; // local only; bytes released +export function evictOldestCursorBlobForBudget(): number; // oldest evictable row; bytes released ``` -Snapshot/metrics read cached fields only. For the 040 registration, `pinnedBytes` includes -live remote-provenance rows and every request-pinned row (without double counting), while -`evictableBytes` includes only currently unpinned local rows. Budget eviction removes the -oldest unpinned local row and returns exact released bytes. Add test-only reset/cap -overrides; production constants remain fixed. +Snapshot/metrics read cached fields only. For the 040 registration the evictable +class is exactly the set `evictOldestCursorBlobForBudget()` draws from (round-3 +audit): unpinned local rows AND expired unpinned remote rows (an expired remote +row has lost its TTL protection — line "expired remote rows become removal +candidates" — so it is evictable, not pinned). `pinnedBytes` covers live +remote-provenance rows and every request-pinned row (without double counting); +`evictableBytes` = total − pinned. `oldestAt` is the `storedAt` timestamp of the +EXACT row budget eviction would remove next (oldest member of the evictable +class), or null when the evictable class is empty — never a pinned row's +timestamp, so 040's cross-store oldest-first comparison operates on genuinely +reclaimable rows. Budget eviction removes that row and returns exact released +bytes. Add test-only reset/cap overrides; production constants remain fixed. +Regression: an expired unpinned remote row is reported evictable, becomes +`oldestAt`, and is removed by budget eviction before any younger local row. ## Antigravity replay diff diff --git a/devlog/_plan/260801_zero_leak_state_stores/040_app_bytes_observability.md b/devlog/_plan/260801_zero_leak_state_stores/040_app_bytes_observability.md index dbdeaaeab..3ba1c981b 100644 --- a/devlog/_plan/260801_zero_leak_state_stores/040_app_bytes_observability.md +++ b/devlog/_plan/260801_zero_leak_state_stores/040_app_bytes_observability.md @@ -118,7 +118,7 @@ Register hooks delivered by 010/020/035 and existing owners: |---|---|---| | logs | `request_log`, `provider_debug`, `injection_debug`, `claude_debug`, `crash_ring` | Remove oldest complete diagnostic row; never truncate an attempt array during budget enforcement. | | caches | `image_normalize`, `vision_descriptions`, `antigravity_replay`, `model_cache`, `usage_summary` | Remove oldest LRU/session/provider value through owner accounting. Preserve “other” usage totals. | -| blobs | `cursor_blobs` | Remove oldest local-regenerated blob only. Live remote blobs report as pinned. | +| blobs | `cursor_blobs` | Remove the oldest EVICTABLE row (unpinned local, or expired unpinned remote — 020 round-4). Live remote and request-pinned blobs report as pinned. | | continuation | `responses_continuation` | Demote oldest resident row through 010 durable spill. Spill stubs/tombstones are not repeatedly demoted. | The request-log owner (`src/server/request-log.ts:150-154,218-246`) must add per-entry From 034d320b82eeaf858123cf0d74c12d2783385f53 Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sat, 1 Aug 2026 10:02:37 +0900 Subject: [PATCH 08/30] feat(caps): bound the blob, replay, vision, and image caches by bytes Cursor blobs gain per-blob 16 MiB / aggregate 64 MiB caps with two-phase atomic admission, request-scope pins that keep a blob alive from construction through hydration, provenance-aware eviction (remote rows stay pinned within TTL; refresh merges to the stronger class), and a typed capacity error before an unstored hash can reach the wire. Antigravity replay sessions cap inner calls at 256/2 MiB with 64 KiB signatures; vision descriptions clamp before insertion (first use and cache hit byte-identical) under a byte-weighted LRU; image-normalize pass/miss sentinels get key-byte accounting and a count cap. Every store exports snapshot/eviction hooks for the 040 budget. Plan: devlog/_plan/260801_zero_leak_state_stores/020 (5 audit rounds). --- src/adapters/anthropic-image-normalize.ts | 130 +++++- src/adapters/cursor/live-transport.ts | 63 ++- src/adapters/cursor/native-exec.ts | 402 ++++++++++++++++-- src/adapters/cursor/protobuf-request.ts | 110 +++-- src/adapters/google-antigravity-replay.ts | 147 ++++++- src/vision/index.ts | 97 ++++- tests/anthropic-image-normalize.test.ts | 99 ++++- tests/cursor-blob.test.ts | 494 +++++++++++++++++++++- tests/cursor-live-transport.test.ts | 80 ++++ tests/google-antigravity-replay.test.ts | 97 ++++- tests/vision-cache.test.ts | 90 ++++ 11 files changed, 1667 insertions(+), 142 deletions(-) diff --git a/src/adapters/anthropic-image-normalize.ts b/src/adapters/anthropic-image-normalize.ts index 6f8f3f540..4e54ead2e 100644 --- a/src/adapters/anthropic-image-normalize.ts +++ b/src/adapters/anthropic-image-normalize.ts @@ -99,49 +99,141 @@ type ProcessResult = * keys, never mutate stored values. */ const CACHE_BYTE_CAP = 64 * MiB; +const CACHE_MAX_ENTRIES = 4_096; +const CACHE_MAX_ENTRY_BYTES = 20 * MiB; // "pass" = validated pass-through; "miss" = this position's ladder cannot meet its hard // cap for these bytes (skip straight to the next position — C-gate round 2, blocker 1). -const cache = new Map(); +type CacheValue = { data: string; mediaType: string } | "pass" | "miss"; +interface CacheEntry { + value: CacheValue; + sizeBytes: number; + metadataBytes: number; + storedAt: number; +} +interface NormalizeCacheLimits { + maxBytes: number; + maxEntries: number; + maxEntryBytes: number; +} +const DEFAULT_CACHE_LIMITS: NormalizeCacheLimits = { + maxBytes: CACHE_BYTE_CAP, + maxEntries: CACHE_MAX_ENTRIES, + maxEntryBytes: CACHE_MAX_ENTRY_BYTES, +}; +const cacheEncoder = new TextEncoder(); +const cache = new Map(); +let cacheLimits = { ...DEFAULT_CACHE_LIMITS }; let cacheBytes = 0; +let cacheMetadataBytes = 0; +let cacheSentinelEntries = 0; let encodeCalls = 0; -function cachePut(key: string, value: { data: string; mediaType: string } | "pass" | "miss"): void { - const size = typeof value === "string" ? 0 : value.data.length; +function cacheEntry(key: string, value: CacheValue): CacheEntry { + const keyBytes = cacheEncoder.encode(key).byteLength; + const valueBytes = typeof value === "string" + ? cacheEncoder.encode(value).byteLength + : cacheEncoder.encode(value.mediaType).byteLength + cacheEncoder.encode(value.data).byteLength; + const metadataBytes = keyBytes + (typeof value === "string" + ? cacheEncoder.encode(value).byteLength + : cacheEncoder.encode(value.mediaType).byteLength); + return { value, sizeBytes: keyBytes + valueBytes, metadataBytes, storedAt: Date.now() }; +} + +function deleteCacheEntry(key: string): number { + const entry = cache.get(key); + if (!entry) return 0; + cache.delete(key); + cacheBytes -= entry.sizeBytes; + cacheMetadataBytes -= entry.metadataBytes; + if (typeof entry.value === "string") cacheSentinelEntries--; + return entry.sizeBytes; +} + +function cachePut(key: string, value: CacheValue): boolean { + const next = cacheEntry(key, value); + if ( + next.sizeBytes > cacheLimits.maxEntryBytes + || next.sizeBytes > cacheLimits.maxBytes + || cacheLimits.maxEntries <= 0 + ) return false; const existing = cache.get(key); if (existing !== undefined) { - cacheBytes -= typeof existing === "string" ? 0 : existing.data.length; - cache.delete(key); // re-insert refreshes recency and prevents double-count on concurrent misses + deleteCacheEntry(key); // re-insert refreshes recency and prevents double-count on concurrent misses } - while (cacheBytes + size > CACHE_BYTE_CAP && cache.size > 0) { - const oldest = cache.keys().next().value as string; - const evicted = cache.get(oldest); - cacheBytes -= typeof evicted === "string" || evicted === undefined ? 0 : evicted.data.length; - cache.delete(oldest); + while (cache.size + 1 > cacheLimits.maxEntries || cacheBytes + next.sizeBytes > cacheLimits.maxBytes) { + const oldest = cache.keys().next().value; + if (oldest === undefined || deleteCacheEntry(oldest) === 0) return false; } - cache.set(key, value); - cacheBytes += size; + cache.set(key, next); + cacheBytes += next.sizeBytes; + cacheMetadataBytes += next.metadataBytes; + if (typeof value === "string") cacheSentinelEntries++; + return true; } /** Read a cache entry, refreshing its recency (true LRU, C-gate round 1 blocker 5). */ -function cacheGet(key: string): { data: string; mediaType: string } | "pass" | "miss" | undefined { - const value = cache.get(key); - if (value !== undefined) { +function cacheGet(key: string): CacheValue | undefined { + const entry = cache.get(key); + if (entry !== undefined) { cache.delete(key); - cache.set(key, value); + entry.storedAt = Date.now(); + cache.set(key, entry); } - return value; + return entry?.value; } /** Test hooks: encoder-invocation counter + cache reset (no production caller). */ -export function getNormalizeStatsForTests(): { encodeCalls: number; cacheEntries: number; cacheBytes: number } { - return { encodeCalls, cacheEntries: cache.size, cacheBytes }; +export function getNormalizeStatsForTests(): { + encodeCalls: number; + cacheEntries: number; + cacheBytes: number; + sentinelEntries: number; + metadataBytes: number; + oldestAt: number | null; +} { + return { + encodeCalls, + cacheEntries: cache.size, + cacheBytes, + sentinelEntries: cacheSentinelEntries, + metadataBytes: cacheMetadataBytes, + oldestAt: cache.values().next().value?.storedAt ?? null, + }; } export function resetNormalizeStateForTests(): void { cache.clear(); cacheBytes = 0; + cacheMetadataBytes = 0; + cacheSentinelEntries = 0; encodeCalls = 0; } +export function setNormalizeCacheLimitsForTests(limits?: Partial): void { + resetNormalizeStateForTests(); + cacheLimits = limits ? { ...DEFAULT_CACHE_LIMITS, ...limits } : { ...DEFAULT_CACHE_LIMITS }; +} + +export function anthropicImageNormalizeRetainedStoreSnapshot(): { + count: number; + bytes: number; + evictableBytes: number; + pinnedBytes: number; + oldestAt: number | null; +} { + return { + count: cache.size, + bytes: cacheBytes, + evictableBytes: cacheBytes, + pinnedBytes: 0, + oldestAt: cache.values().next().value?.storedAt ?? null, + }; +} + +export function evictOldestAnthropicImageNormalizeForBudget(): number { + const oldest = cache.keys().next().value; + return oldest === undefined ? 0 : deleteCacheEntry(oldest); +} + /** Default encoder: Bun.Image resize-to-fit + JPEG at the given quality. */ const bunImageEncode: EncodeFn = async (input, spec, quality) => { const image = new Bun.Image(input); diff --git a/src/adapters/cursor/live-transport.ts b/src/adapters/cursor/live-transport.ts index 78ac830ce..ae21dcfe2 100644 --- a/src/adapters/cursor/live-transport.ts +++ b/src/adapters/cursor/live-transport.ts @@ -41,7 +41,13 @@ import { debugProviderDiagnostic } from "../../lib/debug"; import { classifyCursorError, isCursorBenignCancelError, safeCursorErrorMessage } from "./cursor-errors"; import { mcpArgsFromToolCall } from "./protobuf-events"; import { OCX_RESPONSES_TOOL_PROVIDER } from "./tool-definitions"; -import { handleCursorNativeExec, handleCursorNativeKv, type CursorNativeExecContext } from "./native-exec"; +import { + handleCursorNativeExec, + handleCursorNativeKv, + releaseCursorBlobRequestScope, + type CursorBlobRequestScopeToken, + type CursorNativeExecContext, +} from "./native-exec"; import { effectiveCursorNativeExecAllow } from "./exec-policy"; import { resolveMcpServers } from "./mcp-config"; import { CursorMcpManager } from "./mcp-manager"; @@ -397,6 +403,7 @@ class LiveCursorTransport implements CursorTransport { private readonly desktopDeps: CursorNativeToolDeps; private execContext: CursorNativeExecContext = {}; private mcpPrepared?: Promise; + private blobRequestScope?: CursorBlobRequestScopeToken; // Per-turn diagnostic counters/timestamps when provider debug is on (`ocx debug provider on`). Stamped in open(), cleared on // close; safe to read after a stream failure because open() owns the only writer before run(). private turnStartedAt = 0; @@ -522,24 +529,31 @@ class LiveCursorTransport implements CursorTransport { const prepared = prepareCursorRunRequest(request, { estimateInputTokens: contextUsage.carryForwardTokens === undefined, }); - state = createCursorProtobufEventState({ - clientToolNames: clientToolDefs.map(tool => tool.toolName || tool.name), - parallelToolCalls: request.parallelToolCalls, - toolSchemas, - cursorToolNameMap, - contextUsage, - ...(prepared.estimatedInputTokens !== undefined - ? { estimatedInputTokens: prepared.estimatedInputTokens } - : {}), - }); - - this.open(prepared.bytes, signal, state, push, err => { - failure = err; - wake(); - }, () => { - done = true; - wake(); - }); + this.blobRequestScope = prepared.blobRequestScope; + try { + state = createCursorProtobufEventState({ + clientToolNames: clientToolDefs.map(tool => tool.toolName || tool.name), + parallelToolCalls: request.parallelToolCalls, + toolSchemas, + cursorToolNameMap, + contextUsage, + ...(prepared.estimatedInputTokens !== undefined + ? { estimatedInputTokens: prepared.estimatedInputTokens } + : {}), + }); + this.open(prepared.bytes, signal, state, push, err => { + this.releaseBlobRequestScope(); + failure = err; + wake(); + }, () => { + this.releaseBlobRequestScope(); + done = true; + wake(); + }); + } catch (error) { + this.releaseBlobRequestScope(); + throw error; + } while (!done || queue.length > 0) { while (queue.length > 0) { @@ -582,6 +596,7 @@ class LiveCursorTransport implements CursorTransport { this.clearFirstFrameTimer(); this.stream?.close(); this.session?.close(); + this.releaseBlobRequestScope(); void this.mcpManager?.dispose(); } @@ -596,9 +611,17 @@ class LiveCursorTransport implements CursorTransport { this.stream?.destroy(); } this.session?.close(); + this.releaseBlobRequestScope(); void this.mcpManager?.dispose(); } + private releaseBlobRequestScope(): void { + const scope = this.blobRequestScope; + if (!scope) return; + this.blobRequestScope = undefined; + releaseCursorBlobRequestScope(scope); + } + private clearPendingFinalize(): void { if (this.pendingFinalize) { clearTimeout(this.pendingFinalize); @@ -824,7 +847,7 @@ class LiveCursorTransport implements CursorTransport { if (!this.stream) return; debugProviderDiagnostic("cursor", "frame", describeCursorServerFrame(message)); if (message.message.case === "kvServerMessage") { - this.stream.write(encodeConnectFrame(handleCursorNativeKv(message.message.value))); + this.stream.write(encodeConnectFrame(handleCursorNativeKv(message.message.value, this.blobRequestScope))); return; } if (message.message.case === "execServerMessage") { diff --git a/src/adapters/cursor/native-exec.ts b/src/adapters/cursor/native-exec.ts index f5ae53aef..7de4ad237 100644 --- a/src/adapters/cursor/native-exec.ts +++ b/src/adapters/cursor/native-exec.ts @@ -3,6 +3,7 @@ import { create } from "@bufbuild/protobuf"; import { DiagnosticsErrorSchema, DiagnosticsResultSchema, + ErrorSchema, GetBlobResultSchema, KvClientMessageSchema, McpErrorSchema, @@ -80,42 +81,240 @@ export function cursorUnsafeNativeLocalExecEnabled(input: Pick(); - -function evictStaleBlobs(now: number): void { - if (blobs.size <= BLOB_MAX_ENTRIES) { - // TTL sweep only when the map has any chance of stale entries; Map iterates insertion order. - for (const [k, entry] of blobs) { - if (now - entry.storedAt <= BLOB_TTL_MS) break; - blobs.delete(k); +const BLOB_TTL_MS = 15 * 60_000; +const BLOB_MAX_ENTRIES = 4_096; +const BLOB_MAX_ENTRY_BYTES = 16 * 1024 * 1024; +const BLOB_MAX_TOTAL_BYTES = 64 * 1024 * 1024; + +type CursorBlobProvenance = "local-regenerated" | "remote-setBlobArgs"; +export type CursorBlobRequestScopeToken = symbol; + +interface CursorBlobEntry { + data: Uint8Array; + storedAt: number; + sizeBytes: number; + provenance: CursorBlobProvenance; + requestPins: Set; +} + +type CursorBlobRejectionReason = "entry_too_large" | "pinned_saturation" | "request_pinned_conflict"; +type CursorBlobAdmission = + | { admitted: true; replaced: boolean } + | { admitted: false; reason: CursorBlobRejectionReason }; + +interface CursorBlobLimits { + ttlMs: number; + maxEntries: number; + maxEntryBytes: number; + maxTotalBytes: number; +} + +interface CursorBlobRequestScopeState { + keys: Set; + sealed: boolean; +} + +const DEFAULT_BLOB_LIMITS: CursorBlobLimits = { + ttlMs: BLOB_TTL_MS, + maxEntries: BLOB_MAX_ENTRIES, + maxEntryBytes: BLOB_MAX_ENTRY_BYTES, + maxTotalBytes: BLOB_MAX_TOTAL_BYTES, +}; + +const blobs = new Map(); +const blobRequestScopes = new Map(); +let blobLimits = { ...DEFAULT_BLOB_LIMITS }; +let blobBytes = 0; +let blobLocalBytes = 0; +let blobPinnedBytes = 0; +let blobEvictableBytes = 0; +let blobOldestEvictableAt: number | null = null; +let rejectedEntryTooLarge = 0; +let rejectedPinnedSaturation = 0; +let blobExpiryAccountingTimer: ReturnType | undefined; + +function isExpired(entry: CursorBlobEntry, now: number): boolean { + return now - entry.storedAt >= blobLimits.ttlMs; +} + +function bytesEqual(a: Uint8Array, b: Uint8Array): boolean { + if (a.byteLength !== b.byteLength) return false; + for (let i = 0; i < a.byteLength; i++) if (a[i] !== b[i]) return false; + return true; +} + +function recomputeBlobClassAccounting(): void { + const now = Date.now(); + let localBytes = 0; + let pinnedBytes = 0; + let evictableBytes = 0; + let oldestAt: number | null = null; + for (const entry of blobs.values()) { + const requestPinned = entry.requestPins.size > 0; + const provenancePinned = entry.provenance === "remote-setBlobArgs" && !isExpired(entry, now); + if (entry.provenance === "local-regenerated") localBytes += entry.sizeBytes; + if (requestPinned || provenancePinned) pinnedBytes += entry.sizeBytes; + if (!requestPinned && (entry.provenance === "local-regenerated" || isExpired(entry, now))) { + evictableBytes += entry.sizeBytes; + oldestAt = oldestAt === null ? entry.storedAt : Math.min(oldestAt, entry.storedAt); } - return; } - // Over cap: drop oldest entries first (insertion order approximates recency because re-stores - // delete+set to refresh their position). - const excess = blobs.size - BLOB_MAX_ENTRIES; - let dropped = 0; - for (const k of blobs.keys()) { - if (dropped >= excess) break; - blobs.delete(k); - dropped++; + blobLocalBytes = localBytes; + blobPinnedBytes = pinnedBytes; + blobEvictableBytes = evictableBytes; + blobOldestEvictableAt = oldestAt; + scheduleBlobExpiryAccounting(now); +} + +function scheduleBlobExpiryAccounting(now: number): void { + if (blobExpiryAccountingTimer) clearTimeout(blobExpiryAccountingTimer); + blobExpiryAccountingTimer = undefined; + let nextExpiry = Number.POSITIVE_INFINITY; + for (const entry of blobs.values()) { + if (entry.provenance !== "remote-setBlobArgs" || entry.requestPins.size > 0) continue; + const expiresAt = entry.storedAt + blobLimits.ttlMs; + if (expiresAt > now) nextExpiry = Math.min(nextExpiry, expiresAt); } + if (!Number.isFinite(nextExpiry)) return; + blobExpiryAccountingTimer = setTimeout(() => { + blobExpiryAccountingTimer = undefined; + recomputeBlobClassAccounting(); + }, Math.max(0, nextExpiry - now)); + blobExpiryAccountingTimer.unref?.(); } -function setBlob(k: string, data: Uint8Array): void { +function deleteBlob(k: string, recompute = true): number { + const entry = blobs.get(k); + if (!entry) return 0; + blobs.delete(k); + blobBytes -= entry.sizeBytes; + for (const scope of entry.requestPins) blobRequestScopes.get(scope)?.keys.delete(k); + if (recompute) recomputeBlobClassAccounting(); + return entry.sizeBytes; +} + +function releaseHydratedBlob(k: string, requestScope?: CursorBlobRequestScopeToken): void { + const entry = blobs.get(k); + if (!entry) return; + const scopes = requestScope ? [requestScope] : [...entry.requestPins]; + let changed = false; + for (const scope of scopes) { + if (!entry.requestPins.delete(scope)) continue; + changed = true; + const state = blobRequestScopes.get(scope); + state?.keys.delete(k); + if (state?.sealed && state.keys.size === 0) blobRequestScopes.delete(scope); + } + if (changed) recomputeBlobClassAccounting(); +} + +function setBlob( + k: string, + data: Uint8Array, + provenance: CursorBlobProvenance, + requestScope?: CursorBlobRequestScopeToken, +): CursorBlobAdmission { + if (data.byteLength > blobLimits.maxEntryBytes) { + rejectedEntryTooLarge++; + return { admitted: false, reason: "entry_too_large" }; + } + const now = Date.now(); - blobs.delete(k); // refresh insertion order so live sessions stay newest - blobs.set(k, { data, storedAt: now }); - evictStaleBlobs(now); + const existing = blobs.get(k); + const sameData = existing ? bytesEqual(existing.data, data) : false; + if (existing && !sameData && existing.requestPins.size > 0) { + return { admitted: false, reason: "request_pinned_conflict" }; + } + + const removals = new Set(); + for (const [candidateKey, entry] of blobs) { + if (candidateKey === k && sameData) continue; + if (entry.requestPins.size === 0 && isExpired(entry, now)) removals.add(candidateKey); + } + + const existingRemovedByTtl = existing !== undefined && removals.has(k); + const reuseExisting = existing !== undefined && sameData && !existingRemovedByTtl; + const mergedProvenance: CursorBlobProvenance = reuseExisting && existing.provenance === "remote-setBlobArgs" + ? "remote-setBlobArgs" + : provenance; + const storedAt = reuseExisting && existing.provenance === "remote-setBlobArgs" && provenance === "local-regenerated" + ? existing.storedAt + : now; + + let projectedBytes = blobBytes; + let projectedCount = blobs.size; + for (const candidateKey of removals) { + const entry = blobs.get(candidateKey); + if (!entry) continue; + projectedBytes -= entry.sizeBytes; + projectedCount--; + } + if (existing && !existingRemovedByTtl) { + projectedBytes -= existing.sizeBytes; + projectedCount--; + } + projectedBytes += data.byteLength; + projectedCount++; + + if (projectedBytes > blobLimits.maxTotalBytes || projectedCount > blobLimits.maxEntries) { + const localVictims = [...blobs.entries()].sort((a, b) => a[1].storedAt - b[1].storedAt); + for (const [candidateKey, entry] of localVictims) { + if ( + candidateKey === k + || removals.has(candidateKey) + || entry.requestPins.size > 0 + || entry.provenance !== "local-regenerated" + ) continue; + removals.add(candidateKey); + projectedBytes -= entry.sizeBytes; + projectedCount--; + if (projectedBytes <= blobLimits.maxTotalBytes && projectedCount <= blobLimits.maxEntries) break; + } + } + + if (projectedBytes > blobLimits.maxTotalBytes || projectedCount > blobLimits.maxEntries) { + rejectedPinnedSaturation++; + return { admitted: false, reason: "pinned_saturation" }; + } + + // Carry forward only pins whose scope still has live state; a dead scope's + // token can never be released again (review C1-1). + const requestPins = new Set(); + if (reuseExisting) { + for (const scope of existing.requestPins) { + if (blobRequestScopes.has(scope)) requestPins.add(scope); + } + } + // A pin is only attached when the scope still has live UNSEALED state: + // sealing fixes the advertised key set (locked contract), so a remote + // setBlobArgs arriving after seal must not extend the pin set — and a + // sealed scope whose advertised keys have all hydrated is already deleted, + // so attaching its stale token would create a permanent untracked pin the + // terminal release could never clear (reviews C1-1/C2-1). + const scopeState = requestScope ? blobRequestScopes.get(requestScope) : undefined; + if (requestScope && scopeState && !scopeState.sealed) requestPins.add(requestScope); + const entry: CursorBlobEntry = { + data: reuseExisting ? existing.data : data.slice(), + storedAt, + sizeBytes: data.byteLength, + provenance: mergedProvenance, + requestPins, + }; + + for (const candidateKey of removals) deleteBlob(candidateKey, false); + if (blobs.has(k)) deleteBlob(k, false); + blobs.set(k, entry); + blobBytes += entry.sizeBytes; + for (const scope of entry.requestPins) blobRequestScopes.get(scope)?.keys.add(k); + recomputeBlobClassAccounting(); + return { admitted: true, replaced: existing !== undefined }; } function getBlob(k: string): Uint8Array | undefined { const entry = blobs.get(k); if (!entry) return undefined; - if (Date.now() - entry.storedAt > BLOB_TTL_MS) { - blobs.delete(k); + if (entry.requestPins.size === 0 && isExpired(entry, Date.now())) { + deleteBlob(k); return undefined; } return entry.data; @@ -130,12 +329,136 @@ function key(bytes: Uint8Array): string { * blob id. Cursor's `rootPromptMessagesJson`/turn entries are blob IDS, not inline content — the * server fetches the bytes back via `getBlobArgs`. Mirrors jawcode `createBlobId`/`storeCursorBlob`. */ -export function storeCursorBlob(data: Uint8Array): Uint8Array { +export class CursorBlobAdmissionError extends Error { + readonly code = "cursor_blob_capacity"; + + constructor(readonly reason: CursorBlobRejectionReason) { + super("Cursor blob capacity is exhausted; retry with a smaller request."); + this.name = "CursorBlobAdmissionError"; + } +} + +let blobScopeSequence = 0; + +export function createCursorBlobRequestScope(): CursorBlobRequestScopeToken { + // Unique description per token so debug snapshots expose exact pin + // IDENTITY, not just pin counts (review C2-2: identical descriptions made + // scope-swap bugs invisible to deep comparison). + const scope = Symbol(`cursor-blob-request-${++blobScopeSequence}`); + blobRequestScopes.set(scope, { keys: new Set(), sealed: false }); + return scope; +} + +export function sealCursorBlobRequestScope(scope: CursorBlobRequestScopeToken): void { + const state = blobRequestScopes.get(scope); + if (!state) return; + state.sealed = true; + if (state.keys.size === 0) blobRequestScopes.delete(scope); +} + +export function releaseCursorBlobRequestScope(scope: CursorBlobRequestScopeToken): void { + const state = blobRequestScopes.get(scope); + if (!state) return; + for (const k of state.keys) blobs.get(k)?.requestPins.delete(scope); + blobRequestScopes.delete(scope); + recomputeBlobClassAccounting(); +} + +export function storeCursorBlob(data: Uint8Array, requestScope?: CursorBlobRequestScopeToken): Uint8Array { const blobId = new Uint8Array(createHash("sha256").update(data).digest()); - setBlob(key(blobId), data); + const admission = setBlob(key(blobId), data, "local-regenerated", requestScope); + if (!admission.admitted) throw new CursorBlobAdmissionError(admission.reason); return blobId; } +export interface CursorBlobMetrics { + count: number; + totalBytes: number; + localBytes: number; + pinnedBytes: number; + rejectedEntryTooLarge: number; + rejectedPinnedSaturation: number; + oldestAt: number | null; +} + +export function cursorBlobMetrics(): CursorBlobMetrics { + return { + count: blobs.size, + totalBytes: blobBytes, + localBytes: blobLocalBytes, + pinnedBytes: blobPinnedBytes, + rejectedEntryTooLarge, + rejectedPinnedSaturation, + oldestAt: blobOldestEvictableAt, + }; +} + +export function cursorBlobRetainedStoreSnapshot(): { + count: number; + bytes: number; + evictableBytes: number; + pinnedBytes: number; + oldestAt: number | null; +} { + return { + count: blobs.size, + bytes: blobBytes, + evictableBytes: blobEvictableBytes, + pinnedBytes: blobPinnedBytes, + oldestAt: blobOldestEvictableAt, + }; +} + +export function evictOldestCursorBlobForBudget(): number { + const now = Date.now(); + let oldest: [string, CursorBlobEntry] | undefined; + for (const candidate of blobs) { + const entry = candidate[1]; + if (entry.requestPins.size > 0) continue; + if (entry.provenance !== "local-regenerated" && !isExpired(entry, now)) continue; + if (!oldest || entry.storedAt < oldest[1].storedAt) oldest = candidate; + } + return oldest ? deleteBlob(oldest[0]) : 0; +} + +export function setCursorBlobLimitsForTests(limits?: Partial): void { + resetCursorBlobStateForTests(); + blobLimits = limits ? { ...DEFAULT_BLOB_LIMITS, ...limits } : { ...DEFAULT_BLOB_LIMITS }; +} + +export function resetCursorBlobStateForTests(): void { + if (blobExpiryAccountingTimer) clearTimeout(blobExpiryAccountingTimer); + blobExpiryAccountingTimer = undefined; + blobs.clear(); + blobRequestScopes.clear(); + blobBytes = 0; + rejectedEntryTooLarge = 0; + rejectedPinnedSaturation = 0; + recomputeBlobClassAccounting(); +} + +export function cursorBlobStoreDebugSnapshotForTests(): Array<{ + key: string; + sizeBytes: number; + storedAt: number; + provenance: CursorBlobProvenance; + requestPins: number; + dataDigest: string; + pinTokens: string[]; +}> { + return [...blobs].map(([blobKey, entry]) => ({ + key: blobKey, + sizeBytes: entry.sizeBytes, + storedAt: entry.storedAt, + provenance: entry.provenance, + requestPins: entry.requestPins.size, + // Byte-for-byte content identity + exact pin-set membership so rollback + // tests can deep-compare complete state (review C1-2). + dataDigest: createHash("sha256").update(entry.data).digest("hex"), + pinTokens: [...entry.requestPins].map(scope => String(scope.description ?? "scope")).sort(), + })); +} + export async function handleCursorNativeExec(execMsg: ExecServerMessage, deps: CursorNativeExecContext = {}): Promise { const execCase = execMsg.message.case; if (execCase === "requestContextArgs") { @@ -199,9 +522,14 @@ export async function handleCursorNativeExec(execMsg: ExecServerMessage, deps: C } -export function handleCursorNativeKv(kvMsg: KvServerMessage): Uint8Array { +export function handleCursorNativeKv( + kvMsg: KvServerMessage, + requestScope?: CursorBlobRequestScopeToken, +): Uint8Array { if (kvMsg.message.case === "getBlobArgs") { - const blobData = getBlob(key(kvMsg.message.value.blobId)); + const blobKey = key(kvMsg.message.value.blobId); + const blobData = getBlob(blobKey); + if (blobData) releaseHydratedBlob(blobKey, requestScope); return clientBytes({ message: { case: "kvClientMessage", @@ -213,13 +541,25 @@ export function handleCursorNativeKv(kvMsg: KvServerMessage): Uint8Array { }); } if (kvMsg.message.case === "setBlobArgs") { - setBlob(key(kvMsg.message.value.blobId), kvMsg.message.value.blobData); + const admission = setBlob( + key(kvMsg.message.value.blobId), + kvMsg.message.value.blobData, + "remote-setBlobArgs", + requestScope, + ); return clientBytes({ message: { case: "kvClientMessage", value: create(KvClientMessageSchema, { id: kvMsg.id, - message: { case: "setBlobResult", value: create(SetBlobResultSchema, {}) }, + message: { + case: "setBlobResult", + value: create(SetBlobResultSchema, admission.admitted ? {} : { + error: create(ErrorSchema, { + message: "Cursor blob capacity is exhausted; the blob was not stored.", + }), + }), + }, }), }, }); diff --git a/src/adapters/cursor/protobuf-request.ts b/src/adapters/cursor/protobuf-request.ts index c7030bf5b..f0b08eec2 100644 --- a/src/adapters/cursor/protobuf-request.ts +++ b/src/adapters/cursor/protobuf-request.ts @@ -6,7 +6,13 @@ import { namespacedToolName } from "../../types"; import type { CursorRunRequest } from "./types"; import { isCursorExternalWireModel } from "./discovery"; import { debugProviderDiagnostic } from "../../lib/debug"; -import { storeCursorBlob } from "./native-exec"; +import { + createCursorBlobRequestScope, + releaseCursorBlobRequestScope, + sealCursorBlobRequestScope, + storeCursorBlob, + type CursorBlobRequestScopeToken, +} from "./native-exec"; import { estimateTokens } from "../../lib/token-estimate"; import { AgentClientMessageSchema, @@ -82,8 +88,8 @@ function jsonBlob(value: unknown): { data: Uint8Array; serialized: string } { return { data: encoder.encode(serialized), serialized }; } -type StoredRootBlob = { - id: Uint8Array; +type RootBlobCandidate = { + data: Uint8Array; byteLength: number; /** * The exact JSON handed to storeCursorBlob(). Retained so a token estimate can read @@ -97,14 +103,14 @@ type StoredRootBlob = { text?: string; }; -function storedRootBlob( +function rootBlobCandidate( value: unknown, - role: StoredRootBlob["role"], + role: RootBlobCandidate["role"], opts?: { messageIndex?: number; text?: string }, -): StoredRootBlob { +): RootBlobCandidate { const { data, serialized } = jsonBlob(value); return { - id: storeCursorBlob(data), + data, byteLength: data.byteLength, serialized, role, @@ -113,7 +119,7 @@ function storedRootBlob( }; } -function truncateToolResultBlob(entry: StoredRootBlob, maxBytes: number): StoredRootBlob | null { +function truncateToolResultBlob(entry: RootBlobCandidate, maxBytes: number): RootBlobCandidate | null { if (entry.byteLength <= maxBytes) return entry; if (entry.role !== "toolResult" || entry.text === undefined) return null; const marker = "\n…[truncated for Cursor external replay budget]"; @@ -124,7 +130,7 @@ function truncateToolResultBlob(entry: StoredRootBlob, maxBytes: number): Stored let end = keepBytes; while (end > 0 && end < encoded.byteLength && (encoded[end]! & 0xc0) === 0x80) end -= 1; const truncated = `${decoder.decode(encoded.subarray(0, end))}${marker}`; - const result = storedRootBlob( + const result = rootBlobCandidate( { role: "user", content: [{ type: "text", text: truncated }] }, "toolResult", { messageIndex: entry.messageIndex, text: truncated }, @@ -133,7 +139,7 @@ function truncateToolResultBlob(entry: StoredRootBlob, maxBytes: number): Stored if (end === 0) break; keepBytes = Math.max(0, end - (result.byteLength - maxBytes) - 16); } - const markerOnly = storedRootBlob( + const markerOnly = rootBlobCandidate( { role: "user", content: [{ type: "text", text: marker.trimStart() }] }, "toolResult", { messageIndex: entry.messageIndex, text: marker.trimStart() }, @@ -141,7 +147,7 @@ function truncateToolResultBlob(entry: StoredRootBlob, maxBytes: number): Stored return markerOnly.byteLength <= maxBytes ? markerOnly : null; } -function systemPromptBlobs(request: CursorRunRequest): StoredRootBlob[] { +function systemPromptBlobs(request: CursorRunRequest): RootBlobCandidate[] { const prompts = request.system.length > 0 ? [...request.system] : ["You are a helpful assistant."]; if (cursorRequestHasShellAlias(request.tools)) prompts.push(CURSOR_SHELL_ALIAS_SYSTEM_NOTE); const cursorToolGuidance = buildCursorToolGuidanceSystemNote( @@ -149,7 +155,7 @@ function systemPromptBlobs(request: CursorRunRequest): StoredRootBlob[] { request.toolChoice, ); if (cursorToolGuidance) prompts.push(cursorToolGuidance); - return prompts.map(content => storedRootBlob({ role: "system", content }, "system")); + return prompts.map(content => rootBlobCandidate({ role: "system", content }, "system")); } function assistantRootText( @@ -169,7 +175,7 @@ function assistantRootText( // because it travels in the action. Tool results are rendered as user-role text with a marker, and // each entry is a SHA-256 blob ID (Cursor fetches the bytes back via getBlobArgs). Mirrors the // danger-pi reference buildRootPromptMessagesJson. -function rootPromptMessages(request: CursorRunRequest): { +function rootPromptMessages(request: CursorRunRequest, requestScope: CursorBlobRequestScopeToken): { ids: Uint8Array[]; byteLength: number; historyMessageStart: number; @@ -181,7 +187,7 @@ function rootPromptMessages(request: CursorRunRequest): { const messages = request.rawMessages; if (!messages?.length) { return { - ids: entries.map(entry => entry.id), + ids: entries.map(entry => storeCursorBlob(entry.data, requestScope)), byteLength: entries.reduce((sum, entry) => sum + entry.byteLength, 0), historyMessageStart: 0, serialized: entries.map(entry => entry.serialized), @@ -202,7 +208,7 @@ function rootPromptMessages(request: CursorRunRequest): { // A bare string survives blob hydration but external workers reject the completed replay // before tokenization (`usedTokens: 0`, then invalid_argument). if (text.length > 0) { - entries.push(storedRootBlob({ + entries.push(rootBlobCandidate({ role: "user", content: [{ type: "text", text }], }, "user", { messageIndex: i })); @@ -212,7 +218,7 @@ function rootPromptMessages(request: CursorRunRequest): { // Native Composer state can preserve it through ThinkingMessage/history structures. const text = assistantRootText(message, !externalModel).trim(); if (text.length > 0) { - entries.push(storedRootBlob( + entries.push(rootBlobCandidate( { role: "assistant", content: [{ type: "text", text }] }, "assistant", { messageIndex: i }, @@ -222,7 +228,7 @@ function rootPromptMessages(request: CursorRunRequest): { } else if (message.role === "toolResult") { const prefix = message.isError ? "[Tool Error]" : "[Tool Result]"; const text = `${prefix}\n${toolResultToText(message)}`; - entries.push(storedRootBlob( + entries.push(rootBlobCandidate( { role: "user", content: [{ type: "text", text }] }, "toolResult", { messageIndex: i, text }, @@ -247,7 +253,7 @@ function rootPromptMessages(request: CursorRunRequest): { const active = history .slice(activeStart) .map(entry => truncateToolResultBlob(entry, historyBudget)) - .filter((entry): entry is StoredRootBlob => entry !== null); + .filter((entry): entry is RootBlobCandidate => entry !== null); let activeBytes = active.reduce((sum, entry) => sum + entry.byteLength, 0); while (active.length > 1 && activeBytes > historyBudget) { const dropped = active.shift(); @@ -265,7 +271,7 @@ function rootPromptMessages(request: CursorRunRequest): { } const prior = history.slice(0, activeStart); - const keptPrior: StoredRootBlob[] = []; + const keptPrior: RootBlobCandidate[] = []; let priorBytes = 0; // Take complete turns from the end: a turn starts at a user/developer root entry. let i = prior.length - 1; @@ -298,7 +304,7 @@ function rootPromptMessages(request: CursorRunRequest): { } return { - ids: selected.map(entry => entry.id), + ids: selected.map(entry => storeCursorBlob(entry.data, requestScope)), byteLength: selected.reduce((sum, entry) => sum + entry.byteLength, 0), historyMessageStart, serialized: selected.map(entry => entry.serialized), @@ -345,7 +351,11 @@ function argBytes(value: unknown): Uint8Array { } } -function toolCallStep(part: Extract, result?: OcxToolResultMessage): Uint8Array { +function toolCallStep( + part: Extract, + requestScope: CursorBlobRequestScopeToken, + result?: OcxToolResultMessage, +): Uint8Array { const args: Record = {}; for (const [key, value] of Object.entries(part.arguments ?? {})) args[key] = argBytes(value); const toolName = namespacedToolName(part.namespace, part.name); @@ -368,7 +378,7 @@ function toolCallStep(part: Extract>(); const flush = () => { if (!current) return; - for (const part of pendingToolCalls.values()) current.steps.push(toolCallStep(part)); + for (const part of pendingToolCalls.values()) current.steps.push(toolCallStep(part, requestScope)); turns.push(storeCursorBlob(toBinary(ConversationTurnStructureSchema, create(ConversationTurnStructureSchema, { turn: { case: "agentConversationTurn", value: create(AgentConversationTurnStructureSchema, current), }, - })))); + })), requestScope)); current = undefined; pendingToolCalls.clear(); }; @@ -451,7 +465,7 @@ function conversationTurns(request: CursorRunRequest, historyMessageStart = 0): case: "assistantMessage", value: create(AssistantMessageSchema, { text: part.text }), }, - })))); + })), requestScope)); } continue; } @@ -459,7 +473,7 @@ function conversationTurns(request: CursorRunRequest, historyMessageStart = 0): pendingToolCalls.set(part.id, part); continue; } - const step = assistantStep(part); + const step = assistantStep(part, requestScope); if (step) current.steps.push(step); } continue; @@ -473,12 +487,12 @@ function conversationTurns(request: CursorRunRequest, historyMessageStart = 0): case: "assistantMessage", value: create(AssistantMessageSchema, { text: `${prefix}\n${contentToText(message.content)}` }), }, - })))); + })), requestScope)); continue; } const priorCall = pendingToolCalls.get(message.toolCallId); if (priorCall) { - current.steps.push(toolCallStep(priorCall, message)); + current.steps.push(toolCallStep(priorCall, requestScope, message)); pendingToolCalls.delete(message.toolCallId); } else { current.steps.push(storeCursorBlob(toBinary(ConversationStepSchema, create(ConversationStepSchema, { @@ -486,7 +500,7 @@ function conversationTurns(request: CursorRunRequest, historyMessageStart = 0): case: "assistantMessage", value: create(AssistantMessageSchema, { text: toolResultToText(message) }), }, - })))); + })), requestScope)); } continue; } @@ -495,7 +509,7 @@ function conversationTurns(request: CursorRunRequest, historyMessageStart = 0): userMessage: storeCursorBlob(toBinary(UserMessageSchema, create(UserMessageSchema, { text: contentText(message), messageId: crypto.randomUUID(), - }))), + })), requestScope), steps: [], }; } @@ -537,6 +551,7 @@ function modelVisibleToolText(definition: McpToolDefinition): string { export interface PreparedCursorRunRequest { bytes: Uint8Array; + blobRequestScope: CursorBlobRequestScopeToken; /** Only present when the caller asked for it; see prepareCursorRunRequest(). */ estimatedInputTokens?: number; } @@ -552,8 +567,9 @@ export interface PreparedCursorRunRequest { * it honest: history the pruner dropped and tools the filter removed are already * gone by this point. */ -export function prepareCursorRunRequest( +function buildPreparedCursorRunRequest( request: CursorRunRequest, + requestScope: CursorBlobRequestScopeToken, options?: { estimateInputTokens?: boolean }, ): PreparedCursorRunRequest { const rawText = activePromptText(request); @@ -585,9 +601,9 @@ export function prepareCursorRunRequest( }), }, }); - const rootPromptMessagesState = rootPromptMessages(request); + const rootPromptMessagesState = rootPromptMessages(request, requestScope); const rootPromptMessageIds = rootPromptMessagesState.ids; - const turnIds = conversationTurns(request, rootPromptMessagesState.historyMessageStart); + const turnIds = conversationTurns(request, requestScope, rootPromptMessagesState.historyMessageStart); // Hoisted out of the mcp_tools spread below so the estimate can read the same // filtered definitions the wire carries. Both helpers are pure. const visibleTools = cursorToolsForActivePrompt(request.tools, rawText, request.toolChoice); @@ -661,7 +677,7 @@ export function prepareCursorRunRequest( message: { case: "runRequest", value: runRequest }, }); const bytes = toBinary(AgentClientMessageSchema, message); - if (!options?.estimateInputTokens) return { bytes }; + if (!options?.estimateInputTokens) return { bytes, blobRequestScope: requestScope }; // Same instances that produced `bytes`, so the estimate cannot count history or // tools the payload dropped — the defect that blocked PR #376. @@ -672,10 +688,26 @@ export function prepareCursorRunRequest( ]; return { bytes, + blobRequestScope: requestScope, estimatedInputTokens: estimateTokens(modelVisibleParts.join("\n"), request.modelId), }; } +export function prepareCursorRunRequest( + request: CursorRunRequest, + options?: { estimateInputTokens?: boolean }, +): PreparedCursorRunRequest { + const requestScope = createCursorBlobRequestScope(); + try { + const prepared = buildPreparedCursorRunRequest(request, requestScope, options); + sealCursorBlobRequestScope(requestScope); + return prepared; + } catch (error) { + releaseCursorBlobRequestScope(requestScope); + throw error; + } +} + /** Back-compat wrapper: callers that only need the wire bytes. */ export function encodeCursorRunRequest(request: CursorRunRequest): Uint8Array { return prepareCursorRunRequest(request).bytes; diff --git a/src/adapters/google-antigravity-replay.ts b/src/adapters/google-antigravity-replay.ts index 2fced61b0..5a85e0b49 100644 --- a/src/adapters/google-antigravity-replay.ts +++ b/src/adapters/google-antigravity-replay.ts @@ -10,9 +10,16 @@ * Claude-on-Antigravity uses inline signature sanitization instead (see google-antigravity-wire). */ +interface ReplayCall { + signature: string; + sizeBytes: number; + touchedAtMs: number; +} + interface ReplayEntry { /** thoughtSignature keyed by functionCall identity (name + canonical args). */ - byCall: Map; + byCall: Map; + bytes: number; expiresAtMs: number; } @@ -20,8 +27,26 @@ const MIN_SIGNATURE_LEN = 16; const REPLAY_TTL_MS = 60 * 60 * 1000; // 1h const REPLAY_MAX_ENTRIES = 10_240; const REPLAY_EVICT_BATCH = 128; +const REPLAY_MAX_CALLS_PER_SESSION = 256; +const REPLAY_MAX_BYTES_PER_SESSION = 2 * 1024 * 1024; +const REPLAY_MAX_SIGNATURE_BYTES = 64 * 1024; + +interface ReplayLimits { + maxCallsPerSession: number; + maxBytesPerSession: number; + maxSignatureBytes: number; +} + +const DEFAULT_REPLAY_LIMITS: ReplayLimits = { + maxCallsPerSession: REPLAY_MAX_CALLS_PER_SESSION, + maxBytesPerSession: REPLAY_MAX_BYTES_PER_SESSION, + maxSignatureBytes: REPLAY_MAX_SIGNATURE_BYTES, +}; const replayCache = new Map(); +const utf8 = new TextEncoder(); +let replayLimits = { ...DEFAULT_REPLAY_LIMITS }; +let replayBytes = 0; function replayKey(model: string, sessionId: string): string { return `${model}::session:${sessionId}`; @@ -57,12 +82,44 @@ function extractSignature(part: Record): string | undefined { return undefined; } +function deleteReplaySession(key: string): number { + const entry = replayCache.get(key); + if (!entry) return 0; + replayCache.delete(key); + replayBytes -= entry.bytes; + return entry.bytes; +} + +function deleteExpiredReplaySessions(now: number): void { + for (const [key, entry] of replayCache) if (entry.expiresAtMs <= now) deleteReplaySession(key); +} + +function deleteReplayCall(entry: ReplayEntry, callKey: string): number { + const call = entry.byCall.get(callKey); + if (!call) return 0; + entry.byCall.delete(callKey); + entry.bytes -= call.sizeBytes; + replayBytes -= call.sizeBytes; + return call.sizeBytes; +} + +function evictInnerCalls(entry: ReplayEntry): void { + while ( + entry.byCall.size > replayLimits.maxCallsPerSession + || entry.bytes > replayLimits.maxBytesPerSession + ) { + const oldest = entry.byCall.keys().next().value; + if (oldest === undefined) break; + deleteReplayCall(entry, oldest); + } +} + function evictIfNeeded(): void { if (replayCache.size <= REPLAY_MAX_ENTRIES) return; const oldest = [...replayCache.entries()] .sort((a, b) => a[1].expiresAtMs - b[1].expiresAtMs) .slice(0, REPLAY_EVICT_BATCH); - for (const [key] of oldest) replayCache.delete(key); + for (const [key] of oldest) deleteReplaySession(key); } /** Gemini/Flash/Agent use the replay cache; Claude does not (inline sanitization instead). */ @@ -78,9 +135,11 @@ export function antigravityUsesReplayCache(model: string): boolean { */ export function observeAntigravityReplay(model: string, sessionId: string, parts: unknown[]): void { if (!antigravityUsesReplayCache(model) || !Array.isArray(parts) || parts.length === 0) return; + const now = Date.now(); + deleteExpiredReplaySessions(now); const key = replayKey(model, sessionId); - const entry = replayCache.get(key) ?? { byCall: new Map(), expiresAtMs: 0 }; - let changed = false; + const entry = replayCache.get(key) ?? { byCall: new Map(), bytes: 0, expiresAtMs: 0 }; + let inserted = false; for (const raw of parts) { if (!raw || typeof raw !== "object") continue; const part = raw as Record; @@ -89,10 +148,18 @@ export function observeAntigravityReplay(model: string, sessionId: string, parts const fc = part.functionCall as { name?: unknown; args?: unknown } | undefined; const ck = fc ? functionCallKey(fc.name, fc.args) : undefined; if (!ck) continue; // only function-call signatures are replayable by identity - if (entry.byCall.get(ck) !== sig) { entry.byCall.set(ck, sig); changed = true; } + const signatureBytes = utf8.encode(sig).byteLength; + const sizeBytes = utf8.encode(ck).byteLength + signatureBytes; + if (signatureBytes > replayLimits.maxSignatureBytes || sizeBytes > replayLimits.maxBytesPerSession) continue; + deleteReplayCall(entry, ck); + entry.byCall.set(ck, { signature: sig, sizeBytes, touchedAtMs: now }); + entry.bytes += sizeBytes; + replayBytes += sizeBytes; + inserted = true; } - if (!changed && replayCache.has(key)) return; - entry.expiresAtMs = Date.now() + REPLAY_TTL_MS; + if (!inserted) return; + evictInnerCalls(entry); + entry.expiresAtMs = now + REPLAY_TTL_MS; replayCache.set(key, entry); evictIfNeeded(); } @@ -104,9 +171,10 @@ export function observeAntigravityReplay(model: string, sessionId: string, parts */ export function applyAntigravityReplay(model: string, sessionId: string, contents: unknown[]): unknown[] { if (!antigravityUsesReplayCache(model) || !Array.isArray(contents)) return contents; + const now = Date.now(); + deleteExpiredReplaySessions(now); const entry = replayCache.get(replayKey(model, sessionId)); - if (!entry || entry.expiresAtMs <= Date.now()) { - if (entry) replayCache.delete(replayKey(model, sessionId)); + if (!entry) { return contents; } for (const c of contents as { role?: string; parts?: unknown[] }[]) { @@ -118,8 +186,12 @@ export function applyAntigravityReplay(model: string, sessionId: string, content if (!fc) continue; if (part.thoughtSignature !== undefined || part.thought_signature !== undefined) continue; const ck = functionCallKey(fc.name, fc.args); - const sig = ck ? entry.byCall.get(ck) : undefined; - if (sig) part.thoughtSignature = sig; + const call = ck ? entry.byCall.get(ck) : undefined; + if (call && ck) { + part.thoughtSignature = call.signature; + entry.byCall.delete(ck); + entry.byCall.set(ck, { ...call, touchedAtMs: now }); + } } } return contents; @@ -127,10 +199,61 @@ export function applyAntigravityReplay(model: string, sessionId: string, content /** Drop the cache entry when upstream rejects a signature (clear-on-invalid). */ export function clearAntigravityReplay(model: string, sessionId: string): void { - replayCache.delete(replayKey(model, sessionId)); + deleteReplaySession(replayKey(model, sessionId)); +} + +export function antigravityReplayMetrics(): { + sessions: number; + calls: number; + totalBytes: number; + largestSessionBytes: number; +} { + let calls = 0; + let largestSessionBytes = 0; + for (const entry of replayCache.values()) { + calls += entry.byCall.size; + largestSessionBytes = Math.max(largestSessionBytes, entry.bytes); + } + return { sessions: replayCache.size, calls, totalBytes: replayBytes, largestSessionBytes }; +} + +export function antigravityReplayRetainedStoreSnapshot(): { + count: number; + bytes: number; + evictableBytes: number; + pinnedBytes: number; + oldestAt: number | null; +} { + let oldestAt: number | null = null; + for (const entry of replayCache.values()) { + for (const call of entry.byCall.values()) { + oldestAt = oldestAt === null ? call.touchedAtMs : Math.min(oldestAt, call.touchedAtMs); + } + } + return { count: replayCache.size, bytes: replayBytes, evictableBytes: replayBytes, pinnedBytes: 0, oldestAt }; +} + +export function evictOldestAntigravityReplayForBudget(): number { + let oldestKey: string | undefined; + let oldestAt = Number.POSITIVE_INFINITY; + for (const [key, entry] of replayCache) { + let sessionOldest = entry.expiresAtMs; + for (const call of entry.byCall.values()) sessionOldest = Math.min(sessionOldest, call.touchedAtMs); + if (sessionOldest < oldestAt) { + oldestAt = sessionOldest; + oldestKey = key; + } + } + return oldestKey === undefined ? 0 : deleteReplaySession(oldestKey); +} + +export function setAntigravityReplayLimitsForTests(limits?: Partial): void { + __resetAntigravityReplayCache(); + replayLimits = limits ? { ...DEFAULT_REPLAY_LIMITS, ...limits } : { ...DEFAULT_REPLAY_LIMITS }; } /** Test seam. */ export function __resetAntigravityReplayCache(): void { replayCache.clear(); + replayBytes = 0; } diff --git a/src/vision/index.ts b/src/vision/index.ts index 523fcc730..89cd4d7e6 100644 --- a/src/vision/index.ts +++ b/src/vision/index.ts @@ -16,6 +16,8 @@ const DEFAULT_ANTHROPIC_VISION_MODEL = "claude-sonnet-5"; const DEFAULT_TIMEOUT_MS = 45_000; const DEFAULT_MAX_DESCRIPTIONS_PER_TURN = 8; const DESCRIPTION_CACHE_MAX_ENTRIES = 256; +const DESCRIPTION_CACHE_MAX_BYTES = 1024 * 1024; +const descriptionEncoder = new TextEncoder(); /** Max images described in parallel — keeps first-token latency bounded without flooding the backend. */ const VISION_CONCURRENCY = 3; /** Per-image description hard cap (chars) so multi-image turns can't blow the main model's context. */ @@ -27,47 +29,109 @@ export interface VisionDescriptionCache { get(key: string): string | undefined; set(key: string, value: string): void; clear(): void; + snapshot?(): { count: number; bytes: number; oldestAt: number | null }; + evictOldest?(): number; } class BoundedLruDescriptionCache implements VisionDescriptionCache { - private readonly entries = new Map(); + private readonly entries = new Map(); + private bytes = 0; - constructor(private readonly maxEntries: number) {} + constructor(private readonly maxEntries: number, private readonly maxBytes: number) {} get(key: string): string | undefined { - const value = this.entries.get(key); - if (value === undefined) return undefined; + const entry = this.entries.get(key); + if (entry === undefined) return undefined; this.entries.delete(key); - this.entries.set(key, value); - return value; + entry.storedAt = Date.now(); + this.entries.set(key, entry); + return entry.value; } set(key: string, value: string): void { - this.entries.delete(key); - this.entries.set(key, value); - while (this.entries.size > this.maxEntries) { - const oldest = this.entries.keys().next().value; - if (oldest === undefined) break; - this.entries.delete(oldest); + const existing = this.entries.get(key); + if (existing) { + this.entries.delete(key); + this.bytes -= existing.sizeBytes; + } + const sizeBytes = descriptionEncoder.encode(key).byteLength + descriptionEncoder.encode(value).byteLength; + if (sizeBytes > this.maxBytes || this.maxEntries <= 0) return; + while (this.entries.size + 1 > this.maxEntries || this.bytes + sizeBytes > this.maxBytes) { + if (this.evictOldest() === 0) return; } + this.entries.set(key, { value, sizeBytes, storedAt: Date.now() }); + this.bytes += sizeBytes; } clear(): void { this.entries.clear(); + this.bytes = 0; + } + + snapshot(): { count: number; bytes: number; oldestAt: number | null } { + return { + count: this.entries.size, + bytes: this.bytes, + oldestAt: this.entries.values().next().value?.storedAt ?? null, + }; + } + + evictOldest(): number { + const oldest = this.entries.keys().next().value; + if (oldest === undefined) return 0; + const entry = this.entries.get(oldest); + if (!entry) return 0; + this.entries.delete(oldest); + this.bytes -= entry.sizeBytes; + return entry.sizeBytes; } } -let descriptionCache: VisionDescriptionCache = new BoundedLruDescriptionCache(DESCRIPTION_CACHE_MAX_ENTRIES); +let descriptionCacheLimits = { + maxEntries: DESCRIPTION_CACHE_MAX_ENTRIES, + maxBytes: DESCRIPTION_CACHE_MAX_BYTES, +}; + +function defaultDescriptionCache(): VisionDescriptionCache { + return new BoundedLruDescriptionCache(descriptionCacheLimits.maxEntries, descriptionCacheLimits.maxBytes); +} + +let descriptionCache: VisionDescriptionCache = defaultDescriptionCache(); /** Replace the process cache (primarily for deterministic tests). Passing undefined restores the default LRU. */ export function setVisionDescriptionCache(cache?: VisionDescriptionCache): void { - descriptionCache = cache ?? new BoundedLruDescriptionCache(DESCRIPTION_CACHE_MAX_ENTRIES); + descriptionCache = cache ?? defaultDescriptionCache(); } export function resetVisionDescriptionCache(): void { descriptionCache.clear(); } +export function setVisionDescriptionCacheLimitsForTests( + limits?: { maxEntries?: number; maxBytes?: number }, +): void { + descriptionCacheLimits = limits + ? { maxEntries: limits.maxEntries ?? DESCRIPTION_CACHE_MAX_ENTRIES, maxBytes: limits.maxBytes ?? DESCRIPTION_CACHE_MAX_BYTES } + : { maxEntries: DESCRIPTION_CACHE_MAX_ENTRIES, maxBytes: DESCRIPTION_CACHE_MAX_BYTES }; + descriptionCache = defaultDescriptionCache(); +} + +export function visionDescriptionRetainedStoreSnapshot(): { + count: number; + bytes: number; + evictableBytes: number; + pinnedBytes: number; + oldestAt: number | null; +} { + const snapshot = descriptionCache.snapshot?.(); + if (!snapshot) return { count: 0, bytes: 0, evictableBytes: 0, pinnedBytes: 0, oldestAt: null }; + return { ...snapshot, evictableBytes: snapshot.bytes, pinnedBytes: 0 }; +} + +export function evictOldestVisionDescriptionForBudget(): number { + return descriptionCache.evictOldest?.() ?? 0; +} + /** Runtime config is permissive: zero is intentional; malformed values fall back to the bounded default. */ export function resolveMaxDescriptionsPerTurn(value: unknown): number { if (value === 0) return 0; @@ -338,9 +402,10 @@ export async function describeImagesInPlace( } catch (error) { outcome = { text: "", error: error instanceof Error ? error.message : String(error) }; } - const successfulText = outcome.error ? "" : outcome.text.trim(); + const successfulText = outcome.error ? "" : clamp(outcome.text.trim(), DESC_MAX_CHARS); if (identity.persistent && successfulText) descriptionCache.set(identity.key, successfulText); - resolveOutcome(outcome); + const resolvedOutcome = outcome.error ? outcome : { ...outcome, text: successfulText }; + resolveOutcome(resolvedOutcome); }); } diff --git a/tests/anthropic-image-normalize.test.ts b/tests/anthropic-image-normalize.test.ts index aafa6be53..68d0185a2 100644 --- a/tests/anthropic-image-normalize.test.ts +++ b/tests/anthropic-image-normalize.test.ts @@ -1,8 +1,11 @@ -import { beforeEach, describe, expect, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; import { + anthropicImageNormalizeRetainedStoreSnapshot, + evictOldestAnthropicImageNormalizeForBudget, normalizeAnthropicImages, getNormalizeStatsForTests, resetNormalizeStateForTests, + setNormalizeCacheLimitsForTests, TIER_SPECS, IMAGE_NORMALIZE_CONCURRENCY, normalizeImageTargets, @@ -68,6 +71,100 @@ function sizedEncoder(sizeFor: (maxEdge: number) => number): EncodeFn { } beforeEach(() => resetNormalizeStateForTests()); +afterEach(() => setNormalizeCacheLimitsForTests()); + +describe("bounded normalization cache accounting", () => { + const target = (base64: string): NormalizeTarget => ({ + base64, + mediaType: "image/png", + replace: () => {}, + drop: () => {}, + }); + const validate = () => Promise.resolve(); + + test("unique pass and miss sentinels consume metadata bytes and hit the count cap", async () => { + const pass = fakePngBase64(100, 100, 128); + await normalizeImageTargets([target(pass)], { validate }); + const afterPass = getNormalizeStatsForTests(); + const passKey = `${Bun.hash(pass).toString(36)}:image/png:0`; + const expectedPassBytes = Buffer.byteLength(passKey, "utf8") + Buffer.byteLength("pass", "utf8"); + expect(afterPass.sentinelEntries).toBe(1); + expect(afterPass.metadataBytes).toBe(expectedPassBytes); + expect(afterPass.cacheBytes).toBe(expectedPassBytes); + + const missEncoder = sizedEncoder(() => 3 * 1024 * 1024); + await normalizeImageTargets([target(fakePngBase64(3000, 2000, 256))], { encode: missEncoder }); + expect(getNormalizeStatsForTests().sentinelEntries).toBeGreaterThan(1); + + setNormalizeCacheLimitsForTests({ maxEntries: 2, maxBytes: 1024 * 1024 }); + for (let i = 0; i < 3; i++) { + await normalizeImageTargets([target(fakePngBase64(100 + i, 100 + i, 128 + i))], { validate }); + } + const capped = getNormalizeStatsForTests(); + expect(capped.cacheEntries).toBe(2); + expect(capped.sentinelEntries).toBe(2); + expect(capped.metadataBytes).toBeGreaterThan(0); + }); + + test("encoded replacement keeps aggregate accounting exact", async () => { + let arrivals = 0; + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + let both!: () => void; + const reached = new Promise(resolve => { both = resolve; }); + const encode: EncodeFn = async () => { + arrivals++; + if (arrivals === 2) both(); + await gate; + return { data: "a".repeat(1_000), mediaType: "image/jpeg" }; + }; + const source = fakePngBase64(3000, 2000, 256); + const first = normalizeImageTargets([target(source)], { encode }); + const second = normalizeImageTargets([target(source)], { encode }); + await reached; + release(); + await Promise.all([first, second]); + const stats = getNormalizeStatsForTests(); + expect(stats.cacheEntries).toBe(1); + expect(stats.cacheBytes).toBe(anthropicImageNormalizeRetainedStoreSnapshot().bytes); + expect(stats.cacheBytes).toBeGreaterThan(1_000); + }); + + test("one encoded value above maxEntrySize is returned but not cached", async () => { + setNormalizeCacheLimitsForTests({ maxEntryBytes: 1_000 }); + let replaced = ""; + const oversized: NormalizeTarget = { + ...target(fakePngBase64(3000, 2000, 256)), + replace: data => { replaced = data; }, + }; + await normalizeImageTargets([oversized], { + encode: async () => ({ data: "z".repeat(2_000), mediaType: "image/jpeg" }), + }); + expect(replaced).toHaveLength(2_000); + expect(getNormalizeStatsForTests().cacheEntries).toBe(0); + }); + + test("cache eviction occurs before insertion and never exceeds 64 MiB", async () => { + setNormalizeCacheLimitsForTests({ maxBytes: 2_500, maxEntryBytes: 2_000 }); + const encode: EncodeFn = async () => ({ data: "e".repeat(1_100), mediaType: "image/jpeg" }); + for (let i = 0; i < 3; i++) { + await normalizeImageTargets([target(fakePngBase64(3000 + i, 2000 + i, 256 + i))], { encode }); + expect(getNormalizeStatsForTests().cacheBytes).toBeLessThanOrEqual(2_500); + } + expect(getNormalizeStatsForTests().cacheEntries).toBeLessThanOrEqual(2); + }); + + test("040 snapshot is observe-only and oldest-row eviction returns full metadata-inclusive released bytes", async () => { + await normalizeImageTargets([target(fakePngBase64(100, 100, 128))], { validate }); + await normalizeImageTargets([target(fakePngBase64(101, 101, 129))], { validate }); + const before = anthropicImageNormalizeRetainedStoreSnapshot(); + expect(anthropicImageNormalizeRetainedStoreSnapshot()).toEqual(before); + const released = evictOldestAnthropicImageNormalizeForBudget(); + expect(released).toBeGreaterThan(0); + expect(released).toBeGreaterThanOrEqual(getNormalizeStatsForTests().metadataBytes / Math.max(1, before.count)); + expect(anthropicImageNormalizeRetainedStoreSnapshot().bytes).toBe(before.bytes - released); + }); +}); describe("normalizeAnthropicImages — real Bun.Image path", () => { test("N1: oversized-dimension PNG is resized under the tier-0 edge and 2MiB cap, not dropped", async () => { diff --git a/tests/cursor-blob.test.ts b/tests/cursor-blob.test.ts index e25e0c925..9f57e5d35 100644 --- a/tests/cursor-blob.test.ts +++ b/tests/cursor-blob.test.ts @@ -1,7 +1,21 @@ -import { describe, expect, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; import { createHash } from "node:crypto"; import { create, fromBinary } from "@bufbuild/protobuf"; -import { handleCursorNativeKv, storeCursorBlob } from "../src/adapters/cursor/native-exec"; +import { + createCursorBlobRequestScope, + cursorBlobMetrics, + cursorBlobRetainedStoreSnapshot, + cursorBlobStoreDebugSnapshotForTests, + CursorBlobAdmissionError, + evictOldestCursorBlobForBudget, + handleCursorNativeKv, + releaseCursorBlobRequestScope, + resetCursorBlobStateForTests, + sealCursorBlobRequestScope, + setCursorBlobLimitsForTests, + storeCursorBlob, + type CursorBlobRequestScopeToken, +} from "../src/adapters/cursor/native-exec"; import { CURSOR_EXTERNAL_ROOT_BYTE_LIMIT, CURSOR_EXTERNAL_ROOT_BLOB_LIMIT, @@ -16,8 +30,12 @@ import { ConversationTurnStructureSchema, GetBlobArgsSchema, KvServerMessageSchema, + SetBlobArgsSchema, } from "../src/adapters/cursor/gen/agent_pb"; +beforeEach(() => resetCursorBlobStateForTests()); +afterEach(() => setCursorBlobLimitsForTests()); + function sha256(data: Uint8Array): Uint8Array { return new Uint8Array(createHash("sha256").update(data).digest()); } @@ -33,6 +51,44 @@ function blobData(blobId: Uint8Array): Uint8Array { return kv.message.value.blobData; } +function getBlobReply(blobId: Uint8Array, id = 1, scope?: CursorBlobRequestScopeToken) { + return fromBinary(AgentClientMessageSchema, handleCursorNativeKv(create(KvServerMessageSchema, { + id, + message: { case: "getBlobArgs", value: create(GetBlobArgsSchema, { blobId }) }, + }), scope)); +} + +function setBlobReply(blobId: Uint8Array, blobData: Uint8Array, id = 1, scope?: CursorBlobRequestScopeToken) { + return fromBinary(AgentClientMessageSchema, handleCursorNativeKv(create(KvServerMessageSchema, { + id, + message: { case: "setBlobArgs", value: create(SetBlobArgsSchema, { blobId, blobData }) }, + }), scope)); +} + +function hydrateBlob(blobId: Uint8Array, scope?: CursorBlobRequestScopeToken): Uint8Array { + const reply = getBlobReply(blobId, 1, scope); + expect(reply.message.case).toBe("kvClientMessage"); + if (reply.message.case !== "kvClientMessage") throw new Error("expected kvClientMessage"); + expect(reply.message.value.message.case).toBe("getBlobResult"); + if (reply.message.value.message.case !== "getBlobResult") throw new Error("expected getBlobResult"); + expect(reply.message.value.message.value.blobData).toBeDefined(); + return reply.message.value.message.value.blobData!; +} + +function expectBlobHit(blobId: Uint8Array, expected: Uint8Array, scope?: CursorBlobRequestScopeToken): void { + expect(Array.from(hydrateBlob(blobId, scope))).toEqual(Array.from(expected)); +} + +function expectBlobMiss(blobId: Uint8Array, id = 1, scope?: CursorBlobRequestScopeToken): void { + const reply = getBlobReply(blobId, id, scope); + expect(reply.message.case).toBe("kvClientMessage"); + if (reply.message.case !== "kvClientMessage") throw new Error("expected kvClientMessage"); + expect(reply.message.value.id).toBe(id); + expect(reply.message.value.message.case).toBe("getBlobResult"); + if (reply.message.value.message.case !== "getBlobResult") throw new Error("expected getBlobResult"); + expect(reply.message.value.message.value.blobData).toBeUndefined(); +} + function decodeRootMessages(bytes: Uint8Array): unknown[] { const msg = fromBinary(AgentClientMessageSchema, bytes); const run = msg.message.case === "runRequest" ? msg.message.value : undefined; @@ -657,3 +713,437 @@ describe("prepared request estimate (#373)", () => { ); }); }); + +describe("Cursor bounded blob store", () => { + const bytes = (value: string) => new TextEncoder().encode(value); + + test("admits a local blob exactly at the per-blob byte boundary", () => { + setCursorBlobLimitsForTests({ maxEntryBytes: 4, maxTotalBytes: 8, maxEntries: 2 }); + const first = storeCursorBlob(bytes("1234")); + const second = storeCursorBlob(bytes("5678")); + expectBlobHit(first, bytes("1234")); + expectBlobHit(second, bytes("5678")); + expect(cursorBlobRetainedStoreSnapshot()).toMatchObject({ count: 2, bytes: 8 }); + }); + + test("request construction one byte above the per-blob boundary fails before writing a request and returns no unstored hash", () => { + const serialized = bytes(JSON.stringify({ role: "system", content: "x" })); + setCursorBlobLimitsForTests({ maxEntryBytes: serialized.byteLength - 1, maxTotalBytes: 256 }); + expect(() => prepareCursorRunRequest({ + modelId: "composer-2.5", + conversationId: "oversized", + system: ["x"], + messages: [{ role: "user", content: "hi" }], + })).toThrow(CursorBlobAdmissionError); + expect(cursorBlobRetainedStoreSnapshot()).toEqual({ count: 0, bytes: 0, evictableBytes: 0, pinnedBytes: 0, oldestAt: null }); + }); + + test("request-scope pins preserve every advertised root turn and step until each distinct getBlob hydration completes", () => { + const prepared = prepareCursorRunRequest({ + modelId: "composer-2.5", + conversationId: "pins", + system: ["system"], + messages: [{ role: "user", content: "current" }], + rawMessages: [ + { role: "user", content: "prior", timestamp: 1 }, + { role: "assistant", model: "cursor/composer-2.5", content: [{ type: "text", text: "answer" }], timestamp: 2 }, + { role: "user", content: "current", timestamp: 3 }, + ], + }); + const message = fromBinary(AgentClientMessageSchema, prepared.bytes); + if (message.message.case !== "runRequest") throw new Error("expected runRequest"); + const roots = message.message.value.conversationState?.rootPromptMessagesJson ?? []; + const turns = message.message.value.conversationState?.turns ?? []; + expect(cursorBlobRetainedStoreSnapshot().pinnedBytes).toBeGreaterThan(0); + + for (const root of roots) hydrateBlob(root, prepared.blobRequestScope); + for (const turnId of turns) { + const turn = fromBinary(ConversationTurnStructureSchema, hydrateBlob(turnId, prepared.blobRequestScope)); + if (turn.turn.case !== "agentConversationTurn") throw new Error("expected agent turn"); + hydrateBlob(turn.turn.value.userMessage, prepared.blobRequestScope); + for (const step of turn.turn.value.steps) hydrateBlob(step, prepared.blobRequestScope); + } + expect(cursorBlobRetainedStoreSnapshot().pinnedBytes).toBe(0); + }); + + test("two concurrent streams sharing one blob id release only their own request pin", () => { + const data = bytes("shared"); + const firstScope = createCursorBlobRequestScope(); + const secondScope = createCursorBlobRequestScope(); + const id = storeCursorBlob(data, firstScope); + storeCursorBlob(data, secondScope); + sealCursorBlobRequestScope(firstScope); + sealCursorBlobRequestScope(secondScope); + expect(cursorBlobStoreDebugSnapshotForTests()[0]?.requestPins).toBe(2); + expectBlobHit(id, data, firstScope); + expect(cursorBlobStoreDebugSnapshotForTests()[0]?.requestPins).toBe(1); + expectBlobHit(id, data, secondScope); + expect(cursorBlobStoreDebugSnapshotForTests()[0]?.requestPins).toBe(0); + }); + + test("stream close error and abort release every remaining request-scope pin", () => { + const scope = createCursorBlobRequestScope(); + storeCursorBlob(bytes("a"), scope); + storeCursorBlob(bytes("b"), scope); + sealCursorBlobRequestScope(scope); + expect(cursorBlobRetainedStoreSnapshot().pinnedBytes).toBe(2); + releaseCursorBlobRequestScope(scope); + releaseCursorBlobRequestScope(scope); + expect(cursorBlobRetainedStoreSnapshot().pinnedBytes).toBe(0); + }); + + test("external root pruning stores and pins only selected candidates and cannot fail from discarded history bytes", () => { + const rawMessages = Array.from({ length: 210 }, (_, index) => index % 2 === 0 + ? { role: "user" as const, content: `user-${index}`, timestamp: index } + : { role: "assistant" as const, model: "cursor/gpt-5.6-sol", content: [{ type: "text" as const, text: `assistant-${index}` }], timestamp: index }); + rawMessages.push({ role: "user", content: "active", timestamp: 999 }); + const discarded = bytes(JSON.stringify({ role: "user", content: [{ type: "text", text: "user-0" }] })); + const discardedId = sha256(discarded); + const prepared = prepareCursorRunRequest({ + modelId: "gpt-5.6-sol-xhigh", + conversationId: "selected-only", + system: ["system"], + messages: [{ role: "user", content: "active" }], + rawMessages, + }); + expectBlobMiss(discardedId); + const message = fromBinary(AgentClientMessageSchema, prepared.bytes); + if (message.message.case !== "runRequest") throw new Error("expected runRequest"); + const selected = message.message.value.conversationState?.rootPromptMessagesJson ?? []; + expect(selected.length).toBeLessThanOrEqual(CURSOR_EXTERNAL_ROOT_BLOB_LIMIT); + expect(cursorBlobRetainedStoreSnapshot().pinnedBytes).toBeGreaterThan(0); + const selectedBytes = cursorBlobRetainedStoreSnapshot().bytes; + releaseCursorBlobRequestScope(prepared.blobRequestScope); + setCursorBlobLimitsForTests({ maxTotalBytes: selectedBytes, maxEntryBytes: 1024 * 1024 }); + expect(() => prepareCursorRunRequest({ + modelId: "gpt-5.6-sol-xhigh", + conversationId: "selected-only-repeat", + system: ["system"], + messages: [{ role: "user", content: "active" }], + rawMessages, + })).not.toThrow(); + }); + + test("replacement subtracts old bytes and refreshes local LRU", () => { + setCursorBlobLimitsForTests({ maxEntries: 2, maxEntryBytes: 8, maxTotalBytes: 8 }); + const a = storeCursorBlob(bytes("aaa")); + const b = storeCursorBlob(bytes("bbb")); + storeCursorBlob(bytes("aaa")); + const c = storeCursorBlob(bytes("cccc")); + expectBlobHit(a, bytes("aaa")); + expectBlobMiss(b); + expectBlobHit(c, bytes("cccc")); + expect(cursorBlobRetainedStoreSnapshot().bytes).toBe(7); + }); + + test("aggregate admission evicts oldest local-regenerated blobs first", () => { + setCursorBlobLimitsForTests({ maxEntries: 3, maxEntryBytes: 4, maxTotalBytes: 8 }); + const first = storeCursorBlob(bytes("1111")); + const second = storeCursorBlob(bytes("2222")); + const third = storeCursorBlob(bytes("3333")); + expectBlobMiss(first); + expectBlobHit(second, bytes("2222")); + expectBlobHit(third, bytes("3333")); + expect(cursorBlobRetainedStoreSnapshot().bytes).toBe(8); + }); + + test("remote setBlobArgs remains pinned within TTL while local blobs are evicted", () => { + setCursorBlobLimitsForTests({ maxEntries: 3, maxEntryBytes: 3, maxTotalBytes: 6 }); + const remoteId = sha256(bytes("rem")); + setBlobReply(remoteId, bytes("rem")); + const local = storeCursorBlob(bytes("loc")); + const newest = storeCursorBlob(bytes("new")); + expectBlobHit(remoteId, bytes("rem")); + expectBlobMiss(local); + expectBlobHit(newest, bytes("new")); + }); + + test("expired remote setBlobArgs becomes evictable before aggregate admission", () => { + setCursorBlobLimitsForTests({ ttlMs: 0, maxEntryBytes: 3, maxTotalBytes: 3 }); + const remoteId = sha256(bytes("rem")); + setBlobReply(remoteId, bytes("rem")); + storeCursorBlob(bytes("loc")); + expectBlobMiss(remoteId); + expect(cursorBlobRetainedStoreSnapshot()).toMatchObject({ count: 1, bytes: 3, evictableBytes: 3 }); + expect(cursorBlobStoreDebugSnapshotForTests()[0]?.provenance).toBe("local-regenerated"); + }); + + test("expired remote setBlobArgs is the reported oldestAt victim and budget eviction removes it before a younger local row", () => { + const originalNow = Date.now; + let now = 100; + Date.now = () => now; + try { + setCursorBlobLimitsForTests({ ttlMs: 10, maxEntryBytes: 3, maxTotalBytes: 6 }); + const scope = createCursorBlobRequestScope(); + const remoteId = sha256(bytes("rem")); + setBlobReply(remoteId, bytes("rem"), 1, scope); + sealCursorBlobRequestScope(scope); + now = 111; + const localId = storeCursorBlob(bytes("loc")); + now = 112; + releaseCursorBlobRequestScope(scope); + const snapshot = cursorBlobRetainedStoreSnapshot(); + expect(snapshot).toMatchObject({ bytes: 6, evictableBytes: 6, pinnedBytes: 0, oldestAt: 100 }); + expect(evictOldestCursorBlobForBudget()).toBe(3); + expectBlobMiss(remoteId); + expectBlobHit(localId, bytes("loc")); + } finally { + Date.now = originalNow; + } + }); + + test("pinned saturation returns typed SetBlobResult.error without exceeding aggregate bytes", () => { + setCursorBlobLimitsForTests({ maxEntryBytes: 3, maxTotalBytes: 3 }); + const scope = createCursorBlobRequestScope(); + storeCursorBlob(bytes("pin"), scope); + sealCursorBlobRequestScope(scope); + const rejectedId = sha256(bytes("new")); + const reply = setBlobReply(rejectedId, bytes("new"), 77); + expect(reply.message.case).toBe("kvClientMessage"); + if (reply.message.case !== "kvClientMessage") throw new Error("expected kvClientMessage"); + expect(reply.message.value.id).toBe(77); + expect(reply.message.value.message.case).toBe("setBlobResult"); + if (reply.message.value.message.case !== "setBlobResult") throw new Error("expected setBlobResult"); + expect(reply.message.value.message.value.error?.message).toContain("capacity"); + expectBlobMiss(rejectedId, 78); + expect(cursorBlobRetainedStoreSnapshot()).toMatchObject({ count: 1, bytes: 3, pinnedBytes: 3 }); + }); + + test("getBlob hit preserves the request id includes blobData and releases that key's request pin", () => { + const scope = createCursorBlobRequestScope(); + const id = storeCursorBlob(bytes("hit"), scope); + sealCursorBlobRequestScope(scope); + const hit = getBlobReply(id, 42, scope); + expect(hit.message.case).toBe("kvClientMessage"); + if (hit.message.case !== "kvClientMessage") throw new Error("expected kvClientMessage"); + expect(hit.message.value.id).toBe(42); + expect(hit.message.value.message.case).toBe("getBlobResult"); + if (hit.message.value.message.case !== "getBlobResult") throw new Error("expected getBlobResult"); + expect(hit.message.value.message.value.blobData).toBeDefined(); + expect(cursorBlobRetainedStoreSnapshot().pinnedBytes).toBe(0); + }); + + test("getBlob miss preserves the request id and emits getBlobResult with blobData omitted", () => { + expectBlobMiss(sha256(bytes("absent")), 43); + }); + + test("pinned-saturation get after rejected set uses the same omitted-blobData miss shape", () => { + setCursorBlobLimitsForTests({ maxEntryBytes: 3, maxTotalBytes: 3 }); + const scope = createCursorBlobRequestScope(); + storeCursorBlob(bytes("pin"), scope); + sealCursorBlobRequestScope(scope); + const rejectedId = sha256(bytes("new")); + const setReply = setBlobReply(rejectedId, bytes("new"), 77); + if (setReply.message.case !== "kvClientMessage" || setReply.message.value.message.case !== "setBlobResult") { + throw new Error("expected setBlobResult"); + } + expect(setReply.message.value.message.value.error).toBeDefined(); + expectBlobMiss(rejectedId, 78); + }); + + test("rejected same-key replacement preserves the admitted predecessor", () => { + const scope = createCursorBlobRequestScope(); + const oldData = bytes("old"); + const id = storeCursorBlob(oldData, scope); + sealCursorBlobRequestScope(scope); + const before = cursorBlobStoreDebugSnapshotForTests(); + const rejected = setBlobReply(id, bytes("new")); + if (rejected.message.case !== "kvClientMessage" || rejected.message.value.message.case !== "setBlobResult") throw new Error("expected set result"); + expect(rejected.message.value.message.value.error).toBeDefined(); + expect(cursorBlobStoreDebugSnapshotForTests()).toEqual(before); + expectBlobHit(id, oldData, scope); + }); + + test("atomic pinned-saturation rejection preserves unrelated TTL candidates local victims recency pins counters and same-key predecessor byte-for-byte", () => { + setCursorBlobLimitsForTests({ maxEntryBytes: 9, maxTotalBytes: 12 }); + const scope = createCursorBlobRequestScope(); + const pinned = storeCursorBlob(bytes("pin!"), scope); // 4 bytes, request-pinned + sealCursorBlobRequestScope(scope); + const remote = sha256(bytes("rm")); + setBlobReply(remote, bytes("rm")); // 2 bytes, live remote — 6/12 so far + const localVictim = storeCursorBlob(bytes("lv!")); // 3 bytes, live UNPINNED local (LRU victim class) + // Review C1-2: the rejected transaction must genuinely BUILD its victim + // view — an expired TTL candidate, a live local-LRU victim, AND a growing + // same-key predecessor — + // and then roll back completely. The expired row is inserted LAST: any + // successful insert commits its logical TTL removals, so an earlier + // expired insert would already be gone before the probed rejection. + const realNow = Date.now; + let expired: Uint8Array; + try { + Date.now = () => realNow() - 20 * 60_000; + expired = storeCursorBlob(bytes("exp")); // 3 bytes, expired under real clock + } finally { + Date.now = realNow; + } + const beforeRows = cursorBlobStoreDebugSnapshotForTests(); + const beforeStore = cursorBlobRetainedStoreSnapshot(); + const beforeMetrics = cursorBlobMetrics(); + // Growing same-key replacement (2 → 9 bytes, in-range per-entry): the + // logical victim view must select the expired row (3) AND the live local + // victim (3): 12-3-3-2+9 = 13 > 12 with only pinned mass left — + // pinned_saturation, typed wire error, and BOTH selected victims plus the + // predecessor must survive the rollback untouched. + const rejected = setBlobReply(remote, bytes("remremrem")); + if (rejected.message.case !== "kvClientMessage" || rejected.message.value.message.case !== "setBlobResult") { + throw new Error("expected set result"); + } + expect(rejected.message.value.message.value.error).toBeDefined(); + const afterRows = cursorBlobStoreDebugSnapshotForTests(); + const afterStore = cursorBlobRetainedStoreSnapshot(); + const afterMetrics = cursorBlobMetrics(); + // Complete deep comparison: content digests, exact pin identity (unique + // per-scope tokens), provenance, sizes, recency — byte-for-byte identical, + // including the expired TTL candidate (NOT committed-removed) and the + // same-key predecessor (2-byte original preserved). + expect(afterRows).toEqual(beforeRows); + expect(afterStore).toEqual(beforeStore); + // The ONLY permitted delta is the intentional rejection counter. + expect(afterMetrics).toEqual({ + ...beforeMetrics, + rejectedPinnedSaturation: beforeMetrics.rejectedPinnedSaturation + 1, + }); + expectBlobHit(pinned, bytes("pin!"), scope); + expectBlobHit(remote, bytes("rm")); + expectBlobHit(localVictim, bytes("lv!")); + const expiredKey = Buffer.from(expired).toString("hex"); + expect(cursorBlobStoreDebugSnapshotForTests().some(row => row.key === expiredKey)).toBe(true); + }); + + test("pinned-saturation rejection with an in-range entry preserves complete state including the expired candidate", () => { + setCursorBlobLimitsForTests({ maxEntryBytes: 4, maxTotalBytes: 9 }); + const scope = createCursorBlobRequestScope(); + storeCursorBlob(bytes("pin"), scope); + sealCursorBlobRequestScope(scope); + // Fill remaining budget with pinned rows so saturation is provable even + // after logical TTL removal of the expired row. + const scope2 = createCursorBlobRequestScope(); + storeCursorBlob(bytes("pn2"), scope2); + sealCursorBlobRequestScope(scope2); + // Insert the expired candidate LAST: any successful insert commits its + // logical TTL removals, so an earlier-inserted expired row would already + // be gone before the rejected transaction we are probing. + const realNow = Date.now; + let expired: Uint8Array; + try { + Date.now = () => realNow() - 20 * 60_000; + expired = storeCursorBlob(bytes("exp")); + } finally { + Date.now = realNow; + } + const beforeRows = cursorBlobStoreDebugSnapshotForTests(); + const beforeMetrics = cursorBlobMetrics(); + // 4-byte insert: even after logical TTL removal of the 3-byte expired row, + // pinned rows (3+3) + 4 = 10 > 9 — pinned saturation with the expired + // candidate genuinely in the victim view. + expect(() => storeCursorBlob(bytes("nw44"))).toThrow(CursorBlobAdmissionError); + expect(cursorBlobStoreDebugSnapshotForTests()).toEqual(beforeRows); + expect(cursorBlobMetrics()).toEqual({ + ...beforeMetrics, + rejectedPinnedSaturation: beforeMetrics.rejectedPinnedSaturation + 1, + }); + // The expired row was in the LOGICAL victim view but must not have been + // committed-removed by the failed transaction. + const expiredKey = Buffer.from(expired).toString("hex"); + expect(cursorBlobStoreDebugSnapshotForTests().some(row => row.key === expiredKey)).toBe(true); + }); + + test("a released scope token attached after final hydration cannot create a permanent pin", () => { + // Review C1-1: after the last advertised blob hydrates, the sealed scope's + // state is deleted; a late setBlobArgs carrying that stale token must not + // attach an untracked pin that survives terminal release. + const scope = createCursorBlobRequestScope(); + const advertised = storeCursorBlob(bytes("adv"), scope); + sealCursorBlobRequestScope(scope); + // Hydrate the only advertised key — the scope auto-releases. + getBlobReply(advertised, 1, scope); + // Late remote set carrying the stale token. + const late = sha256(bytes("lat")); + setBlobReply(late, bytes("lat"), 1, scope); + const lateKey = Buffer.from(late).toString("hex"); + const rows = cursorBlobStoreDebugSnapshotForTests(); + const lateRow = rows.find(row => row.key === lateKey); + expect(lateRow).toBeDefined(); + expect(lateRow!.requestPins).toBe(0); + // Terminal release is a no-op; the blob must remain TTL/budget-evictable. + releaseCursorBlobRequestScope(scope); + expect(cursorBlobStoreDebugSnapshotForTests().find(row => row.key === lateKey)!.requestPins).toBe(0); + }); + + test("one request whose construction crosses the aggregate cap fails coherently instead of emitting IDs evicted earlier in that request", () => { + setCursorBlobLimitsForTests({ maxEntryBytes: 1_024, maxTotalBytes: 150 }); + expect(() => prepareCursorRunRequest({ + modelId: "composer-2.5", + conversationId: "aggregate-request", + system: ["a".repeat(40), "b".repeat(40), "c".repeat(40)], + messages: [{ role: "user", content: "hi" }], + })).toThrow(CursorBlobAdmissionError); + const snapshot = cursorBlobRetainedStoreSnapshot(); + expect(snapshot.bytes).toBeLessThanOrEqual(150); + expect(snapshot.pinnedBytes).toBe(0); + }); + + test("cross-provenance refresh keeps or upgrades remote protection", () => { + setCursorBlobLimitsForTests({ maxEntryBytes: 3, maxTotalBytes: 6 }); + const remoteData = bytes("rem"); + const remoteId = sha256(remoteData); + setBlobReply(remoteId, remoteData); + storeCursorBlob(remoteData); // remote -> local refresh must not downgrade + const localVictim = storeCursorBlob(bytes("loc")); + storeCursorBlob(bytes("new")); + expectBlobHit(remoteId, remoteData); + expectBlobMiss(localVictim); + + setCursorBlobLimitsForTests({ maxEntryBytes: 3, maxTotalBytes: 6 }); + const upgradedData = bytes("upg"); + const upgradedId = storeCursorBlob(upgradedData); + setBlobReply(upgradedId, upgradedData); // local -> remote upgrade + const secondVictim = storeCursorBlob(bytes("old")); + storeCursorBlob(bytes("new")); + expectBlobHit(upgradedId, upgradedData); + expectBlobMiss(secondVictim); + }); + + test("remote-to-local same-key refresh keeps remote provenance and its TTL clock", () => { + const originalNow = Date.now; + let now = 1_000; + Date.now = () => now; + try { + setCursorBlobLimitsForTests({ ttlMs: 10, maxEntryBytes: 3, maxTotalBytes: 6 }); + const data = bytes("rem"); + const id = sha256(data); + setBlobReply(id, data); + now = 1_005; + storeCursorBlob(data); + expect(cursorBlobStoreDebugSnapshotForTests()[0]).toMatchObject({ + provenance: "remote-setBlobArgs", + storedAt: 1_000, + }); + } finally { + Date.now = originalNow; + } + }); + + test("local-to-remote same-key refresh upgrades to the stronger remote provenance", () => { + const data = bytes("upg"); + const id = storeCursorBlob(data); + setBlobReply(id, data); + expect(cursorBlobStoreDebugSnapshotForTests()[0]?.provenance).toBe("remote-setBlobArgs"); + }); + + test("blob metrics remain observe-only and exact after reset replacement and eviction", () => { + setCursorBlobLimitsForTests({ maxEntryBytes: 8, maxTotalBytes: 16 }); + const first = storeCursorBlob(bytes("one")); + storeCursorBlob(bytes("two2")); + storeCursorBlob(bytes("one")); + const before = cursorBlobRetainedStoreSnapshot(); + expect(cursorBlobRetainedStoreSnapshot()).toEqual(before); + expect(cursorBlobMetrics()).toMatchObject({ count: 2, totalBytes: 7, localBytes: 7, pinnedBytes: 0 }); + const released = evictOldestCursorBlobForBudget(); + expect(released).toBe(4); + expectBlobHit(first, bytes("one")); + expect(cursorBlobRetainedStoreSnapshot().bytes).toBe(3); + resetCursorBlobStateForTests(); + expect(cursorBlobMetrics()).toMatchObject({ count: 0, totalBytes: 0, localBytes: 0, pinnedBytes: 0 }); + }); +}); diff --git a/tests/cursor-live-transport.test.ts b/tests/cursor-live-transport.test.ts index f3efee05c..41f0e39b8 100644 --- a/tests/cursor-live-transport.test.ts +++ b/tests/cursor-live-transport.test.ts @@ -1,6 +1,12 @@ import { describe, expect, test } from "bun:test"; import { createLiveCursorTransport, CursorMissingCredentialError, parseConnectEndStreamError, resolveCursorToken } from "../src/adapters/cursor/live-transport"; import { prepareCursorRunRequest } from "../src/adapters/cursor/protobuf-request"; +import { + CursorBlobAdmissionError, + cursorBlobRetainedStoreSnapshot, + resetCursorBlobStateForTests, + setCursorBlobLimitsForTests, +} from "../src/adapters/cursor/native-exec"; describe("Cursor live transport", () => { test("fails before network when no Cursor credential is configured", () => { @@ -54,6 +60,80 @@ describe("Cursor live transport", () => { await expect(iterator.next()).rejects.toThrow("Cursor MCP preparation failed: fixture discovery failed"); await transport.close?.(); }); + + test("request construction one byte above the per-blob boundary fails before writing a request and returns no unstored hash", async () => { + resetCursorBlobStateForTests(); + const serialized = new TextEncoder().encode(JSON.stringify({ role: "system", content: "x" })); + setCursorBlobLimitsForTests({ maxEntryBytes: serialized.byteLength - 1, maxTotalBytes: 256 }); + const transport = createLiveCursorTransport({ + provider: { adapter: "cursor", baseUrl: "https://api2.cursor.sh", apiKey: "test-token" }, + headers: new Headers(), + }); + let opened = false; + (transport as unknown as { open(): void }).open = () => { opened = true; }; + const iterator = transport.run({ + modelId: "composer-2.5", + conversationId: "capacity-before-wire", + system: ["x"], + messages: [{ role: "user", content: "hi" }], + })[Symbol.asyncIterator](); + await expect(iterator.next()).rejects.toBeInstanceOf(CursorBlobAdmissionError); + expect(opened).toBe(false); + expect(cursorBlobRetainedStoreSnapshot().pinnedBytes).toBe(0); + transport.close?.(); + setCursorBlobLimitsForTests(); + }); + + test("stream close error and abort release every remaining request-scope pin", async () => { + type OpenFn = ( + encoded: Uint8Array, + signal: AbortSignal | undefined, + state: unknown, + push: unknown, + fail: (error: Error) => void, + finish: () => void, + ) => void; + const runCase = async (kind: "close" | "error" | "abort") => { + resetCursorBlobStateForTests(); + const transport = createLiveCursorTransport({ + provider: { adapter: "cursor", baseUrl: "https://api2.cursor.sh", apiKey: "test-token" }, + headers: new Headers(), + }); + let onOpened!: () => void; + const opened = new Promise(resolve => { onOpened = resolve; }); + let failTurn!: (error: Error) => void; + (transport as unknown as { open: OpenFn }).open = (_encoded, signal, _state, _push, fail) => { + failTurn = fail; + if (kind === "abort") signal?.addEventListener("abort", () => fail(new Error("aborted")), { once: true }); + onOpened(); + }; + const controller = new AbortController(); + const iterator = transport.run({ + modelId: "composer-2.5", + conversationId: `terminal-${kind}`, + system: ["system"], + messages: [{ role: "user", content: "hi" }], + }, controller.signal)[Symbol.asyncIterator](); + const pending = iterator.next(); + await opened; + expect(cursorBlobRetainedStoreSnapshot().pinnedBytes).toBeGreaterThan(0); + if (kind === "close") { + transport.close?.(); + failTurn(new Error("closed")); + } else if (kind === "abort") { + controller.abort(); + } else { + failTurn(new Error("stream error")); + } + await expect(pending).rejects.toBeInstanceOf(Error); + expect(cursorBlobRetainedStoreSnapshot().pinnedBytes).toBe(0); + transport.close?.(); + }; + + await runCase("close"); + await runCase("error"); + await runCase("abort"); + }); }); describe("Cursor end-stream classification", () => { diff --git a/tests/google-antigravity-replay.test.ts b/tests/google-antigravity-replay.test.ts index 67da01182..61b9a015f 100644 --- a/tests/google-antigravity-replay.test.ts +++ b/tests/google-antigravity-replay.test.ts @@ -1,14 +1,17 @@ import { afterEach, describe, expect, test } from "bun:test"; import { + antigravityReplayMetrics, + antigravityReplayRetainedStoreSnapshot, antigravityUsesReplayCache, applyAntigravityReplay, clearAntigravityReplay, + evictOldestAntigravityReplayForBudget, observeAntigravityReplay, - __resetAntigravityReplayCache, + setAntigravityReplayLimitsForTests, } from "../src/adapters/google-antigravity-replay"; import { sanitizeAntigravityClaudeSignatures } from "../src/adapters/google-antigravity-wire"; -afterEach(() => __resetAntigravityReplayCache()); +afterEach(() => setAntigravityReplayLimitsForTests()); const SIG = "sig-1234567890abcdef"; // >= 16 chars const MODEL = "gemini-3-pro"; @@ -107,6 +110,96 @@ describe("antigravity reasoning-replay cache", () => { applyAntigravityReplay("claude-opus-4.6", SESSION, contents); expect((contents[0].parts[0] as { thoughtSignature?: string }).thoughtSignature).toBeUndefined(); }); + + test("preserves 20+ live signed calls below count and byte caps", () => { + for (let i = 0; i < 24; i++) { + observeAntigravityReplay(MODEL, SESSION, [fcPart(`call-${i}`, { i }, `sig-${i}-${"x".repeat(20)}`)]); + } + expect(antigravityReplayMetrics()).toMatchObject({ sessions: 1, calls: 24 }); + const contents = Array.from({ length: 24 }, (_, i) => ({ role: "model", parts: [fcPart(`call-${i}`, { i })] })); + applyAntigravityReplay(MODEL, SESSION, contents); + expect(contents.every(c => typeof (c.parts[0] as { thoughtSignature?: string }).thoughtSignature === "string")).toBe(true); + }); + + test("evicts oldest inner call at the exact per-session count boundary", () => { + setAntigravityReplayLimitsForTests({ maxCallsPerSession: 2 }); + observeAntigravityReplay(MODEL, SESSION, [fcPart("one", {}, "sig-one-aaaaaaaaaaaa")]); + observeAntigravityReplay(MODEL, SESSION, [fcPart("two", {}, "sig-two-bbbbbbbbbbbb")]); + observeAntigravityReplay(MODEL, SESSION, [fcPart("three", {}, "sig-three-cccccccccc")]); + const contents = ["one", "two", "three"].map(name => ({ role: "model", parts: [fcPart(name, {})] })); + applyAntigravityReplay(MODEL, SESSION, contents); + expect((contents[0].parts[0] as { thoughtSignature?: string }).thoughtSignature).toBeUndefined(); + expect((contents[1].parts[0] as { thoughtSignature?: string }).thoughtSignature).toContain("sig-two"); + expect((contents[2].parts[0] as { thoughtSignature?: string }).thoughtSignature).toContain("sig-three"); + }); + + test("evicts oldest inner calls to satisfy aggregate session bytes", () => { + setAntigravityReplayLimitsForTests({ maxBytesPerSession: 100 }); + for (const name of ["one", "two", "three"]) { + observeAntigravityReplay(MODEL, SESSION, [fcPart(name, {}, `sig-${name}-${"x".repeat(32)}`)]); + } + const metrics = antigravityReplayMetrics(); + expect(metrics.totalBytes).toBeLessThanOrEqual(100); + expect(metrics.calls).toBe(2); + const contents = ["one", "two", "three"].map(name => ({ role: "model", parts: [fcPart(name, {})] })); + applyAntigravityReplay(MODEL, SESSION, contents); + expect((contents[0].parts[0] as { thoughtSignature?: string }).thoughtSignature).toBeUndefined(); + }); + + test("does not cache one oversized signature", () => { + setAntigravityReplayLimitsForTests({ maxSignatureBytes: 20 }); + observeAntigravityReplay(MODEL, SESSION, [fcPart("huge", {}, "x".repeat(21))]); + expect(antigravityReplayMetrics()).toEqual({ sessions: 0, calls: 0, totalBytes: 0, largestSessionBytes: 0 }); + }); + + test("apply refreshes matched call recency without extending session TTL", () => { + const originalNow = Date.now; + let now = 1_000; + Date.now = () => now; + try { + observeAntigravityReplay(MODEL, SESSION, [fcPart("one", {}, "sig-one-aaaaaaaaaaaa")]); + now = 1_001; + observeAntigravityReplay(MODEL, SESSION, [fcPart("two", {}, "sig-two-bbbbbbbbbbbb")]); + expect(antigravityReplayRetainedStoreSnapshot().oldestAt).toBe(1_000); + now = 1_002; + applyAntigravityReplay(MODEL, SESSION, [{ role: "model", parts: [fcPart("one", {})] }]); + expect(antigravityReplayRetainedStoreSnapshot().oldestAt).toBe(1_001); + now = 1_001 + 60 * 60 * 1000; + const expired = [{ role: "model", parts: [fcPart("one", {})] }]; + applyAntigravityReplay(MODEL, SESSION, expired); + expect((expired[0].parts[0] as { thoughtSignature?: string }).thoughtSignature).toBeUndefined(); + } finally { + Date.now = originalNow; + } + }); + + test("clear-on-invalid drops the bounded session and all byte accounting", () => { + observeAntigravityReplay(MODEL, SESSION, [fcPart("one", {}, "sig-one-aaaaaaaaaaaa")]); + expect(antigravityReplayMetrics().totalBytes).toBeGreaterThan(0); + clearAntigravityReplay(MODEL, SESSION); + expect(antigravityReplayMetrics()).toEqual({ sessions: 0, calls: 0, totalBytes: 0, largestSessionBytes: 0 }); + }); + + test("040 snapshot is observe-only and oldest-session eviction returns exact released bytes", () => { + const originalNow = Date.now; + let now = 10; + Date.now = () => now; + try { + observeAntigravityReplay(MODEL, "old", [fcPart("one", {}, "sig-one-aaaaaaaaaaaa")]); + now = 20; + observeAntigravityReplay(MODEL, "new", [fcPart("two", {}, "sig-two-bbbbbbbbbbbb")]); + const before = antigravityReplayRetainedStoreSnapshot(); + expect(antigravityReplayRetainedStoreSnapshot()).toEqual(before); + const released = evictOldestAntigravityReplayForBudget(); + expect(released).toBeGreaterThan(0); + expect(antigravityReplayRetainedStoreSnapshot().bytes).toBe(before.bytes - released); + const old = [{ role: "model", parts: [fcPart("one", {})] }]; + applyAntigravityReplay(MODEL, "old", old); + expect((old[0].parts[0] as { thoughtSignature?: string }).thoughtSignature).toBeUndefined(); + } finally { + Date.now = originalNow; + } + }); }); describe("claude-on-antigravity inline signature sanitization", () => { diff --git a/tests/vision-cache.test.ts b/tests/vision-cache.test.ts index c658727da..da8f4278f 100644 --- a/tests/vision-cache.test.ts +++ b/tests/vision-cache.test.ts @@ -7,10 +7,13 @@ import { parseRequest } from "../src/responses/parser"; import type { OcxConfig, OcxContentPart, OcxProviderConfig } from "../src/types"; import { describeImagesInPlace, + evictOldestVisionDescriptionForBudget, resetVisionDescriptionCache, resolveMaxDescriptionsPerTurn, setVisionDescriptionCache, + setVisionDescriptionCacheLimitsForTests, shouldResolveOpenAiVisionSidecar, + visionDescriptionRetainedStoreSnapshot, type VisionPlan, } from "../src/vision"; @@ -119,6 +122,7 @@ describe("vision description cache and per-turn cap", () => { afterEach(() => { globalThis.fetch = originalFetch; + setVisionDescriptionCacheLimitsForTests(); setVisionDescriptionCache(); }); @@ -187,6 +191,19 @@ describe("vision description cache and per-turn cap", () => { expect(calls).toBe(4); }); + test("error outcome reaches the caller unchanged and does not mutate the cache", async () => { + const before = visionDescriptionRetainedStoreSnapshot(); + globalThis.fetch = (async () => new Response("preserve this exact detail", { status: 503 })) as typeof fetch; + const request = parsed([{ type: "input_image", image_url: DATA_A }]); + + await describeImagesInPlace(request, plan(), new Headers({ authorization: "Bearer test" })); + + expect(textParts(request)).toEqual([ + "[An image was attached but could not be processed: vision sidecar HTTP 503: preserve this exact detail]", + ]); + expect(visionDescriptionRetainedStoreSnapshot()).toEqual(before); + }); + test("interleaves hits, misses, and over-cap markers without changing message or part order", async () => { globalThis.fetch = (async (_url, init) => openaiSse(imageCaption(JSON.parse(String(init?.body))))) as typeof fetch; const headers = new Headers({ authorization: "Bearer test" }); @@ -250,4 +267,77 @@ describe("vision description cache and per-turn cap", () => { expect(calls).toBe(5); }); + + test("clamps a successful description before cache insertion and first render", async () => { + let cached = ""; + setVisionDescriptionCache({ + get: () => undefined, + set: (_key, value) => { cached = value; }, + clear: () => {}, + }); + globalThis.fetch = (async () => openaiSse("x".repeat(2_100))) as typeof fetch; + const request = parsed([{ type: "input_image", image_url: DATA_A }]); + await describeImagesInPlace(request, plan(), new Headers({ authorization: "Bearer test" })); + const expected = `${"x".repeat(2_000)}\n…[description truncated]`; + expect(cached).toBe(expected); + expect(textParts(request).join("\n")).toContain(expected); + }); + + test("cache hit returns the same clamped description without a sidecar call", async () => { + let calls = 0; + globalThis.fetch = (async () => { calls++; return openaiSse("y".repeat(2_100)); }) as typeof fetch; + const first = parsed([{ type: "input_image", image_url: DATA_A }]); + await describeImagesInPlace(first, plan(), new Headers({ authorization: "Bearer test" })); + const second = parsed([{ type: "input_image", image_url: DATA_A }]); + await describeImagesInPlace(second, plan(), new Headers({ authorization: "Bearer test" })); + expect(calls).toBe(1); + expect(textParts(second)).toEqual(textParts(first)); + }); + + test("test-only limits make a successful clamped value larger than maxBytes observable but not retained", async () => { + setVisionDescriptionCacheLimitsForTests({ maxBytes: 1 }); + globalThis.fetch = (async () => openaiSse("observable")) as typeof fetch; + const request = parsed([{ type: "input_image", image_url: DATA_A }]); + await describeImagesInPlace(request, plan(), new Headers({ authorization: "Bearer test" })); + expect(textParts(request).join("\n")).toContain("observable"); + expect(visionDescriptionRetainedStoreSnapshot()).toEqual({ + count: 0, bytes: 0, evictableBytes: 0, pinnedBytes: 0, oldestAt: null, + }); + }); + + test("multiple entries fit exactly at the aggregate byte boundary and the next byte evicts the oldest before insert", async () => { + const headers = new Headers({ authorization: "Bearer test" }); + globalThis.fetch = (async () => openaiSse("v")) as typeof fetch; + await describeImagesInPlace(parsed([{ type: "input_image", image_url: DATA_A }]), plan(), headers); + const oneEntryBytes = visionDescriptionRetainedStoreSnapshot().bytes; + + setVisionDescriptionCacheLimitsForTests({ maxEntries: 3, maxBytes: oneEntryBytes * 2 }); + let calls = 0; + globalThis.fetch = (async (_url, init) => { + calls++; + const caption = imageCaption(JSON.parse(String(init?.body))); + return openaiSse(caption === "caption-c" ? "vv" : "v"); + }) as typeof fetch; + await describeImagesInPlace(parsed([{ type: "input_image", image_url: DATA_A }]), plan(), headers); + await describeImagesInPlace(parsed([{ type: "input_image", image_url: DATA_B }]), plan(), headers); + expect(visionDescriptionRetainedStoreSnapshot().bytes).toBe(oneEntryBytes * 2); + await describeImagesInPlace(parsed([{ type: "input_image", image_url: DATA_C }]), plan(), headers); + const after = visionDescriptionRetainedStoreSnapshot(); + expect(after.count).toBe(1); + expect(after.bytes).toBeLessThanOrEqual(oneEntryBytes * 2); + await describeImagesInPlace(parsed([{ type: "input_image", image_url: DATA_A }]), plan(), headers); + expect(calls).toBe(4); + }); + + test("040 snapshot is observe-only and oldest-entry eviction returns exact released bytes", async () => { + globalThis.fetch = (async () => openaiSse("snapshot")) as typeof fetch; + const headers = new Headers({ authorization: "Bearer test" }); + await describeImagesInPlace(parsed([{ type: "input_image", image_url: DATA_A }]), plan(), headers); + await describeImagesInPlace(parsed([{ type: "input_image", image_url: DATA_B }]), plan(), headers); + const before = visionDescriptionRetainedStoreSnapshot(); + expect(visionDescriptionRetainedStoreSnapshot()).toEqual(before); + const released = evictOldestVisionDescriptionForBudget(); + expect(released).toBeGreaterThan(0); + expect(visionDescriptionRetainedStoreSnapshot().bytes).toBe(before.bytes - released); + }); }); From becad999f54bb3375d5a1e969d3a360c6b23368b Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sat, 1 Aug 2026 10:42:38 +0900 Subject: [PATCH 09/30] docs(plan): fence eviction against stale writers and prune by combo target wp4 A-gate (Faraday): GenerationContext gains canonical comboTargets so partial-target removal is prunable, every affected store write path carries a generation fence so late completions cannot resurrect deleted keys, and the PID memo moves to liveness-proof cleanup with a bounded probe budget. --- .../006_roadmap_audit_synthesis.md | 10 + .../030_eviction_mechanisms.md | 187 ++++++++++++++++-- 2 files changed, 178 insertions(+), 19 deletions(-) diff --git a/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md b/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md index c9cf9a0a5..381c2c7b2 100644 --- a/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md +++ b/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md @@ -73,3 +73,13 @@ amended accordingly (failure ladder + admission ladder). | AV-2 | Same-key cross-provenance replacement could downgrade a live remote pin to evictable local | ACCEPT | Admission step 1 now merges provenance to the STRONGER class (remote wins; TTL clock kept on remote→local refresh, upgrade on local→remote); remote→local and local→remote regression tests named. | | AV-3 | Oversized-vision-entry test unreachable (2000-char clamp < 1 MiB cap) | ACCEPT | setVisionDescriptionCacheLimitsForTests() override (production limits restored on undefined) + small-limit fixtures replace the unreachable case. | | AV-4 | Vision clamp snippet left the first-use outcome unclamped, contradicting the byte-identical contract | ACCEPT | Snippet extended: successful outcome is replaced with the clamped text BEFORE resolveOutcome; regression asserts cache value and first-use outcome identical, error outcomes untouched. | + +## wp4 A-gate (freshness auditor, FAIL 3) — adjudication + +| # | Blocker | Decision | Action | +|---|---|---|---| +| WP4-A1 | `GenerationContext.comboIds` cannot reconcile deleted target weights inside a still-live combo; main-Codex and OAuth account-key inclusion/encoding are also undefined | ACCEPT | Added canonical `comboTargets` (`comboId::provider/model`), raw Codex ids with `__main__`, OAuth `provider\0accountId`, partial-target pruning, and regression coverage. | +| WP4-A2 | Post-commit reconciliation has no stale-writer fence, so accepted old-generation flights/requests can resurrect provider-quota, GCP, guardian, routing, combo, or reauth rows | ACCEPT | Added captured writer generation versus per-owner last-reconciled generation at every cited write site; stale deleted-key writes drop, live-key writes remain accepted, and late-completion regressions are required. | +| WP4-A3 | PID memo eviction requires process identity/liveness proof, but 030 names neither an owner export/call site nor its regressions | ACCEPT | Reclassified PID memos to `sweepDeadOcxStartProcessCache(64)`: timer-only round-robin probing, delete only on `process.kill(pid, 0)` `ESRCH`, with live/EPERM/unknown preservation and bounded-cost regressions. | + +All three accepted and incorporated into 030. Re-audit verdict: **PASS**. diff --git a/devlog/_plan/260801_zero_leak_state_stores/030_eviction_mechanisms.md b/devlog/_plan/260801_zero_leak_state_stores/030_eviction_mechanisms.md index 1c9088afc..5c2642b20 100644 --- a/devlog/_plan/260801_zero_leak_state_stores/030_eviction_mechanisms.md +++ b/devlog/_plan/260801_zero_leak_state_stores/030_eviction_mechanisms.md @@ -28,6 +28,7 @@ export interface GenerationContext { generation: number; providerNames: ReadonlySet; comboIds: ReadonlySet; + comboTargets: ReadonlySet; codexAccountIds: ReadonlySet; oauthAccountKeys: ReadonlySet; configRoots: ReadonlySet; @@ -35,6 +36,7 @@ export interface GenerationContext { export interface StateStoreRegistration { name: string; sweepExpired?: (now: number) => number; + sweepLiveness?: () => number; reconcileGeneration?: (context: GenerationContext) => number; } export interface StateSweepResult { storesVisited: number; rowsRemoved: number } @@ -42,23 +44,117 @@ export interface StateSweepResult { storesVisited: number; rowsRemoved: number } export function registerStateStore(registration: StateStoreRegistration): () => void; export function sweepExpired(now?: number): StateSweepResult; export function sweepExpiredOnWrite(now?: number): StateSweepResult; +export function sweepLiveness(): StateSweepResult; +export function captureConfigGeneration(): number; +export function setGenerationContextBuilder(build: () => GenerationContext): void; export function reconcileStateGeneration(context: GenerationContext): StateSweepResult; export function startStateStoreSweeper(options?: StateStoreSweeperOptions): { stop(): void }; export function stopStateStoreSweeper(): void; + +export interface StateStoreSweeperOptions { + /** Override the 60 s interval (tests only; production uses the default). */ + intervalMs?: number; + /** Injectable clock for tests. */ + now?: () => number; +} ``` Registration names are static and unique; replacement does not duplicate callbacks. `startStateStoreSweeper()` replaces the prior singleton, creates one 60-second interval, -and invokes `unref?.()`. One callback failure logs only the static registration name and -does not stop later callbacks. `sweepExpiredOnWrite()` runs synchronously after a -successful owner write; it creates no promise tail. Reconciliation never runs from the -clock timer. - -Start the singleton beside the watchdog in `src/server/index.ts:300-316`; stop it in +and invokes `unref?.()`. Each timer tick runs exact-TTL callbacks followed by bounded +owner-specific liveness callbacks. One callback failure logs only the static registration +name and does not stop later callbacks. `sweepExpiredOnWrite()` runs only TTL callbacks, +synchronously after a successful owner write; it creates no promise tail and performs no +PID probes. Reconciliation never runs from the clock timer. + +Start the singleton beside the watchdog in `src/server/index.ts:300-316` (the current +watchdog call is `startMemoryWatchdog()` at `:306`); stop it in `drainAndShutdown()` at `src/server/lifecycle.ts:70-95`. If side-effect registration would create an import cycle, add explicit wiring in NEW `src/lib/state-store-registrations.ts`; do not add a generic helper module. +### Generation protocol and stale-writer fence (wp4 A-gate B1/B2 — locked) + +- **Canonical owner:** the sweeper module owns a single monotonic + `configGeneration` counter, initial value 0 (pre-first-reconcile state is + generation 0 and is never reconciled against — a reconcile requires a + COMPLETE context). `captureConfigGeneration()` returns the generation a store + writer carries from key selection/flight admission through completion. The + "config generation beside the loaded-config owner" + wording in the locked table above is SUPERSEDED by this section (round-2 + blocker 4): there is exactly ONE counter and the sweeper owns it; config.ts + merely calls the trigger. +- **Context construction:** `buildGenerationContext()` lives in + `state-store-registrations.ts` and reads each owner's CURRENT authoritative + set through owner exports. Exact canonical forms are locked: + - `providerNames` is the exact own-property names in `config.providers`; + - `comboIds` is the exact own-property ids in `config.combos`; + - `comboTargets` is `${comboId}::${targetKey(target)}`, where + `targetKey(target)` is exactly `src/combos/types.ts:48-50`'s + `${target.provider}/${target.model}`. Thus target `a/m1` in combo `free` + is `free::a/m1`; validated ids/providers/models cannot make `::` ambiguous; + - `codexAccountIds` contains raw stable account ids. Added accounts retain + their configured ids; the main account uses the literal + `MAIN_CODEX_ACCOUNT_ID` value `__main__` whenever the canonical OpenAI + forward provider remains configured and enabled, including while it needs + reauth, so safety state is not mistaken for a deleted account; + - `oauthAccountKeys` is `${provider}\0${accountId}`, exactly matching + `src/providers/quota.ts:300-302`. Guardian's + `oauth:${provider}:${accountId}` and other owner-local strings are parsed + by their owner and compared as this canonical pair; the coordinator never + manufactures owner-local keys; + - `configRoots` contains canonical roots supplied by config-ownership. + The combos owner supplies `comboTargets` via a typed topology export; other + owners likewise expose typed live-set exports rather than private map keys. + All reads take the SERVER's long-lived config + object (the `1256/1649` doctrine: an ad-hoc `loadConfig()` snapshot must + not be used — absence in a transient snapshot does not prove not-live; + round-2 blocker 5). The context is built ONCE per reconcile, synchronously + in a single microtask after the triggering commit has REPLACED the server + config, so every owner read observes the same committed state, then stamped + with the next `configGeneration`. +- **Trigger sites (exact):** successful config load/commit + (`loadConfig`/`saveConfig` success paths in config.ts), account + add/remove/reauth commit sites (the locked table's list governs — "rename" + above was shorthand for the same account-mutation commits), and catalog + replacement. "Successful load" means every load that REPLACES the server + config object, including the initial one. Each calls + `reconcileStateGeneration(buildGenerationContext())`. Never from the timer. +- **Failure semantics:** each successful owner atomically records its current + live-key set/topology and `lastReconciledGeneration`. If a callback throws, + log the static registration name, leave that owner on its prior generation, + and schedule ONE retry on the next `sweepExpiredOnWrite` tick (retry uses a + freshly built context — stale contexts are never reused). + To make that retry executable, the sweeper stores the CONTEXT BUILDER, not + a context: `setGenerationContextBuilder(build: () => GenerationContext)` is + called once from `state-store-registrations.ts`; the retry invokes it + fresh (round-2 blocker 4). + A reconciliation attempt stamps its complete context with + `candidateGeneration = configGeneration + 1`. Successful owners may retain + that candidate stamp; the global `configGeneration` advances to it only when + every callback succeeds. On partial failure, writers captured from the still- + current global generation compare stale against any owner already stamped + with the candidate, and the fresh retry completes the remaining owners + without reusing an old context. + Partial reconciliation is deletion-only: a failed callback leaves extra + rows but never deletes a live key. +- **Combo topology reconciliation:** remove a whole selection row when its + `comboId` is absent. For a still-live combo, remove each `currentWeights` + member whose `${comboId}::${targetKey}` is absent from `comboTargets`, while + preserving surviving weights, target order, and the active target/successes + when that active target remains live. +- **Late-completion fence (round-2 blocker 2):** each affected writer captures + `writerGeneration` before work begins. At retained-state commit, + `writerGeneration < lastReconciledGeneration` is stale. A stale write is + accepted when its exact key remains in the owner's current live set, but is + DROPPED when absent; old work cannot resurrect a deleted key, while live-key + writes retain current behavior. The fence never detaches or cancels an + accepted request/flight: it gates only its completion write. +- **Registration timing:** owner modules register at module scope through + `state-store-registrations.ts`, which is imported by `server/index.ts` + before `startStateStoreSweeper()`; a registration arriving after a + reconcile simply joins the next one (no catch-up replay). + ## Locked store-by-store table The inventory shorthand `catalog/*` resolves to `src/codex/catalog/*` in this checkout. @@ -72,27 +168,54 @@ The inventory shorthand `catalog/*` resolves to `src/codex/catalog/*` in this ch | XAI permanent-failure verdicts, `src/oauth/index.ts:54-61,326-327` | TTL sweep | Globally delete the same 30-second-expired verdicts that `cached()` already treats as absent; S3-2 forbids reconciliation-only handling. | | Warning memos, `src/codex/catalog/provider-fetch.ts:219`, `src/codex/catalog/aggregation.ts:37-39,281-283`, `src/router.ts:154-180`, `src/combos/request.ts:4-38`, `src/config.ts:435,2052-2053` | Reconciliation only | Clear/rebuild only after a complete new config generation. No TTL: time expiry would re-emit intentionally suppressed warnings. | | Codex quota, `src/codex/quota.ts:15-27,51,261-326` | Reconciliation only | Remove accounts absent from current account generation and persist the reduced map. The 6 h rule is hydration admission, not live expiry. | -| Provider quota history, `src/providers/quota.ts:52-61,195-239,278-342` | Reconciliation plus admission | Remove dead provider/account keys; cap distinct live flights in 035; never detach an accepted flight. | -| Codex routing health, `src/codex/routing.ts:85-138,209-243` | Reconciliation | Remove account-wide and quota-scope rows only for deleted accounts; preserve live Retry-After and probe generation. | +| Provider quota history, `src/providers/quota.ts:52-61,195-239,278-342` | Reconciliation plus admission | Remove dead provider/account keys; cap distinct live flights in 035; never detach an accepted flight. Capture generation at probe admission and fence retained cache writes at `:259,324,389-401`; a late deleted-account result returns to its caller but is not retained. | +| Codex routing health, `src/codex/routing.ts:85-138,209-243` | Reconciliation | Remove account-wide and quota-scope rows only for deleted accounts; preserve live Retry-After and probe generation. Capture generation with routed account selection and fence health writes at `:230-237,349-562,1225-1404`; late outcomes for a deleted account do not recreate health. | | Model-cache history, `src/codex/model-cache.ts:42-56,114-147` | Reconciliation | Add `reconcileModelCacheProviders(validProviders)` covering cache, failure, status, and live-count maps atomically. | | Pool rotation, `src/codex/pool-rotation.ts:6-12,44-80,180-185` | Reconciliation | Remove deleted pool/account rows while preserving current sticky/RR weights. | -| Combo rotation, `src/combos/resolve.ts:13-20,85-105,161-167` | Reconciliation | Remove deleted combo ids and target weights; preserve current target order. | -| Guardian backoff, `src/oauth/token-guardian.ts:54-99,118-225` | Reconciliation | Remove deleted provider/account rows; keep a configured revoked account at its current backoff. | -| Reauth state, `src/codex/account-runtime-state.ts:1-13` plus OAuth account maps | Reconciliation | Remove only keys absent after a successful persisted account mutation. | -| GCP ADC, `src/lib/gcp-adc.ts:61-66,83-127,274-302` | Reconciliation plus admission | Remove source fingerprints absent from the authoritative ADC source set; cap active source flights in 035; retain current expiry-on-resolve. | +| Combo rotation, `src/combos/resolve.ts:13-20,85-105,161-167` | Reconciliation | Remove deleted combo ids and `currentWeights` targets absent from `comboTargets`; preserve current target order and surviving sticky/RR state. Capture generation at pick/state creation and fence writes at `:54-65,85-105,120-167`. | +| Guardian backoff, `src/oauth/token-guardian.ts:54-99,118-225` | Reconciliation | Remove deleted provider/account rows; keep a configured revoked account at its current backoff. Capture generation when each task is admitted and fence retained backoff writes at `:90-99,137-148,186-218`. | +| Reauth state, `src/codex/account-runtime-state.ts:1-13` plus OAuth account maps | Reconciliation | Remove only keys absent after a successful persisted account mutation. Auth work captures generation before dispatch; fence `markAccountNeedsReauth` at `account-runtime-state.ts:3-5` and equivalent OAuth writes so late failures for deleted accounts are dropped. | +| GCP ADC, `src/lib/gcp-adc.ts:61-66,83-127,274-302` | Reconciliation plus admission | Remove source fingerprints absent from the authoritative ADC source set; cap active source flights in 035; retain current expiry-on-resolve. Capture generation with `expectedSource` and fence `tokenCache.set(source, ...)` at `:273-295`; an old-source flight still resolves to its caller but is not retained. | | Config ownership, `src/lib/config-ownership.ts:79-87,233-258` | Reconciliation | Remove a root only when absent from manifest/current roots and no owned path references it. Never infer inactivity from age. | -| PID and config warnings, `src/config.ts:413-435,2052-2053,2112-2121` | Reconciliation | Delete PID rows only after failed identity/liveness proof; warning rows follow config generation. | +| Config warnings, `src/config.ts:435,2052-2053` | Reconciliation only | Warning rows follow complete config generation; never TTL-expire them. | +| PID command-line memo, `src/config.ts:2112-2121` | Liveness-based owner release | Export/register `sweepDeadOcxStartProcessCache(maxProbes = 64)`. Each timer tick probes at most 64 cached PIDs with `process.kill(pid, 0)`: delete only on `ESRCH`; retain on success, `EPERM`, or unknown errors. Advance a round-robin cursor over a key snapshot so maps larger than 64 receive eventual coverage. Never config-reconcile this owner and never probe it from write-trigger sweeps. | | OAuth login/abort/manual maps, `src/oauth/index.ts:823-904,939-1020` | Reconciliation plus admission | Remove dead provider/account generations; 035 caps flows/probes and pending-code bytes; never evict an in-progress owner. | | Active turns/sockets/workers/slots | Admission only | Implement the hard caps and coherent busy responses in 035; do not register an accepted owner and later sweep it away. | Each owner exports a semantic sweep/reconcile function. The coordinator passes complete -sets; it must not recreate owner key strings. Add a monotonic config generation beside -the canonical loaded-config owner, and call reconciliation only after successful initial -load, successful disk commit, account add/delete/reauth commit, or catalog sync with the -complete provider/combo set. Failed parse/save and speculative routing do not advance it. +sets; it must not recreate owner-local key strings. The single `configGeneration` counter +lives in the sweeper (§Generation protocol supersedes earlier drafts of this paragraph); +config.ts triggers reconciliation beside the canonical loaded-config owner. Call it only +after successful initial load, successful disk commit, account add/delete/reauth commit, +or catalog sync with the complete provider/combo topology. Failed parse/save and +speculative routing do not advance it. ## Windows ACL delete-after-rename contract +### Coverage expansion (wp4 A-gate blocker 1 — every hardened temp producer) + +The two config-writer sites are NOT the only producers of hardened ephemeral +temp paths. Every producer below hardens a unique temp and must forget its +memo entry only after a CONFIRMED unlink/rename; a failed removal retains the +memo (fail-closed): + +| Producer | Anchors | Temp lifecycle | +|---|---|---| +| Config atomic writers | config.ts:96-113, 174-197 | temp → rename to config | +| OpenAI migration backups | config.ts:331, 368, 380 | backup temps | +| Response spill store (NEW in wp2) | spill-store.ts:231, 237 (harden), 248 (unlink) | per-spill temp | +| Tray atomic replacement | tray/windows.ts:222, 227, 230 | icon/state temp | +| Management-token publication | server/management-auth.ts:82, 92, 111 | token temp | + +Implementation shape: the single `forgetHardenedSecretPath(path)` export specified below in +`windows-secret-acl.ts` invoked by each producer's cleanup path (post-rename +and post-unlink), so the memo store stays bounded by the number of LIVE temps +rather than growing one entry per write. Regression coverage per producer: +spill, tray, backup, and management-auth each prove the memo entry disappears +after the temp is gone and is RETAINED when removal fails. wp2/wp4 are +file-disjoint but share this store — the spill-store change is a one-line +cleanup-hook call, inside wp4's write set by this amendment. + Current anchors: - `src/lib/windows-secret-acl.ts:37-40` retains directory/file success paths and timeout keys. @@ -123,8 +246,11 @@ destination to `hardenedPaths`, never reuse one temp's success for the next temp never clear `timedOutPaths[destination]` on rename. This code protects credential/config files and requires explicit security review under -`AGENTS.md`/`MAINTAINERS.md`. The security reviewer must specifically attest that the -second temp for one destination executes `icacls` again and failure remains fail-closed. +`AGENTS.md`/`MAINTAINERS.md`. The security reviewer must specifically attest that, after +a prior **successful** temp harden and rename, the second temp for one destination executes +`icacls` again and real permission/`icacls` failures remain fail-closed. Preserve the +existing timeout-only exception: a destination-keyed timeout memo may skip `icacls` for a +later temp and returns `ok:false`; it must never enter or masquerade as the success memo. ## Diagnostic values: classification, not a clock sweep @@ -149,6 +275,9 @@ Add `tests/state-store-sweeper.test.ts`: - `timer start is singleton unrefed and stop is idempotent` - `reconciliation runs only for a newer complete generation` - `stale or duplicate generation cannot delete current keys` +- `live combo keeps surviving weights while a removed target weight is reconciled` +- `late old-generation completion cannot resurrect a deleted key but can update a still-live key` +- `PID liveness sweep probes at most 64 rotates coverage keeps live EPERM and unknown rows and deletes only ESRCH` Extend nearest owner suites with: @@ -159,6 +288,7 @@ Extend nearest owner suites with: - `Codex quota reconciliation ignores row age and removes only deleted accounts` - `model-cache reconciliation removes all four maps for one deleted provider` - `pool combo guardian GCP ownership and reauth reconciliation preserve current keys` +- `provider quota GCP guardian routing combo and reauth late completions obey the generation fence` Extend `tests/windows-secret-acl.test.ts` and the atomic-writer cases in `tests/config.test.ts`: @@ -166,6 +296,7 @@ Extend `tests/windows-secret-acl.test.ts` and the atomic-writer cases in - `second atomic temp for the same destination is hardened again` - `rename success forgets only the actual temp success memo` - `destination timeout memo never vouches for a new temp` +- `destination timeout memo preserves the existing ok:false no-retry contract` - `failed unlink retains the residual temp success memo` - `later successful residual cleanup forgets that exact memo`. @@ -186,3 +317,21 @@ bun run privacy:scan - No process-wide accounting interface; 030 provides sweeping only. - No provider event semantics, `#820` scheduler architecture, destination-level ACL success memo, or fail-open hardening behavior. + +## wp4 A-gate freshness audit (HEAD `0d1fa7dd5`, 2026-08-01) + +All listed store anchors remain current after wp2/wp3; no landing moved or redefined the +named maps. The server start/stop seams and both atomic writers also remain at the anchors +above. The Windows wording/test list now distinguishes a forgotten success memo from the +existing destination-keyed timeout-only `ok:false` memo. + +All three blockers are accepted and resolved in the locked design above: + +1. `comboTargets` carries exact `comboId::provider/model` topology, with canonical + main-Codex and OAuth account-key forms; partial target removal is executable. +2. Per-owner `lastReconciledGeneration` plus captured writer generations prevent old + completions from resurrecting deleted keys while preserving accepted flights and live keys. +3. `ocxStartProcessCache` is owner-released by a bounded round-robin liveness sweep that + deletes only `process.kill(pid, 0)` `ESRCH` proofs and is outside config reconciliation. + +**wp4 A-gate verdict: PASS.** From c7af1dcc8b53e4e78adbf338ba3b93eb9736106f Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sat, 1 Aug 2026 10:47:58 +0900 Subject: [PATCH 10/30] docs(plan): give reconciliation unique attempt generations and live-object triggers Six audit rounds on the eviction design: candidate generations come from a never-reused attempt counter so a failed attempt cannot strand an owner on an obsolete live set; triggers are the three real lifecycle shapes (startup capture, in-place management commits, OAuth live adoption) and saveConfig alone never reconciles; ACL memo coverage now spans all five hardened-temp producers. --- .../030_eviction_mechanisms.md | 66 +++++++++++++------ 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/devlog/_plan/260801_zero_leak_state_stores/030_eviction_mechanisms.md b/devlog/_plan/260801_zero_leak_state_stores/030_eviction_mechanisms.md index 5c2642b20..02728fd64 100644 --- a/devlog/_plan/260801_zero_leak_state_stores/030_eviction_mechanisms.md +++ b/devlog/_plan/260801_zero_leak_state_stores/030_eviction_mechanisms.md @@ -110,16 +110,34 @@ would create an import cycle, add explicit wiring in NEW object (the `1256/1649` doctrine: an ad-hoc `loadConfig()` snapshot must not be used — absence in a transient snapshot does not prove not-live; round-2 blocker 5). The context is built ONCE per reconcile, synchronously - in a single microtask after the triggering commit has REPLACED the server - config, so every owner read observes the same committed state, then stamped - with the next `configGeneration`. -- **Trigger sites (exact):** successful config load/commit - (`loadConfig`/`saveConfig` success paths in config.ts), account - add/remove/reauth commit sites (the locked table's list governs — "rename" - above was shorthand for the same account-mutation commits), and catalog - replacement. "Successful load" means every load that REPLACES the server - config object, including the initial one. Each calls - `reconcileStateGeneration(buildGenerationContext())`. Never from the timer. + in a single microtask after the triggering commit is VISIBLE ON the live + server config object (whether the trigger replaced it or mutated it in + place — see the three trigger shapes below), so every owner read observes + the same committed state, then stamped with the attempt's unique + `candidateGeneration` (see §Failure semantics — `++attemptSequence`, not + `configGeneration + 1`). +- **Trigger sites (exact, round-4 blocker 5):** the trigger is "the LIVE + server config object now reflects a committed topology change", which in + this codebase happens through THREE distinct shapes, each of which calls + `reconcileStateGeneration(buildGenerationContext())` AFTER its mutation is + visible on the live object: + 1. Initial server startup, after `serverConfig` is captured + (server/index.ts:256) and registrations are wired. + 2. Management routes that mutate the live object IN PLACE (combo/provider/ + account routes, e.g. combo-routes.ts:133/207): the route handler calls + the trigger after its in-place mutation + successful `saveConfig()`. + `saveConfig()` itself is NOT a trigger — it merely persists its argument + (config.ts:1543) and cannot know whether that argument is the live + object or an ad-hoc snapshot. + 3. OAuth flows that load-modify-save a SEPARATE snapshot (oauth/index.ts: + 765): these are NOT triggers at save time. Their changes become + reconciliation-relevant only when adopted into live server state, which + happens through the existing `reconcileOAuthProviders`/account-refresh + path on the live object — THAT adoption site calls the trigger. An + ad-hoc snapshot save with no live adoption never reconciles (correct: + the live topology has not changed). + `buildGenerationContext()` reads ONLY the live server config object and + owner registries — never a `loadConfig()` snapshot. Never from the timer. - **Failure semantics:** each successful owner atomically records its current live-key set/topology and `lastReconciledGeneration`. If a callback throws, log the static registration name, leave that owner on its prior generation, @@ -130,12 +148,20 @@ would create an import cycle, add explicit wiring in NEW called once from `state-store-registrations.ts`; the retry invokes it fresh (round-2 blocker 4). A reconciliation attempt stamps its complete context with - `candidateGeneration = configGeneration + 1`. Successful owners may retain - that candidate stamp; the global `configGeneration` advances to it only when - every callback succeeds. On partial failure, writers captured from the still- - current global generation compare stale against any owner already stamped - with the candidate, and the fresh retry completes the remaining owners - without reusing an old context. + `candidateGeneration = ++attemptSequence` — a SEPARATE monotonic attempt + counter that never resets and never reuses a value (round-4 blocker 2: + deriving candidates from `configGeneration + 1` lets two different attempts + share a candidate after an interleaved config change, so an owner stamped + by attempt 1 would wrongly reject attempt 2's replacement and keep an + obsolete live set). Every attempt therefore carries a unique generation; + owners ALWAYS accept a context whose candidate is strictly greater than + their `lastReconciledGeneration` and replace their live set wholesale. + Successful owners retain the candidate stamp; the global `configGeneration` + advances to the attempt's candidate only when every callback succeeds. On + partial failure, writers captured from the still-current global generation + compare stale against any owner already stamped with the candidate, and the + fresh retry (a NEW attempt with a NEW higher candidate) completes or + re-replaces every owner without reusing an old context. Partial reconciliation is deletion-only: a failed callback leaves extra rows but never deletes a live key. - **Combo topology reconciliation:** remove a whole selection row when its @@ -185,10 +211,10 @@ The inventory shorthand `catalog/*` resolves to `src/codex/catalog/*` in this ch Each owner exports a semantic sweep/reconcile function. The coordinator passes complete sets; it must not recreate owner-local key strings. The single `configGeneration` counter lives in the sweeper (§Generation protocol supersedes earlier drafts of this paragraph); -config.ts triggers reconciliation beside the canonical loaded-config owner. Call it only -after successful initial load, successful disk commit, account add/delete/reauth commit, -or catalog sync with the complete provider/combo topology. Failed parse/save and -speculative routing do not advance it. +the trigger sites are exactly the three shapes in §Generation protocol (startup capture, +in-place management-route commits after successful save, and OAuth live-adoption sites — +`saveConfig()` alone is never a trigger). Failed parse/save and speculative routing do +not advance it. ## Windows ACL delete-after-rename contract From 84897204d363f92c8eca2e973361845a2c2cd20d Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sat, 1 Aug 2026 11:12:19 +0900 Subject: [PATCH 11/30] feat(sweep): reconcile every keyed store against the live topology One sweeper now owns eviction for the expired-key and dead-key store classes: TTL sweeps only where getters already treat expired rows as absent; config-generation reconciliation (unique attempt counters, wholesale owner replacement, deletion-only callbacks) drops keys unreachable from the live server config; late-completion writer fences stop an old flight from resurrecting a deleted key; PID rows are liveness-proven, never generation-swept. Windows ACL success memos are forgotten only after their hardened temp is confirmed gone, across all five producers. Security review: ACL changes affect success-memo lifecycle only; required ACL failures remain fail-closed and destination-keyed timeout memoization is unchanged. Plan: devlog/_plan/260801_zero_leak_state_stores/030 (6 audit rounds). --- src/codex/account-runtime-state.ts | 20 +- src/codex/auth-api.ts | 13 +- src/codex/auth-context.ts | 8 +- src/codex/catalog/aggregation.ts | 15 + src/codex/catalog/provider-fetch.ts | 9 + src/codex/model-cache.ts | 23 ++ src/codex/pool-rotation.ts | 39 ++ src/codex/quota.ts | 34 +- src/codex/routing.ts | 37 +- src/codex/subagent-model-fallback.ts | 12 + src/combos/failover.ts | 32 +- src/combos/request.ts | 9 + src/combos/resolve.ts | 73 +++- src/combos/types.ts | 12 + src/config.ts | 94 ++++- src/lib/config-ownership.ts | 29 ++ src/lib/gcp-adc.ts | 38 +- src/lib/state-store-registrations.ts | 105 ++++++ src/lib/state-store-sweeper.ts | 157 ++++++++ src/lib/windows-secret-acl.ts | 10 + src/oauth/anthropic-routing.ts | 12 + src/oauth/index.ts | 26 +- src/oauth/store.ts | 41 ++- src/oauth/token-guardian.ts | 40 ++- src/providers/key-failover.ts | 12 + src/providers/openai-sidecar.ts | 5 +- src/providers/quota.ts | 75 +++- src/responses/spill-store.ts | 12 +- src/router.ts | 9 + src/server/index.ts | 8 + src/server/lifecycle.ts | 2 + src/server/management-auth.ts | 19 +- src/server/management/combo-routes.ts | 3 + src/server/management/oauth-account-routes.ts | 12 +- src/server/management/provider-routes.ts | 9 + src/server/responses/compact.ts | 3 + src/server/responses/core.ts | 20 +- src/tray/windows.ts | 63 +++- tests/codex-auth-api.test.ts | 29 ++ tests/combos.test.ts | 39 ++ tests/config.test.ts | 88 ++++- tests/oauth-store-multi.test.ts | 35 ++ tests/openai-provider-option-startup.test.ts | 93 +++++ tests/provider-account-quota.test.ts | 43 +++ tests/responses-state.test.ts | 46 +++ tests/server-management-auth.test.ts | 31 ++ tests/state-store-sweeper.test.ts | 340 ++++++++++++++++++ tests/windows-secret-acl.test.ts | 36 ++ tests/windows-tray.test.ts | 61 +++- 49 files changed, 1906 insertions(+), 75 deletions(-) create mode 100644 src/lib/state-store-registrations.ts create mode 100644 src/lib/state-store-sweeper.ts create mode 100644 tests/state-store-sweeper.test.ts diff --git a/src/codex/account-runtime-state.ts b/src/codex/account-runtime-state.ts index 72e8021eb..a2a6495e6 100644 --- a/src/codex/account-runtime-state.ts +++ b/src/codex/account-runtime-state.ts @@ -1,9 +1,27 @@ +import { captureConfigGeneration, type GenerationContext } from "../lib/state-store-sweeper"; + const reauthAccounts = new Set(); +let lastReconciledGeneration = 0; +let liveAccountIds = new Set(); -export function markAccountNeedsReauth(id: string): void { +export function markAccountNeedsReauth(id: string, writerGeneration = captureConfigGeneration()): void { + if (writerGeneration < lastReconciledGeneration && !liveAccountIds.has(id)) return; reauthAccounts.add(id); } +export function reconcileCodexReauthState(context: GenerationContext): number { + if (context.generation <= lastReconciledGeneration) return 0; + let removed = 0; + for (const id of reauthAccounts) { + if (context.codexAccountIds.has(id)) continue; + reauthAccounts.delete(id); + removed += 1; + } + liveAccountIds = new Set(context.codexAccountIds); + lastReconciledGeneration = context.generation; + return removed; +} + export function isAccountNeedsReauth(id: string): boolean { return reauthAccounts.has(id); } diff --git a/src/codex/auth-api.ts b/src/codex/auth-api.ts index 3a925cbe2..1d76d23c6 100644 --- a/src/codex/auth-api.ts +++ b/src/codex/auth-api.ts @@ -51,6 +51,8 @@ export { } from "./quota"; import { extractAccountId, decodeJwtPayload } from "../oauth/chatgpt"; import { getMainAccountPlan, MAIN_CODEX_ACCOUNT_ID, setMainAccountPlan } from "./main-account"; +import { captureConfigGeneration } from "../lib/state-store-sweeper"; +import { reconcileLiveStateStores } from "../lib/state-store-registrations"; import { clearMainAccountInfoCache, getMainAccountInfoCache, @@ -347,6 +349,7 @@ async function retryMainAccountInfoIfIdentityChanged( } async function fetchMainAccountInfoAttempt(forceRefresh: boolean, retriesRemaining: number): Promise { + const writerGeneration = captureConfigGeneration(); reconcileMainCodexAccountRuntimeState(); const tokenRead = readCodexTokensResult(); if (tokenRead.status !== "ok") { @@ -376,7 +379,7 @@ async function fetchMainAccountInfoAttempt(forceRefresh: boolean, retriesRemaini if (retried) return retried; if (terminalAuthFailure) { clearMainAccountInfoCache(); - markAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID); + markAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID, writerGeneration); } return { info: EMPTY_MAIN_ACCOUNT_INFO }; } @@ -398,7 +401,7 @@ async function fetchMainAccountInfoAttempt(forceRefresh: boolean, retriesRemaini // score and auto-switch the main account exactly like a pool account (Option A). setMainAccountPlan(result.plan); if (result.quota) { - setAccountQuotaFromParsed(MAIN_CODEX_ACCOUNT_ID, result.quota); + setAccountQuotaFromParsed(MAIN_CODEX_ACCOUNT_ID, result.quota, writerGeneration); } return { info: result, @@ -509,6 +512,7 @@ async function fetchFreshPoolAccountQuota( configuredPlan?: string, onCredentialGeneration?: (generation: number) => void, ): Promise { + const writerGeneration = captureConfigGeneration(); let requestCredentialGeneration = readCodexAccountRecord(accountId)?.generation; try { const { accessToken, chatgptAccountId, generation } = await getValidCodexToken(accountId); @@ -542,7 +546,7 @@ async function fetchFreshPoolAccountQuota( if (!isCodexAccountGenerationLive(accountId, generation)) { return { quota: null, needsReauth: false, credentialGeneration: generation }; } - setAccountQuotaFromParsed(accountId, quota); + setAccountQuotaFromParsed(accountId, quota, writerGeneration); return { quota: getAccountQuota(accountId), needsReauth: false, @@ -859,6 +863,7 @@ export async function handleCodexAuthAPI( accounts.push(withCodexAccountLogLabel({ id: body.id, email: body.email, plan: body.plan, isMain: false }, accounts)); latestConfig.codexAccounts = accounts; saveRuntimeConfig(config, latestConfig); + reconcileLiveStateStores(); return jsonResponse({ ok: true }); } @@ -873,6 +878,7 @@ export async function handleCodexAuthAPI( } deleteCodexAccount(runtimeConfig, id); saveRuntimeConfig(config, runtimeConfig); + reconcileLiveStateStores(); return jsonResponse({ ok: true }); } @@ -1326,6 +1332,7 @@ export async function handleCodexAuthAPI( latestConfig.codexAccounts = accounts; saveRuntimeConfig(config, latestConfig); } + reconcileLiveStateStores(); codexAuthLoginState.set(flowId, { status: "done", accountId, email, doneAt: Date.now() }); completed = true; } diff --git a/src/codex/auth-context.ts b/src/codex/auth-context.ts index a643703af..fadaa9cb7 100644 --- a/src/codex/auth-context.ts +++ b/src/codex/auth-context.ts @@ -25,12 +25,14 @@ import { formatErrorResponse } from "../bridge"; import { getAccountQuota } from "./quota"; import type { CodexAccountMode, OcxConfig, OcxProviderConfig } from "../types"; import { FORWARD_HEADERS } from "../adapters/openai-responses"; +import { captureConfigGeneration } from "../lib/state-store-sweeper"; export type CodexAuthContext = | { kind: "main"; accountId: null } | { kind: "pool"; accountId: string; + writerGeneration: number; generation: number; accessToken: string; chatgptAccountId: string; @@ -50,6 +52,7 @@ export type CodexAuthContext = // (Option A). Distinct from "main" (passthrough fallback that forwards the client token). kind: "main-pool"; accountId: string; + writerGeneration: number; accessToken: string; chatgptAccountId: string; /** See `pool.probeLeaseId`. */ @@ -198,6 +201,7 @@ export async function resolveCodexAuthContext( mode: CodexAccountMode, options: ResolveCodexAuthContextOptions = {}, ): Promise { + const writerGeneration = captureConfigGeneration(); if (mode === "direct") { if (!hasCallerCodexBearer(headers)) throw new CodexDirectAuthenticationError(); return { kind: "main", accountId: null }; @@ -257,6 +261,7 @@ export async function resolveCodexAuthContext( return { kind: "main-pool", accountId, + writerGeneration, accessToken: token.accessToken, chatgptAccountId: token.chatgptAccountId, ...(quotaScope ? { quotaScope } : {}), @@ -270,6 +275,7 @@ export async function resolveCodexAuthContext( return { kind: "pool", accountId, + writerGeneration, generation: token.generation, accessToken: token.accessToken, chatgptAccountId: token.chatgptAccountId, @@ -281,7 +287,7 @@ export async function resolveCodexAuthContext( if (probeLeaseId && probeQuotaScope) releaseCodexQuotaScopeProbeLease(accountId, probeQuotaScope, probeLeaseId); else if (probeLeaseId) releaseCodexQuotaProbeLease(accountId, probeLeaseId); if (shouldMarkAccountNeedsReauthForCodexAuthFailure(cause)) { - markAccountNeedsReauth(accountId); + markAccountNeedsReauth(accountId, writerGeneration); } throw new CodexAuthContextError(accountId, cause); } diff --git a/src/codex/catalog/aggregation.ts b/src/codex/catalog/aggregation.ts index 1e722d37a..88bfa5eba 100644 --- a/src/codex/catalog/aggregation.ts +++ b/src/codex/catalog/aggregation.ts @@ -281,6 +281,21 @@ export function resetOpenAiApiCatalogWarningStateForTests(): void { export const slugAliasCollisionWarnings = new Set(); export const comboMasqueradeCollisionWarnings = new Set(); +let lastWarningReconciledGeneration = 0; + +export function reconcileCatalogWarningMemos(generation: number): number { + if (generation <= lastWarningReconciledGeneration) return 0; + const removed = openAiApiCollisionWarnings.size + + comboCatalogWarningSignatures.size + + slugAliasCollisionWarnings.size + + comboMasqueradeCollisionWarnings.size; + openAiApiCollisionWarnings.clear(); + comboCatalogWarningSignatures.clear(); + slugAliasCollisionWarnings.clear(); + comboMasqueradeCollisionWarnings.clear(); + lastWarningReconciledGeneration = generation; + return removed; +} export function warnComboMasqueradeCollisionOnce(slug: string): void { if (comboMasqueradeCollisionWarnings.has(slug)) return; diff --git a/src/codex/catalog/provider-fetch.ts b/src/codex/catalog/provider-fetch.ts index fe8449c99..805bbef16 100644 --- a/src/codex/catalog/provider-fetch.ts +++ b/src/codex/catalog/provider-fetch.ts @@ -217,6 +217,15 @@ export function isDatedVariantId(liveId: string, configuredId: string): boolean } export const lastDropWarnSignature = new Map(); +let lastWarningReconciledGeneration = 0; + +export function reconcileProviderFetchWarnings(generation: number): number { + if (generation <= lastWarningReconciledGeneration) return 0; + const removed = lastDropWarnSignature.size; + lastDropWarnSignature.clear(); + lastWarningReconciledGeneration = generation; + return removed; +} export const QUIET_AUTHORITATIVE_CATALOG_PROVIDERS = new Set(["kimi", "xai"]); diff --git a/src/codex/model-cache.ts b/src/codex/model-cache.ts index b116f89fe..ec83af7b4 100644 --- a/src/codex/model-cache.ts +++ b/src/codex/model-cache.ts @@ -8,6 +8,7 @@ * restarts, so an in-memory cache is sufficient here (no SQLite layer needed). */ import type { CatalogModel } from "./catalog"; +import type { GenerationContext } from "../lib/state-store-sweeper"; /** Default freshness window. Matches Codex's own 5-min models cache so the two stay in step. */ export const DEFAULT_MODEL_CACHE_TTL_MS = 5 * 60 * 1000; @@ -54,6 +55,7 @@ const discoveryStatus = new Map(); * custom-only, and the provider's configured fallback would wrongly stay authoritative. */ const liveModelCounts = new Map(); +let lastReconciledGeneration = 0; export function markModelsFetchFailure(provider: string, now = Date.now()): void { failureAt.set(provider, now); @@ -146,3 +148,24 @@ export function clearModelCache(provider?: string): void { liveModelCounts.clear(); } } + +export function reconcileModelCacheProviders( + validProviders: ReadonlySet, + generation = lastReconciledGeneration + 1, +): number { + if (generation <= lastReconciledGeneration) return 0; + const removedProviders = new Set(); + for (const store of [cache, failureAt, discoveryStatus, liveModelCounts]) { + for (const provider of store.keys()) { + if (validProviders.has(provider)) continue; + store.delete(provider); + removedProviders.add(provider); + } + } + lastReconciledGeneration = generation; + return removedProviders.size; +} + +export function reconcileModelCacheGeneration(context: GenerationContext): number { + return reconcileModelCacheProviders(context.providerNames, context.generation); +} diff --git a/src/codex/pool-rotation.ts b/src/codex/pool-rotation.ts index c75f6d34a..47d4576a9 100644 --- a/src/codex/pool-rotation.ts +++ b/src/codex/pool-rotation.ts @@ -1,4 +1,5 @@ import type { OcxAccountPoolRotationStrategy } from "../types"; +import type { GenerationContext } from "../lib/state-store-sweeper"; export const POOL_KEY_CODEX = "codex"; export const POOL_KEY_ANTHROPIC = "anthropic"; @@ -10,6 +11,7 @@ interface SelectionState { } const selectionState = new Map(); +let lastReconciledGeneration = 0; const DEFAULT_STICKY_LIMIT = 1; const MIN_STICKY_LIMIT = 1; @@ -184,3 +186,40 @@ export function clearPoolRotationState(poolKey?: string): void { } selectionState.delete(poolKey); } + +export function reconcilePoolRotationState(context: GenerationContext): number { + if (context.generation <= lastReconciledGeneration) return 0; + const anthropicIds = new Set(); + for (const key of context.oauthAccountKeys) { + const separator = key.indexOf("\0"); + if (separator > 0 && key.slice(0, separator) === "anthropic") { + anthropicIds.add(key.slice(separator + 1)); + } + } + let removed = 0; + for (const [poolKey, state] of selectionState) { + const valid = poolKey === POOL_KEY_ANTHROPIC + ? anthropicIds + : poolKey === POOL_KEY_CODEX || poolKey.startsWith(`${POOL_KEY_CODEX}:`) + ? context.codexAccountIds + : null; + if (!valid) continue; + if (valid.size === 0) { + selectionState.delete(poolKey); + removed += 1; + continue; + } + if (state.activeKey && !valid.has(state.activeKey)) { + delete state.activeKey; + state.successes = 0; + removed += 1; + } + for (const accountId of state.currentWeights.keys()) { + if (valid.has(accountId)) continue; + state.currentWeights.delete(accountId); + removed += 1; + } + } + lastReconciledGeneration = context.generation; + return removed; +} diff --git a/src/codex/quota.ts b/src/codex/quota.ts index 78a3a5f5e..56f303560 100644 --- a/src/codex/quota.ts +++ b/src/codex/quota.ts @@ -1,6 +1,7 @@ import { existsSync, readFileSync, unlinkSync } from "node:fs"; import { join } from "node:path"; import { atomicWriteFile, getConfigDir } from "../config"; +import { captureConfigGeneration, type GenerationContext } from "../lib/state-store-sweeper"; export type StoredAccountQuota = { weeklyPercent?: number; @@ -49,6 +50,12 @@ const MONTHLY_WINDOW_MIN_SECONDS = 28 * 24 * 60 * 60; const MONTHLY_WINDOW_MIN_MINUTES = MONTHLY_WINDOW_MIN_SECONDS / 60; const accountQuota = new Map(); +let lastReconciledGeneration = 0; +let liveAccountIds = new Set(); + +function mayCommitAccountQuota(accountId: string, writerGeneration: number): boolean { + return writerGeneration >= lastReconciledGeneration || liveAccountIds.has(accountId); +} export const CODEX_UNKNOWN_USAGE_SCORE = 100; export const CODEX_EXHAUSTED_USAGE_PERCENT = 100; @@ -125,8 +132,10 @@ function snapshotHasUsage(quota: Omit): boolean export function setAccountQuotaFromParsed( accountId: string, quota: Omit | null, + writerGeneration = captureConfigGeneration(), ): void { if (!quota) return; + if (!mayCommitAccountQuota(accountId, writerGeneration)) return; const existing = accountQuota.get(accountId); const next: StoredAccountQuota = { updatedAt: Date.now() }; const creditsOnly = quota.resetCredits !== undefined && !snapshotHasUsage(quota); @@ -214,10 +223,14 @@ export function parseUpstreamQuotaHeaders(headers: Headers): Omit 0) schedulePersistAccountQuotas(); + return removed; +} + export function parseUsageQuota(data: WhamUsageResponse): Omit | null { const resetCredits = typeof data.rate_limit_reset_credits?.available_count === "number" ? data.rate_limit_reset_credits.available_count diff --git a/src/codex/routing.ts b/src/codex/routing.ts index bbc13f844..65a881e7d 100644 --- a/src/codex/routing.ts +++ b/src/codex/routing.ts @@ -19,6 +19,8 @@ import { CODEX_UNKNOWN_USAGE_SCORE, getAccountQuota } from "./quota"; import { MAIN_CODEX_ACCOUNT_ID, getMainAccountPlan } from "./main-account"; import { isSelectableCodexPoolAccount } from "./account-id"; import type { OcxConfig } from "../types"; +import { captureConfigGeneration, type GenerationContext } from "../lib/state-store-sweeper"; +import { isCanonicalOpenAiForwardProvider } from "../providers/openai-tiers"; type ThreadAffinityEntry = { accountId: string; @@ -115,6 +117,8 @@ const upstreamHealth = new Map(); * from account-wide Retry-After/default throttles and transient health. */ const quotaScopedHealth = new Map>(); +let lastReconciledGeneration = 0; +let liveHealthAccountIds = new Set(); export type CodexUpstreamOutcome = number | "connect_error" | "timeout"; export type CodexUpstreamOutcomeClass = "success" | "credential" | "quota" | "transient" | "caller" | "unknown"; @@ -185,6 +189,8 @@ export type CodexUpstreamOutcomeMeta = { * again (which would advance a round-robin ring twice). */ promoteAccountId?: string; + /** Generation captured when this routed account was selected. */ + writerGeneration?: number; }; function hasConfiguredPoolAccount(config: OcxConfig, accountId: string): boolean { @@ -193,6 +199,15 @@ function hasConfiguredPoolAccount(config: OcxConfig, accountId: string): boolean .some(account => isSelectableCodexPoolAccount(account) && account.id === accountId); } +export function listLiveCodexAccountIds(config: OcxConfig): ReadonlySet { + const ids = new Set((config.codexAccounts ?? []).map(account => account.id)); + const openai = config.providers.openai; + if (openai && openai.disabled !== true && isCanonicalOpenAiForwardProvider(openai)) { + ids.add(MAIN_CODEX_ACCOUNT_ID); + } + return ids; +} + export function clearThreadAccountMap(): void { threadAccountMap.clear(); } @@ -217,6 +232,24 @@ export function clearCodexUpstreamHealthForAccount(accountId: string): void { quotaScopedHealth.delete(accountId); } +export function reconcileCodexRoutingHealth(context: GenerationContext): number { + if (context.generation <= lastReconciledGeneration) return 0; + let removed = 0; + for (const accountId of upstreamHealth.keys()) { + if (context.codexAccountIds.has(accountId)) continue; + upstreamHealth.delete(accountId); + removed += 1; + } + for (const accountId of quotaScopedHealth.keys()) { + if (context.codexAccountIds.has(accountId)) continue; + quotaScopedHealth.delete(accountId); + removed += 1; + } + liveHealthAccountIds = new Set(context.codexAccountIds); + lastReconciledGeneration = context.generation; + return removed; +} + export function getCodexUpstreamHealth( accountId: string, ): CodexUpstreamHealth | null { @@ -1204,6 +1237,8 @@ export function recordCodexUpstreamOutcome( meta: CodexUpstreamOutcomeMeta = {}, ): void { if (!accountId) return; + const writerGeneration = meta.writerGeneration ?? captureConfigGeneration(); + if (writerGeneration < lastReconciledGeneration && !liveHealthAccountIds.has(accountId)) return; const now = meta.now ?? Date.now(); const outcomeClass = classifyCodexUpstreamOutcome(outcome); const quotaScope = codexQuotaScopeForModel(meta.modelId); @@ -1275,7 +1310,7 @@ export function recordCodexUpstreamOutcome( lastFailureAt: now, }); quotaScopedHealth.delete(accountId); - markAccountNeedsReauth(accountId); + markAccountNeedsReauth(accountId, writerGeneration); clearThreadAccountMapForAccount(accountId); return; } diff --git a/src/codex/subagent-model-fallback.ts b/src/codex/subagent-model-fallback.ts index 6e936ec22..f86d31505 100644 --- a/src/codex/subagent-model-fallback.ts +++ b/src/codex/subagent-model-fallback.ts @@ -26,6 +26,7 @@ import { isThreadSpawnRequest } from "../server/effort-policy"; import { PROVIDER_REGISTRY } from "../providers/registry"; import { isCanonicalOpenAiForwardProvider } from "../providers/openai-tiers"; import { routeModel, type RouteResult } from "../router"; +import { sweepExpiredOnWrite } from "../lib/state-store-sweeper"; export const DEFAULT_SUBAGENT_MODEL_FALLBACK_POLL_MS = 60_000; type SubagentQuotaPrimeFn = (config: OcxConfig, reason: string) => Promise; @@ -245,6 +246,17 @@ export function noteSubagentModelFailure( reason: "quota_exhausted", }, ); + sweepExpiredOnWrite(now); +} + +export function sweepExpiredSubagentModelHealth(now = Date.now()): number { + let removed = 0; + for (const [key, health] of modelHealth) { + if (health.unavailableUntil > now) continue; + modelHealth.delete(key); + removed += 1; + } + return removed; } export function resetSubagentModelFallbackStateForTests(): void { diff --git a/src/combos/failover.ts b/src/combos/failover.ts index 2ba677136..95a5bbc83 100644 --- a/src/combos/failover.ts +++ b/src/combos/failover.ts @@ -1,6 +1,11 @@ import { classifyError, isCyberPolicyCode } from "../lib/errors"; import type { OcxComboTarget } from "../types"; import { targetKey } from "./types"; +import { + captureConfigGeneration, + sweepExpiredOnWrite, + type GenerationContext, +} from "../lib/state-store-sweeper"; interface TargetCooldown { cooldownUntil: number; @@ -11,6 +16,8 @@ const MAX_COOLDOWN_MS = 10 * 60_000; /** Map<`${comboId}\0${provider/model}`, TargetCooldown> */ const targetCooldowns = new Map(); +let lastReconciledGeneration = 0; +let liveComboTargets = new Set(); function cooldownMapKey( comboId: string, @@ -55,20 +62,43 @@ export function isComboTargetInCooldown( export function coolComboTarget( comboId: string, target: Pick, - options?: { retryAfter?: string | null; now?: number; cooldownMs?: number }, + options?: { retryAfter?: string | null; now?: number; cooldownMs?: number; writerGeneration?: number }, ): void { const now = options?.now ?? Date.now(); + const writerGeneration = options?.writerGeneration ?? captureConfigGeneration(); + const ownerKey = `${comboId}::${targetKey(target)}`; + if (writerGeneration < lastReconciledGeneration && !liveComboTargets.has(ownerKey)) return; const cooldownMs = options?.cooldownMs ?? parseRetryAfterMs(options?.retryAfter, now) ?? DEFAULT_COOLDOWN_MS; targetCooldowns.set(cooldownMapKey(comboId, target), { cooldownUntil: now + Math.min(Math.max(cooldownMs, 1), MAX_COOLDOWN_MS), }); + sweepExpiredOnWrite(now); +} + +export function reconcileComboTargetCooldowns(context: GenerationContext): number { + if (context.generation <= lastReconciledGeneration) return 0; + liveComboTargets = new Set(context.comboTargets); + lastReconciledGeneration = context.generation; + return 0; +} + +export function sweepExpiredComboTargetCooldowns(now = Date.now()): number { + let removed = 0; + for (const [key, cooldown] of targetCooldowns) { + if (cooldown.cooldownUntil > now) continue; + targetCooldowns.delete(key); + removed += 1; + } + return removed; } export function clearComboTargetCooldowns(comboId?: string): void { if (comboId === undefined) { targetCooldowns.clear(); + liveComboTargets.clear(); + lastReconciledGeneration = 0; return; } const prefix = `${comboId}\0`; diff --git a/src/combos/request.ts b/src/combos/request.ts index aa03862e3..c7d535f7c 100644 --- a/src/combos/request.ts +++ b/src/combos/request.ts @@ -2,6 +2,15 @@ import type { OcxComboDefaultEffort, OcxComboTarget, OcxConfig } from "../types" import { resolveComboId } from "./types"; const warnedUnsupportedDefaults = new Set(); +let lastWarningReconciledGeneration = 0; + +export function reconcileComboWarningMemos(generation: number): number { + if (generation <= lastWarningReconciledGeneration) return 0; + const removed = warnedUnsupportedDefaults.size; + warnedUnsupportedDefaults.clear(); + lastWarningReconciledGeneration = generation; + return removed; +} export function resetComboEffortWarningStateForTests(): void { warnedUnsupportedDefaults.clear(); diff --git a/src/combos/resolve.ts b/src/combos/resolve.ts index b93fdd0c9..7bc3e3b88 100644 --- a/src/combos/resolve.ts +++ b/src/combos/resolve.ts @@ -2,12 +2,17 @@ import type { OcxComboTarget, OcxConfig } from "../types"; import { coolComboTarget, isComboTargetInCooldown } from "./failover"; import { getCombo, resolveComboId, targetKey } from "./types"; import type { NormalizedComboConfig } from "./types"; +import { + captureConfigGeneration, + type GenerationContext, +} from "../lib/state-store-sweeper"; export interface ComboPick { comboId: string; target: Required; targetIndex: number; attempted: string[]; + writerGeneration: number; } interface SelectionState { @@ -17,6 +22,19 @@ interface SelectionState { } const selectionState = new Map(); +let lastReconciledGeneration = 0; +let liveComboTargets = new Set(); +const comboTopologyGeneration = new Map(); + +function comboTargetOwnerKey(comboId: string, key: string): string { + return `${comboId}::${key}`; +} + +function mayCommitComboState(comboId: string, key: string, writerGeneration: number): boolean { + return writerGeneration >= (comboTopologyGeneration.get(comboId) ?? 0) + && (writerGeneration >= lastReconciledGeneration + || liveComboTargets.has(comboTargetOwnerKey(comboId, key))); +} export class UnknownComboError extends Error { constructor(readonly comboId: string) { @@ -74,6 +92,7 @@ export function pickComboTarget( eligible?: (target: Required) => boolean; } = {}, ): ComboPick | null { + const writerGeneration = captureConfigGeneration(); const combo = getCombo(config, comboId); if (!combo) throw new UnknownComboError(comboId); const excluded = new Set(options.exclude ?? []); @@ -114,6 +133,7 @@ export function pickComboTarget( target, targetIndex, attempted: [...excluded, targetKey(target)], + writerGeneration, }; } @@ -121,10 +141,13 @@ export function noteComboSuccess( comboId: string, combo: NormalizedComboConfig, target: Required, + writerGeneration = captureConfigGeneration(), ): void { if (combo.strategy !== "round-robin") return; + const key = targetKey(target); + if (!mayCommitComboState(comboId, key, writerGeneration)) return; const state = selectionState.get(comboId); - if (!state || state.activeKey !== targetKey(target)) return; + if (!state || state.activeKey !== key) return; state.successes += 1; if (state.successes >= combo.stickyLimit) { delete state.activeKey; @@ -132,7 +155,12 @@ export function noteComboSuccess( } } -export function noteComboFailure(comboId: string, target: OcxComboTarget): void { +export function noteComboFailure( + comboId: string, + target: OcxComboTarget, + writerGeneration = captureConfigGeneration(), +): void { + if (!mayCommitComboState(comboId, targetKey(target), writerGeneration)) return; const state = selectionState.get(comboId); if (state?.activeKey === targetKey(target)) { delete state.activeKey; @@ -149,8 +177,11 @@ export function advanceComboAfterFailure( eligible?: (target: Required) => boolean; } = {}, ): ComboPick | null { - noteComboFailure(pick.comboId, pick.target); - coolComboTarget(pick.comboId, pick.target, options); + noteComboFailure(pick.comboId, pick.target, pick.writerGeneration); + coolComboTarget(pick.comboId, pick.target, { + ...options, + writerGeneration: pick.writerGeneration, + }); return pickComboTarget(config, pick.comboId, { exclude: pick.attempted, eligible: target => !isComboTargetInCooldown(pick.comboId, target, options.now) @@ -158,12 +189,46 @@ export function advanceComboAfterFailure( }); } +export function reconcileComboRotationState(context: GenerationContext): number { + if (context.generation <= lastReconciledGeneration) return 0; + let removed = 0; + for (const [comboId, state] of selectionState) { + if (!context.comboIds.has(comboId)) { + selectionState.delete(comboId); + comboTopologyGeneration.set(comboId, context.generation); + removed += 1; + continue; + } + let topologyChanged = false; + if (state.activeKey && !context.comboTargets.has(comboTargetOwnerKey(comboId, state.activeKey))) { + delete state.activeKey; + state.successes = 0; + topologyChanged = true; + removed += 1; + } + for (const key of state.currentWeights.keys()) { + if (context.comboTargets.has(comboTargetOwnerKey(comboId, key))) continue; + state.currentWeights.delete(key); + topologyChanged = true; + removed += 1; + } + if (topologyChanged) comboTopologyGeneration.set(comboId, context.generation); + } + liveComboTargets = new Set(context.comboTargets); + lastReconciledGeneration = context.generation; + return removed; +} + export function clearComboSelectionState(comboId?: string): void { if (comboId === undefined) { selectionState.clear(); + liveComboTargets.clear(); + comboTopologyGeneration.clear(); + lastReconciledGeneration = 0; return; } selectionState.delete(comboId); + comboTopologyGeneration.delete(comboId); } export function tryPickComboModel(config: OcxConfig, modelId: string): ComboPick | null { diff --git a/src/combos/types.ts b/src/combos/types.ts index 7ed952ffd..c5dcd51b6 100644 --- a/src/combos/types.ts +++ b/src/combos/types.ts @@ -304,6 +304,18 @@ export function listComboIds(config: { combos?: Record } return Object.keys(config.combos ?? {}).sort((a, b) => a.localeCompare(b)); } +export function listLiveComboTargetKeys( + config: { combos?: Record }, +): ReadonlySet { + const keys = new Set(); + for (const id of listComboIds(config)) { + const combo = getCombo(config, id); + if (!combo) continue; + for (const target of combo.targets) keys.add(`${id}::${targetKey(target)}`); + } + return keys; +} + export function getCombo( config: { combos?: Record }, id: string, diff --git a/src/config.ts b/src/config.ts index 7ca2c0d4e..7a69343fe 100644 --- a/src/config.ts +++ b/src/config.ts @@ -13,7 +13,12 @@ import { MAIN_CODEX_ACCOUNT_NAMESPACE_TARGET, } from "./codex/account-namespace-match"; import { COMBO_NAMESPACE, comboConfigIssues } from "./combos/types"; -import { hardenSecretDir, hardenSecretPath, hardenSecretPathAsync } from "./lib/windows-secret-acl"; +import { + forgetHardenedSecretPath, + hardenSecretDir, + hardenSecretPath, + hardenSecretPathAsync, +} from "./lib/windows-secret-acl"; import { recordOwnedConfigPath } from "./lib/config-ownership"; import { assertNotRealHomeUnderTest } from "./lib/test-home-guard"; import { providerDestinationConfigError } from "./lib/destination-policy"; @@ -111,6 +116,7 @@ export function atomicWriteFile(path: string, content: string, io: AtomicWriteIO io.harden(tmp); hardened = true; io.rename(tmp, path); + forgetHardenedSecretPath(tmp); } catch (cause) { let scrubbed = false; try { @@ -137,6 +143,7 @@ export function atomicWriteFile(path: string, content: string, io: AtomicWriteIO if (!removed && !hardened) { try { io.harden(tmp); hardened = true; } catch { /* zero-byte residual is reported honestly */ } } + if (removed) forgetHardenedSecretPath(tmp); if (!removed) throw new AtomicWriteResidualTempError(tmp, hardened, { cause }); throw cause; } @@ -195,6 +202,7 @@ export async function atomicWriteFileAsync( await effective.harden(tmp); hardened = true; await effective.rename(tmp, path); + forgetHardenedSecretPath(tmp); } catch (cause) { let scrubbed = false; try { @@ -221,6 +229,7 @@ export async function atomicWriteFileAsync( if (!removed && !hardened) { try { await effective.harden(tmp); hardened = true; } catch { /* zero-byte residual is reported honestly */ } } + if (removed) forgetHardenedSecretPath(tmp); if (!removed) throw new AtomicWriteResidualTempError(tmp, hardened, { cause }); throw cause; } @@ -337,7 +346,10 @@ export function backupConfigBeforeOpenAiTierMigration( const scrubUnpublishedTemp = (): void => { cleanupAttempted = true; - if (!io.exists(temp)) return; + if (!io.exists(temp)) { + forgetHardenedSecretPath(temp); + return; + } let scrubbed = false; try { io.truncate(temp); @@ -352,12 +364,19 @@ export function backupConfigBeforeOpenAiTierMigration( try { io.unlink(temp); removed = true; + forgetHardenedSecretPath(temp); } catch (error) { - if (isMissingPathError(error) || !io.exists(temp)) removed = true; + if (isMissingPathError(error) || !io.exists(temp)) { + removed = true; + forgetHardenedSecretPath(temp); + } else { - try { io.unlink(temp); removed = true; } + try { io.unlink(temp); removed = true; forgetHardenedSecretPath(temp); } catch (retryError) { - if (isMissingPathError(retryError) || !io.exists(temp)) removed = true; + if (isMissingPathError(retryError) || !io.exists(temp)) { + removed = true; + forgetHardenedSecretPath(temp); + } } } } @@ -381,10 +400,18 @@ export function backupConfigBeforeOpenAiTierMigration( published = true; try { io.unlink(temp); - } catch { - try { + forgetHardenedSecretPath(temp); + } catch (firstError) { + if (isMissingPathError(firstError) || !io.exists(temp)) { + forgetHardenedSecretPath(temp); + } else try { io.unlink(temp); - } catch { + forgetHardenedSecretPath(temp); + } catch (secondError) { + if (isMissingPathError(secondError) || !io.exists(temp)) { + forgetHardenedSecretPath(temp); + return "created"; + } // temp and backup are hard links to the same inode. Roll back the backup // link before any truncation so the downgrade snapshot is never zeroed. try { io.unlink(backup); } catch { throw new OpenAiTierBackupRollbackError(); } @@ -436,6 +463,15 @@ function resolveRuntimePortPath(): string { } const warnedConfigFallbacks = new Set(); +let lastWarningReconciledGeneration = 0; + +export function reconcileConfigWarningMemos(generation: number): number { + if (generation <= lastWarningReconciledGeneration) return 0; + const removed = warnedConfigFallbacks.size; + warnedConfigFallbacks.clear(); + lastWarningReconciledGeneration = generation; + return removed; +} const providerConfigSchema = z.object({ adapter: z.string().min(1), @@ -2113,6 +2149,48 @@ export function isOcxStartCommandLine(commandLine: string): boolean { /** Per-process memo: waitForProxy/findLiveProxy used to spawn powershell on every 150ms poll. */ const ocxStartProcessCache = new Map(); +let ocxStartProcessSweepCursor = 0; +let ocxStartProcessProbe: (pid: number) => void = pid => { process.kill(pid, 0); }; + +export function setOcxStartProcessProbeForTests(probe: ((pid: number) => void) | null): void { + ocxStartProcessProbe = probe ?? (pid => { process.kill(pid, 0); }); +} + +export function setOcxStartProcessCacheForTests(entries: Iterable): void { + ocxStartProcessCache.clear(); + for (const [pid, value] of entries) ocxStartProcessCache.set(pid, value); + ocxStartProcessSweepCursor = 0; +} + +export function sweepDeadOcxStartProcessCache(maxProbes = 64): number { + const pids: number[] = []; + let removed = 0; + for (const pid of ocxStartProcessCache.keys()) { + if (Number.isSafeInteger(pid) && pid > 0) pids.push(pid); + else if (ocxStartProcessCache.delete(pid)) removed += 1; + } + if (pids.length === 0 || maxProbes <= 0) { + ocxStartProcessSweepCursor = 0; + return removed; + } + const probeCount = Math.min(Math.floor(maxProbes), pids.length); + const start = ocxStartProcessSweepCursor % pids.length; + for (let offset = 0; offset < probeCount; offset += 1) { + const pid = pids[(start + offset) % pids.length]!; + try { + ocxStartProcessProbe(pid); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") continue; + if (ocxStartProcessCache.delete(pid)) removed += 1; + } + } + ocxStartProcessSweepCursor = (start + probeCount) % pids.length; + return removed; +} + +export function ocxStartProcessCacheSizeForTests(): number { + return ocxStartProcessCache.size; +} function isLikelyOcxStartProcess(pid: number): boolean { const cached = ocxStartProcessCache.get(pid); diff --git a/src/lib/config-ownership.ts b/src/lib/config-ownership.ts index 3dbfa2072..553581e2d 100644 --- a/src/lib/config-ownership.ts +++ b/src/lib/config-ownership.ts @@ -12,6 +12,7 @@ import { } from "node:fs"; import { randomUUID } from "node:crypto"; import { isAbsolute, join, relative, resolve, sep } from "node:path"; +import type { GenerationContext } from "./state-store-sweeper"; export const CONFIG_OWNER_FILE = ".opencodex-owner.json"; export const CONFIG_UNINSTALL_MANIFEST = ".opencodex-uninstall.json"; @@ -80,6 +81,34 @@ const ownershipCache = new Map(); +let lastReconciledGeneration = 0; + +export function listLiveConfigOwnershipRoots(currentConfigDir: string): ReadonlySet { + const currentRoot = ownershipCacheKey(currentConfigDir); + const roots = new Set([currentRoot]); + for (const [root, ownership] of ownershipCache) { + if (root === currentRoot) continue; + if (ownership?.manifest.paths.some(rel => existsSync(join(root, ...rel.split("/"))))) { + roots.add(root); + } + } + return roots; +} + +export function reconcileConfigOwnershipRoots(context: GenerationContext): number { + if (context.generation <= lastReconciledGeneration) return 0; + let removed = 0; + for (const [root, ownership] of ownershipCache) { + if (context.configRoots.has(root)) continue; + const hasLiveOwnedPath = ownership?.manifest.paths.some(rel => + existsSync(join(root, ...rel.split("/")))); + if (hasLiveOwnedPath) continue; + ownershipCache.delete(root); + removed += 1; + } + lastReconciledGeneration = context.generation; + return removed; +} function ownershipCacheKey(configDir: string): string { const key = resolve(configDir); diff --git a/src/lib/gcp-adc.ts b/src/lib/gcp-adc.ts index 691b12aa5..746e6b62a 100644 --- a/src/lib/gcp-adc.ts +++ b/src/lib/gcp-adc.ts @@ -18,6 +18,11 @@ import { Buffer } from "node:buffer"; import * as os from "node:os"; import * as path from "node:path"; import { readFileSync, existsSync, statSync } from "node:fs"; +import { + captureConfigGeneration, + sweepExpiredOnWrite, + type GenerationContext, +} from "./state-store-sweeper"; /** Injectable fetch (tests pass a mock); defaults to the global fetch. */ export type FetchImpl = typeof fetch; @@ -60,6 +65,33 @@ interface TokenResponse { const tokenCache = new Map(); const inflight = new Map>(); +let lastReconciledGeneration = 0; +let liveSources = new Set(); + +export function sweepExpiredGcpAdcTokens(now = Date.now()): number { + const skew = getRefreshSkewMs(); + let removed = 0; + for (const [source, cached] of tokenCache) { + if (cached.expiresAtMs - skew > now) continue; + tokenCache.delete(source); + removed += 1; + } + return removed; +} + +export function reconcileGcpAdcTokens(context: GenerationContext): number { + if (context.generation <= lastReconciledGeneration) return 0; + const currentSource = currentAdcSourceKey(); + let removed = 0; + for (const source of tokenCache.keys()) { + if (source === currentSource) continue; + tokenCache.delete(source); + removed += 1; + } + liveSources = new Set([currentSource]); + lastReconciledGeneration = context.generation; + return removed; +} function getRefreshSkewMs(): number { const raw = Number(process.env.GOOGLE_VERTEX_REFRESH_SKEW_MS); @@ -271,6 +303,7 @@ export async function getVertexAccessToken(options?: { signal?: AbortSignal; fet // Only serve a cached token that matches the source the next resolve would actually use; prune // expired or now-stale (different-source) entries so a credential-source change is honored. const expectedSource = currentAdcSourceKey(); + const writerGeneration = captureConfigGeneration(); for (const [source, cached] of tokenCache) { if (source === expectedSource && cached.expiresAtMs - skew > now) return cached.token; if (cached.expiresAtMs - skew <= now) tokenCache.delete(source); @@ -286,7 +319,10 @@ export async function getVertexAccessToken(options?: { signal?: AbortSignal; fet try { const { source, token } = await resolveAccessTokenUncached(options?.signal, fetchImpl); const expiresAtMs = Date.now() + Math.max(0, token.expires_in * 1000); - tokenCache.set(source, { token: token.access_token, expiresAtMs }); + if (writerGeneration >= lastReconciledGeneration || liveSources.has(source)) { + tokenCache.set(source, { token: token.access_token, expiresAtMs }); + sweepExpiredOnWrite(Date.now()); + } return token.access_token; } finally { inflight.delete(cacheKey); diff --git a/src/lib/state-store-registrations.ts b/src/lib/state-store-registrations.ts new file mode 100644 index 000000000..4ba4b9949 --- /dev/null +++ b/src/lib/state-store-registrations.ts @@ -0,0 +1,105 @@ +import { + getConfigDir, + reconcileConfigWarningMemos, + sweepDeadOcxStartProcessCache, +} from "../config"; +import { reconcileCodexReauthState } from "../codex/account-runtime-state"; +import { reconcileCatalogWarningMemos } from "../codex/catalog/aggregation"; +import { reconcileProviderFetchWarnings } from "../codex/catalog/provider-fetch"; +import { reconcileModelCacheGeneration } from "../codex/model-cache"; +import { reconcilePoolRotationState } from "../codex/pool-rotation"; +import { reconcileCodexQuotaAccounts } from "../codex/quota"; +import { + listLiveCodexAccountIds, + reconcileCodexRoutingHealth, +} from "../codex/routing"; +import { sweepExpiredSubagentModelHealth } from "../codex/subagent-model-fallback"; +import { + reconcileComboTargetCooldowns, + sweepExpiredComboTargetCooldowns, +} from "../combos/failover"; +import { reconcileComboWarningMemos } from "../combos/request"; +import { reconcileComboRotationState } from "../combos/resolve"; +import { listLiveComboTargetKeys } from "../combos/types"; +import { + listLiveConfigOwnershipRoots, + reconcileConfigOwnershipRoots, +} from "./config-ownership"; +import { reconcileGcpAdcTokens } from "./gcp-adc"; +import { + reconcileOAuthFlowState, + sweepExpiredXaiPermanentFailureVerdicts, +} from "../oauth"; +import { sweepExpiredAnthropicRoutingHealth } from "../oauth/anthropic-routing"; +import { listLiveOAuthAccountKeys, reconcileOAuthReauthState } from "../oauth/store"; +import { reconcileGuardianBackoff } from "../oauth/token-guardian"; +import { sweepExpiredApiKeyCooldowns } from "../providers/key-failover"; +import { reconcileProviderAccountQuotaRows } from "../providers/quota"; +import { reconcileRouterWarningMemos } from "../router"; +import type { OcxConfig } from "../types"; +import { + type GenerationContext, + reconcileStateGeneration, + registerStateStore, + setGenerationContextBuilder, + type StateStoreRegistration, +} from "./state-store-sweeper"; + +let liveServerConfig: OcxConfig | null = null; + +export function setLiveStateStoreConfig(config: OcxConfig): void { + liveServerConfig = config; +} + +export function reconcileLiveStateStores() { + if (!liveServerConfig) return { storesVisited: 0, rowsRemoved: 0 }; + return reconcileStateGeneration(buildGenerationContext()); +} + +export function buildGenerationContext(): GenerationContext { + if (!liveServerConfig) throw new Error("live server config is not installed"); + const providerNames = new Set(Object.keys(liveServerConfig.providers)); + return { + generation: 0, + providerNames, + comboIds: new Set(Object.keys(liveServerConfig.combos ?? {})), + comboTargets: listLiveComboTargetKeys(liveServerConfig), + codexAccountIds: listLiveCodexAccountIds(liveServerConfig), + oauthAccountKeys: listLiveOAuthAccountKeys(providerNames), + configRoots: listLiveConfigOwnershipRoots(getConfigDir()), + }; +} + +export const STATE_STORE_REGISTRATIONS = [ + { name: "subagent-model-health", sweepExpired: sweepExpiredSubagentModelHealth }, + { name: "api-key-cooldowns", sweepExpired: sweepExpiredApiKeyCooldowns }, + { + name: "combo-target-cooldowns", + sweepExpired: sweepExpiredComboTargetCooldowns, + reconcileGeneration: reconcileComboTargetCooldowns, + }, + { name: "anthropic-routing-health", sweepExpired: sweepExpiredAnthropicRoutingHealth }, + { name: "xai-refresh-verdicts", sweepExpired: sweepExpiredXaiPermanentFailureVerdicts }, + { name: "config-warning-memos", reconcileGeneration: (context: GenerationContext) => reconcileConfigWarningMemos(context.generation) }, + { name: "catalog-warning-memos", reconcileGeneration: (context: GenerationContext) => reconcileCatalogWarningMemos(context.generation) }, + { name: "provider-fetch-warning-memos", reconcileGeneration: (context: GenerationContext) => reconcileProviderFetchWarnings(context.generation) }, + { name: "combo-warning-memos", reconcileGeneration: (context: GenerationContext) => reconcileComboWarningMemos(context.generation) }, + { name: "router-warning-memos", reconcileGeneration: (context: GenerationContext) => reconcileRouterWarningMemos(context.generation) }, + { name: "codex-quota", reconcileGeneration: reconcileCodexQuotaAccounts }, + { name: "provider-quota-history", reconcileGeneration: reconcileProviderAccountQuotaRows }, + { name: "codex-routing-health", reconcileGeneration: reconcileCodexRoutingHealth }, + { name: "model-cache-history", reconcileGeneration: reconcileModelCacheGeneration }, + { name: "pool-rotation", reconcileGeneration: reconcilePoolRotationState }, + { name: "combo-rotation", reconcileGeneration: reconcileComboRotationState }, + { name: "guardian-backoff", reconcileGeneration: reconcileGuardianBackoff }, + { name: "codex-reauth", reconcileGeneration: reconcileCodexReauthState }, + { name: "oauth-reauth", reconcileGeneration: reconcileOAuthReauthState }, + { name: "gcp-adc", reconcileGeneration: reconcileGcpAdcTokens }, + { name: "config-ownership", reconcileGeneration: reconcileConfigOwnershipRoots }, + { name: "oauth-flow-state", reconcileGeneration: reconcileOAuthFlowState }, + { name: "ocx-start-process-cache", sweepLiveness: sweepDeadOcxStartProcessCache }, +] satisfies readonly StateStoreRegistration[]; + +for (const registration of STATE_STORE_REGISTRATIONS) registerStateStore(registration); + +setGenerationContextBuilder(buildGenerationContext); diff --git a/src/lib/state-store-sweeper.ts b/src/lib/state-store-sweeper.ts new file mode 100644 index 000000000..cc5afb6b9 --- /dev/null +++ b/src/lib/state-store-sweeper.ts @@ -0,0 +1,157 @@ +export const STATE_SWEEP_INTERVAL_MS = 60_000; + +export interface GenerationContext { + generation: number; + providerNames: ReadonlySet; + comboIds: ReadonlySet; + comboTargets: ReadonlySet; + codexAccountIds: ReadonlySet; + oauthAccountKeys: ReadonlySet; + configRoots: ReadonlySet; +} + +export interface StateStoreRegistration { + name: string; + sweepExpired?: (now: number) => number; + sweepLiveness?: () => number; + reconcileGeneration?: (context: GenerationContext) => number; +} + +export interface StateSweepResult { + storesVisited: number; + rowsRemoved: number; +} + +export interface StateStoreSweeperOptions { + /** Override the 60 s interval (tests only; production uses the default). */ + intervalMs?: number; + /** Injectable clock for tests. */ + now?: () => number; +} + +const registrations = new Map(); +let configGeneration = 0; +let attemptSequence = 0; +let generationContextBuilder: (() => GenerationContext) | null = null; +let reconciliationRetryPending = false; +let interval: ReturnType | null = null; + +function logCallbackFailure(name: string): void { + console.warn(`[state-store-sweeper] ${name} failed`); +} + +export function registerStateStore(registration: StateStoreRegistration): () => void { + registrations.set(registration.name, registration); + return () => { + if (registrations.get(registration.name) === registration) { + registrations.delete(registration.name); + } + }; +} + +function runCallbacks( + callback: (registration: StateStoreRegistration) => (() => number) | undefined, +): StateSweepResult { + let storesVisited = 0; + let rowsRemoved = 0; + for (const registration of registrations.values()) { + const invoke = callback(registration); + if (!invoke) continue; + storesVisited += 1; + try { + rowsRemoved += invoke(); + } catch { + logCallbackFailure(registration.name); + } + } + return { storesVisited, rowsRemoved }; +} + +export function sweepExpired(now = Date.now()): StateSweepResult { + return runCallbacks(registration => registration.sweepExpired + ? () => registration.sweepExpired!(now) + : undefined); +} + +export function sweepExpiredOnWrite(now = Date.now()): StateSweepResult { + const result = sweepExpired(now); + if (reconciliationRetryPending && generationContextBuilder) { + try { + reconcileStateGeneration(generationContextBuilder()); + } catch { + // The builder reads authoritative live owners. Keep the single retry pending + // until a later successful owner write can obtain a complete context. + } + } + return result; +} + +export function sweepLiveness(): StateSweepResult { + return runCallbacks(registration => registration.sweepLiveness); +} + +export function captureConfigGeneration(): number { + return configGeneration; +} + +export function setGenerationContextBuilder(build: () => GenerationContext): void { + generationContextBuilder = build; +} + +export function reconcileStateGeneration(context: GenerationContext): StateSweepResult { + const candidateGeneration = ++attemptSequence; + const candidateContext: GenerationContext = { + ...context, + generation: candidateGeneration, + }; + + let storesVisited = 0; + let rowsRemoved = 0; + let failed = false; + for (const registration of registrations.values()) { + if (!registration.reconcileGeneration) continue; + storesVisited += 1; + try { + rowsRemoved += registration.reconcileGeneration(candidateContext); + } catch { + failed = true; + logCallbackFailure(registration.name); + } + } + if (failed) { + reconciliationRetryPending = true; + } else { + configGeneration = candidateGeneration; + reconciliationRetryPending = false; + } + return { storesVisited, rowsRemoved }; +} + +export function startStateStoreSweeper( + options: StateStoreSweeperOptions = {}, +): { stop(): void } { + stopStateStoreSweeper(); + const now = options.now ?? Date.now; + interval = setInterval(() => { + sweepExpired(now()); + sweepLiveness(); + }, options.intervalMs ?? STATE_SWEEP_INTERVAL_MS); + interval.unref?.(); + return { stop: stopStateStoreSweeper }; +} + +export function stopStateStoreSweeper(): void { + if (!interval) return; + clearInterval(interval); + interval = null; +} + +/** Test-only reset for module-global registrations, generation, and lifecycle state. */ +export function resetStateStoreSweeperForTests(): void { + stopStateStoreSweeper(); + registrations.clear(); + configGeneration = 0; + attemptSequence = 0; + generationContextBuilder = null; + reconciliationRetryPending = false; +} diff --git a/src/lib/windows-secret-acl.ts b/src/lib/windows-secret-acl.ts index e5ce2eee9..e46862525 100644 --- a/src/lib/windows-secret-acl.ts +++ b/src/lib/windows-secret-acl.ts @@ -170,6 +170,16 @@ export function resetHardenedStateForTests(): void { timedOutPaths.clear(); } +/** Forget a successful harden only after this exact ephemeral path is gone. */ +export function forgetHardenedSecretPath(targetPath: string): void { + hardenedPaths.delete(targetPath); +} + +/** Test seam for proving ephemeral success memos do not grow across replacements. */ +export function hardenedSecretPathCountForTests(): number { + return hardenedPaths.size; +} + function effectivePlatform(): string { return platformOverride ?? platform; } diff --git a/src/oauth/anthropic-routing.ts b/src/oauth/anthropic-routing.ts index 5422a56a4..e14bd0cee 100644 --- a/src/oauth/anthropic-routing.ts +++ b/src/oauth/anthropic-routing.ts @@ -28,6 +28,7 @@ import { seedPoolRotationAccount, } from "../codex/pool-rotation"; import type { OcxAccountPoolRotationStrategy, OcxConfig } from "../types"; +import { sweepExpiredOnWrite } from "../lib/state-store-sweeper"; const PROVIDER = "anthropic"; const DEFAULT_COOLDOWN_MS = 60_000; @@ -110,6 +111,16 @@ export function clearAnthropicAccountCooldown(accountId: string): boolean { return upstreamHealth.delete(accountId); } +export function sweepExpiredAnthropicRoutingHealth(now = Date.now()): number { + let removed = 0; + for (const [accountId, health] of upstreamHealth) { + if (health.cooldownUntil > now) continue; + upstreamHealth.delete(accountId); + removed += 1; + } + return removed; +} + /** Test / logout helper. */ export function clearAnthropicAccountPoolState(): void { upstreamHealth.clear(); @@ -464,6 +475,7 @@ export function rotateAnthropicAccountOn429( cooldownUntil: now + cooldownMs, cooldownSource: parsedRetry ? "retry-after" : "default", }); + sweepExpiredOnWrite(now); clearAnthropicSessionAffinityForAccount(failedAccountId); notePoolRotationFailure(POOL_KEY_ANTHROPIC, failedAccountId); diff --git a/src/oauth/index.ts b/src/oauth/index.ts index 72e8c6d1f..79b66417c 100644 --- a/src/oauth/index.ts +++ b/src/oauth/index.ts @@ -19,6 +19,7 @@ import { resolveProviderModelDiscoveryUrl } from "../providers/model-discovery"; import { resolveProviderTransport } from "../providers/xai-transport"; import { detectClaudeCodeToken, detectGrokCliToken, hasComparableGrokIdentity, isSameGrokIdentity, shouldAdoptGrokGeneration } from "./local-token-detect"; import { logOAuthEvent } from "./log"; +import { captureConfigGeneration, sweepExpiredOnWrite, type GenerationContext } from "../lib/state-store-sweeper"; export { CODEX_HEALTH_UNAVAILABLE_NOTE, MASKED_ACCOUNT_FALLBACK, @@ -59,6 +60,7 @@ interface AnthropicRefreshDeps { intentLock?:ReturnType; afterPrePersistRead?:()=>void|Promise } function verdictKey(p:string,a:string,c:OAuthCredentials){return `${p}\0${a}\0${credentialGeneration(c)}`;} function cached(p:string,a:string,c:OAuthCredentials,now:()=>number){const k=verdictKey(p,a,c),u=permanentRefreshFailures.get(k);if(u===undefined)return false;if(u<=now()){permanentRefreshFailures.delete(k);return false;}return true;} +export function sweepExpiredXaiPermanentFailureVerdicts(now=Date.now()):number{let removed=0;for(const[key,until]of permanentRefreshFailures){if(until>now)continue;permanentRefreshFailures.delete(key);removed+=1;}return removed;} export interface LoginOpts { forceLogin?: boolean; /** When set, persist into this account slot and require matching identity. */ reauthAccountId?: string } @@ -323,7 +325,7 @@ function merged(fresh: OAuthCredentials, previous: OAuthCredentials): OAuthCrede ...(fresh.kiro === undefined && previous.kiro ? { kiro: previous.kiro } : {}), }; } -export async function refreshXaiAccountWithLock(provider:string,accountId:string,def:OAuthProviderDef,callerCredential:OAuthCredentials,deps:XaiRefreshDeps={}):Promise{const now=deps.now??Date.now;const guard=await(deps.intentLock??createOAuthRefreshIntentLock(provider,accountId)).acquire();try{const stored=getAccountCredential(provider,accountId);if(!stored)throw new OAuthLoginRequiredError(provider);const active=getAccountSet(provider)?.activeAccountId===accountId,candidate=authoritative(stored,active,now);if(credentialGeneration(candidate)!==credentialGeneration(callerCredential)&&candidate.expires>now()+REFRESH_SKEW_MS){if(credentialGeneration(candidate)!==credentialGeneration(stored)){const o=await mergeAccountCredential(provider,accountId,candidate,{expectedGeneration:credentialGeneration(stored),afterPrePersistRead:deps.afterPrePersistRead});if(o.superseded){if(o.stored.expires>now()+REFRESH_SKEW_MS)return o.stored.access;throw new OAuthLoginRequiredError(provider);}}return candidate.access;}if(cached(provider,accountId,candidate,now))throw new OAuthLoginRequiredError(provider);const generation=credentialGeneration(candidate);try{const fresh=merged(await def.refresh(candidate.refresh),candidate);const o=await mergeAccountCredential(provider,accountId,fresh,{expectedGeneration:generation,afterPrePersistRead:deps.afterPrePersistRead});if(o.superseded){if(o.stored.expires>now()+REFRESH_SKEW_MS)return o.stored.access;throw new OAuthLoginRequiredError(provider);}permanentRefreshFailures.delete(verdictKey(provider,accountId,candidate));if(candidate.source==="local-cli")console.warn(XAI_LOCAL_CLI_DETACH_WARNING);return fresh.access;}catch(error){if(!terminal(error))throw error;permanentRefreshFailures.set(verdictKey(provider,accountId,candidate),now()+XAI_PERMANENT_FAILURE_TTL_MS);await markAccountNeedsReauthIfGeneration(provider,accountId,generation);throw new OAuthLoginRequiredError(provider);}}finally{guard.release();}} +export async function refreshXaiAccountWithLock(provider:string,accountId:string,def:OAuthProviderDef,callerCredential:OAuthCredentials,deps:XaiRefreshDeps={}):Promise{const writerGeneration=captureConfigGeneration();const now=deps.now??Date.now;const guard=await(deps.intentLock??createOAuthRefreshIntentLock(provider,accountId)).acquire();try{const stored=getAccountCredential(provider,accountId);if(!stored)throw new OAuthLoginRequiredError(provider);const active=getAccountSet(provider)?.activeAccountId===accountId,candidate=authoritative(stored,active,now);if(credentialGeneration(candidate)!==credentialGeneration(callerCredential)&&candidate.expires>now()+REFRESH_SKEW_MS){if(credentialGeneration(candidate)!==credentialGeneration(stored)){const o=await mergeAccountCredential(provider,accountId,candidate,{expectedGeneration:credentialGeneration(stored),afterPrePersistRead:deps.afterPrePersistRead});if(o.superseded){if(o.stored.expires>now()+REFRESH_SKEW_MS)return o.stored.access;throw new OAuthLoginRequiredError(provider);}}return candidate.access;}if(cached(provider,accountId,candidate,now))throw new OAuthLoginRequiredError(provider);const generation=credentialGeneration(candidate);try{const fresh=merged(await def.refresh(candidate.refresh),candidate);const o=await mergeAccountCredential(provider,accountId,fresh,{expectedGeneration:generation,afterPrePersistRead:deps.afterPrePersistRead});if(o.superseded){if(o.stored.expires>now()+REFRESH_SKEW_MS)return o.stored.access;throw new OAuthLoginRequiredError(provider);}permanentRefreshFailures.delete(verdictKey(provider,accountId,candidate));if(candidate.source==="local-cli")console.warn(XAI_LOCAL_CLI_DETACH_WARNING);return fresh.access;}catch(error){if(!terminal(error))throw error;const failedAt=now();permanentRefreshFailures.set(verdictKey(provider,accountId,candidate),failedAt+XAI_PERMANENT_FAILURE_TTL_MS);sweepExpiredOnWrite(failedAt);await markAccountNeedsReauthIfGeneration(provider,accountId,generation,writerGeneration);throw new OAuthLoginRequiredError(provider);}}finally{guard.release();}} function newerClaudeCredential(stored: OAuthCredentials, now: number): OAuthCredentials | undefined { if (stored.source !== "local-cli") return undefined; @@ -339,6 +341,7 @@ export async function refreshAnthropicAccountWithLock( callerCredential: OAuthCredentials, deps: AnthropicRefreshDeps = {}, ): Promise { + const writerGeneration = captureConfigGeneration(); const now = deps.now ?? Date.now; const guard = await (deps.intentLock ?? createOAuthRefreshIntentLock(provider, accountId)).acquire(); try { @@ -362,7 +365,7 @@ export async function refreshAnthropicAccountWithLock( return disk.access; } if (pendingIntent?.uncertain || pendingIntent?.generation === generation) { - await markAccountNeedsReauthIfGeneration(provider, accountId, generation); + await markAccountNeedsReauthIfGeneration(provider, accountId, generation, writerGeneration); throw new OAuthLoginRequiredError(provider); } if (pendingIntent) clearOAuthRefreshIntent(provider, accountId, pendingIntent.generation); @@ -389,7 +392,7 @@ export async function refreshAnthropicAccountWithLock( return fresh.access; } catch (error) { if (!terminal(error)) throw error; - await markAccountNeedsReauthIfGeneration(provider, accountId, generation); + await markAccountNeedsReauthIfGeneration(provider, accountId, generation, writerGeneration); clearOAuthRefreshIntent(provider, accountId, generation); throw new OAuthLoginRequiredError(provider); } @@ -405,6 +408,7 @@ export async function refreshGenericAccountWithLock( callerCredential: OAuthCredentials, deps: GenericRefreshDeps = {}, ): Promise { + const writerGeneration = captureConfigGeneration(); logOAuthEvent("OAuth refresh started", { provider, accountId }); const guard = await (deps.intentLock ?? createOAuthRefreshIntentLock(provider, accountId)).acquire(); try { @@ -432,7 +436,7 @@ export async function refreshGenericAccountWithLock( return fresh.access; } catch (error) { if (!terminal(error)) throw error; - await markAccountNeedsReauthIfGeneration(provider, accountId, generation); + await markAccountNeedsReauthIfGeneration(provider, accountId, generation, writerGeneration); throw new OAuthLoginRequiredError(provider); } } finally { @@ -831,6 +835,20 @@ interface ManualCodeSlot { expectedState?: string; } const loginManual = new Map(); +let lastOAuthFlowReconciledGeneration = 0; + +export function reconcileOAuthFlowState(context: GenerationContext): number { + if (context.generation <= lastOAuthFlowReconciledGeneration) return 0; + let removed = 0; + for (const [provider, state] of loginState) { + if (context.providerNames.has(provider) || !state.done || loginAbort.has(provider)) continue; + if (loginState.delete(provider)) removed += 1; + if (loginManual.delete(provider)) removed += 1; + if (loginAbort.delete(provider)) removed += 1; + } + lastOAuthFlowReconciledGeneration = context.generation; + return removed; +} function clearManualCodeSlot(provider: string): void { loginManual.delete(provider); diff --git a/src/oauth/store.ts b/src/oauth/store.ts index c0c7c774c..fccef1557 100644 --- a/src/oauth/store.ts +++ b/src/oauth/store.ts @@ -21,10 +21,27 @@ import { join } from "node:path"; import { getConfigDir, atomicWriteFile, backupInvalidConfig, hardenConfigDir, hardenExistingSecret } from "../config"; import { assertNotRealHomeUnderTest } from "../lib/test-home-guard"; import { recordOwnedConfigPath } from "../lib/config-ownership"; +import { + captureConfigGeneration, + type GenerationContext, +} from "../lib/state-store-sweeper"; import { validateCopilotApiBaseUrl } from "./github-copilot"; import type { OAuthCredentialSource, OAuthCredentials, ProviderAccount, ProviderAccountSet } from "./types"; type AuthStore = Record; +let lastReconciledGeneration = 0; +let liveOAuthAccountKeys = new Set(); + +function oauthAccountKey(provider: string, accountId: string): string { + return `${provider}\0${accountId}`; +} + +export function reconcileOAuthReauthState(context: GenerationContext): number { + if (context.generation <= lastReconciledGeneration) return 0; + liveOAuthAccountKeys = new Set(context.oauthAccountKeys); + lastReconciledGeneration = context.generation; + return 0; +} /** Providers whose account set is pinned to a single slot (see module doc). */ const SINGLE_SLOT_PROVIDERS = new Set(["chatgpt"]); @@ -395,6 +412,17 @@ export function listAccounts(provider: string): ProviderAccount[] { return loadAuthStore()[provider]?.accounts ?? []; } +export function listLiveOAuthAccountKeys( + providerNames: ReadonlySet, +): ReadonlySet { + const keys = new Set(); + for (const [provider, accountSet] of Object.entries(loadAuthStore())) { + if (!providerNames.has(provider)) continue; + for (const account of accountSet.accounts) keys.add(`${provider}\0${account.id}`); + } + return keys; +} + export function getAccountCredential(provider: string, accountId: string): OAuthCredentials | null { return loadAuthStore()[provider]?.accounts.find(a => a.id === accountId)?.credential ?? null; } @@ -432,7 +460,7 @@ export async function setAccountAlias(provider: string, accountId: string, alias /** Remove one account by id; active removal promotes the first remaining account. */ export async function removeAccount(provider: string, accountId: string): Promise { - return await mutateStore(store => { + const removed = await mutateStore(store => { const set = store[provider]; if (!set) return false; const before = set.accounts.length; @@ -445,6 +473,7 @@ export async function removeAccount(provider: string, accountId: string): Promis if (set.activeAccountId === accountId) set.activeAccountId = set.accounts[0]!.id; return true; }); + return removed; } /** Replace or clear a provider account set (used for transactional Kiro add-account rollback). */ @@ -470,7 +499,13 @@ export async function replaceProviderAccountSet( }); } -export async function markAccountNeedsReauth(provider: string, accountId: string, needsReauth: boolean): Promise { +export async function markAccountNeedsReauth( + provider: string, + accountId: string, + needsReauth: boolean, + writerGeneration = captureConfigGeneration(), +): Promise { + if (writerGeneration < lastReconciledGeneration && !liveOAuthAccountKeys.has(oauthAccountKey(provider, accountId))) return; await mutateStore(store => { const account = store[provider]?.accounts.find(a => a.id === accountId); if (!account) return; @@ -480,4 +515,4 @@ export async function markAccountNeedsReauth(provider: string, accountId: string } export async function mergeAccountCredential(provider:string,accountId:string,credential:OAuthCredentials,opts:{expectedGeneration?:string;afterPrePersistRead?:()=>void|Promise}={}):Promise<{superseded:false}|{superseded:true;stored:OAuthCredentials}>{const safe=normalizeCredential(credential);if(!safe)throw new Error("Refusing to persist invalid OAuth credential");return await mutateStore(async store=>{await opts.afterPrePersistRead?.();const account=store[provider]?.accounts.find(x=>x.id===accountId);if(!account)throw new Error(`OAuth account disappeared before persist: ${provider}`);if(opts.expectedGeneration!==undefined&&credentialGeneration(account.credential)!==opts.expectedGeneration)return{superseded:true,stored:account.credential};account.credential=safe;delete account.needsReauth;return{superseded:false};});} -export async function markAccountNeedsReauthIfGeneration(provider:string,accountId:string,generation:string):Promise{return await mutateStore(store=>{const account=store[provider]?.accounts.find(x=>x.id===accountId);if(!account?.credential||credentialGeneration(account.credential)!==generation)return false;account.needsReauth=true;return true;});} +export async function markAccountNeedsReauthIfGeneration(provider:string,accountId:string,generation:string,writerGeneration=captureConfigGeneration()):Promise{const key=oauthAccountKey(provider,accountId);if(writerGeneration{const account=store[provider]?.accounts.find(x=>x.id===accountId);if(!account?.credential||credentialGeneration(account.credential)!==generation)return false;if(writerGeneration" / "codex:"). const backoff = new Map(); +let lastReconciledGeneration = 0; +let liveBackoffKeys = new Set(); /** Test hook: clear backoff state between cases. */ export function __resetGuardianState(): void { @@ -87,7 +90,15 @@ function inBackoff(key: string, nowMs: number): boolean { return entry !== undefined && entry.retryAfterMs > nowMs; } -function recordFailure(key: string, nowMs: number, baseSeconds: number, maxSeconds: number, permanent: boolean): void { +function recordFailure( + key: string, + nowMs: number, + baseSeconds: number, + maxSeconds: number, + permanent: boolean, + writerGeneration = captureConfigGeneration(), +): void { + if (writerGeneration < lastReconciledGeneration && !liveBackoffKeys.has(key)) return; const prev = backoff.get(key); const attempts = (prev?.attempts ?? 0) + 1; // Permanent failures (revoked/expired refresh token) wait the full ceiling — nothing but a @@ -116,6 +127,7 @@ async function runWithConcurrency(tasks: Array<() => Promise>, limit: numb * ids only, never tokens). */ export async function guardianSweep(nowMs: number = Date.now()): Promise { + const writerGeneration = captureConfigGeneration(); const config: OcxConfig = loadConfig(); const g = config.tokenGuardian; const result: GuardianSweepResult = { enabled: !!g?.enabled, refreshed: [], warmed: [], failed: [], skippedBackoff: [] }; @@ -143,7 +155,7 @@ export async function guardianSweep(nowMs: number = Date.now()): Promise(); + for (const id of context.codexAccountIds) valid.add(`codex:${id}`); + for (const key of context.oauthAccountKeys) { + const separator = key.indexOf("\0"); + if (separator <= 0) continue; + valid.add(`oauth:${key.slice(0, separator)}:${key.slice(separator + 1)}`); + } + let removed = 0; + for (const key of backoff.keys()) { + if (valid.has(key)) continue; + backoff.delete(key); + removed += 1; + } + liveBackoffKeys = valid; + lastReconciledGeneration = context.generation; + return removed; +} + /** * Start the background sweep loop. Returns a handle whose stop() clears the pending timer (in-flight * refreshes settle on their own). Schedules recursively so each interval gets fresh jitter. The loop diff --git a/src/providers/key-failover.ts b/src/providers/key-failover.ts index b3894fa7c..32a308514 100644 --- a/src/providers/key-failover.ts +++ b/src/providers/key-failover.ts @@ -11,6 +11,7 @@ import { saveConfigPreservingClaudeCode } from "../config"; import type { OcxConfig, OcxProviderConfig } from "../types"; import { resolveProviderTransport, type OcxProviderTransport } from "./xai-transport"; +import { sweepExpiredOnWrite } from "../lib/state-store-sweeper"; // ---- cooldown state (in-memory, same as codex/routing.ts) ---- @@ -99,6 +100,7 @@ export function rotateKeyOn429( keyCooldowns.set(cooldownKey(providerName, currentEntry.id), { cooldownUntil: now + cooldownMs, }); + sweepExpiredOnWrite(now); } // Lost the race: someone already rotated away from the failed key. If the live key is healthy, @@ -131,6 +133,16 @@ export function rotateKeyOn429( return null; } +export function sweepExpiredApiKeyCooldowns(now = Date.now()): number { + let removed = 0; + for (const [key, cooldown] of keyCooldowns) { + if (cooldown.cooldownUntil > now) continue; + keyCooldowns.delete(key); + removed += 1; + } + return removed; +} + interface RotateProviderTransportOptions { retryAfter?: string | null; now?: number; diff --git a/src/providers/openai-sidecar.ts b/src/providers/openai-sidecar.ts index 40728a32f..b3ca44509 100644 --- a/src/providers/openai-sidecar.ts +++ b/src/providers/openai-sidecar.ts @@ -101,7 +101,10 @@ export async function resolveFirstUsableOpenAiSidecar( config, authContext.accountId, outcome, - { probeLeaseId: authContext.probeLeaseId }, + { + probeLeaseId: authContext.probeLeaseId, + writerGeneration: authContext.writerGeneration, + }, ), } : {}), diff --git a/src/providers/quota.ts b/src/providers/quota.ts index 9a5831df7..131bc1358 100644 --- a/src/providers/quota.ts +++ b/src/providers/quota.ts @@ -7,6 +7,11 @@ import { antigravityUserAgent } from "../adapters/client-fingerprint"; import { getProviderRegistryEntry, providerCodexAccountMode } from "./registry"; import type { OcxConfig, OcxProviderConfig } from "../types"; import { isCanonicalOpenAiForwardProvider, OPENAI_CODEX_PROVIDER_ID } from "./openai-tiers"; +import { + captureConfigGeneration, + sweepExpiredOnWrite, + type GenerationContext, +} from "../lib/state-store-sweeper"; /** Match oauth/index REFRESH_SKEW_MS — use stored access without refresh when still fresh. */ const ACCOUNT_TOKEN_SKEW_MS = 60_000; @@ -243,6 +248,8 @@ async function fetchAnthropicQuota(provider: string): Promise(); const accountQuotaInflight = new Map>(); +let lastReconciledGeneration = 0; +let liveAccountQuotaKeys = new Set(); +let liveProviderQuotaKeys = new Set(); + +function mayCommitAccountQuotaKey(key: string, writerGeneration: number): boolean { + return writerGeneration >= lastReconciledGeneration || liveAccountQuotaKeys.has(key); +} + +function mayCommitProviderQuotaKey(key: string, writerGeneration: number): boolean { + return writerGeneration >= lastReconciledGeneration || liveProviderQuotaKeys.has(key); +} export interface ProviderAccountQuota { accountId: string; @@ -324,6 +342,35 @@ export function setCachedProviderAccountQuotaForTests( accountQuotaCache.set(key, { ts: Date.now(), quota }); } +export function sweepExpiredProviderAccountQuotaRows(now = Date.now()): number { + let removed = 0; + for (const [key, entry] of accountQuotaCache) { + if (entry.ts + ACCOUNT_QUOTA_TTL_MS > now) continue; + accountQuotaCache.delete(key); + removed += 1; + } + return removed; +} + +export function reconcileProviderAccountQuotaRows(context: GenerationContext): number { + if (context.generation <= lastReconciledGeneration) return 0; + let removed = 0; + for (const key of accountQuotaCache.keys()) { + if (context.oauthAccountKeys.has(key)) continue; + accountQuotaCache.delete(key); + removed += 1; + } + if (cache) { + const reports = cache.response.reports.filter(report => context.providerNames.has(report.provider)); + removed += cache.response.reports.length - reports.length; + cache = { ...cache, response: { ...cache.response, reports } }; + } + liveAccountQuotaKeys = new Set(context.oauthAccountKeys); + liveProviderQuotaKeys = new Set(context.providerNames); + lastReconciledGeneration = context.generation; + return removed; +} + /** Drop cached per-account rows (all, or just one provider's). */ export function clearAccountQuotaCache(provider?: string): void { if (!provider) { @@ -369,6 +416,7 @@ async function fetchAccountQuota( forceRefresh: boolean, ): Promise { const key = accountCacheKey(provider, accountId); + const writerGeneration = captureConfigGeneration(); const cached = accountQuotaCache.get(key); if (!forceRefresh && cached && Date.now() - cached.ts < ACCOUNT_QUOTA_TTL_MS) return cached; const joinable = accountQuotaInflight.get(key); @@ -386,11 +434,17 @@ async function fetchAccountQuota( quota: cached?.quota ?? null, unavailable: true, }; - accountQuotaCache.set(key, entry); + if (mayCommitAccountQuotaKey(key, writerGeneration)) { + accountQuotaCache.set(key, entry); + sweepExpiredOnWrite(entry.ts); + } return entry; } const entry: AccountQuotaCacheEntry = { ts: Date.now(), quota }; - accountQuotaCache.set(key, entry); + if (mayCommitAccountQuotaKey(key, writerGeneration)) { + accountQuotaCache.set(key, entry); + sweepExpiredOnWrite(entry.ts); + } return entry; } catch { const entry: AccountQuotaCacheEntry = { @@ -398,7 +452,10 @@ async function fetchAccountQuota( quota: cached?.quota ?? null, unavailable: true, }; - accountQuotaCache.set(key, entry); + if (mayCommitAccountQuotaKey(key, writerGeneration)) { + accountQuotaCache.set(key, entry); + sweepExpiredOnWrite(entry.ts); + } return entry; } })().finally(() => { @@ -861,6 +918,7 @@ async function maybeFetchProviderQuota( export async function fetchProviderQuotaReports(config: OcxConfig, forceRefresh = false): Promise { const key = cacheKey(config); + const writerGeneration = captureConfigGeneration(); const now = Date.now(); // The cache fast path must not extend a preserved last-good row past its 30-minute bound: // a row preserved at age 29:59 plus a full 5-minute TTL would otherwise serve until ~35min. @@ -893,7 +951,10 @@ export async function fetchProviderQuotaReports(config: OcxConfig, forceRefresh const response = { generatedAt: Date.now(), reports: [...byProvider.values()] }; // Commit only when this probe still holds authority (no clear/force superseded it). - if (epoch === invalidationEpoch) cache = { key, ts: Date.now(), response }; + if (epoch === invalidationEpoch) { + const reports = response.reports.filter(item => mayCommitProviderQuotaKey(item.provider, writerGeneration)); + cache = { key, ts: Date.now(), response: { ...response, reports } }; + } return response; })(); diff --git a/src/responses/spill-store.ts b/src/responses/spill-store.ts index 669ea22a3..19b030dc2 100644 --- a/src/responses/spill-store.ts +++ b/src/responses/spill-store.ts @@ -17,7 +17,7 @@ import { import { createHash, randomBytes } from "node:crypto"; import { join } from "node:path"; import { getConfigDir } from "../config"; -import { hardenSecretDir, hardenSecretPath } from "../lib/windows-secret-acl"; +import { forgetHardenedSecretPath, hardenSecretDir, hardenSecretPath } from "../lib/windows-secret-acl"; import type { OcxProviderContinuationState } from "../types"; export const RESPONSE_SPILL_VERSION = 1; @@ -145,8 +145,14 @@ function closeFile(fd: number): void { } function unlink(path: string): void { - if (spillIoForTest?.unlink) spillIoForTest.unlink(path); - else unlinkSync(path); + try { + if (spillIoForTest?.unlink) spillIoForTest.unlink(path); + else unlinkSync(path); + forgetHardenedSecretPath(path); + } catch (error) { + if (isErrno(error, "ENOENT")) forgetHardenedSecretPath(path); + throw error; + } } function publishNoReplace(tempPath: string, destinationPath: string): void { diff --git a/src/router.ts b/src/router.ts index fb093d986..8b8f56569 100644 --- a/src/router.ts +++ b/src/router.ts @@ -152,6 +152,15 @@ function configuredOriginForLog(url: string): string { // `routedProviderConfig` runs per request, so warn once per (provider, discarded, effective) triple. // Keyed by the URLs too: editing config.json to a different wrong value warns again. const discardedBaseUrlWarnings = new Set(); +let lastWarningReconciledGeneration = 0; + +export function reconcileRouterWarningMemos(generation: number): number { + if (generation <= lastWarningReconciledGeneration) return 0; + const removed = discardedBaseUrlWarnings.size; + discardedBaseUrlWarnings.clear(); + lastWarningReconciledGeneration = generation; + return removed; +} /** * A pinned registry entry — non-template `baseUrl`, no `allowBaseUrlOverride` — outranks a saved diff --git a/src/server/index.ts b/src/server/index.ts index 4484202e4..899da9492 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -21,6 +21,11 @@ import { import { reconcileOAuthProviders } from "../oauth"; import { invalidateCodexModelsCache } from "../codex/catalog"; import { startMemoryWatchdog } from "./memory-watchdog"; +import { + reconcileLiveStateStores, + setLiveStateStoreConfig, +} from "../lib/state-store-registrations"; +import { startStateStoreSweeper } from "../lib/state-store-sweeper"; import { setStorageCleanupPolicyLiveSink } from "../storage/policy"; import { setStorageCleanupPolicyJobLiveApply } from "../storage/policy-job"; import { scheduleStorageCleanupStartupRun, startStorageCleanupScheduler } from "../storage/policy-scheduler"; @@ -255,12 +260,14 @@ function attachLiveSidebandUpstream(ws: ServerWebSocket): void { export function startServer(port?: number) { const config = runAlibabaRegionStartupMigration(runOpenAiTierStartupMigration(loadConfig())); + setLiveStateStoreConfig(config); applyProxyEnv(config); assertServerAuthConfig(config); const managementAuth = initializeManagementAuthState(config); // Refresh OAuth provider presets (models/noReasoningModels) from the registry so a proxy update // adding/dropping models reaches existing configs on start — not just fresh installs. reconcileOAuthProviders(config); + reconcileLiveStateStores(); // Seed default featured subagent models on first run only (UNSET → defaults). A user-set list, // even [], is left alone so GUI removals persist. if (config.subagentModels === undefined) { @@ -304,6 +311,7 @@ export function startServer(port?: number) { // #314: warn-only RSS observability (unref'd, idempotent — safe under repeated // startServer(0) in tests). Snapshot surfaces via GET /api/system/memory. startMemoryWatchdog(); + startStateStoreSweeper(); // Issue #42 Phase 3: opt-in archived auto-cleanup (default OFF). Unref'd hourly // tick for daily/weekly; startup evaluation is fire-and-forget after listen. // Heavy work runs in a Worker via the single-flight job controller. diff --git a/src/server/lifecycle.ts b/src/server/lifecycle.ts index be1decf76..332bea243 100644 --- a/src/server/lifecycle.ts +++ b/src/server/lifecycle.ts @@ -6,6 +6,7 @@ import { } from "../storage/policy-job"; import { abortRestoreTrashJobAsync } from "../storage/restore-job"; import { stopStorageCleanupScheduler } from "../storage/policy-scheduler"; +import { stopStateStoreSweeper } from "../lib/state-store-sweeper"; import { cancelQueuedStorageWorkerSpawns, drainStorageWorkers, @@ -98,6 +99,7 @@ export async function drainAndShutdown( // Abort each job independently so one wedged join cannot skip the other, // then drain leftovers; failures must not prevent `server.stop`. stopStorageCleanupScheduler(); + stopStateStoreSweeper(); cancelQueuedStorageWorkerSpawns(); const shutdownJoins = await Promise.allSettled([ abortStorageCleanupPolicyJobAsync(), diff --git a/src/server/management-auth.ts b/src/server/management-auth.ts index 52fd46401..f01c23c65 100644 --- a/src/server/management-auth.ts +++ b/src/server/management-auth.ts @@ -13,7 +13,7 @@ import { } from "node:fs"; import { dirname, join } from "node:path"; import { adminApiTokenFilePath } from "../lib/admin-secrets"; -import { hardenSecretDir, hardenSecretPath } from "../lib/windows-secret-acl"; +import { forgetHardenedSecretPath, hardenSecretDir, hardenSecretPath } from "../lib/windows-secret-acl"; import type { OcxConfig } from "../types"; import { isAllowedManagementOrigin, @@ -72,8 +72,17 @@ function readExistingToken(path: string): string { return token; } -function removeBestEffort(path: string): void { - try { unlinkSync(path); } catch { /* fail-closed state is preserved by the caller */ } +export function removeManagementTokenPathBestEffort( + path: string, + remove: (path: string) => void = unlinkSync, +): void { + try { + remove(path); + forgetHardenedSecretPath(path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") forgetHardenedSecretPath(path); + /* other failures retain fail-closed state for the caller */ + } } function createTokenFile(path: string): string { @@ -102,13 +111,13 @@ function createTokenFile(path: string): string { if (!finalHardened.ok) throw new Error("management token file ACL hardening did not complete"); return token; } catch (error) { - if (linked) removeBestEffort(path); + if (linked) removeManagementTokenPathBestEffort(path); throw error; } finally { if (fd !== null) { try { closeSync(fd); } catch { /* best effort */ } } - removeBestEffort(temporary); + removeManagementTokenPathBestEffort(temporary); } } diff --git a/src/server/management/combo-routes.ts b/src/server/management/combo-routes.ts index 57c51e5bb..8e1c64ba3 100644 --- a/src/server/management/combo-routes.ts +++ b/src/server/management/combo-routes.ts @@ -53,6 +53,7 @@ import { } from "../../lib/debug-settings"; import type { OcxClaudeCodeConfig, OcxConfig, OcxCustomModel, OcxProviderConfig } from "../../types"; import { drainAndShutdown } from "../lifecycle"; +import { reconcileLiveStateStores } from "../../lib/state-store-registrations"; import { filterRequestLogs, getRequestLogEntries, type RequestLogEntry } from "../request-log"; import { estimateComboCost, estimateRequestCost, normalizeCostTokens, tokensPerSecond } from "../../usage/cost"; import type { PersistedUsageAttempt } from "../../usage/log"; @@ -186,6 +187,7 @@ export async function handleComboRoutes(ctx: ManagementContext): Promise reconcileLiveConfigFromDisk(config, persistedBaseline), + onSettled: () => { + reconcileLiveConfigFromDisk(config, persistedBaseline); + reconcileLiveStateStores(); + }, }); if (authUrl && !deviceCode) { // Open the browser server-side (the proxy runs on the user's machine) — the GUI's @@ -200,6 +204,7 @@ export async function handleOauthAccountRoutes(ctx: ManagementContext): Promise< const provider = (url.searchParams.get("provider") ?? "").trim().toLowerCase(); if (!isPublicOAuthProvider(provider)) return jsonResponse({ error: "unknown oauth provider" }, 400); await removeCredential(provider); + reconcileLiveStateStores(); clearLoginState(provider); // Drop cached/last-good quota rows tied to the removed credential. const { clearProviderQuotaCache, clearAccountQuotaCache } = await import("../../providers/quota"); @@ -345,6 +350,7 @@ export async function handleOauthAccountRoutes(ctx: ManagementContext): Promise< ...(stickyLimit !== undefined ? { stickyLimit } : {}), }; saveConfigPreservingClaudeCode(config); + reconcileLiveStateStores(); return jsonResponse({ ok: true, provider, @@ -387,6 +393,7 @@ export async function handleOauthAccountRoutes(ctx: ManagementContext): Promise< if (!id) return jsonResponse({ error: "missing id" }, 400); const { removeAccount, getAccountSet } = await import("../../oauth/store"); if (!(await removeAccount(provider, id))) return jsonResponse({ error: "account not found" }, 404); + reconcileLiveStateStores(); if (provider === "anthropic") { const { clearAnthropicAccountCooldown, clearAnthropicSessionAffinityForAccount } = await import("../../oauth/anthropic-routing"); clearAnthropicAccountCooldown(id); @@ -514,6 +521,7 @@ export async function handleOauthAccountRoutes(ctx: ManagementContext): Promise< const entry = { id: randomUUID(), name, key, createdAt: new Date().toISOString() }; config.apiKeys = [...(config.apiKeys ?? []), entry]; saveConfigPreservingClaudeCode(config); + reconcileLiveStateStores(); return jsonResponse({ id: entry.id, name: entry.name, key: entry.key, createdAt: entry.createdAt }, 201, req, config); } @@ -527,6 +535,7 @@ export async function handleOauthAccountRoutes(ctx: ManagementContext): Promise< if (!entry) return jsonResponse({ error: "key not found" }, 404, req, config); entry.name = nameField.value; saveConfigPreservingClaudeCode(config); + reconcileLiveStateStores(); // Never echo key material from a rename. return jsonResponse({ id: entry.id, name: entry.name, createdAt: entry.createdAt }, 200, req, config); } @@ -540,6 +549,7 @@ export async function handleOauthAccountRoutes(ctx: ManagementContext): Promise< // A stale id must not read as a successful revocation. if (config.apiKeys.length === before) return jsonResponse({ error: "key not found" }, 404, req, config); saveConfigPreservingClaudeCode(config); + reconcileLiveStateStores(); return jsonResponse({ success: true }, 200, req, config); } return null; diff --git a/src/server/management/provider-routes.ts b/src/server/management/provider-routes.ts index ffa1724d6..c3933f6ae 100644 --- a/src/server/management/provider-routes.ts +++ b/src/server/management/provider-routes.ts @@ -23,6 +23,7 @@ import { } from "../../oauth"; import { removeCredential } from "../../oauth/store"; import { providerDestinationResolvedError } from "../../lib/destination-policy"; +import { reconcileLiveStateStores } from "../../lib/state-store-registrations"; import { ProviderOutboundPolicyError, providerOutboundGet, providerRedirectError } from "../../lib/provider-outbound"; import { enrichProviderFromCatalog, listKeyLoginProviders } from "../../oauth/key-providers"; import { deriveProviderPresets } from "../../providers/derive"; @@ -135,6 +136,7 @@ export async function handleProviderRoutes(ctx: ManagementContext): Promise recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, { threadId, probeLeaseId: authCtx.probeLeaseId, + writerGeneration: authCtx.writerGeneration, }) : undefined; } @@ -343,7 +344,11 @@ async function retryCodexPoolOnAlternateAccount( const quotaMeta = codexQuotaOutcomeMeta(firstResponse); if (outcomeStatus === 429 || outcomeStatus === 402) { const { applyAccountQuotaFromUpstreamHeaders } = await import("../../codex/auth-api"); - applyAccountQuotaFromUpstreamHeaders(firstAuthCtx.accountId, firstResponse.headers); + applyAccountQuotaFromUpstreamHeaders( + firstAuthCtx.accountId, + firstResponse.headers, + firstAuthCtx.writerGeneration, + ); } if (!shouldDeferCodexResetDerivedCooldown(firstResponse, options.deferCodexResetDerivedCooldown)) { recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, { @@ -352,6 +357,7 @@ async function retryCodexPoolOnAlternateAccount( modelId: route.modelId, probeLeaseId: codexProbeLeaseId(firstAuthCtx), probeQuotaScope: codexProbeQuotaScope(firstAuthCtx), + writerGeneration: firstAuthCtx.writerGeneration, // Retry already advanced the RR ring via excludeAccountId — reuse for promotion. ...(retryAuthCtx.accountId ? { promoteAccountId: retryAuthCtx.accountId } : {}), }); @@ -427,6 +433,7 @@ export function codexForwardTerminalOutcomeRecorder( modelId, probeLeaseId: codexProbeLeaseId(authCtx), probeQuotaScope: codexProbeQuotaScope(authCtx), + writerGeneration: authCtx.writerGeneration, }); return; } @@ -447,6 +454,7 @@ export function codexForwardTerminalOutcomeRecorder( modelId, probeLeaseId: codexProbeLeaseId(authCtx), probeQuotaScope: codexProbeQuotaScope(authCtx), + writerGeneration: authCtx.writerGeneration, }); }; } @@ -999,7 +1007,7 @@ export async function handleComboResponses( ); (logCtx.attempts ??= []).push(attempt); attemptRetained = true; - noteComboSuccess(comboId, combo, pick.target); + noteComboSuccess(comboId, combo, pick.target, pick.writerGeneration); Object.assign(logCtx, childLog, { requestedModel, model: requestedModel, @@ -1485,6 +1493,7 @@ export async function handleResponses( modelId: route.modelId, probeLeaseId: codexProbeLeaseId(authCtx), probeQuotaScope: codexProbeQuotaScope(authCtx), + writerGeneration: authCtx.writerGeneration, }); } const msg = outcome === "timeout" @@ -1581,7 +1590,11 @@ export async function handleResponses( // Prefer primary when present, fall back to secondary for compatibility. const quotaMeta = codexQuotaOutcomeMeta(upstreamResponse); const { applyAccountQuotaFromUpstreamHeaders } = await import("../../codex/auth-api"); - applyAccountQuotaFromUpstreamHeaders(authCtx.accountId, upstreamResponse.headers); + applyAccountQuotaFromUpstreamHeaders( + authCtx.accountId, + upstreamResponse.headers, + authCtx.writerGeneration, + ); if (terminalBodyWillRecord) { options.setTerminalOutcomeRecorder?.((status, httpStatusOverride) => { terminalRecorder(status, httpStatusOverride); @@ -1613,6 +1626,7 @@ export async function handleResponses( modelId: route.modelId, probeLeaseId: codexProbeLeaseId(authCtx), probeQuotaScope: codexProbeQuotaScope(authCtx), + writerGeneration: authCtx.writerGeneration, }); } } diff --git a/src/tray/windows.ts b/src/tray/windows.ts index eec7251af..78c2e681b 100644 --- a/src/tray/windows.ts +++ b/src/tray/windows.ts @@ -5,7 +5,7 @@ import { homedir } from "node:os"; import { join, resolve } from "node:path"; import { expandUserPath, getConfigDir } from "../config"; import { durableBunPath } from "../lib/bun-runtime"; -import { hardenSecretDir, hardenSecretPath } from "../lib/windows-secret-acl"; +import { forgetHardenedSecretPath, hardenSecretDir, hardenSecretPath } from "../lib/windows-secret-acl"; import { recordOwnedConfigPath } from "../lib/config-ownership"; const RUN_KEY = "HKCU\\Software\\Microsoft\\Windows\\CurrentVersion\\Run"; @@ -218,18 +218,45 @@ function readState(): WindowsTrayState | null { } } -function replaceOwnedFile(path: string, contents: string | Buffer): void { +export interface WindowsTrayOwnedFileIO { + write(path: string, contents: string | Buffer): void; + harden(path: string): void; + rename(source: string, destination: string): void; + unlink(path: string): void; +} + +export function replaceWindowsTrayOwnedFile( + path: string, + contents: string | Buffer, + io: WindowsTrayOwnedFileIO = { + write: (target, value) => writeFileSync(target, value, { mode: 0o600 }), + harden: target => { + try { chmodSync(target, 0o600); } catch { /* best-effort */ } + if (process.platform !== "win32") return; + const hardened = hardenSecretPath(target, { required: true }); + if (!hardened.ok) throw new Error("Windows tray ACL hardening did not complete; refusing to persist executable state."); + }, + rename: renameSync, + unlink: unlinkSync, + }, +): void { const temporary = `${path}.${process.pid}.${Date.now()}.tmp`; - writeFileSync(temporary, contents, { mode: 0o600 }); + io.write(temporary, contents); + let renamed = false; try { - try { chmodSync(temporary, 0o600); } catch { /* best-effort */ } - if (process.platform === "win32") { - const hardened = hardenSecretPath(temporary, { required: true }); - if (!hardened.ok) throw new Error("Windows tray ACL hardening did not complete; refusing to persist executable state."); - } - renameSync(temporary, path); + io.harden(temporary); + io.rename(temporary, path); + renamed = true; + forgetHardenedSecretPath(temporary); } finally { - try { if (existsSync(temporary)) unlinkSync(temporary); } catch { /* best-effort */ } + if (!renamed) { + try { + io.unlink(temporary); + forgetHardenedSecretPath(temporary); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") forgetHardenedSecretPath(temporary); + } + } } } @@ -239,7 +266,7 @@ function writeState( runCommand: string, ): void { const path = trayStatePath(); - replaceOwnedFile(path, JSON.stringify({ version: TRAY_STATE_VERSION, ...entry, runValue, runCommand }, null, 2) + "\n"); + replaceWindowsTrayOwnedFile(path, JSON.stringify({ version: TRAY_STATE_VERSION, ...entry, runValue, runCommand }, null, 2) + "\n"); } export function parseWindowsTrayRunValue(output: string, runValue: string): string | null { @@ -580,21 +607,21 @@ export function installWindowsTray(startNow = true): WindowsTrayStatus { ])); const restorePreviousInstall = () => { try { - if (previousScriptBytes) replaceOwnedFile(entry.script, previousScriptBytes); + if (previousScriptBytes) replaceWindowsTrayOwnedFile(entry.script, previousScriptBytes); else if (existsSync(entry.script)) unlinkSync(entry.script); } catch { /* rollback best-effort */ } try { - if (previousLauncherBytes) replaceOwnedFile(launcherPath, previousLauncherBytes); + if (previousLauncherBytes) replaceWindowsTrayOwnedFile(launcherPath, previousLauncherBytes); else if (existsSync(launcherPath)) unlinkSync(launcherPath); } catch { /* rollback best-effort */ } for (const [path, contents] of previousIconBytes) { try { - if (contents) replaceOwnedFile(path, contents); + if (contents) replaceWindowsTrayOwnedFile(path, contents); else if (existsSync(path)) unlinkSync(path); } catch { /* rollback best-effort */ } } try { - if (previousStateBytes) replaceOwnedFile(trayStatePath(), previousStateBytes); + if (previousStateBytes) replaceWindowsTrayOwnedFile(trayStatePath(), previousStateBytes); else if (existsSync(trayStatePath())) unlinkSync(trayStatePath()); } catch { /* rollback best-effort */ } try { @@ -612,9 +639,9 @@ export function installWindowsTray(startNow = true): WindowsTrayStatus { try { const hardenedDir = hardenSecretDir(getConfigDir(), { required: true }); if (!hardenedDir.ok) throw new Error("Windows tray directory ACL hardening did not complete; refusing to install persistence."); - replaceOwnedFile(entry.script, readFileSync(sourceScript)); - for (const pair of iconPairs) replaceOwnedFile(pair.installed, readFileSync(pair.source)); - replaceOwnedFile(launcherPath, Buffer.from("\uFEFF" + buildWindowsTrayLauncherScript(entry), "utf16le")); + replaceWindowsTrayOwnedFile(entry.script, readFileSync(sourceScript)); + for (const pair of iconPairs) replaceWindowsTrayOwnedFile(pair.installed, readFileSync(pair.source)); + replaceWindowsTrayOwnedFile(launcherPath, Buffer.from("\uFEFF" + buildWindowsTrayLauncherScript(entry), "utf16le")); runRegistry(["add", RUN_KEY, "/v", runValue, "/t", "REG_SZ", "/d", runCommand, "/f", "/reg:64"]); writeState(entryWithLauncher, runValue, runCommand); } catch (error) { diff --git a/tests/codex-auth-api.test.ts b/tests/codex-auth-api.test.ts index 12f0b065c..66923073d 100644 --- a/tests/codex-auth-api.test.ts +++ b/tests/codex-auth-api.test.ts @@ -44,6 +44,12 @@ import { saveConfig, setPersistedConfigMutationBeforeCommitForTests, } from "../src/config"; +import { captureConfigGeneration, registerStateStore } from "../src/lib/state-store-sweeper"; +import { + reconcileLiveStateStores, + setLiveStateStoreConfig, + STATE_STORE_REGISTRATIONS, +} from "../src/lib/state-store-registrations"; const TEST_DIR = join(import.meta.dir, ".tmp-codex-auth-api-test"); const TEST_CODEX_HOME = join(TEST_DIR, "codex"); @@ -2393,6 +2399,10 @@ describe("codex-auth API", () => { expiresAt: Date.now() + 5 * 60_000, chatgptAccountId: "acct-delete", }); + for (const registration of STATE_STORE_REGISTRATIONS) registerStateStore(registration); + setLiveStateStoreConfig(config); + reconcileLiveStateStores(); + const preDeletionWriterGeneration = captureConfigGeneration(); updateAccountQuota("pool-delete", 70); expect(resolveCodexAccountForThread("delete-thread", config)).toBe("pool-delete"); recordCodexUpstreamOutcome(config, "pool-delete", 500); @@ -2425,6 +2435,7 @@ describe("codex-auth API", () => { const resp = await handleCodexAuthAPI(req, new URL(req.url), config); expect(resp!.status).toBe(200); + expect(captureConfigGeneration()).toBeGreaterThan(preDeletionWriterGeneration); expect(config.codexAccounts).toEqual([]); expect(config.activeCodexAccountId).toBeUndefined(); expect(config.pausedCodexAccountIds).toBeUndefined(); @@ -2436,6 +2447,24 @@ describe("codex-auth API", () => { expect(cancelled).toBe(true); expect(closed).toEqual([{ code: 4001, reason: "Codex account invalidated" }]); expect(getTrackedCodexWebSocketCountForAccount("pool-delete")).toBe(0); + + updateAccountQuota( + "pool-delete", + 99, + undefined, + undefined, + undefined, + undefined, + preDeletionWriterGeneration, + ); + recordCodexUpstreamOutcome(config, "pool-delete", 429, { + now: Date.now(), + writerGeneration: preDeletionWriterGeneration, + }); + markAccountNeedsReauth("pool-delete", preDeletionWriterGeneration); + expect(getAccountQuota("pool-delete")).toBeNull(); + expect(getCodexUpstreamHealth("pool-delete")).toBeNull(); + expect(isAccountNeedsReauth("pool-delete")).toBe(false); }); test.each([ diff --git a/tests/combos.test.ts b/tests/combos.test.ts index 417665ec1..9e4c1d56c 100644 --- a/tests/combos.test.ts +++ b/tests/combos.test.ts @@ -41,6 +41,7 @@ import { handleResponses } from "../src/server/responses"; import type { OcxConfig } from "../src/types"; import { syncCatalogModels } from "../src/codex/catalog"; import { injectClaudeAgentDefs } from "../src/claude/agents-inject"; +import { reconcileComboRotationState } from "../src/combos/resolve"; const VALID_COMBO = { targets: [{ provider: "a", model: "m1" }] }; @@ -641,3 +642,41 @@ describe("persisted combo config parity", () => { }); }); }); + +describe("combo generation reconciliation", () => { + test("a removed combo member fences a late success even when its target is still active", () => { + clearComboSelectionState(); + const original = rrConfig(2, [1, 1]); + const originalCombo = getCombo(original, "free")!; + const oldPick = pickComboTarget(original, "free")!; + expect(oldPick.target.provider).toBe("a"); + + const removed = reconcileComboRotationState({ + generation: 10_000, + providerNames: new Set(["a", "c"]), + comboIds: new Set(["free"]), + comboTargets: new Set(["free::a/m1", "free::c/m3"]), + codexAccountIds: new Set(), + oauthAccountKeys: new Set(), + configRoots: new Set(), + }); + expect(removed).toBeGreaterThan(0); + + const current = baseConfig({ + combos: { + free: { + strategy: "round-robin", + stickyLimit: 2, + targets: [ + { provider: "a", model: "m1", weight: 1 }, + { provider: "c", model: "m3", weight: 1 }, + ], + }, + }, + }); + expect(pickComboTarget(current, "free")?.target.provider).toBe("a"); + noteComboSuccess("free", originalCombo, oldPick.target, oldPick.writerGeneration); + noteComboSuccess("free", getCombo(current, "free")!, oldPick.target, 10_000); + expect(pickComboTarget(current, "free")?.target.provider).toBe("a"); + }); +}); diff --git a/tests/config.test.ts b/tests/config.test.ts index 1a8f5930a..2032070bf 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; -import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, renameSync, rmSync, truncateSync, unlinkSync, writeFileSync } from "node:fs"; import { homedir, tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { @@ -27,7 +27,7 @@ import { } from "../src/config"; import * as windowsAcl from "../src/lib/windows-secret-acl"; -import { hardenConfigDir, hardenExistingSecret, renameAtomicFile, saveConfig } from "../src/config"; +import { AtomicWriteResidualTempError, atomicWriteFile, hardenConfigDir, hardenExistingSecret, renameAtomicFile, saveConfig } from "../src/config"; let testDir = ""; beforeEach(() => { @@ -1431,6 +1431,90 @@ describe("opencodex config defaults", () => { }); describe("config.ts – Windows ACL hardening integration", () => { + test("successive atomic temps for one destination are each hardened and then forgotten", () => { + const destination = join(testDir, "atomic-secret.json"); + const previousUsername = process.env.USERNAME; + process.env.USERNAME = "ocx-test-user"; + windowsAcl.resetHardenedStateForTests(); + windowsAcl.setPlatformForTests("win32"); + let grants = 0; + windowsAcl.setIcaclsRunnerForTests(args => { + if (args.includes("/grant:r")) grants += 1; + return { success: true, exitCode: 0, timedOut: false, stdout: "" }; + }); + const io = { + write: (path: string, content: string) => writeFileSync(path, content, { mode: 0o600 }), + harden: (path: string) => { + chmodSync(path, 0o600); + windowsAcl.hardenSecretPath(path, { required: true }); + }, + rename: renameSync, + truncate: (path: string) => truncateSync(path, 0), + unlink: unlinkSync, + }; + try { + atomicWriteFile(destination, "first", io); + expect(windowsAcl.hardenedSecretPathCountForTests()).toBe(0); + atomicWriteFile(destination, "second", io); + expect(readFileSync(destination, "utf8")).toBe("second"); + expect(grants).toBe(2); + expect(windowsAcl.hardenedSecretPathCountForTests()).toBe(0); + } finally { + windowsAcl.setIcaclsRunnerForTests(null); + windowsAcl.setPlatformForTests(null); + windowsAcl.resetHardenedStateForTests(); + if (previousUsername === undefined) delete process.env.USERNAME; + else process.env.USERNAME = previousUsername; + } + }); + + test("failed residual unlink retains the exact temp memo until later cleanup", () => { + const destination = join(testDir, "residual-secret.json"); + const previousUsername = process.env.USERNAME; + process.env.USERNAME = "ocx-test-user"; + windowsAcl.resetHardenedStateForTests(); + windowsAcl.setPlatformForTests("win32"); + windowsAcl.setIcaclsRunnerForTests(() => ({ + success: true, + exitCode: 0, + timedOut: false, + stdout: "", + })); + let residual: string | null = null; + try { + atomicWriteFile(destination, "secret", { + write: (path, content) => writeFileSync(path, content, { mode: 0o600 }), + harden: path => { windowsAcl.hardenSecretPath(path, { required: true }); }, + rename: () => { + const error = new Error("rename failed") as NodeJS.ErrnoException; + error.code = "EIO"; + throw error; + }, + truncate: path => truncateSync(path, 0), + unlink: path => { + residual = path; + const error = new Error("unlink failed") as NodeJS.ErrnoException; + error.code = "EPERM"; + throw error; + }, + }); + throw new Error("expected residual error"); + } catch (error) { + expect(error).toBeInstanceOf(AtomicWriteResidualTempError); + expect(windowsAcl.hardenedSecretPathCountForTests()).toBe(1); + expect(residual).not.toBeNull(); + unlinkSync(residual!); + windowsAcl.forgetHardenedSecretPath(residual!); + expect(windowsAcl.hardenedSecretPathCountForTests()).toBe(0); + } finally { + windowsAcl.setIcaclsRunnerForTests(null); + windowsAcl.setPlatformForTests(null); + windowsAcl.resetHardenedStateForTests(); + if (previousUsername === undefined) delete process.env.USERNAME; + else process.env.USERNAME = previousUsername; + } + }); + test("hardenConfigDir delegates to hardenSecretDir with required:false on win32", () => { const origPlatform = process.platform; Object.defineProperty(process, "platform", { value: "win32", configurable: true }); diff --git a/tests/oauth-store-multi.test.ts b/tests/oauth-store-multi.test.ts index 91b0d01b5..a035b1b7b 100644 --- a/tests/oauth-store-multi.test.ts +++ b/tests/oauth-store-multi.test.ts @@ -5,8 +5,12 @@ import { getAccountCredential, getAccountSet, getCredential, + credentialGeneration, listAccounts, markAccountNeedsReauth, + markAccountNeedsReauthIfGeneration, + mutateStore, + reconcileOAuthReauthState, removeAccount, removeCredential, saveAccountCredential, @@ -212,4 +216,35 @@ describe("multi-account auth store", () => { expect(set.accounts.length).toBe(1); expect(set.activeAccountId).toBe("ok"); // dangling active healed }); + + test("queued generation-checked reauth mutation rechecks liveness after reconciliation", async () => { + await saveCredential("xai", cred({ email: "race@example.com", accountId: "race-account" })); + const accountId = getAccountSet("xai")!.activeAccountId; + const generation = credentialGeneration(getAccountCredential("xai", accountId)!); + let release!: () => void; + let entered!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const enteredGate = new Promise(resolve => { entered = resolve; }); + const blocker = mutateStore(async () => { + entered(); + await gate; + }); + await enteredGate; + + const pending = markAccountNeedsReauthIfGeneration("xai", accountId, generation, 0); + reconcileOAuthReauthState({ + generation: 1, + providerNames: new Set(), + comboIds: new Set(), + comboTargets: new Set(), + codexAccountIds: new Set(), + oauthAccountKeys: new Set(), + configRoots: new Set(), + }); + release(); + await blocker; + + expect(await pending).toBe(false); + expect(getAccountSet("xai")!.accounts[0]?.needsReauth).toBeUndefined(); + }); }); diff --git a/tests/openai-provider-option-startup.test.ts b/tests/openai-provider-option-startup.test.ts index 95bc6b271..605b70968 100644 --- a/tests/openai-provider-option-startup.test.ts +++ b/tests/openai-provider-option-startup.test.ts @@ -1,4 +1,18 @@ import { describe, expect, test } from "bun:test"; +import { + chmodSync, + existsSync, + linkSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + truncateSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { atomicWriteFile, AtomicWriteResidualTempError, @@ -14,6 +28,7 @@ import { import { runOpenAiTierStartupMigration } from "../src/providers/openai-tier-startup"; import { OpenAiTierMigrationCollisionError, projectOpenAiTierMigration } from "../src/providers/openai-tiers"; import type { OcxConfig } from "../src/types"; +import * as windowsAcl from "../src/lib/windows-secret-acl"; const config: OcxConfig = { port: 10100, @@ -109,6 +124,28 @@ function virtualBackupIO(initial: Record, fail: { return { io, files, calls }; } +function aclBackupIO(options: { failTempUnlink?: () => boolean; vanishAfterHarden?: boolean } = {}): OpenAiTierBackupIO { + return { + exists: existsSync, + read: path => readFileSync(path), + createExclusive: path => { writeFileSync(path, new Uint8Array(), { flag: "wx", mode: 0o600 }); }, + write: (path, bytes) => writeFileSync(path, bytes), + harden: path => { + chmodSync(path, 0o600); + windowsAcl.hardenSecretPath(path, { required: true }); + if (options.vanishAfterHarden) unlinkSync(path); + }, + publishNoReplace: (temp, backup) => linkSync(temp, backup), + truncate: path => truncateSync(path, 0), + unlink: path => { + if (path.endsWith(".tmp") && options.failTempUnlink?.()) { + throw Object.assign(new Error("injected temp unlink failure"), { code: "EPERM" }); + } + unlinkSync(path); + }, + }; +} + describe("OpenAI provider option startup coordinator", () => { test("fresh default config is already marked with the current OpenAI tier schema", () => { expect(getDefaultConfig().openaiProviderTierVersion).toBe(2); @@ -250,6 +287,62 @@ describe("OpenAI provider option startup coordinator", () => { expect([...state.files.keys()].filter(path => path.endsWith(".tmp"))).toEqual([]); }); + test("backup temp cleanup forgets successful ACL memos and retains failed removals", () => { + const root = mkdtempSync(join(tmpdir(), "ocx-backup-acl-")); + const source = join(root, "config.json"); + const previousUsername = process.env.USERNAME; + process.env.USERNAME = "ocx-test-user"; + windowsAcl.resetHardenedStateForTests(); + windowsAcl.setPlatformForTests("win32"); + windowsAcl.setIcaclsRunnerForTests(() => ({ success: true, exitCode: 0, timedOut: false, stdout: "" })); + try { + writeFileSync(source, "original-secret"); + expect(backupConfigBeforeOpenAiTierMigration(source, aclBackupIO())).toBe("created"); + expect(windowsAcl.hardenedSecretPathCountForTests()).toBe(0); + + unlinkSync(`${source}.pre-openai-tiers-v2.bak`); + let failRemoval = true; + expect(() => backupConfigBeforeOpenAiTierMigration(source, aclBackupIO({ + failTempUnlink: () => failRemoval, + }))).toThrow(OpenAiTierBackupCleanupError); + expect(windowsAcl.hardenedSecretPathCountForTests()).toBe(1); + failRemoval = false; + for (const name of readdirSync(root)) { + if (name.endsWith(".tmp")) unlinkSync(join(root, name)); + } + } finally { + windowsAcl.setIcaclsRunnerForTests(null); + windowsAcl.setPlatformForTests(null); + windowsAcl.resetHardenedStateForTests(); + if (previousUsername === undefined) delete process.env.USERNAME; + else process.env.USERNAME = previousUsername; + rmSync(root, { recursive: true, force: true }); + } + }); + + test("backup confirmed-absent cleanup forgets a memo created before the temp vanished", () => { + const root = mkdtempSync(join(tmpdir(), "ocx-backup-absent-acl-")); + const source = join(root, "config.json"); + const previousUsername = process.env.USERNAME; + process.env.USERNAME = "ocx-test-user"; + windowsAcl.resetHardenedStateForTests(); + windowsAcl.setPlatformForTests("win32"); + windowsAcl.setIcaclsRunnerForTests(() => ({ success: true, exitCode: 0, timedOut: false, stdout: "" })); + try { + writeFileSync(source, "original-secret"); + expect(() => backupConfigBeforeOpenAiTierMigration(source, aclBackupIO({ vanishAfterHarden: true }))) + .toThrow(); + expect(windowsAcl.hardenedSecretPathCountForTests()).toBe(0); + } finally { + windowsAcl.setIcaclsRunnerForTests(null); + windowsAcl.setPlatformForTests(null); + windowsAcl.resetHardenedStateForTests(); + if (previousUsername === undefined) delete process.env.USERNAME; + else process.env.USERNAME = previousUsername; + rmSync(root, { recursive: true, force: true }); + } + }); + test("v2 backup creation never overwrites the historical v1 snapshot", () => { const state = virtualBackupIO({ "/virtual/config.json": "three-tier-state", diff --git a/tests/provider-account-quota.test.ts b/tests/provider-account-quota.test.ts index 16dd5dd3a..a8dafae67 100644 --- a/tests/provider-account-quota.test.ts +++ b/tests/provider-account-quota.test.ts @@ -9,6 +9,8 @@ import { clearProviderQuotaCache, fetchProviderAccountQuotas, fetchProviderQuotaReports, + getCachedProviderAccountQuota, + reconcileProviderAccountQuotaRows, supportsPerAccountQuota, } from "../src/providers/quota"; @@ -373,4 +375,45 @@ describe("fetchProviderAccountQuotas", () => { expect(byId[second!.id]?.quota?.fiveHourPercent).toBe(3); expect(calls).toBe(1); }); + + test("provider removal during an Anthropic report probe cannot recreate its account quota row", async () => { + await seedTwoAccounts(); + const { getAccountSet, setActiveAccount } = await import("../src/oauth/store"); + const first = getAccountSet("anthropic")?.accounts.find(account => account.credential.email === FIRST.email); + expect(first).toBeTruthy(); + await setActiveAccount("anthropic", first!.id); + + let releaseUsage!: () => void; + const usageGate = new Promise(resolve => { releaseUsage = resolve; }); + globalThis.fetch = (async () => { + await usageGate; + return new Response(usageBody(70, 15), { status: 200 }); + }) as typeof fetch; + + const config: OcxConfig = { + port: 1455, + defaultProvider: "anthropic", + providers: { + anthropic: { + adapter: "anthropic", + authMode: "oauth", + baseUrl: "https://api.anthropic.com/v1", + }, + }, + }; + const reportPromise = fetchProviderQuotaReports(config, true); + reconcileProviderAccountQuotaRows({ + generation: 10_000, + providerNames: new Set(), + comboIds: new Set(), + comboTargets: new Set(), + codexAccountIds: new Set(), + oauthAccountKeys: new Set(), + configRoots: new Set(), + }); + releaseUsage(); + await reportPromise; + + expect(getCachedProviderAccountQuota("anthropic", first!.id)).toBeNull(); + }); }); diff --git a/tests/responses-state.test.ts b/tests/responses-state.test.ts index cb582798d..372408403 100644 --- a/tests/responses-state.test.ts +++ b/tests/responses-state.test.ts @@ -42,6 +42,13 @@ import { writeResponseSpillDurably, } from "../src/responses/spill-store"; import { adapterNeedsForcedContinuation, injectDeveloperMessage } from "../src/server/responses"; +import { + hardenSecretPath, + hardenedSecretPathCountForTests, + resetHardenedStateForTests, + setIcaclsRunnerForTests, + setPlatformForTests, +} from "../src/lib/windows-secret-acl"; function feedInspector( inspector: ReturnType, @@ -98,6 +105,9 @@ describe("Responses previous_response_id state", () => { afterEach(() => { setSpillIoForTest(null); + setIcaclsRunnerForTests(null); + setPlatformForTests(null); + resetHardenedStateForTests(); setResponseStateByteCapForTests(null); clearResponseStateForTests(); rmSync(home, { recursive: true, force: true }); @@ -592,6 +602,42 @@ describe("Responses previous_response_id state", () => { expect(events).toEqual(["write", "fsync", "close", "harden", "publish", "stub-swap"]); }); + test("spill temp cleanup forgets successful ACL memos and retains failed removals", () => { + const previousUsername = process.env.USERNAME; + process.env.USERNAME = "ocx-test-user"; + resetHardenedStateForTests(); + setPlatformForTests("win32"); + setIcaclsRunnerForTests(() => ({ success: true, exitCode: 0, timedOut: false, stdout: "" })); + const seedTempMemo = (event: string): void => { + if (event !== "harden") return; + const tempName = readdirSync(responseSpillDirectory(home)).find(name => name.endsWith(".tmp")); + expect(tempName).toBeTruthy(); + hardenSecretPath(join(responseSpillDirectory(home), tempName!), { required: true }); + }; + try { + setSpillIoForTest({ record: seedTempMemo }); + writeResponseSpillDurably("resp_acl_success", { createdAt: Date.now(), items: ["success"] }); + expect(hardenedSecretPathCountForTests()).toBe(0); + + setSpillIoForTest({ + record: seedTempMemo, + unlink: () => { throw Object.assign(new Error("injected unlink failure"), { code: "EPERM" }); }, + }); + expect(() => writeResponseSpillDurably("resp_acl_failure", { + createdAt: Date.now(), + items: ["failure"], + })).toThrow("Response spill write failed"); + expect(hardenedSecretPathCountForTests()).toBe(1); + } finally { + setSpillIoForTest(null); + setIcaclsRunnerForTests(null); + setPlatformForTests(null); + resetHardenedStateForTests(); + if (previousUsername === undefined) delete process.env.USERNAME; + else process.env.USERNAME = previousUsername; + } + }); + test("replays provider metadata and function_call_output history through a spill stub", () => { setResponseStateByteCapForTests(1_024); rememberResponseState( diff --git a/tests/server-management-auth.test.ts b/tests/server-management-auth.test.ts index 1484eb2eb..2bdddbe26 100644 --- a/tests/server-management-auth.test.ts +++ b/tests/server-management-auth.test.ts @@ -10,9 +10,12 @@ import { isProxyAdmissionSecret } from "../src/server/auth-cors"; import { initializeManagementAuthState, issueGuiSession, + removeManagementTokenPathBestEffort, requireManagementAuth, } from "../src/server/management-auth"; import { + hardenSecretPath, + hardenedSecretPathCountForTests, resetHardenedStateForTests, setIcaclsRunnerForTests, setPlatformForTests, @@ -83,6 +86,34 @@ afterEach(() => { }); describe("management and data-plane credential separation", () => { + test("management-token temp cleanup forgets successful ACL memos and retains failed removals", () => { + const temporary = join(testHome, ".admin-token.tmp"); + const previousUsername = process.env.USERNAME; + process.env.USERNAME = "ocx-test-user"; + resetHardenedStateForTests(); + setPlatformForTests("win32"); + setIcaclsRunnerForTests(() => ({ success: true, exitCode: 0, timedOut: false, stdout: "" })); + try { + writeFileSync(temporary, "secret", { mode: 0o600 }); + hardenSecretPath(temporary, { required: true }); + removeManagementTokenPathBestEffort(temporary); + expect(hardenedSecretPathCountForTests()).toBe(0); + + writeFileSync(temporary, "secret", { mode: 0o600 }); + hardenSecretPath(temporary, { required: true }); + removeManagementTokenPathBestEffort(temporary, () => { + throw Object.assign(new Error("injected unlink failure"), { code: "EPERM" }); + }); + expect(hardenedSecretPathCountForTests()).toBe(1); + } finally { + setIcaclsRunnerForTests(null); + setPlatformForTests(null); + resetHardenedStateForTests(); + if (previousUsername === undefined) delete process.env.USERNAME; + else process.env.USERNAME = previousUsername; + } + }); + test("data and management environment tokens authorize only their own planes", async () => { saveConfig(remoteConfig()); const server = startServer(0); diff --git a/tests/state-store-sweeper.test.ts b/tests/state-store-sweeper.test.ts new file mode 100644 index 000000000..e6d609420 --- /dev/null +++ b/tests/state-store-sweeper.test.ts @@ -0,0 +1,340 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + captureConfigGeneration, + reconcileStateGeneration, + registerStateStore, + resetStateStoreSweeperForTests, + setGenerationContextBuilder, + startStateStoreSweeper, + stopStateStoreSweeper, + sweepExpired, + sweepExpiredOnWrite, + sweepLiveness, + type GenerationContext, +} from "../src/lib/state-store-sweeper"; +import { + ocxStartProcessCacheSizeForTests, + setOcxStartProcessCacheForTests, + setOcxStartProcessProbeForTests, + sweepDeadOcxStartProcessCache, +} from "../src/config"; +import { STATE_STORE_REGISTRATIONS } from "../src/lib/state-store-registrations"; +import { getAccountSet, saveCredential } from "../src/oauth/store"; +import { + clearAccountQuotaCache, + clearProviderQuotaCache, + fetchProviderQuotaReports, + getCachedProviderAccountQuota, +} from "../src/providers/quota"; +import type { OcxConfig } from "../src/types"; + +function context( + generation: number, + overrides: Partial> = {}, +): GenerationContext { + return { + generation, + providerNames: new Set(), + comboIds: new Set(), + comboTargets: new Set(), + codexAccountIds: new Set(), + oauthAccountKeys: new Set(), + configRoots: new Set(), + ...overrides, + }; +} + +beforeEach(() => resetStateStoreSweeperForTests()); +afterEach(() => { + resetStateStoreSweeperForTests(); + setOcxStartProcessCacheForTests([]); + setOcxStartProcessProbeForTests(null); +}); + +describe("state-store sweeper", () => { + test("global fake-clock sweep invokes every production clock registration", () => { + const visits: string[] = []; + for (const registration of STATE_STORE_REGISTRATIONS) { + registerStateStore({ + ...registration, + ...(registration.sweepExpired ? { + sweepExpired: (now: number) => { + visits.push(`${registration.name}:ttl:${now}`); + return registration.sweepExpired!(now); + }, + } : {}), + ...(registration.sweepLiveness ? { + sweepLiveness: () => { + visits.push(`${registration.name}:liveness`); + return registration.sweepLiveness!(); + }, + } : {}), + }); + } + + sweepExpired(123); + sweepLiveness(); + expect(visits).toEqual(STATE_STORE_REGISTRATIONS.flatMap(registration => [ + ...(registration.sweepExpired ? [`${registration.name}:ttl:123`] : []), + ...(registration.sweepLiveness ? [`${registration.name}:liveness`] : []), + ])); + }); + + test("expiry boundary removes expired rows and preserves live rows", () => { + const rows = new Map([["past", 99], ["boundary", 100], ["live", 101]]); + registerStateStore({ + name: "ttl", + sweepExpired: now => { + let removed = 0; + for (const [key, deadline] of rows) { + if (deadline > now) continue; + rows.delete(key); + removed += 1; + } + return removed; + }, + }); + + expect(sweepExpired(100).rowsRemoved).toBe(2); + expect([...rows]).toEqual([["live", 101]]); + }); + + test("one throwing owner does not block later sweep owners", () => { + const warning = spyOn(console, "warn").mockImplementation(() => {}); + let later = 0; + registerStateStore({ name: "throws", sweepExpired: () => { throw new Error("secret detail"); } }); + registerStateStore({ name: "later", sweepExpired: () => { later += 1; return 1; } }); + + expect(sweepExpired(0)).toEqual({ storesVisited: 2, rowsRemoved: 1 }); + expect(later).toBe(1); + expect(warning.mock.calls[0]?.[0]).toBe("[state-store-sweeper] throws failed"); + expect(String(warning.mock.calls[0]?.[0])).not.toContain("secret detail"); + warning.mockRestore(); + }); + + test("write-trigger uses the same callbacks synchronously without creating a queue", () => { + const order: string[] = []; + registerStateStore({ name: "a", sweepExpired: () => { order.push("a"); return 1; } }); + registerStateStore({ name: "b", sweepExpired: () => { order.push("b"); return 1; } }); + + const result = sweepExpiredOnWrite(5); + order.push("returned"); + expect(result).toEqual({ storesVisited: 2, rowsRemoved: 2 }); + expect(order).toEqual(["a", "b", "returned"]); + }); + + test("timer start is singleton unrefed and stop is idempotent", () => { + const timers: Array<{ callback: () => void; unrefCalls: number }> = []; + const cleared: unknown[] = []; + const setSpy = spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void) => { + const timer = { callback, unrefCalls: 0, unref() { this.unrefCalls += 1; } }; + timers.push(timer); + return timer; + }) as typeof setInterval); + const clearSpy = spyOn(globalThis, "clearInterval").mockImplementation((timer => { + cleared.push(timer); + }) as typeof clearInterval); + let ttl = 0; + let liveness = 0; + registerStateStore({ + name: "both", + sweepExpired: () => { ttl += 1; return 0; }, + sweepLiveness: () => { liveness += 1; return 0; }, + }); + + startStateStoreSweeper({ intervalMs: 10, now: () => 42 }); + startStateStoreSweeper({ intervalMs: 20, now: () => 43 }); + expect(timers).toHaveLength(2); + expect(timers[0]!.unrefCalls).toBe(1); + expect(timers[1]!.unrefCalls).toBe(1); + expect(cleared).toEqual([timers[0]]); + timers[1]!.callback(); + expect({ ttl, liveness }).toEqual({ ttl: 1, liveness: 1 }); + + stopStateStoreSweeper(); + stopStateStoreSweeper(); + expect(cleared).toEqual([timers[0], timers[1]]); + setSpy.mockRestore(); + clearSpy.mockRestore(); + }); + + test("reconciliation runs only for a newer complete generation", () => { + const seen: number[] = []; + registerStateStore({ name: "owner", reconcileGeneration: next => { seen.push(next.generation); return 1; } }); + + expect(reconcileStateGeneration(context(1)).rowsRemoved).toBe(1); + expect(captureConfigGeneration()).toBe(1); + expect(reconcileStateGeneration(context(1)).rowsRemoved).toBe(1); + expect(captureConfigGeneration()).toBe(2); + expect(seen).toEqual([1, 2]); + }); + + test("stale or duplicate generation cannot delete current keys", () => { + const rows = new Set(["live", "removed"]); + let last = 0; + const reconcileOwner = (next: GenerationContext): number => { + if (next.generation <= last) return 0; + let removed = 0; + for (const row of rows) { + if (next.providerNames.has(row)) continue; + rows.delete(row); + removed += 1; + } + last = next.generation; + return removed; + }; + + reconcileOwner(context(1, { providerNames: new Set(["live"]) })); + expect([...rows]).toEqual(["live"]); + reconcileOwner(context(1, { providerNames: new Set() })); + expect([...rows]).toEqual(["live"]); + }); + + test("partial reconciliation retries every owner with a fresh context and a higher candidate", () => { + const warning = spyOn(console, "warn").mockImplementation(() => {}); + let fail = true; + let builds = 0; + let ownerGeneration = 0; + let liveProviders = new Set(); + registerStateStore({ + name: "ok", + reconcileGeneration: next => { + if (next.generation <= ownerGeneration) return 0; + ownerGeneration = next.generation; + liveProviders = new Set(next.providerNames); + return 0; + }, + }); + registerStateStore({ + name: "flaky", + reconcileGeneration: () => { + if (fail) throw new Error("first"); + return 0; + }, + }); + setGenerationContextBuilder(() => context(0, { + providerNames: new Set([`build-${++builds}`]), + })); + + reconcileStateGeneration(context(0, { providerNames: new Set(["first"]) })); + expect(captureConfigGeneration()).toBe(0); + expect(ownerGeneration).toBe(1); + expect([...liveProviders]).toEqual(["first"]); + fail = false; + sweepExpiredOnWrite(0); + expect(builds).toBe(1); + expect(captureConfigGeneration()).toBe(2); + expect(ownerGeneration).toBe(2); + expect([...liveProviders]).toEqual(["build-1"]); + warning.mockRestore(); + }); + + test("liveness callbacks are separate from write-trigger sweeps", () => { + let probes = 0; + registerStateStore({ name: "pid", sweepLiveness: () => { probes += 1; return 0; } }); + sweepExpiredOnWrite(0); + expect(probes).toBe(0); + expect(sweepLiveness()).toEqual({ storesVisited: 1, rowsRemoved: 0 }); + expect(probes).toBe(1); + }); + + test("provider-quota late completion cannot resurrect a deleted provider or account row", async () => { + const previousHome = process.env.OPENCODEX_HOME; + const home = mkdtempSync(join(tmpdir(), "ocx-sweeper-quota-")); + const originalFetch = globalThis.fetch; + process.env.OPENCODEX_HOME = home; + clearAccountQuotaCache(); + clearProviderQuotaCache(); + try { + await saveCredential("anthropic", { + access: "late-quota-token", + refresh: "late-quota-refresh", + expires: Date.now() + 60_000, + accountId: "late-quota-account", + }); + const accountId = getAccountSet("anthropic")!.activeAccountId; + let release!: () => void; + let started!: () => void; + const startedPromise = new Promise(resolve => { started = resolve; }); + const gate = new Promise(resolve => { release = resolve; }); + let calls = 0; + globalThis.fetch = (async () => { + calls += 1; + started(); + if (calls === 1) await gate; + return new Response(JSON.stringify({ + five_hour: { utilization: 12 }, + seven_day: { utilization: 34 }, + }), { status: 200 }); + }) as typeof fetch; + const config = { + defaultProvider: "anthropic", + providers: { + anthropic: { adapter: "anthropic", authMode: "oauth", baseUrl: "https://api.anthropic.com/v1" }, + }, + } as OcxConfig; + const staleConfig = { ...config, providers: { ...config.providers } } as OcxConfig; + const quotaRegistration = STATE_STORE_REGISTRATIONS.find(row => row.name === "provider-quota-history")!; + registerStateStore(quotaRegistration); + + const late = fetchProviderQuotaReports(config, true); + await startedPromise; + delete config.providers.anthropic; + reconcileStateGeneration(context(0)); + release(); + await late; + + expect(getCachedProviderAccountQuota("anthropic", accountId)).toBeNull(); + expect((await fetchProviderQuotaReports(staleConfig, false)).reports).toEqual([]); + expect(calls).toBe(1); + } finally { + globalThis.fetch = originalFetch; + clearAccountQuotaCache(); + clearProviderQuotaCache(); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + rmSync(home, { recursive: true, force: true }); + } + }); + + test("PID liveness probes at most 64, rotates, and deletes only ESRCH", () => { + setOcxStartProcessCacheForTests(Array.from({ length: 70 }, (_, index) => [index + 1, true] as const)); + const probed: number[] = []; + setOcxStartProcessProbeForTests(pid => { + probed.push(pid); + if (pid === 1) { + const error = new Error("gone") as NodeJS.ErrnoException; + error.code = "ESRCH"; + throw error; + } + if (pid === 2) { + const error = new Error("denied") as NodeJS.ErrnoException; + error.code = "EPERM"; + throw error; + } + if (pid === 3) throw new Error("unknown"); + }); + + expect(sweepDeadOcxStartProcessCache()).toBe(1); + expect(probed).toHaveLength(64); + expect(ocxStartProcessCacheSizeForTests()).toBe(69); + probed.length = 0; + sweepDeadOcxStartProcessCache(); + expect(probed.length).toBeLessThanOrEqual(64); + expect(probed).toContain(70); + expect(ocxStartProcessCacheSizeForTests()).toBe(69); + }); + + test("PID liveness discards invalid keys without probing them", () => { + setOcxStartProcessCacheForTests([[0, true], [-1, true], [1.5, true], [42, true]]); + const probed: number[] = []; + setOcxStartProcessProbeForTests(pid => { probed.push(pid); }); + + expect(sweepDeadOcxStartProcessCache()).toBe(3); + expect(probed).toEqual([42]); + expect(ocxStartProcessCacheSizeForTests()).toBe(1); + }); +}); diff --git a/tests/windows-secret-acl.test.ts b/tests/windows-secret-acl.test.ts index 95871b44b..be37c0d16 100644 --- a/tests/windows-secret-acl.test.ts +++ b/tests/windows-secret-acl.test.ts @@ -15,8 +15,10 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { hardenSecretDir, + forgetHardenedSecretPath, hardenSecretPath, hardenSecretPathAsync, + hardenedSecretPathCountForTests, resetHardenedStateForTests, setAsyncIcaclsRunnerForTests, setIcaclsRunnerForTests, @@ -105,6 +107,40 @@ describe("hardenSecretPath – required mode (required: true)", () => { }); }); +describe("ephemeral harden success memo lifecycle", () => { + test("forgetHardenedSecretPath releases only the actual temp and a second temp hardens again", () => { + const tempA = join(testDir, "config.json.ocx.1.1.tmp"); + const tempB = join(testDir, "config.json.ocx.1.2.tmp"); + writeFileSync(tempA, "first", "utf8"); + writeFileSync(tempB, "second", "utf8"); + setPlatformForTests("win32"); + const previousUsername = process.env.USERNAME; + process.env.USERNAME = "ocx-test-user"; + let grants = 0; + setIcaclsRunnerForTests(args => { + if (args.includes("/grant:r")) grants += 1; + return { success: true, exitCode: 0, timedOut: false, stdout: "" }; + }); + try { + expect(hardenSecretPath(tempA, { required: true })).toEqual({ ok: true }); + expect(hardenedSecretPathCountForTests()).toBe(1); + forgetHardenedSecretPath(tempA); + expect(hardenedSecretPathCountForTests()).toBe(0); + + expect(hardenSecretPath(tempB, { required: true })).toEqual({ ok: true }); + expect(grants).toBe(2); + expect(hardenedSecretPathCountForTests()).toBe(1); + forgetHardenedSecretPath(tempB); + expect(hardenedSecretPathCountForTests()).toBe(0); + } finally { + if (previousUsername === undefined) delete process.env.USERNAME; + else process.env.USERNAME = previousUsername; + setIcaclsRunnerForTests(null); + setPlatformForTests(null); + } + }); +}); + // --------------------------------------------------------------------------- // hardenSecretDir // --------------------------------------------------------------------------- diff --git a/tests/windows-tray.test.ts b/tests/windows-tray.test.ts index 2a9af6035..32d6e3002 100644 --- a/tests/windows-tray.test.ts +++ b/tests/windows-tray.test.ts @@ -1,5 +1,14 @@ import { describe, expect, test } from "bun:test"; -import { copyFileSync, existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { + copyFileSync, + existsSync, + mkdtempSync, + readFileSync, + renameSync, + rmSync, + unlinkSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { @@ -10,6 +19,7 @@ import { parseWindowsTrayRunValue, readWindowsTrayRunValueWithAsyncRunner, readWindowsTrayRunValueWithRunner, + replaceWindowsTrayOwnedFile, windowsTrayProcessArgs, windowsTrayRunValue, windowsTrayStatePathsOwned, @@ -17,6 +27,13 @@ import { windowsRegistryParentShowsRunKey, type WindowsTrayEntry, } from "../src/tray/windows"; +import { + hardenSecretPath, + hardenedSecretPathCountForTests, + resetHardenedStateForTests, + setIcaclsRunnerForTests, + setPlatformForTests, +} from "../src/lib/windows-secret-acl"; import { handleManagementAPI } from "../src/server/management-api"; import type { OcxConfig } from "../src/types"; import { INTERNAL_DEADLINE_MS, SPAWN_BUDGET_MS } from "./helpers/test-budget"; @@ -30,6 +47,46 @@ const entry: WindowsTrayEntry = { }; describe("Windows tray packaging and command safety", () => { + test("owned-file temp cleanup forgets successful ACL memos and retains failed removals", () => { + const root = mkdtempSync(join(tmpdir(), "ocx-tray-acl-")); + const target = join(root, "tray-state.json"); + const previousUsername = process.env.USERNAME; + process.env.USERNAME = "ocx-test-user"; + resetHardenedStateForTests(); + setPlatformForTests("win32"); + setIcaclsRunnerForTests(() => ({ success: true, exitCode: 0, timedOut: false, stdout: "" })); + const write = (path: string, contents: string | Buffer): void => { + writeFileSync(path, contents, { mode: 0o600 }); + }; + const harden = (path: string): void => { + hardenSecretPath(path, { required: true }); + }; + try { + replaceWindowsTrayOwnedFile(target, "success", { + write, + harden, + rename: renameSync, + unlink: unlinkSync, + }); + expect(hardenedSecretPathCountForTests()).toBe(0); + + expect(() => replaceWindowsTrayOwnedFile(target, "failure", { + write, + harden, + rename: () => { throw new Error("injected rename failure"); }, + unlink: () => { throw Object.assign(new Error("injected unlink failure"), { code: "EPERM" }); }, + })).toThrow("injected rename failure"); + expect(hardenedSecretPathCountForTests()).toBe(1); + } finally { + setIcaclsRunnerForTests(null); + setPlatformForTests(null); + resetHardenedStateForTests(); + if (previousUsername === undefined) delete process.env.USERNAME; + else process.env.USERNAME = previousUsername; + rmSync(root, { recursive: true, force: true }); + } + }); + test("uses fixed argv for the hidden PowerShell host", () => { const args = windowsTrayProcessArgs(entry); expect(args).toContain("-NoProfile"); @@ -358,7 +415,7 @@ describe("Windows tray packaging and command safety", () => { expect(tray).toContain('join(getConfigDir(), "opencodex-tray.ps1")'); expect(tray).toContain('join(import.meta.dir, "assets", name)'); expect(tray).toContain("installedTrayIconPaths()"); - expect(tray).toContain("const hardened = hardenSecretPath(temporary, { required: true })"); + expect(tray).toContain("const hardened = hardenSecretPath(target, { required: true })"); expect(tray).toContain("if (!hardened.ok)"); expect(tray).toContain("if (!hardenedDir.ok)"); expect(tray).toContain("refusing to replace its persistent script"); From dcad281868aa118036ce0b410d3585a45a023d53 Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sat, 1 Aug 2026 11:55:10 +0900 Subject: [PATCH 12/30] refactor(sweep): simplify the eviction fences to the generation contract The per-combo topology-generation map duplicated what the global reconciled-generation plus the live-target set already guarantee, so the commit fence keeps the same admissions with one less keyed store. The tier-backup scrub now forgets the hardened secret path in exactly one place, after removal is proven, instead of four racy call sites. --- .../006_roadmap_audit_synthesis.md | 20 ++++++++++ src/combos/resolve.ts | 13 +------ src/config.ts | 18 +++------ tests/combos.test.ts | 30 ++++++++++++--- tests/openai-provider-option-startup.test.ts | 38 ++++++++++++++++++- 5 files changed, 88 insertions(+), 31 deletions(-) diff --git a/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md b/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md index 381c2c7b2..1e996c1f2 100644 --- a/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md +++ b/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md @@ -83,3 +83,23 @@ amended accordingly (failure ladder + admission ladder). | WP4-A3 | PID memo eviction requires process identity/liveness proof, but 030 names neither an owner export/call site nor its regressions | ACCEPT | Reclassified PID memos to `sweepDeadOcxStartProcessCache(64)`: timer-only round-robin probing, delete only on `process.kill(pid, 0)` `ESRCH`, with live/EPERM/unknown preservation and bounded-cost regressions. | All three accepted and incorporated into 030. Re-audit verdict: **PASS**. + +## wp4 C-review round 1 (Hegel, FAIL 6) — adjudication + +| # | Blocker | Decision | Action | +|---|---|---|---| +| B1 | Reconciliation attempts reused a single generation value, so two overlapping sweeps could cross-accept each other's stale writes | ACCEPT | Sweeper issues a unique `attemptSequence` per reconciliation pass; per-owner last-reconciled generation compares against the exact issuing attempt. | +| B2 | Provider-quota rows were reconciled without the stale-writer fence, so an in-flight quota update could resurrect a deleted provider row post-sweep | ACCEPT | Quota store writes now capture writer generation and drop deleted-key commits like every other fenced owner. | +| B3 | Reauth fence was read before the topology snapshot, leaving a window where a reauth completing mid-sweep landed unfenced | ACCEPT | Fence generation is captured inside the same synchronous snapshot section as the topology read. | +| B4 | Dead-process probe accepted PID 0/negative values, where `process.kill(0, 0)` signals the whole process group | ACCEPT | Probe validates `pid > 0` before kill; non-positive PIDs are treated as unknown and preserved. | +| B5 | Windows ACL memo release keyed off `existsSync` returning false, deleting the memo when the file was merely unreadable | ACCEPT | Memo release restricted to successful unlink or ENOENT (round 1 repair; tightened further after round 2). | +| B6 | Regression tests asserted sweep-ran flags rather than observable store state, giving false confidence | ACCEPT | Tests rewritten to assert row presence/absence and byte counts after sweep, not internal flags. | + +Repair delta (13 files) verified: focused 387 pass, typecheck 0, privacy pass. Amended into `819450ac8`. + +## wp4 C-review round 2 (Hegel, FAIL 2) — adjudication + +| # | Blocker | Decision | Action | +|---|---|---|---| +| R2-B5 | `!io.exists(temp)` branches in `src/config.ts` (365–408) still treated a false existence check as proof of removal: unlink EPERM + `exists()` false returned "created", left the temp file on disk, and dropped the ACL memo count to zero | ACCEPT | Memo release now requires a completed rename/unlink or an unlink throwing ENOENT; existence predicates removed from every memo-release decision. Regression: simulated EPERM-with-exists-false asserts memo retention and non-success status. | +| R2-B6 | `comboTopologyGeneration` in `src/combos/resolve.ts` rejected an old completion whenever ANY combo member changed, dropping completions whose exact `${comboId}::${targetKey}` was still live — violating the live-key acceptance rule; `tests/combos.test.ts:642` codified the prohibited drop | ACCEPT | Fencing narrowed to owner generation plus exact live target key: completions for surviving targets accepted, removed targets rejected. Test updated to assert both directions. | diff --git a/src/combos/resolve.ts b/src/combos/resolve.ts index 7bc3e3b88..e2e36601a 100644 --- a/src/combos/resolve.ts +++ b/src/combos/resolve.ts @@ -24,16 +24,14 @@ interface SelectionState { const selectionState = new Map(); let lastReconciledGeneration = 0; let liveComboTargets = new Set(); -const comboTopologyGeneration = new Map(); function comboTargetOwnerKey(comboId: string, key: string): string { return `${comboId}::${key}`; } function mayCommitComboState(comboId: string, key: string, writerGeneration: number): boolean { - return writerGeneration >= (comboTopologyGeneration.get(comboId) ?? 0) - && (writerGeneration >= lastReconciledGeneration - || liveComboTargets.has(comboTargetOwnerKey(comboId, key))); + return writerGeneration >= lastReconciledGeneration + || liveComboTargets.has(comboTargetOwnerKey(comboId, key)); } export class UnknownComboError extends Error { @@ -195,24 +193,19 @@ export function reconcileComboRotationState(context: GenerationContext): number for (const [comboId, state] of selectionState) { if (!context.comboIds.has(comboId)) { selectionState.delete(comboId); - comboTopologyGeneration.set(comboId, context.generation); removed += 1; continue; } - let topologyChanged = false; if (state.activeKey && !context.comboTargets.has(comboTargetOwnerKey(comboId, state.activeKey))) { delete state.activeKey; state.successes = 0; - topologyChanged = true; removed += 1; } for (const key of state.currentWeights.keys()) { if (context.comboTargets.has(comboTargetOwnerKey(comboId, key))) continue; state.currentWeights.delete(key); - topologyChanged = true; removed += 1; } - if (topologyChanged) comboTopologyGeneration.set(comboId, context.generation); } liveComboTargets = new Set(context.comboTargets); lastReconciledGeneration = context.generation; @@ -223,12 +216,10 @@ export function clearComboSelectionState(comboId?: string): void { if (comboId === undefined) { selectionState.clear(); liveComboTargets.clear(); - comboTopologyGeneration.clear(); lastReconciledGeneration = 0; return; } selectionState.delete(comboId); - comboTopologyGeneration.delete(comboId); } export function tryPickComboModel(config: OcxConfig, modelId: string): ComboPick | null { diff --git a/src/config.ts b/src/config.ts index 7a69343fe..1c4e27cff 100644 --- a/src/config.ts +++ b/src/config.ts @@ -346,10 +346,6 @@ export function backupConfigBeforeOpenAiTierMigration( const scrubUnpublishedTemp = (): void => { cleanupAttempted = true; - if (!io.exists(temp)) { - forgetHardenedSecretPath(temp); - return; - } let scrubbed = false; try { io.truncate(temp); @@ -364,22 +360,20 @@ export function backupConfigBeforeOpenAiTierMigration( try { io.unlink(temp); removed = true; - forgetHardenedSecretPath(temp); } catch (error) { - if (isMissingPathError(error) || !io.exists(temp)) { + if (isMissingPathError(error)) { removed = true; - forgetHardenedSecretPath(temp); } else { - try { io.unlink(temp); removed = true; forgetHardenedSecretPath(temp); } + try { io.unlink(temp); removed = true; } catch (retryError) { - if (isMissingPathError(retryError) || !io.exists(temp)) { + if (isMissingPathError(retryError)) { removed = true; - forgetHardenedSecretPath(temp); } } } } + if (removed) forgetHardenedSecretPath(temp); if (!removed && !scrubbed) throw new OpenAiTierBackupSecretResidualError(temp); if (!removed) throw new OpenAiTierBackupCleanupError(); }; @@ -402,13 +396,13 @@ export function backupConfigBeforeOpenAiTierMigration( io.unlink(temp); forgetHardenedSecretPath(temp); } catch (firstError) { - if (isMissingPathError(firstError) || !io.exists(temp)) { + if (isMissingPathError(firstError)) { forgetHardenedSecretPath(temp); } else try { io.unlink(temp); forgetHardenedSecretPath(temp); } catch (secondError) { - if (isMissingPathError(secondError) || !io.exists(temp)) { + if (isMissingPathError(secondError)) { forgetHardenedSecretPath(temp); return "created"; } diff --git a/tests/combos.test.ts b/tests/combos.test.ts index 9e4c1d56c..ae8c4643f 100644 --- a/tests/combos.test.ts +++ b/tests/combos.test.ts @@ -644,10 +644,9 @@ describe("persisted combo config parity", () => { }); describe("combo generation reconciliation", () => { - test("a removed combo member fences a late success even when its target is still active", () => { + test("a surviving target accepts a late completion after a sibling topology change", () => { clearComboSelectionState(); const original = rrConfig(2, [1, 1]); - const originalCombo = getCombo(original, "free")!; const oldPick = pickComboTarget(original, "free")!; expect(oldPick.target.provider).toBe("a"); @@ -674,9 +673,28 @@ describe("combo generation reconciliation", () => { }, }, }); - expect(pickComboTarget(current, "free")?.target.provider).toBe("a"); - noteComboSuccess("free", originalCombo, oldPick.target, oldPick.writerGeneration); - noteComboSuccess("free", getCombo(current, "free")!, oldPick.target, 10_000); - expect(pickComboTarget(current, "free")?.target.provider).toBe("a"); + noteComboFailure("free", oldPick.target, oldPick.writerGeneration); + expect(pickComboTarget(current, "free")?.target.provider).toBe("c"); + }); + + test("a removed target rejects a late completion", () => { + clearComboSelectionState(); + const original = rrConfig(1, [1, 1]); + const originalCombo = getCombo(original, "free")!; + + reconcileComboRotationState({ + generation: 10_000, + providerNames: new Set(["a", "c"]), + comboIds: new Set(["free"]), + comboTargets: new Set(["free::a/m1", "free::c/m3"]), + codexAccountIds: new Set(), + oauthAccountKeys: new Set(), + configRoots: new Set(), + }); + + const removedPick = pickComboTarget(original, "free", { exclude: ["a/m1"] })!; + expect(removedPick.target.provider).toBe("b"); + noteComboSuccess("free", originalCombo, removedPick.target, 0); + expect(pickComboTarget(original, "free")?.target.provider).toBe("b"); }); }); diff --git a/tests/openai-provider-option-startup.test.ts b/tests/openai-provider-option-startup.test.ts index 605b70968..9e9e61c33 100644 --- a/tests/openai-provider-option-startup.test.ts +++ b/tests/openai-provider-option-startup.test.ts @@ -124,9 +124,13 @@ function virtualBackupIO(initial: Record, fail: { return { io, files, calls }; } -function aclBackupIO(options: { failTempUnlink?: () => boolean; vanishAfterHarden?: boolean } = {}): OpenAiTierBackupIO { +function aclBackupIO(options: { + failTempUnlink?: () => boolean; + hideTempFromExists?: boolean; + vanishAfterHarden?: boolean; +} = {}): OpenAiTierBackupIO { return { - exists: existsSync, + exists: path => options.hideTempFromExists && path.endsWith(".tmp") ? false : existsSync(path), read: path => readFileSync(path), createExclusive: path => { writeFileSync(path, new Uint8Array(), { flag: "wx", mode: 0o600 }); }, write: (path, bytes) => writeFileSync(path, bytes), @@ -320,6 +324,36 @@ describe("OpenAI provider option startup coordinator", () => { } }); + test("backup unlink EPERM retains its ACL memo when exists falsely reports the temp absent", () => { + const root = mkdtempSync(join(tmpdir(), "ocx-backup-hidden-acl-")); + const source = join(root, "config.json"); + const backup = `${source}.pre-openai-tiers-v2.bak`; + const previousUsername = process.env.USERNAME; + process.env.USERNAME = "ocx-test-user"; + windowsAcl.resetHardenedStateForTests(); + windowsAcl.setPlatformForTests("win32"); + windowsAcl.setIcaclsRunnerForTests(() => ({ success: true, exitCode: 0, timedOut: false, stdout: "" })); + try { + writeFileSync(source, "original-secret"); + expect(() => backupConfigBeforeOpenAiTierMigration(source, aclBackupIO({ + failTempUnlink: () => true, + hideTempFromExists: true, + }))).toThrow(OpenAiTierBackupCleanupError); + expect(existsSync(backup)).toBe(false); + const residuals = readdirSync(root).filter(name => name.endsWith(".tmp")); + expect(residuals).toHaveLength(1); + expect(existsSync(join(root, residuals[0]!))).toBe(true); + expect(windowsAcl.hardenedSecretPathCountForTests()).toBe(1); + } finally { + windowsAcl.setIcaclsRunnerForTests(null); + windowsAcl.setPlatformForTests(null); + windowsAcl.resetHardenedStateForTests(); + if (previousUsername === undefined) delete process.env.USERNAME; + else process.env.USERNAME = previousUsername; + rmSync(root, { recursive: true, force: true }); + } + }); + test("backup confirmed-absent cleanup forgets a memo created before the temp vanished", () => { const root = mkdtempSync(join(tmpdir(), "ocx-backup-absent-acl-")); const source = join(root, "config.json"); From 3ed4ca0ae24116021b4f5df77f3357ab62522f2b Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sat, 1 Aug 2026 12:00:17 +0900 Subject: [PATCH 13/30] docs(plan): align the B3/B4 adjudications with the repaired contracts The audit synthesis described the reauth fence and dead-PID probe as they stood before repair; the table now states the per-write generation fence and the serialized non-positive-PID discard that actually shipped. The rotation test also learns that a resolved auth context carries its writer generation. --- .../006_roadmap_audit_synthesis.md | 4 ++-- tests/codex-main-rotation.test.ts | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md b/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md index 1e996c1f2..87478f8df 100644 --- a/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md +++ b/devlog/_plan/260801_zero_leak_state_stores/006_roadmap_audit_synthesis.md @@ -90,8 +90,8 @@ All three accepted and incorporated into 030. Re-audit verdict: **PASS**. |---|---|---|---| | B1 | Reconciliation attempts reused a single generation value, so two overlapping sweeps could cross-accept each other's stale writes | ACCEPT | Sweeper issues a unique `attemptSequence` per reconciliation pass; per-owner last-reconciled generation compares against the exact issuing attempt. | | B2 | Provider-quota rows were reconciled without the stale-writer fence, so an in-flight quota update could resurrect a deleted provider row post-sweep | ACCEPT | Quota store writes now capture writer generation and drop deleted-key commits like every other fenced owner. | -| B3 | Reauth fence was read before the topology snapshot, leaving a window where a reauth completing mid-sweep landed unfenced | ACCEPT | Fence generation is captured inside the same synchronous snapshot section as the topology read. | -| B4 | Dead-process probe accepted PID 0/negative values, where `process.kill(0, 0)` signals the whole process group | ACCEPT | Probe validates `pid > 0` before kill; non-positive PIDs are treated as unknown and preserved. | +| B3 | Reauth fence was read before the topology snapshot, leaving a window where a reauth completing mid-sweep landed unfenced | ACCEPT | Each reauth write captures its own writer generation at write time and compares it against the owner's last-reconciled generation inside the reconcile mutation: stale writes for deleted accounts drop, writes for live account ids remain accepted regardless of generation. | +| B4 | Dead-process probe accepted PID 0/negative values, where `process.kill(0, 0)` signals the whole process group | ACCEPT | The sweep rechecks PID validity inside the serialized mutation and discards non-safe-integer/non-positive PIDs without probing; only validated positive PIDs are probed, and deletion still requires an `ESRCH` result. | | B5 | Windows ACL memo release keyed off `existsSync` returning false, deleting the memo when the file was merely unreadable | ACCEPT | Memo release restricted to successful unlink or ENOENT (round 1 repair; tightened further after round 2). | | B6 | Regression tests asserted sweep-ran flags rather than observable store state, giving false confidence | ACCEPT | Tests rewritten to assert row presence/absence and byte counts after sweep, not internal flags. | diff --git a/tests/codex-main-rotation.test.ts b/tests/codex-main-rotation.test.ts index 1c79d5b24..a42f16c45 100644 --- a/tests/codex-main-rotation.test.ts +++ b/tests/codex-main-rotation.test.ts @@ -153,6 +153,7 @@ describe("main account rotation (Option A)", () => { accountId: MAIN_CODEX_ACCOUNT_ID, accessToken: "main_access", chatgptAccountId: "main_acct", + writerGeneration: expect.any(Number), }); expect(isCodexAuthContextUsable(ctx, config)).toBe(true); const headers = headersForCodexAuthContext(new Headers(), ctx); @@ -180,6 +181,7 @@ describe("main account rotation (Option A)", () => { accountId: MAIN_CODEX_ACCOUNT_ID, accessToken: "replacement_access", chatgptAccountId: "replacement_acct", + writerGeneration: expect.any(Number), }); expect(isCodexAccountInCooldown(MAIN_CODEX_ACCOUNT_ID)).toBe(false); expect(isAccountNeedsReauth(MAIN_CODEX_ACCOUNT_ID)).toBe(false); From 6d45c2eeac8fe334f542944edd2797cd9c177bfc Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sat, 1 Aug 2026 12:23:38 +0900 Subject: [PATCH 14/30] docs(plan): retarget the registry caps at the owners that actually leak Freshness audit against the rebased tree moved four caps to their real owners: the turn lease now lives at the fetch boundary with boundary-owned release, the WebSocket reservation precedes the upgrade, the pool-quota cap counts total flight objects across the per-account sets, and the dead 16-live-worker cap gave way to bounding the spawn-closure queue itself. Each quota-probe caller gets a defined busy behavior instead of an imagined shared 409 surface. --- .../035_registry_admission_caps.md | 178 +++++++++++++----- 1 file changed, 135 insertions(+), 43 deletions(-) diff --git a/devlog/_plan/260801_zero_leak_state_stores/035_registry_admission_caps.md b/devlog/_plan/260801_zero_leak_state_stores/035_registry_admission_caps.md index 174ab4e61..07c95fb0b 100644 --- a/devlog/_plan/260801_zero_leak_state_stores/035_registry_admission_caps.md +++ b/devlog/_plan/260801_zero_leak_state_stores/035_registry_admission_caps.md @@ -9,8 +9,10 @@ Binding inputs: `000_state_store_inventory.md` §§3–6, `005_impl_roadmap.md` Close the operational stores omitted by the initial roadmap audit. Every active registry gets a finite hard admission cap and coherent busy result; every single-flight has a -finite distinct-key cap and stale-owner policy; discovery/usage/MCP payloads are bounded -before full materialization; retained diagnostic and affinity strings are truncated at +finite distinct-key cap and stale-owner policy; discovery and usage payloads are bounded +before full materialization, and MCP payloads are bounded in manager-owned retained +copies (the SDK materializes responses before the manager can measure them — see the +MCP section); retained diagnostic and affinity strings are truncated at insertion with a visible marker. Existing accepted owners are never silently untracked. This phase provides retained-byte accounting hooks for 040. It does not implement the @@ -51,17 +53,24 @@ never returns an invalid surrogate fragment. Metrics are scalar-only and monoton Current anchors: -- `src/lib/debug-log-buffer.ts:3-35` retains 2,000 unbounded lines and an unbounded - listener set. +- `src/lib/debug-log-buffer.ts:10-35` retains 2,000 unbounded lines and an unbounded + listener set. `subscribeDebugLogEntries` currently has NO production consumer: + `/api/debug/logs` is a polling JSON route (`src/server/management/logs-usage-routes.ts:141-144`), + not SSE. The cap therefore lands in the owner only; no route change exists or is invented. - `src/lib/injection-debug-log.ts:10-27` retains 2,000 unbounded lines. - `src/claude/inbound-debug.ts:40-106` retains 20 variable metadata rows. - `src/lib/crash-guard.ts:206-245` retains 12 traces with unbounded URL/origin/rejection. -- fixed string slots include `src/lib/sidecar-tracker.ts:10-48`, startup health, - main-account cache, star state, shim error, and project-config warnings listed in - inventory §6. +- fixed string slots include `src/lib/sidecar-tracker.ts:10-17,39-44`, startup health, + main-account cache (`src/codex/main-account-cache.ts:13-20`), shim discovery error + (`src/codex/shim.ts:32-37` — file input already capped at 1 MiB, the retained error + string is not), and project-config warnings (`src/codex/project-config-warnings.ts:290-315`). + GitHub star state (`src/github/star-state.ts:32-38,85-87`) retains only a finite enum + and is EXCLUDED — truncating it would duplicate an existing finite representation. - Codex/Anthropic affinities are count-bounded at - `src/codex/routing.ts:105-109,624-688` and - `src/oauth/anthropic-routing.ts:34-40,62-63,234-241`, but ids are not byte-bounded. + `src/codex/routing.ts:107-108,142,663-721` (now with credential-generation liveness + at :667-669) and `src/oauth/anthropic-routing.ts:36-37,63-64,245-252,361-389,435-489`, + but id components are not byte-bounded. New caps must preserve the generation + validation semantics. Constants and changes: @@ -74,9 +83,9 @@ const MAX_AFFINITY_COMPONENT_BYTES = 512; - `subscribeDebugLogEntries()` throws `ResourceAdmissionError("debug_subscribers",64)` before insertion. Existing subscribers remain active; unsubscribe is idempotent and - updates release-miss metrics. -- The management SSE/tail route catches that error and returns structured 503 with - `Retry-After: 1`; no listener is added on rejection. + updates release-miss metrics. Since no production route subscribes today, this is a + defensive owner-side cap for future consumers; no route-level catch is added in this + phase and the polling `/api/debug/logs` route is untouched. - Debug/injection lines are truncated before ring insertion and before listener fanout; console logging may retain its existing safe line, but no ring stores the original. - Crash trace URL/origin/rejected fields and fixed-slot strings use the 8 KiB cap at @@ -102,9 +111,14 @@ All ring replacements/evictions use centralized subtract helpers. Current anchors: -- `src/server/lifecycle.ts:13-23,37-67` registers every live turn with no admission cap. +- `src/server/lifecycle.ts:19-29,43-73` registers every live turn with no admission cap. + CAUTION: several HTTP paths register the turn only AFTER upstream work and response + construction (`src/server/responses/core.ts:1662-1673,1731-1735,1824-1827`), so a + drop-in `tryRegisterTurn` swap at those sites cannot reject "before handler work". - `src/codex/websocket-registry.ts:4-35,47-73` tracks sockets by account until close. -- `src/storage/worker-lifecycle.ts:25-80` tracks workers until deterministic termination. +- `src/storage/worker-lifecycle.ts:25-85` tracks workers; spawning is already globally + serialized and drains prior workers before creating another (:69-85), so a live-worker + overflow cannot occur through current production paths. - `src/storage/storage-mutation-coordinator.ts:20-64` has one slot per distinct home but no total-home cap. @@ -113,25 +127,57 @@ Production defaults: ```ts export const MAX_ACTIVE_TURNS = 256; export const MAX_TRACKED_CODEX_WEBSOCKETS = 128; -export const MAX_LIVE_STORAGE_WORKERS = 16; export const MAX_ACTIVE_STORAGE_HOME_SLOTS = 32; ``` Change signatures: ```ts -export function tryRegisterTurn(ac: AbortController): AdmissionLease | null; -export function tryRegisterCodexWebSocket(ws: ServerWebSocket): AdmissionLease | null; -export function tryRegisterStorageWorker(worker: Worker): AdmissionLease | null; +export function tryAdmitTurn(): AdmissionLease | null; // called at the request boundary +export function tryReserveCodexWebSocket(): AdmissionLease | null; // called BEFORE upgrade export function tryBeginStorageMutation(...): | { acquired: true; lease: AdmissionLease } | { acquired: false; error: "storage_mutation_busy" }; ``` -Admission must occur before response stream/upgrade/worker creation. HTTP and WS rejects -use structured 503 `server_busy`; storage retains `storage_mutation_busy`. Accepted work -holds one idempotent lease released from every current finish/cancel/error/close/finally -path. Do not throw after a Worker or socket has been created but before it is tracked. +Turn admission moves to the true request boundary: `tryAdmitTurn()` runs in the server +fetch handler before any adapter/upstream work, and the returned lease is threaded to +the existing `registerTurn`/finish sites so one lease covers the whole response +lifecycle (the late `registerTurn` sites in responses/core.ts bind to the +already-acquired lease rather than acquiring a second one). Lease release is +BOUNDARY-OWNED, not delegated to the late sites: the fetch-handler wrapper releases on +handler exception and on non-stream responses (data-plane handlers return both stream +and non-stream responses — `src/server/index.ts:597-705`, `src/server/relay.ts:311-344`); +for streaming responses the lease transfers exactly once to the response-lifetime +wrapper, whose finish/cancel/error paths release it. A lease that was never transferred +is always released by the boundary. WebSocket capacity is +reserved BEFORE `server.upgrade()` (`src/server/index.ts:370-394`) via +`tryReserveCodexWebSocket()`; the reservation is carried through `WsData` and bound to +the SOCKET LIFECYCLE on `open` (`src/server/index.ts:806-812`) — account-registry +binding happens later when pool auth resolves (`src/server/index.ts:915-920`, +`src/codex/websocket-registry.ts:6-18`) — and the reservation is released on upgrade +failure, open-rejection, and close. HTTP and WS rejects use structured 503 +`server_busy`; storage retains `storage_mutation_busy`. Accepted work holds one +idempotent lease released from every current finish/cancel/error/close/finally path. + +The 16-live-storage-worker cap is DROPPED from this phase: production spawning is +already serialized-with-drain, so the cap would be dead code. The remaining unbounded +risk is the queued spawn-closure queue (`spawnGate` in +`src/storage/worker-lifecycle.ts`), which this phase bounds directly: a small +admission cap (`MAX_QUEUED_STORAGE_SPAWNS = 8`) rejects further queued spawn closures +with a typed `StorageSpawnQueueBusyError` before enqueueing. Both gate consumers map +that error to their existing structured failure surfaces instead of a generic worker +failure: policy jobs (`src/storage/policy-job.ts:385-389`) surface it as a +`storage_mutation_busy`-class busy result rather than `policy_worker_failed`, and +restore jobs (`src/storage/restore-job.ts:41-65,230-240`) surface it as busy rather +than `restore_worker_failed`. This supersedes the roadmap's +"active workers hard cap" line (`005_impl_roadmap.md` wp4b row) and the inventory's +"cap creation" note (`000_state_store_inventory.md` §Storage workers/slots): the +bounded resource is the QUEUE, not live workers, because liveness is already +serialized. Tests: `queued storage spawn 9 is rejected busy and accepted spawns drain +normally` (cross-registry file) plus owner-level cases in the policy-job and +restore-job suites proving the externally visible busy result (not +`policy_worker_failed`/`restore_worker_failed`) when the queue is full. Add `activeRegistryMetrics()` returning per-registry `AdmissionMetrics`. `releaseMisses` is the leak signal: increment only when an unregister/finish path attempts to release an @@ -200,7 +246,7 @@ Inventory anchors: - `src/adapters/cursor/live-models.ts:98-125` buffers all RPC chunks before applying the 500-id result cap. -- `src/codex/catalog/provider-fetch.ts:64-129,645-665` keeps one gather Promise per +- `src/codex/catalog/provider-fetch.ts:64-129,654-675` keeps one gather Promise per distinct config fingerprint without a concurrency cap. ```ts @@ -216,21 +262,57 @@ provider requests. Release in `finally` and expose scalar peak/reject counters. ## OAuth flow/probe and pending-code bounds -Current `src/oauth/index.ts:54-61,823-904,939-1020` owns provider/flow maps, XAI probes, -pending pasted code, and refresh flights. The management boundary already rejects pasted -input above 4,096 chars at `src/server/management/oauth-account-routes.ts:183-190`, but -internal callers can bypass that route. +Generic OAuth flow state (`src/oauth/index.ts:827-837,975-990`) is ONE flow per known +provider with an existing busy result, so its active-flow count is already bounded by +the finite provider registry — no new cap there. The management boundary rejects pasted +input above 4,096 chars at `src/server/management/oauth-account-routes.ts:184-193`, but +that is a character limit and internal callers can bypass the route; owner-side UTF-8 +byte validation is still missing at `src/oauth/index.ts:897-921`. + +The actually unbounded flow/probe owners live in `src/codex/auth-api.ts`: + +- `codexAuthLoginState` (`src/codex/auth-api.ts:93,1175-1370`) — random login-state keys + with no distinct-key cap. +- `poolQuotaRefreshInFlight` (`src/codex/auth-api.ts:432-440,579-609`) — a map keyed by + account id whose VALUES are `Set` holding one flight per + credential generation; neither the key count nor the total flight-object count is + capped. ```ts const OAUTH_PENDING_CODE_MAX_BYTES = 4 * 1024; -const MAX_ACTIVE_OAUTH_FLOWS = 32; -const MAX_ACTIVE_OAUTH_PROBES = 16; +const MAX_CODEX_LOGIN_FLOWS = 32; // codexAuthLoginState distinct keys +const MAX_POOL_QUOTA_FLIGHTS = 16; // TOTAL flight objects across ALL account sets ``` -Enforce pending-code UTF-8 bytes again in the owner before assignment. Flow/probe -admission happens before timer, listener, browser/device request, or Promise creation; -reject with the existing 409 busy surface. Existing generation owners remain until their -normal finish/abort timer. Reconciliation of dead provider/account keys remains in 030. +Enforce pending-code UTF-8 bytes in the owner (`src/oauth/index.ts:897-921`) before +assignment. The Codex login-flow and pool-quota-probe caps are implemented in +`src/codex/auth-api.ts` on the exact owners above; admission happens before timer, +listener, browser/device request, or Promise creation. The pool-quota cap counts TOTAL +flight objects across all per-account sets (a per-key cap would not bound the sum); +compatible-generation callers still join the current flight without consuming a new +admission. The caps MUST preserve the credential-generation flight-set semantics +already present in `poolQuotaRefreshInFlight` (writer-generation fencing from 030). + +There is NO existing busy surface for quota probes, so each caller gets a defined +behavior for the typed `PoolQuotaProbeBusyError`: + +- management GET `/api/codex-auth/accounts` (`src/codex/auth-api.ts:818-820`): return + the account list with cached/stale quota values and an additive optional + `quotaProbeSkipped?: true` field on the PER-ACCOUNT `CodexAuthAccountDto` + (`src/codex/auth-api.ts:442-457`) for each account whose probe was skipped — never a + 5xx for a busy probe. +- reset-credit refresh (`src/codex/auth-api.ts:1135-1152`): map the busy error to a + structured 503 `server_busy` with `Retry-After: 1` instead of the current generic 500. +- startup priming (`src/codex/auth-api.ts:631-666`): already best-effort; a busy + rejection is swallowed like any other priming failure. + +Login-flow admission (`MAX_CODEX_LOGIN_FLOWS`) rejects before `startLoginFlow`/browser +work. Note the existing concurrent-flow response is a 409 +(`src/codex/auth-api.ts:1372-1377`, "already in progress"); the CAP overflow is a NEW +structured 503 `server_busy` surface — distinct from that 409, which is preserved +unchanged for the same-provider concurrent case. +Existing generation owners remain until their normal finish/abort timer. Reconciliation +of dead provider/account keys remains in 030. ## MiMo bootstrap value cap @@ -249,9 +331,15 @@ single-flight cleanup. ## Cursor MCP manager payload caps -Current `src/adapters/cursor/mcp-manager.ts:54-70,80-139,152-195,198-223` retains -configured connections/tool schemas and materializes tool/resource payloads without -local count/byte caps. +Current `src/adapters/cursor/mcp-manager.ts:64-70,86-139,152-223` retains configured +connections/tool schemas and materializes tool/resource payloads without local +count/byte caps. SCOPE CORRECTION: the MCP SDK fully materializes responses before the +manager can measure them (`listTools` :123, `callTool` :159-166, `listResources` +:175-177, `readResource` :187-194), so this phase's guarantee is narrowed to bounding +MANAGER-OWNED RETAINED copies — oversized payloads are measured after SDK resolution +and rejected before normalization/copy into manager-owned objects, so nothing over-cap +is retained past the call. Transport/framing-level pre-materialization limits would +require SDK-level changes and are out of scope. ```ts const CURSOR_MCP_MAX_SERVERS = 32; @@ -266,7 +354,7 @@ Validate configured server count in the constructor. During `indexTools`, measur advertised name + description + canonical schema before adding; stop with a typed catalog-too-large error instead of retaining a partial catalog. Resource listings and call/read results are measured before normalization/copy into manager-owned objects; -cancel/reject over cap. `dispose()` remains authoritative and clears accounting before +reject over cap. `dispose()` remains authoritative and clears accounting before awaiting client closes. Do not truncate tool schemas or resource payloads into invalid data. ## Regression tests @@ -280,7 +368,6 @@ Concrete names/fixtures. Put cross-registry lease/accounting cases in NEW - `affinity rejects an oversized key component without colliding or changing routing` - `active turn 257 returns structured server_busy before handler work` - `websocket 129 rejects upgrade without entering account registry` -- `storage worker 17 is not spawned and accepted workers drain normally` - `storage home slot 33 returns storage_mutation_busy without dropping active slots` - `active registry peak rejected and release-miss metrics are monotonic` - `same refresh grant joins a live flight` @@ -292,7 +379,10 @@ Concrete names/fixtures. Put cross-registry lease/accounting cases in NEW - `Cursor model discovery rejects announced and streamed 4 MiB overflow before decode` - `ninth distinct catalog gather is busy while same-fingerprint caller still joins` - `OAuth pending code rejects 4097 UTF-8 bytes in the owner` -- `OAuth flow 33 and probe 17 reject before creating timers or requests` +- `Codex login flow 33 rejects before browser work and pool-quota flight 17 (total across accounts) rejects before request creation while a compatible-generation caller still joins` +- `busy pool-quota probe leaves management GET 200 with cached quota and per-account quotaProbeSkipped` +- `busy pool-quota probe maps reset-credit refresh to 503 server_busy with Retry-After 1` +- `busy pool-quota probe is swallowed by startup priming like other priming failures` - `MiMo accepts exact JWT boundary and rejects one byte over without caching` - `MCP exact catalog boundary admits and one byte over disposes partial state` - `MCP oversized tool result and resource are rejected without truncated payload`. @@ -301,8 +391,8 @@ Verification: ```bash bun test tests/debug.test.ts tests/active-registry-admission.test.ts \ - tests/codex-websocket-registry.test.ts tests/storage-worker-lifecycle.test.ts \ - tests/xai-refresh-lock.test.ts tests/usage-log.test.ts tests/cursor-hardening.test.ts \ + tests/api-debug.test.ts tests/codex-websocket-registry.test.ts \ + tests/codex-account-store.test.ts tests/usage-log.test.ts tests/cursor-hardening.test.ts \ tests/gather-routed-models-single-flight.test.ts tests/oauth-manual-code.test.ts \ tests/codex-auth-api.test.ts tests/mimo-free-provider.test.ts tests/cursor-mcp-manager.test.ts bun run typecheck @@ -311,8 +401,10 @@ bun run privacy:scan ``` File ownership is fixed as follows: debug subscriber/value cases extend -`tests/debug.test.ts`; turns/sockets/workers/slots use the new cross-registry file plus -the existing websocket/worker suites; refresh flights extend `tests/xai-refresh-lock.test.ts`; +`tests/debug.test.ts` and `tests/api-debug.test.ts` (the latter owns the `/api/debug` +route behavior); turns/sockets/slots use the new cross-registry file plus the existing +websocket suite; Codex refresh flights extend `tests/codex-account-store.test.ts` +(NOT `tests/xai-refresh-lock.test.ts`, which owns OAuth/XAI refresh); usage extends `tests/usage-log.test.ts`; Cursor discovery and gather admission extend `tests/cursor-hardening.test.ts` and `tests/gather-routed-models-single-flight.test.ts`; OAuth owner/code and Codex flow/probe cases extend `tests/oauth-manual-code.test.ts` and From 90579d44194545565f8825979d2cf4f830cc096e Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sat, 1 Aug 2026 12:37:41 +0900 Subject: [PATCH 15/30] test(ci): give bulk durable-IO tests a real budget and a clean memo slate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit windows-latest measured 18s and 34s for the two spill tests whose hundreds of fsync'd writes are the assertion itself, against Bun's 5s default — the wait is intrinsic, so they get a dedicated bulk-durable-IO budget with the measurement recorded. The ACL memo-lifecycle test also inherited success memos from earlier win32-override cases and asserted absolute counts, so it now resets the harden state first. --- tests/helpers/test-budget.ts | 7 +++++++ tests/responses-state.test.ts | 5 +++-- tests/windows-secret-acl.test.ts | 4 ++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/helpers/test-budget.ts b/tests/helpers/test-budget.ts index b46139b37..94914ac3e 100644 --- a/tests/helpers/test-budget.ts +++ b/tests/helpers/test-budget.ts @@ -39,6 +39,13 @@ export const SERVER_BUDGET_MS = 30_000; /** Touches SQLite or the filesystem repeatedly. Slow on Windows for reasons outside our code. */ export const STORE_BUDGET_MS = 30_000; +/** + * Hundreds of individually fsync'd durable writes in one test. The fsyncs ARE the + * assertion (durable spill is the product contract), so the wait is intrinsic; the + * orphan-cleanup cap test measured ~34s on windows-latest against Bun's 5s default. + */ +export const BULK_DURABLE_IO_BUDGET_MS = 90_000; + /** * A deadline *inside* a test, for an await that would otherwise hang forever. * diff --git a/tests/responses-state.test.ts b/tests/responses-state.test.ts index 372408403..246705ec6 100644 --- a/tests/responses-state.test.ts +++ b/tests/responses-state.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { BULK_DURABLE_IO_BUDGET_MS } from "./helpers/test-budget"; import { existsSync, linkSync, @@ -986,7 +987,7 @@ describe("Responses previous_response_id state", () => { expect(files.length).toBeLessThanOrEqual(129); expect(files.length).toBeGreaterThan(1); // deferral is real, not immediate unlink void dir; - }); + }, BULK_DURABLE_IO_BUDGET_MS); // 140 fsync'd durable writes ARE the assertion; Windows CI measured ~18s. test("write fsync or publication failure cleans its temp and preserves no unmeasured resident payload", () => { const failures = [ @@ -1048,7 +1049,7 @@ describe("Responses previous_response_id state", () => { expect(result.scanned).toBeLessThanOrEqual(4_096); expect(result.failed).toBe(1); expect(existsSync(join(dir, symlinkName))).toBe(true); - }); + }, BULK_DURABLE_IO_BUDGET_MS); // 521 fsync'd spill writes build the workload; Windows CI measured ~34s. test("orphan scan cap stops enumeration at the limit with an injected directory", () => { // Prove RESPONSE_SPILL_SCAN_MAX itself binds: enumerate more entries than diff --git a/tests/windows-secret-acl.test.ts b/tests/windows-secret-acl.test.ts index be37c0d16..ff1402b79 100644 --- a/tests/windows-secret-acl.test.ts +++ b/tests/windows-secret-acl.test.ts @@ -109,6 +109,10 @@ describe("hardenSecretPath – required mode (required: true)", () => { describe("ephemeral harden success memo lifecycle", () => { test("forgetHardenedSecretPath releases only the actual temp and a second temp hardens again", () => { + // Earlier cases in this file harden real paths under the win32 override and + // legitimately leave success memos behind; this test asserts exact memo + // counts, so it must start from a clean slate rather than inherit them. + resetHardenedStateForTests(); const tempA = join(testDir, "config.json.ocx.1.1.tmp"); const tempB = join(testDir, "config.json.ocx.1.2.tmp"); writeFileSync(tempA, "first", "utf8"); From dd6c60b31f71a3714b98b67ff0a280c3c61150c8 Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sat, 1 Aug 2026 13:05:04 +0900 Subject: [PATCH 16/30] feat(runtime): cap the registries, flights, and retained diagnostics Every operational registry now answers admission before work begins: turns at the fetch boundary, WebSockets before the upgrade, storage workers and mutation homes before spawning, credential-refresh and pool-quota flights before their requests, catalog gathers before providers are contacted. Diagnostic rings and fixed slots account their bytes and truncate on a UTF-8 boundary, discovery and usage reads are byte-bounded with truncation made visible through the management API down to the GUI, and the Cursor MCP manager stages its catalog transactionally under typed limits. The shared gate lives in src/lib/admission.ts; the 035 plan doc carries the four audit rounds that moved each cap onto its real owner. --- .../035_registry_admission_caps.md | 502 +++++++++++++----- .../apikeys-workspace/ApiKeysWorkspace.tsx | 6 +- gui/src/i18n/de.ts | 4 + gui/src/i18n/en.ts | 4 + gui/src/i18n/ja.ts | 4 + gui/src/i18n/ko.ts | 4 + gui/src/i18n/ru.ts | 4 + gui/src/i18n/zh.ts | 4 + gui/src/pages/ApiKeys.tsx | 5 + gui/src/pages/Usage.tsx | 7 +- gui/tests/usage-layout.test.ts | 12 + src/adapters/cursor/live-models.ts | 24 +- src/adapters/cursor/live-transport.ts | 3 + src/adapters/cursor/mcp-manager.ts | 113 +++- src/adapters/cursor/native-exec-mcp.ts | 8 +- src/adapters/mimo-free.ts | 34 +- src/claude/inbound-debug.ts | 59 +- src/cli/catalog-prewarm.ts | 7 +- src/codex/account-store.ts | 104 +++- src/codex/auth-api.ts | 154 +++++- src/codex/auth-context.ts | 4 + src/codex/catalog/provider-fetch.ts | 19 + src/codex/main-account-cache.ts | 9 +- src/codex/project-config-warnings.ts | 13 +- src/codex/routing.ts | 10 + src/codex/shim.ts | 14 +- src/codex/websocket-registry.ts | 27 + src/config.ts | 5 + src/lib/admission.ts | 83 +++ src/lib/crash-guard.ts | 44 +- src/lib/debug-log-buffer.ts | 47 +- src/lib/injection-debug-log.ts | 26 +- src/lib/sidecar-tracker.ts | 7 +- src/oauth/anthropic-routing.ts | 26 +- src/oauth/index.ts | 119 ++++- src/oauth/store.ts | 7 +- src/oauth/token-guardian.ts | 7 + src/server/chat-completions.ts | 4 +- src/server/claude-messages.ts | 4 +- src/server/index.ts | 186 +++++-- src/server/lifecycle.ts | 89 +++- src/server/management-api.ts | 15 +- src/server/management/api-key-usage.ts | 12 +- src/server/management/logs-usage-routes.ts | 23 +- src/server/management/oauth-account-routes.ts | 3 +- src/server/responses/core.ts | 16 +- src/server/startup-health-cache.ts | 15 +- src/server/system-env.ts | 9 +- src/server/ws-bridge.ts | 6 +- src/storage/policy-job.ts | 35 +- src/storage/restore-job.ts | 22 +- src/storage/storage-mutation-coordinator.ts | 42 +- src/storage/worker-lifecycle.ts | 51 ++ src/types.ts | 6 + src/usage/log.ts | 94 +++- tests/active-registry-admission.test.ts | 224 ++++++++ tests/api-debug.test.ts | 18 +- tests/api-usage.test.ts | 21 + tests/claude-inbound-debug.test.ts | 15 +- tests/cli-catalog-prewarm.test.ts | 18 +- tests/codex-account-store.test.ts | 80 ++- tests/codex-auth-api.test.ts | 132 +++++ tests/codex-auth-context.test.ts | 12 +- tests/config.test.ts | 14 + tests/cursor-hardening.test.ts | 19 + tests/cursor-mcp-manager.test.ts | 96 +++- tests/debug.test.ts | 45 +- ...gather-routed-models-single-flight.test.ts | 35 ++ tests/mimo-free-provider.test.ts | 25 + tests/model-visibility-management-api.test.ts | 13 + tests/oauth-manual-code.test.ts | 24 + tests/oauth-refresh.test.ts | 144 ++++- tests/shutdown-drain.test.ts | 19 +- tests/storage-mutation-race.test.ts | 10 + tests/storage-worker-lifecycle.test.ts | 21 +- tests/usage-log.test.ts | 70 ++- 76 files changed, 2815 insertions(+), 401 deletions(-) create mode 100644 src/lib/admission.ts create mode 100644 tests/active-registry-admission.test.ts diff --git a/devlog/_plan/260801_zero_leak_state_stores/035_registry_admission_caps.md b/devlog/_plan/260801_zero_leak_state_stores/035_registry_admission_caps.md index 07c95fb0b..e22644327 100644 --- a/devlog/_plan/260801_zero_leak_state_stores/035_registry_admission_caps.md +++ b/devlog/_plan/260801_zero_leak_state_stores/035_registry_admission_caps.md @@ -8,12 +8,14 @@ Binding inputs: `000_state_store_inventory.md` §§3–6, `005_impl_roadmap.md` ## Outcome Close the operational stores omitted by the initial roadmap audit. Every active registry -gets a finite hard admission cap and coherent busy result; every single-flight has a -finite distinct-key cap and stale-owner policy; discovery and usage payloads are bounded -before full materialization, and MCP payloads are bounded in manager-owned retained -copies (the SDK materializes responses before the manager can measure them — see the -MCP section); retained diagnostic and affinity strings are truncated at -insertion with a visible marker. Existing accepted owners are never silently untracked. +gets a finite hard admission cap and coherent busy result; each identified unbounded +single-flight gets a finite distinct-key cap and identity-checked release, with stale- +owner replacement on refresh flights. Discovery and usage payloads are bounded before +full materialization. MCP payloads are rejected at the earliest enforceable +boundary after SDK materialization and before any manager-owned retained copy (see the +MCP section). Retained diagnostic and affinity strings are truncated at insertion with +a visible marker, but bidirectional protocol-translator payloads are never silently +truncated into invalid data. Existing accepted owners are never silently untracked. This phase provides retained-byte accounting hooks for 040. It does not implement the process-wide eviction policy. @@ -36,7 +38,7 @@ export interface AdmissionMetrics { rejected: number; releaseMisses: number; } -export interface AdmissionLease { release(): void } +export interface AdmissionLease { release(): void } // idempotent, including after forced shutdown export function createAdmissionGate(name: string, limit: number): { tryAcquire(): AdmissionLease | null; metrics(): Readonly; @@ -49,6 +51,34 @@ export function retainedUtf8Bytes(value: string): number; never returns an invalid surrogate fragment. Metrics are scalar-only and monotonic except `active`; they retain no ids, keys, paths, URLs, commands, request bodies, or errors. +## Compatibility configuration for high-risk caps + +Current config ownership is `src/types.ts:531-534,904-923` and +`src/config.ts:470-483,736-788`; Cursor transport receives provider config at +`src/adapters/cursor/transport.ts:16-34` and constructs the MCP manager at +`src/adapters/cursor/live-transport.ts:416-430`. + +The compatibility-sensitive limits keep safe defaults but accept explicit positive- +integer overrides: + +```ts +const MANAGEMENT_USAGE_MAX_READ_BYTES = 64 * 1024 * 1024; +const CURSOR_MCP_MAX_TOOLS = 512; +const CURSOR_MCP_MAX_SCHEMA_BYTES = 256 * 1024; +const CURSOR_MCP_MAX_RESULT_BYTES = 8 * 1024 * 1024; +``` + +Exact config keys are top-level `managementUsageMaxReadBytes` (default 67,108,864), +plus `providers..mcpMaxTools` (default 512), +`providers..mcpMaxSchemaBytes` (default 262,144), and +`providers..mcpMaxResultBytes` (default 8,388,608). Add them to the TypeScript +types and Zod schema; reject non-finite, non-integer, or non-positive writes and thread +the provider values into `CursorMcpManagerOptions`. A configured lower limit is honored +exactly. Do not silently clamp, truncate, or fall back to a larger default: usage +history gets visible qualification and MCP gets a typed visible error. Affinity and +MiMo caps stay fixed (low compatibility risk); all other admission/count caps in this +phase also remain fixed. + ## Debug subscribers and diagnostic rings Current anchors: @@ -58,7 +88,8 @@ Current anchors: `/api/debug/logs` is a polling JSON route (`src/server/management/logs-usage-routes.ts:141-144`), not SSE. The cap therefore lands in the owner only; no route change exists or is invented. - `src/lib/injection-debug-log.ts:10-27` retains 2,000 unbounded lines. -- `src/claude/inbound-debug.ts:40-106` retains 20 variable metadata rows. +- `src/claude/inbound-debug.ts:40-106` retains 20 variable metadata rows; `Object.keys` + at :98 has no key-count or aggregate-row-byte bound. - `src/lib/crash-guard.ts:206-245` retains 12 traces with unbounded URL/origin/rejection. - fixed string slots include `src/lib/sidecar-tracker.ts:10-17,39-44`, startup health, main-account cache (`src/codex/main-account-cache.ts:13-20`), shim discovery error @@ -67,8 +98,8 @@ Current anchors: GitHub star state (`src/github/star-state.ts:32-38,85-87`) retains only a finite enum and is EXCLUDED — truncating it would duplicate an existing finite representation. - Codex/Anthropic affinities are count-bounded at - `src/codex/routing.ts:107-108,142,663-721` (now with credential-generation liveness - at :667-669) and `src/oauth/anthropic-routing.ts:36-37,63-64,245-252,361-389,435-489`, + `src/codex/routing.ts:107-108,142,663-721` (with credential-generation liveness + at :667-669) and `src/oauth/anthropic-routing.ts:36-37,63-64,245-252,345-449`, but id components are not byte-bounded. New caps must preserve the generation validation semantics. @@ -78,16 +109,24 @@ Constants and changes: const MAX_DEBUG_SUBSCRIBERS = 64; const MAX_DEBUG_LINE_BYTES = 16 * 1024; const MAX_DIAGNOSTIC_VALUE_BYTES = 8 * 1024; +const MAX_CLAUDE_INBOUND_METADATA_KEYS = 64; +const MAX_CLAUDE_INBOUND_ROW_BYTES = 32 * 1024; const MAX_AFFINITY_COMPONENT_BYTES = 512; ``` - `subscribeDebugLogEntries()` throws `ResourceAdmissionError("debug_subscribers",64)` before insertion. Existing subscribers remain active; unsubscribe is idempotent and - updates release-miss metrics. Since no production route subscribes today, this is a - defensive owner-side cap for future consumers; no route-level catch is added in this - phase and the polling `/api/debug/logs` route is untouched. + repeated unsubscribe is a no-op; only a foreign owner token updates release-miss + metrics. Since no production route subscribes today, this is a defensive owner-side + cap for future consumers; no route-level catch is added in this phase and the polling + `/api/debug/logs` route is untouched. - Debug/injection lines are truncated before ring insertion and before listener fanout; console logging may retain its existing safe line, but no ring stores the original. +- Claude inbound rows cap `metadataKeys` at 64 and expose `metadataKeysDropped`. + Variable strings use the diagnostic value cap. Build each row in deterministic field + order under the 32 KiB aggregate UTF-8 budget, omitting/truncating only optional + diagnostic fields and setting `rowTruncated: true`; never retain the pre-cap key + array or row. - Crash trace URL/origin/rejected fields and fixed-slot strings use the 8 KiB cap at assignment. Preserve redaction first, then truncate. - Affinity key components are normalized at admission. Oversized thread/scope/account ids @@ -100,98 +139,135 @@ Expose hooks: export function debugBufferMetrics(): { entries: number; bytes: number; subscribers: AdmissionMetrics; oldestAt: number | null }; export function injectionBufferMetrics(): { entries: number; bytes: number; oldestAt: number | null }; export function crashRingMetrics(): { entries: number; bytes: number; oldestAt: number | null }; +export function claudeInboundDebugMetrics(): { entries: number; bytes: number; oldestAt: number | null }; export function evictOldestDebugEntryForBudget(): number; export function evictOldestInjectionEntryForBudget(): number; export function evictOldestCrashTraceForBudget(): number; +export function evictOldestClaudeInboundForBudget(): number; ``` -All ring replacements/evictions use centralized subtract helpers. +All ring replacements, explicit/flag-off clears, and evictions use centralized subtract +helpers so 040/wp5 sees exact current bytes. ## Active turns, WebSockets, workers, and slots Current anchors: -- `src/server/lifecycle.ts:19-29,43-73` registers every live turn with no admission cap. - CAUTION: several HTTP paths register the turn only AFTER upstream work and response - construction (`src/server/responses/core.ts:1662-1673,1731-1735,1824-1827`), so a - drop-in `tryRegisterTurn` swap at those sites cannot reject "before handler work". +- `src/server/lifecycle.ts:19-29,43-73,76-92` registers every live turn with no + admission cap and forcibly clears `activeTurns` after shutdown abort. + `trackStreamLifetime()` registers only after the body already exists (:43-49). +- Response handling also registers late at + `src/server/responses/core.ts:1671-1719,1731-1782,1824-1827,1952-1955,2010-2015,2097-2099,2566-2568`. + A WebSocket frame registers at `src/server/index.ts:856-881` and then enters the same + response pipeline at :915-958, so adding admission inside stream wrappers would + double-register one WS turn. - `src/codex/websocket-registry.ts:4-35,47-73` tracks sockets by account until close. -- `src/storage/worker-lifecycle.ts:25-85` tracks workers; spawning is already globally - serialized and drains prior workers before creating another (:69-85), so a live-worker - overflow cannot occur through current production paths. -- `src/storage/storage-mutation-coordinator.ts:20-64` has one slot per distinct home but - no total-home cap. +- `src/storage/worker-lifecycle.ts:25-85` can register only after a `Worker` exists; + `withStorageWorkerSpawnGate()` drains every predecessor before invoking the next spawn + closure (:69-85). Production workers are created at + `src/storage/policy-job.ts:286-304` and `src/storage/restore-job.ts:144-166`. +- `src/storage/storage-mutation-coordinator.ts:20-64,78-109` has one slot per distinct + home but no total-home cap and releases by recomputing the home key. Production defaults: ```ts export const MAX_ACTIVE_TURNS = 256; export const MAX_TRACKED_CODEX_WEBSOCKETS = 128; +export const MAX_RESERVED_STORAGE_WORKER_SPAWNS = 16; export const MAX_ACTIVE_STORAGE_HOME_SLOTS = 32; ``` Change signatures: ```ts -export function tryAdmitTurn(): AdmissionLease | null; // called at the request boundary -export function tryReserveCodexWebSocket(): AdmissionLease | null; // called BEFORE upgrade +export interface ActiveTurnLease extends AdmissionLease { + bindAbortController(ac: AbortController): void; +} +export interface AdmissionReservation extends AdmissionLease { + bind(value: T): void; +} +export function tryAdmitTurn(): ActiveTurnLease | null; +export function tryReserveCodexWebSocket(): AdmissionReservation> | null; +export function tryReserveStorageWorker(): AdmissionReservation | null; export function tryBeginStorageMutation(...): | { acquired: true; lease: AdmissionLease } | { acquired: false; error: "storage_mutation_busy" }; ``` -Turn admission moves to the true request boundary: `tryAdmitTurn()` runs in the server -fetch handler before any adapter/upstream work, and the returned lease is threaded to -the existing `registerTurn`/finish sites so one lease covers the whole response -lifecycle (the late `registerTurn` sites in responses/core.ts bind to the -already-acquired lease rather than acquiring a second one). Lease release is -BOUNDARY-OWNED, not delegated to the late sites: the fetch-handler wrapper releases on -handler exception and on non-stream responses (data-plane handlers return both stream -and non-stream responses — `src/server/index.ts:597-705`, `src/server/relay.ts:311-344`); -for streaming responses the lease transfers exactly once to the response-lifetime -wrapper, whose finish/cancel/error paths release it. A lease that was never transferred -is always released by the boundary. WebSocket capacity is -reserved BEFORE `server.upgrade()` (`src/server/index.ts:370-394`) via -`tryReserveCodexWebSocket()`; the reservation is carried through `WsData` and bound to -the SOCKET LIFECYCLE on `open` (`src/server/index.ts:806-812`) — account-registry -binding happens later when pool auth resolves (`src/server/index.ts:915-920`, -`src/codex/websocket-registry.ts:6-18`) — and the reservation is released on upgrade -failure, open-rejection, and close. HTTP and WS rejects use structured 503 -`server_busy`; storage retains `storage_mutation_busy`. Accepted work holds one -idempotent lease released from every current finish/cancel/error/close/finally path. - -The 16-live-storage-worker cap is DROPPED from this phase: production spawning is -already serialized-with-drain, so the cap would be dead code. The remaining unbounded -risk is the queued spawn-closure queue (`spawnGate` in -`src/storage/worker-lifecycle.ts`), which this phase bounds directly: a small -admission cap (`MAX_QUEUED_STORAGE_SPAWNS = 8`) rejects further queued spawn closures -with a typed `StorageSpawnQueueBusyError` before enqueueing. Both gate consumers map -that error to their existing structured failure surfaces instead of a generic worker -failure: policy jobs (`src/storage/policy-job.ts:385-389`) surface it as a -`storage_mutation_busy`-class busy result rather than `policy_worker_failed`, and -restore jobs (`src/storage/restore-job.ts:41-65,230-240`) surface it as busy rather -than `restore_worker_failed`. This supersedes the roadmap's -"active workers hard cap" line (`005_impl_roadmap.md` wp4b row) and the inventory's -"cap creation" note (`000_state_store_inventory.md` §Storage workers/slots): the -bounded resource is the QUEUE, not live workers, because liveness is already -serialized. Tests: `queued storage spawn 9 is rejected busy and accepted spawns drain -normally` (cross-registry file) plus owner-level cases in the policy-job and -restore-job suites proving the externally visible busy result (not -`policy_worker_failed`/`restore_worker_failed`) when the queue is full. +Turn admission moves to the true ingress. After draining/auth/origin checks but before +request parsing, adapter work, or upstream I/O, every turn-producing POST branch in +`src/server/index.ts:480-510,513-537,569-637,642-705,711-739` acquires ONE lease. Each +generating WS `response.create` frame acquires ONE lease at :856-881 before +`handleResponses`; warmup frames do not consume a turn lease. Thread the same lease +through handler options and every nested response/stream wrapper. Existing late +`registerTurn`/`trackStreamLifetime` sites become controller binding and idempotent +release sites only: they MUST NOT acquire another gate slot. This preserves one active +count for HTTP translation, sidecars, native passthrough, and WS-to-Responses +translation even when several wrappers participate. + +Lease release is boundary-owned: handler exceptions and non-stream responses release in +the ingress `finally`; streaming responses transfer the same lease exactly once to the +response-lifetime wrapper (`src/server/relay.ts:303-350`), whose settle/cancel/error path +releases it. A WS frame releases in its existing `finally` (:957-961). HTTP admission +failure returns structured 503 `server_busy` with `Retry-After: 1`; an already-upgraded +WS frame gets a typed retryable `server_busy` error frame and starts no handler work. + +Forced shutdown replaces direct `activeTurns.clear()` with one centralized +`abortAndReleaseAllTurns()` operation. It snapshots each admitted owner, aborts every +bound controller, removes the owner, and calls that owner's same idempotent lease once. +Later stream/WS finalizers call the same lease and become no-ops: they do not increment +`releaseMisses`, underflow the gate, or leak `active`. A direct duplicate lease release +is always a no-op; `releaseMisses` is reserved for a genuinely foreign/unknown owner +token. + +WebSocket capacity is reserved before `server.upgrade()` +(`src/server/index.ts:370-394`). Carry the reservation in `WsData` +(`src/server/ws-bridge.ts:21-55`), bind the actual socket on `open` +(`src/server/index.ts:806-812`), and then allow the existing account binding to update +later at :915-920. Upgrade failure, open rejection, and close (:964-972) all release the +same reservation. No API requires a socket object before upgrade. + +Storage worker admission is likewise reservation-level. Call +`tryReserveStorageWorker()` BEFORE entering `withStorageWorkerSpawnGate()` and before +`new Worker`; carry it into the spawn closure, bind immediately after construction, and +release only after deterministic termination, spawn cancellation, or construction +failure. The cap counts queued reservations plus the at-most-one bound live worker. +Because :69-85 drains predecessors, a production "worker 17 concurrently live" state +is unreachable; the regression is instead `storage worker reservation 17 rejects +before enqueue while the first 16 spawn serially and drain`. Policy and restore callers +retain their returned reservation until their `terminateStorageWorker()` join settles. + +Storage mutation callers must also retain the returned lease, not recompute a home key +on release. `withStorageMutationSlot()`/`runPolicyStorageMutation()` +(`src/storage/storage-mutation-coordinator.ts:78-109`), cleanup +(`src/storage/cleanup-job.ts:35-53`), restore (`src/storage/restore-job.ts:253-273`), +and the policy job (`src/storage/policy-job.ts:350-389`) release that exact lease in +their authoritative `finally`, after worker termination where applicable. + +Admission leases are NEVER released by state-store sweeper reconciliation. Normal +settle/cancel/close/termination is authoritative; reconciliation may report stale +owners but cannot decrement an active gate or silently untrack accepted work. Add `activeRegistryMetrics()` returning per-registry `AdmissionMetrics`. `releaseMisses` -is the leak signal: increment only when an unregister/finish path attempts to release an -unknown owner; never remove another owner to hide it. +is the leak signal under the foreign-owner rule above; never remove another owner to +hide it. ## Codex credential refresh flights -Current `src/codex/account-store.ts:268-270,349-465` deduplicates by grant fingerprint +Current `src/codex/account-store.ts:235-270,349-465` deduplicates by grant fingerprint and deletes in `finally`, but distinct fingerprints have no cap and a stuck Promise is joined forever. ```ts const MAX_CODEX_REFRESH_FLIGHTS = 32; const CODEX_REFRESH_FLIGHT_STALE_MS = 120_000; +export class CodexCredentialRefreshBusyError extends Error { + readonly code = "CODEX_REFRESH_BUSY"; +} +export class CodexCredentialRefreshStaleError extends Error { + readonly code = "CODEX_REFRESH_STALE"; +} interface RefreshFlight { promise: Promise; startedAt: number; @@ -213,6 +289,28 @@ Admission order: An aborted stale owner remains awaited by its original callers and settles with a typed retryable error; it is not silently detached. The replacement is the only mapped owner. +Both new errors are RETRYABLE credential outcomes, never evidence for reauthentication. +Enumerate them explicitly at every current consumer so they cannot fall through an +`unknown` branch: + +- `src/codex/auth-context.ts:186-189`: both return `false` from + `shouldMarkAccountNeedsReauthForCodexAuthFailure()`. +- `src/oauth/token-guardian.ts:210-230`: record ordinary transient backoff and a skipped/ + retryable result; do not call `markCodexAccountValidationFailed()` and never mark the + failure permanent. +- `src/codex/auth-api.ts:559-566`: quota reads keep cached quota with + `needsReauth: false` and expose the same optional retryable-skip marker used for a busy + quota probe. +- `src/codex/auth-api.ts:1354-1377`: login background state records a retryable busy + message, while a synchronous route rejection returns structured 503 `server_busy` + with `Retry-After: 1`; neither path becomes the generic 500 or reauth-worthy state. + +Refresh-flight owner tests live in `tests/codex-account-store.test.ts`, alongside the +existing grant/file-lock contract at :229-339. Consumer classification extends +`tests/codex-auth-context.test.ts`, `tests/token-guardian.test.ts`, and +`tests/codex-auth-api.test.ts`; do not put Codex grant-flight tests in +`tests/xai-refresh-lock.test.ts`. + ## Management usage-read bound Current `src/usage/log.ts:389-403,470-511` reads `stat.size` into one Buffer, converts all @@ -224,21 +322,51 @@ const MANAGEMENT_USAGE_MAX_ENTRIES = 200_000; const MANAGEMENT_USAGE_FLIGHT_STALE_MS = 30_000; interface ManagementUsageSnapshot { entries: PersistedUsageEntry[]; - revision: UsageLogRevision; + revision: UsageLogRevision | null; truncatedPrefixBytes: number; + entriesTruncated: boolean; + entriesDropped: number; } ``` -- Read at most the newest 64 MiB through bounded 1 MiB chunks; when starting mid-file, +- Read at most the newest effective `managementUsageMaxReadBytes` (64 MiB by default) + through bounded 1 MiB chunks; when starting mid-file, discard the first partial line. - Parse batches cooperatively and retain at most the newest 200,000 valid rows. -- Return `truncatedPrefixBytes` so callers/GUI cannot present capped history as complete. +- `truncatedPrefixBytes` counts omitted file-prefix bytes (including the discarded + partial line). Independently, `entriesTruncated` and `entriesDropped` report valid + parsed rows dropped by the 200,000-entry cap. A byte-truncated prefix has an unknown + row count, so never fold it into `entriesDropped`. - A 30-second-stale same-revision flight is aborted through a local controller and - replaced. At most one management usage-read flight exists. + replaced. At most one management usage-read flight exists. Include the effective byte + limit in the flight and cache key so a live config change cannot join/reuse a snapshot + built under another limit. - Never allocate `Buffer.allocUnsafe(stat.size)` for a file over the cap. -Update usage summary/routes to preserve totals for the returned window and expose an -additive `historyTruncated` boolean; do not fabricate lifetime totals. +Every consumer derives `historyTruncated = truncatedPrefixBytes > 0 || +entriesTruncated`; totals describe only the returned window: + +- `/api/usage` at `src/server/management/logs-usage-routes.ts:187-205` carries + `historyTruncated`, `truncatedPrefixBytes`, `entriesTruncated`, and `entriesDropped` + through both fresh and cached summaries. The read-failure fallback returns explicit + false/zero metadata. Do not fabricate lifetime totals. +- `UsageResponse` at `gui/src/pages/Usage.tsx:77-87` accepts those fields. Render a + persistent localized qualification banner whenever `historyTruncated` is true, and + change the active "All" option at :243-255 to localized "Available history" (not + "All"). The notice applies to every range because the omitted prefix may overlap any + requested time window. +- `readApiKeyUsageRollup()` at + `src/server/management/api-key-usage.ts:131-162` propagates snapshot truncation in + `ApiKeyUsageSnapshot`; `/api/keys` carries it at + `src/server/management/oauth-account-routes.ts:482-506`. Existing `totalRequests` + remains additive compatibility data but means "requests in available history" when + truncated. `gui/src/pages/ApiKeys.tsx:28-57,175-186` passes the flag to + `gui/src/components/apikeys-workspace/ApiKeysWorkspace.tsx:358-387`, which visibly + qualifies both the total label and attribution-since date. + +Add the new Usage/API-key strings in every existing GUI locale. Capped history must be +visibly qualified on all three surfaces (usage API, Usage GUI, API-key GUI); a boolean +that no consumer renders is insufficient. ## Cursor model discovery and gather flights @@ -252,46 +380,110 @@ Inventory anchors: ```ts const CURSOR_MODEL_DISCOVERY_MAX_BYTES = 4 * 1024 * 1024; const MAX_CONCURRENT_CATALOG_GATHERS = 8; +export class CatalogGatherBusyError extends Error { + readonly code = "catalog_busy"; + readonly retryAfterSeconds = 1; +} ``` Reject an announced `content-length` above 4 MiB before reading and cancel the body once streamed chunks exceed 4 MiB. Decode only after admission. Keep the existing 500 model-id result cap. Add a gate around distinct gather fingerprints; same-fingerprint callers -still join, while the ninth distinct gather receives a typed busy result and starts no -provider requests. Release in `finally` and expose scalar peak/reject counters. +still join, while the ninth distinct gather rejects with `CatalogGatherBusyError` and +starts no provider requests. `gatherRoutedModels()` keeps its existing +`Promise` signature (no result union), so existing callers keep +compiling. Release the distinct-flight lease in the flight's own `finally` even if +`clearGatherRoutedModelsInflight()` removes its map entry; cache clearing never releases +active admission. + +Caller mapping is explicit: + +- The central management dispatch at `src/server/management-api.ts:82-136` catches only + `CatalogGatherBusyError` and returns structured 503 `catalog_busy` with + `Retry-After: 1`; other errors retain existing handling. The public `/v1/models` + boundary at `src/server/index.ts:410-474` uses the same 503 mapping. +- Startup prewarm at `src/cli/catalog-prewarm.ts:15-22` warns once that discovery was + skipped because it was busy, then settles successfully; it no longer silently + swallows this typed condition. +- System-env context discovery at `src/server/system-env.ts:235-241` treats busy as a + bounded skip and continues with native/default windows only. +- Direct maintenance/catalog sync at `src/codex/catalog/sync.ts:485-509` remains + error-propagating so its owning route/CLI can report failure; it is not converted to + an empty catalog. ## OAuth flow/probe and pending-code bounds +Account-scoped access-token refresh is another unbounded owner: +`tokenRefreshes` at `src/oauth/index.ts:55,235-268` stores one Promise per distinct +`provider\0accountId` with no cap or stale replacement. + +```ts +const MAX_OAUTH_TOKEN_REFRESH_FLIGHTS = 32; +const OAUTH_TOKEN_REFRESH_FLIGHT_STALE_MS = 120_000; +interface OAuthTokenRefreshFlight { + promise: Promise; + startedAt: number; + abort: AbortController; +} +export class OAuthTokenRefreshBusyError extends Error { + readonly code = "OAUTH_TOKEN_REFRESH_BUSY"; +} +export class OAuthTokenRefreshStaleError extends Error { + readonly code = "OAUTH_TOKEN_REFRESH_STALE"; +} +``` + +Same-key callers join a flight younger than 120 seconds. An older same-key flight is +aborted with typed retryable `OAuthTokenRefreshStaleError`, removed only by identity, +and replaced. A 33rd distinct key throws typed retryable +`OAuthTokenRefreshBusyError` before provider refresh/network work. Thread the local +abort signal through the provider refresh path, and let the original stale callers +observe settlement; `finally` deletes only the matching flight. Neither typed error +deletes credentials or marks an OAuth account as needing reauthentication. + Generic OAuth flow state (`src/oauth/index.ts:827-837,975-990`) is ONE flow per known provider with an existing busy result, so its active-flow count is already bounded by -the finite provider registry — no new cap there. The management boundary rejects pasted -input above 4,096 chars at `src/server/management/oauth-account-routes.ts:184-193`, but +the seven-entry static provider registry, and a same-provider second flow already +rejects at :982-985. A generic `MAX_ACTIVE_OAUTH_FLOWS = 32` would therefore be +vacuous and is not added. The management boundary rejects pasted input above 4,096 +characters at `src/server/management/oauth-account-routes.ts:184-193` (the check is +at :191), but that is a character limit and internal callers can bypass the route; owner-side UTF-8 byte validation is still missing at `src/oauth/index.ts:897-921`. The actually unbounded flow/probe owners live in `src/codex/auth-api.ts`: -- `codexAuthLoginState` (`src/codex/auth-api.ts:93,1175-1370`) — random login-state keys +- `codexAuthLoginState` (`src/codex/auth-api.ts:93,1175-1377`) — random login-state keys with no distinct-key cap. -- `poolQuotaRefreshInFlight` (`src/codex/auth-api.ts:432-440,579-609`) — a map keyed by +- `poolQuotaRefreshInFlight` (`src/codex/auth-api.ts:432-440,583-609`) — a map keyed by account id whose VALUES are `Set` holding one flight per credential generation; neither the key count nor the total flight-object count is capped. ```ts const OAUTH_PENDING_CODE_MAX_BYTES = 4 * 1024; -const MAX_CODEX_LOGIN_FLOWS = 32; // codexAuthLoginState distinct keys +const MAX_CODEX_LOGIN_STATE_ROWS = 32; // all retained codexAuthLoginState keys const MAX_POOL_QUOTA_FLIGHTS = 16; // TOTAL flight objects across ALL account sets +export class CodexLoginStateBusyError extends ResourceAdmissionError {} +export class PoolQuotaProbeBusyError extends ResourceAdmissionError {} ``` Enforce pending-code UTF-8 bytes in the owner (`src/oauth/index.ts:897-921`) before -assignment. The Codex login-flow and pool-quota-probe caps are implemented in -`src/codex/auth-api.ts` on the exact owners above; admission happens before timer, -listener, browser/device request, or Promise creation. The pool-quota cap counts TOTAL -flight objects across all per-account sets (a per-key cap would not bound the sum); -compatible-generation callers still join the current flight without consuming a new -admission. The caps MUST preserve the credential-generation flight-set semantics -already present in `poolQuotaRefreshInFlight` (writer-generation fencing from 030). +assignment. For Codex login, prune terminal rows whose 300-second retention expired, +then evict oldest terminal rows if needed; pending/starting owners are never evicted. +Insert one identity-bearing `starting` row before `startLoginFlow`, browser work, polling +Promise, or cleanup timer. If 32 pending/retained rows still occupy the map, return +typed 503 `server_busy`. Start failure deletes the same row immediately; terminal +completion keeps it for polling and schedules an identity-checked delete at 300 seconds, +so an old timer cannot remove a replacement row. + +The pool-quota cap counts TOTAL flight objects across all account sets (a per-key cap +would not bound the sum). Check for a compatible live generation and join it first; +otherwise reserve before `fetchFreshPoolAccountQuota()` creates a Promise. Retain the +exact flight owner in its account set and release only that owner in `finally`, deleting +the account set only when it is still the mapped set and becomes empty. The cap MUST +preserve the writer-generation fencing already present in +`poolQuotaRefreshInFlight` (030). There is NO existing busy surface for quota probes, so each caller gets a defined behavior for the typed `PoolQuotaProbeBusyError`: @@ -306,7 +498,7 @@ behavior for the typed `PoolQuotaProbeBusyError`: - startup priming (`src/codex/auth-api.ts:631-666`): already best-effort; a busy rejection is swallowed like any other priming failure. -Login-flow admission (`MAX_CODEX_LOGIN_FLOWS`) rejects before `startLoginFlow`/browser +Login-state admission (`MAX_CODEX_LOGIN_STATE_ROWS`) rejects before `startLoginFlow`/browser work. Note the existing concurrent-flow response is a 409 (`src/codex/auth-api.ts:1372-1377`, "already in progress"); the CAP overflow is a NEW structured 503 `server_busy` surface — distinct from that 409, which is preserved @@ -331,15 +523,15 @@ single-flight cleanup. ## Cursor MCP manager payload caps -Current `src/adapters/cursor/mcp-manager.ts:64-70,86-139,152-223` retains configured -connections/tool schemas and materializes tool/resource payloads without local -count/byte caps. SCOPE CORRECTION: the MCP SDK fully materializes responses before the -manager can measure them (`listTools` :123, `callTool` :159-166, `listResources` -:175-177, `readResource` :187-194), so this phase's guarantee is narrowed to bounding -MANAGER-OWNED RETAINED copies — oversized payloads are measured after SDK resolution -and rejected before normalization/copy into manager-owned objects, so nothing over-cap -is retained past the call. Transport/framing-level pre-materialization limits would -require SDK-level changes and are out of scope. +Current `src/adapters/cursor/mcp-manager.ts:64-70,80-101,121-139,152-223` retains +configured connections/tool schemas and materializes tool/resource payloads without +local count/byte caps. The MCP SDK fully materializes `listTools` (:123), `callTool` +(:159-166), `listResources` (:175-177), and `readResource` (:187-194) before the manager +regains control. No manager-only patch can truthfully promise a transport/framing limit. +This phase therefore measures immediately after each SDK Promise resolves, rejects with +a typed error before normalization/copy/registry commit, and retains NO over-limit +manager-owned state. SDK/transport pre-materialization limits remain explicitly out of +scope. ```ts const CURSOR_MCP_MAX_SERVERS = 32; @@ -350,12 +542,37 @@ const CURSOR_MCP_MAX_CATALOG_BYTES = 4 * 1024 * 1024; const CURSOR_MCP_MAX_RESULT_BYTES = 8 * 1024 * 1024; ``` -Validate configured server count in the constructor. During `indexTools`, measure -advertised name + description + canonical schema before adding; stop with a typed -catalog-too-large error instead of retaining a partial catalog. Resource listings and -call/read results are measured before normalization/copy into manager-owned objects; -reject over cap. `dispose()` remains authoritative and clears accounting before -awaiting client closes. Do not truncate tool schemas or resource payloads into invalid data. +The tool/schema/result defaults resolve through the compatibility config above; server, +resource, and aggregate catalog caps stay fixed. Add typed +`McpCatalogLimitError`/`McpPayloadTooLargeError` classes with stable codes that +`src/adapters/cursor/native-exec-mcp.ts:52-105` maps to the existing protobuf error +cases, preserving a visible failure instead of a partial success. + +Validate server count in the constructor. Discovery is transactional for the WHOLE +manager: `connectOne()` returns a staged connection plus staged tool handles, and +`connectAll()` validates advertised name + description + canonical schema bytes and the +global tool/catalog totals before committing either `servers` or `toolIndex`. If any +limit fails, close every staged client, clear staging, and reject `ensureConnected()`; +no prefix of the catalog survives. The broad catch at +`src/adapters/cursor/mcp-manager.ts:90-101` may continue isolating ordinary per-server +connect/protocol failures, but it MUST rethrow limit errors. A server whose discovery +fails ordinarily is closed and omitted rather than left as a connected empty partial. + +`listResources()` stages all normalized listings and commits its return only after the +count/byte checks pass. `callTool()` and `readResource()` measure the fully materialized +SDK value immediately and reject before `normalizeContent`, blob decode, or manager +retention. `dispose()` remains authoritative and clears accounting before awaiting +client closes. + +Image translation needs a second-allocation guard. At +`src/adapters/cursor/native-exec-mcp.ts:115-142`, compute decoded base64 length from the +encoded string without allocating, include it in the aggregate result budget, and throw +`McpPayloadTooLargeError` BEFORE `Buffer.from`/`Uint8Array.from`; apply the same rule to +resource blobs at `src/adapters/cursor/mcp-manager.ts:183-195`. Exact-boundary data is +decoded once. Oversized schemas, tool results, resources, and images are REJECTED with +typed errors—never truncated, sliced, compressed beyond existing valid image semantics, +or flattened/restored into syntactically valid-looking partial protocol data. opencodex +remains a bidirectional translator, so byte admission cannot change payload meaning. ## Regression tests @@ -363,53 +580,90 @@ Concrete names/fixtures. Put cross-registry lease/accounting cases in NEW `tests/active-registry-admission.test.ts`; extend existing owner suites for the rest: - `debug subscriber 65 is rejected while the first 64 still receive entries` -- `subscriber unsubscribe is idempotent and leak metric records unknown release` +- `subscriber unsubscribe is idempotent and only a foreign owner records release miss` +- `polling debug-log route remains JSON and never invents an SSE admission response` - `debug injection crash and fixed-slot strings truncate on UTF-8 boundary with marker` +- `Claude inbound metadata key 65 and aggregate row overflow are visibly capped and wp5 hooks account exact bytes` - `affinity rejects an oversized key component without colliding or changing routing` - `active turn 257 returns structured server_busy before handler work` +- `one HTTP or WS turn through nested stream wrappers consumes one lease only` +- `forced shutdown abort releases every lease and later finalizers cause no miss or underflow` - `websocket 129 rejects upgrade without entering account registry` +- `storage worker reservation 17 rejects before enqueue and the first 16 spawn serially and drain` - `storage home slot 33 returns storage_mutation_busy without dropping active slots` +- `cleanup restore and policy retain their exact mutation lease through worker join` - `active registry peak rejected and release-miss metrics are monotonic` - `same refresh grant joins a live flight` - `33rd distinct refresh grant is rejected before file lock and fetch` - `stale refresh flight is aborted and replaced without deleting the replacement` +- `Codex refresh busy and stale stay retryable in auth context guardian quota and login consumers` - `usage reader never requests more than 64 MiB from an oversized log` -- `usage reader returns newest complete capped rows and historyTruncated` +- `usage byte-prefix truncation and entry-count truncation report independent metadata` +- `usage route cache preserves truncation metadata and invalidates when configured byte limit changes` +- `Usage All becomes Available history and API-key lifetime totals are qualified when capped` - `stale usage-read flight is replaced and old completion cannot clear new owner` - `Cursor model discovery rejects announced and streamed 4 MiB overflow before decode` - `ninth distinct catalog gather is busy while same-fingerprint caller still joins` +- `catalog busy maps management and v1 models to 503 startup to warn-skip and system-env to skip` - `OAuth pending code rejects 4097 UTF-8 bytes in the owner` -- `Codex login flow 33 rejects before browser work and pool-quota flight 17 (total across accounts) rejects before request creation while a compatible-generation caller still joins` +- `OAuth token-refresh flight 33 rejects and a stale same-key owner cannot delete replacement` +- `Codex login state row 33 rejects before browser work and terminal TTL deletes only its owner` +- `pool-quota flight 17 total across accounts rejects before request creation while compatible generation joins` - `busy pool-quota probe leaves management GET 200 with cached quota and per-account quotaProbeSkipped` - `busy pool-quota probe maps reset-credit refresh to 503 server_busy with Retry-After 1` - `busy pool-quota probe is swallowed by startup priming like other priming failures` - `MiMo accepts exact JWT boundary and rejects one byte over without caching` -- `MCP exact catalog boundary admits and one byte over disposes partial state` -- `MCP oversized tool result and resource are rejected without truncated payload`. +- `MCP exact transactional catalog boundary admits and one byte over closes staging with empty committed catalog` +- `MCP list tool call resource and read bounds are proven after SDK receipt with no retained partial` +- `MCP oversized base64 image rejects before its second decode allocation` +- `usage and MCP config overrides change the effective bound while defaults remain compatible`. Verification: ```bash bun test tests/debug.test.ts tests/active-registry-admission.test.ts \ - tests/api-debug.test.ts tests/codex-websocket-registry.test.ts \ - tests/codex-account-store.test.ts tests/usage-log.test.ts tests/cursor-hardening.test.ts \ - tests/gather-routed-models-single-flight.test.ts tests/oauth-manual-code.test.ts \ - tests/codex-auth-api.test.ts tests/mimo-free-provider.test.ts tests/cursor-mcp-manager.test.ts + tests/api-debug.test.ts tests/claude-inbound-debug.test.ts \ + tests/codex-websocket-registry.test.ts tests/storage-worker-lifecycle.test.ts \ + tests/storage-mutation-race.test.ts tests/codex-account-store.test.ts \ + tests/codex-auth-context.test.ts tests/token-guardian.test.ts +bun test tests/usage-log.test.ts tests/api-usage.test.ts tests/api-key-attribution.test.ts \ + gui/tests/usage-layout.test.ts tests/cursor-hardening.test.ts \ + tests/gather-routed-models-single-flight.test.ts tests/cli-catalog-prewarm.test.ts \ + tests/system-env.test.ts tests/model-visibility-management-api.test.ts +bun test tests/oauth-refresh.test.ts tests/oauth-manual-code.test.ts \ + tests/codex-auth-api.test.ts tests/mimo-free-provider.test.ts \ + tests/cursor-mcp-manager.test.ts tests/config.test.ts bun run typecheck +bun run lint:gui +bun run build:gui bun run test bun run privacy:scan ``` File ownership is fixed as follows: debug subscriber/value cases extend `tests/debug.test.ts` and `tests/api-debug.test.ts` (the latter owns the `/api/debug` -route behavior); turns/sockets/slots use the new cross-registry file plus the existing -websocket suite; Codex refresh flights extend `tests/codex-account-store.test.ts` -(NOT `tests/xai-refresh-lock.test.ts`, which owns OAuth/XAI refresh); -usage extends `tests/usage-log.test.ts`; Cursor discovery and gather admission extend -`tests/cursor-hardening.test.ts` and `tests/gather-routed-models-single-flight.test.ts`; -OAuth owner/code and Codex flow/probe cases extend `tests/oauth-manual-code.test.ts` and -`tests/codex-auth-api.test.ts`; MiMo extends `tests/mimo-free-provider.test.ts`; MCP -extends `tests/cursor-mcp-manager.test.ts`. Do not invent parallel test harness names. +route behavior); Claude row/accounting cases extend `tests/claude-inbound-debug.test.ts`. +Turns use the new cross-registry file; sockets, worker reservations, and mutation lease +retention extend `tests/codex-websocket-registry.test.ts`, +`tests/storage-worker-lifecycle.test.ts`, and `tests/storage-mutation-race.test.ts`. +Codex refresh-flight ownership extends `tests/codex-account-store.test.ts` (NOT +`tests/xai-refresh-lock.test.ts`, which owns OAuth/XAI refresh), while consumer retry +classification extends `tests/codex-auth-context.test.ts`, `tests/token-guardian.test.ts`, +and `tests/codex-auth-api.test.ts`. + +Usage reader, route, API-key rollup, and rendered GUI qualification extend +`tests/usage-log.test.ts`, `tests/api-usage.test.ts`, +`tests/api-key-attribution.test.ts`, and `gui/tests/usage-layout.test.ts`, respectively. +Cursor discovery and gather admission/caller mapping extend +`tests/cursor-hardening.test.ts`, `tests/gather-routed-models-single-flight.test.ts`, +`tests/cli-catalog-prewarm.test.ts`, `tests/system-env.test.ts`, and +`tests/model-visibility-management-api.test.ts`. OAuth token refresh and pending code +extend `tests/oauth-refresh.test.ts` and `tests/oauth-manual-code.test.ts`; Codex login/ +quota owners extend `tests/codex-auth-api.test.ts`. MiMo extends +`tests/mimo-free-provider.test.ts`. MCP bounding is proven at the actual enforcement +layer in `tests/cursor-mcp-manager.test.ts`, including SDK-already-materialized fixtures +and native base64 decode; do not claim transport-level proof. Config defaults/validation +extend `tests/config.test.ts`. Do not invent parallel test harness names. ## Commit @@ -417,10 +671,14 @@ extends `tests/cursor-mcp-manager.test.ts`. Do not invent parallel test harness ## Explicitly not changed -- No forced eviction/untracking of accepted turns, sockets, workers, mutation slots, - refresh grants, OAuth flows, or probes. +- No forced eviction/untracking of accepted ACTIVE turns, sockets, workers, mutation + slots, refresh grants, pending OAuth flows, or probes. Expired/old terminal Codex + login-status rows may be removed because their work has already settled. - No `#820` scheduler/session-lane architecture. - No credential, token, account id, URL, path, command, or body in metrics. -- No truncation of JSON tool schemas/results into syntactically valid-looking partials. +- No truncation of MCP schemas/results/resources or flattened/restored translator data + into syntactically valid-looking partials. +- No MCP SDK fork or transport/framing limit claim; this phase bounds only what can be + enforced immediately after SDK receipt. - No change to provider retry/rotation, MCP tool execution semantics, or usage-log disk format. - No process-wide demotion; 040 consumes only the retained-ring accounting hooks. diff --git a/gui/src/components/apikeys-workspace/ApiKeysWorkspace.tsx b/gui/src/components/apikeys-workspace/ApiKeysWorkspace.tsx index 82ff6b22e..90985a57b 100644 --- a/gui/src/components/apikeys-workspace/ApiKeysWorkspace.tsx +++ b/gui/src/components/apikeys-workspace/ApiKeysWorkspace.tsx @@ -26,6 +26,7 @@ export interface ApiKeysWorkspaceProps { /** Dataset-level. Absent means nothing is attributable yet — a different * statement from a key whose counters read zero. */ attributionSince?: string; + historyTruncated?: boolean; authMatrix: ApiAuthMatrixRow[]; keysLoading: boolean; keysLoadFailed: boolean; @@ -62,6 +63,7 @@ export interface ApiKeysWorkspaceProps { export default function ApiKeysWorkspace({ keys, attributionSince, + historyTruncated, authMatrix, keysLoading, keysLoadFailed, @@ -371,7 +373,7 @@ export default function ApiKeysWorkspace({
{selected.usage.requests7d.toLocaleString(localeTag)}
-
{t("api.attribution.totalRequests")}
+
{historyTruncated ? t("api.attribution.totalRequestsAvailable") : t("api.attribution.totalRequests")}
{selected.usage.totalRequests.toLocaleString(localeTag)}
@@ -381,7 +383,7 @@ export default function ApiKeysWorkspace({ : t("api.attribution.neverUsed")}
-
{t("api.attribution.since")}
+
{historyTruncated ? t("api.attribution.sinceAvailable") : t("api.attribution.since")}
{formatCreatedDate(attributionSince, localeTag)}
diff --git a/gui/src/i18n/de.ts b/gui/src/i18n/de.ts index 57c907d52..dcb621f57 100644 --- a/gui/src/i18n/de.ts +++ b/gui/src/i18n/de.ts @@ -609,6 +609,8 @@ export const de: Record = { "usage.empty": "Noch keine Nutzung erfasst. Sende eine Anfrage über den Proxy, um Aktivität hier zu sehen.", "usage.loadError": "Nutzungsdaten konnten nicht geladen werden.", "usage.range.all": "Alle", + "usage.range.available": "Verfügbarer Verlauf", + "usage.historyTruncated": "Die Summen beziehen sich nur auf den verfügbaren Verlauf, da ältere Nutzungsdaten nicht geladen wurden.", "usage.range.30d": "30d", "usage.range.7d": "7d", "usage.card.requests": "Anfragen", @@ -904,6 +906,8 @@ export const de: Record = { "api.attribution.title": "Zugeordnete Nutzung", "api.attribution.requests7d": "Anfragen, letzte 7 Tage", "api.attribution.totalRequests": "Zugeordnete Anfragen gesamt", + "api.attribution.totalRequestsAvailable": "Anfragen im verfügbaren Verlauf", + "api.attribution.sinceAvailable": "Verfügbare Zuordnung seit", "api.attribution.lastUsed": "Zuletzt verwendet", "api.attribution.since": "Zuordnung verfügbar seit", "api.attribution.neverUsed": "Seit Beginn der Zuordnung nicht verwendet", diff --git a/gui/src/i18n/en.ts b/gui/src/i18n/en.ts index a81d812a3..e130743dc 100644 --- a/gui/src/i18n/en.ts +++ b/gui/src/i18n/en.ts @@ -638,6 +638,8 @@ export const en = { "usage.empty": "No usage recorded yet. Send a request through the proxy to see activity here.", "usage.loadError": "Could not load usage data.", "usage.range.all": "All", + "usage.range.available": "Available history", + "usage.historyTruncated": "Totals cover available history only because older usage was not loaded.", "usage.range.30d": "30d", "usage.range.7d": "7d", "usage.card.requests": "Requests", @@ -1352,6 +1354,8 @@ export const en = { "api.attribution.title": "Attributed usage", "api.attribution.requests7d": "Requests, last 7 days", "api.attribution.totalRequests": "Total attributed requests", + "api.attribution.totalRequestsAvailable": "Requests in available history", + "api.attribution.sinceAvailable": "Available attribution since", "api.attribution.lastUsed": "Last used", "api.attribution.since": "Attribution available since", "api.attribution.neverUsed": "Not used since attribution began", diff --git a/gui/src/i18n/ja.ts b/gui/src/i18n/ja.ts index 1dcb49e40..ae8109ed7 100644 --- a/gui/src/i18n/ja.ts +++ b/gui/src/i18n/ja.ts @@ -598,6 +598,8 @@ export const ja: Record = { "usage.empty": "まだ使用量が記録されていません。プロキシ経由でリクエストを送信するとここにアクティビティが表示されます。", "usage.loadError": "使用量データを読み込めませんでした。", "usage.range.all": "すべて", + "usage.range.available": "利用可能な履歴", + "usage.historyTruncated": "古い利用履歴が読み込まれていないため、合計は利用可能な履歴のみを対象とします。", "usage.range.30d": "30日", "usage.range.7d": "7日", "usage.card.requests": "リクエスト", @@ -1300,6 +1302,8 @@ export const ja: Record = { "api.attribution.title": "キー別の使用状況", "api.attribution.requests7d": "直近 7 日のリクエスト", "api.attribution.totalRequests": "集計済みリクエスト総数", + "api.attribution.totalRequestsAvailable": "利用可能な履歴のリクエスト", + "api.attribution.sinceAvailable": "利用可能な集計開始日", "api.attribution.lastUsed": "最終使用", "api.attribution.since": "集計開始", "api.attribution.neverUsed": "集計開始以降は未使用", diff --git a/gui/src/i18n/ko.ts b/gui/src/i18n/ko.ts index 45be93e66..d5f16aae2 100644 --- a/gui/src/i18n/ko.ts +++ b/gui/src/i18n/ko.ts @@ -631,6 +631,8 @@ export const ko: Record = { "usage.empty": "아직 기록된 사용량이 없습니다. 프록시로 요청을 보내면 여기에 표시됩니다.", "usage.loadError": "사용량 데이터를 불러오지 못했습니다.", "usage.range.all": "전체", + "usage.range.available": "사용 가능한 기록", + "usage.historyTruncated": "이전 사용 기록을 불러오지 않아 합계는 사용 가능한 기록만 포함합니다.", "usage.range.30d": "30일", "usage.range.7d": "7일", "usage.card.requests": "요청", @@ -931,6 +933,8 @@ export const ko: Record = { "api.attribution.title": "키별 사용량", "api.attribution.requests7d": "최근 7일 요청", "api.attribution.totalRequests": "집계된 전체 요청", + "api.attribution.totalRequestsAvailable": "사용 가능한 기록의 요청", + "api.attribution.sinceAvailable": "사용 가능한 집계 시작일", "api.attribution.lastUsed": "마지막 사용", "api.attribution.since": "집계 시작", "api.attribution.neverUsed": "집계 이후 사용 없음", diff --git a/gui/src/i18n/ru.ts b/gui/src/i18n/ru.ts index ac0c7f37f..a0bf4eb01 100644 --- a/gui/src/i18n/ru.ts +++ b/gui/src/i18n/ru.ts @@ -630,6 +630,8 @@ export const ru: Record = { "usage.empty": "Данных об использовании пока нет. Отправьте запрос через прокси, чтобы увидеть здесь активность.", "usage.loadError": "Не удалось загрузить данные об использовании.", "usage.range.all": "Все", + "usage.range.available": "Доступная история", + "usage.historyTruncated": "Итоги охватывают только доступную историю, поскольку старые данные не загружены.", "usage.range.30d": "30 дн.", "usage.range.7d": "7 дн.", "usage.card.requests": "Запросы", @@ -1342,6 +1344,8 @@ export const ru: Record = { "api.attribution.title": "Использование по ключам", "api.attribution.requests7d": "Запросы за 7 дней", "api.attribution.totalRequests": "Всего учтённых запросов", + "api.attribution.totalRequestsAvailable": "Запросы в доступной истории", + "api.attribution.sinceAvailable": "Доступная атрибуция с", "api.attribution.lastUsed": "Последнее использование", "api.attribution.since": "Учёт ведётся с", "api.attribution.neverUsed": "Не использовался с начала учёта", diff --git a/gui/src/i18n/zh.ts b/gui/src/i18n/zh.ts index 289fa2572..03e48ddc4 100644 --- a/gui/src/i18n/zh.ts +++ b/gui/src/i18n/zh.ts @@ -624,6 +624,8 @@ export const zh: Record = { "usage.empty": "尚无用量记录。通过代理发送请求后将在此显示。", "usage.loadError": "无法加载用量数据。", "usage.range.all": "全部", + "usage.range.available": "可用历史", + "usage.historyTruncated": "由于未加载较早的使用记录，合计仅涵盖可用历史。", "usage.range.30d": "30 天", "usage.range.7d": "7 天", "usage.card.requests": "请求数", @@ -924,6 +926,8 @@ export const zh: Record = { "api.attribution.title": "按密钥统计的用量", "api.attribution.requests7d": "最近 7 天请求数", "api.attribution.totalRequests": "已归属请求总数", + "api.attribution.totalRequestsAvailable": "可用历史中的请求", + "api.attribution.sinceAvailable": "可用归属记录起始时间", "api.attribution.lastUsed": "最近使用", "api.attribution.since": "统计起始", "api.attribution.neverUsed": "统计开始后未使用", diff --git a/gui/src/pages/ApiKeys.tsx b/gui/src/pages/ApiKeys.tsx index 33ee543a0..9bf316697 100644 --- a/gui/src/pages/ApiKeys.tsx +++ b/gui/src/pages/ApiKeys.tsx @@ -32,6 +32,7 @@ interface KeysResponse { // could not read. keys?: Array & { usage?: ApiKeyEntry["usage"] }>; attributionSince?: string; + historyTruncated?: boolean; authMatrix?: unknown; endpoint?: string; baseUrl?: string; @@ -53,6 +54,7 @@ type CachedKeysShape = { /** Dataset-level: absent means nothing is attributable yet, which is a * different statement from a key whose counters are zero. */ attributionSince?: string; + historyTruncated?: boolean; authMatrix: ApiAuthMatrixRow[]; }; @@ -129,6 +131,7 @@ export default function ApiKeys({ apiBase }: { apiBase: string }) { }, claudeCodeEnabled: data.claudeCodeEnabled !== false, ...(data.attributionSince ? { attributionSince: data.attributionSince } : {}), + ...(data.historyTruncated === true ? { historyTruncated: true } : {}), authMatrix: data.authMatrix, }; // Prefixes only — never the secret key material. @@ -180,6 +183,7 @@ export default function ApiKeys({ apiBase }: { apiBase: string }) { const endpoints = keysData?.endpoints ?? seedEndpointsFromApiBase(apiBase); const claudeCodeEnabled = keysData?.claudeCodeEnabled ?? true; const attributionSince = keysData?.attributionSince; + const historyTruncated = keysData?.historyTruncated === true; // `?? []` only ever fires while there is no key data at all — both the network // and cache paths reject a payload without a valid matrix, so an empty table // can never be presented as the server's answer. @@ -412,6 +416,7 @@ export default function ApiKeys({ apiBase }: { apiBase: string }) {
{(["all", "30d", "7d"] as Range[]).map(choice => { - const label = t(`usage.range.${choice}`); + const label = choice === "all" ? t("usage.range.available") : t(`usage.range.${choice}`); return (