From ca5e7993bd8f03f27901b035e8d8fa1238c94c90 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 06:06:00 +0000 Subject: [PATCH] =?UTF-8?q?test(runtime):=20add=20the=20#3259=20flake=20ca?= =?UTF-8?q?nary=20=E2=80=94=20nested=20writes=20stay=20green=20at=20the=20?= =?UTF-8?q?stock=20250ms=20CPU=20budget=20under=20load?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The verification "flake canary" from the ADR-0102 plan (#3275). It drives the real nested-write scenario (child afterInsert -> nested parent.update -> the parent's own hook = a second re-entrant VM) across 128 chains CONCURRENTLY through one shared stock runner, and asserts every invocation lands the correct rollup with zero CPU-budget / wall-ceiling errors at the stock 250ms budget. Closes the loop on ADR-0102 D1. #3259 was a LOAD flake: under the old wall-clock budget, the time an invocation spent parked at an `await` while other concurrent invocations ran was charged to it, so it blew 250ms under oversubscription even though its own VM did microseconds of work. D1 charges script CPU-time only, so parked time is not counted. Validated as a real regression gate, not a no-op: temporarily reverting the interrupt handler to a `Date.now() - start` wall budget turns this suite RED from ~64 concurrent chains up, while the shipped CPU budget stays green through 256. It is robust (not itself flaky) because concurrency stresses the parked-time accounting D1 excludes, not slice size — the charged CPU per tiny rollup body is invariant to how many chains are in flight. `OS_CANARY_CONCURRENCY` overrides the default for an ad-hoc heavier soak. Test-only; no changeset. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_011iJLjqToxNv1aYP3syQtRp --- .../src/sandbox/nested-write.canary.test.ts | 179 ++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 packages/runtime/src/sandbox/nested-write.canary.test.ts diff --git a/packages/runtime/src/sandbox/nested-write.canary.test.ts b/packages/runtime/src/sandbox/nested-write.canary.test.ts new file mode 100644 index 0000000000..a3ba278d4a --- /dev/null +++ b/packages/runtime/src/sandbox/nested-write.canary.test.ts @@ -0,0 +1,179 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * Flake canary for #3259 (ADR-0102 D1 — the CPU-time budget). + * + * #3259 was a LOAD flake, not a logic bug: on an oversubscribed CI runner the + * fixed per-invocation WASM-module creation cost alone tripped the 250ms hook + * deadline while the VM was still progressing — because the budget was + * WALL-CLOCK. When many hook invocations run concurrently on one JS thread, any + * single invocation spends most of its wall time PARKED at an `await` (host call + * / `setImmediate` yield) while the OTHER invocations run; the old budget charged + * that parked time to the parked invocation, so it blew 250ms under load even + * though its own VM did microseconds of work. D1 made the budget script + * CPU-time: only VM-active slices are charged, parked time is not, so the stock + * 250ms budget is meaningful again under load. + * + * This canary reproduces that exact condition and asserts the fix holds: it runs + * the real nested-write scenario (child `afterInsert` → nested `parent.update` + * → the parent's OWN hook = a second re-entrant VM) across many chains + * CONCURRENTLY through one shared stock runner. Every invocation runs at the + * STOCK 250ms budget and must land the correct rollup with NOT ONE CPU-budget / + * wall-ceiling error. Revert D1 to a wall-clock budget and the parked-while- + * others-run time reappears on the charge → this goes red. That is the + * regression it guards. + * + * Why the canary is itself robust (not flaky): the stressor is CONCURRENCY, not + * unrelated CPU busy-loops. JS runs each synchronous VM slice to completion, so + * concurrent chains interleave only BETWEEN slices (at `await` points), never + * mid-slice — the `Date.now()`-measured slices stay accurate and no legitimate + * tiny rollup body ever approaches 250ms of charged VM-active time regardless of + * how many chains are in flight. + */ + +import { describe, it, expect } from 'vitest'; +import { ObjectQL, bindHooksToEngine } from '@objectstack/objectql'; +import { hookBodyRunnerFactory } from './body-runner.js'; +import { QuickJSScriptRunner } from './quickjs-runner.js'; + +const parent = { + name: 'parent', + label: 'Parent', + fields: { + id: { name: 'id', type: 'text' as const, primaryKey: true }, + total_amount: { name: 'total_amount', type: 'number' as const }, + child_count: { name: 'child_count', type: 'number' as const }, + }, +}; +const child = { + name: 'child', + label: 'Child', + fields: { + id: { name: 'id', type: 'text' as const, primaryKey: true }, + amount: { name: 'amount', type: 'number' as const }, + parent_id: { name: 'parent_id', type: 'text' as const }, + }, +}; + +function makeMemoryDriver() { + const stores = new Map>>(); + const storeFor = (o: string) => { + let s = stores.get(o); + if (!s) { s = new Map(); stores.set(o, s); } + return s; + }; + let nextId = 0; + const matches = (row: Record, where: any): boolean => { + if (!where || typeof where !== 'object') return true; + for (const [k, v] of Object.entries(where)) { + if (k.startsWith('$')) continue; + const exp = (v && typeof v === 'object' && '$eq' in (v as any)) ? (v as any).$eq : v; + if ((row[k] ?? null) !== (exp ?? null)) return false; + } + return true; + }; + const driver: any = { + name: 'memory', version: '0.0.0', supports: {}, + async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, + async find(o: string, ast: any) { return Array.from(storeFor(o).values()).filter((r) => matches(r, ast?.where)); }, + findStream() { throw new Error('ns'); }, + async findOne(o: string, ast: any) { for (const r of storeFor(o).values()) if (matches(r, ast?.where)) return r; return null; }, + async create(o: string, data: Record) { + nextId += 1; const id = (data.id as string) ?? `r_${nextId}`; const row = { ...data, id }; storeFor(o).set(id, row); return row; + }, + async update(o: string, id: string, data: Record) { + const s = storeFor(o); const cur = s.get(id); if (!cur) throw new Error(`nf ${o}/${id}`); + const up = { ...cur, ...data, id }; s.set(id, up); return up; + }, + async upsert(o: string, data: Record) { const id = data.id as string | undefined; return id && storeFor(o).has(id) ? this.update(o, id, data) : this.create(o, data); }, + async delete(o: string, id: string) { return storeFor(o).delete(id); }, + async count(o: string, ast: any) { return (await this.find(o, ast)).length; }, + async bulkCreate(o: string, rows: Record[]) { return Promise.all(rows.map((r) => this.create(o, r))); }, + async bulkUpdate() { return []; }, async bulkDelete() {}, + async beginTransaction() { return { commit: async () => {}, rollback: async () => {} }; }, async commit() {}, async rollback() {}, + }; + return { driver, stores }; +} + +// The rollup hook the issue says authors could not write, plus a trivial parent +// hook whose mere presence forces a SECOND, re-entrant sandbox VM while the +// child's hook VM is still in flight. +const hooks = [ + { + name: 'rollup_parent_total', + object: 'child', + events: ['afterInsert', 'afterUpdate'], + body: { + language: 'js', + source: ` + const pid = ctx.input.parent_id || (ctx.previous && ctx.previous.parent_id); + const rows = await ctx.api.object('child').find({ where: { parent_id: pid } }); + const total = rows.reduce((s, r) => s + (r.amount || 0), 0); + await ctx.api.object('parent').update({ id: pid, total_amount: total, child_count: rows.length }); + `, + capabilities: ['api.read', 'api.write'], + }, + } as any, + { + name: 'parent_touch_marker', + object: 'parent', + events: ['afterUpdate'], + body: { language: 'js', source: "return { seen: ctx.input.id };", capabilities: [] }, + } as any, +]; + +describe('#3259 flake canary — nested writes stay green at the stock 250ms CPU budget under concurrency', () => { + // One shared stock runner (250ms CPU budget), exactly as production wires it — + // many concurrent invocations, each still getting its own isolated WASM module. + const runner = new QuickJSScriptRunner(); + + async function rollupChain(n: number) { + const engine = new ObjectQL(); + const { driver } = makeMemoryDriver(); + engine.registerDriver(driver, true); + await engine.init(); + for (const o of [parent, child]) engine.registry.registerObject(o as any); + engine.setDefaultBodyRunner(hookBodyRunnerFactory(runner, { ql: engine, appId: 'test' })); + bindHooksToEngine(engine, hooks, { packageId: 'test' }); + + const pid = `p_${n}`; + await engine.insert('parent', { id: pid, total_amount: 0, child_count: 0 }); + await engine.insert('child', { id: `c_${n}_1`, amount: 100, parent_id: pid }); + await engine.insert('child', { id: `c_${n}_2`, amount: 50, parent_id: pid }); + return (await engine.findOne('parent', { where: { id: pid } })) as any; + } + + it('runs many concurrent nested-write rollups with no CPU-budget / wall-ceiling errors', async () => { + // Enough chains in flight that, under the OLD wall-clock budget, each one's + // parked-while-others-run time blows past 250ms; under the CPU budget it does + // not, because that parked time is not charged. Validated during development: + // temporarily reverting the interrupt handler to a `Date.now() - start` wall + // budget turns this suite RED from ~64 chains up, while the shipped CPU budget + // stays green through 256 — so this is a real regression gate, not a no-op. + // `OS_CANARY_CONCURRENCY` overrides it for an ad-hoc heavier soak. + const CONCURRENCY = Number(process.env.OS_CANARY_CONCURRENCY) || 128; + + const results = await Promise.allSettled( + Array.from({ length: CONCURRENCY }, (_, n) => rollupChain(n)), + ); + + const reasons = results + .filter((r): r is PromiseRejectedResult => r.status === 'rejected') + .map((r) => String(r.reason?.message ?? r.reason)); + + // The signature of a #3259 regression: a load-induced budget/ceiling trip. + // Assert it separately so a failure names the actual cause, not a generic count. + const budgetTrips = reasons.filter((m) => /CPU budget|wall-clock ceiling/.test(m)); + expect(budgetTrips, 'no load-induced CPU-budget / wall-ceiling trips at the stock 250ms budget').toEqual([]); + + // No failures of any kind, and every rollup landed the correct denormalized total. + expect(reasons, 'all concurrent nested-write chains succeed').toEqual([]); + for (const r of results) { + expect(r.status).toBe('fulfilled'); + if (r.status === 'fulfilled') { + expect(r.value.total_amount).toBe(150); + expect(r.value.child_count).toBe(2); + } + } + }, 120000); +});