Benchmark validity: close the 3 audit-flagged holes (sense-leak + 2 factsCore gaming surfaces)

adapters/providers/ClaudeAdapter.ts

@@ -20,7 +20,8 @@ export class ClaudeAdapter implements TranslationProvider, Provider {
       request.content,
       request.settings,
       request.history,
-      request.fanTranslation
+      request.fanTranslation,
+      request.abortSignal
     );
   }
 
@@ -69,7 +70,9 @@ export class ClaudeAdapter implements TranslationProvider, Provider {
     const startTime = performance.now();
     try {
       const claude = new Anthropic({ apiKey, dangerouslyAllowBrowser: true });
-      const response = await claude.messages.create(requestPayload);
+      // Thread the signal into the SDK so an abort cancels the in-flight request, not just the
+      // post-return check below (review #3).
+      const response = await claude.messages.create(requestPayload, { signal: input.abortSignal });
 
       if (input.abortSignal?.aborted) {
         throw new DOMException('Aborted', 'AbortError');

adapters/providers/GeminiAdapter.ts

@@ -90,7 +90,11 @@ export class GeminiAdapter implements TranslationProvider, Provider {
     try {
       const schema = getTranslationOnlyResponseGeminiSchema();
 
-      // Make API call
+      // Make API call. Pass the signal in SingleRequestOptions so a Translator timeout actually
+      // CANCELS the in-flight request (review #3) — the SDK (@google/generative-ai ≥0.24) supports
+      // it natively; the previous "Gemini doesn't support native abort" note was outdated, so the
+      // signal was only checked AFTER the call returned, by which point the original was still
+      // running and billing while the retry fired.
       result = await model.generateContent({
         contents: [{ role: 'user', parts: [{ text: fullPrompt }] }],
         generationConfig: {
@@ -99,9 +103,9 @@ export class GeminiAdapter implements TranslationProvider, Provider {
           responseMimeType: 'application/json',
           responseSchema: schema
         },
-      });
+      }, { signal: abortSignal });
 
-      // Handle abort signal manually (Gemini doesn't support native abort)
+      // Belt-and-braces: if the signal fired without the SDK surfacing an AbortError, bail here.
       if (abortSignal?.aborted) {
         throw new DOMException('Aborted', 'AbortError');
       }
@@ -166,7 +170,7 @@ export class GeminiAdapter implements TranslationProvider, Provider {
       result = await model.generateContent({
         contents: [{ role: 'user', parts: [{ text: prompt }] }],
         generationConfig,
-      });
+      }, { signal: input.abortSignal });
 
       if (input.abortSignal?.aborted) {
         throw new DOMException('Aborted', 'AbortError');

adapters/providers/OpenAIAdapter.ts

@@ -626,11 +626,6 @@ ${schemaString}`;
     const finishReason = choice?.finish_reason || (choice as any)?.native_finish_reason || null;
 
     const responseText = choice?.message?.content;
-    if (!responseText) {
-      throw new Error('Empty response from API');
-    }
-
-    dlogFull('Raw response text:', responseText.substring(0, 500));
 
     // Cost and timing are computed up-front, from response.usage. The provider bills for this
     // call whether or not we can parse it, so every exit below records a metric — a
@@ -671,6 +666,17 @@ ${schemaString}`;
       throw new Error(message);
     };
 
+    // An empty response was still billed (usage tokens are present), so record the spend before
+    // failing — a thrown-before-recording empty response was invisible to the budget ledger
+    // (TECH-DEBT P1.4). Explicit record-then-throw (not failWith) so TS narrows responseText to
+    // a string for the code below.
+    if (!responseText) {
+      await recordMetric(false);
+      throw new Error('Empty response from API');
+    }
+
+    dlogFull('Raw response text:', responseText.substring(0, 500));
+
     // Strip fences BEFORE the truncation check: a ```json-wrapped response does not end with
     // `}`, and reading that as truncation would throw length_cap and trigger a chunked re-bill.
     const cleanedText = this.stripMarkdownCodeFences(responseText);

config/suttaStudioExamples.ts

@@ -426,25 +426,31 @@ export const SUTTA_STUDIO_LEXICO_COMPOUND_EXAMPLE_JSON = JSON.stringify(SUTTA_ST
 // When selecting different senses, nearby ghost words may need adjustment.
 // Use ripples to override ghost text when sense choice affects grammar.
 //
-// Example: "viharati" with ghost "was" before it
-// - If sense = "dwells" (present) → remove "was" ghost
-// - If sense = "was staying" (past continuous) → keep "was"
-// - If sense = "stayed" (simple past) → remove "was" ghost
+// Example: "gacchati" with ghost "was" before it
+// - If sense = "goes" (present) → remove "was" ghost
+// - If sense = "was going" (past continuous) → keep "was"
+// - If sense = "went" (simple past) → remove "was" ghost
+//
+// NOTE: this worked example is shown in the lexicographer PROMPT, so its words must NOT
+// be any word graded in a ranked benchmark phase — otherwise the taught gloss leaks the
+// answer on the sense axis (senseF1). The previous example used "viharati → dwells", which
+// is graded in 8 ranked phases; it was reglossed to "gacchati → goes" (absent from every
+// ranked phase). The lexico-example-leak test enforces this contract.
 //
 // The ripple key is the English token ID (e.g., "e10" for the ghost "was")
 export const SUTTA_STUDIO_LEXICO_RIPPLE_EXAMPLE: LexicographerPass = {
   id: 'phase-ripple',
   senses: [
     {
-      wordId: 'p5',  // viharati
+      wordId: 'p5',  // gacchati
       wordClass: 'content',
       senses: [
         // Present tense - remove "was" ghost (e10)
-        { english: 'dwells', nuance: 'habitual residence', ripples: { e10: '' } },
+        { english: 'goes', nuance: 'habitual motion', ripples: { e10: '' } },
         // Past continuous - keep "was" as-is (no ripple needed, or explicit)
-        { english: 'staying', nuance: 'temporary residence', ripples: { e10: 'was' } },
+        { english: 'going', nuance: 'motion in progress', ripples: { e10: 'was' } },
         // Simple past - remove "was" ghost
-        { english: 'stayed', nuance: 'completed action', ripples: { e10: '' } },
+        { english: 'went', nuance: 'completed action', ripples: { e10: '' } },
       ],
     },
     {

docs/adr/SUTTA-013-facts-vs-prose-scoping.md

@@ -154,3 +154,29 @@ so no correctness reference exists.
 ## Explicitly out of scope here
 The pedagogical probe (a student model answering teacher questions from the compiled output
 alone) stays the north-star experiment; it needs its own design round.
+
+## Amendment — 2026-07-16: v2.2 formula implemented (final-score weights)
+
+The reviewer flagged (#3) that "40/30/30" was underspecified — the *fidelity*-internal weights
+nested inside the v2.1 `0.60·fidelity + 0.25·usability + 0.15·richness`, or the final score?
+**Operator decision: they are the FINAL-score weights.** So the ranked score is now:
+
+```
+overall = gateFactor × (0.40·segmentationF1 + 0.30·factsCore + 0.30·senseF1)
+```
+
+Implemented in `quality-scorer.ts` (`RUBRIC_VERSION` → `2.2`; leaderboard `RANKED_RUBRIC_VERSION`
+→ `2.2`):
+- `segmentationF1` — morpheme-boundary micro-F1 vs golden (unchanged).
+- `factsCore` — root + word-class macro from `scoreFactsDetail`. Morphology is EXCLUDED (advisory)
+  this cycle, per the recommendation; its gaming + vocabulary were fixed separately (review #4), so
+  it can be promoted later.
+- `senseF1` — strict sense-english micro-F1 (`scoreSenseFidelityDetail`, SUTTA-012 drop-penalised).
+- **Usability and richness are RETIRED from rank** (still computed for display). Alignment, the LLM
+  judge and the reader probe stay advisory.
+- Missing components (e.g. a function-only phase with no golden senses) renormalise over the present
+  weights rather than being charged 0.
+
+This is a rubric bump: v2.1 and v2.2 scores are not comparable (the version gate enforces it), so it
+lands BEFORE the next multi-model pass and needs a full fleet re-run. Board columns for factsCore /
+senseF1 and the fleet re-run are the remaining follow-ups.

docs/roadmaps/GOLDEN-CONTRACT-REPAIR.md

@@ -0,0 +1,103 @@
+# Golden/Prompt Contract Repair (review finding #1)
+
+**Status:** DONE. Part 1 (16 mechanical `wordRange` fixes) and Part 2 (8 sandhi/omission phases,
+reconciled under the operator-approved one-word-per-whitespace-token policy) are both complete —
+**30/30 ranked phases satisfy the contract.**
+**Gate:** `tests/scripts/sutta-studio/golden-prompt-contract.test.ts` (pending list now empty).
+
+## The contract
+
+A phase is scored by aligning the model's analysis to a **golden** and grading the matches. For
+that to be fair, the golden must cover the **same Pāli word sequence the prompt shows the model**.
+The prompt sequence is derived by the runner as: the phase's canonical segments' `pali`, joined and
+whitespace-split, then sliced by the phase's optional `wordRange`
+(`benchmark.ts` → `getSegmentsForPhase` / `applyWordRangeToSegments`).
+
+When the golden covers only a **slice** of the prompt but the phase has **no `wordRange`**, the
+model is prompted with the whole segment yet graded on a fraction of it — so it can report 100% Pāli
+coverage while ignoring most of what it was given. This is the shared-segment problem ADR SUTTA-003
+opened and left unfinished beyond 7 phases.
+
+## Audit (30-phase ranked set, mn10)
+
+| State | MATCH | of 30 |
+|---|---|---|
+| Before repair | **6** | 6/30 — confirms the review's mechanical count |
+| After Part 1 (`wordRange`) | **22** | +16 |
+| After Part 2 (sandhi/omission) | **30** | +8 → **all pass** |
+
+Categories of the 24 failures:
+- **16 SLICE** — golden is a correct contiguous sub-span of the prompt; the phase just lacked a
+  `wordRange`. **Fixed mechanically** (Part 1) — no golden content changed.
+- **8 SANDHI / OMISSION** — the golden tokenizes differently from the prompt (splits a joined token)
+  or omits a word. Needs golden content edits → **Part 2, operator decision**.
+
+## Part 1 — the 16 `wordRange` fixes (DONE)
+
+Added to `test-fixtures/sutta-studio-anatomist-golden.json` `_phases[].wordRange`, slicing each
+phase's prompt to exactly its existing golden span (the golden words were already correct):
+
+| phase | wordRange | phase | wordRange | phase | wordRange | phase | wordRange |
+|---|---|---|---|---|---|---|---|
+| phase-d | [5,9) | phase-af | [0,3) | phase-ao | [9,15) | phase-bc | [4,8) |
+| phase-z | [3,6) | phase-ag | [3,6) | phase-ap | [15,18) | phase-bd | [8,12) |
+| phase-ab | [9,12) | phase-ai | [0,3) | phase-az | [6,11) | phase-bf | [7,10) |
+| phase-ad | [3,6) | phase-aj | [3,6) | phase-ba | [11,16) | phase-bg | [10,18) |
+
+Reproduce/verify: the contract test above passes for these 22 and goes RED without the `wordRange`
+additions (all 16 listed as violations).
+
+## Part 2 — the 8 sandhi/omission phases (DONE, policy: one word per whitespace token)
+
+These goldens split a joined Pāli token the prompt presents as ONE whitespace token, or omitted a
+word. Resolved under the operator-approved policy — **the golden now tokenizes exactly as the
+Anatomist is instructed to (one word per space-separated token), with sandhi living in morpheme
+SEGMENTS, not word boundaries.** Concretely: each split pair was merged into one word whose segments
+(unchanged) still reconstruct the surface (verified), relations were retargeted to the merged word
+and intra-word relations dropped, and the two omitted function words were added. All 8 now match.
+
+### 2a. Quotative / sandhi splits (6 phases)
+
+The golden splits a sandhi compound or the `ti`/`'ti` quotative into two words; the Anatomist's own
+rule is **one word per space-separated token** (see the Anatomist prompt's CRITICAL WORD BOUNDARY
+RULE). So the golden diverges from the tokenization the model is instructed to use.
+
+| phase | prompt token(s) | golden words | joined form |
+|---|---|---|---|
+| phase-f | `"bhikkhavo"ti.` | `Bhikkhavo` + `ti` | bhikkhavoti |
+| phase-h | `Bhagavā` `etadavoca:` | `Bhagavā` `etad` + `avoca` | etadavoca |
+| phase-as | …`assasāmī'ti`… | `assasāmī` + `'ti` (×2) | assasāmīti |
+| phase-at | …`assasissāmī'ti`… | `assasissāmī` + `'ti` (×2) | assasissāmīti |
+| phase-av | …`añchāmī'ti`… | `añchāmī` + `'ti` (×2) | añchāmīti |
+| phase-ax | …`assasissāmī'ti`… | `assasissāmī` + `'ti` (×2) | assasissāmīti |
+
+**Recommended resolution:** make the golden follow the Anatomist word-boundary rule — merge each
+split back into ONE word (`etadavoca`, `assasāmīti`, …) and move the split to **morpheme segments**
+inside that word (`etad` + `avoca`; `assasāmī` + `ti`). The morphological analysis is preserved; only
+the WORD boundary moves to match the prompt. Then add a `wordRange` to slice the prompt.
+
+**Alternative (not recommended):** split the sandhi in the canonical source text so the prompt shows
+two tokens. Rejected — the canonical SuttaCentral segmentation is the authority and should not be
+re-tokenized to fit the golden.
+
+**Applied.** Each pair merged into one word; the morpheme split (`etad`·`avoca`; `assasāmī`·`ti`) is
+preserved in that word's segments, which still reconstruct the surface.
+
+### 2b. Omissions (2 phases)
+
+The golden drops a word that is in the prompt:
+
+| phase | prompt (relevant span) | golden | missing |
+|---|---|---|---|
+| phase-an | `…araññagato vā rukkhamūlagato vā suññāgāragato…` | `…araññagato vā rukkhamūlagato suññāgāragato` | the 2nd `vā` |
+| phase-aq | `So satova assasati satova passasati` | `So sato va assasati passasati` | the 2nd `satova` (and the 1st is sandhi-split `sato`+`va`) |
+
+**Applied.** Added the omitted `vā` (phase-an, with `wordRange [0,8)`); for phase-aq merged
+`sato`+`va`→`satova` and added the second `satova`.
+
+## Not in scope of this repair
+
+- The 40/30/30 formula decision and the 12-model roster (separate, operator-directed).
+- Downstream golden parity: the lexicographer/weaver/typesetter goldens for the repaired phases
+  should be re-checked against the sliced prompt once Part 2 lands. The contract test here covers the
+  Anatomist (primary) golden only.

scripts/sutta-studio/benchmark-config.ts

@@ -108,18 +108,30 @@ export const BENCHMARK_CONFIG = {
   // ─────────────────────────────────────────────────────────────────────────
   // ALL 51 PHASES - Full coverage benchmark
   // ─────────────────────────────────────────────────────────────────────────
-  // HELD-OUT test set (30 phases) — the honest, uncontaminated ranking set.
-  // The only phases embedded in prompts as worked examples are phase-a/b/aa
-  // (config/suttaStudioExamples); none appear below, so ranking here is clean.
+  // HELD-OUT test set (27 phases) — the honest, uncontaminated ranking set.
+  // The worked examples embedded in prompts are phase-a/b/aa (config/suttaStudioExamples). The
+  // prior claim that "none appear below" was FALSE: phase-ad, phase-ag and phase-aj each grade the
+  // exact `ātāpī sampajāno satimā` sequence that phase-aa teaches as a worked example (the
+  // satipaṭṭhāna refrain recurs verbatim), so those three were the answer key in the prompt. They
+  // are removed from the ranked set — re-goldening can't help, the sequence IS the example. The
+  // refrain is still exercised once, as the phase-aa example. Guarded by
+  // tests/scripts/sutta-studio/phase-contract.test.ts.
   // (Full 51-phase list preserved in git history / the *Fixture.phases arrays.)
   phasesToTest: [
     'phase-d', 'phase-f', 'phase-h',
     'phase-2', 'phase-4', 'phase-6', 'phase-7',
     'phase-x', 'phase-z',
-    'phase-ab', 'phase-ad', 'phase-af', 'phase-ag', 'phase-ai', 'phase-aj', 'phase-al',
+    'phase-ab', 'phase-af', 'phase-ai', 'phase-al',
     'phase-an', 'phase-ao', 'phase-ap', 'phase-aq', 'phase-as', 'phase-at', 'phase-av', 'phase-ax',
     'phase-az', 'phase-ba', 'phase-bc', 'phase-bd', 'phase-bf', 'phase-bg',
   ],
+  // Fail-closed cumulative spend cap for a fleet run, in USD. The runner tracks spend across all
+  // model/phase calls and ABORTS once this is reached, so a mispriced model or a runaway loop
+  // can't spend unbounded. A call whose cost can't be measured (missing usage or no pricing) is
+  // charged UNPRICED_CALL_ESTIMATE_USD so unpriced spend still drives toward the cap instead of
+  // being invisible. Set null to disable (not recommended for paid runs).
+  maxSpendUsd: 50 as number | null,
+
   // Filter to specific model IDs (empty = all)
   // gemini-2-flash dropped: slug google/gemini-2.0-flash-001 is deprecated on
   // OpenRouter ("No endpoints found"). 5 live models below.

scripts/sutta-studio/benchmark.ts

@@ -23,6 +23,7 @@ import type {
   WeaverPass,
 } from '../../types/suttaStudio';
 import { BENCHMARK_CONFIG, resolveSettingsForModel, type AnatomistFixtureConfig } from './benchmark-config';
+import { SpendGuard } from './spend-guard';
 import {
   assemblePipelineToPacket,
   type PipelinePhaseOutput,
@@ -550,6 +551,10 @@ type ProviderPreferences = {
 /** Request timeout in ms (10 minutes - some reasoning models take longer) */
 const LLM_REQUEST_TIMEOUT_MS = 10 * 60 * 1000;
 
+// Fail-closed cumulative spend guard (review #6). Every LLM call accrues; the run ABORTS at the
+// next loop boundary (assertUnderBudget) once BENCHMARK_CONFIG.maxSpendUsd is hit. See spend-guard.ts.
+const spendGuard = new SpendGuard(BENCHMARK_CONFIG.maxSpendUsd);
+
 const createOpenRouterLLMCaller = (modelProviderPrefs?: ProviderPreferences): LLMCaller => {
   return async ({ settings, messages, signal, maxTokens, options }) => {
     const apiKey = settings.apiKeyOpenRouter || process.env.OPENROUTER_API_KEY;
@@ -711,6 +716,10 @@ const createOpenRouterLLMCaller = (modelProviderPrefs?: ProviderPreferences): LL
     }
   }
 
+  // Accrue toward the fail-closed cap. Enforcement happens at loop boundaries (assertUnderBudget),
+  // not here — a throw inside a pass runner is caught and demoted to a phase failure.
+  spendGuard.accrue(costUsd);
+
     return {
       text,
       tokens: {
@@ -1174,6 +1183,7 @@ const runBenchmark = async () => {
     // runsToExecute is now calculated above with runsTotal
 
     for (const run of runsToExecute) {
+      spendGuard.assertUnderBudget(`model "${run.id}"`); // fail-closed: don't start a new model over budget
       const runIndex = BENCHMARK_CONFIG.runs.indexOf(run);
       const baseSettings = resolveSettingsForModel(run.model);
       const dependencyMode = BENCHMARK_CONFIG.dependencyMode;
@@ -1427,6 +1437,7 @@ const runBenchmark = async () => {
           let earlyStopReason: string | null = null;
 
           for (const testPhaseId of phasesToTest) {
+            spendGuard.assertUnderBudget(`phase "${testPhaseId}"`); // fail-closed between phases (finest boundary)
             const phaseSegments = getSegmentsForPhase(testPhaseId, goldenFixtures);
             if (phaseSegments.length === 0) {
               console.warn(`[Pipeline] No segments found for phase ${testPhaseId}, skipping`);