From 2451dc257d37dbbfb3b388c32ccb847811a90c47 Mon Sep 17 00:00:00 2001 From: "H. Furkan Bozkurt" Date: Mon, 13 Jul 2026 23:19:40 +0000 Subject: [PATCH 01/16] =?UTF-8?q?report(bench):=20single=20results=20table?= =?UTF-8?q?=20with=20per-cell=20colored=20delta,=20drop=20Overview/=CE=94?= =?UTF-8?q?=20column?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Collapse the two-table report (Overview glyphs + Detailed numbers) into ONE results table. Each metric cell now shows the current value on line 1 and the signed delta vs the main baseline on line 2 (via
), colored by the significance+direction of the change: - green/red beyond a per-metric threshold, yellow within the noise band, ⚪ when the baseline has no value for that field (value still shown, tagged (new)). - thresholds centralized in DELTA_THRESHOLDS: composite/score ±5, judge ±0.3, tests ±1, cost ±10%, tokens ±10%; direction per metric (tests/judge/score up, cost/tokens down). No arrows — color only. Drops the whole-row schema-2 gate (baselineHasMetrics/perMetricBaseline): the baseline is diffed PER FIELD, so a field the baseline lacks degrades to ⚪ (new) for that cell only instead of forcing the entire row to 🆕. Removes renderOverview and the separate Δ-vs-base column. Presentational only — buildAggregate/ diffAgainstBaseline data shape (current/delta) unchanged, so analyze.mjs is unaffected. --- scripts/agent-bench/steps/lib/overview.mjs | 312 ++++++++---------- .../agent-bench/steps/lib/overview.test.mjs | 286 +++++++--------- scripts/agent-bench/steps/summary.mjs | 56 ++-- 3 files changed, 265 insertions(+), 389 deletions(-) diff --git a/scripts/agent-bench/steps/lib/overview.mjs b/scripts/agent-bench/steps/lib/overview.mjs index 09ad79a3..b321c04a 100644 --- a/scripts/agent-bench/steps/lib/overview.mjs +++ b/scripts/agent-bench/steps/lib/overview.mjs @@ -1,8 +1,9 @@ -// PR-vs-baseline overview + detailed-results helpers, kept as PURE functions (no fs/env/process) so -// the diff math + coloring are unit-testable under `node --test`. summary.mjs does the I/O and calls -// these to render two tables from the same rows: renderOverview (colors only) and renderDetailed -// (colors + numbers). Baseline = commit-keyed aggregate (bench/runs/latest-main.json); no baseline → -// absolute numbers, all 🆕. Composite/cost/score all come from lib/scoring.mjs. +// PR-vs-baseline results helpers, kept as PURE functions (no fs/env/process) so the diff math + +// coloring are unit-testable under `node --test`. summary.mjs does the I/O and calls these to render +// ONE results table: renderDetailed. Each metric cell shows the CURRENT value plus a color for the +// SIGNIFICANCE + DIRECTION of its change vs the baseline, with the signed delta on a second line. +// Baseline = most recent main bench (bench/runs/latest-main.json); a missing baseline VALUE for a +// field renders ⚪ + the current value + "(new)". Composite/cost/score all come from lib/scoring.mjs. import { SCORE_PER_DOLLAR, cellCost, @@ -24,49 +25,45 @@ const numOrNull = (v) => (typeof v === 'number' && Number.isFinite(v) ? v : null export const GREEN = '🟢'; export const YELLOW = '🟡'; export const RED = '🔴'; -export const NEW = '🆕'; +export const WHITE = '⚪'; // no baseline value for this field/cell — current value still shown, tagged "(new)" export const GONE = '🗑️'; export const NONE = '—'; -// The single tunable separating 🟡 (worse within noise) from 🔴 (worse beyond): a 5% relative margin. -export const MARGIN_PCT = 0.05; - -// Default boundary policy: worse-but-within-margin renders 🟡. Set true to treat within-margin as 🟢. -export const MARGIN_IS_GREEN = false; - // SCORE is higher-better iff scoring.mjs computes composite-per-$ (default); imported so one knob drives both. export const SCORE_HIGHER_BETTER = SCORE_PER_DOLLAR; +const SCORE_DIR = SCORE_HIGHER_BETTER ? 'up' : 'down'; -/** - * Absolute tolerance around a baseline below which a WORSE move stays 🟡. MARGIN_PCT of |baseline|; - * floored to 1 for integer metrics so a ±1 nudge on a small integer reads as within-margin. - * @param {number} baseline - * @param {boolean} integer - * @returns {number} - */ -export function marginAbs(baseline, integer) { - const raw = Math.abs(baseline) * MARGIN_PCT; - return integer ? Math.max(1, Math.round(raw)) : raw; -} +// Per-metric significance thresholds for the delta coloring — the ONE place they live. A change within +// ±threshold reads as 🟡 (noise, since N=1); beyond it, 🟢 (improved) / 🔴 (regressed) by direction. +// Absolute units unless the key ends in `Pct` (then it's a fraction of |baseline|). +export const DELTA_THRESHOLDS = { + composite: 5, // composite points — also the headline mean-delta band (deltaBall) + score: 5, // composite-per-$ points + judge: 0.3, // judge score (0-10) + tests: 1, // test pass count (a ±1 nudge is noise) + costPct: 0.1, // cost: ±10% of the baseline cost + tokensPct: 0.1, // tokens (in+out combined): ±10% of the baseline total +}; /** - * Color one metric vs its baseline. Returns 🟢/🟡/🔴, or `null` when either side is missing. - * `direction` 'up' = higher-better (tests/judge/score), 'down' = lower-better (cost/tokens). Equal = 🟢. - * A worse move within {@link marginAbs} is 🟡 (or 🟢 when MARGIN_IS_GREEN); beyond it, 🔴. + * Color a metric by the SIGNIFICANCE + DIRECTION of its delta vs baseline (NOT absolute quality). + * `direction` 'up' = higher-is-better (tests/judge/score), 'down' = lower-is-better (cost/tokens). + * A change beyond `threshold` in the improving direction → 🟢; beyond it in the worsening direction → + * 🔴; within ±threshold (either way) → 🟡. Missing either side → ⚪. * @param {number|null|undefined} baseline * @param {number|null|undefined} pr - * @param {{direction?: 'up'|'down', integer?: boolean}} [opts] - * @returns {'🟢'|'🟡'|'🔴'|null} + * @param {number} threshold absolute tolerance in the metric's own units + * @param {'up'|'down'} direction + * @returns {'🟢'|'🟡'|'🔴'|'⚪'} */ -export function metricColor(baseline, pr, opts = {}) { - const direction = opts.direction ?? 'up'; - const integer = opts.integer ?? false; - if (baseline === null || baseline === undefined || Number.isNaN(baseline)) return null; - if (pr === null || pr === undefined || Number.isNaN(pr)) return null; - const better = direction === 'up' ? pr >= baseline : pr <= baseline; - if (better) return GREEN; - const worseBy = direction === 'up' ? baseline - pr : pr - baseline; // > 0 - return worseBy <= marginAbs(baseline, integer) ? (MARGIN_IS_GREEN ? GREEN : YELLOW) : RED; +export function deltaColor(baseline, pr, threshold, direction = 'up') { + if (baseline === null || baseline === undefined || Number.isNaN(baseline)) return WHITE; + if (pr === null || pr === undefined || Number.isNaN(pr)) return WHITE; + const raw = pr - baseline; // signed change in the metric's units + const improvement = direction === 'up' ? raw : -raw; // > 0 means "better" + if (improvement > threshold) return GREEN; + if (improvement < -threshold) return RED; + return YELLOW; } // ── Cell scoring (shared with the mean/headline) ───────────────────────────── @@ -96,8 +93,8 @@ export function meanComposite(cells) { /** * Build the compact schema-2 aggregate persisted to S3 as the commit-keyed baseline: per-cell * composite/verdict/klass, test counts, judge overall + per-dimension, tokens, $ cost, score-per-$, - * plus mean + provenance. Artifact-unreadable cells dropped. Schema-1 baselines still diff (missing - * fields render 🆕/— until a main bench records schema-2). + * plus mean + provenance. Artifact-unreadable cells dropped. An older baseline that lacks some + * per-metric fields still diffs per-field: fields it carries color, fields it lacks render ⚪ "(new)". * @param {object[]} cells finalized result.json cells for this run * @param {{sha?: string, base_sha?: string, pr_number?: string, event?: string, generated_at?: string}} [meta] * @returns {object} @@ -142,20 +139,21 @@ export function buildAggregate(cells, meta = {}) { } /** - * Status ball for a COMPOSITE delta, reusing the report's 🟢/🟡/🔴 convention. ±5 near-equal band - * (wide: N=1, so a small delta is as likely variance as signal): improved beyond it 🟢, regressed - * beyond it 🔴, within it (flat / noise) 🟡. `''` for a missing/NaN delta. + * Status ball for a COMPOSITE delta (headline mean-delta), over the ±{@link DELTA_THRESHOLDS}.composite + * band: improved beyond it 🟢, regressed beyond it 🔴, within it (flat / noise) 🟡. `''` for a + * missing/NaN delta. * @param {number|null} delta * @returns {'🟢'|'🟡'|'🔴'|''} */ export function deltaBall(delta) { if (delta === null || delta === undefined || Number.isNaN(delta)) return ''; - if (delta > 5) return GREEN; - if (delta < -5) return RED; + if (delta > DELTA_THRESHOLDS.composite) return GREEN; + if (delta < -DELTA_THRESHOLDS.composite) return RED; return YELLOW; } -// The metric fields the tables read, defaulted to null so a schema-1/partial baseline degrades to 🆕/—. +// The per-cell metric fields the table reads, defaulted to null so any field an older baseline lacks +// degrades to ⚪ "(new)" for THAT metric only (never forcing the whole row). function cellMetrics(c) { return { composite: numOrNull(c?.composite), @@ -171,29 +169,16 @@ function cellMetrics(c) { }; } -/** - * True iff the baseline is schema 2+, i.e. carries the per-metric set the tables diff (test counts, - * per-dimension judge, tokens, cost, score). A schema-1 baseline is treated as "no per-metric - * baseline" — every column (Judge included) renders 🆕 — so coloring stays consistent; the composite - * mean/delta still uses `composite` (present in schema 1) for the headline + analysis roll-up. - * @param {object|null|undefined} baseline - * @returns {boolean} - */ -export function baselineHasMetrics(baseline) { - return !!baseline && (numOrNull(baseline.schema) ?? 0) >= 2; -} - /** * Diff the current run's aggregate against a baseline (or `null`). Cells matched by {@link cellKey}. - * Each row carries the COMPOSITE delta plus `pr`/`base` metric objects the tables color. `base` is - * populated only for a schema-2 baseline ({@link baselineHasMetrics}); against schema-1 every metric - * renders 🆕. A baseline-only cell surfaces as a `removed` row. + * Each row carries the COMPOSITE delta plus `pr`/`base` metric objects the table colors. `base` is + * populated for ANY matched baseline cell (per-field: fields it lacks simply read null → ⚪ "(new)"). + * A baseline-only cell surfaces as a `removed` row. * @param {object} current aggregate from {@link buildAggregate} for this run * @param {object|null} baseline aggregate fetched for the base commit, or null - * @returns {{rows: object[], meanCurrent: number|null, meanBaseline: number|null, meanDelta: number|null, hasBaseline: boolean, perMetricBaseline: boolean}} + * @returns {{rows: object[], meanCurrent: number|null, meanBaseline: number|null, meanDelta: number|null, hasBaseline: boolean}} */ export function diffAgainstBaseline(current, baseline) { - const perMetricBaseline = baselineHasMetrics(baseline); const baseCells = new Map((baseline?.cells ?? []).map((c) => [cellKey(c), c])); const curKeys = new Set((current?.cells ?? []).map((c) => cellKey(c))); const rows = (current?.cells ?? []).map((c) => { @@ -212,8 +197,8 @@ export function diffAgainstBaseline(current, baseline) { hasBaselineCell: !!base, removed: false, pr: cellMetrics(c), - // Per-metric diff only against a schema-2 baseline; schema-1 → every column 🆕. - base: base && perMetricBaseline ? cellMetrics(base) : null, + // Per-field diff against whatever the baseline cell carries (missing fields → ⚪ "(new)"). + base: base ? cellMetrics(base) : null, }; }); // Baseline-only cells (removed/renamed) get their OWN row so a dropped cell stays visible. @@ -229,14 +214,14 @@ export function diffAgainstBaseline(current, baseline) { hasBaselineCell: true, removed: true, pr: null, - base: perMetricBaseline ? cellMetrics(base) : null, + base: cellMetrics(base), }); } rows.sort((a, b) => a.key.localeCompare(b.key)); const meanCurrent = numOrNull(current?.mean_composite); const meanBaseline = baseline ? numOrNull(baseline.mean_composite) : null; const meanDelta = meanCurrent !== null && meanBaseline !== null ? round1(meanCurrent - meanBaseline) : null; - return { rows, meanCurrent, meanBaseline, meanDelta, hasBaseline: !!baseline, perMetricBaseline }; + return { rows, meanCurrent, meanBaseline, meanDelta, hasBaseline: !!baseline }; } // ── Formatters ─────────────────────────────────────────────────────────────── @@ -259,150 +244,117 @@ export function fmtScore(s) { return String(+s.toFixed(1)); } -// Metric spec shared by both renderers so a metric is colored/directed identically in both tables. -const SCORE_DIR = SCORE_HIGHER_BETTER ? 'up' : 'down'; - -// Glyph for the colors-only Overview: metric color, else 🆕 (scored now, no baseline) / — (nothing to diff). -function overviewGlyph(baseVal, prVal, opts) { - if (prVal === null || prVal === undefined) return NONE; - const col = metricColor(baseVal, prVal, opts); - if (col) return col; - return baseVal === null || baseVal === undefined ? NEW : NONE; +/** Judge score 0-10: integer as-is, else 1 decimal. 8 → "8", 7.5 → "7.5". */ +export function fmtJudge(j) { + if (j === null || j === undefined || Number.isNaN(j)) return NONE; + return Number.isInteger(j) ? String(j) : String(+j.toFixed(1)); } -// "baseline -> pr" for the Detailed table, colored. `fmtVal` formats each side. -function detailPair(baseVal, prVal, fmtVal, opts) { - if (prVal === null || prVal === undefined) return NONE; - const col = metricColor(baseVal, prVal, opts); - if (col === null) return `${NEW} ${fmtVal(prVal)}`; // scored now, no baseline - return `${col} ${fmtVal(baseVal)} -> ${fmtVal(prVal)}`; -} +// Signed-delta formatters for the second line of each cell (no arrows — sign only). +const signOf = (n) => (n > 0 ? '+' : n < 0 ? '-' : ''); +const signedInt = (d) => `${d > 0 ? '+' : ''}${Math.round(d)}`; +const signed1 = (d) => { + const v = Math.round(d * 10) / 10; + return `${v > 0 ? '+' : ''}${v}`; +}; +const signedCost = (d) => `${signOf(d)}$${+Math.abs(d).toFixed(2)}`; +const signedTokens = (d) => `${signOf(d)}${humanTokens(Math.abs(d))}`; + +// ── Metric cells ────────────────────────────────────────────────────────────── +// Every cell is two lines joined by
: line 1 ` `, line 2 `()`. +// No baseline value for the field → `⚪
(new)` (the current value is ALWAYS shown). +// A missing CURRENT value → NONE (nothing this run to show). -// Union of judge-dimension keys across baseline + pr (pr first), stable order for the multi-line cell. -function dimKeys(baseDims, prDims) { - const keys = []; - for (const k of Object.keys(prDims ?? {})) if (!keys.includes(k)) keys.push(k); - for (const k of Object.keys(baseDims ?? {})) if (!keys.includes(k)) keys.push(k); - return keys; +// TESTS: "passed/denom", colored by the pass COUNT (higher better, ±1 noise). denom 0 → NONE. +function testsCell(base, pr) { + const pp = numOrNull(pr?.tests_passed); + const pd = numOrNull(pr?.tests_denom); + if (pp === null || pd === null || pd === 0) return NONE; + const value = `${pp}/${pd}`; + const bp = numOrNull(base?.tests_passed); + if (bp === null) return `${WHITE} ${value}
(new)`; + return `${deltaColor(bp, pp, DELTA_THRESHOLDS.tests, 'up')} ${value}
(${signedInt(pp - bp)})`; } -// Multi-line judge cell for the Detailed table: one " base -> pr" line per dim,
-joined. -function judgeDetailCell(base, pr) { - const baseDims = base?.judge_dimensions ?? null; - const prDims = pr?.judge_dimensions ?? null; - const keys = dimKeys(baseDims, prDims); - if (keys.length === 0) return NONE; - const lines = keys.map((k) => { - const bv = baseDims ? numOrNull(baseDims[k]) : null; - const pv = prDims ? numOrNull(prDims[k]) : null; - if (pv === null) return `${NONE} ${k} ${bv ?? NONE} -> ${NONE}`; - const col = metricColor(bv, pv, { direction: 'up', integer: true }); - if (col === null) return `${NEW} ${k} ${pv}`; - return `${col} ${k} ${bv} -> ${pv}`; - }); - return lines.join('
'); +// JUDGE: overall judge score (0-10, higher better, ±0.3 noise). +function judgeCell(base, pr) { + const pv = numOrNull(pr?.judge_score); + if (pv === null) return NONE; + const value = fmtJudge(pv); + const bv = numOrNull(base?.judge_score); + if (bv === null) return `${WHITE} ${value}
(new)`; + return `${deltaColor(bv, pv, DELTA_THRESHOLDS.judge, 'up')} ${value}
(${signed1(pv - bv)})`; } -// Per-row base→PR COMPOSITE delta as a status ball: 🟢/🟡/🔴 over the same ±5-point band as the -// headline; 🆕 for a scored cell with no baseline counterpart; — for an unscored/undiffable cell. -function overviewDeltaGlyph(r) { - if (r.delta !== null && r.delta !== undefined) return deltaBall(r.delta); - return r.current !== null && r.current !== undefined ? NEW : NONE; +// COST: $ builder spend (lower better, ±10% of baseline noise). +function costCell(base, pr) { + const pv = numOrNull(pr?.cost); + if (pv === null) return NONE; + const value = fmtCost(pv); + const bv = numOrNull(base?.cost); + if (bv === null) return `${WHITE} ${value}
(new)`; + const threshold = DELTA_THRESHOLDS.costPct * Math.abs(bv); + return `${deltaColor(bv, pv, threshold, 'down')} ${value}
(${signedCost(pv - bv)})`; } -// Detailed variant: the ball plus the signed numeric delta (e.g. "🟢 +12.4"); 🆕 / — like above. -function detailDeltaCell(r) { - if (r.delta !== null && r.delta !== undefined) { - return `${deltaBall(r.delta)} ${r.delta > 0 ? '+' : ''}${r.delta.toFixed(1)}`; - } - return r.current !== null && r.current !== undefined ? NEW : NONE; +// TOKENS (in/out): value shows both; colored by the COMBINED total (lower better, ±10% noise). +function tokensCell(base, pr) { + const pin = numOrNull(pr?.tokens_in); + const pout = numOrNull(pr?.tokens_out); + if (pin === null && pout === null) return NONE; + const value = `${humanTokens(pr?.tokens_in)}/${humanTokens(pr?.tokens_out)}`; + const prTotal = (pin ?? 0) + (pout ?? 0); + const bin = numOrNull(base?.tokens_in); + const bout = numOrNull(base?.tokens_out); + if (bin === null && bout === null) return `${WHITE} ${value}
(new)`; + const baseTotal = (bin ?? 0) + (bout ?? 0); + const threshold = DELTA_THRESHOLDS.tokensPct * Math.abs(baseTotal); + return `${deltaColor(baseTotal, prTotal, threshold, 'down')} ${value}
(${signedTokens(prTotal - baseTotal)})`; } -// ── Render: Overview (colors only) ─────────────────────────────────────────── -/** - * Colors-only overview: TASK | TEMPLATE | TESTS | JUDGE | COST | TOKENS (in/out) | SCORE | Δ VS BASE, - * one glyph per metric vs baseline (🆕 new, — nothing to diff). The trailing Δ vs base column is the - * cell's COMPOSITE change vs the same cell on the baseline as a {@link deltaBall}. See - * {@link renderDetailed} for numbers. - * @param {ReturnType} diff - * @param {{heading?: string, note?: string}} [opts] - * @returns {string[]} - */ -export function renderOverview(diff, opts = {}) { - const lines = [opts.heading ?? '## Overview', '']; - if (opts.note) lines.push(opts.note, ''); - lines.push( - '| Task | Template | Tests | Judge | Cost | Tokens (in/out) | Score | Δ vs base |', - '|------|----------|:-----:|:-----:|:----:|:---------------:|:-----:|:---------:|', - ); - for (const r of diff.rows) { - if (r.removed) { - lines.push(`| ${r.task ?? NONE} | ${r.template ?? NONE} | ${GONE} | ${GONE} | ${GONE} | ${GONE} | ${GONE} | ${GONE} |`); - continue; - } - const b = r.base; - const p = r.pr ?? {}; - const tests = overviewGlyph(b?.tests_passed, p.tests_passed, { direction: 'up', integer: true }); - const judge = overviewGlyph(b?.judge_score, p.judge_score, { direction: 'up' }); - const cost = overviewGlyph(b?.cost, p.cost, { direction: 'down' }); - const tin = overviewGlyph(b?.tokens_in, p.tokens_in, { direction: 'down' }); - const tout = overviewGlyph(b?.tokens_out, p.tokens_out, { direction: 'down' }); - const score = overviewGlyph(b?.score, p.score, { direction: SCORE_DIR }); - const vsBase = overviewDeltaGlyph(r); - lines.push( - `| ${r.task ?? NONE} | ${r.template ?? NONE} | ${tests} | ${judge} | ${cost} | ${tin}/${tout} | ${score} | ${vsBase} |`, - ); - } - lines.push(''); - return lines; +// SCORE: composite-per-$ (direction from SCORE_HIGHER_BETTER, ±5 noise). +function scoreCell(base, pr) { + const pv = numOrNull(pr?.score); + if (pv === null) return NONE; + const value = fmtScore(pv); + const bv = numOrNull(base?.score); + if (bv === null) return `${WHITE} ${value}
(new)`; + return `${deltaColor(bv, pv, DELTA_THRESHOLDS.score, SCORE_DIR)} ${value}
(${signed1(pv - bv)})`; } -// ── Render: Detailed results (numbers) ─────────────────────────────────────── +// ── Render: single results table ────────────────────────────────────────────── /** - * The Overview rows widened WITH numbers: TESTS (🟡 10/14 -> 9/14) | JUDGE (one colored dim per line) - * | COST | TOKENS (in/out) | SCORE (base -> pr) | Δ VS BASE (ball + signed composite delta) | STOP - * REASON. Colors/directions match the Overview. + * The one results table: TASK | TEMPLATE | TESTS | JUDGE | COST | TOKENS (in/out) | SCORE | STOP REASON. + * Every metric cell is a two-line `
()` (⚪ "(new)" when + * the baseline has no value for it). Color = significance + direction of the change (see {@link deltaColor}). * @param {ReturnType} diff * @param {{heading?: string, note?: string}} [opts] * @returns {string[]} */ export function renderDetailed(diff, opts = {}) { - const lines = [opts.heading ?? '## Detailed results', '']; + const lines = []; + if (opts.heading) lines.push(opts.heading, ''); if (opts.note) lines.push(opts.note, ''); lines.push( - '| Task | Template | Tests | Judge | Cost | Tokens | Score | Δ vs base | Stop reason |', - '|------|----------|-------|-------|------|--------|-------|-----------|-------------|', + '| Task | Template | Tests | Judge | Cost | Tokens (in/out) | Score | Stop reason |', + '|------|----------|-------|-------|------|-----------------|-------|-------------|', ); for (const r of diff.rows) { if (r.removed) { - const was = r.base && r.base.tests_passed !== null ? ` (was ${r.base.tests_passed}/${r.base.tests_denom})` : ''; - lines.push(`| ${r.task ?? NONE} | ${r.template ?? NONE} | ${GONE} removed${was} | ${NONE} | ${NONE} | ${NONE} | ${NONE} | ${GONE} | ${NONE} |`); + const was = + r.base && r.base.tests_passed !== null && r.base.tests_passed !== undefined + ? ` (was ${r.base.tests_passed}/${r.base.tests_denom})` + : ''; + lines.push( + `| ${r.task ?? NONE} | ${r.template ?? NONE} | ${GONE} removed${was} | ${NONE} | ${NONE} | ${NONE} | ${NONE} | ${NONE} |`, + ); continue; } const b = r.base; const p = r.pr ?? {}; - // TESTS is "passed/denom" colored by the passed COUNT, built explicitly rather than via detailPair. - const fmtTests = (m) => `${m.tests_passed ?? NONE}/${m.tests_denom ?? NONE}`; - let testsCell; - if (p.tests_passed === null || p.tests_passed === undefined || p.tests_denom === 0) { - testsCell = NONE; - } else if (!b || b.tests_passed === null) { - testsCell = `${NEW} ${fmtTests(p)}`; - } else { - const col = metricColor(b.tests_passed, p.tests_passed, { direction: 'up', integer: true }); - testsCell = `${col} ${fmtTests(b)} -> ${fmtTests(p)}`; - } - const judge = judgeDetailCell(b, p); - const cost = detailPair(b?.cost ?? null, p.cost, fmtCost, { direction: 'down' }); - const tinCell = detailPair(b?.tokens_in ?? null, p.tokens_in, humanTokens, { direction: 'down' }); - const toutCell = detailPair(b?.tokens_out ?? null, p.tokens_out, humanTokens, { direction: 'down' }); - const tokens = - p.tokens_in === null && p.tokens_out === null ? NONE : `in ${tinCell}
out ${toutCell}`; - const score = detailPair(b?.score ?? null, p.score, fmtScore, { direction: SCORE_DIR }); - const vsBase = detailDeltaCell(r); const stop = p.stop_reason || NONE; lines.push( - `| ${r.task ?? NONE} | ${r.template ?? NONE} | ${testsCell} | ${judge} | ${cost} | ${tokens} | ${score} | ${vsBase} | ${stop} |`, + `| ${r.task ?? NONE} | ${r.template ?? NONE} | ${testsCell(b, p)} | ${judgeCell(b, p)} | ${costCell(b, p)} | ${tokensCell(b, p)} | ${scoreCell(b, p)} | ${stop} |`, ); } lines.push(''); diff --git a/scripts/agent-bench/steps/lib/overview.test.mjs b/scripts/agent-bench/steps/lib/overview.test.mjs index 53bd206a..55319913 100644 --- a/scripts/agent-bench/steps/lib/overview.test.mjs +++ b/scripts/agent-bench/steps/lib/overview.test.mjs @@ -1,24 +1,22 @@ -// Unit tests for the PR-vs-baseline report helpers (overview.mjs): the diff math, margin/color engine, -// formatters, and the two render modes (Overview = colors, Detailed = numbers). Run under bare `node --test`. +// Unit tests for the PR-vs-baseline report helpers (overview.mjs): the diff math, the delta color +// engine, formatters, and the single results table (renderDetailed). Run under bare `node --test`. import assert from 'node:assert/strict'; import { describe, it } from 'node:test'; import { - MARGIN_PCT, - baselineHasMetrics, + DELTA_THRESHOLDS, buildAggregate, cellComposite, cellKey, deltaBall, + deltaColor, diffAgainstBaseline, fmtCost, + fmtJudge, fmtScore, humanTokens, - marginAbs, meanComposite, - metricColor, renderDetailed, - renderOverview, } from './overview.mjs'; const DIMS = { functional_completeness: 8, selector_contract: 8, persistence: 8, code_quality: 8, blocks_fidelity: 8 }; @@ -53,44 +51,37 @@ describe('meanComposite(cells)', () => { }); }); -describe('marginAbs(baseline, integer) — one tunable, MARGIN_PCT', () => { - it('MARGIN_PCT is the documented 5%', () => { - assert.equal(MARGIN_PCT, 0.05); - }); - it('integer metrics floor the margin to 1 (a ±1 nudge is always within-margin)', () => { - assert.equal(marginAbs(8, true), 1); // round(0.4)=0 → floored to 1 - assert.equal(marginAbs(11, true), 1); // round(0.55)=1 - assert.equal(marginAbs(40, true), 2); // round(2.0)=2 - assert.equal(marginAbs(0, true), 1); - }); - it('continuous metrics use exact 5% of |baseline| (no floor)', () => { - assert.equal(marginAbs(100, false), 5); - assert.equal(marginAbs(2, false), 0.1); - assert.equal(marginAbs(0, false), 0); +describe('DELTA_THRESHOLDS — the one place the bands live', () => { + it('holds the documented per-metric thresholds', () => { + assert.equal(DELTA_THRESHOLDS.composite, 5); + assert.equal(DELTA_THRESHOLDS.score, 5); + assert.equal(DELTA_THRESHOLDS.judge, 0.3); + assert.equal(DELTA_THRESHOLDS.tests, 1); + assert.equal(DELTA_THRESHOLDS.costPct, 0.1); + assert.equal(DELTA_THRESHOLDS.tokensPct, 0.1); }); }); -describe('metricColor(baseline, pr, {direction, integer})', () => { - it('equal or better is 🟢 (both directions)', () => { - assert.equal(metricColor(8, 8, { direction: 'up' }), '🟢'); - assert.equal(metricColor(8, 9, { direction: 'up' }), '🟢'); - assert.equal(metricColor(2, 2, { direction: 'down' }), '🟢'); - assert.equal(metricColor(2, 1, { direction: 'down' }), '🟢'); - }); - it('worse within the (integer) margin is 🟡, beyond is 🔴 — matches the judge-dim example', () => { - assert.equal(metricColor(8, 7, { direction: 'up', integer: true }), '🟡'); // 8→7 within 1 - assert.equal(metricColor(9, 5, { direction: 'up', integer: true }), '🔴'); // 9→5 beyond 1 - assert.equal(metricColor(11, 10, { direction: 'up', integer: true }), '🟡'); // 11→10 - assert.equal(metricColor(11, 9, { direction: 'up', integer: true }), '🔴'); // 11→9 - }); - it('worse within/beyond the (continuous) margin for a lower-is-better metric (cost/tokens)', () => { - assert.equal(metricColor(100, 103, { direction: 'down' }), '🟡'); // +3 within 5 - assert.equal(metricColor(100, 110, { direction: 'down' }), '🔴'); // +10 beyond 5 - }); - it('null when either side is missing (caller renders 🆕 / —)', () => { - assert.equal(metricColor(null, 5, { direction: 'up' }), null); - assert.equal(metricColor(5, null, { direction: 'up' }), null); - assert.equal(metricColor(undefined, undefined), null); +describe('deltaColor(baseline, pr, threshold, direction) — significance + direction of the change', () => { + it('higher-is-better: 🟢 beyond threshold up, 🔴 beyond threshold down, 🟡 within (either way)', () => { + assert.equal(deltaColor(5, 11, 5, 'up'), '🟢'); // +6 beyond +5 → improved + assert.equal(deltaColor(11, 5, 5, 'up'), '🔴'); // -6 beyond -5 → regressed + assert.equal(deltaColor(10, 13, 5, 'up'), '🟡'); // +3 within → noise + assert.equal(deltaColor(10, 10, 5, 'up'), '🟡'); // flat → noise + }); + it('lower-is-better (cost/tokens): a DROP is 🟢, a RISE is 🔴', () => { + assert.equal(deltaColor(100, 80, 5, 'down'), '🟢'); // -20 → improved (cheaper) + assert.equal(deltaColor(100, 120, 5, 'down'), '🔴'); // +20 → regressed (pricier) + assert.equal(deltaColor(100, 103, 5, 'down'), '🟡'); // +3 within → noise + }); + it('the threshold boundary is inclusive-of-noise (exactly ±threshold is 🟡, not 🟢/🔴)', () => { + assert.equal(deltaColor(0, 5, 5, 'up'), '🟡'); // exactly +threshold + assert.equal(deltaColor(0, -5, 5, 'up'), '🟡'); // exactly -threshold + }); + it('⚪ when either side is missing (no baseline value for the field)', () => { + assert.equal(deltaColor(null, 5, 5, 'up'), '⚪'); + assert.equal(deltaColor(5, null, 5, 'up'), '⚪'); + assert.equal(deltaColor(undefined, undefined, 5, 'up'), '⚪'); }); }); @@ -112,6 +103,11 @@ describe('formatters', () => { assert.equal(fmtScore(108.3), '108.3'); assert.equal(fmtScore(null), '—'); }); + it('fmtJudge: integer as-is, else 1 decimal', () => { + assert.equal(fmtJudge(8), '8'); + assert.equal(fmtJudge(7.5), '7.5'); + assert.equal(fmtJudge(null), '—'); + }); }); describe('buildAggregate(cells, meta) — schema 2', () => { @@ -155,7 +151,7 @@ describe('buildAggregate(cells, meta) — schema 2', () => { }); }); -describe('deltaBall(delta)', () => { +describe('deltaBall(delta) — headline composite mean-delta', () => { it('🟢 / 🔴 beyond ±5, 🟡 within (inclusive), empty for null', () => { assert.equal(deltaBall(5.2), '🟢'); assert.equal(deltaBall(-6.4), '🔴'); @@ -165,13 +161,17 @@ describe('deltaBall(delta)', () => { }); }); +function round1Delta(a, b) { + return Math.round((a - b) * 10) / 10; +} + describe('diffAgainstBaseline(current, baseline)', () => { const current = buildAggregate([PASS, PARTIAL, { task: 'brand-new', template: 'demo', tests_passed: 2, tests_failed: 0, judge_score: 10, tokens_in: 50000, tokens_out: 5000 }], {}); const baseline = { schema: 2, mean_composite: 70, cells: [ - { task: 'auth-notes', template: 'demo', composite: 80, judge_score: 7, tests_passed: 4, tests_denom: 4, judge_dimensions: { functional_completeness: 9, selector_contract: 8, persistence: 8, code_quality: 7, blocks_fidelity: 8 }, cost: 2.0, score: 40, tokens_in: 190000, tokens_out: 28000 }, + { task: 'auth-notes', template: 'demo', composite: 80, judge_score: 7, tests_passed: 4, tests_denom: 4, cost: 2.0, score: 40, tokens_in: 190000, tokens_out: 28000 }, { task: 'file-gallery', template: 'bare', composite: 70, tests_passed: 3, tests_denom: 4, cost: 0.6, score: 116.7, tokens_in: 100000, tokens_out: 20000 }, { task: 'removed', template: 'x', composite: 50, tests_passed: 5, tests_denom: 5 }, ], @@ -184,13 +184,12 @@ describe('diffAgainstBaseline(current, baseline)', () => { assert.equal(byKey['file-gallery/bare'].delta, -5); // 65 - 70 assert.equal(diff.meanDelta, round1Delta(diff.meanCurrent, 70)); }); - it('attaches pr + base metric objects for the tables', () => { + it('attaches pr + base metric objects for the table', () => { const r = byKey['auth-notes/demo']; assert.equal(r.pr.cost, 1.75); assert.equal(r.base.cost, 2.0); assert.equal(r.pr.tests_passed, 4); assert.equal(r.base.judge_score, 7); - assert.deepEqual(r.base.judge_dimensions.functional_completeness, 9); }); it('a new cell has base=null; a removed cell surfaces with pr=null', () => { assert.equal(byKey['brand-new/demo'].base, null); @@ -205,69 +204,71 @@ describe('diffAgainstBaseline(current, baseline)', () => { assert.equal(noBase.hasBaseline, false); assert.equal(noBase.rows.every((r) => r.base === null), true); }); -}); - -function round1Delta(a, b) { - return Math.round((a - b) * 10) / 10; -} - -describe('renderOverview(diff) — colors only', () => { - const current = buildAggregate([PASS], {}); - const baseline = { - schema: 2, - mean_composite: 92, - cells: [{ task: 'auth-notes', template: 'demo', composite: 92, judge_score: 8, tests_passed: 4, tests_denom: 4, cost: 2.0, score: 46, tokens_in: 190000, tokens_out: 28000 }], - }; - const md = renderOverview(diffAgainstBaseline(current, baseline), { heading: '## Overview' }).join('\n'); - - it('has the colors-only column header incl. the Δ vs base column and no baseline->pr numbers', () => { - assert.match(md, /\| Task \| Template \| Tests \| Judge \| Cost \| Tokens \(in\/out\) \| Score \| Δ vs base \|/); - assert.doesNotMatch(md, /->/); // Overview is glyphs only - }); - it('colors each metric vs baseline + a per-row composite Δ ball', () => { - // tests 4/4 vs 4/4 → 🟢 ; judge 8 vs 8 → 🟢 ; cost $1.75 vs $2.0 (lower) → 🟢 ; - // tokens_in 200k vs 190k (higher=worse, beyond 5%) → 🔴 ; score 52.6 vs 46 (higher) → 🟢 ; - // composite 92 vs 92 → Δ 0 → 🟡 (flat, within the ±5-point band) - const row = md.split('\n').find((l) => l.includes('auth-notes')); - assert.match(row, /\| auth-notes \| demo \| 🟢 \| 🟢 \| 🟢 \| 🔴\/🔴 \| 🟢 \| 🟡 \|/); - }); - it('no-baseline mode flags every metric AND the Δ column 🆕', () => { - const noBase = renderOverview(diffAgainstBaseline(current, null), {}).join('\n'); - const row = noBase.split('\n').find((l) => l.includes('auth-notes')); - assert.match(row, /\| auth-notes \| demo \| 🆕 \| 🆕 \| 🆕 \| 🆕\/🆕 \| 🆕 \| 🆕 \|/); + it('attaches base PER-FIELD: an older baseline missing some fields still attaches (fields read null)', () => { + const partialBase = { + mean_composite: 80, + cells: [{ task: 'auth-notes', template: 'demo', composite: 80, judge_score: 7, test_rate: 1 }], // no tests_passed/cost/tokens/score + }; + const r = diffAgainstBaseline(buildAggregate([PASS], {}), partialBase).rows.find((x) => x.key === 'auth-notes/demo'); + assert.notEqual(r.base, null); // base IS attached (no whole-row gate) + assert.equal(r.base.judge_score, 7); // a field it HAS + assert.equal(r.base.cost, null); // a field it LACKS → null (renders ⚪ "(new)") + assert.equal(r.base.tests_passed, null); + assert.equal(r.delta, 12); // composite still comparable (92 - 80) }); }); -describe('renderDetailed(diff) — numbers', () => { +describe('renderDetailed(diff) — the single results table', () => { const current = buildAggregate([PASS], {}); const baseline = { schema: 2, mean_composite: 92, - cells: [{ task: 'auth-notes', template: 'demo', composite: 92, judge_score: 8, tests_passed: 4, tests_denom: 4, judge_dimensions: { functional_completeness: 9, selector_contract: 8, persistence: 8, code_quality: 7, blocks_fidelity: 8 }, cost: 2.0, score: 46, tokens_in: 190000, tokens_out: 28000 }], + cells: [{ task: 'auth-notes', template: 'demo', composite: 92, judge_score: 8, tests_passed: 4, tests_denom: 4, cost: 2.0, score: 46, tokens_in: 190000, tokens_out: 28000 }], }; const md = renderDetailed(diffAgainstBaseline(current, baseline), {}).join('\n'); const row = md.split('\n').find((l) => l.includes('auth-notes')); - it('has the detailed column header incl. Δ vs base and Stop reason', () => { - assert.match(md, /\| Task \| Template \| Tests \| Judge \| Cost \| Tokens \| Score \| Δ vs base \| Stop reason \|/); - }); - it('renders tests as colored baseline->pr counts', () => { - assert.match(row, /🟢 4\/4 -> 4\/4/); - }); - it('renders the judge cell multi-line per dimension with color + baseline->pr', () => { - // functional_completeness 9→8 (down 1, within margin 1) → 🟡 ; code_quality 7→8 (better) → 🟢 - assert.match(row, /🟡 functional_completeness 9 -> 8/); - assert.match(row, /🟢 code_quality 7 -> 8/); - assert.match(row, /
/); // dimensions on separate lines - }); - it('renders cost, tokens (in/out), score, Δ vs base, and stop reason with numbers', () => { - assert.match(row, /🟢 \$2 -> \$1\.75/); // cost lower = better - assert.match(row, /in 🔴 190K -> 200K
out 🔴 28K -> 30K/); - assert.match(row, /🟢 46 -> 52\.6/); // score higher = better - assert.match(row, /🟡 0\.0 \| end_turn \|/); // composite 92 vs 92 → Δ 0 → 🟡 flat, ball + signed number - assert.match(row, /\| end_turn \|/); - }); - it('a removed cell shows a 🗑️ marker', () => { + it('has the single-table header — no separate Overview, no Δ vs base column', () => { + assert.match(md, /\| Task \| Template \| Tests \| Judge \| Cost \| Tokens \(in\/out\) \| Score \| Stop reason \|/); + assert.doesNotMatch(md, /Δ vs base/); + assert.doesNotMatch(md, /->/); // no arrows anywhere + }); + it('each metric cell is two lines: `
()`', () => { + assert.match(row, /🟡 4\/4
\(0\)/); // tests 4→4, within ±1 → 🟡, delta 0 + assert.match(row, /🟡 8
\(0\)/); // judge 8→8, within ±0.3 → 🟡 + assert.match(row, /🟢 \$1\.75
\(-\$0\.25\)/); // cost $2→$1.75 (−12.5% beyond ±10%) → 🟢 improved + assert.match(row, /🟡 200K\/30K
\(\+12K\)/); // tokens 218K→230K total (+5.5% within ±10%) → 🟡 + assert.match(row, /🟢 52\.6
\(\+6\.6\)/); // score 46→52.6 (+6.6 beyond ±5) → 🟢 improved + assert.match(row, /\| end_turn \|$/); // stop reason retained + }); + it('uses the literal `
` HTML break (NOT a newline) so GFM keeps the two lines inside one table cell', () => { + // A raw \n would break the markdown table row; code fences would show a literal "
". Guard both. + assert.ok(row.includes('
'), 'cell must contain the literal
separator'); + assert.doesNotMatch(row, /\n/); // the row is a single physical line + assert.doesNotMatch(row, /`[^`]*
[^`]*`/); //
is never inside backticks/code span + }); + it('no baseline at all → ⚪ + the CURRENT value + (new) for every metric (value never hidden)', () => { + const noBase = renderDetailed(diffAgainstBaseline(current, null), {}).join('\n'); + const r = noBase.split('\n').find((l) => l.includes('auth-notes')); + assert.match(r, /⚪ 4\/4
\(new\)/); + assert.match(r, /⚪ 8
\(new\)/); + assert.match(r, /⚪ \$1\.75
\(new\)/); + assert.match(r, /⚪ 200K\/30K
\(new\)/); + assert.match(r, /⚪ 52\.6
\(new\)/); + }); + it('PER-FIELD fallback: an older baseline colors the fields it has, ⚪ "(new)" for those it lacks', () => { + const partialBase = { + mean_composite: 80, + cells: [{ task: 'auth-notes', template: 'demo', composite: 80, judge_score: 7, test_rate: 1 }], // judge only + }; + const r = renderDetailed(diffAgainstBaseline(current, partialBase), {}).join('\n').split('\n').find((l) => l.includes('auth-notes')); + assert.match(r, /🟢 8
\(\+1\)/); // judge 7→8 beyond ±0.3 → colored 🟢 + assert.match(r, /⚪ 4\/4
\(new\)/); // tests: baseline lacks the field → ⚪ (new) + assert.match(r, /⚪ \$1\.75
\(new\)/); // cost: ⚪ (new) + assert.match(r, /⚪ 200K\/30K
\(new\)/); // tokens: ⚪ (new) + assert.match(r, /⚪ 52\.6
\(new\)/); // score: ⚪ (new) + }); + it('a removed cell shows a 🗑️ marker with the last-known test count', () => { const withRemoved = diffAgainstBaseline(current, { schema: 2, mean_composite: 80, @@ -281,86 +282,21 @@ describe('renderDetailed(diff) — numbers', () => { }); }); -describe('Δ vs base column — per-row composite delta ball', () => { - const mk = (task, tp, tf, judge) => ({ task, template: 'demo', tests_passed: tp, tests_failed: tf, judge_score: judge, tokens_in: 100000, tokens_out: 10000, stop_reason: 'end_turn' }); - // composites: imp 4/4·8 → 92 ; reg 1/4·3 → 27 ; flat 4/4·8 → 92 ; fresh 4/4·8 → 92 (no baseline cell) - const current = buildAggregate([mk('imp', 4, 0, 8), mk('reg', 1, 3, 3), mk('flat', 4, 0, 8), mk('fresh', 4, 0, 8)], {}); +describe('renderDetailed — per-metric DIRECTION (improve vs regress colors correctly)', () => { + // Current: cheaper cost + fewer tokens (both improvements for lower-is-better), fewer tests passed + // (a regression for higher-is-better), same judge. Verifies each metric's direction independently. + const current = buildAggregate([{ task: 't', template: 'demo', tests_passed: 2, tests_failed: 2, judge_score: 8, tokens_in: 100000, tokens_out: 10000, stop_reason: 'end_turn' }], {}); const baseline = { schema: 2, - mean_composite: 80, - cells: [ - { task: 'imp', template: 'demo', composite: 60 }, // 92 - 60 = +32 → 🟢 improved - { task: 'reg', template: 'demo', composite: 90 }, // 27 - 90 = -63 → 🔴 regressed - { task: 'flat', template: 'demo', composite: 90 }, // 92 - 90 = +2 → 🟡 flat (within ±5) - // 'fresh' has no baseline cell → 🆕 - ], - }; - const ov = renderOverview(diffAgainstBaseline(current, baseline), {}).join('\n'); - const de = renderDetailed(diffAgainstBaseline(current, baseline), {}).join('\n'); - const cellOf = (md, task) => md.split('\n').find((l) => l.includes(`| ${task} |`)); - - it('Overview shows 🟢 improved / 🔴 regressed / 🟡 flat / 🆕 no-base as the trailing column', () => { - assert.match(cellOf(ov, 'imp'), /🟢 \|$/); - assert.match(cellOf(ov, 'reg'), /🔴 \|$/); - assert.match(cellOf(ov, 'flat'), /🟡 \|$/); - assert.match(cellOf(ov, 'fresh'), /🆕 \|$/); - }); - it('Detailed shows the ball + signed delta (🆕 for a no-base cell)', () => { - assert.match(cellOf(de, 'imp'), /🟢 \+32\.0 \| end_turn \|/); - assert.match(cellOf(de, 'reg'), /🔴 -63\.0 \| end_turn \|/); - assert.match(cellOf(de, 'flat'), /🟡 \+2\.0 \| end_turn \|/); - assert.match(cellOf(de, 'fresh'), /🆕 \| end_turn \|/); - }); -}); - -describe('baselineHasMetrics(baseline) — the per-metric gate', () => { - it('true only for a schema-2+ baseline (carries the per-metric set)', () => { - assert.equal(baselineHasMetrics({ schema: 2, cells: [] }), true); - assert.equal(baselineHasMetrics({ schema: 3, cells: [] }), true); - }); - it('false for schema-1, a missing schema, or null (NOT per-metric comparable)', () => { - assert.equal(baselineHasMetrics({ schema: 1, cells: [] }), false); - assert.equal(baselineHasMetrics({ cells: [] }), false); - assert.equal(baselineHasMetrics(null), false); - assert.equal(baselineHasMetrics(undefined), false); - }); -}); - -// REGRESSION GUARD for the "Judge colored while everything else is 🆕" bug: a schema-1 baseline lacked -// the per-metric fields, so coloring lit up Judge alone. The fix gates all per-metric coloring on -// baseline completeness → schema-1 renders every column 🆕 while the composite mean/delta stays comparable. -describe('schema-1 baseline → every column (Judge included) is 🆕', () => { - // Exactly what the OLD buildAggregate wrote: composite/judge_score/test_rate only. - const SCHEMA1 = { - schema: 1, - mean_composite: 80, - cells: [{ task: 'auth-notes', template: 'demo', composite: 80, verdict: 'pass', klass: null, judge_score: 7, test_rate: 1 }], + mean_composite: 90, + cells: [{ task: 't', template: 'demo', composite: 90, judge_score: 8, tests_passed: 4, tests_denom: 4, cost: 2.0, score: 20, tokens_in: 300000, tokens_out: 40000 }], }; - const diff = diffAgainstBaseline(buildAggregate([PASS], {}), SCHEMA1); + const row = renderDetailed(diffAgainstBaseline(current, baseline), {}).join('\n').split('\n').find((l) => l.includes('| t |')); - it('is recognized as a baseline, but NOT a per-metric one', () => { - assert.equal(diff.hasBaseline, true); // a baseline WAS found in S3… - assert.equal(diff.perMetricBaseline, false); // …but it can't diff per-metric - assert.equal(diff.rows.every((r) => r.base === null), true); // so no base metrics - }); - it('keeps the composite mean/delta comparable (the headline still works)', () => { - assert.equal(diff.rows.find((r) => r.key === 'auth-notes/demo').delta, 12); // 92 - 80 - assert.equal(diff.meanDelta, round1Delta(diff.meanCurrent, 80)); - }); - it('Overview: per-METRIC columns render 🆕 (the bug) — but the composite Δ still colors', () => { - const row = renderOverview(diff, {}).find((l) => l.includes('auth-notes')); - // metric columns 🆕 (schema-1 has no per-metric baseline), Δ vs base 🟢 (composite 92 vs 80 → +12) - assert.match(row, /\| auth-notes \| demo \| 🆕 \| 🆕 \| 🆕 \| 🆕\/🆕 \| 🆕 \| 🟢 \|/); - // The composite Δ is the ONLY colored cell — composite persists in schema 1, so its delta is comparable. - const metricsOnly = row.replace(/ 🟢 \|$/, ' |'); - assert.doesNotMatch(metricsOnly, /🟢|🟡|🔴/); // no PER-METRIC cell colors against a schema-1 baseline - }); - it('Detailed: per-metric columns are 🆕 + numbers — only the composite Δ colors', () => { - const row = renderDetailed(diff, {}).find((l) => l.includes('auth-notes')); - assert.match(row, /🟢 \+12\.0/); // composite 92 vs 80 → Δ +12 → 🟢 (comparable even for schema-1) - const metricsOnly = row.replace(/🟢 \+12\.0 \|/, '|'); - assert.doesNotMatch(metricsOnly, /🟢|🟡|🔴/); // no PER-METRIC cell colors against a schema-1 baseline - assert.doesNotMatch(row, /->/); // 🆕 cells show the pr value only, no baseline->pr - assert.match(row, /🆕 functional_completeness 8/); // judge dims are 🆕, not colored + it('tests regressed (4→2 passes) → 🔴; cost/tokens dropped → 🟢; judge flat → 🟡', () => { + assert.match(row, /🔴 2\/4
\(-2\)/); // fewer passes, beyond ±1 + assert.match(row, /🟢 .*
\(-\$/); // cost fell beyond 10% → 🟢 + assert.match(row, /🟢 100K\/10K
\(-/); // tokens fell (340K→110K) → 🟢 + assert.match(row, /🟡 8
\(0\)/); // judge unchanged → 🟡 }); }); diff --git a/scripts/agent-bench/steps/summary.mjs b/scripts/agent-bench/steps/summary.mjs index 6d5a63bc..bb31e55d 100644 --- a/scripts/agent-bench/steps/summary.mjs +++ b/scripts/agent-bench/steps/summary.mjs @@ -1,12 +1,12 @@ // "Render summary": read every bench-result-*/result.json artifact and render a markdown report to -// $GITHUB_STEP_SUMMARY (Overview colors + Detailed numbers, glossary on top; exec summary/analysis -// appended later by analyze.mjs). N=1 per cell. Formulas live in ./lib/scoring.mjs. Baseline = most -// recent main bench (bench/runs/latest-main.json). Headline = mean composite over scored cells; -// observational unless BENCH_MIN_SCORE gates. +// $GITHUB_STEP_SUMMARY (ONE results table with a value + colored delta per metric, glossary on top; +// exec summary/analysis appended later by analyze.mjs). N=1 per cell. Formulas live in ./lib/scoring.mjs. +// Baseline = most recent main bench (bench/runs/latest-main.json). Headline = mean composite over +// scored cells; observational unless BENCH_MIN_SCORE gates. import { appendFileSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { cellCost, compositeBand, isScoredCell, scorePerDollar, testRate, testStats, verdictOf } from './lib/scoring.mjs'; -import { buildAggregate, cellComposite, deltaBall, diffAgainstBaseline, renderDetailed, renderOverview } from './lib/overview.mjs'; +import { buildAggregate, cellComposite, deltaBall, diffAgainstBaseline, renderDetailed } from './lib/overview.mjs'; const RESULTS_DIR = process.env.RESULTS_DIR ?? 'results'; @@ -121,19 +121,16 @@ const md = []; // 1) Glossary & notes — collapsed, at the very top. const lastRun = (process.env.GITHUB_RUN_STARTED_AT || new Date().toISOString()).replace(/\.\d{3}Z$/, 'Z'); md.push('
'); -md.push('📖 Glossary & notes — scoring, colors, the ±5% margin (click to expand)'); +md.push('📖 Glossary & notes — scoring, colors, per-metric thresholds (click to expand)'); md.push(''); md.push('- **N = 1** — one rep per cell, so a small delta may be model variance, not a real change; re-run for certainty.'); md.push( - '- **Colors (vs baseline, per metric):** 🟢 same-or-better · 🟡 worse but within the margin · 🔴 worse beyond it · 🆕 no comparable baseline value (a new cell, or a baseline that predates this metric) · — nothing to diff this run · 🗑️ cell gone since the baseline.', + '- **Colors (change vs baseline, per metric):** 🟢 meaningful improvement · 🟡 change within the noise band · 🔴 meaningful regression · ⚪ no baseline value yet (a new cell, or a metric the baseline predates) — the current value is still shown, tagged `(new)` · — nothing to show this run · 🗑️ cell gone since the baseline. Each cell shows the current value on top and the signed delta vs `main` below it.', ); md.push( - '- **Δ vs base (per row):** the cell\'s COMPOSITE change vs the same cell on the baseline — 🟢 improved · 🟡 flat · 🔴 regressed, over the same ±5-point band as the headline delta (wider than the per-metric ±5% margin — it\'s absolute composite points, since N=1 makes a small swing likely noise). 🆕 no baseline cell to diff. The Detailed table also shows the signed number.', + '- **Thresholds (per metric, `DELTA_THRESHOLDS` in `overview.mjs`):** composite/score ±5 points · judge ±0.3 · tests ±1 pass · cost ±10% · tokens ±10% (in+out combined). A change within the threshold is 🟡 (noise, since N=1); beyond it, 🟢/🔴 by direction. Edit that one map to tune them.', ); -md.push( - '- **Margin = ±5%** (`MARGIN_PCT` in `overview.mjs`) — relative to the baseline value; for integer metrics (test counts, 0-10 judge dims) it is floored to 1, so a single-point nudge is 🟡, never 🔴. Edit that one constant to widen/narrow it.', -); -md.push('- **Directions:** tests ↑, judge ↑, score ↑ are better (higher = 🟢); cost ↓, tokens ↓ are better (lower = 🟢).'); +md.push('- **Directions:** higher is better for tests, judge, and score; lower is better for cost and tokens.'); md.push( '- **Composite (0-100)** = `round(60·test_rate + 4·judge·min(1, 4·test_rate), 1)` — 60% objective pass-rate + 40% judge, the judge term gated below a 25% pass-rate.', ); @@ -152,38 +149,29 @@ md.push(''); md.push('
'); md.push(''); -// 2) Overview — colors only. +// 2) Results — one table: current value + colored delta per metric, headline underneath the heading. if (cells.length > 0) { - let heading = '## Overview — PR vs `main` baseline'; + let heading = '## Results — PR vs `main` baseline'; let note; - const legend = '🟢 better/equal · 🟡 worse within ±5% · 🔴 worse beyond · 🆕 new/uncomparable.'; - const perMetric = diff.perMetricBaseline; + const legend = '🟢 improved · 🟡 within noise · 🔴 regressed · ⚪ no baseline yet (value shown, tagged `(new)`).'; const baseLabel = baseline?.sha ? `\`${String(baseline.sha).slice(0, 7)}\`` : 'the recorded baseline'; - // A baseline predating the per-metric (schema-2) aggregate can compare only the composite mean; - // per-metric cells show 🆕 until the next main bench records the new schema. - const staleNote = `A \`main\` baseline exists (${baseLabel}) but predates the per-metric schema, so per-metric cells show 🆕 — only the composite mean (in the headline) is comparable. Full per-metric coloring returns once a \`main\` bench records the new schema.`; if (benchEvent === 'push') { // A push-to-main run IS the new baseline; diffs against the previous main bench, else absolute. - heading = '## Overview — baseline run'; + heading = '## Results — baseline run'; const rec = `Baseline run (push to \`main\`): recorded as the new \`main\` baseline for \`${benchSha.slice(0, 7) || '(unknown)'}\`.`; - if (!baseline) note = `${rec} No earlier baseline to diff — absolute values (all 🆕).`; - else if (perMetric) note = `${rec} Colored vs the PREVIOUS \`main\` baseline ${baseLabel}. ${legend}`; - else note = `${rec} ${staleNote}`; - } else if (perMetric) { - note = `Each metric colored vs the latest \`main\` baseline ${baseLabel}. ${legend}`; + note = baseline + ? `${rec} Each metric colored vs the PREVIOUS \`main\` baseline ${baseLabel}. ${legend}` + : `${rec} No earlier baseline to diff — current values only (every metric ⚪).`; } else if (baseline) { - note = staleNote; + note = `Each metric shows its current value colored by the change vs the latest \`main\` baseline ${baseLabel}. ${legend}`; } else { - note = 'No `main` baseline recorded yet — showing absolute values (every metric 🆕). PR-vs-`main` deltas appear once a `main` bench has stored one.'; + note = 'No `main` baseline recorded yet — showing current values only (every metric ⚪). Colored deltas vs `main` appear once a `main` bench has stored one.'; } - md.push(...renderOverview(diff, { heading, note })); - // Deterministic headline directly under the Overview. + md.push(heading, ''); + md.push(note, ''); + // Deterministic headline directly under the heading, above the table. md.push(headlineLine(), ''); -} - -// 3) Detailed results — numbers. -if (cells.length > 0) { - md.push(...renderDetailed(diff, { heading: '## Detailed results', note: 'Same rows, colored `baseline -> pr`.' })); + md.push(...renderDetailed(diff, {})); } // 4) Compact caveats (deterministic) — excluded / harness / judge-error cells. From 8c53c18ac2b5acfbf85299d6b631c1cce3a6412c Mon Sep 17 00:00:00 2001 From: "H. Furkan Bozkurt" Date: Mon, 13 Jul 2026 23:24:37 +0000 Subject: [PATCH 02/16] fix(report): always emit all 8 columns for crashed/null-metric cells MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Harden renderDetailed so a crashed/errored cell (0 tests, no tokens → null cost/score, no judge) can never drop a metric column and misalign the row: build the 5 metric cells into an array and coerce any empty string to the NONE placeholder before joining, guaranteeing the 8-column invariant structurally even if a future cell helper returns ''. Add a crashed-cell regression test that asserts every rendered row has exactly 8 columns with no empty cell. Presentational only. --- scripts/agent-bench/steps/lib/overview.mjs | 8 +++++--- scripts/agent-bench/steps/lib/overview.test.mjs | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/scripts/agent-bench/steps/lib/overview.mjs b/scripts/agent-bench/steps/lib/overview.mjs index b321c04a..9ce34c30 100644 --- a/scripts/agent-bench/steps/lib/overview.mjs +++ b/scripts/agent-bench/steps/lib/overview.mjs @@ -353,9 +353,11 @@ export function renderDetailed(diff, opts = {}) { const b = r.base; const p = r.pr ?? {}; const stop = p.stop_reason || NONE; - lines.push( - `| ${r.task ?? NONE} | ${r.template ?? NONE} | ${testsCell(b, p)} | ${judgeCell(b, p)} | ${costCell(b, p)} | ${tokensCell(b, p)} | ${scoreCell(b, p)} | ${stop} |`, - ); + // Guarantee the 8-column invariant: coerce any empty metric cell to NONE so a crashed/null-metric + // cell (no tokens/score/composite) can never drop a column and misalign the row. + const cell = (v) => (typeof v === 'string' && v.trim() ? v : NONE); + const cells = [testsCell(b, p), judgeCell(b, p), costCell(b, p), tokensCell(b, p), scoreCell(b, p)].map(cell); + lines.push(`| ${r.task ?? NONE} | ${r.template ?? NONE} | ${cells.join(' | ')} | ${cell(stop)} |`); } lines.push(''); return lines; diff --git a/scripts/agent-bench/steps/lib/overview.test.mjs b/scripts/agent-bench/steps/lib/overview.test.mjs index 55319913..b0d64e3c 100644 --- a/scripts/agent-bench/steps/lib/overview.test.mjs +++ b/scripts/agent-bench/steps/lib/overview.test.mjs @@ -268,6 +268,23 @@ describe('renderDetailed(diff) — the single results table', () => { assert.match(r, /⚪ 200K\/30K
\(new\)/); // tokens: ⚪ (new) assert.match(r, /⚪ 52\.6
\(new\)/); // score: ⚪ (new) }); + it('EVERY row emits exactly 8 columns — even a crashed cell with null tokens/score/composite/judge', () => { + // A crashed cell (agent_fail/PGlite crash): 0 tests, no tokens → cost/score null, no judge. Its + // metric cells fall back to placeholders, but the row must NEVER drop a column and misalign. + const CRASH = { task: 'crash', template: 'nextjs', klass: 'agent_fail', tests_passed: 0, tests_failed: 0 }; // no tokens/judge → cost/score null + const withCrash = buildAggregate([PASS, CRASH], {}); + const cmd = renderDetailed(diffAgainstBaseline(withCrash, null), {}).join('\n'); + const dataRows = cmd.split('\n').filter((l) => l.startsWith('| ') && !l.startsWith('| Task') && !l.startsWith('|--')); + assert.equal(dataRows.length, 2); + for (const line of dataRows) { + // Inner cells between the outer pipes must be exactly 8 (Task|Template|Tests|Judge|Cost|Tokens|Score|Stop). + const inner = line.split('|').slice(1, -1); + assert.equal(inner.length, 8, `row must have 8 columns, got ${inner.length}: ${line}`); + assert.ok(inner.every((c) => c.trim().length > 0), `no column may be empty: ${line}`); + } + const crashRow = dataRows.find((l) => l.includes('| crash |')); + assert.match(crashRow, /\| crash \| nextjs \| — \| — \| — \| — \| — \| — \|/); // every crashed metric → placeholder, 8 cols + }); it('a removed cell shows a 🗑️ marker with the last-known test count', () => { const withRemoved = diffAgainstBaseline(current, { schema: 2, From 4560659d01d21abd80704482e2144e0e15627466 Mon Sep 17 00:00:00 2001 From: "H. Furkan Bozkurt" Date: Tue, 14 Jul 2026 04:19:12 +0000 Subject: [PATCH 03/16] docs(bench): update README + changeset for single-table report; note judge_dimensions is aggregate-only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sync the docs to the shipped single-table render: - README: replace the two-table (Overview colors-only + Detailed baseline->pr) description with the one results table (renderDetailed) — each metric cell is current value + colored signed delta (two lines via
), ⚪ (new) when the baseline lacks that field, value always shown. Swap MARGIN_PCT/metricColor/ renderOverview for DELTA_THRESHOLDS + deltaColor and list the per-metric bands and directions. Drop the baseline->pr / multi-line per-dimension Judge-cell wording from Re-derivability. - overview.mjs: one-line comment that cellMetrics.judge_dimensions is carried for the persisted S3 aggregate / re-derivability only and is NOT consumed by the rendered table (behavior unchanged). - changeset: refresh the body to describe the single table + baked-in per-cell signed delta + DELTA_THRESHOLDS coloring; still non-releasing. Docs + one comment only — no functional code change. --- .changeset/bench-report-redesign.md | 29 +++++--- scripts/agent-bench/README.md | 79 ++++++++++++---------- scripts/agent-bench/steps/lib/overview.mjs | 2 + 3 files changed, 62 insertions(+), 48 deletions(-) diff --git a/.changeset/bench-report-redesign.md b/.changeset/bench-report-redesign.md index 41f5d890..d268f696 100644 --- a/.changeset/bench-report-redesign.md +++ b/.changeset/bench-report-redesign.md @@ -5,17 +5,24 @@ feat(bench): redesign agent-bench PR-vs-`main` report + fix baseline selection Internal CI tooling — no published-package changes. -- The report is now two tables built from the same rows: a colors-only - **Overview** (🟢/🟡/🔴 per metric) and a numbers **Detailed results** - (`baseline -> pr`, with a multi-line per-dimension judge cell + stop reason), - preceded by a collapsible glossary and followed by a short executive summary - (paragraph + bullets), a **Potential issues** section, and a collapsed - per-cell analysis. Dropped the old build/verdict/composite columns and the raw - per-dimension blurb. -- New per-metric coloring vs the `main` baseline with a single tunable - `MARGIN_PCT` (±5%), and a new **SCORE = composite ÷ cost** (composite points - per dollar) priced from builder tokens at Bedrock Opus 4.8 rates - (`PRICING`/`cellCost`/`scorePerDollar` in `scoring.mjs`). +- The report is now a SINGLE results table built from the run's cells + (`TASK · TEMPLATE · TESTS · JUDGE · COST · TOKENS · SCORE · STOP REASON`), + preceded by a collapsible glossary + the headline (mean composite + delta) and + followed by a short executive summary (paragraph + bullets), a **Potential + issues** section, and a collapsed per-cell analysis. Replaces the old + two-table layout (a colors-only Overview + a numbers Detailed table) and drops + the standalone `Δ vs base` column. +- Each metric cell is two lines: the current value on top and the signed delta + vs the `main` baseline underneath, colored by the significance + direction of + the change via the per-metric `DELTA_THRESHOLDS` (composite/score ±5, judge + ±0.3, tests ±1, cost/tokens ±10%; tests/judge/score higher-better, cost/tokens + lower-better; within band → 🟡, beyond → 🟢/🔴). When the baseline has no value + for a field, the cell still shows the current value tagged `⚪ (new)` — decided + per field, so a partial baseline no longer forces the whole row to be treated + as new. +- New **SCORE = composite ÷ cost** (composite points per dollar) priced from + builder tokens at Bedrock Opus 4.8 rates (`PRICING`/`cellCost`/`scorePerDollar` + in `scoring.mjs`). - Baseline-selection fix: a PR now always diffs against `latest-main.json` (the current `main` tip), never the PR's stale recorded `base.sha`; a push to `main` diffs against the preceding main commit (`github.event.before`) by exact sha. diff --git a/scripts/agent-bench/README.md b/scripts/agent-bench/README.md index 6335ab39..f559b534 100644 --- a/scripts/agent-bench/README.md +++ b/scripts/agent-bench/README.md @@ -137,14 +137,18 @@ and a broken (composite 0) cell scores 0 no matter how cheap it was. A cell with no recorded tokens renders `—` (never a fake `$0`). Flip the single `SCORE_PER_DOLLAR` constant in `lib/scoring.mjs` for cost-per-point instead. -**Colors vs baseline (per metric).** In the report every metric cell is colored -against the baseline value for that metric: 🟢 same-or-better · 🟡 worse but -within the margin · 🔴 worse beyond it · 🆕 no baseline value (new cell) · `—` -nothing to diff · 🗑️ cell gone since the baseline. The margin is a SINGLE tunable, -`MARGIN_PCT` (5% relative) in `lib/overview.mjs`; for integer metrics (test -counts, 0–10 judge dims) it is floored to 1, so a single-point nudge reads 🟡, -never 🔴. Directions: tests ↑, judge ↑, score ↑ are better (higher = 🟢); cost ↓, -tokens ↓ are better (lower = 🟢). +**Colors vs baseline (per metric).** In the report every metric cell shows its +current value on top and, underneath, the **signed delta vs the baseline value** +for that metric, colored by the SIGNIFICANCE + DIRECTION of the change (not +absolute quality): 🟢 meaningful improvement · 🟡 change within the noise band · +🔴 meaningful regression · ⚪ no baseline value for that field (a new cell, or a +metric the baseline predates) — the current value is still shown, tagged `(new)` +· `—` nothing to show this run · 🗑️ cell gone since the baseline. Directions: +tests, judge, score are higher-better; cost, tokens are lower-better. The +per-metric noise bands are a SINGLE tunable, `DELTA_THRESHOLDS` in +`lib/overview.mjs`, consumed by `deltaColor`: composite/score ±5 points, judge +±0.3, tests ±1 pass, cost ±10% (relative), tokens ±10% (in+out combined). A +change within its band reads 🟡 (noise, since N=1); beyond it, 🟢/🔴 by direction. **Verdict tiers** are pure pass-rate — the judge plays no part, so an LLM failure can never flip a verdict: @@ -209,19 +213,18 @@ data so identifiers stay collision-free across a spec's internal navigation. **Re-derivability.** Each `result.json` publishes `tests_passed`/`tests_total`, `test_rate`, the raw per-dimension judge scores (pre-cap `judge_dimensions_raw` and post-cap `judge_dimensions`), `judge_overall`, `composite`, `verdict`, -`klass`, the builder `tokens_in`/`tokens_out`, and `stop_reason`. The **Detailed -results** table renders every post-cap dimension inline (one colored -`baseline -> pr` line per dimension in its Judge cell), and the cost + score are -derived from the published tokens via `lib/scoring.mjs`. A reader can re-derive — -or re-weight — every composite, cost and score from the published data without -re-running anything. +`klass`, the builder `tokens_in`/`tokens_out`, and `stop_reason`. The results +table renders the overall judge score, tests, cost, tokens and score per cell, +and the cost + score are derived from the published tokens via `lib/scoring.mjs`. +A reader can re-derive — or re-weight — every composite, cost and score from the +published data without re-running anything. **Gating.** Observational by default: with the repo/org variable `BENCH_MIN_SCORE` unset the summary only reports the mean composite. Set it to a number to gate — the summary job exits non-zero when the mean composite across scored cells falls below it; this is the **one** intentional exception to green-regardless (below). There is no baseline-*delta* gate — the PR-vs-baseline -overview (below) is observational only. +results table (below) is observational only. **Check status — green regardless.** A bench cell never turns the PR check red. Every fallible cell step (`npm ci`, OIDC, `1-init`, `2-agent`, `3-build-and-test`, @@ -233,24 +236,25 @@ the run summary, not the check status, and the summary job is green too (unless `BENCH_MIN_SCORE` is set and trips). A new commit cancels the prior in-flight run via the workflow `concurrency` group. -**The report — Overview + Detailed vs the `main` baseline.** Each run writes a -compact **aggregate** (per cell: composite/verdict, test counts, per-dimension +**The report — a single results table vs the `main` baseline.** Each run writes +a compact **aggregate** (per cell: composite/verdict, test counts, per-dimension judge scores, builder tokens, cost, and score-per-$, plus the mean) to S3 at `bench/runs//results.json`; a push-to-`main` run also updates the stable pointer `bench/runs/latest-main.json`. The summary job fetches a baseline and -renders TWO tables from the SAME rows (`lib/overview.mjs`): - -- **Overview** — colors ONLY (🟢/🟡/🔴 per metric vs baseline), at-a-glance: - `TASK · TEMPLATE · TESTS · JUDGE · COST · TOKENS (in/out) · SCORE`. -- **Detailed results** — the same rows widened WITH numbers (`baseline -> pr`), - including a multi-line per-dimension Judge cell and the cell's stop reason. - -A collapsible **Glossary** (scoring, colors, the ±5% margin) sits at the very -top; a best-effort roll-up step (`analyze.mjs`) then appends an **Executive -summary** (a short paragraph + bullets), a **Potential issues** section (fed by -each cell's own analysis), and a collapsed **Per-cell analysis** (each cell also -collapsed within it). The old `build` / `verdict` / `composite` columns and the -raw per-dimension blurb are gone — folded into the two tables above. +renders ONE results table (`renderDetailed` in `lib/overview.mjs`): +`TASK · TEMPLATE · TESTS · JUDGE · COST · TOKENS (in/out) · SCORE · STOP REASON`. +Each metric cell is two lines (joined by `
`): the current value on top and +the signed delta vs the baseline underneath (`⚪ (new)` when the baseline has no +value for that field — the value is still shown), colored by `deltaColor` per the +`DELTA_THRESHOLDS` bands. There is no separate colors-only Overview and no +standalone `Δ vs base` column — the delta is baked into every cell. + +A collapsible **Glossary** (scoring, colors, the per-metric thresholds) sits at +the very top; the deterministic **headline** (mean composite + delta) prints just +above the table; a best-effort roll-up step (`analyze.mjs`) then appends an +**Executive summary** (a short paragraph + bullets), a **Potential issues** +section (fed by each cell's own analysis), and a collapsed **Per-cell analysis** +(each cell also collapsed within it). **Baseline selection.** The baseline a **PR** diffs against is ALWAYS the most recent `main`-branch bench, `bench/runs/latest-main.json` — the current tip of @@ -260,8 +264,9 @@ so an exact-base-sha lookup can silently diff against an outdated or never-bench commit; `latest-main.json`, refreshed on every push to `main`, can't. A **push to `main`** instead diffs against the immediately preceding main commit (`github.event.before`) by exact sha — the ONLY place a commit-keyed baseline is -read — with `latest-main` as the fallback. With no baseline found the tables show -absolute values (every metric 🆕) and a "no baseline" note (never an error). +read — with `latest-main` as the fallback. With no baseline found the table shows +current values only (every metric ⚪ `(new)`) and a "no baseline" note (never an +error). Reading/writing the baseline uses the same OIDC role (`s3:GetObject` / `s3:PutObject` on `bench/*`); a missing grant just degrades to "no baseline". @@ -279,12 +284,12 @@ Reading/writing the baseline uses the same OIDC role (`s3:GetObject` / | `steps/4-judge.ts` | Judge agent (Strands + Bedrock); one vended `bash` tool over a spec-blinded, disposable source-only copy; grades on the fixed shared rubric (`COMMON_DIMENSIONS`) and applies hard caps | | `steps/lib/run-shell.ts` | Shared shell infrastructure for the builder + judge: the `WorkspaceSandbox` (host-execution Sandbox rooted at a fixed dir) + a backgrounded-process-safe runner. The containment fix lives here once; imported by `2-agent-run.ts` and `4-judge.ts` | | `steps/lib/scoring.mjs` | **Single source of truth** for scoring: `classifyCell`, `testStats`/`testRate`, `verdict`/`verdictOf`, `composite`/`compositeBand`, `isScoredCell`, plus the cost/score model — `PRICING`/`BUILDER_PRICING`, `cellCost`, `scorePerDollar` (+ the `SCORE_PER_DOLLAR` knob). Imported by finalize, summary, and overview | -| `steps/lib/overview.mjs` | Pure helpers for the two report tables: `buildAggregate` (schema-2 aggregate — per-cell composite/tests/judge-dims/tokens/cost/score/stop_reason + mean), `diffAgainstBaseline`, the color engine (`MARGIN_PCT`, `metricColor`), formatters, and `renderOverview` (colors) + `renderDetailed` (numbers). Imported by summary + analyze | +| `steps/lib/overview.mjs` | Pure helpers for the report table: `buildAggregate` (schema-2 aggregate — per-cell composite/tests/judge-dims/tokens/cost/score/stop_reason + mean), `diffAgainstBaseline`, the color engine (`DELTA_THRESHOLDS`, `deltaColor`), formatters, and `renderDetailed` (the single results table: current value + colored signed delta per cell). Imported by summary + analyze | | `steps/lib/analysis.mjs` | Shared, mostly-pure helpers for the trace/metrics analysis feature: trace trimming, prompt builders, and `parseCellAnalysis` (splits the per-cell model output into an analysis string + a bounded potential-issues list). Imported by `analyze-cell.mjs` (per-cell) and `analyze.mjs` (roll-up) | | `steps/analyze-cell.mjs` | Step 4b: per-cell trace/metrics analysis via the judge model; writes a concise `analysis` string **and** an `analysis_issues[]` (potential issues) back into the cell's `result.json` | | `steps/analyze.mjs` | Summary-job roll-up: synthesizes the per-cell analyses into a short **Executive summary** (paragraph + bullets) via one best-effort Bedrock call, aggregates a **Potential issues** section, and renders a collapsed **Per-cell analysis** (each cell also collapsed) | | `steps/finalize-result.mjs` | Run with `if: always()`; stamps `status` + `failed_at` from per-step outcomes, then `klass`, `test_rate`, `verdict`, `composite` via `lib/scoring.mjs` | -| `steps/summary.mjs` | Render the report to `$GITHUB_STEP_SUMMARY`: a collapsible **Glossary**, the colors-only **Overview** + numbers **Detailed results** tables (vs the `main` baseline), and a deterministic caveats block; reads one `result.json` per cell (N=1); writes the run's schema-2 aggregate (+ Athena NDJSON) for the S3 baseline; optional `BENCH_MIN_SCORE` gate | +| `steps/summary.mjs` | Render the report to `$GITHUB_STEP_SUMMARY`: a collapsible **Glossary**, the headline (mean composite + delta), and the single **results table** — current value + colored signed delta per cell — vs the `main` baseline, plus a deterministic caveats block; reads one `result.json` per cell (N=1); writes the run's schema-2 aggregate (+ Athena NDJSON) for the S3 baseline; optional `BENCH_MIN_SCORE` gate | | `package.json` | Workspace metadata; `private: true` | Failure handling: every cell starts with `0-init-result.mjs` writing a @@ -296,9 +301,9 @@ upload step also runs with `if: always()`, so the cell always shows up in the summary table — never silently missing. The report is written to the **GitHub Actions run summary** -(`$GITHUB_STEP_SUMMARY`) and renders in the run UI — the Glossary, the Overview + -Detailed tables, then the executive summary / potential issues / per-cell -analysis. The bench posts **no PR comment** — the github-script commenting step in +(`$GITHUB_STEP_SUMMARY`) and renders in the run UI — the Glossary, the headline, +the single results table, then the executive summary / potential issues / +per-cell analysis. The bench posts **no PR comment** — the github-script commenting step in `agent-bench.yml` is intentionally left in place but commented out, so it can be restored if commenting is ever wanted again. When the bench matrix produces no results, `summary.mjs` renders a benign "no results" note and still exits 0. diff --git a/scripts/agent-bench/steps/lib/overview.mjs b/scripts/agent-bench/steps/lib/overview.mjs index 9ce34c30..35ca89f9 100644 --- a/scripts/agent-bench/steps/lib/overview.mjs +++ b/scripts/agent-bench/steps/lib/overview.mjs @@ -160,6 +160,8 @@ function cellMetrics(c) { tests_passed: numOrNull(c?.tests_passed), tests_denom: numOrNull(c?.tests_denom), judge_score: numOrNull(c?.judge_score), + // Carried for the persisted S3 aggregate / re-derivability only — the single results table renders + // the overall judge_score, NOT the per-dimension breakdown. Kept so downstream readers can re-weight. judge_dimensions: c?.judge_dimensions && typeof c.judge_dimensions === 'object' ? c.judge_dimensions : null, tokens_in: numOrNull(c?.tokens_in), tokens_out: numOrNull(c?.tokens_out), From 402d16d6c08bb3ab3641ee2f5059fdd256487aea Mon Sep 17 00:00:00 2001 From: "H. Furkan Bozkurt" Date: Tue, 14 Jul 2026 09:25:02 +0000 Subject: [PATCH 04/16] =?UTF-8?q?feat(bench):=20expanded=20report=20?= =?UTF-8?q?=E2=80=94=20per-dim=20judge,=204-line=20tokens,=20turns/LOC/fil?= =?UTF-8?q?es=20cols,=20inline=20delta,=20preword?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Render (overview.mjs, summary.mjs): - Scalar cells (tests/cost/turns/score) now inline: (<Δ>), dropping the two-line
form. - JUDGE column stacks one
-joined line per rubric dimension (functional/selectors/persistence/code/blocks) with per-dim baseline deltas; overall judge folded into the preword. - TOKENS column stacks in / out / cached in / cached out; cache lines render ⚪ '(new)' until a baseline carries them. - New columns: Turns (cycle_count, lower-better), LOC (created/edited), Files (created/edited). - LOC & Files are NEUTRAL — always ⚪, value + signed delta, never green/red (no inherent good/bad direction). Cache tokens are DISPLAYED only, not deducted from cost/SCORE. - New DELTA_THRESHOLDS bands: turns ±3, cache read/write ±20%. humanTokens now compacts ≥1M to 'M'. - renderPreword: bulleted run summary (mean composite + Δ vs main, verdict counts, totals, biggest gains/drops, config). Replaces the old headline. - N-column guard extended to 11 columns; every row (incl. crashed/removed) emits all 11. Aggregate (overview.mjs buildAggregate/cellMetrics): - Persist cycle_count, cache_read_tokens, cache_write_tokens, loc_created/edited, files_created/edited so future baselines carry them. Instrumentation (2-agent-run.ts, partial-envelope.mjs): - Accumulate usage.cacheReadInputTokens/cacheWriteInputTokens in the ModelStreamUpdate hook; persist cache_read/write_tokens on every envelope path (checkpoint, partial, error, success), success taking max(hook, winnerUsage). - workspace-diff.mjs: snapshot the scaffolded workspace before the agent runs via a throwaway external GIT_DIR (never touches the workspace's own .git), diff after for LOC/files churn on the success path. Degrades to null on any failure. Tests: overview.test.mjs updated for the 11-column layout (per-dim judge, 4-line tokens, turns/LOC/files, ⚪ '(new)', 11-column guard, M formatting); workspace-diff.test.mjs added (real git against a real temp workspace + pure parseChurn). tsc clean; 182 pass / 4 skipped (isolation tests need unshare). --- scripts/agent-bench/steps/2-agent-run.ts | 34 ++ scripts/agent-bench/steps/lib/overview.mjs | 304 ++++++++++++++---- .../agent-bench/steps/lib/overview.test.mjs | 267 ++++++++++----- .../steps/lib/partial-envelope.mjs | 8 +- .../agent-bench/steps/lib/workspace-diff.mjs | 146 +++++++++ .../steps/lib/workspace-diff.test.mjs | 104 ++++++ scripts/agent-bench/steps/summary.mjs | 62 ++-- 7 files changed, 766 insertions(+), 159 deletions(-) create mode 100644 scripts/agent-bench/steps/lib/workspace-diff.mjs create mode 100644 scripts/agent-bench/steps/lib/workspace-diff.test.mjs diff --git a/scripts/agent-bench/steps/2-agent-run.ts b/scripts/agent-bench/steps/2-agent-run.ts index 68782b0f..0062fd1f 100644 --- a/scripts/agent-bench/steps/2-agent-run.ts +++ b/scripts/agent-bench/steps/2-agent-run.ts @@ -21,6 +21,7 @@ import { sleep, } from './lib/bedrock-retry.ts'; import { buildCheckpointEnvelope, writeEnvelopeAtomic } from './lib/partial-envelope.mjs'; +import { beginWorkspaceDiff, finishWorkspaceDiff } from './lib/workspace-diff.mjs'; import { BENCH_AGENT_USER, WorkspaceSandbox, @@ -76,6 +77,8 @@ process.stderr.write( // envelope from the signal handler. Counters at module scope so they survive invoke retries and the handler. let partialTokensIn = 0; let partialTokensOut = 0; +let partialCacheRead = 0; +let partialCacheWrite = 0; let partialCycles = 0; // Fresh agent per invoke attempt (mirrors the judge): a mid-stream failure can leave a half-built @@ -99,6 +102,10 @@ function makeBuilderAgent(): Agent { if (inner.type === 'modelMetadataEvent' && inner.usage) { partialTokensIn += inner.usage.inputTokens ?? 0; partialTokensOut += inner.usage.outputTokens ?? 0; + // Cache-read/write tokens are DISPLAYED only (never fed into cost/SCORE); accumulated with + // the same per-cycle += as tokens_in/out so a mid-run kill still leaves the spend on disk. + partialCacheRead += inner.usage.cacheReadInputTokens ?? 0; + partialCacheWrite += inner.usage.cacheWriteInputTokens ?? 0; partialCycles += 1; // Checkpoint the moment usage advances, so an UNGRACEFUL kill (a pkill storm tearing down // this harness before the SIGTERM flush) still leaves nonzero tokens + partial cycles on OUTPUT. @@ -124,6 +131,8 @@ function writeCheckpoint(): void { startedMs: started, tokensIn: partialTokensIn, tokensOut: partialTokensOut, + cacheRead: partialCacheRead, + cacheWrite: partialCacheWrite, cycles: partialCycles, isolationActive: ISOLATE, }), @@ -153,6 +162,8 @@ function writePartialEnvelopeAndExit(signal: string): void { duration_sec: Math.round((Date.now() - started) / 1000), tokens_in: partialTokensIn, tokens_out: partialTokensOut, + cache_read_tokens: partialCacheRead, + cache_write_tokens: partialCacheWrite, stop_reason: stopReason, cycle_count: partialCycles, final_message: '', @@ -189,6 +200,12 @@ process.on('SIGINT', () => writePartialEnvelopeAndExit('SIGINT')); // have to assume isolation was off. A later model cycle (or a terminal exit) overwrites this. writeCheckpoint(); +// Snapshot the scaffolded workspace NOW (before the agent touches it) so a post-run diff can report +// LOC/files churn. Best-effort: null on any failure → the cell persists no loc/files (renders ⚪ "(new)"). +// Uses a throwaway external git dir, so it never disturbs the workspace/app or the agent's own git. +const snapshot = beginWorkspaceDiff(WORKSPACE); +process.stderr.write(`[bench] workspace churn snapshot: ${snapshot ? 'captured' : 'unavailable (loc/files → null)'}\n`); + // Startup stagger: spread each wave's FIRST Bedrock call so N cells don't all hit invoke at once and // trip the account TPM ceiling (the burst that failed 6/10 cells in run 28967475174). Keyed to the // matrix job index for a deterministic fan-out, staggering by within-wave slot (index % waveSize) so @@ -242,6 +259,8 @@ if (!result) { duration_sec: Math.round((Date.now() - started) / 1000), tokens_in: partialTokensIn, tokens_out: partialTokensOut, + cache_read_tokens: partialCacheRead, + cache_write_tokens: partialCacheWrite, stop_reason: 'error', cycle_count: partialCycles, final_message: '', @@ -266,6 +285,15 @@ const duration_sec = Math.round((Date.now() - started) / 1000); const winnerUsage = result.metrics?.accumulatedUsage; const tokensIn = Math.max(partialTokensIn, winnerUsage?.inputTokens ?? 0); const tokensOut = Math.max(partialTokensOut, winnerUsage?.outputTokens ?? 0); +// Cache tokens follow the SAME max(hook, winner) rule as tokens_in/out. Displayed only — NOT part of +// cost/SCORE (see scoring.mjs cellCost); a future net-of-cache cost model could subtract these. +const cacheRead = Math.max(partialCacheRead, winnerUsage?.cacheReadInputTokens ?? 0); +const cacheWrite = Math.max(partialCacheWrite, winnerUsage?.cacheWriteInputTokens ?? 0); + +// LOC/files churn from the pre-run scaffold snapshot (best-effort; null on any failure — see +// workspace-diff.mjs). Computed only on the success path: a timed-out/errored cell left the workspace +// mid-flight, so its diff would be misleading — those paths persist no loc/files (→ ⚪ "(new)"). +const churn = finishWorkspaceDiff(snapshot, WORKSPACE); writeFileSync( OUTPUT, @@ -275,8 +303,14 @@ writeFileSync( duration_sec, tokens_in: tokensIn, tokens_out: tokensOut, + cache_read_tokens: cacheRead, + cache_write_tokens: cacheWrite, stop_reason: result.stopReason, cycle_count: result.metrics?.cycleCount ?? 0, + loc_created: churn.loc_created, + loc_edited: churn.loc_edited, + files_created: churn.files_created, + files_edited: churn.files_edited, final_message: messageText(result.lastMessage), isolation_active: ISOLATE, }, diff --git a/scripts/agent-bench/steps/lib/overview.mjs b/scripts/agent-bench/steps/lib/overview.mjs index 35ca89f9..0e8fc7d6 100644 --- a/scripts/agent-bench/steps/lib/overview.mjs +++ b/scripts/agent-bench/steps/lib/overview.mjs @@ -14,6 +14,7 @@ import { testStats, verdictOf, } from './scoring.mjs'; +import { COMMON_DIMENSIONS } from './scoring.mjs'; // Stable cross-run identity for a cell (task + template, since a task may run on multiple templates). export const cellKey = (c) => `${c?.task ?? ''}/${c?.template ?? ''}`; @@ -39,10 +40,16 @@ const SCORE_DIR = SCORE_HIGHER_BETTER ? 'up' : 'down'; export const DELTA_THRESHOLDS = { composite: 5, // composite points — also the headline mean-delta band (deltaBall) score: 5, // composite-per-$ points - judge: 0.3, // judge score (0-10) + judge: 0.3, // judge score (0-10) — applied per-dimension too tests: 1, // test pass count (a ±1 nudge is noise) costPct: 0.1, // cost: ±10% of the baseline cost - tokensPct: 0.1, // tokens (in+out combined): ±10% of the baseline total + tokensPct: 0.1, // tokens (in / out): ±10% of the baseline, per stream + turns: 3, // cycle_count (lower-better): ±3 turns is within run-to-run noise at N=1 + cacheReadPct: 0.2, // cache-read tokens: ±20% relative (no strong prior; cache hits swing more than raw tokens) + cacheWritePct: 0.2, // cache-write tokens: ±20% relative + // LOC & Files are INTENTIONALLY absent: more/fewer lines or files is neither good nor bad on its + // own (a bigger app can be the right call), so those cells are rendered NEUTRAL (⚪, value + signed + // delta, never 🟢/🔴) — see locCell/filesCell. Add a band here only if a direction is ever agreed. }; /** @@ -129,6 +136,15 @@ export function buildAggregate(cells, meta = {}) { c.judge_dimensions && typeof c.judge_dimensions === 'object' ? c.judge_dimensions : null, tokens_in: numOrNull(c.tokens_in), tokens_out: numOrNull(c.tokens_out), + // Newer per-cell signals — persisted so the NEXT main bench writes them into the baseline + // (older baselines lack them → the table renders those metrics ⚪ "(new)" until then). + cycle_count: numOrNull(c.cycle_count), + cache_read_tokens: numOrNull(c.cache_read_tokens), + cache_write_tokens: numOrNull(c.cache_write_tokens), + loc_created: numOrNull(c.loc_created), + loc_edited: numOrNull(c.loc_edited), + files_created: numOrNull(c.files_created), + files_edited: numOrNull(c.files_edited), cost, score: scorePerDollar(comp, cost), stop_reason: typeof c.stop_reason === 'string' && c.stop_reason ? c.stop_reason : null, @@ -160,11 +176,18 @@ function cellMetrics(c) { tests_passed: numOrNull(c?.tests_passed), tests_denom: numOrNull(c?.tests_denom), judge_score: numOrNull(c?.judge_score), - // Carried for the persisted S3 aggregate / re-derivability only — the single results table renders - // the overall judge_score, NOT the per-dimension breakdown. Kept so downstream readers can re-weight. + // Per-dimension judge scores (capped), rendered as the JUDGE column's per-dim rows. Baselines + // that carry them color per-dim; older ones lacking them fall back to the overall judge score. judge_dimensions: c?.judge_dimensions && typeof c.judge_dimensions === 'object' ? c.judge_dimensions : null, tokens_in: numOrNull(c?.tokens_in), tokens_out: numOrNull(c?.tokens_out), + cycle_count: numOrNull(c?.cycle_count), + cache_read_tokens: numOrNull(c?.cache_read_tokens), + cache_write_tokens: numOrNull(c?.cache_write_tokens), + loc_created: numOrNull(c?.loc_created), + loc_edited: numOrNull(c?.loc_edited), + files_created: numOrNull(c?.files_created), + files_edited: numOrNull(c?.files_edited), cost: numOrNull(c?.cost), score: numOrNull(c?.score), stop_reason: typeof c?.stop_reason === 'string' && c.stop_reason ? c.stop_reason : null, @@ -231,7 +254,8 @@ export function diffAgainstBaseline(current, baseline) { export function humanTokens(n) { if (n === null || n === undefined || Number.isNaN(n)) return NONE; if (n < 1000) return String(Math.round(n)); - return `${+(n / 1000).toFixed(1)}K`; + if (n < 1_000_000) return `${+(n / 1000).toFixed(1)}K`; + return `${+(n / 1_000_000).toFixed(1)}M`; } /** USD, trailing zeros trimmed: 3.5 → "$3.5", 2 → "$2", 1.05 → "$1.05", null → "—". */ @@ -252,7 +276,7 @@ export function fmtJudge(j) { return Number.isInteger(j) ? String(j) : String(+j.toFixed(1)); } -// Signed-delta formatters for the second line of each cell (no arrows — sign only). +// Signed-delta formatters for the delta suffix of each cell (no arrows — sign only). const signOf = (n) => (n > 0 ? '+' : n < 0 ? '-' : ''); const signedInt = (d) => `${d > 0 ? '+' : ''}${Math.round(d)}`; const signed1 = (d) => { @@ -261,11 +285,48 @@ const signed1 = (d) => { }; const signedCost = (d) => `${signOf(d)}$${+Math.abs(d).toFixed(2)}`; const signedTokens = (d) => `${signOf(d)}${humanTokens(Math.abs(d))}`; +const wholeNum = (v) => String(Math.round(v)); // ── Metric cells ────────────────────────────────────────────────────────────── -// Every cell is two lines joined by
: line 1 ` `, line 2 `()`. -// No baseline value for the field → `⚪
(new)` (the current value is ALWAYS shown). -// A missing CURRENT value → NONE (nothing this run to show). +// Scalar cells are ONE inline line: ` (<Δ>)`. Multi-value cells (judge / tokens / loc / +// files) stack one `