Skip to content

Commit 7526375

Browse files
committed
feat(v3-7): AblationsTab math-divergence assertions
Closes V3-7 / cppmega-mlx-bws. Adds per-row testid ablation-final-{variant} so e2e tests can read the final-loss column directly. New spec 13_ablation_math.spec.ts replaces v2's vacuous "rows >= 2" check with two math-divergence assertions: 1. activation axis (glu, swiglu, gelu, 4 steps): at least one pairwise |final_loss_a - final_loss_b| > 1e-3. Different activation kernels MUST produce numerically distinct outputs; identical losses would prove the axis switch never reached the model. 2. optimizer axis (adamw, lion, 4 steps): |final_loss_adamw - final_loss_lion| > 1e-4. AdamW (adaptive 2nd-moment) and Lion (sign-based) are fundamentally different update rules — equal losses would indicate B1-style optimizer fallthrough. 2/2 e2e green, 166/166 vitest, testid rename avoided prefix collision with existing 'ablation-row-' selectors.
1 parent 7ba1661 commit 7526375

2 files changed

Lines changed: 95 additions & 1 deletion

File tree

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
// V3-7: AblationsTab axis variants must produce real loss / weight
2+
// divergence — not just N rows in a table. Closes the "AblationsTab
3+
// Run produces results table with variants" vacuous gap from v2.
4+
5+
import { test, expect } from "@playwright/test";
6+
import { gotoApp, selectPreset } from "../fixtures";
7+
8+
async function runAblation(
9+
page: import("@playwright/test").Page,
10+
axis: "activation" | "optimizer" | "norm" | "schedule",
11+
variants: string[],
12+
numSteps: number,
13+
): Promise<void> {
14+
await page.getByTestId("sidebar-tab-ablations").click();
15+
await page.getByTestId("ablations-tab").waitFor();
16+
await page.getByTestId("ablation-axis").selectOption(axis);
17+
18+
// Tick only the requested variants — uncheck whatever's pre-selected.
19+
const allLabels = page.locator("[data-testid^='ablation-variant-']");
20+
const count = await allLabels.count();
21+
for (let i = 0; i < count; i++) {
22+
const label = allLabels.nth(i);
23+
const tid = await label.getAttribute("data-testid");
24+
const v = tid?.replace("ablation-variant-", "") ?? "";
25+
const checkbox = label.locator("input[type='checkbox']");
26+
const want = variants.includes(v);
27+
const isChecked = await checkbox.isChecked();
28+
if (want !== isChecked) await checkbox.click();
29+
}
30+
31+
await page.getByTestId("ablation-num-steps").fill(String(numSteps));
32+
await page.getByTestId("ablation-run").click();
33+
await page.getByTestId("ablation-results").waitFor({ timeout: 60_000 });
34+
}
35+
36+
async function rowFinal(
37+
page: import("@playwright/test").Page, variant: string,
38+
): Promise<number> {
39+
const text = await page.getByTestId(`ablation-final-${variant}`)
40+
.textContent();
41+
return parseFloat(text?.trim() ?? "NaN");
42+
}
43+
44+
// ---------------------------------------------------------------------------
45+
// Activation axis: different activations should produce DIFFERENT final loss
46+
// ---------------------------------------------------------------------------
47+
48+
test("activation ablation: glu vs swiglu vs gelu produce different losses",
49+
async ({ page }) => {
50+
test.setTimeout(120_000);
51+
await gotoApp(page);
52+
await selectPreset(page, "llama3_8b");
53+
await runAblation(page, "activation", ["glu", "swiglu", "gelu"], 4);
54+
55+
const glu = await rowFinal(page, "glu");
56+
const swiglu = await rowFinal(page, "swiglu");
57+
const gelu = await rowFinal(page, "gelu");
58+
59+
expect([glu, swiglu, gelu].every(Number.isFinite)).toBe(true);
60+
61+
// At least one pair must differ by > 1e-3 in absolute terms — proves
62+
// the activation swap actually reached the model. Synthetic data
63+
// makes exact ordering noisy but identical results are impossible.
64+
const maxDiff = Math.max(
65+
Math.abs(glu - swiglu),
66+
Math.abs(swiglu - gelu),
67+
Math.abs(glu - gelu),
68+
);
69+
expect(maxDiff).toBeGreaterThan(1e-3);
70+
});
71+
72+
// ---------------------------------------------------------------------------
73+
// Optimizer axis: AdamW vs Lion → different final loss (math divergence)
74+
// ---------------------------------------------------------------------------
75+
76+
test("optimizer ablation: adamw vs lion final losses differ", async ({
77+
page,
78+
}) => {
79+
test.setTimeout(120_000);
80+
await gotoApp(page);
81+
await selectPreset(page, "llama3_8b");
82+
await runAblation(page, "optimizer", ["adamw", "lion"], 4);
83+
84+
const adamw = await rowFinal(page, "adamw");
85+
const lion = await rowFinal(page, "lion");
86+
87+
expect(Number.isFinite(adamw) && Number.isFinite(lion)).toBe(true);
88+
89+
// AdamW and Lion are fundamentally different update rules — sign-only
90+
// vs second-moment adaptive. Their final losses after 4 steps must
91+
// differ. V2 only counted rows, this is the math-divergence proof.
92+
expect(Math.abs(adamw - lion)).toBeGreaterThan(1e-4);
93+
});

vbgui/src/components/sidebar/AblationsTab.tsx

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,8 @@ export function AblationsTab({
216216
)}
217217
<code>{r.variant}</code>
218218
</td>
219-
<td style={td}>{final?.toFixed(4) ?? "—"}</td>
219+
<td data-testid={`ablation-final-${r.variant}`}
220+
style={td}>{final?.toFixed(4) ?? "—"}</td>
220221
<td style={{ ...td,
221222
color: delta == null ? "#9ca3af"
222223
: delta > 0 ? "#dc2626" : "#16a34a" }}>

0 commit comments

Comments
 (0)