|
| 1 | +// V3-7: AblationsTab axis variants must produce real loss / weight |
| 2 | +// divergence — not just N rows in a table. Closes the "AblationsTab |
| 3 | +// Run produces results table with variants" vacuous gap from v2. |
| 4 | + |
| 5 | +import { test, expect } from "@playwright/test"; |
| 6 | +import { gotoApp, selectPreset } from "../fixtures"; |
| 7 | + |
| 8 | +async function runAblation( |
| 9 | + page: import("@playwright/test").Page, |
| 10 | + axis: "activation" | "optimizer" | "norm" | "schedule", |
| 11 | + variants: string[], |
| 12 | + numSteps: number, |
| 13 | +): Promise<void> { |
| 14 | + await page.getByTestId("sidebar-tab-ablations").click(); |
| 15 | + await page.getByTestId("ablations-tab").waitFor(); |
| 16 | + await page.getByTestId("ablation-axis").selectOption(axis); |
| 17 | + |
| 18 | + // Tick only the requested variants — uncheck whatever's pre-selected. |
| 19 | + const allLabels = page.locator("[data-testid^='ablation-variant-']"); |
| 20 | + const count = await allLabels.count(); |
| 21 | + for (let i = 0; i < count; i++) { |
| 22 | + const label = allLabels.nth(i); |
| 23 | + const tid = await label.getAttribute("data-testid"); |
| 24 | + const v = tid?.replace("ablation-variant-", "") ?? ""; |
| 25 | + const checkbox = label.locator("input[type='checkbox']"); |
| 26 | + const want = variants.includes(v); |
| 27 | + const isChecked = await checkbox.isChecked(); |
| 28 | + if (want !== isChecked) await checkbox.click(); |
| 29 | + } |
| 30 | + |
| 31 | + await page.getByTestId("ablation-num-steps").fill(String(numSteps)); |
| 32 | + await page.getByTestId("ablation-run").click(); |
| 33 | + await page.getByTestId("ablation-results").waitFor({ timeout: 60_000 }); |
| 34 | +} |
| 35 | + |
| 36 | +async function rowFinal( |
| 37 | + page: import("@playwright/test").Page, variant: string, |
| 38 | +): Promise<number> { |
| 39 | + const text = await page.getByTestId(`ablation-final-${variant}`) |
| 40 | + .textContent(); |
| 41 | + return parseFloat(text?.trim() ?? "NaN"); |
| 42 | +} |
| 43 | + |
| 44 | +// --------------------------------------------------------------------------- |
| 45 | +// Activation axis: different activations should produce DIFFERENT final loss |
| 46 | +// --------------------------------------------------------------------------- |
| 47 | + |
| 48 | +test("activation ablation: glu vs swiglu vs gelu produce different losses", |
| 49 | + async ({ page }) => { |
| 50 | + test.setTimeout(120_000); |
| 51 | + await gotoApp(page); |
| 52 | + await selectPreset(page, "llama3_8b"); |
| 53 | + await runAblation(page, "activation", ["glu", "swiglu", "gelu"], 4); |
| 54 | + |
| 55 | + const glu = await rowFinal(page, "glu"); |
| 56 | + const swiglu = await rowFinal(page, "swiglu"); |
| 57 | + const gelu = await rowFinal(page, "gelu"); |
| 58 | + |
| 59 | + expect([glu, swiglu, gelu].every(Number.isFinite)).toBe(true); |
| 60 | + |
| 61 | + // At least one pair must differ by > 1e-3 in absolute terms — proves |
| 62 | + // the activation swap actually reached the model. Synthetic data |
| 63 | + // makes exact ordering noisy but identical results are impossible. |
| 64 | + const maxDiff = Math.max( |
| 65 | + Math.abs(glu - swiglu), |
| 66 | + Math.abs(swiglu - gelu), |
| 67 | + Math.abs(glu - gelu), |
| 68 | + ); |
| 69 | + expect(maxDiff).toBeGreaterThan(1e-3); |
| 70 | + }); |
| 71 | + |
| 72 | +// --------------------------------------------------------------------------- |
| 73 | +// Optimizer axis: AdamW vs Lion → different final loss (math divergence) |
| 74 | +// --------------------------------------------------------------------------- |
| 75 | + |
| 76 | +test("optimizer ablation: adamw vs lion final losses differ", async ({ |
| 77 | + page, |
| 78 | +}) => { |
| 79 | + test.setTimeout(120_000); |
| 80 | + await gotoApp(page); |
| 81 | + await selectPreset(page, "llama3_8b"); |
| 82 | + await runAblation(page, "optimizer", ["adamw", "lion"], 4); |
| 83 | + |
| 84 | + const adamw = await rowFinal(page, "adamw"); |
| 85 | + const lion = await rowFinal(page, "lion"); |
| 86 | + |
| 87 | + expect(Number.isFinite(adamw) && Number.isFinite(lion)).toBe(true); |
| 88 | + |
| 89 | + // AdamW and Lion are fundamentally different update rules — sign-only |
| 90 | + // vs second-moment adaptive. Their final losses after 4 steps must |
| 91 | + // differ. V2 only counted rows, this is the math-divergence proof. |
| 92 | + expect(Math.abs(adamw - lion)).toBeGreaterThan(1e-4); |
| 93 | +}); |
0 commit comments