|
| 1 | +// H06: UI N=100 long-run real-corpus walk through Visual Builder GUI. |
| 2 | +// |
| 3 | +// Closes V5-G15 "long-N convergence on real corpus" from the UI side: |
| 4 | +// pick the real parquet+tokenizer pair from MATRIX.json, set Train |
| 5 | +// steps to 100, dispatch Train through the TopBar, then assert that |
| 6 | +// the resulting extras prove the model actually trained for 100 steps |
| 7 | +// on tokens (not synthetic random), made non-trivial parameter |
| 8 | +// movement, and the inference probe distinguishes pre-vs-post weights. |
| 9 | +// |
| 10 | +// This is a long-running test (~30-90s) and is tagged @long-running so |
| 11 | +// CI can opt-in. Local runs include it by default. |
| 12 | + |
| 13 | +import { test, expect } from "@playwright/test"; |
| 14 | +import { gotoApp, selectPreset, clickTab, closeModal } from "../fixtures"; |
| 15 | +import { loadMatrix } from "../utils/matrix"; |
| 16 | +import { readTrainExtras } from "../utils/train_extras"; |
| 17 | + |
| 18 | +const MATRIX = loadMatrix(); |
| 19 | +const REAL_PARQUET = MATRIX.parquets.T2_gpt2_small__P1_minimal.path; |
| 20 | +const REAL_TOKENIZER = MATRIX.tokenizers.T2_gpt2_small.path; |
| 21 | + |
| 22 | +test("H06 @long-running: N=100 real-corpus walk through UI", async ({ page }) => { |
| 23 | + // 180s budget; plan target is 120s but Apple-Silicon CI variance is real. |
| 24 | + test.setTimeout(180_000); |
| 25 | + |
| 26 | + await gotoApp(page); |
| 27 | + await selectPreset(page, "llama3_8b"); |
| 28 | + |
| 29 | + // Wire real parquet via Data tab. |
| 30 | + await clickTab(page, "data"); |
| 31 | + await page.getByTestId("data-inspector").waitFor(); |
| 32 | + await page.getByTestId("data-path").fill(REAL_PARQUET); |
| 33 | + await page.getByTestId("data-load").click(); |
| 34 | + await page.getByTestId("data-metrics").waitFor({ timeout: 8_000 }); |
| 35 | + await page.getByTestId("data-use-for-train").click(); |
| 36 | + |
| 37 | + // Wire real tokenizer via Tokenizer tab. |
| 38 | + await clickTab(page, "tokenizer"); |
| 39 | + await page.getByTestId("tokenizer-playground").waitFor(); |
| 40 | + await page.getByTestId("add-panel").click(); |
| 41 | + await page.getByTestId("tokenizer-source-0").fill(REAL_TOKENIZER); |
| 42 | + await page.getByTestId("tokenizer-encode-0").click(); |
| 43 | + await page.getByTestId("tokenizer-metrics-0").waitFor({ timeout: 8_000 }); |
| 44 | + await page.getByTestId("tokenizer-use-for-train-0").click(); |
| 45 | + |
| 46 | + // Drive N=100 Train from TopBar Train menu on Canvas tab. |
| 47 | + await clickTab(page, "canvas"); |
| 48 | + await page.getByTestId("run-pipeline-toggle").click(); |
| 49 | + await page.getByTestId("train-num-steps").fill("100"); |
| 50 | + await page.getByTestId("run-pipeline-train").click(); |
| 51 | + const modal = page.getByTestId("run-result-modal"); |
| 52 | + await modal.waitFor({ timeout: 150_000 }); |
| 53 | + |
| 54 | + const extras = await readTrainExtras(page); |
| 55 | + |
| 56 | + // (H06.2) exactly 100 steps executed. |
| 57 | + expect(extras.num_steps).toBe(100); |
| 58 | + expect(extras.losses.length).toBe(100); |
| 59 | + expect(extras.losses.every((l) => Number.isFinite(l))).toBe(true); |
| 60 | + |
| 61 | + // (H06.3) losses_smoothed final window < initial window |
| 62 | + // — the smoothed series strips per-step noise so a monotone-window |
| 63 | + // check is meaningful. We read the smoothed array directly. |
| 64 | + const smoothed = await arrayOf( |
| 65 | + page, "run-result-extras-train-losses_smoothed"); |
| 66 | + expect(smoothed.length).toBe(100); |
| 67 | + const headWin = smoothed.slice(0, 20); |
| 68 | + const tailWin = smoothed.slice(-20); |
| 69 | + const headAvg = headWin.reduce((a, b) => a + b, 0) / headWin.length; |
| 70 | + const tailAvg = tailWin.reduce((a, b) => a + b, 0) / tailWin.length; |
| 71 | + expect(tailAvg).toBeLessThan(headAvg); |
| 72 | + |
| 73 | + // (H06.4) weights moved meaningfully on a 100-step real-token run. |
| 74 | + expect(extras.weight_delta_norm).toBeGreaterThan(0.01); |
| 75 | + |
| 76 | + // (H06.5) inference probe shows real divergence (100 steps is significant). |
| 77 | + const l2 = parseFloat(await textOf( |
| 78 | + page, "run-result-extras-train-inference_probe-l2_diff")); |
| 79 | + expect(l2).toBeGreaterThan(0.1); |
| 80 | + |
| 81 | + // Real-data path activated (not synthetic). |
| 82 | + const dataSource = await page.getByTestId( |
| 83 | + "run-result-extras-train-data_source").textContent(); |
| 84 | + expect(dataSource?.trim()).toBe("parquet_tokenized"); |
| 85 | + |
| 86 | + await closeModal(page); |
| 87 | +}); |
| 88 | + |
| 89 | +import type { Page } from "@playwright/test"; |
| 90 | + |
| 91 | +async function textOf(page: Page, testid: string): Promise<string> { |
| 92 | + const t = await page.getByTestId(testid).textContent(); |
| 93 | + if (t == null) throw new Error(`testid ${testid} produced no text`); |
| 94 | + return t.trim(); |
| 95 | +} |
| 96 | + |
| 97 | +async function arrayOf(page: Page, base: string): Promise<number[]> { |
| 98 | + const items = page.locator(`[data-testid^='${base}-']`); |
| 99 | + const count = await items.count(); |
| 100 | + const out: number[] = []; |
| 101 | + for (let i = 0; i < count; i++) { |
| 102 | + const t = await page.getByTestId(`${base}-${i}`).textContent(); |
| 103 | + if (t == null) throw new Error(`testid ${base}-${i} empty`); |
| 104 | + out.push(parseFloat(t.trim())); |
| 105 | + } |
| 106 | + return out; |
| 107 | +} |
0 commit comments