Skip to content

Commit 90deea9

Browse files
committed
feat(h06): UI N=100 real-corpus long-run walk e2e
Closes V5-G15 from the UI side: drive a 100-step Train through the Visual Builder GUI on a real parquet + tokenizer (MATRIX.json T2_gpt2_small__P1_minimal + T2_gpt2_small) and assert the resulting train extras prove real training happened on real tokens. Assertions: - num_steps == 100, losses.length == 100, all finite - losses_smoothed tail-window mean < head-window mean - weight_delta_norm > 0.01 - inference_probe.l2_diff > 0.1 - data_source == "parquet_tokenized" Tagged `@long-running` so CI can opt-in. 180s budget. Tests: - vbgui e2e 60_long_run_ui.spec.ts: 1/1 passing - vbgui e2e V6 suite (55–60): 8/8 passing — no regression
1 parent 5516781 commit 90deea9

1 file changed

Lines changed: 107 additions & 0 deletions

File tree

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
// H06: UI N=100 long-run real-corpus walk through Visual Builder GUI.
2+
//
3+
// Closes V5-G15 "long-N convergence on real corpus" from the UI side:
4+
// pick the real parquet+tokenizer pair from MATRIX.json, set Train
5+
// steps to 100, dispatch Train through the TopBar, then assert that
6+
// the resulting extras prove the model actually trained for 100 steps
7+
// on tokens (not synthetic random), made non-trivial parameter
8+
// movement, and the inference probe distinguishes pre-vs-post weights.
9+
//
10+
// This is a long-running test (~30-90s) and is tagged @long-running so
11+
// CI can opt-in. Local runs include it by default.
12+
13+
import { test, expect } from "@playwright/test";
14+
import { gotoApp, selectPreset, clickTab, closeModal } from "../fixtures";
15+
import { loadMatrix } from "../utils/matrix";
16+
import { readTrainExtras } from "../utils/train_extras";
17+
18+
const MATRIX = loadMatrix();
19+
const REAL_PARQUET = MATRIX.parquets.T2_gpt2_small__P1_minimal.path;
20+
const REAL_TOKENIZER = MATRIX.tokenizers.T2_gpt2_small.path;
21+
22+
test("H06 @long-running: N=100 real-corpus walk through UI", async ({ page }) => {
23+
// 180s budget; plan target is 120s but Apple-Silicon CI variance is real.
24+
test.setTimeout(180_000);
25+
26+
await gotoApp(page);
27+
await selectPreset(page, "llama3_8b");
28+
29+
// Wire real parquet via Data tab.
30+
await clickTab(page, "data");
31+
await page.getByTestId("data-inspector").waitFor();
32+
await page.getByTestId("data-path").fill(REAL_PARQUET);
33+
await page.getByTestId("data-load").click();
34+
await page.getByTestId("data-metrics").waitFor({ timeout: 8_000 });
35+
await page.getByTestId("data-use-for-train").click();
36+
37+
// Wire real tokenizer via Tokenizer tab.
38+
await clickTab(page, "tokenizer");
39+
await page.getByTestId("tokenizer-playground").waitFor();
40+
await page.getByTestId("add-panel").click();
41+
await page.getByTestId("tokenizer-source-0").fill(REAL_TOKENIZER);
42+
await page.getByTestId("tokenizer-encode-0").click();
43+
await page.getByTestId("tokenizer-metrics-0").waitFor({ timeout: 8_000 });
44+
await page.getByTestId("tokenizer-use-for-train-0").click();
45+
46+
// Drive N=100 Train from TopBar Train menu on Canvas tab.
47+
await clickTab(page, "canvas");
48+
await page.getByTestId("run-pipeline-toggle").click();
49+
await page.getByTestId("train-num-steps").fill("100");
50+
await page.getByTestId("run-pipeline-train").click();
51+
const modal = page.getByTestId("run-result-modal");
52+
await modal.waitFor({ timeout: 150_000 });
53+
54+
const extras = await readTrainExtras(page);
55+
56+
// (H06.2) exactly 100 steps executed.
57+
expect(extras.num_steps).toBe(100);
58+
expect(extras.losses.length).toBe(100);
59+
expect(extras.losses.every((l) => Number.isFinite(l))).toBe(true);
60+
61+
// (H06.3) losses_smoothed final window < initial window
62+
// — the smoothed series strips per-step noise so a monotone-window
63+
// check is meaningful. We read the smoothed array directly.
64+
const smoothed = await arrayOf(
65+
page, "run-result-extras-train-losses_smoothed");
66+
expect(smoothed.length).toBe(100);
67+
const headWin = smoothed.slice(0, 20);
68+
const tailWin = smoothed.slice(-20);
69+
const headAvg = headWin.reduce((a, b) => a + b, 0) / headWin.length;
70+
const tailAvg = tailWin.reduce((a, b) => a + b, 0) / tailWin.length;
71+
expect(tailAvg).toBeLessThan(headAvg);
72+
73+
// (H06.4) weights moved meaningfully on a 100-step real-token run.
74+
expect(extras.weight_delta_norm).toBeGreaterThan(0.01);
75+
76+
// (H06.5) inference probe shows real divergence (100 steps is significant).
77+
const l2 = parseFloat(await textOf(
78+
page, "run-result-extras-train-inference_probe-l2_diff"));
79+
expect(l2).toBeGreaterThan(0.1);
80+
81+
// Real-data path activated (not synthetic).
82+
const dataSource = await page.getByTestId(
83+
"run-result-extras-train-data_source").textContent();
84+
expect(dataSource?.trim()).toBe("parquet_tokenized");
85+
86+
await closeModal(page);
87+
});
88+
89+
import type { Page } from "@playwright/test";
90+
91+
async function textOf(page: Page, testid: string): Promise<string> {
92+
const t = await page.getByTestId(testid).textContent();
93+
if (t == null) throw new Error(`testid ${testid} produced no text`);
94+
return t.trim();
95+
}
96+
97+
async function arrayOf(page: Page, base: string): Promise<number[]> {
98+
const items = page.locator(`[data-testid^='${base}-']`);
99+
const count = await items.count();
100+
const out: number[] = [];
101+
for (let i = 0; i < count; i++) {
102+
const t = await page.getByTestId(`${base}-${i}`).textContent();
103+
if (t == null) throw new Error(`testid ${base}-${i} empty`);
104+
out.push(parseFloat(t.trim()));
105+
}
106+
return out;
107+
}

0 commit comments

Comments
 (0)