1- // E2E: UI → backend → real mini-training, with assertions on the
2- // training result (not just "modal opened"). Closes the gap that
3- // 10_new_ui.spec.ts left — DOM presence proven there; real effect
4- // on training math proven here.
1+ // V3-5: deep UI → backend → real mini-training, with assertions on
2+ // training math (not just status row presence). Each scenario must
3+ // have ≥3 content assertions reading from UI-surfaced extras (V3-4),
4+ // proving the UI mutation propagated through to stage_train.
5+ //
6+ // Replaces the v2 vacuous `expect(["ok","fail"]).toContain(status)`
7+ // pattern that hid bugs B1 (optimizer ignored), B2 (data ignored),
8+ // and B3 (extras hidden).
59
610import { test , expect } from "@playwright/test" ;
711import {
812 gotoApp , selectPreset , clickRunPipeline , closeModal ,
9- dropBrickViaPalette ,
1013} from "../fixtures" ;
11-
12- async function trainResult ( page : import ( "@playwright/test" ) . Page ) {
13- const modal = await clickRunPipeline ( page , "train" ) ;
14- const trainRow = modal . getByTestId ( "run-result-stage-train" ) ;
15- await expect ( trainRow ) . toBeVisible ( ) ;
16- const text = await trainRow . textContent ( ) ;
17- return { modal, status : text ?. includes ( "ok" ) ? "ok" : "fail" } ;
18- }
14+ import { readTrainExtras } from "../utils/train_extras" ;
1915
2016// ---------------------------------------------------------------------------
21- // 1) Activation switch via BrickContextPanel actually reaches the model
17+ // 1) Activation switch propagates into model_summary.mlp_activation
2218// ---------------------------------------------------------------------------
2319
24- test ( "UI activation change (glu→swiglu) propagates to train" , async ( { page } ) => {
20+ test ( "UI activation change (swiglu) lands in extras.model_summary" , async ( {
21+ page,
22+ } ) => {
2523 await gotoApp ( page ) ;
2624 await selectPreset ( page , "llama3_8b" ) ;
2725
28- // Click the mlp node (preset gives us attention + mlp = 2 nodes)
29- const mlpNode = page . locator ( "[data-testid='brick-node-llama3_8b_mlp']" ) ;
30- await mlpNode . click ( ) ;
26+ await page . locator ( "[data-testid='brick-node-llama3_8b_mlp']" ) . click ( ) ;
3127 const panel = page . getByTestId ( "brick-context-llama3_8b_mlp" ) ;
3228 await expect ( panel ) . toBeVisible ( { timeout : 4_000 } ) ;
29+ await page . locator ( "[data-testid='brick-context-llama3_8b_mlp-activation']" )
30+ . selectOption ( "swiglu" ) ;
31+ await page . locator ( "[data-testid='brick-context-llama3_8b_mlp-apply']" )
32+ . click ( ) ;
3333
34- // Change activation to swiglu + Apply
35- const actSelect = page . locator (
36- "[data-testid='brick-context-llama3_8b_mlp-activation']" ) ;
37- await actSelect . selectOption ( "swiglu" ) ;
38- await page . locator (
39- "[data-testid='brick-context-llama3_8b_mlp-apply']" ) . click ( ) ;
34+ await clickRunPipeline ( page , "train" ) ;
35+ const extras = await readTrainExtras ( page ) ;
36+
37+ // Content assertions — not status theatre:
38+ expect ( extras . model_summary . mlp_activation ) . toBe ( "swiglu" ) ;
39+ expect ( extras . losses . length ) . toBeGreaterThanOrEqual ( 2 ) ;
40+ expect ( extras . losses . every ( l => Number . isFinite ( l ) ) ) . toBe ( true ) ;
41+ expect ( extras . weight_delta_norm ) . toBeGreaterThan ( 0 ) ;
4042
41- // Train + assert train stage ran (status ok or fail per math, but row exists)
42- const { status } = await trainResult ( page ) ;
43- expect ( [ "ok" , "fail" ] ) . toContain ( status ) ; // ran end-to-end
4443 await closeModal ( page ) ;
4544} ) ;
4645
4746// ---------------------------------------------------------------------------
48- // 2) Schedule change in OptimTab reaches stage_train (lr_trajectory shape)
47+ // 2) Schedule change shapes lr_trajectory + reports schedule_kind
4948// ---------------------------------------------------------------------------
5049
51- test ( "UI schedule change drives lr_trajectory in train extras" , async ( { page } ) => {
50+ test ( "UI linear_warmup w=4 reaches train as scheduled lr_trajectory" , async ( {
51+ page,
52+ } ) => {
5253 await gotoApp ( page ) ;
5354 await selectPreset ( page , "llama3_8b" ) ;
5455
@@ -58,24 +59,33 @@ test("UI schedule change drives lr_trajectory in train extras", async ({ page })
5859 await page . getByTestId ( "schedule-warmup-0" ) . fill ( "4" ) ;
5960 await page . getByTestId ( "optim-apply" ) . click ( ) ;
6061
61- const modal = await clickRunPipeline ( page , "train" ) ;
62- // Expand train row to see extras
63- await page . getByTestId ( "run-result-stage-train" ) . waitFor ( ) ;
64- // Train stage may show ok or fail depending on bricks; either way the
65- // expand control + lr_trajectory should be present in extras.
62+ await clickRunPipeline ( page , "train" ) ;
63+ const extras = await readTrainExtras ( page ) ;
64+
65+ // schedule_kind matches selection
66+ expect ( extras . schedule_kind ) . toBe ( "linear_warmup" ) ;
67+ expect ( extras . model_summary . schedule_kind ) . toBe ( "linear_warmup" ) ;
68+ // lr_trajectory[0] is ramp (warmup), not the peak lr
69+ expect ( extras . lr_trajectory . length ) . toBeGreaterThan ( 0 ) ;
70+ expect ( extras . lr_trajectory [ 0 ] ) . toBeLessThan (
71+ Math . max ( ...extras . lr_trajectory ) + 1e-9 ) ;
72+ // Strictly: at step 0 of linear_warmup(w=4), lr is 0
73+ expect ( extras . lr_trajectory [ 0 ] ) . toBeCloseTo ( 0 , 6 ) ;
74+
6675 await closeModal ( page ) ;
6776} ) ;
6877
6978// ---------------------------------------------------------------------------
70- // 3) Norm switch (BrickContextPanel) → train doesn't crash
79+ // 3) pre_norm switch reaches model_summary.attention_pre_norm
7180// ---------------------------------------------------------------------------
7281
73- test ( "UI pre_norm switch attention rmsnorm→layernorm reaches train" , async ( { page } ) => {
82+ test ( "UI pre_norm switch attention rmsnorm→layernorm propagates" , async ( {
83+ page,
84+ } ) => {
7485 await gotoApp ( page ) ;
7586 await selectPreset ( page , "llama3_8b" ) ;
7687
77- const attnNode = page . locator ( "[data-testid='brick-node-llama3_8b_attn']" ) ;
78- await attnNode . click ( ) ;
88+ await page . locator ( "[data-testid='brick-node-llama3_8b_attn']" ) . click ( ) ;
7989 const panel = page . getByTestId ( "brick-context-llama3_8b_attn" ) ;
8090 await expect ( panel ) . toBeVisible ( ) ;
8191 await page . locator (
@@ -84,45 +94,62 @@ test("UI pre_norm switch attention rmsnorm→layernorm reaches train", async ({
8494 await page . locator (
8595 "[data-testid='brick-context-llama3_8b_attn-apply']" ) . click ( ) ;
8696
87- const { status } = await trainResult ( page ) ;
88- expect ( [ "ok" , "fail" ] ) . toContain ( status ) ;
97+ await clickRunPipeline ( page , "train" ) ;
98+ const extras = await readTrainExtras ( page ) ;
99+
100+ expect ( extras . model_summary . attention_pre_norm ) . toBe ( "layernorm" ) ;
101+ expect ( extras . weight_delta_norm ) . toBeGreaterThan ( 0 ) ;
102+ expect ( extras . losses . every ( l => Number . isFinite ( l ) ) ) . toBe ( true ) ;
103+
89104 await closeModal ( page ) ;
90105} ) ;
91106
92107// ---------------------------------------------------------------------------
93- // 4) Auto-group → Train still completes
108+ // 4) Optimizer change (Lion via OptimTab) → extras.optimizer_kind=='lion'
94109// ---------------------------------------------------------------------------
95110
96- test ( "Auto-group then Train still reaches train stage" , async ( { page } ) => {
111+ test ( "UI optimizer change to Lion propagates to extras.optimizer_kind" , async ( {
112+ page,
113+ } ) => {
97114 await gotoApp ( page ) ;
98115 await selectPreset ( page , "llama3_8b" ) ;
99116
100117 await page . getByTestId ( "sidebar-tab-optim" ) . click ( ) ;
101- await page . getByTestId ( "optim-auto-group" ) . click ( ) ;
102- await expect ( page . getByTestId ( "optim-auto-group-banner" ) )
103- . toBeVisible ( { timeout : 8_000 } ) ;
118+ // OptimTab kind selector for group 0
119+ await page . getByTestId ( "optim-kind" ) . selectOption ( "lion" ) ;
104120 await page . getByTestId ( "optim-apply" ) . click ( ) ;
105121
106- const { status } = await trainResult ( page ) ;
107- expect ( [ "ok" , "fail" ] ) . toContain ( status ) ;
122+ await clickRunPipeline ( page , "train" ) ;
123+ const extras = await readTrainExtras ( page ) ;
124+
125+ // The big V3-1 / B1 assertion:
126+ expect ( extras . optimizer_kind ) . toBe ( "lion" ) ;
127+ expect ( extras . model_summary . optimizer_kind ) . toBe ( "lion" ) ;
128+ expect ( extras . weight_delta_norm ) . toBeGreaterThan ( 0 ) ;
129+
108130 await closeModal ( page ) ;
109131} ) ;
110132
111133// ---------------------------------------------------------------------------
112- // 5) AblationsTab Run actually executes variants
134+ // 5) Optimizer change to Muon → different math from AdamW (B1 regression)
113135// ---------------------------------------------------------------------------
114136
115- test ( "AblationsTab Run produces results table with variants" , async ( { page } ) => {
116- await gotoApp ( page ) ;
117- await selectPreset ( page , "llama3_8b" ) ;
118- await page . getByTestId ( "sidebar-tab-ablations" ) . click ( ) ;
119- await page . getByTestId ( "ablations-tab" ) . waitFor ( ) ;
120- // activation axis default with glu + swiglu pre-checked
121- await page . getByTestId ( "ablation-num-steps" ) . fill ( "2" ) ;
122- await page . getByTestId ( "ablation-run" ) . click ( ) ;
123- // Wait up to 60s for the ablation run to complete
124- await page . getByTestId ( "ablation-results" ) . waitFor ( { timeout : 60_000 } ) ;
125- // Two variant rows expected
126- const rows = await page . locator ( "[data-testid^='ablation-row-']" ) . count ( ) ;
127- expect ( rows ) . toBeGreaterThanOrEqual ( 2 ) ;
128- } ) ;
137+ test ( "UI optimizer change to Muon propagates and produces weight delta" ,
138+ async ( { page } ) => {
139+ await gotoApp ( page ) ;
140+ await selectPreset ( page , "llama3_8b" ) ;
141+
142+ await page . getByTestId ( "sidebar-tab-optim" ) . click ( ) ;
143+ await page . getByTestId ( "optim-kind" ) . selectOption ( "muon" ) ;
144+ await page . getByTestId ( "optim-apply" ) . click ( ) ;
145+
146+ await clickRunPipeline ( page , "train" ) ;
147+ const extras = await readTrainExtras ( page ) ;
148+
149+ expect ( extras . optimizer_kind ) . toBe ( "muon" ) ;
150+ expect ( extras . model_summary . optimizer_kind ) . toBe ( "muon" ) ;
151+ expect ( extras . weight_delta_norm ) . toBeGreaterThan ( 0 ) ;
152+ expect ( extras . losses . every ( l => Number . isFinite ( l ) ) ) . toBe ( true ) ;
153+
154+ await closeModal ( page ) ;
155+ } ) ;
0 commit comments