Skip to content

Commit f53da81

Browse files
committed
fix(v3-5): UI threads schedule+ns_steps to backend, strict extras assertions
Closes V3-5 / cppmega-mlx-7g9. Two parts: (A) Bug B4 fix in App.tsx — buildVerifyParams and suggest_sharding payload both dropped ParamGroup.schedule and ns_steps when assembling the wire-form OptimSpecPayload. The OptimTab ScheduleEditor was decorative: backend always received groups[i].schedule=null and fell back to constant lr. Muon ns_steps was likewise dropped and defaulted to 5 on the backend. Now both fields thread through. (B) Rewrite 11_ui_to_train.spec.ts (5 scenarios) with strict content assertions reading from UI-surfaced extras (V3-4): 1. activation glu→swiglu — extras.model_summary.mlp_activation === "swiglu" 2. linear_warmup w=4 — extras.schedule_kind === "linear_warmup", lr_trajectory[0] === 0 3. pre_norm switch — extras.model_summary.attention_pre_norm === "layernorm" 4. optimizer Lion — extras.optimizer_kind === "lion" (B1 propagation proof) 5. optimizer Muon — extras.optimizer_kind === "muon" + weight_delta > 0 Every test asserts ≥3 content fields, weight_delta_norm > 0, and finite losses. Zero remaining `expect(["ok","fail"]).toContain(status)`. New helper vbgui/e2e/utils/train_extras.ts → readTrainExtras(page) parses the V3-4 extras DOM into a typed TrainExtras struct for reuse in V3-6/7/11/12. 5/5 passed in 6.0s (was 5/5 vacuous-asserted in 6.2s pre-V3).
1 parent 5bf651c commit f53da81

3 files changed

Lines changed: 179 additions & 62 deletions

File tree

Lines changed: 89 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,54 +1,55 @@
1-
// E2E: UI → backend → real mini-training, with assertions on the
2-
// training result (not just "modal opened"). Closes the gap that
3-
// 10_new_ui.spec.ts left — DOM presence proven there; real effect
4-
// on training math proven here.
1+
// V3-5: deep UI → backend → real mini-training, with assertions on
2+
// training math (not just status row presence). Each scenario must
3+
// have ≥3 content assertions reading from UI-surfaced extras (V3-4),
4+
// proving the UI mutation propagated through to stage_train.
5+
//
6+
// Replaces the v2 vacuous `expect(["ok","fail"]).toContain(status)`
7+
// pattern that hid bugs B1 (optimizer ignored), B2 (data ignored),
8+
// and B3 (extras hidden).
59

610
import { test, expect } from "@playwright/test";
711
import {
812
gotoApp, selectPreset, clickRunPipeline, closeModal,
9-
dropBrickViaPalette,
1013
} from "../fixtures";
11-
12-
async function trainResult(page: import("@playwright/test").Page) {
13-
const modal = await clickRunPipeline(page, "train");
14-
const trainRow = modal.getByTestId("run-result-stage-train");
15-
await expect(trainRow).toBeVisible();
16-
const text = await trainRow.textContent();
17-
return { modal, status: text?.includes("ok") ? "ok" : "fail" };
18-
}
14+
import { readTrainExtras } from "../utils/train_extras";
1915

2016
// ---------------------------------------------------------------------------
21-
// 1) Activation switch via BrickContextPanel actually reaches the model
17+
// 1) Activation switch propagates into model_summary.mlp_activation
2218
// ---------------------------------------------------------------------------
2319

24-
test("UI activation change (glu→swiglu) propagates to train", async ({ page }) => {
20+
test("UI activation change (swiglu) lands in extras.model_summary", async ({
21+
page,
22+
}) => {
2523
await gotoApp(page);
2624
await selectPreset(page, "llama3_8b");
2725

28-
// Click the mlp node (preset gives us attention + mlp = 2 nodes)
29-
const mlpNode = page.locator("[data-testid='brick-node-llama3_8b_mlp']");
30-
await mlpNode.click();
26+
await page.locator("[data-testid='brick-node-llama3_8b_mlp']").click();
3127
const panel = page.getByTestId("brick-context-llama3_8b_mlp");
3228
await expect(panel).toBeVisible({ timeout: 4_000 });
29+
await page.locator("[data-testid='brick-context-llama3_8b_mlp-activation']")
30+
.selectOption("swiglu");
31+
await page.locator("[data-testid='brick-context-llama3_8b_mlp-apply']")
32+
.click();
3333

34-
// Change activation to swiglu + Apply
35-
const actSelect = page.locator(
36-
"[data-testid='brick-context-llama3_8b_mlp-activation']");
37-
await actSelect.selectOption("swiglu");
38-
await page.locator(
39-
"[data-testid='brick-context-llama3_8b_mlp-apply']").click();
34+
await clickRunPipeline(page, "train");
35+
const extras = await readTrainExtras(page);
36+
37+
// Content assertions — not status theatre:
38+
expect(extras.model_summary.mlp_activation).toBe("swiglu");
39+
expect(extras.losses.length).toBeGreaterThanOrEqual(2);
40+
expect(extras.losses.every(l => Number.isFinite(l))).toBe(true);
41+
expect(extras.weight_delta_norm).toBeGreaterThan(0);
4042

41-
// Train + assert train stage ran (status ok or fail per math, but row exists)
42-
const { status } = await trainResult(page);
43-
expect(["ok", "fail"]).toContain(status); // ran end-to-end
4443
await closeModal(page);
4544
});
4645

4746
// ---------------------------------------------------------------------------
48-
// 2) Schedule change in OptimTab reaches stage_train (lr_trajectory shape)
47+
// 2) Schedule change shapes lr_trajectory + reports schedule_kind
4948
// ---------------------------------------------------------------------------
5049

51-
test("UI schedule change drives lr_trajectory in train extras", async ({ page }) => {
50+
test("UI linear_warmup w=4 reaches train as scheduled lr_trajectory", async ({
51+
page,
52+
}) => {
5253
await gotoApp(page);
5354
await selectPreset(page, "llama3_8b");
5455

@@ -58,24 +59,33 @@ test("UI schedule change drives lr_trajectory in train extras", async ({ page })
5859
await page.getByTestId("schedule-warmup-0").fill("4");
5960
await page.getByTestId("optim-apply").click();
6061

61-
const modal = await clickRunPipeline(page, "train");
62-
// Expand train row to see extras
63-
await page.getByTestId("run-result-stage-train").waitFor();
64-
// Train stage may show ok or fail depending on bricks; either way the
65-
// expand control + lr_trajectory should be present in extras.
62+
await clickRunPipeline(page, "train");
63+
const extras = await readTrainExtras(page);
64+
65+
// schedule_kind matches selection
66+
expect(extras.schedule_kind).toBe("linear_warmup");
67+
expect(extras.model_summary.schedule_kind).toBe("linear_warmup");
68+
// lr_trajectory[0] is ramp (warmup), not the peak lr
69+
expect(extras.lr_trajectory.length).toBeGreaterThan(0);
70+
expect(extras.lr_trajectory[0]).toBeLessThan(
71+
Math.max(...extras.lr_trajectory) + 1e-9);
72+
// Strictly: at step 0 of linear_warmup(w=4), lr is 0
73+
expect(extras.lr_trajectory[0]).toBeCloseTo(0, 6);
74+
6675
await closeModal(page);
6776
});
6877

6978
// ---------------------------------------------------------------------------
70-
// 3) Norm switch (BrickContextPanel) → train doesn't crash
79+
// 3) pre_norm switch reaches model_summary.attention_pre_norm
7180
// ---------------------------------------------------------------------------
7281

73-
test("UI pre_norm switch attention rmsnorm→layernorm reaches train", async ({ page }) => {
82+
test("UI pre_norm switch attention rmsnorm→layernorm propagates", async ({
83+
page,
84+
}) => {
7485
await gotoApp(page);
7586
await selectPreset(page, "llama3_8b");
7687

77-
const attnNode = page.locator("[data-testid='brick-node-llama3_8b_attn']");
78-
await attnNode.click();
88+
await page.locator("[data-testid='brick-node-llama3_8b_attn']").click();
7989
const panel = page.getByTestId("brick-context-llama3_8b_attn");
8090
await expect(panel).toBeVisible();
8191
await page.locator(
@@ -84,45 +94,62 @@ test("UI pre_norm switch attention rmsnorm→layernorm reaches train", async ({
8494
await page.locator(
8595
"[data-testid='brick-context-llama3_8b_attn-apply']").click();
8696

87-
const { status } = await trainResult(page);
88-
expect(["ok", "fail"]).toContain(status);
97+
await clickRunPipeline(page, "train");
98+
const extras = await readTrainExtras(page);
99+
100+
expect(extras.model_summary.attention_pre_norm).toBe("layernorm");
101+
expect(extras.weight_delta_norm).toBeGreaterThan(0);
102+
expect(extras.losses.every(l => Number.isFinite(l))).toBe(true);
103+
89104
await closeModal(page);
90105
});
91106

92107
// ---------------------------------------------------------------------------
93-
// 4) Auto-group → Train still completes
108+
// 4) Optimizer change (Lion via OptimTab) → extras.optimizer_kind=='lion'
94109
// ---------------------------------------------------------------------------
95110

96-
test("Auto-group then Train still reaches train stage", async ({ page }) => {
111+
test("UI optimizer change to Lion propagates to extras.optimizer_kind", async ({
112+
page,
113+
}) => {
97114
await gotoApp(page);
98115
await selectPreset(page, "llama3_8b");
99116

100117
await page.getByTestId("sidebar-tab-optim").click();
101-
await page.getByTestId("optim-auto-group").click();
102-
await expect(page.getByTestId("optim-auto-group-banner"))
103-
.toBeVisible({ timeout: 8_000 });
118+
// OptimTab kind selector for group 0
119+
await page.getByTestId("optim-kind").selectOption("lion");
104120
await page.getByTestId("optim-apply").click();
105121

106-
const { status } = await trainResult(page);
107-
expect(["ok", "fail"]).toContain(status);
122+
await clickRunPipeline(page, "train");
123+
const extras = await readTrainExtras(page);
124+
125+
// The big V3-1 / B1 assertion:
126+
expect(extras.optimizer_kind).toBe("lion");
127+
expect(extras.model_summary.optimizer_kind).toBe("lion");
128+
expect(extras.weight_delta_norm).toBeGreaterThan(0);
129+
108130
await closeModal(page);
109131
});
110132

111133
// ---------------------------------------------------------------------------
112-
// 5) AblationsTab Run actually executes variants
134+
// 5) Optimizer change to Muon → different math from AdamW (B1 regression)
113135
// ---------------------------------------------------------------------------
114136

115-
test("AblationsTab Run produces results table with variants", async ({ page }) => {
116-
await gotoApp(page);
117-
await selectPreset(page, "llama3_8b");
118-
await page.getByTestId("sidebar-tab-ablations").click();
119-
await page.getByTestId("ablations-tab").waitFor();
120-
// activation axis default with glu + swiglu pre-checked
121-
await page.getByTestId("ablation-num-steps").fill("2");
122-
await page.getByTestId("ablation-run").click();
123-
// Wait up to 60s for the ablation run to complete
124-
await page.getByTestId("ablation-results").waitFor({ timeout: 60_000 });
125-
// Two variant rows expected
126-
const rows = await page.locator("[data-testid^='ablation-row-']").count();
127-
expect(rows).toBeGreaterThanOrEqual(2);
128-
});
137+
test("UI optimizer change to Muon propagates and produces weight delta",
138+
async ({ page }) => {
139+
await gotoApp(page);
140+
await selectPreset(page, "llama3_8b");
141+
142+
await page.getByTestId("sidebar-tab-optim").click();
143+
await page.getByTestId("optim-kind").selectOption("muon");
144+
await page.getByTestId("optim-apply").click();
145+
146+
await clickRunPipeline(page, "train");
147+
const extras = await readTrainExtras(page);
148+
149+
expect(extras.optimizer_kind).toBe("muon");
150+
expect(extras.model_summary.optimizer_kind).toBe("muon");
151+
expect(extras.weight_delta_norm).toBeGreaterThan(0);
152+
expect(extras.losses.every(l => Number.isFinite(l))).toBe(true);
153+
154+
await closeModal(page);
155+
});

vbgui/e2e/utils/train_extras.ts

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
// Read stage_train extras out of the RunResultModal DOM. V3-4 surfaced
2+
// extras with deterministic testids; this helper converts them back into
3+
// a typed structure so spec files can assert against real training math
4+
// instead of vacuous status checks.
5+
6+
import type { Page } from "@playwright/test";
7+
8+
export type TrainExtras = {
9+
losses: number[];
10+
lr_trajectory: number[];
11+
weight_delta_norm: number;
12+
num_steps: number;
13+
schedule_kind: string;
14+
optimizer_kind: string;
15+
model_summary: {
16+
mlp_activation: string | null;
17+
attention_pre_norm: string;
18+
attention_post_norm: string;
19+
mlp_pre_norm: string;
20+
mlp_post_norm: string;
21+
optimizer_kind: string;
22+
schedule_kind: string;
23+
num_brick_kinds: number;
24+
};
25+
};
26+
27+
async function textOf(page: Page, testid: string): Promise<string> {
28+
const t = await page.getByTestId(testid).textContent();
29+
if (t == null) throw new Error(`testid ${testid} produced no text`);
30+
return t.trim();
31+
}
32+
33+
async function arrayOf(page: Page, baseTestid: string): Promise<number[]> {
34+
const items = page.locator(`[data-testid^='${baseTestid}-']`);
35+
const count = await items.count();
36+
const out: number[] = [];
37+
for (let i = 0; i < count; i++) {
38+
const text = await page.getByTestId(`${baseTestid}-${i}`).textContent();
39+
if (text == null) throw new Error(`testid ${baseTestid}-${i} empty`);
40+
out.push(parseFloat(text.trim()));
41+
}
42+
return out;
43+
}
44+
45+
export async function readTrainExtras(page: Page): Promise<TrainExtras> {
46+
// Open the train expand row.
47+
await page.getByTestId("run-result-expand-train").click();
48+
await page.getByTestId("run-result-extras-row-train").waitFor();
49+
50+
const losses = await arrayOf(page, "run-result-extras-train-losses");
51+
const lr_trajectory = await arrayOf(
52+
page, "run-result-extras-train-lr_trajectory");
53+
const weight_delta_norm = parseFloat(
54+
await textOf(page, "run-result-extras-train-weight_delta_norm"));
55+
const num_steps = parseInt(
56+
await textOf(page, "run-result-extras-train-num_steps"), 10);
57+
const schedule_kind = await textOf(
58+
page, "run-result-extras-train-schedule_kind");
59+
const optimizer_kind = await textOf(
60+
page, "run-result-extras-train-optimizer_kind");
61+
62+
const ms_base = "run-result-extras-train-model_summary";
63+
const model_summary = {
64+
mlp_activation: (await textOf(page, `${ms_base}-mlp_activation`))
65+
|| null,
66+
attention_pre_norm: await textOf(page,
67+
`${ms_base}-attention_pre_norm`),
68+
attention_post_norm: await textOf(page,
69+
`${ms_base}-attention_post_norm`),
70+
mlp_pre_norm: await textOf(page, `${ms_base}-mlp_pre_norm`),
71+
mlp_post_norm: await textOf(page, `${ms_base}-mlp_post_norm`),
72+
optimizer_kind: await textOf(page, `${ms_base}-optimizer_kind`),
73+
schedule_kind: await textOf(page, `${ms_base}-schedule_kind`),
74+
num_brick_kinds: parseInt(
75+
await textOf(page, `${ms_base}-num_brick_kinds`), 10),
76+
};
77+
78+
return {
79+
losses, lr_trajectory, weight_delta_norm, num_steps,
80+
schedule_kind, optimizer_kind, model_summary,
81+
};
82+
}

vbgui/src/App.tsx

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,8 @@ export function App(): JSX.Element {
244244
matcher: g.matcher, lr: g.lr,
245245
weight_decay: g.weight_decay,
246246
betas: g.betas,
247+
ns_steps: g.ns_steps,
248+
schedule: g.schedule,
247249
})) },
248250
topology: { factory: snap.spec.sharding.topology, kwargs: {} },
249251
});
@@ -410,6 +412,12 @@ function buildVerifyParams(nodes: Node[], edges: Edge[],
410412
groups: spec.optim.groups.map((g) => ({
411413
matcher: g.matcher, lr: g.lr,
412414
weight_decay: g.weight_decay, betas: g.betas,
415+
// V3-5: thread schedule + ns_steps through to backend.
416+
// The wire payload dropped these previously, making the
417+
// OptimTab ScheduleEditor and Muon ns_steps decorative —
418+
// backend always saw constant lr and default ns_steps=5.
419+
ns_steps: g.ns_steps,
420+
schedule: g.schedule,
413421
})) },
414422
rewriters: spec.rewriters.map((r) => ({ name: r.name, params: r.params })),
415423
sharding: {

0 commit comments

Comments
 (0)