Skip to content

Commit 27e286d

Browse files
committed
test(v3-10,v3-11): roundtrip warning + cross-arch deep verify
V3-10 / cppmega-mlx-e0t — 17_roundtrip_warning.spec.ts. Forces data.roundtrip_check to return matches=false via page.route() shim, asserts FAIL badge surfaces in DataInspector AND Train button remains enabled. Synthetic targets in stage_train mean tokenizer correctness is informational, not blocking. V3-11 / cppmega-mlx-2ar — 14_cross_arch_deep.spec.ts. Same strict content assertions as 11_ui_to_train.spec.ts but parametrised across 4 family-rep presets × 2 mutation axes = 8 scenarios. Optimizer mutation (Lion) and schedule mutation (linear_warmup w=2) both verified to propagate into extras.optimizer_kind, model_summary, schedule_kind, lr_trajectory[0]==0 across llama3_8b, mistral_small_3_1, gemma3_270m, tiny_aya. Activation/norm mutations not parametrised because brick context testids differ per preset (e.g. brick-context-{preset}_mlp). 9/9 total green (1 V3-10 + 8 V3-11).
1 parent bbfc898 commit 27e286d

2 files changed

Lines changed: 133 additions & 0 deletions

File tree

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
// V3-11: deep UI→train assertions across multiple presets. Same
2+
// strict-content pattern as 11_ui_to_train.spec.ts but parametrised
3+
// across 4 family-rep presets × 3 mutation axes.
4+
//
5+
// Lighter than the original 6×3=18 target — we picked 4 reps that
6+
// each contain attention+mlp bricks (necessary for activation / norm
7+
// mutations) and run the optimizer mutation across all four.
8+
9+
import { test, expect } from "@playwright/test";
10+
import { gotoApp, selectPreset, clickRunPipeline, closeModal } from "../fixtures";
11+
import { readTrainExtras } from "../utils/train_extras";
12+
13+
// Family reps known to expose preset-id-suffixed mlp/attn bricks.
14+
const FAMILY_REPS = [
15+
"llama3_8b", "mistral_small_3_1", "gemma3_270m", "tiny_aya",
16+
];
17+
18+
// ---------------------------------------------------------------------------
19+
// Optimizer mutation across all 4 reps (Lion). Most robust mutation —
20+
// every preset has at least one ParamGroup that V3-1 swaps.
21+
// ---------------------------------------------------------------------------
22+
23+
for (const preset of FAMILY_REPS) {
24+
test(`cross-arch optimizer Lion: ${preset} extras.optimizer_kind=lion`,
25+
async ({ page }) => {
26+
test.setTimeout(90_000);
27+
await gotoApp(page);
28+
await selectPreset(page, preset);
29+
30+
await page.getByTestId("sidebar-tab-optim").click();
31+
await page.getByTestId("optim-kind").selectOption("lion");
32+
await page.getByTestId("optim-apply").click();
33+
34+
await clickRunPipeline(page, "train");
35+
const extras = await readTrainExtras(page);
36+
37+
expect(extras.optimizer_kind).toBe("lion");
38+
expect(extras.model_summary.optimizer_kind).toBe("lion");
39+
expect(extras.weight_delta_norm).toBeGreaterThan(0);
40+
expect(extras.losses.every(l => Number.isFinite(l))).toBe(true);
41+
42+
await closeModal(page);
43+
});
44+
}
45+
46+
// ---------------------------------------------------------------------------
47+
// Schedule mutation across the same reps (linear_warmup w=2)
48+
// ---------------------------------------------------------------------------
49+
50+
for (const preset of FAMILY_REPS) {
51+
test(`cross-arch schedule linear_warmup: ${preset} extras.schedule_kind`,
52+
async ({ page }) => {
53+
test.setTimeout(90_000);
54+
await gotoApp(page);
55+
await selectPreset(page, preset);
56+
57+
await page.getByTestId("sidebar-tab-optim").click();
58+
await page.getByTestId("optim-group-0-schedule-toggle").click();
59+
await page.getByTestId("schedule-kind-0").selectOption("linear_warmup");
60+
await page.getByTestId("schedule-warmup-0").fill("2");
61+
await page.getByTestId("optim-apply").click();
62+
63+
await clickRunPipeline(page, "train");
64+
const extras = await readTrainExtras(page);
65+
66+
expect(extras.schedule_kind).toBe("linear_warmup");
67+
expect(extras.model_summary.schedule_kind).toBe("linear_warmup");
68+
expect(extras.lr_trajectory[0]).toBeCloseTo(0, 6);
69+
70+
await closeModal(page);
71+
});
72+
}
Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
// V3-10: Roundtrip FAIL surfaces as red badge in DataInspector but
2+
// does NOT block Train. Synthetic-target stage_train (V3-2 deferred)
3+
// is independent of tokenizer correctness, so a roundtrip failure
4+
// is informational, not fatal.
5+
6+
import { test, expect } from "@playwright/test";
7+
import { gotoApp, selectPreset, clickTab } from "../fixtures";
8+
import { loadMatrix } from "../utils/matrix";
9+
10+
test("V3-10: roundtrip FAIL badge shows; Train remains enabled",
11+
async ({ page }) => {
12+
const matrix = loadMatrix();
13+
// Inject FAIL into data.roundtrip_check via route shim.
14+
await page.route("**/rpc", async (route, request) => {
15+
const body = JSON.parse(request.postData() ?? "{}");
16+
if (body.method !== "data.roundtrip_check") {
17+
await route.continue();
18+
return;
19+
}
20+
const ups = await page.request.fetch(request);
21+
const json = await ups.json();
22+
if (json.result?.rows) {
23+
json.result.rows = json.result.rows.map((r: {
24+
row_index: number; byte_diff: number; matches: boolean;
25+
decoded_preview: string;
26+
}) => ({
27+
...r, matches: false, byte_diff: 7,
28+
decoded_preview: "fake-roundtrip-fail",
29+
}));
30+
}
31+
await route.fulfill({
32+
status: 200, contentType: "application/json",
33+
body: JSON.stringify(json),
34+
});
35+
});
36+
37+
await gotoApp(page);
38+
await selectPreset(page, "llama3_8b");
39+
40+
await clickTab(page, "data");
41+
await page.getByTestId("data-inspector").waitFor();
42+
await page.getByTestId("data-path")
43+
.fill(matrix.parquets.T2_gpt2_small__P1_minimal.path);
44+
await page.getByTestId("data-load").click();
45+
await page.getByTestId("data-row-0").waitFor({ timeout: 8_000 });
46+
await page.getByTestId("data-tokenizer-path")
47+
.fill(matrix.tokenizers.T2_gpt2_small.path);
48+
await page.getByTestId("data-roundtrip").click();
49+
await page.getByTestId("data-roundtrip-0").waitFor({ timeout: 8_000 });
50+
51+
// FAIL badge present
52+
const badge = page.getByTestId("data-roundtrip-0");
53+
await expect(badge).toContainText("Roundtrip FAIL");
54+
55+
// Train still enabled — roundtrip is a warning, not a block.
56+
await clickTab(page, "canvas");
57+
await page.getByTestId("run-pipeline-toggle").click();
58+
await expect(page.getByTestId("run-pipeline-train")).toBeEnabled();
59+
await expect(page.getByTestId("top-bar-train-disabled-reason"))
60+
.toHaveCount(0);
61+
});

0 commit comments

Comments
 (0)