Skip to content

Commit 9efbabc

Browse files
committed
feat(v4-9): hybrid optimizer bucket sizes in extras
Closes V4-9 / cppmega-mlx-9e7. Closes G8 from V4 audit (split was wire-only proven). Backend (stage_train): - When optimizer_kind === 'muon_adamw_hybrid', call split_param_groups on all_modules.parameters() to count parameter elements routed to Muon (2-D matmul) vs AdamW (1-D / 3-D+). - extras.muon_group_size + extras.adamw_group_size populated. - Both null for non-hybrid optimizers. E2E 23_hybrid_optimizer_split.spec.ts: UI selects muon_adamw_hybrid, Train, asserts both bucket sizes > 0 AND muon >= adamw (matmul weights dominate norm scalars + biases by orders of magnitude). 1/1 green; stage_train pytest 27/27 still passing.
1 parent 6225f20 commit 9efbabc

2 files changed

Lines changed: 60 additions & 0 deletions

File tree

cppmega_v4/runner/stages.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -457,6 +457,23 @@ def loss_fn(model: nn.Module, emb: mx.array, tgt: mx.array) -> mx.array:
457457
pass
458458
all_modules = nn.Sequential(*modules, lm_head)
459459
opt, optimizer_kind = _build_optimizer(spec_optim, lr)
460+
# V4-9: when hybrid, count params routed to each bucket so e2e can
461+
# prove the split predicate actually saw 2D vs 1D/3D parameters.
462+
muon_group_size: int | None = None
463+
adamw_group_size: int | None = None
464+
if optimizer_kind == "muon_adamw_hybrid":
465+
try:
466+
from cppmega_mlx.training.optimizers import split_param_groups
467+
muon_t, adamw_t = split_param_groups(
468+
all_modules.parameters())
469+
def _count(tree: Any) -> int:
470+
flat = dict(nn.utils.tree_flatten(tree))
471+
return sum(int(v.size) for v in flat.values()
472+
if hasattr(v, "size"))
473+
muon_group_size = _count(muon_t)
474+
adamw_group_size = _count(adamw_t)
475+
except Exception:
476+
pass
460477
loss_and_grad = nn.value_and_grad(all_modules, loss_fn)
461478

462479
losses: list[float] = []
@@ -552,6 +569,8 @@ def loss_fn(model: nn.Module, emb: mx.array, tgt: mx.array) -> mx.array:
552569
if getattr(ctx.spec, "loss", None) is not None
553570
else "cross_entropy"
554571
),
572+
"muon_group_size": muon_group_size,
573+
"adamw_group_size": adamw_group_size,
555574
"model_summary": _summarize_model(
556575
ctx.spec, optimizer_kind, schedule_kind_label),
557576
},
Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
// V4-9: muon_adamw_hybrid optimizer routes 2D matmul weights to Muon
2+
// and 1D/3D+ tensors to AdamW. extras.muon_group_size + adamw_group_size
3+
// must both be > 0 to prove the split predicate actually fired.
4+
5+
import { test, expect } from "@playwright/test";
6+
import { gotoApp, selectPreset, closeModal } from "../fixtures";
7+
import { readTrainExtras } from "../utils/train_extras";
8+
9+
test("V4-9: hybrid optimizer split reports non-empty muon + adamw buckets",
10+
async ({ page }) => {
11+
test.setTimeout(60_000);
12+
await gotoApp(page);
13+
await selectPreset(page, "llama3_8b");
14+
15+
await page.getByTestId("sidebar-tab-optim").click();
16+
await page.getByTestId("optim-kind").selectOption("muon_adamw_hybrid");
17+
await page.getByTestId("optim-apply").click();
18+
19+
await page.getByTestId("run-pipeline-toggle").click();
20+
await page.getByTestId("run-pipeline-train").click();
21+
const modal = page.getByTestId("run-result-modal");
22+
await modal.waitFor({ timeout: 60_000 });
23+
const extras = await readTrainExtras(page);
24+
25+
expect(extras.optimizer_kind).toBe("muon_adamw_hybrid");
26+
27+
const muonSize = parseInt(
28+
(await page.getByTestId("run-result-extras-train-muon_group_size")
29+
.textContent()) ?? "0", 10);
30+
const adamwSize = parseInt(
31+
(await page.getByTestId("run-result-extras-train-adamw_group_size")
32+
.textContent()) ?? "0", 10);
33+
34+
expect(muonSize).toBeGreaterThan(0);
35+
expect(adamwSize).toBeGreaterThan(0);
36+
// Sanity: Muon handles big matmul weights → its bucket should be
37+
// at least as large as the AdamW bucket (norm scalars + biases).
38+
expect(muonSize).toBeGreaterThanOrEqual(adamwSize);
39+
40+
await closeModal(page);
41+
});

0 commit comments

Comments
 (0)