Skip to content

Commit 3f02343

Browse files
committed
feat(v7-a01): llama3_8b preset e2e at scaled hidden=512
Closes V7-A01 (cppmega-mlx-c30.1.5): per user redirect ('конструктор из кирпичиков'), pins that the llama3_8b preset's brick composition runs through stage_train at hidden=512 with finite losses + non-zero weight delta. Full H=4096 perf gate lives under V7-I05 (production-preset baseline). Tests (tests/v4/test_llama3_8b_scaled_e2e.py): 2/2 — * llama3_8b preset at H=512 runs through train(2) with finite losses + weight_delta > 0. * Preset composition exposes attention + mlp brick kinds.
1 parent 75e342b commit 3f02343

1 file changed

Lines changed: 70 additions & 0 deletions

File tree

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
"""V7-A01: llama3_8b preset instantiated end-to-end at scaled dims.
2+
3+
Per user redirect ("у нас же конструктор из кирпичиков"), the
4+
acceptance is that the llama3_8b preset's brick composition runs
5+
through stage_train at a realistic-but-mini scale (H=512 with
6+
multi-repeat). Hidden=4096 full-scale is HBM-bound on a single
7+
Mac and tracked as a separate perf gate (V7-I05).
8+
"""
9+
10+
from __future__ import annotations
11+
12+
import pytest
13+
14+
from cppmega_v4.architectures.presets import build_preset_specs
15+
from cppmega_v4.jsonrpc.schema import VerifyParams
16+
from cppmega_v4.runner import Pipeline, run_pipeline
17+
18+
19+
def _spec_from_preset(name: str, hidden: int = 512) -> VerifyParams:
20+
specs = build_preset_specs(name, hidden_size=hidden)
21+
return VerifyParams.model_validate({
22+
"graph": {
23+
"nodes": [
24+
{"id": f"n{i}", "kind": s["kind"],
25+
"params": s.get("params", {})}
26+
for i, s in enumerate(specs)
27+
],
28+
"edges": [
29+
{"src": f"n{i}", "dst": f"n{i + 1}"}
30+
for i in range(len(specs) - 1)
31+
],
32+
},
33+
"dim_env": {"B": 1, "S": 8, "H": hidden,
34+
"nh": max(2, hidden // 64),
35+
"nkv": max(1, hidden // 128),
36+
"head_dim": 64,
37+
"num_experts": 4, "top_k": 2},
38+
"loss": {"kind": "cross_entropy",
39+
"head_outputs": [f"n{len(specs) - 1}"]},
40+
"optim": {"kind": "adamw",
41+
"groups": [{"matcher": "all", "lr": 1e-3,
42+
"weight_decay": 0.01,
43+
"betas": [0.9, 0.95]}]},
44+
})
45+
46+
47+
def test_v7_a01_llama3_8b_preset_runs_at_h512():
48+
rep = run_pipeline(
49+
_spec_from_preset("llama3_8b", hidden=512),
50+
Pipeline.from_dict({
51+
"stages": ["parse", "verify_build_spec", "build_model",
52+
"train"],
53+
"stage_options": {"train": {"num_steps": 2}},
54+
}),
55+
)
56+
tr = next(s for s in rep.stages if s.name == "train")
57+
assert tr.status == "ok", f"train failed: {tr.error}"
58+
assert len(tr.extras["losses"]) == 2
59+
for L in tr.extras["losses"]:
60+
assert -1e10 < L < 1e10
61+
assert tr.extras["weight_delta_norm"] > 0
62+
63+
64+
def test_v7_a01_preset_compose_yields_known_brick_kinds():
65+
"""llama3_8b preset must produce attention + mlp bricks (the
66+
canonical LLaMA repeat unit)."""
67+
specs = build_preset_specs("llama3_8b", hidden_size=128)
68+
kinds = {s["kind"] for s in specs}
69+
assert "attention" in kinds
70+
assert "mlp" in kinds

0 commit comments

Comments
 (0)