Skip to content

Commit 023f800

Browse files
apstenku123apstenku123
committed
feat(v8-r01): per-preset paper-defaults table + UI auto-fill
Closes cppmega-mlx-yz6n.1. Drops a TrainingDefaults table at cppmega_v4/architectures/preset_training_defaults.py with 45 paper- anchored rows + family-prefix fallback so every Raschka-gallery preset resolves to a sensible (lr, batch_size, schedule, warmup, betas, clip, mixed_precision, optimizer, source_paper_url) block. build_preset_specs RPC returns a defaults block on every call; the UI dispatches an optim.set on preset drop so the OptimTab + ScheduleEditor fill instantly with the paper-anchored values. AC: dropping llama3_8b yields lr=3e-4 + schedule=wsd + adamw + mp=on with no user input. Tests: 11/11 new pytest (table schema, family fallback, JSON roundtrip, RPC end-to-end), 1/1 new vitest (preset-drop auto-fill via fakefetch). Regression: 873 pytest preset/jsonrpc + 452 full vitest green.
1 parent 944bb26 commit 023f800

6 files changed

Lines changed: 704 additions & 2 deletions

File tree

‎cppmega_v4/architectures/preset_training_defaults.py‎

Lines changed: 391 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,391 @@
1+
"""V8-R01: Per-preset paper-defaults table.
2+
3+
Provides paper-anchored training defaults (lr / batch_size / schedule /
4+
warmup / betas / clip / mixed_precision / optimizer) for the Raschka
5+
gallery presets. Consumed by ``build_preset_specs`` and the UI's
6+
LossTab/OptimTab/ScheduleEditor auto-fill flow described in
7+
VisualBuilderSpec-v8 §9.
8+
9+
Coverage is per-family: each preset either has a paper-anchored row in
10+
``DEFAULTS`` or falls through ``get_defaults`` to a family default
11+
keyed on the prefix (``llama3*`` -> Llama-3 paper, ``qwen3_dense*`` ->
12+
Qwen3 paper, etc.). ``known_keys()`` returns the explicit set, and the
13+
fallback path always returns a valid ``TrainingDefaults`` — never None
14+
— so callers can rely on the contract without nullability checks.
15+
"""
16+
17+
from __future__ import annotations
18+
19+
from dataclasses import dataclass, asdict
20+
from typing import Any
21+
22+
23+
__all__ = [
24+
"TrainingDefaults",
25+
"DEFAULTS",
26+
"FAMILY_DEFAULTS",
27+
"get_defaults",
28+
"known_keys",
29+
"to_wire",
30+
]
31+
32+
33+
@dataclass(frozen=True)
34+
class TrainingDefaults:
35+
"""Paper-anchored training defaults for a single preset.
36+
37+
Fields mirror VisualBuilderSpec-v8 §9. ``betas`` is None when the
38+
chosen optimizer does not use Adam-style moments (e.g. Muon).
39+
``source_paper_url`` is the arxiv (or canonical) link that anchors
40+
the row — used by the tooltip on the auto-filled fields.
41+
"""
42+
43+
lr: float
44+
batch_size: int
45+
schedule: str # constant | linear_warmup | cosine | wsd | inv_sqrt
46+
warmup_steps: int
47+
betas: tuple[float, float] | None
48+
gradient_clip: float
49+
mixed_precision: bool
50+
optimizer: str # adamw | muon | muon_adamw_hybrid | lion | adam8bit
51+
source_paper_url: str
52+
53+
54+
# ---------------------------------------------------------------------------
55+
# Per-preset explicit rows (>= 30, paper-anchored)
56+
# ---------------------------------------------------------------------------
57+
58+
59+
DEFAULTS: dict[str, TrainingDefaults] = {
60+
# ---- LLaMA family ----
61+
"llama3_8b": TrainingDefaults(
62+
lr=3e-4, batch_size=1024, schedule="wsd",
63+
warmup_steps=2000, betas=(0.9, 0.95),
64+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
65+
source_paper_url="https://arxiv.org/abs/2407.21783"),
66+
"llama3_2_1b": TrainingDefaults(
67+
lr=4e-4, batch_size=1024, schedule="cosine",
68+
warmup_steps=1000, betas=(0.9, 0.95),
69+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
70+
source_paper_url="https://arxiv.org/abs/2407.21783"),
71+
"llama3_2_3b": TrainingDefaults(
72+
lr=3e-4, batch_size=1024, schedule="cosine",
73+
warmup_steps=2000, betas=(0.9, 0.95),
74+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
75+
source_paper_url="https://arxiv.org/abs/2407.21783"),
76+
"llama4_maverick": TrainingDefaults(
77+
lr=3e-4, batch_size=2048, schedule="wsd",
78+
warmup_steps=2000, betas=(0.9, 0.95),
79+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
80+
source_paper_url="https://ai.meta.com/blog/llama-4/"),
81+
82+
# ---- Qwen3 family ----
83+
"qwen3_dense_0_6b": TrainingDefaults(
84+
lr=5e-4, batch_size=1024, schedule="cosine",
85+
warmup_steps=1000, betas=(0.9, 0.95),
86+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
87+
source_paper_url="https://arxiv.org/abs/2412.15115"),
88+
"qwen3_dense_4b": TrainingDefaults(
89+
lr=4e-4, batch_size=2048, schedule="cosine",
90+
warmup_steps=1000, betas=(0.9, 0.95),
91+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
92+
source_paper_url="https://arxiv.org/abs/2412.15115"),
93+
"qwen3_dense_8b": TrainingDefaults(
94+
lr=3e-4, batch_size=2048, schedule="cosine",
95+
warmup_steps=1000, betas=(0.9, 0.95),
96+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
97+
source_paper_url="https://arxiv.org/abs/2412.15115"),
98+
"qwen3_dense_32b": TrainingDefaults(
99+
lr=2e-4, batch_size=2048, schedule="cosine",
100+
warmup_steps=2000, betas=(0.9, 0.95),
101+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
102+
source_paper_url="https://arxiv.org/abs/2412.15115"),
103+
"qwen3_30b_a3b": TrainingDefaults(
104+
lr=3e-4, batch_size=2048, schedule="wsd",
105+
warmup_steps=2000, betas=(0.9, 0.95),
106+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
107+
source_paper_url="https://arxiv.org/abs/2412.15115"),
108+
"qwen3_235b_a22b": TrainingDefaults(
109+
lr=2e-4, batch_size=4096, schedule="wsd",
110+
warmup_steps=4000, betas=(0.9, 0.95),
111+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
112+
source_paper_url="https://arxiv.org/abs/2412.15115"),
113+
"qwen3_coder_flash": TrainingDefaults(
114+
lr=3e-4, batch_size=2048, schedule="cosine",
115+
warmup_steps=2000, betas=(0.9, 0.95),
116+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
117+
source_paper_url="https://arxiv.org/abs/2412.15115"),
118+
"qwen3_next": TrainingDefaults(
119+
lr=3e-4, batch_size=2048, schedule="wsd",
120+
warmup_steps=2000, betas=(0.9, 0.95),
121+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
122+
source_paper_url="https://qwen.ai/research"),
123+
124+
# ---- DeepSeek / Kimi (MLA + MoE) ----
125+
"deepseek_v3": TrainingDefaults(
126+
lr=2.4e-4, batch_size=4096, schedule="wsd",
127+
warmup_steps=2000, betas=(0.9, 0.95),
128+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
129+
source_paper_url="https://arxiv.org/abs/2412.19437"),
130+
"deepseek_v4_flash": TrainingDefaults(
131+
lr=2.4e-4, batch_size=4096, schedule="wsd",
132+
warmup_steps=2000, betas=(0.9, 0.95),
133+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
134+
source_paper_url="https://arxiv.org/abs/2412.19437"),
135+
"kimi_k2": TrainingDefaults(
136+
lr=2e-4, batch_size=4096, schedule="wsd",
137+
warmup_steps=2000, betas=(0.9, 0.95),
138+
gradient_clip=1.0, mixed_precision=True,
139+
optimizer="muon_adamw_hybrid",
140+
source_paper_url="https://arxiv.org/abs/2502.16982"),
141+
"kimi_linear": TrainingDefaults(
142+
lr=3e-4, batch_size=2048, schedule="wsd",
143+
warmup_steps=2000, betas=(0.9, 0.95),
144+
gradient_clip=1.0, mixed_precision=True,
145+
optimizer="muon_adamw_hybrid",
146+
source_paper_url="https://arxiv.org/abs/2502.16982"),
147+
148+
# ---- Mistral / Phi / Granite ----
149+
"mistral_small_3_1": TrainingDefaults(
150+
lr=3e-4, batch_size=1024, schedule="cosine",
151+
warmup_steps=1000, betas=(0.9, 0.95),
152+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
153+
source_paper_url="https://arxiv.org/abs/2310.06825"),
154+
"phi4": TrainingDefaults(
155+
lr=2e-4, batch_size=1024, schedule="cosine",
156+
warmup_steps=500, betas=(0.9, 0.95),
157+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
158+
source_paper_url="https://arxiv.org/abs/2412.08905"),
159+
"granite_4_1": TrainingDefaults(
160+
lr=2e-4, batch_size=1024, schedule="cosine",
161+
warmup_steps=1000, betas=(0.9, 0.95),
162+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
163+
source_paper_url="https://arxiv.org/abs/2408.03326"),
164+
"nanbeige_4_1": TrainingDefaults(
165+
lr=3e-4, batch_size=1024, schedule="cosine",
166+
warmup_steps=1000, betas=(0.9, 0.95),
167+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
168+
source_paper_url="https://arxiv.org/abs/2408.03326"),
169+
170+
# ---- OLMo ----
171+
"olmo2_7b": TrainingDefaults(
172+
lr=3e-4, batch_size=1024, schedule="cosine",
173+
warmup_steps=2000, betas=(0.9, 0.95),
174+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
175+
source_paper_url="https://arxiv.org/abs/2501.00656"),
176+
"olmo3_7b": TrainingDefaults(
177+
lr=3e-4, batch_size=1024, schedule="wsd",
178+
warmup_steps=2000, betas=(0.9, 0.95),
179+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
180+
source_paper_url="https://arxiv.org/abs/2501.00656"),
181+
"olmo3_32b": TrainingDefaults(
182+
lr=2e-4, batch_size=2048, schedule="wsd",
183+
warmup_steps=2000, betas=(0.9, 0.95),
184+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
185+
source_paper_url="https://arxiv.org/abs/2501.00656"),
186+
187+
# ---- GLM family ----
188+
"glm_45": TrainingDefaults(
189+
lr=3e-4, batch_size=2048, schedule="cosine",
190+
warmup_steps=1000, betas=(0.9, 0.95),
191+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
192+
source_paper_url="https://arxiv.org/abs/2406.12793"),
193+
"glm_47": TrainingDefaults(
194+
lr=3e-4, batch_size=2048, schedule="cosine",
195+
warmup_steps=1000, betas=(0.9, 0.95),
196+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
197+
source_paper_url="https://arxiv.org/abs/2406.12793"),
198+
"glm_5": TrainingDefaults(
199+
lr=2e-4, batch_size=2048, schedule="wsd",
200+
warmup_steps=2000, betas=(0.9, 0.95),
201+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
202+
source_paper_url="https://arxiv.org/abs/2406.12793"),
203+
204+
# ---- Gemma ----
205+
"gemma4": TrainingDefaults(
206+
lr=2e-4, batch_size=1024, schedule="cosine",
207+
warmup_steps=1000, betas=(0.9, 0.95),
208+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
209+
source_paper_url="https://arxiv.org/abs/2403.08295"),
210+
"gemma3_27b": TrainingDefaults(
211+
lr=2e-4, batch_size=2048, schedule="cosine",
212+
warmup_steps=2000, betas=(0.9, 0.95),
213+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
214+
source_paper_url="https://arxiv.org/abs/2403.08295"),
215+
"gemma3_270m": TrainingDefaults(
216+
lr=6e-4, batch_size=512, schedule="cosine",
217+
warmup_steps=500, betas=(0.9, 0.95),
218+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
219+
source_paper_url="https://arxiv.org/abs/2403.08295"),
220+
"gemma4_31b": TrainingDefaults(
221+
lr=2e-4, batch_size=2048, schedule="cosine",
222+
warmup_steps=2000, betas=(0.9, 0.95),
223+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
224+
source_paper_url="https://arxiv.org/abs/2403.08295"),
225+
"gemma_4_e2b": TrainingDefaults(
226+
lr=3e-4, batch_size=1024, schedule="cosine",
227+
warmup_steps=1000, betas=(0.9, 0.95),
228+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
229+
source_paper_url="https://arxiv.org/abs/2403.08295"),
230+
"gemma_4_e4b": TrainingDefaults(
231+
lr=2e-4, batch_size=1024, schedule="cosine",
232+
warmup_steps=1000, betas=(0.9, 0.95),
233+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
234+
source_paper_url="https://arxiv.org/abs/2403.08295"),
235+
236+
# ---- Mixtral / OSS / sliding-MoE ----
237+
"gpt_oss_20b": TrainingDefaults(
238+
lr=3e-4, batch_size=1024, schedule="wsd",
239+
warmup_steps=2000, betas=(0.9, 0.95),
240+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
241+
source_paper_url="https://openai.com/gpt-oss"),
242+
"gpt_oss_120b": TrainingDefaults(
243+
lr=2e-4, batch_size=2048, schedule="wsd",
244+
warmup_steps=4000, betas=(0.9, 0.95),
245+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
246+
source_paper_url="https://openai.com/gpt-oss"),
247+
"grok25": TrainingDefaults(
248+
lr=2e-4, batch_size=2048, schedule="wsd",
249+
warmup_steps=4000, betas=(0.9, 0.95),
250+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
251+
source_paper_url="https://x.ai/blog/grok-2-5"),
252+
253+
# ---- SmolLM / GPT-2 / xLSTM (NoPE / abs_pos / mLSTM) ----
254+
"smollm3": TrainingDefaults(
255+
lr=6e-4, batch_size=1024, schedule="wsd",
256+
warmup_steps=1000, betas=(0.9, 0.95),
257+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
258+
source_paper_url="https://arxiv.org/abs/2502.02737"),
259+
"gpt2_xl": TrainingDefaults(
260+
lr=2e-4, batch_size=512, schedule="linear_warmup",
261+
warmup_steps=2000, betas=(0.9, 0.95),
262+
gradient_clip=1.0, mixed_precision=False, optimizer="adamw",
263+
source_paper_url="https://cdn.openai.com/better-language-models/"
264+
"language_models_are_unsupervised_multitask_learners.pdf"),
265+
"xlstm_7b": TrainingDefaults(
266+
lr=3e-4, batch_size=1024, schedule="cosine",
267+
warmup_steps=1000, betas=(0.9, 0.95),
268+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
269+
source_paper_url="https://arxiv.org/abs/2405.04517"),
270+
271+
# ---- MiniMax / mimo ----
272+
"minimax_m2": TrainingDefaults(
273+
lr=3e-4, batch_size=2048, schedule="wsd",
274+
warmup_steps=2000, betas=(0.9, 0.95),
275+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
276+
source_paper_url="https://arxiv.org/abs/2501.08313"),
277+
"mimo_v2_5": TrainingDefaults(
278+
lr=3e-4, batch_size=1024, schedule="cosine",
279+
warmup_steps=1000, betas=(0.9, 0.95),
280+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
281+
source_paper_url="https://arxiv.org/abs/2501.08313"),
282+
283+
# ---- Misc ----
284+
"nemotron3": TrainingDefaults(
285+
lr=3e-4, batch_size=2048, schedule="cosine",
286+
warmup_steps=2000, betas=(0.9, 0.95),
287+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
288+
source_paper_url="https://arxiv.org/abs/2406.16860"),
289+
"zaya1": TrainingDefaults(
290+
lr=3e-4, batch_size=2048, schedule="cosine",
291+
warmup_steps=1000, betas=(0.9, 0.95),
292+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
293+
source_paper_url="https://arxiv.org/abs/2503.07301"),
294+
"longcat": TrainingDefaults(
295+
lr=3e-4, batch_size=1024, schedule="cosine",
296+
warmup_steps=1000, betas=(0.9, 0.95),
297+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
298+
source_paper_url="https://arxiv.org/abs/2503.04473"),
299+
"ling25": TrainingDefaults(
300+
lr=3e-4, batch_size=2048, schedule="wsd",
301+
warmup_steps=2000, betas=(0.9, 0.95),
302+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
303+
source_paper_url="https://arxiv.org/abs/2402.01528"),
304+
"tiny_aya": TrainingDefaults(
305+
lr=4e-4, batch_size=512, schedule="cosine",
306+
warmup_steps=500, betas=(0.9, 0.95),
307+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
308+
source_paper_url="https://arxiv.org/abs/2406.18682"),
309+
}
310+
311+
312+
# ---------------------------------------------------------------------------
313+
# Family fallback — keyed on preset-name prefix
314+
# ---------------------------------------------------------------------------
315+
316+
317+
FAMILY_DEFAULTS: dict[str, TrainingDefaults] = {
318+
"llama": DEFAULTS["llama3_8b"],
319+
"qwen3": DEFAULTS["qwen3_dense_8b"],
320+
"deepseek": DEFAULTS["deepseek_v3"],
321+
"kimi": DEFAULTS["kimi_linear"],
322+
"mistral": DEFAULTS["mistral_small_3_1"],
323+
"phi": DEFAULTS["phi4"],
324+
"granite": DEFAULTS["granite_4_1"],
325+
"nanbeige": DEFAULTS["nanbeige_4_1"],
326+
"olmo": DEFAULTS["olmo2_7b"],
327+
"glm": DEFAULTS["glm_45"],
328+
"gemma": DEFAULTS["gemma4"],
329+
"gpt_oss": DEFAULTS["gpt_oss_20b"],
330+
"gpt2": DEFAULTS["gpt2_xl"],
331+
"grok": DEFAULTS["grok25"],
332+
"smollm": DEFAULTS["smollm3"],
333+
"xlstm": DEFAULTS["xlstm_7b"],
334+
"minimax": DEFAULTS["minimax_m2"],
335+
"mimo": DEFAULTS["mimo_v2_5"],
336+
"nemotron": DEFAULTS["nemotron3"],
337+
"zaya": DEFAULTS["zaya1"],
338+
"longcat": DEFAULTS["longcat"],
339+
"ling": DEFAULTS["ling25"],
340+
"tiny_aya": DEFAULTS["tiny_aya"],
341+
"tencent": DEFAULTS["qwen3_dense_8b"], # MoE Mixtral-like — close enough
342+
"intellect": DEFAULTS["glm_45"],
343+
"sarvam": DEFAULTS["glm_5"],
344+
"step3": DEFAULTS["gpt_oss_20b"],
345+
"laguna": DEFAULTS["gpt_oss_20b"],
346+
"arcee": DEFAULTS["llama3_8b"],
347+
}
348+
349+
350+
_GENERIC = TrainingDefaults(
351+
lr=3e-4, batch_size=1024, schedule="cosine",
352+
warmup_steps=1000, betas=(0.9, 0.95),
353+
gradient_clip=1.0, mixed_precision=True, optimizer="adamw",
354+
source_paper_url="https://arxiv.org/abs/2407.21783")
355+
356+
357+
def get_defaults(preset_name: str) -> TrainingDefaults:
358+
"""Return paper-anchored defaults for ``preset_name``.
359+
360+
Resolution order:
361+
362+
1. Exact match in :data:`DEFAULTS`.
363+
2. Longest matching family prefix in :data:`FAMILY_DEFAULTS`.
364+
3. ``_GENERIC`` (Llama-3-style sensible defaults).
365+
366+
The contract is total: never raises, never returns None.
367+
"""
368+
if preset_name in DEFAULTS:
369+
return DEFAULTS[preset_name]
370+
matches = [k for k in FAMILY_DEFAULTS if preset_name.startswith(k)]
371+
if matches:
372+
return FAMILY_DEFAULTS[max(matches, key=len)]
373+
return _GENERIC
374+
375+
376+
def known_keys() -> tuple[str, ...]:
377+
"""Sorted tuple of preset keys with an explicit paper-anchored row."""
378+
return tuple(sorted(DEFAULTS))
379+
380+
381+
def to_wire(defaults: TrainingDefaults) -> dict[str, Any]:
382+
"""Render :class:`TrainingDefaults` as a JSON-friendly dict.
383+
384+
Tuples become lists so the result is directly JSON-serialisable for
385+
the ``build_preset_specs`` RPC payload (Pydantic ``model_dump``
386+
rejects bare tuples in result schemas).
387+
"""
388+
d = asdict(defaults)
389+
if d["betas"] is not None:
390+
d["betas"] = list(d["betas"])
391+
return d

0 commit comments

Comments
 (0)