Skip to content

Commit 5696392

Browse files
committed
feat(v5-g12): checkpoint save/resume via safetensors
Closes V5-G12 / cppmega-mlx-t8i. stage_train opts: - checkpoint_load_path: safetensors → tree_unflatten → model.update (before training, so first step starts from loaded weights) - checkpoint_save_path: model.parameters → tree_flatten → safetensors save (after training, weights persist for next session) extras.checkpoint = {saved_path, loaded_path}; both null when opts absent; load failure non-fatal. 4 new pytest. UI textinputs for paths deferred — backend contract first.
1 parent 629c1c0 commit 5696392

2 files changed

Lines changed: 93 additions & 0 deletions

File tree

cppmega_v4/runner/stages.py

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -773,6 +773,20 @@ def _count(tree: Any) -> int:
773773
probe_key: str | None = None
774774
probe_before: mx.array | None = None
775775

776+
# G12: optional checkpoint load before training. Reads safetensors
777+
# weights into the model. Failure is non-fatal — log via extras.
778+
checkpoint_loaded: str | None = None
779+
ckpt_load = opts.get("checkpoint_load_path")
780+
if ckpt_load:
781+
try:
782+
import safetensors.mlx as _stmlx
783+
loaded = _stmlx.load_file(ckpt_load)
784+
all_modules.update(
785+
nn.utils.tree_unflatten(list(loaded.items())))
786+
checkpoint_loaded = str(ckpt_load)
787+
except Exception:
788+
pass
789+
776790
# G09: check abort flag set via opts.abort or _ABORT_TOKENS set
777791
abort_token = opts.get("abort_token")
778792
for step in range(n_steps):
@@ -846,6 +860,18 @@ def _count(tree: Any) -> int:
846860
except Exception:
847861
pass
848862

863+
# G12: optional checkpoint save after training.
864+
checkpoint_saved: str | None = None
865+
ckpt_save = opts.get("checkpoint_save_path")
866+
if ckpt_save:
867+
try:
868+
import safetensors.mlx as _stmlx
869+
flat = dict(nn.utils.tree_flatten(all_modules.parameters()))
870+
_stmlx.save_file(flat, ckpt_save)
871+
checkpoint_saved = str(ckpt_save)
872+
except Exception:
873+
pass
874+
849875
# G10: cache opt.state for future warm-start lookups (capped LRU)
850876
try:
851877
_RUN_CACHE[run_id] = opt.state
@@ -965,6 +991,10 @@ def _count(tree: Any) -> int:
965991
"moe": moe_extras,
966992
"opt_state_carried": opt_state_carried,
967993
"run_id": run_id,
994+
"checkpoint": {
995+
"saved_path": checkpoint_saved,
996+
"loaded_path": checkpoint_loaded,
997+
},
968998
"mtp": _compute_mtp_extras(
969999
all_modules, mtp_k, mtp_betas, vocab_size,
9701000
batch, seq, hidden, targets,
Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
"""G12: checkpoint save/resume."""
2+
3+
from __future__ import annotations
4+
5+
import pathlib
6+
7+
import pytest
8+
9+
from cppmega_v4.jsonrpc.schema import VerifyParams
10+
from cppmega_v4.runner import Pipeline, run_pipeline
11+
12+
13+
def _spec() -> VerifyParams:
14+
return VerifyParams.model_validate({
15+
"graph": {"nodes": [
16+
{"id": "attn", "kind": "attention", "params": {}},
17+
{"id": "mlp", "kind": "mlp",
18+
"params": {"intermediate_size": 64, "activation": "swiglu"}},
19+
], "edges": [{"src": "attn", "dst": "mlp"}]},
20+
"dim_env": {"B": 1, "S": 8, "H": 32, "nh": 2, "nkv": 1, "head_dim": 16},
21+
"loss": {"kind": "cross_entropy", "head_outputs": ["mlp"]},
22+
"optim": {"kind": "adamw",
23+
"groups": [{"matcher": "all", "lr": 1e-3,
24+
"weight_decay": 0.01, "betas": [0.9, 0.95]}]},
25+
})
26+
27+
28+
def _run(opts: dict) -> dict:
29+
report = run_pipeline(_spec(), Pipeline.from_dict({
30+
"stages": ["parse", "verify_build_spec", "build_model", "train"],
31+
"stage_options": {"train": opts},
32+
}))
33+
train = next(s for s in report.stages if s.name == "train")
34+
return train.extras
35+
36+
37+
def test_checkpoint_save_writes_file(tmp_path):
38+
save_path = str(tmp_path / "ckpt.safetensors")
39+
extras = _run({"num_steps": 2, "checkpoint_save_path": save_path})
40+
assert extras["checkpoint"]["saved_path"] == save_path
41+
assert extras["checkpoint"]["loaded_path"] is None
42+
assert pathlib.Path(save_path).exists()
43+
44+
45+
def test_checkpoint_load_reads_file(tmp_path):
46+
save_path = str(tmp_path / "ckpt.safetensors")
47+
a = _run({"num_steps": 2, "checkpoint_save_path": save_path})
48+
assert a["checkpoint"]["saved_path"] == save_path
49+
b = _run({"num_steps": 2, "checkpoint_load_path": save_path})
50+
assert b["checkpoint"]["loaded_path"] == save_path
51+
52+
53+
def test_checkpoint_load_missing_file_is_non_fatal():
54+
extras = _run({"num_steps": 2,
55+
"checkpoint_load_path": "/nonexistent/ckpt.safetensors"})
56+
# train still succeeds; loaded_path is None
57+
assert extras["checkpoint"]["loaded_path"] is None
58+
59+
60+
def test_no_checkpoint_opts_yields_null_paths():
61+
extras = _run({"num_steps": 2})
62+
assert extras["checkpoint"]["saved_path"] is None
63+
assert extras["checkpoint"]["loaded_path"] is None

0 commit comments

Comments
 (0)