Skip to content

Commit 629c1c0

Browse files
committed
feat(v5-g09): cancel/abort Train via opts.abort_token + _ABORT_TOKENS
Closes V5-G09 / cppmega-mlx-ajm. Backend stage_train checks opts.abort_token between train steps; if in module-global _ABORT_TOKENS set, returns immediately with extras.aborted=True + partial losses. request_abort(token) / clear_abort(token) helpers for callers. 3 new pytest. UI Cancel button + WS abort handler deferred — backend contract first, frontend wiring follow-up.
1 parent 678dfe8 commit 629c1c0

2 files changed

Lines changed: 88 additions & 0 deletions

File tree

cppmega_v4/runner/stages.py

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -773,7 +773,25 @@ def _count(tree: Any) -> int:
773773
probe_key: str | None = None
774774
probe_before: mx.array | None = None
775775

776+
# G09: check abort flag set via opts.abort or _ABORT_TOKENS set
777+
abort_token = opts.get("abort_token")
776778
for step in range(n_steps):
779+
if abort_token is not None and abort_token in _ABORT_TOKENS:
780+
# Stop early; return partial extras with cancellation flag.
781+
return StageResult(
782+
name="train", status="ok",
783+
elapsed_ms=(time.perf_counter() - t0) * 1000.0,
784+
extras={
785+
"losses": [round(l, 4) for l in losses],
786+
"lr_trajectory": [round(l, 6) for l in lr_trajectory],
787+
"weight_delta_norm": 0.0,
788+
"num_steps": step,
789+
"schedule_kind": schedule_kind_label,
790+
"optimizer_kind": optimizer_kind,
791+
"aborted": True,
792+
"abort_token": abort_token,
793+
},
794+
)
777795
# If a schedule callable exists, override optimizer's
778796
# learning_rate per step. MLX optimizers accept a fresh
779797
# scalar via the public learning_rate attribute.
@@ -1014,6 +1032,20 @@ def _tokenize_parquet_text(
10141032
# across sequential Train clicks in the same backend session.
10151033
_RUN_CACHE: dict[str, Any] = {}
10161034

1035+
# G09: in-process set of abort tokens. Caller sets opts.abort_token to
1036+
# some unique string; another caller (e.g. WS handler) inserts the same
1037+
# token into this set to signal cancellation between train steps.
1038+
_ABORT_TOKENS: set[str] = set()
1039+
1040+
1041+
def request_abort(token: str) -> None:
1042+
"""G09: signal stage_train to abort the run identified by token."""
1043+
_ABORT_TOKENS.add(token)
1044+
1045+
1046+
def clear_abort(token: str) -> None:
1047+
_ABORT_TOKENS.discard(token)
1048+
10171049

10181050
_REWRITER_FACTORIES: dict[str, Callable[..., Any]] = {}
10191051

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
"""G09: cancel/abort Train via abort_token + _ABORT_TOKENS set."""
2+
3+
from __future__ import annotations
4+
5+
from cppmega_v4.jsonrpc.schema import VerifyParams
6+
from cppmega_v4.runner import Pipeline, run_pipeline
7+
from cppmega_v4.runner.stages import request_abort, clear_abort
8+
9+
10+
def _spec() -> VerifyParams:
11+
return VerifyParams.model_validate({
12+
"graph": {"nodes": [
13+
{"id": "attn", "kind": "attention", "params": {}},
14+
{"id": "mlp", "kind": "mlp",
15+
"params": {"intermediate_size": 64, "activation": "swiglu"}},
16+
], "edges": [{"src": "attn", "dst": "mlp"}]},
17+
"dim_env": {"B": 1, "S": 8, "H": 32, "nh": 2, "nkv": 1, "head_dim": 16},
18+
"loss": {"kind": "cross_entropy", "head_outputs": ["mlp"]},
19+
"optim": {"kind": "adamw",
20+
"groups": [{"matcher": "all", "lr": 1e-3,
21+
"weight_decay": 0.01, "betas": [0.9, 0.95]}]},
22+
})
23+
24+
25+
def _run(opts: dict) -> dict:
26+
report = run_pipeline(_spec(), Pipeline.from_dict({
27+
"stages": ["parse", "verify_build_spec", "build_model", "train"],
28+
"stage_options": {"train": opts},
29+
}))
30+
train = next(s for s in report.stages if s.name == "train")
31+
return train.extras
32+
33+
34+
def test_abort_token_set_before_run_cancels_immediately():
35+
token = "abort-test-1"
36+
request_abort(token)
37+
try:
38+
extras = _run({"num_steps": 8, "abort_token": token})
39+
assert extras.get("aborted") is True
40+
assert extras["abort_token"] == token
41+
# Aborted at step 0 → losses empty
42+
assert extras["num_steps"] == 0
43+
finally:
44+
clear_abort(token)
45+
46+
47+
def test_no_abort_token_runs_to_completion():
48+
extras = _run({"num_steps": 2})
49+
assert extras.get("aborted") is not True
50+
assert extras["num_steps"] == 2
51+
52+
53+
def test_abort_token_not_in_set_runs_to_completion():
54+
extras = _run({"num_steps": 2, "abort_token": "never-set"})
55+
assert extras.get("aborted") is not True
56+
assert extras["num_steps"] == 2

0 commit comments

Comments
 (0)