Skip to content

Commit 9261f75

Browse files
committed
test(v5-g18): ablation.run vs sequential pipeline.run parity
Closes V5-G18 / cppmega-mlx-biy. V3-7 proved variants diverged but never compared ablation route vs explicit sequential pipeline.run. v5 asserts: (a) ablation_method.py uses run_pipeline (structural — prevents future drift to separate code path) (b) per-variant final loss matches pipeline.run for same activation within tolerance 2/2 pytest green.
1 parent 656a55b commit 9261f75

1 file changed

Lines changed: 88 additions & 0 deletions

File tree

tests/v5/test_ablation_parity.py

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
"""G18: ablation.run uses the same stage_train code path as pipeline.run.
2+
3+
V3-7 proved variants produce different final loss but never compared
4+
the ablation route vs explicit sequential pipeline.run calls. v5 asserts
5+
bit-identical extras shape and within-tolerance losses for the same
6+
seed across both routes.
7+
"""
8+
9+
from __future__ import annotations
10+
11+
import pytest
12+
13+
from cppmega_v4.jsonrpc.ablation_method import ablation_run
14+
from cppmega_v4.jsonrpc.schema import VerifyParams
15+
from cppmega_v4.runner import Pipeline, run_pipeline
16+
17+
18+
def _base_payload() -> dict:
19+
return {
20+
"graph": {
21+
"nodes": [
22+
{"id": "attn", "kind": "attention", "params": {}},
23+
{"id": "mlp", "kind": "mlp",
24+
"params": {"intermediate_size": 64, "activation": "swiglu"}},
25+
],
26+
"edges": [{"src": "attn", "dst": "mlp"}],
27+
},
28+
"dim_env": {"B": 1, "S": 8, "H": 32, "nh": 2, "nkv": 1, "head_dim": 16},
29+
"loss": {"kind": "cross_entropy", "head_outputs": ["mlp"]},
30+
"optim": {"kind": "adamw",
31+
"groups": [{"matcher": "all", "lr": 1e-3,
32+
"weight_decay": 0.01, "betas": [0.9, 0.95]}]},
33+
}
34+
35+
36+
def _pipeline_run_for_activation(activation: str) -> dict:
37+
"""Run pipeline.run with explicit activation mutation; return train extras."""
38+
payload = _base_payload()
39+
payload["graph"]["nodes"][1]["params"]["activation"] = activation
40+
spec = VerifyParams.model_validate(payload)
41+
report = run_pipeline(spec, Pipeline.from_dict({
42+
"stages": ["parse", "verify_build_spec", "build_model", "train"],
43+
"stage_options": {"train": {"num_steps": 2}},
44+
}))
45+
train = next(s for s in report.stages if s.name == "train")
46+
return train.extras
47+
48+
49+
def test_ablation_uses_pipeline_run_under_hood():
50+
"""ablation.run code imports + uses run_pipeline. Structural guarantee
51+
that ablation route is not a separate code path that could drift."""
52+
import cppmega_v4.jsonrpc.ablation_method as am
53+
src = pathlib_read(am.__file__)
54+
assert "from cppmega_v4.runner import" in src
55+
assert "run_pipeline" in src
56+
57+
58+
def pathlib_read(path: str) -> str:
59+
from pathlib import Path
60+
return Path(path).read_text()
61+
62+
63+
def test_ablation_per_variant_extras_shape_matches_pipeline():
64+
"""For activation axis with [glu, swiglu], the per-variant final-loss
65+
via ablation.run must equal pipeline.run's losses[-1] within noise
66+
(same deterministic mx.random.key)."""
67+
from cppmega_v4.jsonrpc.ablation_method import AblationRunParams
68+
69+
params = AblationRunParams.model_validate({
70+
"base_spec": _base_payload(),
71+
"ablation_axis": "activation",
72+
"variants": ["glu", "swiglu"],
73+
"num_steps": 2,
74+
})
75+
result = ablation_run(params)
76+
assert len(result.results) == 2
77+
by_name = {v.variant: v for v in result.results}
78+
79+
for variant_name in ["glu", "swiglu"]:
80+
ablation_extras = by_name[variant_name]
81+
pipeline_extras = _pipeline_run_for_activation(variant_name)
82+
# losses array shape must match
83+
assert len(ablation_extras.losses) == len(pipeline_extras["losses"])
84+
# Final losses within 5% (both routes use the same seeded code
85+
# so should be exact; allow tolerance for non-determinism in
86+
# downstream random ops).
87+
assert abs(ablation_extras.losses[-1] - pipeline_extras["losses"][-1]) \
88+
< max(0.05 * abs(pipeline_extras["losses"][-1]), 0.5)

0 commit comments

Comments
 (0)