Skip to content

Commit 61d145d

Browse files
committed
feat(probe-b): contract_probe runner + alternatives + dry-forward gate
Stage B of the Contract Probe epic (cppmega-mlx-728). Adds the solver, requirements tables, alternative generator, and synthetic dry-forward gate. - requirements.py: BRICK_REQUIREMENTS (covers every BLOCK_BUILDERS key — engram needs call_edges, csa_hca needs type_edges, the other 23 bricks accept input_ids only) + LOSS_REQUIREMENTS (CE/MTP/IFIM/ MHC/CUSTOM with explicit satisfied_by chains). - alternatives.py: deterministic generator (swap_loss, swap_tokenizer, add_column, drop_brick, relax_requirement). Capped at 3 per finding, ordered by cost (low→medium→high). - dry_forward.py: synthetic (B=1, S=8, H=64) walk through the instantiated graph; mean-reduce predecessors at fan-in for parallel-block topologies. Never raises — returns a verdict tuple. - probe.py: contract_probe() composes capability snapshots + per- component evaluation + dry-forward into one read-only ContractProbeReport with is_clean / blocking accessors. Tests (39): every brick kind has a requirements entry, satisfied-by alternative-key logic, end-to-end probe on llama3_8b with CE/IFIM/ MTP/MHC variants, alternative shape + cap + cost-ordering, dry_forward on linear and parallel-block graphs, sub-2s gate on qwen3_235b_a22b. Full v4 regression: 1927 passed / 5 skipped / 15 xfailed / 0 failed. Closes cppmega-mlx-728.2.
1 parent 12129b4 commit 61d145d

6 files changed

Lines changed: 813 additions & 0 deletions

File tree

cppmega_v4/probe/__init__.py

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,18 +9,40 @@
99

1010
from __future__ import annotations
1111

12+
from cppmega_v4.probe.alternatives import Alternative, generate_alternatives
1213
from cppmega_v4.probe.capabilities import (
1314
ColumnSpec,
1415
ParquetCapabilities,
1516
TokenizerCapabilities,
1617
introspect_parquet,
1718
introspect_tokenizer,
1819
)
20+
from cppmega_v4.probe.dry_forward import DryForwardResult, dry_forward
21+
from cppmega_v4.probe.probe import (
22+
ContractProbeReport,
23+
ProbeFinding,
24+
contract_probe,
25+
)
26+
from cppmega_v4.probe.requirements import (
27+
BRICK_REQUIREMENTS,
28+
LOSS_REQUIREMENTS,
29+
DataRequirement,
30+
)
1931

2032
__all__ = [
33+
"Alternative",
34+
"BRICK_REQUIREMENTS",
2135
"ColumnSpec",
36+
"ContractProbeReport",
37+
"DataRequirement",
38+
"DryForwardResult",
39+
"LOSS_REQUIREMENTS",
2240
"ParquetCapabilities",
41+
"ProbeFinding",
2342
"TokenizerCapabilities",
43+
"contract_probe",
44+
"dry_forward",
45+
"generate_alternatives",
2446
"introspect_parquet",
2547
"introspect_tokenizer",
2648
]

cppmega_v4/probe/alternatives.py

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
"""Generators for explicit alternatives to unsatisfied requirements.
2+
3+
Each generator is a pure function:
4+
(requirement, build_spec, tokenizer_caps, parquet_caps) -> tuple[Alternative, ...]
5+
6+
No side effects, no LLM, no model calls. Alternatives are ranked
7+
deterministically by ``cost`` (low < medium < high) and then by
8+
``action`` to keep test output stable across runs.
9+
10+
Each Alternative carries a JSON-Patch-shaped ``diff`` that the GUI/CLI
11+
applies to the canonical spec representation. The probe never applies
12+
the diff itself.
13+
"""
14+
15+
from __future__ import annotations
16+
17+
from dataclasses import dataclass
18+
from typing import Literal, Mapping
19+
20+
from cppmega_v4.buildspec.loss_spec import LossKind
21+
from cppmega_v4.buildspec.model_build_spec import ModelBuildSpec
22+
from cppmega_v4.probe.capabilities import (
23+
ParquetCapabilities,
24+
TokenizerCapabilities,
25+
)
26+
from cppmega_v4.probe.requirements import DataRequirement
27+
28+
29+
_COST_ORDER: dict[str, int] = {"low": 0, "medium": 1, "high": 2}
30+
31+
32+
@dataclass(frozen=True)
33+
class Alternative:
34+
"""One way the user can resolve an unsatisfied requirement."""
35+
36+
action: Literal[
37+
"swap_loss", "swap_tokenizer", "add_column",
38+
"drop_brick", "relax_requirement",
39+
]
40+
target: str
41+
diff: Mapping[str, object]
42+
cost: Literal["low", "medium", "high"]
43+
reason: str
44+
45+
46+
_MAX_ALTERNATIVES_PER_FINDING: int = 3
47+
48+
49+
def generate_alternatives(
50+
requirement: DataRequirement,
51+
component: str,
52+
build_spec: ModelBuildSpec,
53+
tokenizer_caps: TokenizerCapabilities,
54+
parquet_caps: ParquetCapabilities,
55+
) -> tuple[Alternative, ...]:
56+
"""Return ordered alternatives that, if applied, would satisfy
57+
``requirement``. Caps the result at three per design ceiling."""
58+
out: list[Alternative] = []
59+
60+
# Loss-level requirements — most cheaply resolved by swapping loss.
61+
if component.startswith("loss:"):
62+
if build_spec.loss.kind != LossKind.CROSS_ENTROPY:
63+
out.append(Alternative(
64+
action="swap_loss", target=component,
65+
diff={"op": "replace", "path": "/loss/kind",
66+
"value": LossKind.CROSS_ENTROPY.value},
67+
cost="low",
68+
reason="cross-entropy has no side-channel data requirements",
69+
))
70+
# FIM-specific: suggest tokenizer swap when FIM ids are missing.
71+
if requirement.origin == "tokenizer" and requirement.key.startswith("FIM_"):
72+
out.append(Alternative(
73+
action="swap_tokenizer", target="tokenizer",
74+
diff={"op": "replace", "path": "/tokenizer/source",
75+
"value": "<tokenizer with full FIM trio>"},
76+
cost="medium",
77+
reason=f"need a tokenizer that defines {requirement.key}",
78+
))
79+
80+
# Brick-level requirements — drop the brick OR add the missing column.
81+
if component.startswith("brick:"):
82+
brick_name = component.split(":", 1)[1]
83+
if requirement.origin == "parquet":
84+
out.append(Alternative(
85+
action="add_column", target=f"parquet:{requirement.key}",
86+
diff={"op": "add", "path": f"/parquet/columns/{requirement.key}",
87+
"value": "<run enrichment pipeline>"},
88+
cost="high",
89+
reason=f"parquet shard is missing {requirement.key!r}; "
90+
"enrichment scripts live under scripts/nanochat_data/",
91+
))
92+
out.append(Alternative(
93+
action="drop_brick", target=component,
94+
diff={"op": "remove", "path": f"/graph/nodes/{brick_name}"},
95+
cost="medium",
96+
reason=f"remove brick {brick_name!r} which requires "
97+
f"{requirement.key!r}",
98+
))
99+
100+
# Universal escape hatch — only when the requirement is non-required.
101+
if not requirement.required:
102+
out.append(Alternative(
103+
action="relax_requirement", target=component,
104+
diff={"op": "add", "path": "/probe/allowlist",
105+
"value": requirement.key},
106+
cost="low",
107+
reason="requirement is soft; explicitly accept missing data",
108+
))
109+
110+
out.sort(key=lambda a: (_COST_ORDER[a.cost], a.action))
111+
return tuple(out[:_MAX_ALTERNATIVES_PER_FINDING])

cppmega_v4/probe/dry_forward.py

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
"""Dry forward pass — synthetic input_ids on the instantiated graph.
2+
3+
Last gate after static checks. Each brick already has a unit test that
4+
its forward preserves (B, S, H) at small sizes; here we chain the
5+
instantiated graph and run **one** forward at hidden=64, batch=1, seq=8
6+
to catch residual brick-to-brick coupling that the resolver missed.
7+
8+
Returns a one-word verdict + an optional exception trace. Never raises.
9+
"""
10+
11+
from __future__ import annotations
12+
13+
from dataclasses import dataclass
14+
from typing import Literal
15+
16+
import mlx.core as mx
17+
18+
from cppmega_v4.fusion.brick_graph import BrickGraph
19+
from cppmega_v4.models.unified_superblock_v4 import BLOCK_BUILDERS
20+
21+
22+
@dataclass(frozen=True)
23+
class DryForwardResult:
24+
verdict: Literal["ok", "shape_mismatch", "exception"]
25+
detail: str = ""
26+
27+
28+
def dry_forward(
29+
graph: BrickGraph,
30+
*,
31+
hidden_size: int = 64,
32+
seq_len: int = 8,
33+
batch: int = 1,
34+
) -> DryForwardResult:
35+
"""Walk ``graph`` in declared order, forwarding a synthetic activation.
36+
37+
Parallel-block topologies are handled by averaging branch outputs at
38+
the fan-in point — matches the convention BrickGraph uses for
39+
Tiny-Aya style ``GQA‖MLP``.
40+
"""
41+
try:
42+
# Build modules fresh — caller may have passed instantiate=False.
43+
modules: dict[str, object] = {}
44+
for node in graph.nodes:
45+
builder = BLOCK_BUILDERS.get(node.kind)
46+
if builder is None:
47+
return DryForwardResult(
48+
verdict="exception",
49+
detail=f"unknown brick kind {node.kind!r}",
50+
)
51+
modules[node.name] = builder(hidden_size, dict(node.params))
52+
53+
x0 = mx.random.normal((batch, seq_len, hidden_size))
54+
# Topo: pre-compute predecessors map; if a node has multiple
55+
# predecessors, mean-reduce their outputs before forwarding.
56+
outputs: dict[str, mx.array] = {}
57+
roots = [n.name for n in graph.nodes if not graph.predecessors(n.name)]
58+
for name in roots:
59+
outputs[name] = modules[name](x0)
60+
# Iterate remaining nodes in declared order (graph is already a
61+
# topological declaration thanks to from_block_specs).
62+
for node in graph.nodes:
63+
if node.name in outputs:
64+
continue
65+
preds = graph.predecessors(node.name)
66+
if not preds:
67+
outputs[node.name] = modules[node.name](x0)
68+
continue
69+
if len(preds) == 1:
70+
inp = outputs[preds[0]]
71+
else:
72+
stacked = mx.stack([outputs[p] for p in preds], axis=0)
73+
inp = mx.mean(stacked, axis=0)
74+
outputs[node.name] = modules[node.name](inp)
75+
76+
last = graph.nodes[-1].name
77+
y = outputs[last]
78+
if y.shape != (batch, seq_len, hidden_size):
79+
return DryForwardResult(
80+
verdict="shape_mismatch",
81+
detail=f"final shape {y.shape} != "
82+
f"(batch={batch}, seq={seq_len}, H={hidden_size})",
83+
)
84+
return DryForwardResult(verdict="ok")
85+
except Exception as exc:
86+
return DryForwardResult(
87+
verdict="exception",
88+
detail=f"{type(exc).__name__}: {exc}",
89+
)

0 commit comments

Comments
 (0)