Skip to content

Commit 7cc15a4

Browse files
committed
feat(mamba3-pathc): Stage 3 — register 3 _bwd descriptors + extend compile-site interpose
Mirrors the forward Stage-2 descriptor+interpose layer for the backward B0/B1/B2: - 3 _bwd brick-schedule descriptors (mamba3_chunk_scan_combine_bwd / mamba3_inter_chunk_recur_bwd / mamba3_chunk_precompute_bwd), each FLAT loop policy + shadow fragment_emitter, production_source -> build_*_bwd_metal. - 3 _bwd fragment emitters (shadow markers; never the live kernel — interpose bypasses them when the flag is ON). - Extended _MAMBA3_CHUNKED_GRID_DELEGATION_OPS with the 3 _bwd op-names so the existing _mamba3_chunked_grid_delegation_prim resolves+compiles them (same build_*_metal ABI / positional dims as the forward). Verified by tests/test_mamba3_chunked_backward_b0b1b2.py (8 tests): - 3 _bwd descriptors register + select-resolve (no 'no descriptor target'). - flag OFF: B2 segment is NOT a grid kernel (merge-safe). - flag ON: all 3 _bwd op-names emit REAL build_*_bwd_metal grid JITKernels via the interpose (RULE #1: shadow marker never live). - chained B2->B1->B0 per-grad parity vs the MLX backward proto: WORST 3.7e-4 (S=256) / 3.8e-4 (S=512), all 8 grads < 1e-3 at chunk=64. Flag-OFF unchanged: 136 path_c template/IR tests + 19 forward F0/F1/F2 tests green. Flag CPPMEGA_PATH_C_MAMBA3_CHUNKED_SCAN still default OFF (merge-safe). The live region-build backward autograd 1->3 surface flip is the remaining piece.
1 parent 648085a commit 7cc15a4

2 files changed

Lines changed: 489 additions & 0 deletions

File tree

cppmega_mlx/runtime/path_c_fusion_schedules.py

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1494,6 +1494,100 @@ def default_path_c_brick_schedule_descriptor_registry() -> (
14941494
preferred_loop_policy=DESCRIPTOR_LOOP_POLICY_FLAT,
14951495
fragment_emitter=_emit_mamba3_inter_chunk_recur_source,
14961496
),
1497+
# --- Stage 3 BACKWARD chunked descriptors (B2 / B1 / B0) --- #
1498+
# The analytic transpose of the forward F2/F1/F0 (design §2/§7 Stage 3).
1499+
# Each is a single-entry GRID kernel; the compile-site interpose
1500+
# substitutes the real build_*_bwd_metal grid prim when the flag is ON.
1501+
PathCBrickScheduleDescriptor(
1502+
op_name="mamba3_chunk_scan_combine_bwd",
1503+
implementation_status="descriptor_codegen_ready",
1504+
required_codegen_steps=(
1505+
"mamba3_chunk_scan_combine_bwd_descriptor",
1506+
"mamba3_chunk_scan_combine_bwd_grid_kernel",
1507+
),
1508+
description=(
1509+
"Mamba3 chunked SSD scan+combine BACKWARD (B2) brick "
1510+
"descriptor; GRID-launched output/Y transpose; delegates to "
1511+
"chunk_scan_combine_bwd_metal_prim (dC/dx/dz/dchunk_states/"
1512+
"dinp/dA_cumsum_y/dD)"
1513+
),
1514+
production_source=(
1515+
"cppmega_mlx.nn._tilelang.mamba3_chunked_backward_core:"
1516+
"build_chunk_scan_combine_bwd_metal/"
1517+
"chunk_scan_combine_bwd_metal_prim"
1518+
),
1519+
production_fragment_status="region_fragment_inlined_unoptimized",
1520+
production_fragment_reason=(
1521+
"B2 scan+combine backward is a grid-launched kernel (own "
1522+
"T.Kernel grid) delegating to chunk_scan_combine_bwd_metal_prim "
1523+
"— the analytic transpose of the forward F2. Validated vs the "
1524+
"MLX backward proto (worst grad 3.68e-4) by "
1525+
"scratch/test_b0b1b2_metal_vs_proto.py (Stage 3)"
1526+
),
1527+
supports_backward=False,
1528+
preferred_loop_policy=DESCRIPTOR_LOOP_POLICY_FLAT,
1529+
fragment_emitter=_emit_mamba3_chunk_scan_combine_bwd_source,
1530+
),
1531+
PathCBrickScheduleDescriptor(
1532+
op_name="mamba3_inter_chunk_recur_bwd",
1533+
implementation_status="descriptor_codegen_ready",
1534+
required_codegen_steps=(
1535+
"mamba3_inter_chunk_recur_bwd_descriptor",
1536+
"mamba3_inter_chunk_recur_bwd_grid_kernel",
1537+
),
1538+
description=(
1539+
"Mamba3 chunked SSD inter-chunk recurrence BACKWARD (B1) brick "
1540+
"descriptor; the NEW O(S/C) REVERSE upper-tri combiner; "
1541+
"delegates to inter_chunk_recur_bwd_metal_prim (reuses forward "
1542+
"prev_states -> dstates/dh0/dA_cumsum_tail; no 8x replay)"
1543+
),
1544+
production_source=(
1545+
"cppmega_mlx.nn._tilelang.mamba3_chunked_backward_core:"
1546+
"build_inter_chunk_recur_bwd_metal/"
1547+
"inter_chunk_recur_bwd_metal_prim"
1548+
),
1549+
production_fragment_status="region_fragment_inlined_unoptimized",
1550+
production_fragment_reason=(
1551+
"B1 inter-chunk recurrence backward is the REVERSE O(S/C) "
1552+
"upper-tri combiner (own T.Kernel(batch,nheads) grid; the chunk "
1553+
"axis is the only serial REVERSE carry) — the adjoint of the "
1554+
"forward F1 lower-tri recurrence. REUSES the forward-"
1555+
"materialized prev_states (the 8x checkpoint-replay elimination, "
1556+
"design §3/§6). Validated by scratch/test_b0b1b2_metal_vs_proto.py"
1557+
),
1558+
supports_backward=False,
1559+
preferred_loop_policy=DESCRIPTOR_LOOP_POLICY_FLAT,
1560+
fragment_emitter=_emit_mamba3_inter_chunk_recur_bwd_source,
1561+
),
1562+
PathCBrickScheduleDescriptor(
1563+
op_name="mamba3_chunk_precompute_bwd",
1564+
implementation_status="descriptor_codegen_ready",
1565+
required_codegen_steps=(
1566+
"mamba3_chunk_precompute_bwd_descriptor",
1567+
"mamba3_chunk_precompute_bwd_grid_kernel",
1568+
),
1569+
description=(
1570+
"Mamba3 chunked SSD precompute BACKWARD (B0) brick descriptor; "
1571+
"GRID-launched precompute transpose; delegates to "
1572+
"chunk_precompute_bwd_metal_prim (decay_states transpose + dinp "
1573+
"assembly + cumsum/segsum VJP -> dx/dB/dlog_decay/ddt)"
1574+
),
1575+
production_source=(
1576+
"cppmega_mlx.nn._tilelang.mamba3_chunked_backward_core:"
1577+
"build_chunk_precompute_bwd_metal/"
1578+
"chunk_precompute_bwd_metal_prim"
1579+
),
1580+
production_fragment_status="region_fragment_inlined_unoptimized",
1581+
production_fragment_reason=(
1582+
"B0 precompute backward is a grid-launched kernel (own T.Kernel "
1583+
"grid) delegating to chunk_precompute_bwd_metal_prim — the "
1584+
"transpose of the forward F0 precompute, assembling the final "
1585+
"input grads. Validated by scratch/test_b0b1b2_metal_vs_proto.py"
1586+
),
1587+
supports_backward=False,
1588+
preferred_loop_policy=DESCRIPTOR_LOOP_POLICY_FLAT,
1589+
fragment_emitter=_emit_mamba3_chunk_precompute_bwd_source,
1590+
),
14971591
PathCBrickScheduleDescriptor(
14981592
op_name="mamba3_mimo_bwd",
14991593
implementation_status="descriptor_codegen_ready",
@@ -6588,9 +6682,16 @@ def mamba3_chunked_forward_scan_grid(
65886682
# ``CPPMEGA_PATH_C_MAMBA3_CHUNKED_SCAN`` flag is ON.
65896683
_MAMBA3_CHUNKED_GRID_DELEGATION_OPS = frozenset(
65906684
{
6685+
# forward F0/F1/F2 (Stage 2)
65916686
"mamba3_chunk_precompute",
65926687
"mamba3_inter_chunk_recur",
65936688
"mamba3_chunk_scan_combine",
6689+
# backward B0/B1/B2 (Stage 3) — the analytic transpose of F0/F1/F2.
6690+
# Same builder ABI (build_*_metal / positional dims); the interpose
6691+
# resolves them identically (RULE #1: one delegation path each).
6692+
"mamba3_chunk_precompute_bwd",
6693+
"mamba3_inter_chunk_recur_bwd",
6694+
"mamba3_chunk_scan_combine_bwd",
65946695
}
65956696
)
65966697

@@ -13220,6 +13321,98 @@ def _emit_mamba3_inter_chunk_recur_source(
1322013321
)
1322113322

1322213323

13324+
def _emit_mamba3_chunk_scan_combine_bwd_source(
13325+
node: _ScheduleNodeView,
13326+
dtype_by_buffer: dict[str, str],
13327+
access_by_buffer: dict[str, str],
13328+
) -> _ScheduleNodeFragment:
13329+
"""Fragment emitter for the Path-C B2 ``mamba3_chunk_scan_combine_bwd`` segment.
13330+
13331+
Design: ``docs/MAMBA3-PATHC-MULTIKERNEL-DESIGN.md`` §2 (B2 row) / §7 Stage 3.
13332+
13333+
B2 is the analytic TRANSPOSE of the forward F2 scan+combine (output/gate +
13334+
Y_diag + Y_off transpose). Its kernel is the GRID-launched core
13335+
``chunk_scan_combine_bwd_metal_prim`` (own ``T.Kernel(batch*nchunks, nheads)``
13336+
grid). Real codegen is the single delegation
13337+
``mamba3_chunked_backward_core.build_chunk_scan_combine_bwd_metal`` — validated
13338+
vs the MLX backward proto (worst grad 3.68e-4) by
13339+
``scratch/test_b0b1b2_metal_vs_proto.py``. This marker only needs to RESOLVE
13340+
via ``select``; the live interpose substitutes the real grid kernel when the
13341+
flag is ON. RULE #1: no silent fallback — one delegation path.
13342+
"""
13343+
marker = _scratch_name(node, "mamba3_chunk_scan_combine_bwd_shadow")
13344+
return _ScheduleNodeFragment(
13345+
allocations=(f"{marker} = T.alloc_local((1,), \"float32\")",),
13346+
statements=(
13347+
"# B2 mamba3_chunk_scan_combine_bwd: delegates to "
13348+
"cppmega_mlx.nn._tilelang.mamba3_chunked_backward_core."
13349+
"build_chunk_scan_combine_bwd_metal (grid output/Y transpose -> "
13350+
"dC/dx/dz/dchunk_states/dinp/dA_cumsum_y/dD)",
13351+
f"{marker}[0] = 0.0",
13352+
),
13353+
)
13354+
13355+
13356+
def _emit_mamba3_chunk_precompute_bwd_source(
13357+
node: _ScheduleNodeView,
13358+
dtype_by_buffer: dict[str, str],
13359+
access_by_buffer: dict[str, str],
13360+
) -> _ScheduleNodeFragment:
13361+
"""Fragment emitter for the Path-C B0 ``mamba3_chunk_precompute_bwd`` segment.
13362+
13363+
Design: ``docs/MAMBA3-PATHC-MULTIKERNEL-DESIGN.md`` §2 (B0 row) / §7 Stage 3.
13364+
13365+
B0 is the transpose of the forward F0 precompute (decay_states transpose +
13366+
dinp assembly + cumsum/segsum VJP). Real codegen delegation
13367+
``mamba3_chunked_backward_core.build_chunk_precompute_bwd_metal`` -> the final
13368+
input grads ``dx/dB/dlog_decay/ddt``. This marker only needs to RESOLVE via
13369+
``select``; the live interpose substitutes the real grid kernel when the flag
13370+
is ON. RULE #1: no silent fallback — one delegation path.
13371+
"""
13372+
marker = _scratch_name(node, "mamba3_chunk_precompute_bwd_shadow")
13373+
return _ScheduleNodeFragment(
13374+
allocations=(f"{marker} = T.alloc_local((1,), \"float32\")",),
13375+
statements=(
13376+
"# B0 mamba3_chunk_precompute_bwd: delegates to "
13377+
"cppmega_mlx.nn._tilelang.mamba3_chunked_backward_core."
13378+
"build_chunk_precompute_bwd_metal (grid precompute transpose -> "
13379+
"dx/dB/dlog_decay/ddt input grads)",
13380+
f"{marker}[0] = 0.0",
13381+
),
13382+
)
13383+
13384+
13385+
def _emit_mamba3_inter_chunk_recur_bwd_source(
13386+
node: _ScheduleNodeView,
13387+
dtype_by_buffer: dict[str, str],
13388+
access_by_buffer: dict[str, str],
13389+
) -> _ScheduleNodeFragment:
13390+
"""Fragment emitter for the Path-C B1 ``mamba3_inter_chunk_recur_bwd`` segment.
13391+
13392+
Design: ``docs/MAMBA3-PATHC-MULTIKERNEL-DESIGN.md`` §2 (B1 row) / §7 Stage 3.
13393+
13394+
B1 is the ONE genuinely new kernel: the REVERSE O(S/C) inter-chunk combiner
13395+
(upper-tri ``decay_chunk`` contraction — the adjoint of the forward F1 lower-tri
13396+
recurrence). It REUSES the forward-materialized ``prev_states`` boundary states
13397+
(the 8x checkpoint-replay elimination, design §3/§6). Real codegen delegation
13398+
``mamba3_chunked_backward_core.build_inter_chunk_recur_bwd_metal`` ->
13399+
``dstates/dh0/dA_cumsum_tail``. This marker only needs to RESOLVE via
13400+
``select``; the live interpose substitutes the real grid kernel when the flag
13401+
is ON. RULE #1: no silent fallback — one delegation path.
13402+
"""
13403+
marker = _scratch_name(node, "mamba3_inter_chunk_recur_bwd_shadow")
13404+
return _ScheduleNodeFragment(
13405+
allocations=(f"{marker} = T.alloc_local((1,), \"float32\")",),
13406+
statements=(
13407+
"# B1 mamba3_inter_chunk_recur_bwd: delegates to "
13408+
"cppmega_mlx.nn._tilelang.mamba3_chunked_backward_core."
13409+
"build_inter_chunk_recur_bwd_metal (REVERSE O(S/C) upper-tri combiner "
13410+
"-> dstates/dh0/dA_cumsum_tail; reuses forward prev_states)",
13411+
f"{marker}[0] = 0.0",
13412+
),
13413+
)
13414+
13415+
1322313416
def _emit_residual_rmsnorm_source(
1322413417
node: _ScheduleNodeView,
1322513418
dtype_by_buffer: dict[str, str],

0 commit comments

Comments
 (0)