@@ -1494,6 +1494,100 @@ def default_path_c_brick_schedule_descriptor_registry() -> (
14941494 preferred_loop_policy=DESCRIPTOR_LOOP_POLICY_FLAT,
14951495 fragment_emitter=_emit_mamba3_inter_chunk_recur_source,
14961496 ),
1497+ # --- Stage 3 BACKWARD chunked descriptors (B2 / B1 / B0) --- #
1498+ # The analytic transpose of the forward F2/F1/F0 (design §2/§7 Stage 3).
1499+ # Each is a single-entry GRID kernel; the compile-site interpose
1500+ # substitutes the real build_*_bwd_metal grid prim when the flag is ON.
1501+ PathCBrickScheduleDescriptor(
1502+ op_name="mamba3_chunk_scan_combine_bwd",
1503+ implementation_status="descriptor_codegen_ready",
1504+ required_codegen_steps=(
1505+ "mamba3_chunk_scan_combine_bwd_descriptor",
1506+ "mamba3_chunk_scan_combine_bwd_grid_kernel",
1507+ ),
1508+ description=(
1509+ "Mamba3 chunked SSD scan+combine BACKWARD (B2) brick "
1510+ "descriptor; GRID-launched output/Y transpose; delegates to "
1511+ "chunk_scan_combine_bwd_metal_prim (dC/dx/dz/dchunk_states/"
1512+ "dinp/dA_cumsum_y/dD)"
1513+ ),
1514+ production_source=(
1515+ "cppmega_mlx.nn._tilelang.mamba3_chunked_backward_core:"
1516+ "build_chunk_scan_combine_bwd_metal/"
1517+ "chunk_scan_combine_bwd_metal_prim"
1518+ ),
1519+ production_fragment_status="region_fragment_inlined_unoptimized",
1520+ production_fragment_reason=(
1521+ "B2 scan+combine backward is a grid-launched kernel (own "
1522+ "T.Kernel grid) delegating to chunk_scan_combine_bwd_metal_prim "
1523+ "— the analytic transpose of the forward F2. Validated vs the "
1524+ "MLX backward proto (worst grad 3.68e-4) by "
1525+ "scratch/test_b0b1b2_metal_vs_proto.py (Stage 3)"
1526+ ),
1527+ supports_backward=False,
1528+ preferred_loop_policy=DESCRIPTOR_LOOP_POLICY_FLAT,
1529+ fragment_emitter=_emit_mamba3_chunk_scan_combine_bwd_source,
1530+ ),
1531+ PathCBrickScheduleDescriptor(
1532+ op_name="mamba3_inter_chunk_recur_bwd",
1533+ implementation_status="descriptor_codegen_ready",
1534+ required_codegen_steps=(
1535+ "mamba3_inter_chunk_recur_bwd_descriptor",
1536+ "mamba3_inter_chunk_recur_bwd_grid_kernel",
1537+ ),
1538+ description=(
1539+ "Mamba3 chunked SSD inter-chunk recurrence BACKWARD (B1) brick "
1540+ "descriptor; the NEW O(S/C) REVERSE upper-tri combiner; "
1541+ "delegates to inter_chunk_recur_bwd_metal_prim (reuses forward "
1542+ "prev_states -> dstates/dh0/dA_cumsum_tail; no 8x replay)"
1543+ ),
1544+ production_source=(
1545+ "cppmega_mlx.nn._tilelang.mamba3_chunked_backward_core:"
1546+ "build_inter_chunk_recur_bwd_metal/"
1547+ "inter_chunk_recur_bwd_metal_prim"
1548+ ),
1549+ production_fragment_status="region_fragment_inlined_unoptimized",
1550+ production_fragment_reason=(
1551+ "B1 inter-chunk recurrence backward is the REVERSE O(S/C) "
1552+ "upper-tri combiner (own T.Kernel(batch,nheads) grid; the chunk "
1553+ "axis is the only serial REVERSE carry) — the adjoint of the "
1554+ "forward F1 lower-tri recurrence. REUSES the forward-"
1555+ "materialized prev_states (the 8x checkpoint-replay elimination, "
1556+ "design §3/§6). Validated by scratch/test_b0b1b2_metal_vs_proto.py"
1557+ ),
1558+ supports_backward=False,
1559+ preferred_loop_policy=DESCRIPTOR_LOOP_POLICY_FLAT,
1560+ fragment_emitter=_emit_mamba3_inter_chunk_recur_bwd_source,
1561+ ),
1562+ PathCBrickScheduleDescriptor(
1563+ op_name="mamba3_chunk_precompute_bwd",
1564+ implementation_status="descriptor_codegen_ready",
1565+ required_codegen_steps=(
1566+ "mamba3_chunk_precompute_bwd_descriptor",
1567+ "mamba3_chunk_precompute_bwd_grid_kernel",
1568+ ),
1569+ description=(
1570+ "Mamba3 chunked SSD precompute BACKWARD (B0) brick descriptor; "
1571+ "GRID-launched precompute transpose; delegates to "
1572+ "chunk_precompute_bwd_metal_prim (decay_states transpose + dinp "
1573+ "assembly + cumsum/segsum VJP -> dx/dB/dlog_decay/ddt)"
1574+ ),
1575+ production_source=(
1576+ "cppmega_mlx.nn._tilelang.mamba3_chunked_backward_core:"
1577+ "build_chunk_precompute_bwd_metal/"
1578+ "chunk_precompute_bwd_metal_prim"
1579+ ),
1580+ production_fragment_status="region_fragment_inlined_unoptimized",
1581+ production_fragment_reason=(
1582+ "B0 precompute backward is a grid-launched kernel (own T.Kernel "
1583+ "grid) delegating to chunk_precompute_bwd_metal_prim — the "
1584+ "transpose of the forward F0 precompute, assembling the final "
1585+ "input grads. Validated by scratch/test_b0b1b2_metal_vs_proto.py"
1586+ ),
1587+ supports_backward=False,
1588+ preferred_loop_policy=DESCRIPTOR_LOOP_POLICY_FLAT,
1589+ fragment_emitter=_emit_mamba3_chunk_precompute_bwd_source,
1590+ ),
14971591 PathCBrickScheduleDescriptor(
14981592 op_name="mamba3_mimo_bwd",
14991593 implementation_status="descriptor_codegen_ready",
@@ -6588,9 +6682,16 @@ def mamba3_chunked_forward_scan_grid(
65886682# ``CPPMEGA_PATH_C_MAMBA3_CHUNKED_SCAN`` flag is ON.
65896683_MAMBA3_CHUNKED_GRID_DELEGATION_OPS = frozenset(
65906684 {
6685+ # forward F0/F1/F2 (Stage 2)
65916686 "mamba3_chunk_precompute",
65926687 "mamba3_inter_chunk_recur",
65936688 "mamba3_chunk_scan_combine",
6689+ # backward B0/B1/B2 (Stage 3) — the analytic transpose of F0/F1/F2.
6690+ # Same builder ABI (build_*_metal / positional dims); the interpose
6691+ # resolves them identically (RULE #1: one delegation path each).
6692+ "mamba3_chunk_precompute_bwd",
6693+ "mamba3_inter_chunk_recur_bwd",
6694+ "mamba3_chunk_scan_combine_bwd",
65946695 }
65956696)
65966697
@@ -13220,6 +13321,98 @@ def _emit_mamba3_inter_chunk_recur_source(
1322013321 )
1322113322
1322213323
13324+ def _emit_mamba3_chunk_scan_combine_bwd_source(
13325+ node: _ScheduleNodeView,
13326+ dtype_by_buffer: dict[str, str],
13327+ access_by_buffer: dict[str, str],
13328+ ) -> _ScheduleNodeFragment:
13329+ """Fragment emitter for the Path-C B2 ``mamba3_chunk_scan_combine_bwd`` segment.
13330+
13331+ Design: ``docs/MAMBA3-PATHC-MULTIKERNEL-DESIGN.md`` §2 (B2 row) / §7 Stage 3.
13332+
13333+ B2 is the analytic TRANSPOSE of the forward F2 scan+combine (output/gate +
13334+ Y_diag + Y_off transpose). Its kernel is the GRID-launched core
13335+ ``chunk_scan_combine_bwd_metal_prim`` (own ``T.Kernel(batch*nchunks, nheads)``
13336+ grid). Real codegen is the single delegation
13337+ ``mamba3_chunked_backward_core.build_chunk_scan_combine_bwd_metal`` — validated
13338+ vs the MLX backward proto (worst grad 3.68e-4) by
13339+ ``scratch/test_b0b1b2_metal_vs_proto.py``. This marker only needs to RESOLVE
13340+ via ``select``; the live interpose substitutes the real grid kernel when the
13341+ flag is ON. RULE #1: no silent fallback — one delegation path.
13342+ """
13343+ marker = _scratch_name(node, "mamba3_chunk_scan_combine_bwd_shadow")
13344+ return _ScheduleNodeFragment(
13345+ allocations=(f"{marker} = T.alloc_local((1,), \"float32\")",),
13346+ statements=(
13347+ "# B2 mamba3_chunk_scan_combine_bwd: delegates to "
13348+ "cppmega_mlx.nn._tilelang.mamba3_chunked_backward_core."
13349+ "build_chunk_scan_combine_bwd_metal (grid output/Y transpose -> "
13350+ "dC/dx/dz/dchunk_states/dinp/dA_cumsum_y/dD)",
13351+ f"{marker}[0] = 0.0",
13352+ ),
13353+ )
13354+
13355+
13356+ def _emit_mamba3_chunk_precompute_bwd_source(
13357+ node: _ScheduleNodeView,
13358+ dtype_by_buffer: dict[str, str],
13359+ access_by_buffer: dict[str, str],
13360+ ) -> _ScheduleNodeFragment:
13361+ """Fragment emitter for the Path-C B0 ``mamba3_chunk_precompute_bwd`` segment.
13362+
13363+ Design: ``docs/MAMBA3-PATHC-MULTIKERNEL-DESIGN.md`` §2 (B0 row) / §7 Stage 3.
13364+
13365+ B0 is the transpose of the forward F0 precompute (decay_states transpose +
13366+ dinp assembly + cumsum/segsum VJP). Real codegen delegation
13367+ ``mamba3_chunked_backward_core.build_chunk_precompute_bwd_metal`` -> the final
13368+ input grads ``dx/dB/dlog_decay/ddt``. This marker only needs to RESOLVE via
13369+ ``select``; the live interpose substitutes the real grid kernel when the flag
13370+ is ON. RULE #1: no silent fallback — one delegation path.
13371+ """
13372+ marker = _scratch_name(node, "mamba3_chunk_precompute_bwd_shadow")
13373+ return _ScheduleNodeFragment(
13374+ allocations=(f"{marker} = T.alloc_local((1,), \"float32\")",),
13375+ statements=(
13376+ "# B0 mamba3_chunk_precompute_bwd: delegates to "
13377+ "cppmega_mlx.nn._tilelang.mamba3_chunked_backward_core."
13378+ "build_chunk_precompute_bwd_metal (grid precompute transpose -> "
13379+ "dx/dB/dlog_decay/ddt input grads)",
13380+ f"{marker}[0] = 0.0",
13381+ ),
13382+ )
13383+
13384+
13385+ def _emit_mamba3_inter_chunk_recur_bwd_source(
13386+ node: _ScheduleNodeView,
13387+ dtype_by_buffer: dict[str, str],
13388+ access_by_buffer: dict[str, str],
13389+ ) -> _ScheduleNodeFragment:
13390+ """Fragment emitter for the Path-C B1 ``mamba3_inter_chunk_recur_bwd`` segment.
13391+
13392+ Design: ``docs/MAMBA3-PATHC-MULTIKERNEL-DESIGN.md`` §2 (B1 row) / §7 Stage 3.
13393+
13394+ B1 is the ONE genuinely new kernel: the REVERSE O(S/C) inter-chunk combiner
13395+ (upper-tri ``decay_chunk`` contraction — the adjoint of the forward F1 lower-tri
13396+ recurrence). It REUSES the forward-materialized ``prev_states`` boundary states
13397+ (the 8x checkpoint-replay elimination, design §3/§6). Real codegen delegation
13398+ ``mamba3_chunked_backward_core.build_inter_chunk_recur_bwd_metal`` ->
13399+ ``dstates/dh0/dA_cumsum_tail``. This marker only needs to RESOLVE via
13400+ ``select``; the live interpose substitutes the real grid kernel when the flag
13401+ is ON. RULE #1: no silent fallback — one delegation path.
13402+ """
13403+ marker = _scratch_name(node, "mamba3_inter_chunk_recur_bwd_shadow")
13404+ return _ScheduleNodeFragment(
13405+ allocations=(f"{marker} = T.alloc_local((1,), \"float32\")",),
13406+ statements=(
13407+ "# B1 mamba3_inter_chunk_recur_bwd: delegates to "
13408+ "cppmega_mlx.nn._tilelang.mamba3_chunked_backward_core."
13409+ "build_inter_chunk_recur_bwd_metal (REVERSE O(S/C) upper-tri combiner "
13410+ "-> dstates/dh0/dA_cumsum_tail; reuses forward prev_states)",
13411+ f"{marker}[0] = 0.0",
13412+ ),
13413+ )
13414+
13415+
1322313416def _emit_residual_rmsnorm_source(
1322413417 node: _ScheduleNodeView,
1322513418 dtype_by_buffer: dict[str, str],
0 commit comments