5555 "MAMBA3_INTER_CHUNK_RECUR_BWD_OP_NAME" ,
5656 "MAMBA3_CHUNK_PRECOMPUTE_BWD_OP_NAME" ,
5757 "chunk_scan_combine_bwd_metal_prim" ,
58+ "chunk_scan_combine_bwd_cuda_prim" ,
5859 "inter_chunk_recur_bwd_metal_prim" ,
5960 "chunk_precompute_bwd_metal_prim" ,
6061 "chunk_scan_combine_bwd_grid" ,
@@ -391,6 +392,303 @@ def main(
391392 return main
392393
393394
395+ def chunk_scan_combine_bwd_cuda_prim (
396+ batch : int ,
397+ seqlen : int ,
398+ chunk_size : int ,
399+ ngroups : int ,
400+ nheads : int ,
401+ headdim : int ,
402+ dstate : int ,
403+ * ,
404+ threads : int = 128 ,
405+ ) -> Any :
406+ """CUDA (gb10 / sm_121) twin of :func:`chunk_scan_combine_bwd_metal_prim`.
407+
408+ IDENTICAL analytic backward math (it is the transpose of the forward F2 scan +
409+ combine). The ONLY delta vs the Metal prim is the THREAD MAPPING of the two
410+ terms the Metal prim funnels through lane 0:
411+
412+ * the ``dC = dC_off + dC_diag + dstate_decay`` block (Metal lines 274-311),
413+ whose dominant ``dC_diag`` inner did
414+ ``L*dstate*headdim*Sum_{ll}(ll+1) ~= 8.5M`` serial MACs on ONE lane, and
415+ * the ``Y_diag`` ``dseg`` segsum-VJP block (Metal lines 360-386), another
416+ ``headdim*Sum(ll+1) ~= 133K`` serial MACs on lane 0.
417+
418+ Re-grid (same ``(batch*nchunks, nheads)`` grid, 128 threads, NO new grid dims —
419+ the per-threadgroup ``dAcs_acc``/``dY``/``DYX`` tiles must stay co-resident):
420+
421+ 1. Stage ``x[base+0..L-1, head, 0..headdim-1]`` into a shared ``XT[L,headdim]``
422+ tile (built once via ``T.Parallel(L*headdim)``).
423+ 2. Build the lower-tri shared tile
424+ ``DYX[ll,ss] = sum_p dY[ll,pp]*XT[ss,pp]`` (ss<=ll) ONCE, mapped over
425+ ``T.Parallel(L*L)`` with each thread doing the headdim reduction. This is
426+ the recompute-killer: the Metal prim recomputed this exact quantity
427+ ``dstate`` times inside ``dC_diag`` AND a second time as ``dlmat`` inside
428+ ``dseg``; here it is built once and consumed by BOTH (matches the proto /
429+ forward F2 ``cb`` decay+dt+mask reuse, just transposed).
430+ 3. ``dC_off + dC_diag + dstate_decay``: map ``(ll,nn)`` over
431+ ``T.Parallel(L*dstate)``. Each thread owns one ``dC[*,base+ll,head,nn]``
432+ cell (UNIQUE per thread -> race-free direct write): the serial ``pp``
433+ reduction gives ``accn`` (dC_off), the serial ``ss<=ll`` reduction over
434+ the precomputed ``DYX`` gives ``cdiag`` (dC_diag), and the per-(ll)
435+ ``dstate_decay`` (an ``nn``-reduction of ``accn*C[l,n]``) is folded into
436+ ``dAcs_acc[ll]`` via ``T.atomic_add`` (the ONLY multi-writer cell).
437+ 4. ``dseg`` (Y_diag dA grad): lane-strided over the lower-tri ``(ll,ss)``
438+ pairs (mirrors the existing ``dinp`` lane-strided pattern), reusing
439+ ``DYX`` (zero recompute), with the ``+dAcs_acc[ll] / -dAcs_acc[ss]``
440+ segsum-VJP scatter done via ``T.atomic_add`` (both contributions race on
441+ shared ``dAcs_acc``).
442+
443+ The fast already-threaded parts (dz/dx/dD ``T.Parallel(L*headdim)``,
444+ ``dchunk_states`` ``T.Parallel(headdim*dstate)``, ``dinp`` lane-strided) are
445+ copied VERBATIM — they already match the B0 ~111ms threaded pattern.
446+
447+ dD PARITY NOTE (RULE #1): the dD path (the ``dy_v*x_v`` reduction +
448+ ``T.atomic_add(dD[head], ...)``) is UNCHANGED and is already fp32-correct
449+ (proven bit-exact vs fp32 gold when fed fp32 inputs). The 1.40e-3 gate miss is
450+ a GOLD-vs-kernel INPUT-PRECISION mismatch (gold read fp32 x/z/dout while the
451+ kernel reads the fp16 forward cache), NOT a kernel bug — so it is fixed in the
452+ probe's gold (align the dD VJP to the SAME fp16 cache the kernel consumes), NOT
453+ by changing this kernel. See scratch/probe_chunked_backward_cuda_gb10.py.
454+
455+ Same pass_configs (``tl.disable_tma_lower`` / ``tl.disable_warp_specialized``)
456+ and out_idx [11..17] as the Metal compile-site; selected only when the resolved
457+ target is CUDA (the Metal prim stays byte-identical). RULE #1: this is the ONE
458+ CUDA path; compile/parity failures RAISE (no fallback to the slow prim/numpy).
459+ """
460+ import tilelang
461+ import tilelang .language as T
462+
463+ if seqlen % chunk_size != 0 :
464+ raise ValueError (
465+ f"chunk_scan_combine_bwd_cuda_prim: seqlen ({ seqlen } ) must be "
466+ f"divisible by chunk_size ({ chunk_size } ); no padding (RULE #1)"
467+ )
468+ if nheads % ngroups != 0 :
469+ raise ValueError (
470+ f"chunk_scan_combine_bwd_cuda_prim: nheads ({ nheads } ) must be "
471+ f"divisible by ngroups ({ ngroups } )"
472+ )
473+
474+ dtype = T .float16
475+ accum_dtype = T .float32
476+ nchunks = seqlen // chunk_size
477+ heads_per_group = nheads // ngroups
478+ L = chunk_size
479+ p = _LOG2E
480+
481+ @T .prim_func
482+ def main (
483+ dout : T .Tensor ((batch , seqlen , nheads , headdim ), dtype ), # type: ignore
484+ cb : T .Tensor ((batch , nchunks , ngroups , chunk_size , chunk_size ), dtype ), # type: ignore
485+ x : T .Tensor ((batch , seqlen , nheads , headdim ), dtype ), # type: ignore
486+ z : T .Tensor ((batch , seqlen , nheads , headdim ), dtype ), # type: ignore
487+ dt : T .Tensor ((batch , nheads , nchunks , chunk_size ), dtype ), # type: ignore
488+ dA_cumsum : T .Tensor ((batch , nheads , nchunks , chunk_size ), dtype ), # type: ignore
489+ C : T .Tensor ((batch , seqlen , ngroups , dstate ), dtype ), # type: ignore
490+ B : T .Tensor ((batch , seqlen , ngroups , dstate ), dtype ), # type: ignore
491+ prev_states : T .Tensor ((batch , nchunks , nheads , headdim , dstate ), accum_dtype ), # type: ignore
492+ D : T .Tensor ((nheads ), dtype ), # type: ignore
493+ y : T .Tensor ((batch , seqlen , nheads , headdim ), dtype ), # type: ignore
494+ dC : T .Tensor ((batch , seqlen , nheads , dstate ), accum_dtype ), # type: ignore
495+ dx : T .Tensor ((batch , seqlen , nheads , headdim ), accum_dtype ), # type: ignore
496+ dz : T .Tensor ((batch , seqlen , nheads , headdim ), accum_dtype ), # type: ignore
497+ dchunk_states : T .Tensor ((batch , nchunks , nheads , headdim , dstate ), accum_dtype ), # type: ignore
498+ dinp : T .Tensor ((batch , seqlen , nheads , headdim , dstate ), accum_dtype ), # type: ignore
499+ dA_cumsum_y : T .Tensor ((batch , nheads , nchunks , chunk_size ), accum_dtype ), # type: ignore
500+ dD : T .Tensor ((nheads ), accum_dtype ), # type: ignore
501+ ):
502+ with T .Kernel (batch * nchunks , nheads , threads = threads ) as (bx , by ):
503+ batch_idx = bx % batch
504+ chunk_idx = bx // batch
505+ head_idx = by
506+ group_idx = head_idx // heads_per_group
507+ base = chunk_idx * chunk_size
508+
509+ dacs = T .alloc_shared ((L ,), accum_dtype ) # dA_cumsum row (this head/chunk)
510+ dY = T .alloc_shared ((L , headdim ), accum_dtype ) # dY[l,p] (post split)
511+ dAcs_acc = T .alloc_shared ((L ,), accum_dtype ) # accumulated dA_cumsum grad
512+ XT = T .alloc_shared ((L , headdim ), accum_dtype ) # staged x[base+s, head, p]
513+ DYX = T .alloc_shared ((L , L ), accum_dtype ) # dyx[l,s] = sum_p dY[l,p]*x[s,p]
514+
515+ # --- load dA_cumsum row, init dA grad accumulator ---
516+ for l in T .Parallel (L ):
517+ dacs [l ] = T .Cast (accum_dtype , dA_cumsum [batch_idx , head_idx , chunk_idx , l ])
518+ dAcs_acc [l ] = T .Cast (accum_dtype , 0 )
519+ T .sync_threads ()
520+
521+ # --- output/gate + D-skip transpose: dY[l,p], dz, dx, dD ---
522+ # (VERBATIM from the Metal prim — already threaded over L*headdim;
523+ # dD is fp32-correct, the 1.40e-3 gate miss is an input-precision
524+ # GOLD mismatch fixed in the probe, NOT here. RULE #1.)
525+ # out = gate*y ; gate=silu(z) ; y = Y + D*x
526+ # dgate = dout*y ; dy = dout*gate ; dz = dgate*silu'(z)
527+ # dx_skip = D*dy ; dD += dy*x ; dY = dy
528+ dD_local = T .alloc_local ((1 ,), accum_dtype )
529+ dD_local [0 ] = T .Cast (accum_dtype , 0 )
530+ for lp in T .Parallel (L * headdim ):
531+ ll = lp // headdim
532+ pp = lp % headdim
533+ s = base + ll
534+ z_v = T .Cast (accum_dtype , z [batch_idx , s , head_idx , pp ])
535+ gate = z_v / (T .Cast (accum_dtype , 1.0 ) + T .exp (- z_v ))
536+ y_v = T .Cast (accum_dtype , y [batch_idx , s , head_idx , pp ])
537+ dout_v = T .Cast (accum_dtype , dout [batch_idx , s , head_idx , pp ])
538+ dgate = dout_v * y_v
539+ dy_v = dout_v * gate
540+ dz [batch_idx , s , head_idx , pp ] = dgate * _silu_grad_expr (T , z_v , accum_dtype )
541+ d_v = T .Cast (accum_dtype , D [head_idx ])
542+ x_v = T .Cast (accum_dtype , x [batch_idx , s , head_idx , pp ])
543+ dx [batch_idx , s , head_idx , pp ] = d_v * dy_v
544+ dD_local [0 ] = dD_local [0 ] + dy_v * x_v
545+ dY [ll , pp ] = dy_v
546+ # Stage x for THIS chunk/head into shared once (XT[s,p]); reused by
547+ # the DYX build below (the dC_diag + dseg recompute-killer).
548+ XT [ll , pp ] = x_v
549+ T .sync_threads ()
550+ T .atomic_add (dD [head_idx ], dD_local [0 ])
551+
552+ # First zero the dC / dinp / dchunk_states slices this threadgroup owns
553+ # (each (batch,chunk,head) owns disjoint slices). VERBATIM from Metal.
554+ for ln in T .Parallel (L * dstate ):
555+ ll = ln // dstate
556+ nn = ln % dstate
557+ dC [batch_idx , base + ll , head_idx , nn ] = T .Cast (accum_dtype , 0 )
558+ for pn in T .Parallel (headdim * dstate ):
559+ pp = pn // dstate
560+ nn = pn % dstate
561+ dchunk_states [batch_idx , chunk_idx , head_idx , pp , nn ] = T .Cast (
562+ accum_dtype , 0
563+ )
564+ for lpn0 in T .serial (0 , L * headdim * dstate , threads ):
565+ lane = T .get_thread_binding (0 )
566+ idx = lpn0 + lane
567+ if idx < L * headdim * dstate :
568+ ll = idx // (headdim * dstate )
569+ rem = idx % (headdim * dstate )
570+ pp = rem // dstate
571+ nn = rem % dstate
572+ dinp [batch_idx , base + ll , head_idx , pp , nn ] = T .Cast (
573+ accum_dtype , 0
574+ )
575+ T .sync_threads ()
576+
577+ # ---- BUILD shared DYX[l,s] = sum_p dY[l,p]*x[s,p] (ss<=ll, lower-tri) ----
578+ # This is the SINGLE quantity the Metal prim recomputed dstate-times in
579+ # dC_diag (lines 299-305) AND again as dlmat in dseg (lines 371-378).
580+ # Built ONCE over L*L work items spread across 128 threads.
581+ for ls in T .Parallel (L * L ):
582+ ll = ls // L
583+ ss = ls % L
584+ acc = T .alloc_local ((1 ,), accum_dtype )
585+ acc [0 ] = T .Cast (accum_dtype , 0 )
586+ if ss <= ll :
587+ for pp in T .serial (headdim ):
588+ acc [0 ] = acc [0 ] + dY [ll , pp ] * XT [ss , pp ]
589+ DYX [ll , ss ] = acc [0 ]
590+ T .sync_threads ()
591+
592+ # ---- dC = dC_off + dC_diag (per (l,n)); dstate_decay -> dAcs_acc ----
593+ # dC_off[l,n] = sum_p dY[l,p]*prev_states[p,n]*state_decay[l]
594+ # dC_diag[l,n] = sum_{s<=l} Lmat[l,s]*dt_s*DYX[l,s]*B[s,n]
595+ # Lmat[l,s] = exp(dacs[l]-dacs[s]). inp=dt*x⊗B so dt_s appears here.
596+ # dstate_decay[l] = sum_n (dC_off-inner)*C[l,n] ; dAcs_acc[l] += dsd*sd
597+ # Re-gridded: each (ll,nn) thread owns one dC cell (UNIQUE -> race-free
598+ # write); dstate_decay's nn-reduction folds into dAcs_acc via atomic_add.
599+ for ln in T .Parallel (L * dstate ):
600+ ll = ln // dstate
601+ nn = ln % dstate
602+ s = base + ll
603+ sd = T .exp2 (dacs [ll ] * p )
604+ # dC_off inner: accn = sum_p dY[l,p]*prev_states[p,n]
605+ accn = T .alloc_local ((1 ,), accum_dtype )
606+ accn [0 ] = T .Cast (accum_dtype , 0 )
607+ for pp in T .serial (headdim ):
608+ cs = prev_states [batch_idx , chunk_idx , head_idx , pp , nn ]
609+ accn [0 ] = accn [0 ] + dY [ll , pp ] * cs
610+ # dC_diag inner: cdiag = sum_{s2<=l} Lmat[l,s2]*dt_s2*DYX[l,s2]*B[s2,n]
611+ cdiag = T .alloc_local ((1 ,), accum_dtype )
612+ cdiag [0 ] = T .Cast (accum_dtype , 0 )
613+ for ss in T .serial (0 , ll + 1 ):
614+ lmat = T .exp2 ((dacs [ll ] - dacs [ss ]) * p )
615+ dt_s = T .Cast (accum_dtype , dt [batch_idx , head_idx , chunk_idx , ss ])
616+ b_v = T .Cast (accum_dtype , B [batch_idx , base + ss , group_idx , nn ])
617+ cdiag [0 ] = cdiag [0 ] + lmat * dt_s * DYX [ll , ss ] * b_v
618+ dC [batch_idx , s , head_idx , nn ] = accn [0 ] * sd + cdiag [0 ]
619+ # dstate_decay nn-reduction -> dAcs_acc[ll] (multi-writer: atomic).
620+ c_v = T .Cast (accum_dtype , C [batch_idx , s , group_idx , nn ])
621+ T .atomic_add (dAcs_acc [ll ], accn [0 ] * c_v * sd )
622+ T .sync_threads ()
623+
624+ # dchunk_states[p,n] += sum_l dY[l,p]*C[l,n]*state_decay[l] (VERBATIM).
625+ for pn in T .Parallel (headdim * dstate ):
626+ pp = pn // dstate
627+ nn = pn % dstate
628+ acc = T .alloc_local ((1 ,), accum_dtype )
629+ acc [0 ] = T .Cast (accum_dtype , 0 )
630+ for ll in T .serial (L ):
631+ sd = T .exp2 (dacs [ll ] * p )
632+ c_v = T .Cast (accum_dtype , C [batch_idx , base + ll , group_idx , nn ])
633+ acc [0 ] = acc [0 ] + dY [ll , pp ] * c_v * sd
634+ dchunk_states [batch_idx , chunk_idx , head_idx , pp , nn ] = acc [0 ]
635+ T .sync_threads ()
636+
637+ # ---- Y_diag transpose (intra-chunk) -> dinp (VERBATIM, already threaded) ----
638+ # dinp[s,p,n] (grad wrt inp=dt*x⊗B) = sum_{l>=s} dY[l,p]*C[l,n]*Lmat[l,s]
639+ for sp in T .serial (0 , L * headdim , threads ):
640+ lane = T .get_thread_binding (0 )
641+ spi = sp + lane
642+ if spi < L * headdim :
643+ ss = spi // headdim
644+ pp = spi % headdim
645+ sidx = base + ss
646+ for nn in T .serial (dstate ):
647+ acc = T .alloc_local ((1 ,), accum_dtype )
648+ acc [0 ] = T .Cast (accum_dtype , 0 )
649+ for ll in T .serial (ss , L ): # l >= s (lower-tri)
650+ lmat = T .exp2 ((dacs [ll ] - dacs [ss ]) * p )
651+ c_v = T .Cast (
652+ accum_dtype , C [batch_idx , base + ll , group_idx , nn ]
653+ )
654+ acc [0 ] = acc [0 ] + dY [ll , pp ] * c_v * lmat
655+ dinp [batch_idx , sidx , head_idx , pp , nn ] = (
656+ dinp [batch_idx , sidx , head_idx , pp , nn ] + acc [0 ]
657+ )
658+ T .sync_threads ()
659+
660+ # ---- Y_diag dA grad (dseg) — RE-GRIDDED off lane 0 ----
661+ # dLmat[l,s] = sum_p dY[l,p]*x[s,p] == DYX[l,s] (reuse, zero recompute)
662+ # M[l,s] = cb*lmat*dt_s ; dseg = DYX[l,s]*M[l,s]
663+ # segsum-vjp: +dAcs_acc[l] over l>=s, -dAcs_acc[s] over s<l (strictly l>s).
664+ # Lane-strided over the L*L (ll,ss) grid with the ll>ss mask (mirrors the
665+ # dinp lane-strided pattern); the +/- scatter races on dAcs_acc -> atomic.
666+ for ls0 in T .serial (0 , L * L , threads ):
667+ lane = T .get_thread_binding (0 )
668+ lsi = ls0 + lane
669+ if lsi < L * L :
670+ ll = lsi // L
671+ ss = lsi % L
672+ if ll > ss :
673+ cb_v = T .Cast (
674+ accum_dtype , cb [batch_idx , chunk_idx , group_idx , ll , ss ]
675+ )
676+ lmat = T .exp2 ((dacs [ll ] - dacs [ss ]) * p )
677+ dt_s = T .Cast (
678+ accum_dtype , dt [batch_idx , head_idx , chunk_idx , ss ]
679+ )
680+ m_ls = cb_v * lmat * dt_s
681+ dseg = DYX [ll , ss ] * m_ls
682+ T .atomic_add (dAcs_acc [ll ], dseg )
683+ T .atomic_add (dAcs_acc [ss ], - dseg )
684+ T .sync_threads ()
685+
686+ for l in T .Parallel (L ):
687+ dA_cumsum_y [batch_idx , head_idx , chunk_idx , l ] = dAcs_acc [l ]
688+
689+ return main
690+
691+
394692def build_chunk_scan_combine_bwd_metal (
395693 batch : int ,
396694 seqlen : int ,
@@ -416,20 +714,28 @@ def build_chunk_scan_combine_bwd_metal(
416714 _resolve_chunked_compile_target ,
417715 )
418716
419- prim = chunk_scan_combine_bwd_metal_prim (
420- batch , seqlen , chunk_size , ngroups , nheads , headdim , dstate , ** kwargs
421- )
422717 resolved_target = _resolve_chunked_compile_target (target )
718+ # B2 is — like the forward F2 — the ONE backward stage whose prim BODY differs
719+ # by backend: the CUDA twin RE-GRIDS the two lane-0 funnels (dC_diag + dseg)
720+ # across all 128 threads (the dominant ~8.5M-serial-MAC dC_diag hotspot, and
721+ # the dseg segsum-VJP), staging a shared DYX[L,L] recompute-killer tile. The
722+ # Metal prim (chunk_scan_combine_bwd_metal_prim) stays BYTE-IDENTICAL so Metal
723+ # callers are unaffected. Select the matching prim by resolved target kind.
423724 # CUDA (sm_121): mirror the forward F2 compile-site EXACTLY — thread the same
424725 # two pass_configs so the CUDA codegen surface is identical to the validated
425- # forward. The B2 prim BODY has NO T.gemm / TMA-eligible copy (grep-confirmed:
426- # zero T.gemm / shared.dyn / make_swizzled_layout), so there is no tensormap
427- # descriptor to mis-align; disabling the TMA + warp-specialized lowering is a
428- # no-op-or-safer escape hatch that keeps the compile path byte-identical to
429- # the forward. The Metal branch is UNCHANGED (no pass_configs). RULE #1:
430- # explicit per-target codegen choice, never a silent fallback.
726+ # forward. The B2 cuda prim BODY has NO T.gemm / TMA-eligible copy (the new
727+ # DYX/XT tiles are plain static shared filled by elementwise T.Parallel loads),
728+ # so there is no tensormap descriptor to mis-align; disabling the TMA +
729+ # warp-specialized lowering is a no-op-or-safer escape hatch that keeps the
730+ # compile path byte-identical to the forward. The Metal branch is UNCHANGED (no
731+ # pass_configs). RULE #1: explicit per-target codegen choice, never a silent
732+ # fallback — the CUDA prim is the ONE CUDA path; a compile/parity failure
733+ # propagates (no fallback to the slow Metal prim or numpy).
431734 kind = str (getattr (getattr (resolved_target , "kind" , None ), "name" , "" )).lower ()
432735 if "cuda" in kind :
736+ prim = chunk_scan_combine_bwd_cuda_prim (
737+ batch , seqlen , chunk_size , ngroups , nheads , headdim , dstate , ** kwargs
738+ )
433739 pass_configs = {
434740 "tl.disable_tma_lower" : True ,
435741 "tl.disable_warp_specialized" : True ,
@@ -440,6 +746,9 @@ def build_chunk_scan_combine_bwd_metal(
440746 target = resolved_target ,
441747 pass_configs = pass_configs ,
442748 )
749+ prim = chunk_scan_combine_bwd_metal_prim (
750+ batch , seqlen , chunk_size , ngroups , nheads , headdim , dstate , ** kwargs
751+ )
443752 return tilelang .compile (
444753 prim ,
445754 out_idx = [11 , 12 , 13 , 14 , 15 , 16 , 17 ],
0 commit comments