@@ -511,29 +511,13 @@ def main(
511511 dAcs_acc = T .alloc_shared ((L ,), accum_dtype ) # accumulated dA_cumsum grad
512512 XT = T .alloc_shared ((L , headdim ), accum_dtype ) # staged x[base+s, head, p]
513513 DYX = T .alloc_shared ((L , L ), accum_dtype ) # dyx[l,s] = sum_p dY[l,p]*x[s,p]
514- LMAT = T .alloc_shared ((L , L ), accum_dtype ) # exp2((dacs[l]-dacs[s])*p), l>=s
515514
516515 # --- load dA_cumsum row, init dA grad accumulator ---
517516 for l in T .Parallel (L ):
518517 dacs [l ] = T .Cast (accum_dtype , dA_cumsum [batch_idx , head_idx , chunk_idx , l ])
519518 dAcs_acc [l ] = T .Cast (accum_dtype , 0 )
520519 T .sync_threads ()
521520
522- # --- precompute the lower-tri intra-chunk decay LMAT[l,s] ONCE ---
523- # lmat[l,s] = exp2((dacs[l]-dacs[s])*p) for s<=l (else 0). The Metal prim
524- # recomputed this exp2 inside dC_diag (per (l,n,s)), dinp (per (s,p,n,l))
525- # AND dseg (per (l,s)) — ~65K exp2/threadgroup in dinp alone. Built once
526- # over L*L threads (4096 exp2), then read from shared everywhere below.
527- for ls in T .Parallel (L * L ):
528- ll = ls // L
529- ss = ls % L
530- LMAT [ll , ss ] = T .if_then_else (
531- ss <= ll ,
532- T .exp2 ((dacs [ll ] - dacs [ss ]) * p ),
533- T .Cast (accum_dtype , 0 ),
534- )
535- T .sync_threads ()
536-
537521 # --- output/gate + D-skip transpose: dY[l,p], dz, dx, dD ---
538522 # (VERBATIM from the Metal prim — already threaded over L*headdim;
539523 # dD is fp32-correct, the 1.40e-3 gate miss is an input-precision
@@ -627,7 +611,7 @@ def main(
627611 cdiag = T .alloc_local ((1 ,), accum_dtype )
628612 cdiag [0 ] = T .Cast (accum_dtype , 0 )
629613 for ss in T .serial (0 , ll + 1 ):
630- lmat = LMAT [ll , ss ]
614+ lmat = T . exp2 (( dacs [ll ] - dacs [ ss ]) * p )
631615 dt_s = T .Cast (accum_dtype , dt [batch_idx , head_idx , chunk_idx , ss ])
632616 b_v = T .Cast (accum_dtype , B [batch_idx , base + ss , group_idx , nn ])
633617 cdiag [0 ] = cdiag [0 ] + lmat * dt_s * DYX [ll , ss ] * b_v
@@ -663,7 +647,7 @@ def main(
663647 acc = T .alloc_local ((1 ,), accum_dtype )
664648 acc [0 ] = T .Cast (accum_dtype , 0 )
665649 for ll in T .serial (ss , L ): # l >= s (lower-tri)
666- lmat = LMAT [ll , ss ] # shared, precomputed (no exp2 here )
650+ lmat = T . exp2 (( dacs [ll ] - dacs [ ss ]) * p )
667651 c_v = T .Cast (
668652 accum_dtype , C [batch_idx , base + ll , group_idx , nn ]
669653 )
@@ -693,7 +677,7 @@ def main(
693677 ll = ls // L
694678 ss = ls % L
695679 cb_v = T .Cast (accum_dtype , cb [batch_idx , chunk_idx , group_idx , ll , ss ])
696- lmat = LMAT [ll , ss ] # shared, precomputed (no exp2 here )
680+ lmat = T . exp2 (( dacs [ll ] - dacs [ ss ]) * p )
697681 dt_s = T .Cast (accum_dtype , dt [batch_idx , head_idx , chunk_idx , ss ])
698682 tri = T .if_then_else (
699683 ss < ll , T .Cast (accum_dtype , 1 ), T .Cast (accum_dtype , 0 )
0 commit comments