Skip to content

Commit bf158eb

Browse files
committed
fix(path-c): sparse_mla_fp8 contiguity (_flat_1d_view) + graph-route dkv zero-clear (2 RULE #1 FIRES)
FIRE rank-1 (contiguity): _flat_1d_view did `array.reshape((array.size,))` -- reshape preserves the non-contiguity of the prepared FP8 inputs (q_fp8/kv_fp8 from per-token/tensor scaling) -> tvm-ffi DLPack rejects (Tensor is not contiguous), re-raised by the force_path_c guard. Fix: `mx.contiguous(array.reshape((array.size,)))` (no-op when contiguous, dtype-preserving). One helper, every fwd/bwd dispatch routes through it. FIRE rank-4 (NaN dkv): the no-owner-output bwd route allocated dkv via tvm-ffi out_idx (uninitialized NaN-poison) while the kernel T.atomic_add ACCUMULATES into dkv -> 128/128 NaN. An in-kernel T.clear is unsafe (no cross-block global barrier -> clearing block races other blocks' scatters), so allocate dq/dkv as mx.zeros on the graph branch and route both cases through the single owner-output out=(dq,dkv) dispatch -> self-clearing regardless of allocator state. Metal: sparse_mla_fp8-prepared-small parity max_abs 1.22e-4 PASS; no-owner dkv finite 0/128 NaN (was 128/128); test_..._clears_atomic_dkv FAIL->PASS; test_tilelang_sparse_mla_fp8.py 55/1 -> 56/0. No fallback (force_path_c fail-loud raises preserved).
1 parent 7753e39 commit bf158eb

1 file changed

Lines changed: 45 additions & 45 deletions

File tree

cppmega_mlx/nn/_tilelang/sparse_mla_fp8_path_c.py

Lines changed: 45 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -2558,7 +2558,12 @@ def _owner_output_graph_tuple(
25582558

25592559

25602560
def _flat_1d_view(array: mx.array) -> mx.array:
2561-
return array.reshape((int(array.size),))
2561+
# reshape preserves the non-contiguity of prepared FP8 inputs
2562+
# (q_fp8/kv_fp8 produced by per-token/tensor scaling), and tvm-ffi DLPack
2563+
# rejects non-contiguous tensors. mx.contiguous is a no-op when the array
2564+
# is already contiguous and preserves dtype, so the 1-D view is always a
2565+
# contiguous DLPack-exportable buffer.
2566+
return mx.contiguous(array.reshape((int(array.size),)))
25622567

25632568

25642569
def _clear_fp8_bwd_dkv_buffer(dkv_buffer: mx.array) -> mx.array:
@@ -2624,9 +2629,16 @@ def _dispatch_fp8_bwd_owner_output_path_c(
26242629
dkv_needs_clear = True
26252630
graph_output_route = False
26262631
if dq_buffer is None and dkv_buffer is None:
2632+
# GRAPH-OUTPUT route: no caller-owned buffers were supplied. Allocating
2633+
# the dkv output via tvm-ffi out_idx hands back uninitialized (NaN-poison)
2634+
# memory, and the bwd kernel's T.atomic_add scatters *accumulating* into
2635+
# dkv -- so the buffer MUST be pre-zeroed. Allocate the gradient buffers
2636+
# here (mx.zeros, native graph outputs) and route through the same
2637+
# owner-output dispatch with a zeroed dkv, making the graph route
2638+
# self-clearing regardless of allocator state.
26272639
graph_output_route = True
2628-
dq_owner = None
2629-
dkv_owner = None
2640+
dq_owner = mx.zeros(dq_shape, dtype=mx.float32)
2641+
dkv_owner = mx.zeros(dkv_shape, dtype=mx.float32)
26302642
dkv_needs_clear = False
26312643
elif (dq_buffer is None) != (dkv_buffer is None):
26322644
raise SparseMLAFp8PathCDirectError(
@@ -2660,54 +2672,42 @@ def _dispatch_fp8_bwd_owner_output_path_c(
26602672
d_out_dtype,
26612673
index_dtype,
26622674
)
2663-
if graph_output_route:
2664-
dq_flat = None
2665-
dkv_flat = None
2666-
returned = kernel(
2667-
_flat_1d_view(q_fp8),
2668-
_flat_1d_view(q_scale),
2669-
_flat_1d_view(kv_fp8),
2670-
_flat_1d_view(kv_scale),
2671-
_flat_1d_view(d_out),
2672-
_flat_1d_view(indices),
2673-
sm_scale_buf,
2674-
)
2675-
else:
2676-
if dkv_needs_clear:
2677-
dkv_owner = _clear_fp8_bwd_dkv_buffer(cast(mx.array, dkv_owner))
2678-
dq_flat = _flat_1d_view(cast(mx.array, dq_owner))
2679-
dkv_flat = _flat_1d_view(cast(mx.array, dkv_owner))
2680-
returned = kernel(
2681-
_flat_1d_view(q_fp8),
2682-
_flat_1d_view(q_scale),
2683-
_flat_1d_view(kv_fp8),
2684-
_flat_1d_view(kv_scale),
2685-
_flat_1d_view(d_out),
2686-
_flat_1d_view(indices),
2687-
sm_scale_buf,
2688-
out=(dq_flat, dkv_flat),
2689-
)
2675+
# Both routes now dispatch into pre-zeroed owner buffers via out=:
2676+
# the graph route zeroed dq/dkv at allocation above; the owner route
2677+
# zero-clears the caller's dkv via the dedicated clear kernel. This
2678+
# guarantees the bwd kernel's accumulating atomic_add into dkv always
2679+
# starts from zero instead of NaN-poison freed memory.
2680+
if dkv_needs_clear:
2681+
dkv_owner = _clear_fp8_bwd_dkv_buffer(cast(mx.array, dkv_owner))
2682+
dq_flat = _flat_1d_view(cast(mx.array, dq_owner))
2683+
dkv_flat = _flat_1d_view(cast(mx.array, dkv_owner))
2684+
returned = kernel(
2685+
_flat_1d_view(q_fp8),
2686+
_flat_1d_view(q_scale),
2687+
_flat_1d_view(kv_fp8),
2688+
_flat_1d_view(kv_scale),
2689+
_flat_1d_view(d_out),
2690+
_flat_1d_view(indices),
2691+
sm_scale_buf,
2692+
out=(dq_flat, dkv_flat),
2693+
)
26902694
except Exception as exc:
26912695
if force_path_c:
2696+
route_label = (
2697+
"graph-output (self-allocated zeroed buffers)"
2698+
if graph_output_route
2699+
else "caller-owned"
2700+
)
26922701
raise RuntimeError(
2693-
"sparse_mla_fp8_bwd_path_c: direct tvm-ffi owner-output "
2702+
f"sparse_mla_fp8_bwd_path_c: direct tvm-ffi {route_label} "
26942703
f"dispatch failed: {type(exc).__name__}: {exc}"
26952704
) from exc
26962705
return None
2697-
if graph_output_route:
2698-
if not isinstance(returned, (list, tuple)) or len(returned) != 2:
2699-
raise SparseMLAFp8PathCDirectError(
2700-
"direct tvm-ffi FP8 Sparse-MLA backward graph-output route "
2701-
"did not return dq/dkv"
2702-
)
2703-
dq_flat = cast(mx.array, returned[0])
2704-
dkv_flat = cast(mx.array, returned[1])
2705-
else:
2706-
dq_flat, dkv_flat = _owner_output_graph_tuple(
2707-
returned,
2708-
expected=(cast(mx.array, dq_flat), cast(mx.array, dkv_flat)),
2709-
op_name="direct tvm-ffi FP8 Sparse-MLA backward",
2710-
)
2706+
dq_flat, dkv_flat = _owner_output_graph_tuple(
2707+
returned,
2708+
expected=(cast(mx.array, dq_flat), cast(mx.array, dkv_flat)),
2709+
op_name="direct tvm-ffi FP8 Sparse-MLA backward",
2710+
)
27112711
return dq_flat.reshape(dq_shape), dkv_flat.reshape(dkv_shape)
27122712

27132713

0 commit comments

Comments
 (0)