Skip to content

Commit 989c4f8

Browse files
committed
refactor: RULE #1 - kill 4 P0 silent fallbacks (surface hidden Path C bugs)
Replace 4 automated/silent fallbacks with immediate raises (RULE #1: no automated fallbacks, fail loud). Each kill surfaced a real, previously-hidden bug: - GDN dispatch (linear_attention_paths.py:108) + KDA dispatch (kda_paths.py:96): _dispatch_failure_or_fallback silently ran Path A when the selected Path C/D/E kernel crashed -> now raises. SURFACED: GDN/KDA Path C broken (SystemError: ffi.Function returned NULL at the tvm-ffi boundary). - Mamba3 dispatch (mamba3.py:502): `except RuntimeError: pass` silently dropped a crashed Path C to Path B/ref (falsely commented fail-closed) -> now raises. SURFACED: Mamba3 Path C DLPackConversionError (tensor not contiguous). - Engine dispatch (_engine_dispatch.py:63): `except Exception: pass` swallowed FP8-intrinsic registration failure -> now only genuine import-absent is skipped; the assert propagates. 4 tests were green ONLY because the fallback handed them Path A output (never exercised Path C) -> they now fail correctly; the fallback was NOT restored. The surfaced bugs (GDN/KDA Path C ffi-NULL, Mamba3 Path C contiguity) are the next fixes. Staged: _cuda_eager writeback swallows, compiled.py eager-replaces-fused, gated matmul2d, MSL-shim fallbacks.
1 parent c0fdae3 commit 989c4f8

4 files changed

Lines changed: 52 additions & 14 deletions

File tree

cppmega_mlx/nn/_tilelang/_engine_dispatch.py

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,20 @@ def _with_pass_context(pass_configs: dict[str, Any] | None):
6161

6262

6363
def _ensure_path_c_metal_intrinsics_registered() -> None:
64+
# RULE #1 (no automated/silent fallbacks): if the Metal FP8 intrinsics are
65+
# not registered, a Path C FP8 kernel would compile/run against MISSING
66+
# intrinsics and silently produce a wrong result. The only acceptable
67+
# silent skip is the genuine "feature not present in this tilelang build"
68+
# case (the assert helper itself does not exist) — that is a
69+
# pre-existing-feature-not-available check, not an on-failure fallback.
6470
try:
6571
from tilelang.language.fp8_op import assert_metal_fp8_intrinsics_registered
66-
67-
assert_metal_fp8_intrinsics_registered()
68-
except Exception:
69-
pass
72+
except (ImportError, ModuleNotFoundError):
73+
# Older tilelang without the FP8 intrinsics module — nothing to assert.
74+
return
75+
# A failure of the assert itself means the intrinsics are genuinely NOT
76+
# registered: surface it instead of swallowing and running a broken kernel.
77+
assert_metal_fp8_intrinsics_registered()
7078

7179

7280
def _engine_compile(

cppmega_mlx/nn/mamba3.py

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -499,10 +499,21 @@ def _dispatch_mamba3_scan(
499499
y, h_last = mamba3_mimo_apply_with_state_path_c_fwd_path_b_bwd(
500500
x, B, C, z, A, dt, D, h0
501501
)
502-
except RuntimeError:
503-
# AUTO is fail-closed: graph/DLPack boundary failures keep the
504-
# production Path B route rather than allocating staging buffers.
505-
pass
502+
except RuntimeError as exc:
503+
# RULE #1 (no automated/silent fallbacks): a RUNTIME crash in
504+
# the selected Mamba3 Path C kernel is a bug in that clear
505+
# path. Silently dropping to Path B / the pure-MLX reference
506+
# would return a degraded result and hide the bug. RAISE so
507+
# the graph/DLPack boundary failure surfaces and we fix the
508+
# root cause. (An UNAVAILABLE Path C is handled before this
509+
# try-block by the status check, not here.)
510+
raise RuntimeError(
511+
"mamba3_mimo dispatch: selected Path C "
512+
f"({auto_mode!r}) crashed at runtime "
513+
f"({type(exc).__name__}: {exc}). Refusing to silently fall "
514+
"back to Path B / pure-MLX reference (RULE #1) — this "
515+
"points at a real bug in the Path C graph/DLPack boundary."
516+
) from exc
506517
else:
507518
record_dispatch(
508519
"mamba3_mimo",

cppmega_v4/_tilelang/kda_paths.py

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -100,9 +100,18 @@ def _dispatch_failure_or_fallback(
100100
*args,
101101
**kwargs,
102102
):
103-
if allow_fallback:
104-
return _path_a_call(*args, **kwargs)
105-
raise RuntimeError(f"KDA {path} dispatch failed and fallback disabled: {exc}") from exc
103+
# RULE #1 (no automated/silent fallbacks): a RUNTIME crash inside the
104+
# selected KDA kernel is a bug in that clear path. Silently switching to
105+
# Path A here would return a *different* (degraded) result and hide the
106+
# bug. Always RAISE so the failure surfaces and we fix the root cause.
107+
# (Explicit *unavailability* routing still goes through
108+
# ``_fallback_or_raise``; that is selection, not an on-failure fallback.)
109+
del allow_fallback, args, kwargs
110+
raise RuntimeError(
111+
f"KDA dispatch: selected {path} crashed at runtime "
112+
f"({type(exc).__name__}: {exc}). Refusing to silently fall back to "
113+
f"Path A (RULE #1) — this points at a real bug in {path}."
114+
) from exc
106115

107116

108117
def _path_b_call(*args, allow_fallback: bool = True, **kwargs):

cppmega_v4/_tilelang/linear_attention_paths.py

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -112,9 +112,19 @@ def _dispatch_failure_or_fallback(
112112
*args,
113113
**kwargs,
114114
):
115-
if allow_fallback:
116-
return _path_a_call(*args, **kwargs)
117-
raise RuntimeError(f"GDN {path} dispatch failed and fallback disabled: {exc}") from exc
115+
# RULE #1 (no automated/silent fallbacks): a RUNTIME crash inside the
116+
# selected GDN kernel is a bug in that clear path. Silently switching to
117+
# Path A here would return a *different* (degraded) result and hide the
118+
# bug. Always RAISE so the failure surfaces and we fix the root cause.
119+
# (Explicit *unavailability* routing — when a path advertises it cannot
120+
# run — still goes through ``_fallback_or_raise``; that is selection, not
121+
# an on-failure fallback.)
122+
del allow_fallback, args, kwargs
123+
raise RuntimeError(
124+
f"GDN dispatch: selected {path} crashed at runtime "
125+
f"({type(exc).__name__}: {exc}). Refusing to silently fall back to "
126+
f"Path A (RULE #1) — this points at a real bug in {path}."
127+
) from exc
118128

119129

120130
def _path_b_call(*args, allow_fallback: bool = True, **kwargs):

0 commit comments

Comments
 (0)