Skip to content

Commit 26b34dc

Browse files
committed
[None][perf] optimize DSV4 FP8 o_a decode epilogue
Signed-off-by: Mingyang Hao <200044211+mingyangHao@users.noreply.github.com>
1 parent 6ce2609 commit 26b34dc

3 files changed

Lines changed: 303 additions & 13 deletions

File tree

tensorrt_llm/_torch/custom_ops/cute_dsl_custom_ops.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3821,6 +3821,9 @@ def forward_fp8out(
38213821
sf_n = ceil_div(n, 128)
38223822
aligned_mn = pad_up(m, 4)
38233823
n_tiles_per_group = ceil_div(n, 128)
3824+
# The cooperative SMEM epilogue wins only for the single-token
3825+
# decode shape; retain the register epilogue for every other M.
3826+
use_fp8_smem_epilogue = m == 1
38243827

38253828
if c_tensor.dtype != torch.float8_e4m3fn or c_tensor.shape != (
38263829
batch_size, m, n):
@@ -3840,6 +3843,7 @@ def forward_fp8out(
38403843
mma_tiler_mn,
38413844
cluster_shape_mn,
38423845
self.use_tvm_ffi,
3846+
use_fp8_smem_epilogue,
38433847
)
38443848
if cache_key not in self.__class__.kernel_cache:
38453849
a_ptr = make_ptr(cutlass.Float8E4M3FN,
@@ -3894,6 +3898,7 @@ def forward_fp8out(
38943898
sf_out_ptr,
38953899
aligned_mn,
38963900
n_tiles_per_group,
3901+
use_fp8_smem_epilogue,
38973902
options="--opt-level 2 --enable-tvm-ffi",
38983903
)
38993904
self.__class__.kernel_cache[cache_key] = compiled_gemm

0 commit comments

Comments
 (0)