Skip to content

Commit b36d8db

Browse files
author
Guenther Schmuelling
authored
enable dynamic max_k_step in FA for nvidia (microsoft#28511)
apply microsoft#27780 for nvidia. I see 10-15% performance improvement for prefill on rtx5060ti
1 parent a8ba94a commit b36d8db

1 file changed

Lines changed: 5 additions & 4 deletions

File tree

onnxruntime/contrib_ops/webgpu/bert/flash_attention.h

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -92,11 +92,12 @@ class FlashAttentionProgram final : public Program<FlashAttentionProgram> {
9292
q_BNSH_(q_BNSH),
9393
use_seqlen_k_(use_seqlen_k),
9494
has_head_sink_(has_head_sink) {
95-
if (is_apple) {
96-
// On Apple GPUs, use an optimized loop-based path with dynamic max_k_step.
97-
// Compute max_k_step from shared memory budget: k_tile + v_tile = 2 * element_size * head_size * max_k_step
95+
if (is_apple || is_nvidia) {
96+
// On Apple and NVIDIA, use an optimized loop-based path with dynamic max_k_step.
97+
// Compute max_k_step from workgroup shared memory budget: k_tile + v_tile = 2 * element_size * head_size * max_k_step
9898
const int element_size = is_fp16 ? 2 : 4;
99-
int max_k_from_shm = 16384 / (2 * element_size * qkv_head_size);
99+
constexpr int kMinWorkgroupStorageBudgetBytes = 16384;
100+
int max_k_from_shm = kMinWorkgroupStorageBudgetBytes / (2 * element_size * qkv_head_size);
100101
if (max_k_from_shm >= 32) {
101102
max_k_step_ = 32;
102103
} else {

0 commit comments

Comments
 (0)