Skip to content

Commit 2d0ced8

Browse files
committed
docs: explain empty sequence graph padding
1 parent f258610 commit 2d0ced8

1 file changed

Lines changed: 3 additions & 0 deletions

File tree

xllm/python/model_executor/runners/decode_cuda_graph.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -295,6 +295,9 @@ def _fill_host_metadata(
295295
out=entry.host_seq_lens[:batch_size],
296296
)
297297
if padded_batch_size > batch_size:
298+
# Keep padding sequences empty in FlashInfer's host planner
299+
# metadata. Using seq_len=1 dummy sequences would schedule the
300+
# padding rows as real decode work and add unnecessary overhead.
298301
entry.host_seq_lens[batch_size:padded_batch_size].zero_()
299302
torch.add(
300303
entry.host_seq_lens,

0 commit comments

Comments
 (0)