Skip to content

Commit 894c948

Browse files
authored
[CUDA] Fix qmm_naive K-tail dispatch for FP quantized kernels (#3445)
1 parent 211e57b commit 894c948

1 file changed

Lines changed: 3 additions & 2 deletions

File tree

  • mlx/backend/cuda/quantized/qmm

mlx/backend/cuda/quantized/qmm/qmm.cu

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,7 @@ void qmm_naive(
265265
if constexpr (k_major.value) {
266266
if (has_k_residue) {
267267
throw std::invalid_argument(
268-
"[quantized_matmul] K must be multiples of group_size.");
268+
"[quantized_matmul] K must be multiples of max(64, group_size).");
269269
}
270270
f.template operator()<false>();
271271
} else {
@@ -276,7 +276,8 @@ void qmm_naive(
276276
};
277277
int m = out.ndim() > 1 ? out.shape(-2) : 1;
278278
int k = x.shape(-1);
279-
bool has_k_residue = k % group_size != 0;
279+
int tile_k = std::max(64, group_size);
280+
bool has_k_residue = k % tile_k != 0;
280281
bool sm80 = encoder.device().compute_capability_major() >= 8;
281282
dispatch_bool(transpose, [&](auto k_major) {
282283
dispatch_k(k_major, has_k_residue, [&]<bool HasKResidue>() {

0 commit comments

Comments
 (0)