Skip to content

CUDA: route single-column f32 mul_mat through mmvf (transpose-free operand swap) #49

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
roberteg16 wants to merge 1 commit into
base: gfx11
Choose a base branch
from
+29 −0
Open

CUDA: route single-column f32 mul_mat through mmvf (transpose-free operand swap) #49

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions ggml/src/ggml-cuda/ggml-cuda.cu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1759,6 +1759,27 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor
ggml_cuda_mul_mat_vec_f(ctx, src0, src1, nullptr, dst);
return;
}
// A single-column f32 weight (ne01 == 1) times a wide activation (ne11 > MMVF_MAX_BATCH_SIZE)
// misses mmvf above because ne11 is the batch dim, so it falls back to a wasteful f32 cuBLAS
// GEMM. The output is a vector, so swapping the operands is transpose-free: the activation
// becomes the matrix and the single-column weight the lone F32 vector, with dst reinterpreted
// as [ne11, 1].
if (ne01 == 1 && ne11 > MMVF_MAX_BATCH_SIZE && ne2 == 1 && ne3 == 1
&& src0->type == GGML_TYPE_F32
&& ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst)
&& ggml_cuda_should_use_mmvf(src1->type, cc, src1->ne, src1->nb, /*ne11 =*/ 1)) {
// dst is [ne00=1, ne11] (one output feature, ne11 tokens); the swapped GEMV instead
// produces [ne11, 1] (ne11 rows, a single vector column). Both are the same ne11
// contiguous floats, so we only relabel the shape of a shallow copy - no data moves:
ggml_tensor dst_vec = *dst;
dst_vec.ne[0] = ne11; // rows: one result per token
dst_vec.ne[1] = 1; // a single output vector column
dst_vec.nb[1] = dst_vec.nb[0]*ne11; // nb[0] (element stride) stays; row dim spans ne11
dst_vec.nb[2] = dst_vec.nb[1]; // higher dims are 1 -> same stride
dst_vec.nb[3] = dst_vec.nb[1];
ggml_cuda_mul_mat_vec_f(ctx, src1, src0, nullptr, &dst_vec);
return;
}
if (ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, ne11, /*mul_mat_id =*/ false)) {
ggml_cuda_mul_mat_f(ctx, src0, src1, nullptr, dst);
return;
Expand Down
8 changes: 8 additions & 0 deletions tests/test-backend-ops.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8555,6 +8555,14 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1}));
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_MXFP4, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1}));

// single-row f32 weight (m == 1) against a wide activation (n): the second column count sweeps
// across MMVF_MAX_BATCH_SIZE (8) so both the direct mmvf path (n <= 8) and the CUDA/HIP
// operand-swap GEMV path (n > 8, which would otherwise fall back to an f32 cuBLAS GEMM) are
// exercised and checked against the CPU reference. Regression guard for the mul_mat_vec_f swap.
for (int64_t n : {1, 7, 8, 9, 16, 128, 512}) {
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 1, n, 2048, {1, 1}, {1, 1}));
}


#if 0
{
Expand Down
Loading