Skip to content

Commit 47b145d

Browse files
am17anam17an
committed
ggml-cpu: add repack for mxfp4
1 parent 2943210 commit 47b145d

5 files changed

Lines changed: 623 additions & 2 deletions

File tree

‎ggml/src/ggml-cpu/arch-fallback.h‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@
4848
#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K
4949
#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
5050
#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
51+
#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0
52+
#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0
5153
#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0
5254
#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0
5355
#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0
@@ -62,15 +64,19 @@
6264
#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K
6365
#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
6466
#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
67+
#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0
68+
#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0
6569
#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0
6670
#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0
6771
#elif defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64)
6872
// repack.cpp
6973
#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4
7074
#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8
7175
#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
76+
#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0
7277
#define ggml_gemv_q2_K_8x8_q8_K_generic ggml_gemv_q2_K_8x8_q8_K
7378
#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
79+
#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0
7480
#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K
7581
#elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64)
7682
// repack.cpp
@@ -84,6 +90,7 @@
8490
#define ggml_gemv_q6_K_8x4_q8_K_generic ggml_gemv_q6_K_8x4_q8_K
8591
#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K
8692
#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
93+
#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0
8794
#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0
8895
#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0
8996
#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0
@@ -94,6 +101,7 @@
94101
#define ggml_gemm_q6_K_8x4_q8_K_generic ggml_gemm_q6_K_8x4_q8_K
95102
#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K
96103
#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
104+
#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0
97105
#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0
98106
#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0
99107
#elif defined(__POWERPC__) || defined(__powerpc__)
@@ -120,6 +128,8 @@
120128
#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K
121129
#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
122130
#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
131+
#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0
132+
#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0
123133
#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0
124134
#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0
125135
#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0
@@ -134,6 +144,8 @@
134144
#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K
135145
#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
136146
#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
147+
#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0
148+
#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0
137149
#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0
138150
#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0
139151
#elif defined(__loongarch64)
@@ -160,6 +172,8 @@
160172
#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K
161173
#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
162174
#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
175+
#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0
176+
#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0
163177
#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0
164178
#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0
165179
#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0
@@ -174,6 +188,8 @@
174188
#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K
175189
#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
176190
#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
191+
#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0
192+
#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0
177193
#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0
178194
#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0
179195
#elif defined(__riscv)
@@ -201,6 +217,8 @@
201217
#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K
202218
#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
203219
#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
220+
#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0
221+
#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0
204222
#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0
205223
#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0
206224
#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0
@@ -214,6 +232,8 @@
214232
#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K
215233
#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
216234
#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
235+
#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0
236+
#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0
217237
#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0
218238
#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0
219239
#elif defined(__s390x__)
@@ -246,6 +266,8 @@
246266
#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K
247267
#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
248268
#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
269+
#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0
270+
#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0
249271
#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0
250272
#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0
251273
#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0
@@ -260,6 +282,8 @@
260282
#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K
261283
#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
262284
#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
285+
#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0
286+
#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0
263287
#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0
264288
#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0
265289
#elif defined(__wasm__)
@@ -294,6 +318,8 @@
294318
#define ggml_gemv_q6_K_8x8_q8_K_generic ggml_gemv_q6_K_8x8_q8_K
295319
#define ggml_gemv_iq4_nl_4x4_q8_0_generic ggml_gemv_iq4_nl_4x4_q8_0
296320
#define ggml_gemv_iq4_nl_8x8_q8_0_generic ggml_gemv_iq4_nl_8x8_q8_0
321+
#define ggml_gemv_mxfp4_4x4_q8_0_generic ggml_gemv_mxfp4_4x4_q8_0
322+
#define ggml_gemv_mxfp4_8x8_q8_0_generic ggml_gemv_mxfp4_8x8_q8_0
297323
#define ggml_gemv_q8_0_4x4_q8_0_generic ggml_gemv_q8_0_4x4_q8_0
298324
#define ggml_gemv_q8_0_4x8_q8_0_generic ggml_gemv_q8_0_4x8_q8_0
299325
#define ggml_gemm_q4_0_4x4_q8_0_generic ggml_gemm_q4_0_4x4_q8_0
@@ -308,6 +334,8 @@
308334
#define ggml_gemm_q6_K_8x8_q8_K_generic ggml_gemm_q6_K_8x8_q8_K
309335
#define ggml_gemm_iq4_nl_4x4_q8_0_generic ggml_gemm_iq4_nl_4x4_q8_0
310336
#define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0
337+
#define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0
338+
#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0
311339
#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0
312340
#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0
313341
#endif

‎ggml/src/ggml-cpu/arch/arm/repack.cpp‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -498,6 +498,81 @@ void ggml_gemv_iq4_nl_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const
498498
ggml_gemv_iq4_nl_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);
499499
}
500500

501+
void ggml_gemv_mxfp4_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {
502+
const int qk = QK8_0;
503+
const int nb = n / qk;
504+
const int ncols_interleaved = 4;
505+
const int blocklen = 4;
506+
507+
assert (n % qk == 0);
508+
assert (nc % ncols_interleaved == 0);
509+
510+
UNUSED(s);
511+
UNUSED(bs);
512+
UNUSED(vx);
513+
UNUSED(vy);
514+
UNUSED(nr);
515+
UNUSED(nc);
516+
UNUSED(nb);
517+
UNUSED(ncols_interleaved);
518+
UNUSED(blocklen);
519+
520+
#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)
521+
const int8x16_t kvalues = vld1q_s8(kvalues_mxfp4);
522+
const block_q8_0 * a_ptr = (const block_q8_0 *) vy;
523+
float * res_ptr = s;
524+
525+
for (int x = 0; x < nc / ncols_interleaved; x++) {
526+
const block_mxfp4x4 * b_ptr = (const block_mxfp4x4 *) vx + (x * nb);
527+
528+
float32x4_t sumf = vdupq_n_f32(0);
529+
for (int l = 0; l < nb; l++) {
530+
uint8x16_t b_0 = vld1q_u8(b_ptr[l].qs + 0);
531+
uint8x16_t b_1 = vld1q_u8(b_ptr[l].qs + 16);
532+
uint8x16_t b_2 = vld1q_u8(b_ptr[l].qs + 32);
533+
uint8x16_t b_3 = vld1q_u8(b_ptr[l].qs + 48);
534+
535+
int8x16_t b_0_hi = vqtbl1q_s8(kvalues, b_0 >> 4);
536+
int8x16_t b_0_lo = vqtbl1q_s8(kvalues, b_0 & 0x0F);
537+
int8x16_t b_1_hi = vqtbl1q_s8(kvalues, b_1 >> 4);
538+
int8x16_t b_1_lo = vqtbl1q_s8(kvalues, b_1 & 0x0F);
539+
int8x16_t b_2_hi = vqtbl1q_s8(kvalues, b_2 >> 4);
540+
int8x16_t b_2_lo = vqtbl1q_s8(kvalues, b_2 & 0x0F);
541+
int8x16_t b_3_hi = vqtbl1q_s8(kvalues, b_3 >> 4);
542+
int8x16_t b_3_lo = vqtbl1q_s8(kvalues, b_3 & 0x0F);
543+
544+
int8x16_t a_0 = vld1q_s8(a_ptr[l].qs + 0);
545+
int8x16_t a_1 = vld1q_s8(a_ptr[l].qs + 16);
546+
547+
int32x4_t sumi = vdupq_n_s32(0);
548+
sumi = vdotq_laneq_s32(sumi, b_0_lo, a_0, 0);
549+
sumi = vdotq_laneq_s32(sumi, b_0_hi, a_1, 0);
550+
sumi = vdotq_laneq_s32(sumi, b_1_lo, a_0, 1);
551+
sumi = vdotq_laneq_s32(sumi, b_1_hi, a_1, 1);
552+
sumi = vdotq_laneq_s32(sumi, b_2_lo, a_0, 2);
553+
sumi = vdotq_laneq_s32(sumi, b_2_hi, a_1, 2);
554+
sumi = vdotq_laneq_s32(sumi, b_3_lo, a_0, 3);
555+
sumi = vdotq_laneq_s32(sumi, b_3_hi, a_1, 3);
556+
557+
float32x4_t a_d = vcvt_f32_f16(vld1_dup_f16((const float16_t *)&a_ptr[l].d));
558+
float32x4_t b_d = {
559+
GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[0]),
560+
GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[1]),
561+
GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[2]),
562+
GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[3]),
563+
};
564+
float32x4_t d = a_d * b_d;
565+
566+
sumf = vmlaq_f32(sumf, d, vcvtq_f32_s32(sumi));
567+
}
568+
569+
vst1q_f32(res_ptr + x * 4, sumf);
570+
}
571+
return;
572+
#endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON)
573+
ggml_gemv_mxfp4_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);
574+
}
575+
501576
void ggml_gemv_q4_K_8x4_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {
502577
constexpr int qk = QK_K;
503578
const int nb = n / qk;
@@ -3164,6 +3239,87 @@ void ggml_gemm_iq4_nl_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const
31643239
ggml_gemm_iq4_nl_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);
31653240
}
31663241

3242+
void ggml_gemm_mxfp4_4x4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {
3243+
const int qk = QK8_0;
3244+
const int nb = n / qk;
3245+
const int ncols_interleaved = 4;
3246+
const int blocklen = 4;
3247+
3248+
assert (n % qk == 0);
3249+
assert (nr % 4 == 0);
3250+
assert (nc % ncols_interleaved == 0);
3251+
3252+
UNUSED(s);
3253+
UNUSED(bs);
3254+
UNUSED(vx);
3255+
UNUSED(vy);
3256+
UNUSED(nr);
3257+
UNUSED(nc);
3258+
UNUSED(nb);
3259+
UNUSED(ncols_interleaved);
3260+
UNUSED(blocklen);
3261+
3262+
#if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)
3263+
const int8x16_t kvalues = vld1q_s8(kvalues_mxfp4);
3264+
3265+
for (int y = 0; y < nr / 4; y++) {
3266+
const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);
3267+
for (int x = 0; x < nc / ncols_interleaved; x++) {
3268+
const block_mxfp4x4 * b_ptr = (const block_mxfp4x4 *) vx + (x * nb);
3269+
3270+
float32x4_t sumf[4];
3271+
for (int m = 0; m < 4; m++) {
3272+
sumf[m] = vdupq_n_f32(0);
3273+
}
3274+
3275+
for (int l = 0; l < nb; l++) {
3276+
float32x4_t a_d = vcvt_f32_f16(vld1_f16((const float16_t *)a_ptr[l].d));
3277+
float32x4_t b_d = {
3278+
GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[0]),
3279+
GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[1]),
3280+
GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[2]),
3281+
GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[l].e[3]),
3282+
};
3283+
3284+
int32x4_t sumi_0 = vdupq_n_s32(0);
3285+
int32x4_t sumi_1 = vdupq_n_s32(0);
3286+
int32x4_t sumi_2 = vdupq_n_s32(0);
3287+
int32x4_t sumi_3 = vdupq_n_s32(0);
3288+
3289+
for (int k = 0; k < 4; k++) {
3290+
int8x16_t a_0 = vld1q_s8(a_ptr[l].qs + 16 * k + 0);
3291+
int8x16_t a_1 = vld1q_s8(a_ptr[l].qs + 16 * k + 64);
3292+
3293+
uint8x16_t b = vld1q_u8(b_ptr[l].qs + 16 * k);
3294+
int8x16_t b_hi = vqtbl1q_s8(kvalues, b >> 4);
3295+
int8x16_t b_lo = vqtbl1q_s8(kvalues, b & 0xF);
3296+
3297+
sumi_0 = vdotq_laneq_s32(sumi_0, b_lo, a_0, 0);
3298+
sumi_1 = vdotq_laneq_s32(sumi_1, b_lo, a_0, 1);
3299+
sumi_2 = vdotq_laneq_s32(sumi_2, b_lo, a_0, 2);
3300+
sumi_3 = vdotq_laneq_s32(sumi_3, b_lo, a_0, 3);
3301+
sumi_0 = vdotq_laneq_s32(sumi_0, b_hi, a_1, 0);
3302+
sumi_1 = vdotq_laneq_s32(sumi_1, b_hi, a_1, 1);
3303+
sumi_2 = vdotq_laneq_s32(sumi_2, b_hi, a_1, 2);
3304+
sumi_3 = vdotq_laneq_s32(sumi_3, b_hi, a_1, 3);
3305+
}
3306+
3307+
sumf[0] = vmlaq_f32(sumf[0], vmulq_laneq_f32(b_d, a_d, 0), vcvtq_f32_s32(sumi_0));
3308+
sumf[1] = vmlaq_f32(sumf[1], vmulq_laneq_f32(b_d, a_d, 1), vcvtq_f32_s32(sumi_1));
3309+
sumf[2] = vmlaq_f32(sumf[2], vmulq_laneq_f32(b_d, a_d, 2), vcvtq_f32_s32(sumi_2));
3310+
sumf[3] = vmlaq_f32(sumf[3], vmulq_laneq_f32(b_d, a_d, 3), vcvtq_f32_s32(sumi_3));
3311+
}
3312+
3313+
for (int m = 0; m < 4; m++) {
3314+
vst1q_f32(s + (y * 4 + m) * bs + x * 4, sumf[m]);
3315+
}
3316+
}
3317+
}
3318+
return;
3319+
#endif // #if ! ((defined(_MSC_VER)) && ! defined(__clang__)) && defined(__aarch64__) && defined(__ARM_NEON)
3320+
ggml_gemm_mxfp4_4x4_q8_0_generic(n, s, bs, vx, vy, nr, nc);
3321+
}
3322+
31673323
void ggml_gemm_q4_K_8x4_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, const void * GGML_RESTRICT vy, int nr, int nc) {
31683324
constexpr int qk = QK_K;
31693325
const int nb = n / qk;

0 commit comments

Comments
 (0)