Skip to content

Commit dcaf676

Browse files
mudlermudler
committed
feat: add TurboQuant support
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
1 parent 9f102a1 commit dcaf676

13 files changed

Lines changed: 337 additions & 1 deletion

File tree

‎ggml/include/ggml.h‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -428,7 +428,11 @@ extern "C" {
428428
// GGML_TYPE_IQ4_NL_8_8 = 38,
429429
GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block)
430430
GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale)
431-
GGML_TYPE_COUNT = 41,
431+
GGML_TYPE_TBQ3_0 = 41, // TurboQuant Q_mse 3-bit
432+
GGML_TYPE_TBQ4_0 = 42, // TurboQuant Q_mse 4-bit
433+
GGML_TYPE_TBQP3_0 = 43, // TurboQuant Q_prod 3-bit
434+
GGML_TYPE_TBQP4_0 = 44, // TurboQuant Q_prod 4-bit
435+
GGML_TYPE_COUNT = 45,
432436
};
433437

434438
// precision
@@ -465,6 +469,10 @@ extern "C" {
465469
GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors
466470
GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors
467471
GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors
472+
GGML_FTYPE_MOSTLY_TBQ3_0 = 27, // except 1d tensors
473+
GGML_FTYPE_MOSTLY_TBQ4_0 = 28, // except 1d tensors
474+
GGML_FTYPE_MOSTLY_TBQP3_0 = 29, // except 1d tensors
475+
GGML_FTYPE_MOSTLY_TBQP4_0 = 30, // except 1d tensors
468476
};
469477

470478
// available tensor operations:

‎ggml/src/CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,9 @@ add_library(ggml-base
205205
ggml-threading.h
206206
ggml-quants.c
207207
ggml-quants.h
208+
ggml-turboq.c
209+
ggml-turboq.h
210+
ggml-turboq-tables.h
208211
gguf.cpp)
209212

210213
set_target_properties(ggml-base PROPERTIES

‎ggml/src/ggml-common.h‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,45 @@ typedef struct {
266266
} block_tq2_0;
267267
static_assert(sizeof(block_tq2_0) == sizeof(ggml_half) + QK_K / 4, "wrong tq2_0 block size/padding");
268268

269+
//
270+
// TurboQuant quantization structures
271+
// Uses SRHT (random rotation) + Max-Lloyd codebook quantization
272+
//
273+
274+
// TurboQuant Q_mse 3-bit: 3.0625 bpw
275+
// 256 elements * 3 bits = 96 bytes packed indices + 2 bytes scale
276+
typedef struct {
277+
uint8_t qs[QK_K * 3 / 8]; // 96 bytes: 3-bit codebook indices, packed
278+
ggml_half d; // row-level scale (L2 norm)
279+
} block_tbq3_0;
280+
static_assert(sizeof(block_tbq3_0) == sizeof(ggml_half) + QK_K * 3 / 8, "wrong tbq3_0 block size/padding");
281+
282+
// TurboQuant Q_mse 4-bit: 4.0625 bpw
283+
// 256 elements * 4 bits = 128 bytes packed indices + 2 bytes scale
284+
typedef struct {
285+
uint8_t qs[QK_K / 2]; // 128 bytes: 4-bit codebook indices as nibbles
286+
ggml_half d; // row-level scale (L2 norm)
287+
} block_tbq4_0;
288+
static_assert(sizeof(block_tbq4_0) == sizeof(ggml_half) + QK_K / 2, "wrong tbq4_0 block size/padding");
289+
290+
// TurboQuant Q_prod 3-bit: 3.0625 bpw
291+
// 2-bit codebook (64B) + 1-bit QJL signs (32B) + 2 bytes scale
292+
typedef struct {
293+
uint8_t qs[QK_K / 4]; // 64 bytes: 2-bit codebook indices
294+
uint8_t signs[QK_K / 8]; // 32 bytes: 1-bit QJL sign per element
295+
ggml_half d; // L2 norm (used for inner-product estimation)
296+
} block_tbqp3_0;
297+
static_assert(sizeof(block_tbqp3_0) == sizeof(ggml_half) + QK_K / 4 + QK_K / 8, "wrong tbqp3_0 block size/padding");
298+
299+
// TurboQuant Q_prod 4-bit: 4.0625 bpw
300+
// 3-bit codebook (96B) + 1-bit QJL signs (32B) + 2 bytes scale
301+
typedef struct {
302+
uint8_t qs[QK_K * 3 / 8]; // 96 bytes: 3-bit codebook indices, packed
303+
uint8_t signs[QK_K / 8]; // 32 bytes: 1-bit QJL sign per element
304+
ggml_half d; // L2 norm (used for inner-product estimation)
305+
} block_tbqp4_0;
306+
static_assert(sizeof(block_tbqp4_0) == sizeof(ggml_half) + QK_K * 3 / 8 + QK_K / 8, "wrong tbqp4_0 block size/padding");
307+
269308
//
270309
// Super-block quantization structures
271310
//

‎ggml/src/ggml-cpu/arch-fallback.h‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@
1818
#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0
1919
#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K
2020
#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K
21+
#define ggml_vec_dot_tbq3_0_q8_K_generic ggml_vec_dot_tbq3_0_q8_K
22+
#define ggml_vec_dot_tbq4_0_q8_K_generic ggml_vec_dot_tbq4_0_q8_K
23+
#define ggml_vec_dot_tbqp3_0_q8_K_generic ggml_vec_dot_tbqp3_0_q8_K
24+
#define ggml_vec_dot_tbqp4_0_q8_K_generic ggml_vec_dot_tbqp4_0_q8_K
2125
#define ggml_vec_dot_q2_K_q8_K_generic ggml_vec_dot_q2_K_q8_K
2226
#define ggml_vec_dot_q3_K_q8_K_generic ggml_vec_dot_q3_K_q8_K
2327
#define ggml_vec_dot_q4_K_q8_K_generic ggml_vec_dot_q4_K_q8_K
@@ -82,6 +86,10 @@
8286
#elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64)
8387
// quants.c
8488
#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0
89+
#define ggml_vec_dot_tbq3_0_q8_K_generic ggml_vec_dot_tbq3_0_q8_K
90+
#define ggml_vec_dot_tbq4_0_q8_K_generic ggml_vec_dot_tbq4_0_q8_K
91+
#define ggml_vec_dot_tbqp3_0_q8_K_generic ggml_vec_dot_tbqp3_0_q8_K
92+
#define ggml_vec_dot_tbqp4_0_q8_K_generic ggml_vec_dot_tbqp4_0_q8_K
8593
// repack.cpp
8694
#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4
8795
#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4
@@ -114,6 +122,10 @@
114122
#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0
115123
#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K
116124
#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K
125+
#define ggml_vec_dot_tbq3_0_q8_K_generic ggml_vec_dot_tbq3_0_q8_K
126+
#define ggml_vec_dot_tbq4_0_q8_K_generic ggml_vec_dot_tbq4_0_q8_K
127+
#define ggml_vec_dot_tbqp3_0_q8_K_generic ggml_vec_dot_tbqp3_0_q8_K
128+
#define ggml_vec_dot_tbqp4_0_q8_K_generic ggml_vec_dot_tbqp4_0_q8_K
117129
#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K
118130
// repack.cpp
119131
#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4
@@ -157,6 +169,10 @@
157169
#define quantize_row_q8_K_generic quantize_row_q8_K
158170
#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K
159171
#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K
172+
#define ggml_vec_dot_tbq3_0_q8_K_generic ggml_vec_dot_tbq3_0_q8_K
173+
#define ggml_vec_dot_tbq4_0_q8_K_generic ggml_vec_dot_tbq4_0_q8_K
174+
#define ggml_vec_dot_tbqp3_0_q8_K_generic ggml_vec_dot_tbqp3_0_q8_K
175+
#define ggml_vec_dot_tbqp4_0_q8_K_generic ggml_vec_dot_tbqp4_0_q8_K
160176
#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K
161177
#define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0
162178
#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0
@@ -242,6 +258,10 @@
242258
#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0
243259
#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K
244260
#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K
261+
#define ggml_vec_dot_tbq3_0_q8_K_generic ggml_vec_dot_tbq3_0_q8_K
262+
#define ggml_vec_dot_tbq4_0_q8_K_generic ggml_vec_dot_tbq4_0_q8_K
263+
#define ggml_vec_dot_tbqp3_0_q8_K_generic ggml_vec_dot_tbqp3_0_q8_K
264+
#define ggml_vec_dot_tbqp4_0_q8_K_generic ggml_vec_dot_tbqp4_0_q8_K
245265
#define ggml_vec_dot_q2_K_q8_K_generic ggml_vec_dot_q2_K_q8_K
246266
#define ggml_vec_dot_iq2_xxs_q8_K_generic ggml_vec_dot_iq2_xxs_q8_K
247267
#define ggml_vec_dot_iq2_xs_q8_K_generic ggml_vec_dot_iq2_xs_q8_K
@@ -292,6 +312,10 @@
292312
#define ggml_vec_dot_q4_1_q8_1_generic ggml_vec_dot_q4_1_q8_1
293313
#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K
294314
#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K
315+
#define ggml_vec_dot_tbq3_0_q8_K_generic ggml_vec_dot_tbq3_0_q8_K
316+
#define ggml_vec_dot_tbq4_0_q8_K_generic ggml_vec_dot_tbq4_0_q8_K
317+
#define ggml_vec_dot_tbqp3_0_q8_K_generic ggml_vec_dot_tbqp3_0_q8_K
318+
#define ggml_vec_dot_tbqp4_0_q8_K_generic ggml_vec_dot_tbqp4_0_q8_K
295319
#define ggml_vec_dot_iq2_xxs_q8_K_generic ggml_vec_dot_iq2_xxs_q8_K
296320
#define ggml_vec_dot_iq2_xs_q8_K_generic ggml_vec_dot_iq2_xs_q8_K
297321
#define ggml_vec_dot_iq2_s_q8_K_generic ggml_vec_dot_iq2_s_q8_K

‎ggml/src/ggml-cpu/ggml-cpu.c‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -390,6 +390,30 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = {
390390
.vec_dot_type = GGML_TYPE_Q8_K,
391391
.nrows = 1,
392392
},
393+
[GGML_TYPE_TBQ3_0] = {
394+
.from_float = quantize_row_tbq3_0,
395+
.vec_dot = ggml_vec_dot_tbq3_0_q8_K,
396+
.vec_dot_type = GGML_TYPE_Q8_K,
397+
.nrows = 1,
398+
},
399+
[GGML_TYPE_TBQ4_0] = {
400+
.from_float = quantize_row_tbq4_0,
401+
.vec_dot = ggml_vec_dot_tbq4_0_q8_K,
402+
.vec_dot_type = GGML_TYPE_Q8_K,
403+
.nrows = 1,
404+
},
405+
[GGML_TYPE_TBQP3_0] = {
406+
.from_float = quantize_row_tbqp3_0,
407+
.vec_dot = ggml_vec_dot_tbqp3_0_q8_K,
408+
.vec_dot_type = GGML_TYPE_Q8_K,
409+
.nrows = 1,
410+
},
411+
[GGML_TYPE_TBQP4_0] = {
412+
.from_float = quantize_row_tbqp4_0,
413+
.vec_dot = ggml_vec_dot_tbqp4_0_q8_K,
414+
.vec_dot_type = GGML_TYPE_Q8_K,
415+
.nrows = 1,
416+
},
393417
[GGML_TYPE_I32] = {
394418
.from_float = (ggml_from_float_t) ggml_cpu_fp32_to_i32,
395419
},

‎ggml/src/ggml-cpu/quants.c‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,32 @@ void quantize_row_tq2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy,
108108
quantize_row_tq2_0_ref(x, y, k);
109109
}
110110

111+
// ====================== TurboQuant (TBQ) quantization ========================
112+
113+
void quantize_row_tbq3_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {
114+
assert(k % QK_K == 0);
115+
block_tbq3_0 * GGML_RESTRICT y = vy;
116+
quantize_row_tbq3_0_ref(x, y, k);
117+
}
118+
119+
void quantize_row_tbq4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {
120+
assert(k % QK_K == 0);
121+
block_tbq4_0 * GGML_RESTRICT y = vy;
122+
quantize_row_tbq4_0_ref(x, y, k);
123+
}
124+
125+
void quantize_row_tbqp3_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {
126+
assert(k % QK_K == 0);
127+
block_tbqp3_0 * GGML_RESTRICT y = vy;
128+
quantize_row_tbqp3_0_ref(x, y, k);
129+
}
130+
131+
void quantize_row_tbqp4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {
132+
assert(k % QK_K == 0);
133+
block_tbqp4_0 * GGML_RESTRICT y = vy;
134+
quantize_row_tbqp4_0_ref(x, y, k);
135+
}
136+
111137
//===================================== Q8_K ==============================================
112138

113139
void quantize_row_q8_K_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
@@ -456,6 +482,120 @@ void ggml_vec_dot_tq2_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs,
456482
*s = sumf;
457483
}
458484

485+
// ====================== TurboQuant vec_dot (dequant-then-dot reference) ===========
486+
//
487+
// These implementations dequantize the TBQ row to float, then compute the dot
488+
// product with the Q8_K dequantized row. This is the correctness-first approach.
489+
490+
void ggml_vec_dot_tbq3_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
491+
assert(nrc == 1);
492+
UNUSED(nrc);
493+
UNUSED(bx);
494+
UNUSED(by);
495+
UNUSED(bs);
496+
497+
// Dequantize TBQ3 to float, then dot with dequantized Q8_K
498+
float * tmp = (float *)malloc(n * sizeof(float));
499+
dequantize_row_tbq3_0((const block_tbq3_0 *)vx, tmp, n);
500+
501+
const block_q8_K * GGML_RESTRICT y = vy;
502+
const int nb = n / QK_K;
503+
504+
float sumf = 0.0f;
505+
int64_t idx = 0;
506+
for (int i = 0; i < nb; i++) {
507+
const float d = y[i].d;
508+
for (int j = 0; j < QK_K; j++) {
509+
sumf += tmp[idx] * (d * y[i].qs[j]);
510+
idx++;
511+
}
512+
}
513+
514+
free(tmp);
515+
*s = sumf;
516+
}
517+
518+
void ggml_vec_dot_tbq4_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
519+
assert(nrc == 1);
520+
UNUSED(nrc);
521+
UNUSED(bx);
522+
UNUSED(by);
523+
UNUSED(bs);
524+
525+
float * tmp = (float *)malloc(n * sizeof(float));
526+
dequantize_row_tbq4_0((const block_tbq4_0 *)vx, tmp, n);
527+
528+
const block_q8_K * GGML_RESTRICT y = vy;
529+
const int nb = n / QK_K;
530+
531+
float sumf = 0.0f;
532+
int64_t idx = 0;
533+
for (int i = 0; i < nb; i++) {
534+
const float d = y[i].d;
535+
for (int j = 0; j < QK_K; j++) {
536+
sumf += tmp[idx] * (d * y[i].qs[j]);
537+
idx++;
538+
}
539+
}
540+
541+
free(tmp);
542+
*s = sumf;
543+
}
544+
545+
void ggml_vec_dot_tbqp3_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
546+
assert(nrc == 1);
547+
UNUSED(nrc);
548+
UNUSED(bx);
549+
UNUSED(by);
550+
UNUSED(bs);
551+
552+
float * tmp = (float *)malloc(n * sizeof(float));
553+
dequantize_row_tbqp3_0((const block_tbqp3_0 *)vx, tmp, n);
554+
555+
const block_q8_K * GGML_RESTRICT y = vy;
556+
const int nb = n / QK_K;
557+
558+
float sumf = 0.0f;
559+
int64_t idx = 0;
560+
for (int i = 0; i < nb; i++) {
561+
const float d = y[i].d;
562+
for (int j = 0; j < QK_K; j++) {
563+
sumf += tmp[idx] * (d * y[i].qs[j]);
564+
idx++;
565+
}
566+
}
567+
568+
free(tmp);
569+
*s = sumf;
570+
}
571+
572+
void ggml_vec_dot_tbqp4_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
573+
assert(nrc == 1);
574+
UNUSED(nrc);
575+
UNUSED(bx);
576+
UNUSED(by);
577+
UNUSED(bs);
578+
579+
float * tmp = (float *)malloc(n * sizeof(float));
580+
dequantize_row_tbqp4_0((const block_tbqp4_0 *)vx, tmp, n);
581+
582+
const block_q8_K * GGML_RESTRICT y = vy;
583+
const int nb = n / QK_K;
584+
585+
float sumf = 0.0f;
586+
int64_t idx = 0;
587+
for (int i = 0; i < nb; i++) {
588+
const float d = y[i].d;
589+
for (int j = 0; j < QK_K; j++) {
590+
sumf += tmp[idx] * (d * y[i].qs[j]);
591+
idx++;
592+
}
593+
}
594+
595+
free(tmp);
596+
*s = sumf;
597+
}
598+
459599
void ggml_vec_dot_q2_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
460600
assert(nrc == 1);
461601
UNUSED(nrc);

‎ggml/src/ggml-cpu/quants.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,11 @@ void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in
3232
void quantize_row_tq1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
3333
void quantize_row_tq2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
3434

35+
void quantize_row_tbq3_0 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
36+
void quantize_row_tbq4_0 (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
37+
void quantize_row_tbqp3_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
38+
void quantize_row_tbqp4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
39+
3540
void quantize_row_iq4_nl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
3641
void quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
3742

@@ -54,6 +59,11 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
5459
void ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
5560
void ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
5661

62+
void ggml_vec_dot_tbq3_0_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
63+
void ggml_vec_dot_tbq4_0_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
64+
void ggml_vec_dot_tbqp3_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
65+
void ggml_vec_dot_tbqp4_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
66+
5767
void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
5868
void ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
5969
void ggml_vec_dot_iq2_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);

‎ggml/src/ggml-quants.c‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5399,6 +5399,23 @@ bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbyte
53995399
VALIDATE_ROW_DATA_D_F16_IMPL(block_iq4_nl, data, nb);
54005400
} break;
54015401

5402+
case GGML_TYPE_TBQ3_0:
5403+
{
5404+
VALIDATE_ROW_DATA_D_F16_IMPL(block_tbq3_0, data, nb);
5405+
} break;
5406+
case GGML_TYPE_TBQ4_0:
5407+
{
5408+
VALIDATE_ROW_DATA_D_F16_IMPL(block_tbq4_0, data, nb);
5409+
} break;
5410+
case GGML_TYPE_TBQP3_0:
5411+
{
5412+
VALIDATE_ROW_DATA_D_F16_IMPL(block_tbqp3_0, data, nb);
5413+
} break;
5414+
case GGML_TYPE_TBQP4_0:
5415+
{
5416+
VALIDATE_ROW_DATA_D_F16_IMPL(block_tbqp4_0, data, nb);
5417+
} break;
5418+
54025419
case GGML_TYPE_I8:
54035420
case GGML_TYPE_I16:
54045421
case GGML_TYPE_I32:

0 commit comments

Comments
 (0)