Skip to content

Commit 921f10e

Browse files
romgenieromgenie
committed
Add TurboQuant Vulkan turbo4 SET_ROWS support [skip ci]
1 parent e1adfd4 commit 921f10e

103 files changed

Lines changed: 23230 additions & 413 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎common/arg.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -406,6 +406,9 @@ const std::vector<ggml_type> kv_cache_types = {
406406
GGML_TYPE_IQ4_NL,
407407
GGML_TYPE_Q5_0,
408408
GGML_TYPE_Q5_1,
409+
GGML_TYPE_TURBO2_0,
410+
GGML_TYPE_TURBO3_0,
411+
GGML_TYPE_TURBO4_0,
409412
};
410413

411414
static ggml_type kv_cache_type_from_str(const std::string & s) {

‎common/common.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,12 @@
4444
#include <string.h>
4545
#include <fcntl.h>
4646
#include <io.h>
47+
#ifndef fileno
48+
#define fileno _fileno
49+
#endif
50+
#ifndef isatty
51+
#define isatty _isatty
52+
#endif
4753
#else
4854
#include <sys/ioctl.h>
4955
#include <sys/stat.h>

‎ggml/include/ggml-rpc.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,12 @@
66
extern "C" {
77
#endif
88

9-
#define RPC_PROTO_MAJOR_VERSION 4
9+
#define RPC_PROTO_MAJOR_VERSION 5
1010
#define RPC_PROTO_MINOR_VERSION 0
1111
#define RPC_PROTO_PATCH_VERSION 0
1212

1313
#ifdef __cplusplus
14-
static_assert(GGML_OP_COUNT == 96, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
14+
static_assert(GGML_OP_COUNT == 97, "GGML_OP_COUNT has changed - update RPC protocol version");
1515
#endif
1616

1717
#define GGML_RPC_MAX_SERVERS 16

‎ggml/include/ggml.h‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -429,7 +429,12 @@ extern "C" {
429429
GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block)
430430
GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale)
431431
GGML_TYPE_Q1_0 = 41,
432-
GGML_TYPE_COUNT = 42,
432+
GGML_TYPE_TURBO2_0 = 42, // TurboQuant 2-bit KV cache: WHT + 2-bit PolarQuant
433+
GGML_TYPE_TURBO3_0 = 43, // TurboQuant 3-bit KV cache: WHT + 3-bit PolarQuant
434+
GGML_TYPE_TURBO4_0 = 44, // TurboQuant 4-bit KV cache: WHT + 4-bit PolarQuant
435+
GGML_TYPE_TQ3_1S = 45, // TurboQuant 3-bit weight: WHT-rotated 8-level Lloyd-Max, block_size=32
436+
GGML_TYPE_TQ4_1S = 46, // TurboQuant 4-bit weight: WHT-rotated 16-level Lloyd-Max, block_size=32
437+
GGML_TYPE_COUNT = 47,
433438
};
434439

435440
// precision
@@ -567,6 +572,7 @@ extern "C" {
567572
GGML_OP_RWKV_WKV7,
568573
GGML_OP_SOLVE_TRI,
569574
GGML_OP_GATED_DELTA_NET,
575+
GGML_OP_TURBO_WHT,
570576

571577
GGML_OP_UNARY,
572578

@@ -2555,6 +2561,16 @@ extern "C" {
25552561
struct ggml_tensor * beta,
25562562
struct ggml_tensor * state);
25572563

2564+
// TurboQuant Walsh-Hadamard Transform (O(d log d) rotation for KV cache compression)
2565+
// Applies WHT rotation to 128-element groups along ne[0]: sign1 → butterfly → sign2 → normalize
2566+
// direction: 0 = forward (signs1 → WHT → signs2), 1 = inverse (signs2 → WHT → signs1)
2567+
GGML_API struct ggml_tensor * ggml_turbo_wht(
2568+
struct ggml_context * ctx,
2569+
struct ggml_tensor * a,
2570+
int direction,
2571+
int group_size, // 0 = auto (64 or 128 from ne[0])
2572+
struct ggml_tensor * scale); // NULL = no InnerQ scaling
2573+
25582574
// custom operators
25592575

25602576
typedef void (*ggml_custom1_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, int ith, int nth, void * userdata);

‎ggml/src/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,7 @@ add_library(ggml-base
206206
ggml-threading.h
207207
ggml-quants.c
208208
ggml-quants.h
209+
ggml-turbo-quant.c
209210
gguf.cpp)
210211

211212
set_target_properties(ggml-base PROPERTIES

‎ggml/src/ggml-common.h‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,97 @@ typedef struct {
277277
} block_tq2_0;
278278
static_assert(sizeof(block_tq2_0) == sizeof(ggml_half) + QK_K / 4, "wrong tq2_0 block size/padding");
279279

280+
// TurboQuant 3-bit MSE-only: 3-bit PolarQuant indices (no QJL)
281+
// Storage block size = 32 (matches q4_0 for optimal GPU parallelism)
282+
// Transform group size = 128 (head_dim, for rotation Gaussianization)
283+
// Per block: norm(fp16) + 2-bit indices (8 bytes) + 1-bit extra (4 bytes) = 14 bytes per 32 values
284+
// = 3.5 bits/value → 4.6× compression vs fp16
285+
// The 3-bit index is split: lower 2 bits in qs[], upper 1 bit in signs[]
286+
#define QK_TURBO3 128 // Block size 128: one block per rotation group, eliminates redundant norms
287+
#define QK_TURBO3_GROUP 128 // rotation group size = head_dim
288+
// Derived: FA template nl parameters (auto-scale with block size)
289+
#define NL_TURBO3 (QK_TURBO3 / 16) // non-vec FA iterations per block
290+
#define NL_TURBO3_VEC (QK_TURBO3 / 4) // vec FA iterations per block
291+
typedef struct {
292+
ggml_half norm; // 2 bytes: vector L2 norm (for rescaling)
293+
uint8_t qs[QK_TURBO3 / 4]; // 8 bytes: lower 2-bit indices (4 per byte)
294+
uint8_t signs[QK_TURBO3 / 8]; // 4 bytes: upper 1-bit of 3-bit index (8 per byte)
295+
} block_turbo3_0; // 14 bytes total
296+
static_assert(sizeof(block_turbo3_0) == sizeof(ggml_half) + QK_TURBO3/4 + QK_TURBO3/8, "wrong turbo3_0 block size/padding");
297+
298+
// TurboQuant 4-bit: 3-bit PolarQuant indices + 1-bit QJL signs
299+
// TURBO4_USE_4BIT: switch between 4-bit PolarQuant (new) and 3-bit+QJL (legacy)
300+
// Default: 4-bit on all backends (Metal + CUDA validated)
301+
#ifndef TURBO4_USE_4BIT
302+
# define TURBO4_USE_4BIT 1
303+
#endif
304+
305+
#define QK_TURBO4 128
306+
307+
#if TURBO4_USE_4BIT
308+
// 4-bit PolarQuant: 16 optimal centroids, nibble packed, no QJL
309+
// Per block: norm(fp16) + rnorm(fp16, reserved) + 4-bit indices (64 bytes)
310+
// = 68 bytes per 128 values = 4.25 bits/value → 3.8× compression vs fp16
311+
typedef struct {
312+
ggml_half norm; // 2 bytes
313+
ggml_half rnorm; // 2 bytes (reserved, unused in 4-bit mode)
314+
uint8_t qs[QK_TURBO4 / 2]; // 64 bytes: 4-bit PolarQuant indices (nibble packed)
315+
} block_turbo4_0; // 68 bytes total
316+
static_assert(sizeof(block_turbo4_0) == 68, "wrong turbo4_0 block size");
317+
#else
318+
// Legacy 3-bit PolarQuant + 1-bit QJL (original paper design)
319+
// Per block: norm(fp16) + rnorm(fp16) + 3-bit indices (48 bytes) + 1-bit QJL signs (16 bytes)
320+
// = 68 bytes per 128 values = 4.25 bits/value → 3.8× compression vs fp16
321+
typedef struct {
322+
ggml_half norm; // 2 bytes
323+
ggml_half rnorm; // 2 bytes: residual norm for QJL scale
324+
uint8_t qs[QK_TURBO4 * 3 / 8]; // 48 bytes: 3-bit PolarQuant indices
325+
uint8_t signs[QK_TURBO4 / 8]; // 16 bytes: 1-bit QJL signs
326+
} block_turbo4_0; // 68 bytes total
327+
static_assert(sizeof(block_turbo4_0) == 2*sizeof(ggml_half) + QK_TURBO4*3/8 + QK_TURBO4/8, "wrong turbo4_0 block size");
328+
#endif
329+
330+
static_assert(QK_TURBO4 == 128, "turbo4 kernels assume QK_TURBO4 == 128");
331+
332+
// TurboQuant 2-bit: 2-bit PolarQuant indices only (no QJL)
333+
// Per block: norm(fp16) + 2-bit indices (8 bytes) = 10 bytes per 32 values
334+
// = 2.5 bits/value → 6.4× compression vs fp16
335+
// 4 centroids (Lloyd-Max for N(0, 1/128)): {-0.133462, -0.039994, 0.039994, 0.133462}
336+
#define QK_TURBO2 128 // Block size 128: one block per rotation group
337+
#define QK_TURBO2_GROUP 128 // rotation group size = head_dim
338+
// Derived: FA template nl parameters (auto-scale with block size)
339+
#define NL_TURBO2 (QK_TURBO2 / 16) // non-vec FA iterations per block
340+
#define NL_TURBO2_VEC (QK_TURBO2 / 4) // vec FA iterations per block
341+
typedef struct {
342+
ggml_half norm; // 2 bytes: corrected L2 norm
343+
uint8_t qs[QK_TURBO2 / 4]; // 8 bytes: 2-bit indices (4 per byte)
344+
} block_turbo2_0; // 10 bytes total
345+
static_assert(sizeof(block_turbo2_0) == sizeof(ggml_half) + QK_TURBO2/4, "wrong turbo2_0 block size/padding");
346+
347+
// TQ3_1S: WHT-rotated 3-bit weight quantization (8-level Lloyd-Max for N(0,1))
348+
// Block size 32, dual half-block scales (d0 for [0..15], d1 for [16..31])
349+
// Per block: d0(fp16) + d1(fp16) + 3-bit indices packed (12 bytes) = 16 bytes per 32 values
350+
// = 4.0 bits/value
351+
#define QK_TQ3_0 32
352+
typedef struct {
353+
ggml_half d0; // 2 bytes: scale for first 16 elements
354+
ggml_half d1; // 2 bytes: scale for last 16 elements
355+
uint8_t qs[QK_TQ3_0 * 3 / 8]; // 12 bytes: 3-bit indices packed (4 groups of 8 in 3 bytes)
356+
} block_tq3_1s; // 16 bytes total
357+
static_assert(sizeof(block_tq3_1s) == 16, "wrong tq3_1s block size");
358+
359+
// TQ4_1S: WHT-rotated 4-bit weight quantization (16-level Lloyd-Max for N(0,1))
360+
// Block size 32, dual half-block scales (d0 for [0..15], d1 for [16..31])
361+
// Per block: d0(fp16) + d1(fp16) + 4-bit indices packed (16 bytes) = 20 bytes per 32 values
362+
// = 5.0 bits/value
363+
#define QK_TQ4_1S 32
364+
typedef struct {
365+
ggml_half d0; // 2 bytes: scale for first 16 elements
366+
ggml_half d1; // 2 bytes: scale for last 16 elements
367+
uint8_t qs[QK_TQ4_1S / 2]; // 16 bytes: 4-bit indices nibble-packed
368+
} block_tq4_1s; // 20 bytes total
369+
static_assert(sizeof(block_tq4_1s) == 20, "wrong tq4_1s block size");
370+
280371
//
281372
// Super-block quantization structures
282373
//

‎ggml/src/ggml-cpu/ggml-cpu.c‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
#include "ggml-cpu-impl.h"
88
#include "ggml-impl.h"
99
#include "quants.h"
10+
#include "ggml-quants.h"
1011
#include "ggml-threading.h"
1112
#include "unary-ops.h"
1213
#include "binary-ops.h"
@@ -208,6 +209,23 @@ typedef pthread_t ggml_thread_t;
208209
#include <TargetConditionals.h>
209210
#endif
210211

212+
// Forward declarations — defined below, after utility functions
213+
static void ggml_vec_dot_turbo3_0_f32(int n, float * GGML_RESTRICT s, size_t bs,
214+
const void * GGML_RESTRICT vx, size_t bx,
215+
const void * GGML_RESTRICT vy, size_t by, int nrc);
216+
static void ggml_vec_dot_turbo2_0_f32(int n, float * GGML_RESTRICT s, size_t bs,
217+
const void * GGML_RESTRICT vx, size_t bx,
218+
const void * GGML_RESTRICT vy, size_t by, int nrc);
219+
static void ggml_vec_dot_turbo4_0_f32(int n, float * GGML_RESTRICT s, size_t bs,
220+
const void * GGML_RESTRICT vx, size_t bx,
221+
const void * GGML_RESTRICT vy, size_t by, int nrc);
222+
static void ggml_vec_dot_tq3_1s_q8_0(int n, float * GGML_RESTRICT s, size_t bs,
223+
const void * GGML_RESTRICT vx, size_t bx,
224+
const void * GGML_RESTRICT vy, size_t by, int nrc);
225+
static void ggml_vec_dot_tq4_1s_q8_0(int n, float * GGML_RESTRICT s, size_t bs,
226+
const void * GGML_RESTRICT vx, size_t bx,
227+
const void * GGML_RESTRICT vy, size_t by, int nrc);
228+
211229
static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = {
212230
[GGML_TYPE_F32] = {
213231
.from_float = (ggml_from_float_t) ggml_cpu_fp32_to_fp32,
@@ -403,6 +421,36 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = {
403421
[GGML_TYPE_I32] = {
404422
.from_float = (ggml_from_float_t) ggml_cpu_fp32_to_i32,
405423
},
424+
[GGML_TYPE_TURBO3_0] = {
425+
.from_float = (ggml_from_float_t) quantize_row_turbo3_0_ref,
426+
.vec_dot = (ggml_vec_dot_t) ggml_vec_dot_turbo3_0_f32,
427+
.vec_dot_type = GGML_TYPE_F32,
428+
.nrows = 1,
429+
},
430+
[GGML_TYPE_TURBO2_0] = {
431+
.from_float = (ggml_from_float_t) quantize_row_turbo2_0_ref,
432+
.vec_dot = (ggml_vec_dot_t) ggml_vec_dot_turbo2_0_f32,
433+
.vec_dot_type = GGML_TYPE_F32,
434+
.nrows = 1,
435+
},
436+
[GGML_TYPE_TURBO4_0] = {
437+
.from_float = (ggml_from_float_t) quantize_row_turbo4_0_ref,
438+
.vec_dot = (ggml_vec_dot_t) ggml_vec_dot_turbo4_0_f32,
439+
.vec_dot_type = GGML_TYPE_F32,
440+
.nrows = 1,
441+
},
442+
[GGML_TYPE_TQ3_1S] = {
443+
.from_float = (ggml_from_float_t) quantize_row_tq3_1s_ref,
444+
.vec_dot = (ggml_vec_dot_t) ggml_vec_dot_tq3_1s_q8_0,
445+
.vec_dot_type = GGML_TYPE_Q8_0,
446+
.nrows = 1,
447+
},
448+
[GGML_TYPE_TQ4_1S] = {
449+
.from_float = (ggml_from_float_t) quantize_row_tq4_1s_ref,
450+
.vec_dot = (ggml_vec_dot_t) ggml_vec_dot_tq4_1s_q8_0,
451+
.vec_dot_type = GGML_TYPE_Q8_0,
452+
.nrows = 1,
453+
},
406454
};
407455

408456
const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type) {
@@ -2047,6 +2095,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
20472095
{
20482096
ggml_compute_forward_gated_delta_net(params, tensor);
20492097
} break;
2098+
case GGML_OP_TURBO_WHT:
2099+
{
2100+
ggml_compute_forward_turbo_wht(params, tensor);
2101+
} break;
20502102
case GGML_OP_MAP_CUSTOM1:
20512103
{
20522104
ggml_compute_forward_map_custom1(params, tensor);
@@ -2227,6 +2279,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
22272279
case GGML_OP_COUNT_EQUAL:
22282280
case GGML_OP_SOLVE_TRI:
22292281
case GGML_OP_GATED_DELTA_NET:
2282+
case GGML_OP_TURBO_WHT:
22302283
{
22312284
n_tasks = n_threads;
22322285
} break;
@@ -2947,6 +3000,10 @@ struct ggml_cplan ggml_graph_plan(
29473000
const int64_t per_thread = S_v + (K > 1 ? S_v * S_v : 0);
29483001
cur = per_thread * sizeof(float) * n_tasks;
29493002
} break;
3003+
case GGML_OP_TURBO_WHT:
3004+
{
3005+
cur = 0; // no extra workspace needed
3006+
} break;
29503007
case GGML_OP_COUNT:
29513008
{
29523009
GGML_ABORT("fatal error");
@@ -3385,6 +3442,112 @@ enum ggml_status ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct g
33853442
return ggml_graph_compute(cgraph, &cplan);
33863443
}
33873444

3445+
// TurboQuant3 vec_dot: dequantize turbo3 block to f32, then dot with f32 operand.
3446+
// Used by CPU flash attention for models with D not supported by CUDA FA (e.g. D=192).
3447+
static void ggml_vec_dot_turbo3_0_f32(int n, float * GGML_RESTRICT s, size_t bs,
3448+
const void * GGML_RESTRICT vx, size_t bx,
3449+
const void * GGML_RESTRICT vy, size_t by, int nrc) {
3450+
GGML_ASSERT(nrc == 1);
3451+
GGML_UNUSED(bs); GGML_UNUSED(bx); GGML_UNUSED(by); GGML_UNUSED(nrc);
3452+
3453+
// Dequantize turbo3 to f32 stack scratch, then dot.
3454+
float * tmp = (float *)alloca((size_t)n * sizeof(float));
3455+
GGML_ASSERT(tmp != NULL);
3456+
ggml_get_type_traits(GGML_TYPE_TURBO3_0)->to_float(vx, tmp, n);
3457+
3458+
const float * y = (const float *)vy;
3459+
float sum = 0.0f;
3460+
for (int i = 0; i < n; i++) {
3461+
sum += tmp[i] * y[i];
3462+
}
3463+
*s = sum;
3464+
}
3465+
3466+
// TurboQuant2 vec_dot: dequantize turbo2 block to f32, then dot with f32 operand.
3467+
static void ggml_vec_dot_turbo2_0_f32(int n, float * GGML_RESTRICT s, size_t bs,
3468+
const void * GGML_RESTRICT vx, size_t bx,
3469+
const void * GGML_RESTRICT vy, size_t by, int nrc) {
3470+
GGML_ASSERT(nrc == 1);
3471+
GGML_UNUSED(bs); GGML_UNUSED(bx); GGML_UNUSED(by); GGML_UNUSED(nrc);
3472+
3473+
float * tmp = (float *)alloca((size_t)n * sizeof(float));
3474+
GGML_ASSERT(tmp != NULL);
3475+
ggml_get_type_traits(GGML_TYPE_TURBO2_0)->to_float(vx, tmp, n);
3476+
3477+
const float * y = (const float *)vy;
3478+
float sum = 0.0f;
3479+
for (int i = 0; i < n; i++) {
3480+
sum += tmp[i] * y[i];
3481+
}
3482+
*s = sum;
3483+
}
3484+
3485+
// TurboQuant4 vec_dot: dequantize turbo4 block to f32, then dot with f32 operand.
3486+
static void ggml_vec_dot_turbo4_0_f32(int n, float * GGML_RESTRICT s, size_t bs,
3487+
const void * GGML_RESTRICT vx, size_t bx,
3488+
const void * GGML_RESTRICT vy, size_t by, int nrc) {
3489+
GGML_ASSERT(nrc == 1);
3490+
GGML_UNUSED(bs); GGML_UNUSED(bx); GGML_UNUSED(by); GGML_UNUSED(nrc);
3491+
3492+
float * tmp = (float *)alloca((size_t)n * sizeof(float));
3493+
GGML_ASSERT(tmp != NULL);
3494+
ggml_get_type_traits(GGML_TYPE_TURBO4_0)->to_float(vx, tmp, n);
3495+
3496+
const float * y = (const float *)vy;
3497+
float sum = 0.0f;
3498+
for (int i = 0; i < n; i++) {
3499+
sum += tmp[i] * y[i];
3500+
}
3501+
*s = sum;
3502+
}
3503+
3504+
// TQ3_1S vec_dot: dequantize tq3_1s block to f32, then dot with q8_0.
3505+
// TODO: optimize with SIMD intrinsics for ARM NEON / AVX2
3506+
static void ggml_vec_dot_tq3_1s_q8_0(int n, float * GGML_RESTRICT s, size_t bs,
3507+
const void * GGML_RESTRICT vx, size_t bx,
3508+
const void * GGML_RESTRICT vy, size_t by, int nrc) {
3509+
GGML_ASSERT(nrc == 1);
3510+
GGML_UNUSED(bs); GGML_UNUSED(bx); GGML_UNUSED(by); GGML_UNUSED(nrc);
3511+
3512+
float * tmp = (float *)alloca((size_t)n * sizeof(float));
3513+
GGML_ASSERT(tmp != NULL);
3514+
ggml_get_type_traits(GGML_TYPE_TQ3_1S)->to_float(vx, tmp, n);
3515+
3516+
// Dequantize q8_0 and dot
3517+
float * tmp2 = (float *)alloca((size_t)n * sizeof(float));
3518+
GGML_ASSERT(tmp2 != NULL);
3519+
ggml_get_type_traits(GGML_TYPE_Q8_0)->to_float(vy, tmp2, n);
3520+
3521+
float sum = 0.0f;
3522+
for (int i = 0; i < n; i++) {
3523+
sum += tmp[i] * tmp2[i];
3524+
}
3525+
*s = sum;
3526+
}
3527+
3528+
// TQ4_1S vec_dot: dequantize tq4_1s block to f32, then dot with q8_0.
3529+
// TODO: optimize with SIMD intrinsics
3530+
static void ggml_vec_dot_tq4_1s_q8_0(int n, float * GGML_RESTRICT s, size_t bs,
3531+
const void * GGML_RESTRICT vx, size_t bx,
3532+
const void * GGML_RESTRICT vy, size_t by, int nrc) {
3533+
GGML_ASSERT(nrc == 1);
3534+
GGML_UNUSED(bs); GGML_UNUSED(bx); GGML_UNUSED(by); GGML_UNUSED(nrc);
3535+
3536+
float * tmp = (float *)alloca((size_t)n * sizeof(float));
3537+
GGML_ASSERT(tmp != NULL);
3538+
ggml_get_type_traits(GGML_TYPE_TQ4_1S)->to_float(vx, tmp, n);
3539+
3540+
float * tmp2 = (float *)alloca((size_t)n * sizeof(float));
3541+
GGML_ASSERT(tmp2 != NULL);
3542+
ggml_get_type_traits(GGML_TYPE_Q8_0)->to_float(vy, tmp2, n);
3543+
3544+
float sum = 0.0f;
3545+
for (int i = 0; i < n; i++) {
3546+
sum += tmp[i] * tmp2[i];
3547+
}
3548+
*s = sum;
3549+
}
3550+
33883551
void ggml_cpu_fp32_to_fp32(const float * x, float * y, int64_t n) {
33893552
memcpy(y, x, n * sizeof(float));
33903553
}

0 commit comments

Comments
 (0)