Skip to content

Commit d0b79aa

Browse files
authored
ggml : add native AVX512-FP16 support for F16 operations (ggml-org#20529)
The overall benchmark speed remains almost the same because the CPU is now calculating faster than the RAM can deliver the data. (See perf stat results below showing 2.7 billion fewer instructions). Also note that this path will be only enabled for native build or with custom flags. now: ``` Performance counter stats for 'build/bin/llama-bench -m Qwen3-0.6B-f16.gguf -p 512 -n 128': 189,073.52 msec task-clock # 14.658 CPUs utilized 404 context-switches # 2.137 /sec 19 cpu-migrations # 0.100 /sec 372,390 page-faults # 1.970 K/sec 310,877,195,595 instructions # 0.54 insn per cycle 581,071,530,602 cycles # 3.073 GHz 19,352,107,994 branches # 102.352 M/sec 48,304,438 branch-misses # 0.25% of all branches 84,998,431,152 L1-dcache-loads # 449.552 M/sec 12,186,410,279 L1-dcache-load-misses # 14.34% of all L1-dcache accesses 12.899358742 seconds time elapsed 187.823044000 seconds user 1.253416000 seconds sys ``` before: ``` Performance counter stats for 'build/bin/llama-bench -m Qwen3-0.6B-f16.gguf -p 512 -n 128': 190,594.56 msec task-clock # 14.652 CPUs utilized 436 context-switches # 2.288 /sec 22 cpu-migrations # 0.115 /sec 372,782 page-faults # 1.956 K/sec 313,574,921,966 instructions # 0.54 insn per cycle 586,064,970,425 cycles # 3.075 GHz 19,585,778,563 branches # 102.761 M/sec 48,437,488 branch-misses # 0.25% of all branches 86,219,336,628 L1-dcache-loads # 452.370 M/sec 12,232,085,771 L1-dcache-load-misses # 14.19% of all L1-dcache accesses 13.007923164 seconds time elapsed 189.395316000 seconds user 1.202612000 seconds sys ``` Signed-off-by: Adrien Gallouët <angt@huggingface.co>
1 parent f2c0dfb commit d0b79aa

1 file changed

Lines changed: 43 additions & 3 deletions

File tree

ggml/src/ggml-cpu/simd-mappings.h

Lines changed: 43 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -479,13 +479,51 @@ do { \
479479

480480
// F16 AVX512
481481

482-
// F16 AVX
482+
#if defined(__AVX512FP16__)
483+
484+
#define GGML_F16_STEP 128
485+
#define GGML_F16_EPR 32
486+
487+
#define GGML_F16x32 __m512h
488+
#define GGML_F16x32_ZERO _mm512_setzero_ph()
489+
#define GGML_F16x32_SET1(x) _mm512_set1_ph(__extension__(_Float16)(x))
490+
#define GGML_F16x32_LOAD(x) _mm512_loadu_ph(x)
491+
#define GGML_F16x32_STORE(x, y) _mm512_storeu_ph(x, y)
492+
#define GGML_F16x32_FMA(a, b, c) _mm512_fmadd_ph(b, c, a)
493+
#define GGML_F16x32_ADD _mm512_add_ph
494+
#define GGML_F16x32_MUL _mm512_mul_ph
495+
#define GGML_F16x32_REDUCE(res, x) \
496+
do { \
497+
int offset = GGML_F16_ARR >> 1; \
498+
for (int i = 0; i < offset; ++i) { \
499+
x[i] = _mm512_add_ph(x[i], x[offset+i]); \
500+
} \
501+
offset >>= 1; \
502+
for (int i = 0; i < offset; ++i) { \
503+
x[i] = _mm512_add_ph(x[i], x[offset+i]); \
504+
} \
505+
offset >>= 1; \
506+
for (int i = 0; i < offset; ++i) { \
507+
x[i] = _mm512_add_ph(x[i], x[offset+i]); \
508+
} \
509+
res = (ggml_float) _mm512_reduce_add_ph(x[0]); \
510+
} while (0)
511+
512+
#define GGML_F16_VEC GGML_F16x32
513+
#define GGML_F16_VEC_ZERO GGML_F16x32_ZERO
514+
#define GGML_F16_VEC_SET1 GGML_F16x32_SET1
515+
#define GGML_F16_VEC_LOAD(p, i) GGML_F16x32_LOAD(p)
516+
#define GGML_F16_VEC_STORE(p, r, i) GGML_F16x32_STORE(p, r[i])
517+
#define GGML_F16_VEC_FMA GGML_F16x32_FMA
518+
#define GGML_F16_VEC_ADD GGML_F16x32_ADD
519+
#define GGML_F16_VEC_MUL GGML_F16x32_MUL
520+
#define GGML_F16_VEC_REDUCE GGML_F16x32_REDUCE
521+
522+
#else // Fallback FP16 <-> FP32
483523

484524
#define GGML_F16_STEP 64
485525
#define GGML_F16_EPR 16
486526

487-
// AVX512 has FP16 extension (AVX512_FP16) but I don't have it on my machine so I use FP32 instead
488-
489527
#define GGML_F32Cx16 __m512
490528
#define GGML_F32Cx16_ZERO _mm512_setzero_ps()
491529
#define GGML_F32Cx16_SET1(x) _mm512_set1_ps(x)
@@ -525,6 +563,8 @@ do { \
525563
#define GGML_F16_VEC_MUL GGML_F32Cx16_MUL
526564

527565
#define GGML_F16_VEC_REDUCE GGML_F32Cx16_REDUCE
566+
567+
#endif // __AVX512FP16__
528568
#elif defined(__AVX__)
529569

530570
#define GGML_SIMD

0 commit comments

Comments
 (0)