Skip to content

Commit c16c35b

Browse files
tdakhranggerganov
tdakhran
and
ggerganov
authored
ggml-cpu: fix SVE leftover path in ggml_vec_dot_f32 (ggml-org#24699)
* ggml-cpu: fix SVE leftover path in ggml_vec_dot_f32 2D convolutions with kernel size 9 produced different results on SVE enabled ARM devices. After debugging it turned out that ggml_vec_dot_f32 was using data from inactive lanes. Use svmla_f32_m(pg, sum1, ax1, ay1) so inactive lanes retain sum1. * cont : clean-up --------- Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
1 parent 1a87dcd commit c16c35b

2 files changed

Lines changed: 63 additions & 30 deletions

File tree

‎ggml/src/ggml-cpu/vec.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,12 +75,12 @@ void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * G
7575
ay1 = GGML_F32_VEC_LOAD(y + i);
7676
sum1 = GGML_F32_VEC_FMA(sum1, ax1, ay1);
7777
}
78-
// maximum number of leftover elements will be less that ggml_f32_epr. Apply predicated svmad on available elements only
78+
// maximum number of leftover elements will be less that ggml_f32_epr. Apply predicated svmla on available elements only
7979
if (np2 < n) {
8080
svbool_t pg = svwhilelt_b32(np2, n);
8181
ax1 = svld1_f32(pg, x + np2);
8282
ay1 = svld1_f32(pg, y + np2);
83-
sum1 = svmad_f32_m(pg, ax1, ay1, sum1);
83+
sum1 = svmla_f32_m(pg, sum1, ax1, ay1);
8484
}
8585
// reduce sum1,sum2 to sum1
8686
GGML_F32_VEC_REDUCE(sumf, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum8);

‎tests/test-quantize-fns.cpp‎

Lines changed: 61 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -102,33 +102,41 @@ static float dot_product_error(const ggml_type_traits * qfns, const ggml_type_tr
102102
return fabsf(result - dot_ref) / test_size;
103103
}
104104

105-
int main(int argc, char * argv[]) {
106-
bool verbose = false;
107-
const size_t test_size = 32 * 128;
108-
109-
std::string arg;
110-
for (int i = 1; i < argc; i++) {
111-
arg = argv[i];
112-
113-
if (arg == "-v") {
114-
verbose = true;
115-
} else {
116-
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
117-
return 1;
105+
static int test_vec_dot_f32(bool verbose) {
106+
const auto * f32 = ggml_get_type_traits_cpu(GGML_TYPE_F32);
107+
int num_failed = 0;
108+
for (int n : {1, 2, 3, 5, 7, 8, 15, 16, 17, 31, 33, 63, 67, 127, 129, 193, 255, 1023}) {
109+
std::vector<float> a(n);
110+
std::vector<float> b(n);
111+
generate_data(0.0, n, a.data());
112+
generate_data(1.0, n, b.data());
113+
114+
float result = 0.0f;
115+
f32->vec_dot(n, &result, 0, a.data(), 0, b.data(), 0, 1);
116+
const float ref = dot_product(a.data(), b.data(), n);
117+
const float error = fabsf(result - ref) / n;
118+
119+
const bool failed = !(error < MAX_QUANTIZATION_REFERENCE_ERROR);
120+
num_failed += failed;
121+
if (failed || verbose) {
122+
printf(" f32 vec_dot n=%4d: %s (ref=%f got=%f err=%f)\n",
123+
n, RESULT_STR[failed], ref, result, error);
118124
}
119125
}
126+
return num_failed;
127+
}
128+
129+
static int test_vec_dot_q(bool verbose) {
130+
int num_failed = 0;
131+
132+
const size_t test_size = 32 * 128;
120133

121134
std::vector<float> test_data(test_size);
122135
std::vector<float> test_data2(test_size);
123136

124137
generate_data(0.0, test_data.size(), test_data.data());
125138
generate_data(1.0, test_data2.size(), test_data2.data());
126139

127-
ggml_cpu_init();
128-
129-
int num_failed = 0;
130-
bool failed = false;
131-
132140
for (int i = 0; i < GGML_TYPE_COUNT; i++) {
133141
ggml_type type = (ggml_type) i;
134142
const auto * qfns = ggml_get_type_traits(type);
@@ -156,7 +164,7 @@ int main(int argc, char * argv[]) {
156164
type == GGML_TYPE_IQ3_S ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS :
157165
type == GGML_TYPE_IQ3_XXS ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS_XXS :
158166
type == GGML_TYPE_NVFP4 ? MAX_QUANTIZATION_TOTAL_ERROR_FP4 : MAX_QUANTIZATION_TOTAL_ERROR;
159-
failed = !(total_error < max_quantization_error);
167+
bool failed = !(total_error < max_quantization_error);
160168
num_failed += failed;
161169
if (failed || verbose) {
162170
printf("%5s absolute quantization error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], total_error);
@@ -171,15 +179,15 @@ int main(int argc, char * argv[]) {
171179

172180
const float vec_dot_error = dot_product_error(qfns, qfns_cpu, test_size, test_data.data(), test_data2.data());
173181
const float max_allowed_error = type == GGML_TYPE_Q2_K || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ2_XXS ||
174-
type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S || type == GGML_TYPE_IQ2_S
175-
? MAX_DOT_PRODUCT_ERROR_LOWBIT
176-
: type == GGML_TYPE_Q1_0
177-
? MAX_DOT_PRODUCT_ERROR_BINARY
178-
: type == GGML_TYPE_TQ1_0 || type == GGML_TYPE_TQ2_0
179-
? MAX_DOT_PRODUCT_ERROR_TERNARY
180-
: type == GGML_TYPE_NVFP4
181-
? MAX_DOT_PRODUCT_ERROR_FP4
182-
: MAX_DOT_PRODUCT_ERROR;
182+
type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S || type == GGML_TYPE_IQ2_S
183+
? MAX_DOT_PRODUCT_ERROR_LOWBIT
184+
: type == GGML_TYPE_Q1_0
185+
? MAX_DOT_PRODUCT_ERROR_BINARY
186+
: type == GGML_TYPE_TQ1_0 || type == GGML_TYPE_TQ2_0
187+
? MAX_DOT_PRODUCT_ERROR_TERNARY
188+
: type == GGML_TYPE_NVFP4
189+
? MAX_DOT_PRODUCT_ERROR_FP4
190+
: MAX_DOT_PRODUCT_ERROR;
183191
failed = !(vec_dot_error < max_allowed_error);
184192
num_failed += failed;
185193
if (failed || verbose) {
@@ -188,6 +196,31 @@ int main(int argc, char * argv[]) {
188196
}
189197
}
190198

199+
return num_failed;
200+
}
201+
202+
int main(int argc, char * argv[]) {
203+
bool verbose = false;
204+
205+
std::string arg;
206+
for (int i = 1; i < argc; i++) {
207+
arg = argv[i];
208+
209+
if (arg == "-v") {
210+
verbose = true;
211+
} else {
212+
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
213+
return 1;
214+
}
215+
}
216+
217+
ggml_cpu_init();
218+
219+
int num_failed = 0;
220+
221+
num_failed += test_vec_dot_f32(verbose);
222+
num_failed += test_vec_dot_q(verbose);
223+
191224
if (num_failed || verbose) {
192225
printf("%d tests failed\n", num_failed);
193226
}

0 commit comments

Comments
 (0)