Skip to content

Commit 65108ac

Browse files
author
Kwok Cheung Yeung
committed
Add improved CGEMM and ZGEMM kernels for the RISC-V ZVL128B and ZVL256B architectures
This applies the changes made to the SGEMM/DGEMM kernels to their complex CGEMM/ZGEMM equivalents.
1 parent 744eda8 commit 65108ac

9 files changed

Lines changed: 669 additions & 0 deletions

‎kernel/riscv64/cgemm_kernel_zvl128b.c‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
/***************************************************************************
2+
Copyright (c) 2025, The OpenBLAS Project
3+
All rights reserved.
4+
Redistribution and use in source and binary forms, with or without
5+
modification, are permitted provided that the following conditions are
6+
met:
7+
1. Redistributions of source code must retain the above copyright
8+
notice, this list of conditions and the following disclaimer.
9+
2. Redistributions in binary form must reproduce the above copyright
10+
notice, this list of conditions and the following disclaimer in
11+
the documentation and/or other materials provided with the
12+
distribution.
13+
3. Neither the name of the OpenBLAS project nor the names of
14+
its contributors may be used to endorse or promote products
15+
derived from this software without specific prior written permission.
16+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19+
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
20+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
25+
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
*****************************************************************************/
27+
28+
#include "common.h"
29+
30+
#define M_BLOCKSIZE CGEMM_UNROLL_M
31+
#define N_BLOCKSIZE CGEMM_UNROLL_N
32+
33+
#if M_BLOCKSIZE == 4
34+
#define RVV_MUL __riscv_vfmul_vf_f32m1
35+
#define RVV_LOAD __riscv_vlse32_v_f32m1
36+
#define RVV_STORE __riscv_vsse32_v_f32m1
37+
#define VECTOR_T vfloat32m1_t
38+
#elif M_BLOCKSIZE == 8
39+
#define RVV_MUL __riscv_vfmul_vf_f32m2
40+
#define RVV_LOAD __riscv_vlse32_v_f32m2
41+
#define RVV_STORE __riscv_vsse32_v_f32m2
42+
#define VECTOR_T vfloat32m2_t
43+
#elif M_BLOCKSIZE == 16
44+
#define RVV_MUL __riscv_vfmul_vf_f32m4
45+
#define RVV_LOAD __riscv_vlse32_v_f32m4
46+
#define RVV_STORE __riscv_vsse32_v_f32m4
47+
#define VECTOR_T vfloat32m4_t
48+
#else
49+
#error "Unsupported M_BLOCKSIZE value"
50+
#endif
51+
52+
#include "zgemm_kernel_rvv_vlv_common.h"

‎kernel/riscv64/cgemm_kernel_zvl256b.c‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/***************************************************************************
2+
Copyright (c) 2025, The OpenBLAS Project
3+
All rights reserved.
4+
Redistribution and use in source and binary forms, with or without
5+
modification, are permitted provided that the following conditions are
6+
met:
7+
1. Redistributions of source code must retain the above copyright
8+
notice, this list of conditions and the following disclaimer.
9+
2. Redistributions in binary form must reproduce the above copyright
10+
notice, this list of conditions and the following disclaimer in
11+
the documentation and/or other materials provided with the
12+
distribution.
13+
3. Neither the name of the OpenBLAS project nor the names of
14+
its contributors may be used to endorse or promote products
15+
derived from this software without specific prior written permission.
16+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19+
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
20+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
25+
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
*****************************************************************************/
27+
28+
#include "common.h"
29+
30+
31+
#define M_BLOCKSIZE CGEMM_UNROLL_M
32+
#define N_BLOCKSIZE CGEMM_UNROLL_N
33+
34+
#if M_BLOCKSIZE == 8
35+
#define RVV_MUL __riscv_vfmul_vf_f32m1
36+
#define RVV_LOAD __riscv_vlse32_v_f32m1
37+
#define RVV_STORE __riscv_vsse32_v_f32m1
38+
#define VECTOR_T vfloat32m1_t
39+
#elif M_BLOCKSIZE == 16
40+
#define RVV_MUL __riscv_vfmul_vf_f32m2
41+
#define RVV_LOAD __riscv_vlse32_v_f32m2
42+
#define RVV_STORE __riscv_vsse32_v_f32m2
43+
#define VECTOR_T vfloat32m2_t
44+
#else
45+
#error "Unsupported M_BLOCKSIZE value"
46+
#endif
47+
48+
#include "zgemm_kernel_rvv_vlv_common.h"

‎kernel/riscv64/zgemm_kernel_rvv_vlv_common.h‎

Lines changed: 268 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,268 @@
1+
/***************************************************************************
2+
Copyright (c) 2025, The OpenBLAS Project
3+
All rights reserved.
4+
Redistribution and use in source and binary forms, with or without
5+
modification, are permitted provided that the following conditions are
6+
met:
7+
1. Redistributions of source code must retain the above copyright
8+
notice, this list of conditions and the following disclaimer.
9+
2. Redistributions in binary form must reproduce the above copyright
10+
notice, this list of conditions and the following disclaimer in
11+
the documentation and/or other materials provided with the
12+
distribution.
13+
3. Neither the name of the OpenBLAS project nor the names of
14+
its contributors may be used to endorse or promote products
15+
derived from this software without specific prior written permission.
16+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19+
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
20+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
25+
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
*****************************************************************************/
27+
28+
#if defined(NN) || defined(NT) || defined(TN) || defined(TT)
29+
#define S0 1
30+
#define S1 -1
31+
#define S2 1
32+
#define S3 1
33+
#define VFMACC_RR __riscv_vfmsac
34+
#define VFMACC_RI __riscv_vfmacc
35+
#endif
36+
#if defined(NR) || defined(NC) || defined(TR) || defined(TC)
37+
#define S0 1
38+
#define S1 1
39+
#define S2 1
40+
#define S3 -1
41+
#define VFMACC_RR __riscv_vfmacc
42+
#define VFMACC_RI __riscv_vfmsac
43+
#endif
44+
#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
45+
#define S0 1
46+
#define S1 1
47+
#define S2 -1
48+
#define S3 1
49+
#define VFMACC_RR __riscv_vfmacc
50+
#define VFMACC_RI __riscv_vfnmsac
51+
#endif
52+
#if defined(RR) || defined(RC) || defined(CR) || defined(CC)
53+
#define S0 1
54+
#define S1 -1
55+
#define S2 -1
56+
#define S3 -1
57+
#define VFMACC_RR __riscv_vfmsac
58+
#define VFMACC_RI __riscv_vfnmacc
59+
#endif
60+
61+
#define RISCV_REPEAT_1(INSN, BLEN, ...) \
62+
do { \
63+
INSN (0, 0, __VA_ARGS__); \
64+
if (BLEN == 1) break; \
65+
INSN (1, 1, __VA_ARGS__); \
66+
if (BLEN == 2) break; \
67+
INSN (2, 2, __VA_ARGS__); \
68+
INSN (3, 3, __VA_ARGS__); \
69+
} while (0)
70+
71+
#define RISCV_REPEAT_2(INSN, BLEN, ...) \
72+
do { \
73+
if (BLEN <= 4) break; \
74+
INSN (0, 4, __VA_ARGS__); \
75+
INSN (1, 5, __VA_ARGS__); \
76+
INSN (2, 6, __VA_ARGS__); \
77+
INSN (3, 7, __VA_ARGS__); \
78+
} while (0)
79+
80+
#define RISCV_MUL(M, N, DESTr, DESTi, Ar, Ai, Bi, GVL) \
81+
DESTr##M = RVV_MUL(Ai, Bi[N], GVL); \
82+
DESTi##M = RVV_MUL(Ar, Bi[N], GVL);
83+
84+
#define RISCV_VFMACC(M, N, DESTr, DESTi, Ar, Ai, Br, GVL) \
85+
DESTr##M = VFMACC_RR(DESTr##M, Br[N], Ar, GVL); \
86+
DESTi##M = VFMACC_RI(DESTi##M, Br[N], Ai, GVL);
87+
88+
#define RISCV_ACC_MUL_CONSTR(M, N, DESTr, DESTi, Ar, Ai, B, GVL) \
89+
DESTr##N = __riscv_vfmacc(DESTr##N, B, Ar##N, GVL); \
90+
DESTi##N = __riscv_vfmacc(DESTi##N, B, Ai##N, GVL);
91+
92+
#define RISCV_ACC_MUL_CONSTI(M, N, DESTr, DESTi, Ar, Ai, B, GVL) \
93+
DESTr##N = __riscv_vfnmsac(DESTr##N, B, Ai##N, GVL); \
94+
DESTi##N = __riscv_vfmacc(DESTi##N, B, Ar##N, GVL);
95+
96+
#define RISCV_LOAD(M, N, DESTr, DESTi, SRC, OFFSET, LDC, GVL) \
97+
DESTr##N = RVV_LOAD((SRC) + ((OFFSET) + N*(LDC)) * 2, sizeof(FLOAT)*2, GVL); \
98+
DESTi##N = RVV_LOAD((SRC) + ((OFFSET) + N*(LDC)) * 2 + 1, sizeof(FLOAT)*2, GVL);
99+
100+
#define RISCV_STORE(M, N, DEST, SRCr, SRCi, OFFSET, LDC, GVL) \
101+
RVV_STORE((DEST) + ((OFFSET) + N*(LDC)) * 2, sizeof(FLOAT)*2, SRCr##N, GVL); \
102+
RVV_STORE((DEST) + ((OFFSET) + N*(LDC)) * 2 + 1, sizeof(FLOAT)*2, SRCi##N, GVL);
103+
104+
#define RISCV_LOAD_COLUMN(DESTR, DESTI, SRC, OFFSET, GVL) \
105+
DESTR = RVV_LOAD((SRC) + (OFFSET), sizeof (FLOAT)*2, GVL); \
106+
DESTI = RVV_LOAD((SRC) + (OFFSET) + 1, sizeof (FLOAT)*2, GVL);
107+
108+
#define COPY_TMP(M, N, DESTr, DESTi, SRCr, SRCi) \
109+
DESTr##N = SRCr##M; \
110+
DESTi##N = SRCi##M;
111+
112+
#define RISCV_ADD(M, N, DESTr, DESTi, SRCr, SRCi, GVL) \
113+
DESTr##N = __riscv_vfadd(DESTr##N, SRCr##M, GVL); \
114+
DESTi##N = __riscv_vfadd(DESTi##N, SRCi##M, GVL);
115+
116+
#define COPY_ROW(DESTR, DESTI, SRC, OFFSET, LEN) \
117+
do { \
118+
DESTR[0] = SRC[OFFSET]; \
119+
DESTI[0] = SRC[OFFSET + 1]; \
120+
if (LEN == 1) break; \
121+
DESTR[1] = SRC[OFFSET + 2]; \
122+
DESTI[1] = SRC[OFFSET + 3]; \
123+
if (LEN == 2) break; \
124+
DESTR[2] = SRC[OFFSET + 4]; \
125+
DESTI[2] = SRC[OFFSET + 5]; \
126+
DESTR[3] = SRC[OFFSET + 6]; \
127+
DESTI[3] = SRC[OFFSET + 7]; \
128+
if (LEN == 4) break; \
129+
DESTR[4] = SRC[OFFSET + 8]; \
130+
DESTI[4] = SRC[OFFSET + 9]; \
131+
DESTR[5] = SRC[OFFSET + 10]; \
132+
DESTI[5] = SRC[OFFSET + 11]; \
133+
DESTR[6] = SRC[OFFSET + 12]; \
134+
DESTI[6] = SRC[OFFSET + 13]; \
135+
DESTR[7] = SRC[OFFSET + 14]; \
136+
DESTI[7] = SRC[OFFSET + 15]; \
137+
} while (0)
138+
139+
/* Perform matrix multiplication between submatrices:
140+
A(m_size,K) * B(K,n_size) = C(m_size,n_size) */
141+
142+
static inline __attribute__((always_inline))
143+
BLASLONG kernel (BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alphar, FLOAT alphai,
144+
FLOAT* A, FLOAT* B, FLOAT* C, BLASLONG ldc,
145+
BLASLONG m_top, BLASLONG n_top,
146+
BLASLONG m_size, BLASLONG n_size)
147+
{
148+
BLASLONG ai = m_top*K*2;
149+
BLASLONG bi = n_top*K*2;
150+
151+
/* b_r[0..n_size-1] = real(B(0, n_top)..B(0, n_top+n_size-1))
152+
b_i[0..n_size-1] = imag(B(0, n_top)..B(0, n_top+n_size-1)) */
153+
FLOAT b_r[N_BLOCKSIZE], b_i[N_BLOCKSIZE];
154+
COPY_ROW (b_r, b_i, B, bi, n_size);
155+
bi += n_size * 2;
156+
157+
/* a_r[0..m_size-1] = real(A(m_top, 0)..A(m_top+m_size-1, 0))
158+
a_i[0..m_size-1] = imag(A(m_top, 0)..A(m_top+m_size-1, 0)) */
159+
VECTOR_T a_r, a_i;
160+
RISCV_LOAD_COLUMN (a_r, a_i, A, ai, m_size);
161+
ai += m_size * 2;
162+
163+
/* for I = 0..n_size-1
164+
acc_rI[0..m_size-1] = real(A(m_top..m_top+msize-1, 0) * B(0, ntop+I))
165+
acc_iI[0..m_size-1] = imag(A(m_top..m_top+msize-1, 0) * B(0, ntop+I)) */
166+
VECTOR_T tmp_r0, tmp_i0, tmp_r1, tmp_i1, tmp_r2, tmp_i2, tmp_r3, tmp_i3;
167+
VECTOR_T acc_r0, acc_i0, acc_r1, acc_i1, acc_r2, acc_i2, acc_r3, acc_i3;
168+
VECTOR_T acc_r4, acc_i4, acc_r5, acc_i5, acc_r6, acc_i6, acc_r7, acc_i7;
169+
RISCV_REPEAT_1 (RISCV_MUL, n_size, tmp_r, tmp_i, a_r, a_i, b_i, m_size);
170+
RISCV_REPEAT_1 (RISCV_VFMACC, n_size, tmp_r, tmp_i, a_r, a_i, b_r, m_size);
171+
RISCV_REPEAT_1 (COPY_TMP, n_size, acc_r, acc_i, tmp_r, tmp_i);
172+
RISCV_REPEAT_2 (RISCV_MUL, n_size, tmp_r, tmp_i, a_r, a_i, b_i, m_size);
173+
RISCV_REPEAT_2 (RISCV_VFMACC, n_size, tmp_r, tmp_i, a_r, a_i, b_r, m_size);
174+
RISCV_REPEAT_2 (COPY_TMP, n_size, acc_r, acc_i, tmp_r, tmp_i);
175+
176+
for (BLASLONG k = 1; k < K; k++) {
177+
/* b_r[0..n_size-1] = real(B(k, n_top)..B(k, n_top+n_size-1))
178+
b_i[0..n_size-1] = imag(B(k, n_top)..B(k, n_top+n_size-1)) */
179+
COPY_ROW (b_r, b_i, B, bi, n_size);
180+
bi += n_size * 2;
181+
182+
/* a_r[0..m_size-1] = real(A(m_top, k)..A(m_top+m_size-1, k))
183+
a_i[0..m_size-1] = imag(A(m_top, k)..A(m_top+m_size-1, k)) */
184+
RISCV_LOAD_COLUMN (a_r, a_i, A, ai, m_size);
185+
ai += m_size * 2;
186+
187+
/* for I = 0..n_size-1
188+
acc_rI[0..m_size-1] += real(A(m_top..m_top+msize-1, k) * B(k, ntop+I))
189+
acc_iI[0..m_size-1] += imag(A(m_top..m_top+msize-1, k) * B(k, ntop+I)) */
190+
RISCV_REPEAT_1 (RISCV_MUL, n_size, tmp_r, tmp_i, a_r, a_i, b_i, m_size);
191+
RISCV_REPEAT_1 (RISCV_VFMACC, n_size, tmp_r, tmp_i, a_r, a_i, b_r, m_size);
192+
RISCV_REPEAT_1 (RISCV_ADD, n_size, acc_r, acc_i, tmp_r, tmp_i, m_size);
193+
RISCV_REPEAT_2 (RISCV_MUL, n_size, tmp_r, tmp_i, a_r, a_i, b_i, m_size);
194+
RISCV_REPEAT_2 (RISCV_VFMACC, n_size, tmp_r, tmp_i, a_r, a_i, b_r, m_size);
195+
RISCV_REPEAT_2 (RISCV_ADD, n_size, acc_r, acc_i, tmp_r, tmp_i, m_size);
196+
}
197+
198+
BLASLONG ci = n_top * ldc + m_top;
199+
VECTOR_T c_r0, c_i0, c_r1, c_i1, c_r2, c_i2, c_r3, c_i3;
200+
VECTOR_T c_r4, c_i4, c_r5, c_i5, c_r6, c_i6, c_r7, c_i7;
201+
202+
/* for I = 0..nsize-1
203+
c_rI[0..m_size-1] = real(C(m_top..m_top+m_size-1, n_top+I))
204+
c_iI[0..m_size-1] = imag(C(m_top..m_top+m_size-1, n_top+I))
205+
c_rI[0..m_size-1] += alpha_r * acc_rI[0..m_size-1]
206+
c_iI[0..m_size-1] += alpha_i * acc_iI[0..m_size-1]
207+
real(C(mtop..m_top+m_size-1, n_top+I)) = c_rI[0..m_size-1]
208+
imag(C(mtop..m_top+m_size-1, n_top+I)) = c_iI[0..m_size-1] */
209+
RISCV_REPEAT_1 (RISCV_LOAD, n_size, c_r, c_i, C, ci, ldc, m_size);
210+
RISCV_REPEAT_2 (RISCV_LOAD, n_size, c_r, c_i, C, ci, ldc, m_size);
211+
RISCV_REPEAT_1 (RISCV_ACC_MUL_CONSTR, n_size, c_r, c_i, acc_r, acc_i, alphar, m_size);
212+
RISCV_REPEAT_2 (RISCV_ACC_MUL_CONSTR, n_size, c_r, c_i, acc_r, acc_i, alphar, m_size);
213+
RISCV_REPEAT_1 (RISCV_ACC_MUL_CONSTI, n_size, c_r, c_i, acc_r, acc_i, alphai, m_size);
214+
RISCV_REPEAT_2 (RISCV_ACC_MUL_CONSTI, n_size, c_r, c_i, acc_r, acc_i, alphai, m_size);
215+
RISCV_REPEAT_1 (RISCV_STORE, n_size, C, c_r, c_i, ci, ldc, m_size);
216+
RISCV_REPEAT_2 (RISCV_STORE, n_size, C, c_r, c_i, ci, ldc, m_size);
217+
218+
return m_top + m_size;
219+
}
220+
221+
/* Perform matrix multiplication between submatrices:
222+
A(M,K) * B(K,n_size) = C(M, n_size) */
223+
224+
static inline __attribute__((always_inline))
225+
BLASLONG kernel_column (BLASLONG M, BLASLONG N, BLASLONG K,
226+
FLOAT alphar, FLOAT alphai,
227+
FLOAT* A, FLOAT* B, FLOAT* C, BLASLONG ldc,
228+
BLASLONG n_top, BLASLONG n_size)
229+
{
230+
BLASLONG m_top = 0;
231+
232+
for (BLASLONG i = 0; i < M / M_BLOCKSIZE; i++)
233+
m_top = kernel (M, N, K, alphar, alphai, A, B, C, ldc, m_top, n_top, M_BLOCKSIZE, n_size);
234+
235+
if (M & (M_BLOCKSIZE - 1))
236+
kernel (M, N, K, alphar, alphai, A, B, C, ldc, m_top, n_top, M - m_top, n_size);
237+
238+
return n_top + n_size;
239+
}
240+
241+
#define xstr(s) str(s)
242+
#define str(s) #s
243+
244+
/* Perform matrix multiplication between matrices:
245+
A(M,K) * B(K,N) = C(M,N) */
246+
247+
int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alphar, FLOAT alphai, FLOAT* A, FLOAT* B, FLOAT* C, BLASLONG ldc)
248+
{
249+
//fprintf(stderr, "%s (with VLV): M=%ld, N=%ld, K=%ld, ldc=%ld\n", xstr(CNAME), M, N, K, ldc);
250+
BLASLONG n_top = 0;
251+
252+
for (BLASLONG j = 0; j < N / N_BLOCKSIZE; j++)
253+
n_top = kernel_column (M, N, K, alphar, alphai, A, B, C, ldc, n_top, N_BLOCKSIZE);
254+
255+
#if N_BLOCKSIZE > 4
256+
if (N & 4)
257+
n_top = kernel_column (M, N, K, alphar, alphai, A, B, C, ldc, n_top, 4);
258+
#endif
259+
#if N_BLOCKSIZE > 2
260+
if (N & 2)
261+
n_top = kernel_column (M, N, K, alphar, alphai, A, B, C, ldc, n_top, 2);
262+
#endif
263+
#if N_BLOCKSIZE > 1
264+
if (N & 1)
265+
kernel_column (M, N, K, alphar, alphai, A, B, C, ldc, n_top, 1);
266+
#endif
267+
return 0;
268+
}

0 commit comments

Comments
 (0)