Skip to content

Commit 744eda8

Browse files
author
Kwok Cheung Yeung
committed
Add new SGEMM/DGEMM kernels for RISC-V ZVL128B/ZVL256B architectures
These kernels use the same algorithm as the original ones, but improve on them in several respects: - Common code is shared between the two kernel types and architectures, reducing the overall lines of code. - Instead of using auto-generated code with each vector operation repeated multiple times (necessitated in part due to the vector types not being storable in arrays), macros and forced inlining are used to achieve the same effect directly for better clarity and a major reduction in lines of code. - The tiling sizes can be modified within a certain range by modifying the unroll parameters. - The RVV extension is not restricted to having the number of elements in a vector register being a power of two, so the tails of the matrix involving vector operations can be dealt with in a single operation rather than in decreasing powers of two, thereby improving performance. The tcopy operations need to be modified to take this into account.
1 parent 68ff451 commit 744eda8

9 files changed

Lines changed: 586 additions & 0 deletions

‎kernel/riscv64/dgemm_kernel_zvl128b.c‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/***************************************************************************
2+
Copyright (c) 2025, The OpenBLAS Project
3+
All rights reserved.
4+
Redistribution and use in source and binary forms, with or without
5+
modification, are permitted provided that the following conditions are
6+
met:
7+
1. Redistributions of source code must retain the above copyright
8+
notice, this list of conditions and the following disclaimer.
9+
2. Redistributions in binary form must reproduce the above copyright
10+
notice, this list of conditions and the following disclaimer in
11+
the documentation and/or other materials provided with the
12+
distribution.
13+
3. Neither the name of the OpenBLAS project nor the names of
14+
its contributors may be used to endorse or promote products
15+
derived from this software without specific prior written permission.
16+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19+
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
20+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
25+
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
*****************************************************************************/
27+
28+
#include "common.h"
29+
30+
#define M_BLOCKSIZE DGEMM_UNROLL_M
31+
#define N_BLOCKSIZE DGEMM_UNROLL_N
32+
33+
#if M_BLOCKSIZE == 2
34+
#define RVV_MUL __riscv_vfmul_vf_f64m1
35+
#define RVV_MACC __riscv_vfmacc_vf_f64m1
36+
#define RVV_LOAD __riscv_vle64_v_f64m1
37+
#define RVV_STORE __riscv_vse64_v_f64m1
38+
#define VECTOR_T vfloat64m1_t
39+
#elif M_BLOCKSIZE == 4
40+
#define RVV_MUL __riscv_vfmul_vf_f64m2
41+
#define RVV_MACC __riscv_vfmacc_vf_f64m2
42+
#define RVV_LOAD __riscv_vle64_v_f64m2
43+
#define RVV_STORE __riscv_vse64_v_f64m2
44+
#define VECTOR_T vfloat64m2_t
45+
#elif M_BLOCKSIZE == 8
46+
#define RVV_MUL __riscv_vfmul_vf_f64m4
47+
#define RVV_MACC __riscv_vfmacc_vf_f64m4
48+
#define RVV_LOAD __riscv_vle64_v_f64m4
49+
#define RVV_STORE __riscv_vse64_v_f64m4
50+
#define VECTOR_T vfloat64m4_t
51+
#else
52+
#error "Unsupported M_BLOCKSIZE value"
53+
#endif
54+
55+
#include "gemm_kernel_rvv_vlv_common.h"

‎kernel/riscv64/dgemm_kernel_zvl256b.c‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/***************************************************************************
2+
Copyright (c) 2025, The OpenBLAS Project
3+
All rights reserved.
4+
Redistribution and use in source and binary forms, with or without
5+
modification, are permitted provided that the following conditions are
6+
met:
7+
1. Redistributions of source code must retain the above copyright
8+
notice, this list of conditions and the following disclaimer.
9+
2. Redistributions in binary form must reproduce the above copyright
10+
notice, this list of conditions and the following disclaimer in
11+
the documentation and/or other materials provided with the
12+
distribution.
13+
3. Neither the name of the OpenBLAS project nor the names of
14+
its contributors may be used to endorse or promote products
15+
derived from this software without specific prior written permission.
16+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19+
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
20+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
25+
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
*****************************************************************************/
27+
28+
#include "common.h"
29+
30+
#define M_BLOCKSIZE DGEMM_UNROLL_M
31+
#define N_BLOCKSIZE DGEMM_UNROLL_N
32+
33+
#if M_BLOCKSIZE == 4
34+
#define RVV_MUL __riscv_vfmul_vf_f64m1
35+
#define RVV_MACC __riscv_vfmacc_vf_f64m1
36+
#define RVV_LOAD __riscv_vle64_v_f64m1
37+
#define RVV_STORE __riscv_vse64_v_f64m1
38+
#define VECTOR_T vfloat64m1_t
39+
#elif M_BLOCKSIZE == 8
40+
#define RVV_MUL __riscv_vfmul_vf_f64m2
41+
#define RVV_MACC __riscv_vfmacc_vf_f64m2
42+
#define RVV_LOAD __riscv_vle64_v_f64m2
43+
#define RVV_STORE __riscv_vse64_v_f64m2
44+
#define VECTOR_T vfloat64m2_t
45+
#elif M_BLOCKSIZE == 16
46+
#define RVV_MUL __riscv_vfmul_vf_f64m4
47+
#define RVV_MACC __riscv_vfmacc_vf_f64m4
48+
#define RVV_LOAD __riscv_vle64_v_f64m4
49+
#define RVV_STORE __riscv_vse64_v_f64m4
50+
#define VECTOR_T vfloat64m4_t
51+
#else
52+
#error "Unsupported M_BLOCKSIZE value"
53+
#endif
54+
55+
#include "gemm_kernel_rvv_vlv_common.h"

‎kernel/riscv64/gemm_kernel_rvv_vlv_common.h‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
/***************************************************************************
2+
Copyright (c) 2025, The OpenBLAS Project
3+
All rights reserved.
4+
Redistribution and use in source and binary forms, with or without
5+
modification, are permitted provided that the following conditions are
6+
met:
7+
1. Redistributions of source code must retain the above copyright
8+
notice, this list of conditions and the following disclaimer.
9+
2. Redistributions in binary form must reproduce the above copyright
10+
notice, this list of conditions and the following disclaimer in
11+
the documentation and/or other materials provided with the
12+
distribution.
13+
3. Neither the name of the OpenBLAS project nor the names of
14+
its contributors may be used to endorse or promote products
15+
derived from this software without specific prior written permission.
16+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19+
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
20+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
25+
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
*****************************************************************************/
27+
28+
#define RISCV_REPEAT(INSN, BLEN, ...) \
29+
do { \
30+
INSN (0, __VA_ARGS__); \
31+
if (BLEN == 1) break; \
32+
INSN (1, __VA_ARGS__); \
33+
if (BLEN == 2) break; \
34+
INSN (2, __VA_ARGS__); \
35+
INSN (3, __VA_ARGS__); \
36+
if (BLEN == 4) break; \
37+
INSN (4, __VA_ARGS__); \
38+
INSN (5, __VA_ARGS__); \
39+
INSN (6, __VA_ARGS__); \
40+
INSN (7, __VA_ARGS__); \
41+
} while (0)
42+
43+
#define RISCV_MUL(N, DEST, A, B, LEN) \
44+
DEST##N = RVV_MUL(A, B[N], LEN);
45+
46+
#define RISCV_ACC_MUL(N, DEST, A, B, LEN) \
47+
DEST##N = RVV_MACC(DEST##N, B[N], A, LEN);
48+
49+
#define RISCV_ACC_MUL_CONST(N, DEST, A, B, LEN) \
50+
DEST##N = RVV_MACC(DEST##N, B, A##N, LEN);
51+
52+
#define RISCV_LOAD(N, DEST, SRC, OFFSET, LDC, LEN) \
53+
DEST##N = RVV_LOAD((SRC) + (OFFSET) + N*(LDC), LEN);
54+
55+
#define RISCV_STORE(N, DEST, SRC, OFFSET, LDC, LEN) \
56+
RVV_STORE((DEST) + (OFFSET) + N*(LDC), SRC##N, LEN);
57+
58+
#define RISCV_LOAD_COLUMN(DEST, SRC, OFFSET, LEN) \
59+
DEST = RVV_LOAD((SRC) + (OFFSET), LEN);
60+
61+
#define COPY_ROW(DEST, SRC, OFFSET, LEN) \
62+
do { \
63+
DEST[0] = SRC[OFFSET]; \
64+
if (LEN == 1) break; \
65+
DEST[1] = SRC[OFFSET + 1]; \
66+
if (LEN == 2) break; \
67+
DEST[2] = SRC[OFFSET + 2]; \
68+
DEST[3] = SRC[OFFSET + 3]; \
69+
if (LEN == 4) break; \
70+
DEST[4] = SRC[OFFSET + 4]; \
71+
DEST[5] = SRC[OFFSET + 5]; \
72+
DEST[6] = SRC[OFFSET + 6]; \
73+
DEST[7] = SRC[OFFSET + 7]; \
74+
} while (0)
75+
76+
/* Perform matrix multiplication between submatrices:
77+
A(m_size,K) * B(K,n_size) = C(m_size,n_size) */
78+
79+
static inline __attribute__((always_inline))
80+
BLASLONG kernel (BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha,
81+
FLOAT* A, FLOAT* B, FLOAT* C, BLASLONG ldc,
82+
BLASLONG m_top, BLASLONG n_top,
83+
BLASLONG m_size, BLASLONG n_size)
84+
{
85+
BLASLONG ai = m_top*K;
86+
BLASLONG bi = n_top*K;
87+
88+
/* b[0..n_size-1] = B(0, n_top)..B(0, n_top+n_size-1) */
89+
FLOAT b[N_BLOCKSIZE];
90+
COPY_ROW (b, B, bi, n_size);
91+
bi += n_size;
92+
93+
/* a[0..m_size-1] = A(m_top, 0)..A(m_top+m_size-1, 0) */
94+
VECTOR_T a;
95+
RISCV_LOAD_COLUMN (a, A, ai, m_size);
96+
ai += m_size;
97+
98+
/* for I = 0..n_size-1
99+
resultI[0..m_size-1] = A(m_top..m_top+msize-1, 0) * B(0, ntop+I) */
100+
VECTOR_T result0, result1, result2, result3;
101+
VECTOR_T result4, result5, result6, result7;
102+
RISCV_REPEAT (RISCV_MUL, n_size, result, a, b, m_size);
103+
104+
for (BLASLONG k = 1; k < K; k++) {
105+
/* b[0..n_size-1] = B(k, n_top)..B(k, n_top+n_size-1) */
106+
COPY_ROW (b, B, bi, n_size);
107+
bi += n_size;
108+
109+
/* a[0..m_size-1] = A(m_top, k)..A(m_top+m_size-1, k) */
110+
RISCV_LOAD_COLUMN (a, A, ai, m_size);
111+
ai += m_size;
112+
113+
/* for I = 0..n_size-1
114+
resultI[0..m_size-1] += A(m_top..m_top+msize-1, k) * B(k, ntop+I) */
115+
RISCV_REPEAT (RISCV_ACC_MUL, n_size, result, a, b, m_size);
116+
}
117+
118+
BLASLONG ci = n_top * ldc + m_top;
119+
VECTOR_T c0, c1, c2, c3, c4, c5, c6, c7;
120+
121+
/* for I = 0..nsize-1
122+
cI[0..m_size-1] = C(m_top..m_top+m_size-1, n_top+I)
123+
cI[0..m_size-1] += alpha * resultI[0..m_size-1]
124+
C(mtop..m_top+m_size-1, n_top+I) = cI[0..m_size-1] */
125+
RISCV_REPEAT (RISCV_LOAD, n_size, c, C, ci, ldc, m_size);
126+
RISCV_REPEAT (RISCV_ACC_MUL_CONST, n_size, c, result, alpha, m_size);
127+
RISCV_REPEAT (RISCV_STORE, n_size, C, c, ci, ldc, m_size);
128+
129+
return m_top + m_size;
130+
}
131+
132+
/* Perform matrix multiplication between submatrices:
133+
A(M,K) * B(K,n_size) = C(M, n_size) */
134+
135+
static inline __attribute__((always_inline))
136+
BLASLONG kernel_column (BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha,
137+
FLOAT* A, FLOAT* B, FLOAT* C, BLASLONG ldc,
138+
BLASLONG n_top, BLASLONG n_size)
139+
{
140+
BLASLONG m_top = 0;
141+
142+
for (BLASLONG i = 0; i < M / M_BLOCKSIZE; i++)
143+
m_top = kernel (M, N, K, alpha, A, B, C, ldc, m_top, n_top, M_BLOCKSIZE, n_size);
144+
145+
if (M & (M_BLOCKSIZE - 1))
146+
kernel (M, N, K, alpha, A, B, C, ldc, m_top, n_top, M - m_top, n_size);
147+
148+
return n_top + n_size;
149+
}
150+
151+
#define xstr(s) str(s)
152+
#define str(s) #s
153+
154+
/* Perform matrix multiplication between matrices:
155+
A(M,K) * B(K,N) = C(M,N) */
156+
157+
int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, FLOAT* A, FLOAT* B, FLOAT* C, BLASLONG ldc)
158+
{
159+
//fprintf(stderr, "%s (with VLV): M=%ld, N=%ld, K=%ld, ldc=%ld, m_blocksize=%d, n_blocksize=%d\n", xstr(CNAME), M, N, K, ldc, M_BLOCKSIZE, N_BLOCKSIZE);
160+
BLASLONG n_top = 0;
161+
162+
for (BLASLONG j = 0; j < N / N_BLOCKSIZE; j++)
163+
n_top = kernel_column (M, N, K, alpha, A, B, C, ldc, n_top, N_BLOCKSIZE);
164+
165+
#if N_BLOCKSIZE > 4
166+
if (N & 4)
167+
n_top = kernel_column (M, N, K, alpha, A, B, C, ldc, n_top, 4);
168+
#endif
169+
#if N_BLOCKSIZE > 2
170+
if (N & 2)
171+
n_top = kernel_column (M, N, K, alpha, A, B, C, ldc, n_top, 2);
172+
#endif
173+
#if N_BLOCKSIZE > 1
174+
if (N & 1)
175+
kernel_column (M, N, K, alpha, A, B, C, ldc, n_top, 1);
176+
#endif
177+
178+
return 0;
179+
}

‎kernel/riscv64/gemm_tcopy_16_rvv_max.c‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/***************************************************************************
2+
Copyright (c) 2025, The OpenBLAS Project
3+
All rights reserved.
4+
Redistribution and use in source and binary forms, with or without
5+
modification, are permitted provided that the following conditions are
6+
met:
7+
1. Redistributions of source code must retain the above copyright
8+
notice, this list of conditions and the following disclaimer.
9+
2. Redistributions in binary form must reproduce the above copyright
10+
notice, this list of conditions and the following disclaimer in
11+
the documentation and/or other materials provided with the
12+
distribution.
13+
3. Neither the name of the OpenBLAS project nor the names of
14+
its contributors may be used to endorse or promote products
15+
derived from this software without specific prior written permission.
16+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19+
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
20+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
25+
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
*****************************************************************************/
27+
28+
#include "common.h"
29+
30+
#if !defined(DOUBLE)
31+
#define FLOAT_V_T vfloat32m4_t
32+
#define VLEV_FLOAT __riscv_vle32_v_f32m4
33+
#define VSEV_FLOAT __riscv_vse32_v_f32m4
34+
#else
35+
#define FLOAT_V_T vfloat64m8_t
36+
#define VLEV_FLOAT __riscv_vle64_v_f64m8
37+
#define VSEV_FLOAT __riscv_vse64_v_f64m8
38+
#endif
39+
40+
#define BLOCKSIZE 16
41+
42+
#include "gemm_tcopy_rvv_max_common.h"

‎kernel/riscv64/gemm_tcopy_4_rvv_max.c‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/***************************************************************************
2+
Copyright (c) 2025, The OpenBLAS Project
3+
All rights reserved.
4+
Redistribution and use in source and binary forms, with or without
5+
modification, are permitted provided that the following conditions are
6+
met:
7+
1. Redistributions of source code must retain the above copyright
8+
notice, this list of conditions and the following disclaimer.
9+
2. Redistributions in binary form must reproduce the above copyright
10+
notice, this list of conditions and the following disclaimer in
11+
the documentation and/or other materials provided with the
12+
distribution.
13+
3. Neither the name of the OpenBLAS project nor the names of
14+
its contributors may be used to endorse or promote products
15+
derived from this software without specific prior written permission.
16+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19+
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
20+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
25+
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
*****************************************************************************/
27+
28+
#include "common.h"
29+
30+
#if !defined(DOUBLE)
31+
#define FLOAT_V_T vfloat32m1_t
32+
#define VLEV_FLOAT __riscv_vle32_v_f32m2
33+
#define VSEV_FLOAT __riscv_vse32_v_f32m2
34+
#else
35+
#define FLOAT_V_T vfloat64m2_t
36+
#define VLEV_FLOAT __riscv_vle64_v_f64m2
37+
#define VSEV_FLOAT __riscv_vse64_v_f64m2
38+
#endif
39+
40+
#define BLOCKSIZE 4
41+
42+
#include "gemm_tcopy_rvv_max_common.h"

0 commit comments

Comments
 (0)