Optimized BGEMV for NEOVERSEN2, NEOVERSEV1 and NEOVERSEV2 targets

- Adds bgemv T based off of sbgemv T kernel
- Adds bgemv N which is slightly alterated to not use Y as an
accumulator due to the output being bf16 which results in loss of
precision
- Extends NEOVERSEV2 from NEOVERSEN2 to match existing BF16 kernels with
SVE enabled
- Re-uses the existing kernels for BGEMM and BGEMV in NEOVERSEN2 and
NEOVERSEV2 targets
- Enables BGEMM_GEMV_FORWARD to proxy BGEMM to BGEMV with new kernels

Author: Mousius

Makefile.system

@@ -277,6 +277,7 @@ endif
 ifeq ($(ARCH), arm64)
 GEMM_GEMV_FORWARD = 1
 SBGEMM_GEMV_FORWARD = 1
+BGEMM_GEMV_FORWARD = 1
 endif
 ifeq ($(ARCH), riscv)
 GEMM_GEMV_FORWARD = 1
@@ -296,6 +297,9 @@ endif
 ifeq ($(SBGEMM_GEMV_FORWARD), 1)
 CCOMMON_OPT += -DSBGEMM_GEMV_FORWARD
 endif
+ifeq ($(BGEMM_GEMV_FORWARD), 1)
+CCOMMON_OPT += -DBGEMM_GEMV_FORWARD
+endif
 endif
 
 # This operation is expensive, so execution should be once.

benchmark/Makefile

@@ -84,7 +84,7 @@ GOTO_LAPACK_TARGETS=
 endif
 
 ifeq ($(BUILD_BFLOAT16),1)
-GOTO_BFLOAT_TARGETS=bgemm.goto sbgemm.goto
+GOTO_BFLOAT_TARGETS=bgemm.goto sbgemm.goto bgemv.goto sbgemv.goto
 else
 GOTO_BFLOAT_TARGETS=
 endif
@@ -667,6 +667,10 @@ bgemm.goto : bgemm.$(SUFFIX) ../$(LIBNAME)
 	$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm
 sbgemm.goto : sbgemm.$(SUFFIX) ../$(LIBNAME)
 	$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm
+bgemv.goto : bgemv.$(SUFFIX) ../$(LIBNAME)
+	$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm
+sbgemv.goto : sbgemv.$(SUFFIX) ../$(LIBNAME)
+	$(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm
 endif
 
 ifeq ($(BUILD_HFLOAT16),1)
@@ -3146,6 +3150,13 @@ dgemv.$(SUFFIX) : gemv.c
 cgemv.$(SUFFIX) : gemv.c
 	$(CC) $(CFLAGS) -c -DCOMPLEX -UDOUBLE -o $(@F) $^
 
+ifeq ($(BUILD_BFLOAT16),1) 
+bgemv.$(SUFFIX) : gemv.c
+	$(CC) $(CFLAGS) -c -DBFLOAT16 -DBGEMM -UCOMPLEX -UDOUBLE -o $(@F) $^
+sbgemv.$(SUFFIX) : gemv.c
+	$(CC) $(CFLAGS) -c -DBFLOAT16 -UCOMPLEX -UDOUBLE -o $(@F) $^
+endif ()
+
 zgemv.$(SUFFIX) : gemv.c
 	$(CC) $(CFLAGS) -c -DCOMPLEX -DDOUBLE -o $(@F) $^
 

benchmark/gemv.c

@@ -1,5 +1,5 @@
 /***************************************************************************
-Copyright (c) 2014, The OpenBLAS Project
+Copyright (c) 2014, 2025 The OpenBLAS Project
 All rights reserved.
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are
@@ -34,6 +34,12 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #ifdef DOUBLE
 #define GEMV   BLASFUNC(dgemv)
+#elif defined(BFLOAT16) && defined(BGEMM)
+#define GEMV   BLASFUNC(bgemv)
+#elif defined(BFLOAT16)
+#define GEMV   BLASFUNC(sbgemv)
+#undef IFLOAT
+#define IFLOAT bfloat16
 #else
 #define GEMV   BLASFUNC(sgemv)
 #endif
@@ -49,9 +55,20 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #endif
 int main(int argc, char *argv[]){
 
-  FLOAT *a, *x, *y;
-  FLOAT alpha[] = {1.0, 1.0};
-  FLOAT beta [] = {1.0, 0.0};
+  IFLOAT *a, *x;
+  FLOAT *y;
+#ifdef BGEMM
+  blasint one=1;
+  blasint two=2;
+  float alpha_in[] = {1.0, 0.0};
+  float beta_in[] = {0.0, 0.0};
+  FLOAT alpha[2], beta[2];
+  sbstobf16_(&two, alpha_in, &one, alpha, &one);
+  sbstobf16_(&two, beta_in, &one, beta, &one);
+#else
+  FLOAT alpha[] = {1.0, 0.0};
+  FLOAT beta [] = {0.0, 0.0};
+#endif
   char trans='N';
   blasint m, i, j;
   blasint inc_x=1,inc_y=1;
@@ -97,11 +114,11 @@ int main(int argc, char *argv[]){
 
   fprintf(stderr, "From : %3d  To : %3d Step = %3d Trans = '%c' Inc_x = %d Inc_y = %d Loops = %d\n", from, to, step,trans,inc_x,inc_y,loops);
 
-  if (( a = (FLOAT *)malloc(sizeof(FLOAT) * tomax * tomax * COMPSIZE)) == NULL){
+  if (( a = (IFLOAT *)malloc(sizeof(IFLOAT) * tomax * tomax * COMPSIZE)) == NULL){
     fprintf(stderr,"Out of Memory!!\n");exit(1);
   }
 
-  if (( x = (FLOAT *)malloc(sizeof(FLOAT) * tomax * abs(inc_x) * COMPSIZE)) == NULL){
+  if (( x = (IFLOAT *)malloc(sizeof(IFLOAT) * tomax * abs(inc_x) * COMPSIZE)) == NULL){
     fprintf(stderr,"Out of Memory!!\n");exit(1);
   }
 
@@ -125,15 +142,15 @@ int main(int argc, char *argv[]){
    		fprintf(stderr, " %6dx%d : ", (int)m,(int)n);
    		for(j = 0; j < m; j++){
       			for(i = 0; i < n * COMPSIZE; i++){
-				a[(long)i + (long)j * (long)m * COMPSIZE] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
+				a[(long)i + (long)j * (long)m * COMPSIZE] = ((IFLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
       			}
    		}
 
     		for (l=0; l<loops; l++)
     		{
 
    			for(i = 0; i < n * COMPSIZE * abs(inc_x); i++){
-				x[i] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
+				x[i] = ((IFLOAT) rand() / (FLOAT) RAND_MAX) - 0.5;
    			}
 
    			for(i = 0; i < m * COMPSIZE * abs(inc_y); i++){

kernel/arm64/KERNEL.NEOVERSEN2

@@ -188,6 +188,23 @@ ZGEMMOTCOPY    =  ../generic/zgemm_tcopy_$(ZGEMM_UNROLL_N).c
 ZGEMMONCOPYOBJ =  zgemm_oncopy$(TSUFFIX).$(SUFFIX)
 ZGEMMOTCOPYOBJ =  zgemm_otcopy$(TSUFFIX).$(SUFFIX)
 
+ifeq ($(BUILD_BFLOAT16), 1)
+BGEMM_BETA    =  bgemm_beta_neon.c
+BGEMMKERNEL    = bgemm_kernel_$(BGEMM_UNROLL_M)x$(BGEMM_UNROLL_N)_neoversev1.c
+ifneq ($(BGEMM_UNROLL_M), $(BGEMM_UNROLL_N))
+BGEMMINCOPY    = sbgemm_ncopy_$(SBGEMM_UNROLL_M)_neoversev1.c
+BGEMMITCOPY    = sbgemm_tcopy_$(SBGEMM_UNROLL_M)_neoversev1.c
+BGEMMINCOPYOBJ =  bgemm_incopy$(TSUFFIX).$(SUFFIX)
+BGEMMITCOPYOBJ =  bgemm_itcopy$(TSUFFIX).$(SUFFIX)
+endif
+BGEMMONCOPY    = sbgemm_ncopy_$(BGEMM_UNROLL_N)_neoversev1.c
+BGEMMOTCOPY    = sbgemm_tcopy_$(BGEMM_UNROLL_N)_neoversev1.c
+BGEMMONCOPYOBJ =  bgemm_oncopy$(TSUFFIX).$(SUFFIX)
+BGEMMOTCOPYOBJ =  bgemm_otcopy$(TSUFFIX).$(SUFFIX)
+
+BGEMVTKERNEL = sbgemv_t_bfdot.c
+BGEMVNKERNEL = bgemv_n_sve_v3x4.c
+
 SBGEMM_BETA    =  sbgemm_beta_neoversen2.c
 SBGEMMKERNEL    = sbgemm_kernel_$(SBGEMM_UNROLL_M)x$(SBGEMM_UNROLL_N)_neoversen2.c
 SBGEMMINCOPY    = sbgemm_ncopy_$(SBGEMM_UNROLL_M)_neoversen2.c
@@ -199,4 +216,5 @@ SBGEMMITCOPYOBJ =  sbgemm_itcopy$(TSUFFIX).$(SUFFIX)
 SBGEMMONCOPYOBJ =  sbgemm_oncopy$(TSUFFIX).$(SUFFIX)
 SBGEMMOTCOPYOBJ =  sbgemm_otcopy$(TSUFFIX).$(SUFFIX)
 SBGEMVTKERNEL = sbgemv_t_bfdot.c
-SBGEMVNKERNEL = sbgemv_n_neon.c
+SBGEMVNKERNEL = sbgemv_n_neon.c
+endif

kernel/arm64/KERNEL.NEOVERSEV1

@@ -46,6 +46,9 @@ BGEMMOTCOPY    = sbgemm_tcopy_$(BGEMM_UNROLL_N)_neoversev1.c
 BGEMMONCOPYOBJ =  bgemm_oncopy$(TSUFFIX).$(SUFFIX)
 BGEMMOTCOPYOBJ =  bgemm_otcopy$(TSUFFIX).$(SUFFIX)
 
+BGEMVTKERNEL = sbgemv_t_bfdot.c
+BGEMVNKERNEL = bgemv_n_sve_v3x4.c
+
 SBGEMM_BETA    =  sbgemm_beta_neoversev1.c
 SBGEMMKERNEL    = sbgemm_kernel_$(SBGEMM_UNROLL_M)x$(SBGEMM_UNROLL_N)_neoversev1.c
 ifneq ($(SBGEMM_UNROLL_M), $(SBGEMM_UNROLL_N))

kernel/arm64/KERNEL.NEOVERSEV2

@@ -1,6 +1 @@
-include $(KERNELDIR)/KERNEL.ARMV8SVE
-
-ifeq ($(BUILD_BFLOAT16), 1)
-SBGEMVTKERNEL = sbgemv_t_bfdot.c
-SBGEMVNKERNEL = sbgemv_n_neon.c
-endif
+include $(KERNELDIR)/KERNEL.NEOVERSEN2