Merge pull request #5898 from moluopro/develop

Fix ARM64 SME direct-kernel build and dispatch

Author: martin-frbg

cmake/cc.cmake

@@ -313,13 +313,13 @@ endif ()
 if (${CORE} STREQUAL ARMV9SME)
   if (NOT DYNAMIC_ARCH)
     if (${CMAKE_C_COMPILER_ID} STREQUAL "NVHPC" AND NOT NO_SVE)
-	    set (CCOMMON_OPT  "${CCOMMON_OPT} -tp=host")
+      set (CCOMMON_OPT  "${CCOMMON_OPT} -tp=host")
     else ()
-    set (CCOMMON_OPT "${CCOMMON_OPT} -march=armv9-a+sme")
-	if (${OSNAME} STREQUAL Windows AND ${CMAKE_C_COMPILER_ID} MATCHES "Clang" )
-	set (CCOMMON_OPT "${CCOMMON_OPT} --aarch64-stack-hazard-size=0")
+      set (CCOMMON_OPT "${CCOMMON_OPT} -march=armv9-a+sme")
+      if (CMAKE_SYSTEM_NAME STREQUAL "Windows" AND CMAKE_C_COMPILER_ID MATCHES "Clang")
+        set (CCOMMON_OPT "${CCOMMON_OPT} --aarch64-stack-hazard-size=0")
+      endif ()
     endif ()
-	endif ()
   endif ()
 endif ()
 

interface/trsm.c

@@ -361,6 +361,7 @@ void CNAME(enum CBLAS_ORDER order,
 
   if (args.m == 0 || args.n == 0) return;
 
+#if defined(TRMM)
 #if !defined(COMPLEX) && !defined(DOUBLE) && !defined(BFLOAT16)  && !defined(HFLOAT16)
 #if defined(ARCH_ARM64) && (defined(USE_STRMM_KERNEL_DIRECT)||defined(DYNAMIC_ARCH))
 #if defined(DYNAMIC_ARCH)
@@ -371,15 +372,17 @@ if (strcmp(gotoblas_corename(), "armv9sme") == 0
 )
 #endif
   if (order == CblasRowMajor && Diag == CblasNonUnit && Side == CblasLeft && m == lda && n == ldb) {
-    if (Trans ==  CblasNoTrans) {
+    if (Trans == CblasNoTrans || Trans == CblasConjNoTrans) {
       (Uplo == CblasUpper ? STRMM_DIRECT_LNUN : STRMM_DIRECT_LNLN)(m, n, alpha, a, lda, b, ldb);
-    } else if (Trans == CblasTrans) {
+      return;
+    } else if (Trans == CblasTrans || Trans == CblasConjTrans) {
       (Uplo == CblasUpper ? STRMM_DIRECT_LTUN : STRMM_DIRECT_LTLN)(m, n, alpha, a, lda, b, ldb);
+      return;
     }
-    return;
   }
 #endif
 #endif
+#endif
 
 
 

kernel/CMakeLists.txt

@@ -292,38 +292,38 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
     if (USE_DIRECT_SSYMM)
       if (ARM64)
 	  set (SSYMMDIRECTKERNEL_ALPHA_BETA ssymm_direct_alpha_beta_arm64_sme1.c)
-      GenerateNamedObjects("${KERNELDIR}/${SSYMMDIRECTKERNEL_ALPHA_BETA}" "" "symm_direct_alpha_betaLU" false "" "" false SINGLE)
-      GenerateNamedObjects("${KERNELDIR}/${SSYMMDIRECTKERNEL_ALPHA_BETA}" "" "symm_direct_alpha_betaLL" false "" "" false SINGLE)
+      GenerateNamedObjects("${KERNELDIR}/${SSYMMDIRECTKERNEL_ALPHA_BETA}" "LEFT;UPPER" "symm_direct_alpha_betaLU" false "" "" false SINGLE)
+      GenerateNamedObjects("${KERNELDIR}/${SSYMMDIRECTKERNEL_ALPHA_BETA}" "LEFT;LOWER" "symm_direct_alpha_betaLL" false "" "" false SINGLE)
    	  endif ()
     endif()
 
     if (USE_DIRECT_STRMM)
         if (ARM64)
         set (STRMMDIRECTKERNEL strmm_direct_arm64_sme1.c)
-        GenerateNamedObjects("${KERNELDIR}/${STRMMDIRECTKERNEL}" "" "trmm_direct_LNUN" false "" "" false SINGLE)
+        GenerateNamedObjects("${KERNELDIR}/${STRMMDIRECTKERNEL}" "UPPER" "trmm_direct_LNUN" false "" "" false SINGLE)
         GenerateNamedObjects("${KERNELDIR}/${STRMMDIRECTKERNEL}" "" "trmm_direct_LNLN" false "" "" false SINGLE)
-        GenerateNamedObjects("${KERNELDIR}/${STRMMDIRECTKERNEL}" "" "trmm_direct_LTUN" false "" "" false SINGLE)
-        GenerateNamedObjects("${KERNELDIR}/${STRMMDIRECTKERNEL}" "" "trmm_direct_LTLN" false "" "" false SINGLE)
+        GenerateNamedObjects("${KERNELDIR}/${STRMMDIRECTKERNEL}" "TRANSA;UPPER" "trmm_direct_LTUN" false "" "" false SINGLE)
+        GenerateNamedObjects("${KERNELDIR}/${STRMMDIRECTKERNEL}" "TRANSA" "trmm_direct_LTLN" false "" "" false SINGLE)
         endif ()
       endif ()
 
     if (USE_DIRECT_SSYRK)
       if (ARM64)
       set (SSYRKDIRECTKERNEL_ALPHA_BETA ssyrk_direct_alpha_beta_arm64_sme1.c)
-      GenerateNamedObjects("${KERNELDIR}/${SSYRKDIRECTKERNEL_ALPHA_BETA}" "" "syrk_direct_alpha_betaUN" false "" "" false SINGLE)
-      GenerateNamedObjects("${KERNELDIR}/${SSYRKDIRECTKERNEL_ALPHA_BETA}" "" "syrk_direct_alpha_betaUT" false "" "" false SINGLE)
+      GenerateNamedObjects("${KERNELDIR}/${SSYRKDIRECTKERNEL_ALPHA_BETA}" "UPPER" "syrk_direct_alpha_betaUN" false "" "" false SINGLE)
+      GenerateNamedObjects("${KERNELDIR}/${SSYRKDIRECTKERNEL_ALPHA_BETA}" "TRANSA;UPPER" "syrk_direct_alpha_betaUT" false "" "" false SINGLE)
       GenerateNamedObjects("${KERNELDIR}/${SSYRKDIRECTKERNEL_ALPHA_BETA}" "" "syrk_direct_alpha_betaLN" false "" "" false SINGLE)
-      GenerateNamedObjects("${KERNELDIR}/${SSYRKDIRECTKERNEL_ALPHA_BETA}" "" "syrk_direct_alpha_betaLT" false "" "" false SINGLE)
+      GenerateNamedObjects("${KERNELDIR}/${SSYRKDIRECTKERNEL_ALPHA_BETA}" "TRANSA" "syrk_direct_alpha_betaLT" false "" "" false SINGLE)
       endif ()
     endif()
 
     if (USE_DIRECT_SSYR2K)
       if (ARM64)
 	    set (SSYR2KDIRECTKERNEL_ALPHA_BETA ssyr2k_direct_alpha_beta_arm64_sme1.c)
-      GenerateNamedObjects("${KERNELDIR}/${SSYR2KDIRECTKERNEL_ALPHA_BETA}" "" "syr2k_direct_alpha_betaUN" false "" "" false SINGLE)
-      GenerateNamedObjects("${KERNELDIR}/${SSYR2KDIRECTKERNEL_ALPHA_BETA}" "" "syr2k_direct_alpha_betaUT" false "" "" false SINGLE)
+      GenerateNamedObjects("${KERNELDIR}/${SSYR2KDIRECTKERNEL_ALPHA_BETA}" "UPPER" "syr2k_direct_alpha_betaUN" false "" "" false SINGLE)
+      GenerateNamedObjects("${KERNELDIR}/${SSYR2KDIRECTKERNEL_ALPHA_BETA}" "TRANSA;UPPER" "syr2k_direct_alpha_betaUT" false "" "" false SINGLE)
       GenerateNamedObjects("${KERNELDIR}/${SSYR2KDIRECTKERNEL_ALPHA_BETA}" "" "syr2k_direct_alpha_betaLN" false "" "" false SINGLE)
-      GenerateNamedObjects("${KERNELDIR}/${SSYR2KDIRECTKERNEL_ALPHA_BETA}" "" "syr2k_direct_alpha_betaLT" false "" "" false SINGLE)
+      GenerateNamedObjects("${KERNELDIR}/${SSYR2KDIRECTKERNEL_ALPHA_BETA}" "TRANSA" "syr2k_direct_alpha_betaLT" false "" "" false SINGLE)
       endif ()
     endif()
 
@@ -1555,4 +1555,3 @@ else ()
   set(KERNEL_DEFINITIONS "")
   build_core("${TARGET_CORE}" "${KDIR}" "${TSUFFIX}" "${KERNEL_DEFINITIONS}")
 endif ()
-