[MLAS] RVV-Optimized LLM Operators for RISC-V (#28518)

## Description
Added RVV implementations for a subset of LLM inference operators.
Optimization of activation functions is in #28308.
All tests were conducted on a Spacemit K3 CPU (VLEN=256).

| Operator | File | Speedup vs Scalar | Precision |
| :--- | :--- | :--- | :--- |
| FP16 GEMM | `riscv64/halfgemm_kernel_rvv.cpp` | 51–191x | max_abs ≤
0.0005 (PASS) |
| FP16↔FP32 Cast | `riscv64/cast_kernel_rvv.cpp` | 4–12x | Bit-exact
(PASS) |
| RotaryEmbedding | `riscv64/rotary_embedding_kernel_rvv.cpp` | 3.1x |
max_abs ~6e-08 (PASS) |
| SimplifiedLayerNorm | `layer_norm_impl.cc` (inline RVV) | 4.3x |
Bit-exact (PASS) |

## Operator Performance

### **FP16 GEMM**

| Shape (M×N×K) | ORT Scalar | RVV | Speedup | Max Abs Error |
| :--- | :--- | :--- | :--- | :--- |
| 1×768×768 | 7.32 ms | 0.14 ms | 51.6x | 1.22e-04 |
| 32×768×768 | 235 ms | 1.25 ms | 187.9x | 3.66e-04 |
| 64×768×768 | 469 ms | 2.49 ms | 188.8x | 2.44e-04 |
| 128×3072×768 | 5718 ms | 30.4 ms | 187.8x | 4.88e-04 |

---

### **FP16↔FP32 Cast**

| Elements | Direction | ORT Scalar | RVV | Speedup |
| :--- | :--- | :--- | :--- | :--- |
| 1K | F16→F32 | 0.002 ms | 0.000 ms | 9.6x |
| 1K | F32→F16 | 0.002 ms | 0.000 ms | 10.5x |
| 64K | F16→F32 | 0.127 ms | 0.013 ms | 9.7x |
| 64K | F32→F16 | 0.150 ms | 0.013 ms | 11.3x |
| 1M | F16→F32 | 2.03 ms | 0.26 ms | 7.7x |
| 1M | F32→F16 | 2.39 ms | 0.50 ms | 4.8x |

---

### **RotaryEmbedding**

| Dim | Mode | ORT Scalar | RVV | Speedup | Max Abs Error |
| :--- | :--- | :--- | :--- | :--- | :--- |
| 64 | non-interleaved | 0.32 us | 0.05 us | 7.1x | 5.96e-08 |
| 64 | interleaved | 0.22 us | 0.07 us | 3.2x | 0 |
| 128 | non-interleaved | 0.64 us | 0.07 us | 9.7x | 5.96e-08 |
| 128 | interleaved | 0.44 us | 0.13 us | 3.4x | 0 |
| 256 | non-interleaved | 1.28 us | 0.10 us | 13.0x | 1.19e-07 |
| 256 | interleaved | 1.05 us | 0.25 us | 4.3x | 0 |

---

### **RMSNorm**

| Hidden | ORT Scalar | RVV | Speedup | Max Abs Error |
| :--- | :--- | :--- | :--- | :--- |
| 512 | 2.31 us | 0.38 us | 6.0x | 2.38e-07 |
| 1024 | 4.64 us | 0.71 us | 6.5x | 2.38e-07 |
| 2048 | 9.24 us | 1.42 us | 6.5x | 3.58e-07 |
| 4096 | 18.5 us | 2.82 us | 6.6x | 3.58e-06 |
> **Note**: ORT's LayerNorm ComputeJob is in an anonymous namespace —
there's no public API to call it separately. So I rewrite the benchmark
using the same algorithm as ORT's ComputeJob.

## Model Performance
The ONNX model comes from:
https://huggingface.co/onnx-community/Qwen3-0.6B-ONNX

| Metric | FP32 | FP16 |
| :--- | :--- | :--- |
| Prompt processing | 61.1 tok/s (255 ms p50) | 58.8 tok/s (272 ms p50)
|
| Token generation | 6.5 tok/s (152 ms p50) | 6.1 tok/s (162 ms p50) |
| E2E (16+32 tokens) | 4987 ms p50 | 5357 ms p50 |
| Peak memory | 3.1 GB | 4.1 GB |

> **Note**: FP32 is slightly faster because it runs SGEMM directly
without the FP16↔FP32 cast overhead. FP16 uses ~1 GB less storage on
disk but more runtime memory (the cast creates FP32 copies). Both use
the RVV SGEMM kernel for the actual compute.

Author: velonica0

cmake/CMakeLists.txt

@@ -90,6 +90,7 @@ option(onnxruntime_USE_DNNL "Build with DNNL support" OFF)
 option(onnxruntime_USE_JSEP "Build with JavaScript implemented kernels support" OFF)
 option(onnxruntime_USE_SVE "Build with SVE support in MLAS" OFF)
 option(onnxruntime_USE_RVV "Build with RISC-V Vector support in MLAS" OFF)
+option(onnxruntime_USE_RVV_ZVFH "Build with RISC-V Zvfh (FP16 vector) support in MLAS" OFF)
 option(onnxruntime_USE_ARM_NEON_NCHWC "Build with ARM Neon NCHWc kernels in MLAS" OFF)
 
 option(onnxruntime_USE_KLEIDIAI "Build with KleidiAI integration in MLAS" OFF)

cmake/onnxruntime_mlas.cmake

@@ -57,6 +57,7 @@ onnxruntime_add_static_library(onnxruntime_mlas
   ${MLAS_SRC_DIR}/flashattn.cpp
   ${MLAS_SRC_DIR}/qkv_quant.cpp
   ${MLAS_SRC_DIR}/cast.cpp
+  ${MLAS_SRC_DIR}/layernorm.cpp
   ${MLAS_SRC_DIR}/rotary_embedding.h
   ${MLAS_SRC_DIR}/rotary_embedding.cpp
   ${MLAS_SRC_DIR}/softmax.h
@@ -959,6 +960,8 @@ endif()
               ${MLAS_SRC_DIR}/riscv64/softmax_kernel_rvv.cpp
               ${MLAS_SRC_DIR}/riscv64/sconv_depthwise_kernel_rvv.cpp
               ${MLAS_SRC_DIR}/riscv64/sconv_nchwc_kernel_rvv.cpp
+              ${MLAS_SRC_DIR}/riscv64/rotary_embedding_kernel_rvv.cpp
+              ${MLAS_SRC_DIR}/riscv64/layernorm_kernel_rvv.cpp
             )
             list(REMOVE_ITEM mlas_platform_srcs
               "${MLAS_SRC_DIR}/sconv_nchw_depthwise_multiplier_1.cpp")
@@ -968,8 +971,22 @@ endif()
               ${MLAS_SRC_DIR}/riscv64/softmax_kernel_rvv.cpp
               ${MLAS_SRC_DIR}/riscv64/sconv_depthwise_kernel_rvv.cpp
               ${MLAS_SRC_DIR}/riscv64/sconv_nchwc_kernel_rvv.cpp
+              ${MLAS_SRC_DIR}/riscv64/rotary_embedding_kernel_rvv.cpp
+              ${MLAS_SRC_DIR}/riscv64/layernorm_kernel_rvv.cpp
               PROPERTIES COMPILE_FLAGS "-march=rv64gcv -mabi=lp64d")
             list(APPEND mlas_private_compile_definitions MLAS_USE_RVV=1)
+
+            if(onnxruntime_USE_RVV_ZVFH)
+              list(APPEND mlas_platform_srcs
+                ${MLAS_SRC_DIR}/riscv64/halfgemm_kernel_rvv.cpp
+                ${MLAS_SRC_DIR}/riscv64/cast_kernel_rvv.cpp
+              )
+              set_source_files_properties(
+                ${MLAS_SRC_DIR}/riscv64/halfgemm_kernel_rvv.cpp
+                ${MLAS_SRC_DIR}/riscv64/cast_kernel_rvv.cpp
+                PROPERTIES COMPILE_FLAGS "-march=rv64gcv_zvfh -mabi=lp64d")
+              list(APPEND mlas_private_compile_definitions MLAS_USE_RVV_ZVFH=1)
+            endif()
           else()
             message(
               WARNING

cmake/onnxruntime_unittests.cmake

@@ -1450,6 +1450,50 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
       PRIVATE ${ONNXRUNTIME_MLAS_LIBS} onnxruntime_common ${CMAKE_DL_LIBS})
     target_compile_definitions(onnxruntime_mlas_softmax_riscv_compare PRIVATE ${mlas_private_compile_definitions})
     set_target_properties(onnxruntime_mlas_softmax_riscv_compare PROPERTIES FOLDER "ONNXRuntimeTest")
+
+    onnxruntime_add_executable(
+      onnxruntime_mlas_halfgemm_rvv_bench
+      ${MLAS_RISCV64_BENCH_DIR}/halfgemm_rvv_bench.cpp)
+    target_include_directories(onnxruntime_mlas_halfgemm_rvv_bench PRIVATE
+      ${ONNXRUNTIME_ROOT}/core/mlas/inc ${ONNXRUNTIME_ROOT}/core/mlas/lib)
+    target_link_libraries(
+      onnxruntime_mlas_halfgemm_rvv_bench
+      PRIVATE ${ONNXRUNTIME_MLAS_LIBS} onnxruntime_common ${CMAKE_DL_LIBS})
+    target_compile_definitions(onnxruntime_mlas_halfgemm_rvv_bench PRIVATE ${mlas_private_compile_definitions})
+    set_target_properties(onnxruntime_mlas_halfgemm_rvv_bench PROPERTIES FOLDER "ONNXRuntimeTest")
+
+    onnxruntime_add_executable(
+      onnxruntime_mlas_cast_rvv_bench
+      ${MLAS_RISCV64_BENCH_DIR}/cast_rvv_bench.cpp)
+    target_include_directories(onnxruntime_mlas_cast_rvv_bench PRIVATE
+      ${ONNXRUNTIME_ROOT}/core/mlas/inc ${ONNXRUNTIME_ROOT}/core/mlas/lib)
+    target_link_libraries(
+      onnxruntime_mlas_cast_rvv_bench
+      PRIVATE ${ONNXRUNTIME_MLAS_LIBS} onnxruntime_common ${CMAKE_DL_LIBS})
+    target_compile_definitions(onnxruntime_mlas_cast_rvv_bench PRIVATE ${mlas_private_compile_definitions})
+    set_target_properties(onnxruntime_mlas_cast_rvv_bench PROPERTIES FOLDER "ONNXRuntimeTest")
+
+    onnxruntime_add_executable(
+      onnxruntime_mlas_rope_rvv_bench
+      ${MLAS_RISCV64_BENCH_DIR}/rope_rvv_bench.cpp)
+    target_include_directories(onnxruntime_mlas_rope_rvv_bench PRIVATE
+      ${ONNXRUNTIME_ROOT}/core/mlas/inc ${ONNXRUNTIME_ROOT}/core/mlas/lib)
+    target_link_libraries(
+      onnxruntime_mlas_rope_rvv_bench
+      PRIVATE ${ONNXRUNTIME_MLAS_LIBS} onnxruntime_common ${CMAKE_DL_LIBS})
+    target_compile_definitions(onnxruntime_mlas_rope_rvv_bench PRIVATE ${mlas_private_compile_definitions})
+    set_target_properties(onnxruntime_mlas_rope_rvv_bench PROPERTIES FOLDER "ONNXRuntimeTest")
+
+    onnxruntime_add_executable(
+      onnxruntime_mlas_rmsnorm_rvv_bench
+      ${MLAS_RISCV64_BENCH_DIR}/rmsnorm_rvv_bench.cpp)
+    target_include_directories(onnxruntime_mlas_rmsnorm_rvv_bench PRIVATE
+      ${ONNXRUNTIME_ROOT}/core/mlas/inc ${ONNXRUNTIME_ROOT}/core/mlas/lib)
+    target_link_libraries(
+      onnxruntime_mlas_rmsnorm_rvv_bench
+      PRIVATE ${ONNXRUNTIME_MLAS_LIBS} onnxruntime_common ${CMAKE_DL_LIBS})
+    target_compile_definitions(onnxruntime_mlas_rmsnorm_rvv_bench PRIVATE ${mlas_private_compile_definitions})
+    set_target_properties(onnxruntime_mlas_rmsnorm_rvv_bench PROPERTIES FOLDER "ONNXRuntimeTest")
   endif()
 
   if(WIN32)

onnxruntime/core/common/cpuid_arch_definition.h

@@ -12,3 +12,7 @@
 #if defined(_M_ARM64) || defined(_M_ARM64EC) || defined(__aarch64__) || defined(_M_ARM) || defined(__arm__)
 #define CPUIDINFO_ARCH_ARM
 #endif  // ARM or ARM64
+
+#if defined(__riscv) && __riscv_xlen == 64
+#define CPUIDINFO_ARCH_RISCV64
+#endif

onnxruntime/core/common/cpuid_info.cc

@@ -47,6 +47,16 @@
 
 #endif  // ARM
 
+#if defined(CPUIDINFO_ARCH_RISCV64)
+#include <asm/hwprobe.h>
+#ifndef RISCV_HWPROBE_EXT_ZVFH
+#define RISCV_HWPROBE_EXT_ZVFH (1 << 30)
+#endif
+#ifndef RISCV_HWPROBE_IMA_V
+#define RISCV_HWPROBE_IMA_V (1 << 2)
+#endif
+#endif  // RISCV64
+
 #endif  // Linux
 
 #if _WIN32
@@ -334,6 +344,17 @@ void CPUIDInfo::ArmAppleInit() {
 
 #endif  // defined(CPUIDINFO_ARCH_ARM)
 
+#if defined(CPUIDINFO_ARCH_RISCV64) && defined(__linux__)
+void CPUIDInfo::RiscvLinuxInit() {
+  struct riscv_hwprobe pairs[] = {
+      {RISCV_HWPROBE_KEY_IMA_EXT_0, 0},
+  };
+  if (syscall(__NR_riscv_hwprobe, pairs, 1, 0, nullptr, 0) == 0) {
+    has_fp16_ = (pairs[0].value & RISCV_HWPROBE_EXT_ZVFH) != 0;
+  }
+}
+#endif  // defined(CPUIDINFO_ARCH_RISCV64) && defined(__linux__)
+
 uint32_t CPUIDInfo::GetCurrentCoreIdx() const {
 #ifdef _WIN32
   return GetCurrentProcessorNumber();
@@ -377,5 +398,11 @@ CPUIDInfo::CPUIDInfo() {
   ArmAppleInit();
 #endif
 #endif  // defined(CPUIDINFO_ARCH_ARM)
+
+#if defined(CPUIDINFO_ARCH_RISCV64)
+#if defined(__linux__)
+  RiscvLinuxInit();
+#endif
+#endif  // defined(CPUIDINFO_ARCH_RISCV64)
 }
 }  // namespace onnxruntime

onnxruntime/core/common/cpuid_info.h

@@ -135,6 +135,12 @@ class CPUIDInfo {
 
 #endif  // defined(CPUIDINFO_ARCH_ARM)
 
+#if defined(CPUIDINFO_ARCH_RISCV64)
+#if defined(__linux__)
+  void RiscvLinuxInit();
+#endif
+#endif  // defined(CPUIDINFO_ARCH_RISCV64)
+
 #if defined(CPUINFO_SUPPORTED)
   bool pytorch_cpuinfo_init_{false};
 #endif  // defined(CPUINFO_SUPPORTED)

onnxruntime/core/mlas/inc/mlas.h

@@ -1665,6 +1665,27 @@ MlasRotaryEmbedOneRow(
     T* output
 );
 
+/**
+ * @brief Compute LayerNorm or RMSNorm (simplified) for one row of float data.
+ *        Uses platform-optimized kernel if available, otherwise returns false.
+ *        Any platform (AMD64/ARM64/RISC-V) can register a LayerNormF32Kernel.
+ *
+ * @return true if an optimized kernel was used, false if caller should fall back
+ */
+bool
+MLASCALL
+MlasLayerNormF32(
+    const float* Input,
+    const float* Scale,
+    const float* Bias,
+    float* Output,
+    float* MeanOut,
+    float* InvStdDevOut,
+    size_t NormSize,
+    float Epsilon,
+    bool Simplified
+);
+
 /**
  * @brief Supply matrices data information to half precision gemm functions
  */

onnxruntime/core/mlas/lib/halfgemm.cpp

@@ -27,6 +27,8 @@ MlasFp16AccelerationSupported()
 {
 #ifdef MLAS_F16VEC_INTRINSICS_SUPPORTED
     return MLAS_CPUIDINFO::GetCPUIDInfo().HasFp16VectorAcceleration();
+#elif defined(MLAS_TARGET_RISCV64) && defined(MLAS_USE_RVV_ZVFH)
+    return MLAS_CPUIDINFO::GetCPUIDInfo().HasFp16VectorAcceleration();
 #else
     return false;
 #endif

onnxruntime/core/mlas/lib/halfgemm.h

@@ -503,12 +503,21 @@ extern const MLAS_HALFGEMM_DISPATCH MlasHalfGemmDispatchDefault;
 extern const MLAS_HALFGEMM_DISPATCH MlasHalfGemmDispatchNeon;
 #endif
 
+#if defined(MLAS_TARGET_RISCV64) && defined(MLAS_USE_RVV_ZVFH)
+extern const MLAS_HALFGEMM_DISPATCH MlasHalfGemmDispatchRvv;
+#endif
+
 MLAS_FORCEINLINE
 const MLAS_HALFGEMM_DISPATCH*
 MlasHalfGemmGetDispatch()
 {
 #if defined(MLAS_F16VEC_INTRINSICS_SUPPORTED) && defined(MLAS_TARGET_ARM64)
     return &MlasHalfGemmDispatchNeon;
+#elif defined(MLAS_TARGET_RISCV64) && defined(MLAS_USE_RVV_ZVFH)
+    if (MLAS_CPUIDINFO::GetCPUIDInfo().HasFp16VectorAcceleration()) {
+        return &MlasHalfGemmDispatchRvv;
+    }
+    return &MlasHalfGemmDispatchDefault;
 #else
     return &MlasHalfGemmDispatchDefault;
 #endif

onnxruntime/core/mlas/lib/layernorm.cpp

@@ -0,0 +1,41 @@
+/*++
+
+Copyright (c) Microsoft Corporation. All rights reserved.
+
+Licensed under the MIT License.
+
+Module Name:
+
+    layernorm.cpp
+
+Abstract:
+
+    This module implements the dispatch for platform-optimized
+    LayerNorm/RMSNorm kernels.
+
+--*/
+
+#include "mlasi.h"
+
+bool
+    MLASCALL
+    MlasLayerNormF32(
+        const float* Input,
+        const float* Scale,
+        const float* Bias,
+        float* Output,
+        float* MeanOut,
+        float* InvStdDevOut,
+        size_t NormSize,
+        float Epsilon,
+        bool Simplified
+    )
+{
+    auto kernel = GetMlasPlatform().LayerNormF32Kernel;
+    if (kernel == nullptr) {
+        return false;
+    }
+
+    kernel(Input, Scale, Bias, Output, MeanOut, InvStdDevOut, NormSize, Epsilon, Simplified);
+    return true;
+}