[RVV] add rvv support for f16-vcmul #9971
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -15,26 +15,24 @@ | |
| #include "src/xnnpack/microfnptr.h" | ||
| #include "src/xnnpack/vbinary.h" | ||
|
|
||
| static struct xnn_cmul_config f16_cmul_config = {0}; | ||
| static struct xnn_cmul_config f32_cmul_config = {0}; | ||
|
|
||
| XNN_INIT_ONCE_GUARD(f16_cmul); | ||
| XNN_INIT_ONCE_GUARD(f32_cmul); | ||
|
|
||
| // Macros to log the microkernel names if and when they are registered. | ||
| #define XNN_INIT_CMUL_UKERNEL(ukernel) \ | ||
| (xnn_vbinary_ukernel_fn) ukernel; \ | ||
| xnn_log_info("Using cmul microkernel '%s'.", #ukernel); | ||
|
|
||
| static void init_f16_cmul_config(void) { | ||
| #if XNN_ENABLE_ARM_FP16_VECTOR && (XNN_ARCH_ARM || XNN_ARCH_ARM64) | ||
| f16_cmul_config.ukernel = XNN_INIT_CMUL_UKERNEL(xnn_f16_vcmul_ukernel__neonfp16arith_u16); | ||
| #elif XNN_ARCH_RISCV && XNN_ENABLE_RISCV_FP16_VECTOR | ||
| f16_cmul_config.ukernel = XNN_INIT_CMUL_UKERNEL(xnn_f16_vcmul_ukernel__rvvfp16arith_u2v); | ||
| #endif | ||
| } | ||
|
|
||
| static void init_f32_cmul_config(void) { | ||
| #if XNN_ARCH_ARM | ||
|
|
@@ -81,16 +79,12 @@ static void init_f32_cmul_config(void) { | |
| } | ||
|
|
||
| const struct xnn_cmul_config* xnn_init_f16_cmul_config() { | ||
| const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config(); | ||
| if (hardware_config == NULL || !xnn_is_f16_compatible_config(hardware_config)) { | ||
| return NULL; | ||
| } | ||
| XNN_INIT_ONCE(f16_cmul); | ||
| return f16_cmul_config.ukernel ? &f16_cmul_config : NULL; | ||
|
Contributor
Author
There was a problem hiding this comment. @dsharletg I modified this (line 87) to protect against the unconfigured ukernel case. (Sorry it seems your previous comments here have been dropped). |
||
| } | ||
|
|
||
| const struct xnn_cmul_config* xnn_init_f32_cmul_config() { | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| // clang-format off | ||
| // Auto-generated file. Do not edit! | ||
| // Template: src/f16-vcmul/rvv.c.in | ||
| // Generator: tools/xngen | ||
| // | ||
| // Copyright 2026 Google LLC | ||
| // | ||
| // This source code is licensed under the BSD-style license found in the | ||
| // LICENSE file in the root directory of this source tree. | ||
|
|
||
| #include <assert.h> | ||
|
|
||
| #include <riscv_vector.h> | ||
|
|
||
| #include "src/xnnpack/vbinary.h" | ||
|
|
||
| void xnn_f16_vcmul_ukernel__rvvfp16arith_u1v( | ||
| size_t batch, | ||
| const xnn_float16* input_a, | ||
| const xnn_float16* input_b, | ||
| xnn_float16* output, | ||
| const struct xnn_f16_default_params* restrict params) | ||
| { | ||
| assert(batch != 0); | ||
| assert(batch % sizeof(xnn_float16) == 0); | ||
| assert(input_a != NULL); | ||
| assert(input_b != NULL); | ||
| assert(output != NULL); | ||
|
|
||
| batch >>= XNN_LOG2_SIZEOF_FLOAT16; | ||
|
|
||
| const xnn_float16* ar = input_a; | ||
| const xnn_float16* ai = input_a + batch; | ||
| const xnn_float16* br = input_b; | ||
| const xnn_float16* bi = input_b + batch; | ||
| xnn_float16* or = output; | ||
| xnn_float16* oi = output + batch; | ||
|
|
||
| do { | ||
| size_t n = __riscv_vsetvl_e16m1(batch); batch -= n; | ||
| vfloat16m1_t var = __riscv_vle16_v_f16m1(ar, n); ar += n; | ||
| vfloat16m1_t vai = __riscv_vle16_v_f16m1(ai, n); ai += n; | ||
| vfloat16m1_t vbr = __riscv_vle16_v_f16m1(br, n); br += n; | ||
| vfloat16m1_t vbi = __riscv_vle16_v_f16m1(bi, n); bi += n; | ||
| vfloat16m1_t vaccr = __riscv_vfmul(var, vbr, n); | ||
| vfloat16m1_t vacci = __riscv_vfmul(var, vbi, n); | ||
| vaccr = __riscv_vfnmsac(vaccr, vai, vbi, n); | ||
| vacci = __riscv_vfmacc(vacci, vai, vbr, n); | ||
| __riscv_vse16(or, vaccr, n); or += n; | ||
| __riscv_vse16(oi, vacci, n); oi += n; | ||
| } while (batch > 0); | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| // clang-format off | ||
| // Auto-generated file. Do not edit! | ||
| // Template: src/f16-vcmul/rvv.c.in | ||
| // Generator: tools/xngen | ||
| // | ||
| // Copyright 2026 Google LLC | ||
| // | ||
| // This source code is licensed under the BSD-style license found in the | ||
| // LICENSE file in the root directory of this source tree. | ||
|
|
||
| #include <assert.h> | ||
|
|
||
| #include <riscv_vector.h> | ||
|
|
||
| #include "src/xnnpack/vbinary.h" | ||
|
|
||
| void xnn_f16_vcmul_ukernel__rvvfp16arith_u2v( | ||
| size_t batch, | ||
| const xnn_float16* input_a, | ||
| const xnn_float16* input_b, | ||
| xnn_float16* output, | ||
| const struct xnn_f16_default_params* restrict params) | ||
| { | ||
| assert(batch != 0); | ||
| assert(batch % sizeof(xnn_float16) == 0); | ||
| assert(input_a != NULL); | ||
| assert(input_b != NULL); | ||
| assert(output != NULL); | ||
|
|
||
| batch >>= XNN_LOG2_SIZEOF_FLOAT16; | ||
|
|
||
| const xnn_float16* ar = input_a; | ||
| const xnn_float16* ai = input_a + batch; | ||
| const xnn_float16* br = input_b; | ||
| const xnn_float16* bi = input_b + batch; | ||
| xnn_float16* or = output; | ||
| xnn_float16* oi = output + batch; | ||
|
|
||
| do { | ||
| size_t n = __riscv_vsetvl_e16m2(batch); batch -= n; | ||
| vfloat16m2_t var = __riscv_vle16_v_f16m2(ar, n); ar += n; | ||
| vfloat16m2_t vai = __riscv_vle16_v_f16m2(ai, n); ai += n; | ||
| vfloat16m2_t vbr = __riscv_vle16_v_f16m2(br, n); br += n; | ||
| vfloat16m2_t vbi = __riscv_vle16_v_f16m2(bi, n); bi += n; | ||
| vfloat16m2_t vaccr = __riscv_vfmul(var, vbr, n); | ||
| vfloat16m2_t vacci = __riscv_vfmul(var, vbi, n); | ||
| vaccr = __riscv_vfnmsac(vaccr, vai, vbi, n); | ||
| vacci = __riscv_vfmacc(vacci, vai, vbr, n); | ||
| __riscv_vse16(or, vaccr, n); or += n; | ||
| __riscv_vse16(oi, vacci, n); oi += n; | ||
| } while (batch > 0); | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| // clang-format off | ||
| // Auto-generated file. Do not edit! | ||
| // Template: src/f16-vcmul/rvv.c.in | ||
| // Generator: tools/xngen | ||
| // | ||
| // Copyright 2026 Google LLC | ||
| // | ||
| // This source code is licensed under the BSD-style license found in the | ||
| // LICENSE file in the root directory of this source tree. | ||
|
|
||
| #include <assert.h> | ||
|
|
||
| #include <riscv_vector.h> | ||
|
|
||
| #include "src/xnnpack/vbinary.h" | ||
|
|
||
| void xnn_f16_vcmul_ukernel__rvvfp16arith_u4v( | ||
| size_t batch, | ||
| const xnn_float16* input_a, | ||
| const xnn_float16* input_b, | ||
| xnn_float16* output, | ||
| const struct xnn_f16_default_params* restrict params) | ||
| { | ||
| assert(batch != 0); | ||
| assert(batch % sizeof(xnn_float16) == 0); | ||
| assert(input_a != NULL); | ||
| assert(input_b != NULL); | ||
| assert(output != NULL); | ||
|
|
||
| batch >>= XNN_LOG2_SIZEOF_FLOAT16; | ||
|
|
||
| const xnn_float16* ar = input_a; | ||
| const xnn_float16* ai = input_a + batch; | ||
| const xnn_float16* br = input_b; | ||
| const xnn_float16* bi = input_b + batch; | ||
| xnn_float16* or = output; | ||
| xnn_float16* oi = output + batch; | ||
|
|
||
| do { | ||
| size_t n = __riscv_vsetvl_e16m4(batch); batch -= n; | ||
| vfloat16m4_t var = __riscv_vle16_v_f16m4(ar, n); ar += n; | ||
| vfloat16m4_t vai = __riscv_vle16_v_f16m4(ai, n); ai += n; | ||
| vfloat16m4_t vbr = __riscv_vle16_v_f16m4(br, n); br += n; | ||
| vfloat16m4_t vbi = __riscv_vle16_v_f16m4(bi, n); bi += n; | ||
| vfloat16m4_t vaccr = __riscv_vfmul(var, vbr, n); | ||
| vfloat16m4_t vacci = __riscv_vfmul(var, vbi, n); | ||
| vaccr = __riscv_vfnmsac(vaccr, vai, vbi, n); | ||
| vacci = __riscv_vfmacc(vacci, vai, vbr, n); | ||
| __riscv_vse16(or, vaccr, n); or += n; | ||
| __riscv_vse16(oi, vacci, n); oi += n; | ||
| } while (batch > 0); | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| // Copyright 2026 Google LLC | ||
| // | ||
| // This source code is licensed under the BSD-style license found in the | ||
| // LICENSE file in the root directory of this source tree. | ||
|
|
||
| $assert LMUL in [1, 2, 4] | ||
| #include <assert.h> | ||
|
|
||
| #include <riscv_vector.h> | ||
|
|
||
| #include "src/xnnpack/vbinary.h" | ||
|
|
||
| void xnn_f16_vcmul_ukernel__rvvfp16arith_u${LMUL}v( | ||
| size_t batch, | ||
| const xnn_float16* input_a, | ||
| const xnn_float16* input_b, | ||
| xnn_float16* output, | ||
| const struct xnn_f16_default_params* restrict params) | ||
| { | ||
| assert(batch != 0); | ||
| assert(batch % sizeof(xnn_float16) == 0); | ||
| assert(input_a != NULL); | ||
| assert(input_b != NULL); | ||
| assert(output != NULL); | ||
|
|
||
| batch >>= XNN_LOG2_SIZEOF_FLOAT16; | ||
|
|
||
| const xnn_float16* ar = input_a; | ||
| const xnn_float16* ai = input_a + batch; | ||
| const xnn_float16* br = input_b; | ||
| const xnn_float16* bi = input_b + batch; | ||
| xnn_float16* or = output; | ||
| xnn_float16* oi = output + batch; | ||
|
|
||
| do { | ||
| size_t n = __riscv_vsetvl_e16m${LMUL}(batch); batch -= n; | ||
| vfloat16m${LMUL}_t var = __riscv_vle16_v_f16m${LMUL}(ar, n); ar += n; | ||
| vfloat16m${LMUL}_t vai = __riscv_vle16_v_f16m${LMUL}(ai, n); ai += n; | ||
| vfloat16m${LMUL}_t vbr = __riscv_vle16_v_f16m${LMUL}(br, n); br += n; | ||
| vfloat16m${LMUL}_t vbi = __riscv_vle16_v_f16m${LMUL}(bi, n); bi += n; | ||
| vfloat16m${LMUL}_t vaccr = __riscv_vfmul(var, vbr, n); | ||
| vfloat16m${LMUL}_t vacci = __riscv_vfmul(var, vbi, n); | ||
| vaccr = __riscv_vfnmsac(vaccr, vai, vbi, n); | ||
| vacci = __riscv_vfmacc(vacci, vai, vbr, n); | ||
| __riscv_vse16(or, vaccr, n); or += n; | ||
| __riscv_vse16(oi, vacci, n); oi += n; | ||
| } while (batch > 0); | ||
| } |
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.