Skip to content

Commit e29c390

Browse files
hanno-beckermkannwischer
authored andcommitted
AArch64: Gate NEON use on runtime capability
Previously, the native backend and ABI checker assumed that NEON was present on every AArch64 system. Add MLK_SYS_CAP_AARCH64_NEON and gate every NEON-dependent native and FIPS202 entry point on it. This allows integrators to provide runtime capability detection and fall back to C. Mark NEON requirements in the assembly ABI metadata and teach generated ABI checks to skip unsupported kernels. Route ABI checks through C call-stub wrappers. On AArch64, pass the runtime capability into the assembly stub so it only saves, seeds, captures, and restores vector registers when NEON is available. Keep the GPR self-test unconditional and run vector corrupters only when NEON is supported. Extend the custom ID-register test configuration to detect AdvSIMD through ID_AA64PFR0_EL1. Signed-off-by: Hanno Becker <beckphan@amazon.co.uk>
1 parent af4c5ab commit e29c390

89 files changed

Lines changed: 552 additions & 149 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

dev/aarch64_clean/meta.h

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,10 @@
2828
MLK_MUST_CHECK_RETURN_VALUE
2929
static MLK_INLINE int mlk_ntt_native(int16_t data[MLKEM_N])
3030
{
31+
if (!mlk_sys_check_capability(MLK_SYS_CAP_NEON))
32+
{
33+
return MLK_NATIVE_FUNC_FALLBACK;
34+
}
3135
mlk_ntt_aarch64_asm(data, mlk_aarch64_ntt_zetas_layer12345,
3236
mlk_aarch64_ntt_zetas_layer67);
3337
return MLK_NATIVE_FUNC_SUCCESS;
@@ -36,6 +40,10 @@ static MLK_INLINE int mlk_ntt_native(int16_t data[MLKEM_N])
3640
MLK_MUST_CHECK_RETURN_VALUE
3741
static MLK_INLINE int mlk_intt_native(int16_t data[MLKEM_N])
3842
{
43+
if (!mlk_sys_check_capability(MLK_SYS_CAP_NEON))
44+
{
45+
return MLK_NATIVE_FUNC_FALLBACK;
46+
}
3947
mlk_intt_aarch64_asm(data, mlk_aarch64_invntt_zetas_layer12345,
4048
mlk_aarch64_invntt_zetas_layer67);
4149
return MLK_NATIVE_FUNC_SUCCESS;
@@ -44,13 +52,21 @@ static MLK_INLINE int mlk_intt_native(int16_t data[MLKEM_N])
4452
MLK_MUST_CHECK_RETURN_VALUE
4553
static MLK_INLINE int mlk_poly_reduce_native(int16_t data[MLKEM_N])
4654
{
55+
if (!mlk_sys_check_capability(MLK_SYS_CAP_NEON))
56+
{
57+
return MLK_NATIVE_FUNC_FALLBACK;
58+
}
4759
mlk_poly_reduce_aarch64_asm(data);
4860
return MLK_NATIVE_FUNC_SUCCESS;
4961
}
5062

5163
MLK_MUST_CHECK_RETURN_VALUE
5264
static MLK_INLINE int mlk_poly_tomont_native(int16_t data[MLKEM_N])
5365
{
66+
if (!mlk_sys_check_capability(MLK_SYS_CAP_NEON))
67+
{
68+
return MLK_NATIVE_FUNC_FALLBACK;
69+
}
5470
mlk_poly_tomont_aarch64_asm(data);
5571
return MLK_NATIVE_FUNC_SUCCESS;
5672
}
@@ -59,6 +75,10 @@ MLK_MUST_CHECK_RETURN_VALUE
5975
static MLK_INLINE int mlk_poly_mulcache_compute_native(int16_t x[MLKEM_N / 2],
6076
const int16_t y[MLKEM_N])
6177
{
78+
if (!mlk_sys_check_capability(MLK_SYS_CAP_NEON))
79+
{
80+
return MLK_NATIVE_FUNC_FALLBACK;
81+
}
6282
mlk_poly_mulcache_compute_aarch64_asm(
6383
x, y, mlk_aarch64_zetas_mulcache_native,
6484
mlk_aarch64_zetas_mulcache_twisted_native);
@@ -71,6 +91,10 @@ static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k2_native(
7191
int16_t r[MLKEM_N], const int16_t a[2 * MLKEM_N],
7292
const int16_t b[2 * MLKEM_N], const int16_t b_cache[2 * (MLKEM_N / 2)])
7393
{
94+
if (!mlk_sys_check_capability(MLK_SYS_CAP_NEON))
95+
{
96+
return MLK_NATIVE_FUNC_FALLBACK;
97+
}
7498
mlk_polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm(r, a, b, b_cache);
7599
return MLK_NATIVE_FUNC_SUCCESS;
76100
}
@@ -82,6 +106,10 @@ static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k3_native(
82106
int16_t r[MLKEM_N], const int16_t a[3 * MLKEM_N],
83107
const int16_t b[3 * MLKEM_N], const int16_t b_cache[3 * (MLKEM_N / 2)])
84108
{
109+
if (!mlk_sys_check_capability(MLK_SYS_CAP_NEON))
110+
{
111+
return MLK_NATIVE_FUNC_FALLBACK;
112+
}
85113
mlk_polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm(r, a, b, b_cache);
86114
return MLK_NATIVE_FUNC_SUCCESS;
87115
}
@@ -93,6 +121,10 @@ static MLK_INLINE int mlk_polyvec_basemul_acc_montgomery_cached_k4_native(
93121
int16_t r[MLKEM_N], const int16_t a[4 * MLKEM_N],
94122
const int16_t b[4 * MLKEM_N], const int16_t b_cache[4 * (MLKEM_N / 2)])
95123
{
124+
if (!mlk_sys_check_capability(MLK_SYS_CAP_NEON))
125+
{
126+
return MLK_NATIVE_FUNC_FALLBACK;
127+
}
96128
mlk_polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm(r, a, b, b_cache);
97129
return MLK_NATIVE_FUNC_SUCCESS;
98130
}
@@ -102,6 +134,10 @@ MLK_MUST_CHECK_RETURN_VALUE
102134
static MLK_INLINE int mlk_poly_tobytes_native(uint8_t r[MLKEM_POLYBYTES],
103135
const int16_t a[MLKEM_N])
104136
{
137+
if (!mlk_sys_check_capability(MLK_SYS_CAP_NEON))
138+
{
139+
return MLK_NATIVE_FUNC_FALLBACK;
140+
}
105141
mlk_poly_tobytes_aarch64_asm(r, a);
106142
return MLK_NATIVE_FUNC_SUCCESS;
107143
}
@@ -111,8 +147,8 @@ static MLK_INLINE int mlk_rej_uniform_native(int16_t *r, unsigned len,
111147
const uint8_t *buf,
112148
unsigned buflen)
113149
{
114-
if (len != MLKEM_N ||
115-
buflen % 24 != 0) /* NEON support is mandatory for AArch64 */
150+
if (!mlk_sys_check_capability(MLK_SYS_CAP_NEON) || len != MLKEM_N ||
151+
buflen % 24 != 0)
116152
{
117153
return MLK_NATIVE_FUNC_FALLBACK;
118154
}

dev/aarch64_clean/src/intt_aarch64_asm.S

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
ABI:
2727
Architecture: aarch64
2828
CallingConvention: AAPCS64
29+
Features: [NEON]
2930
x0:
3031
type: buffer
3132
size_bytes: 512

dev/aarch64_clean/src/ntt_aarch64_asm.S

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
ABI:
2727
Architecture: aarch64
2828
CallingConvention: AAPCS64
29+
Features: [NEON]
2930
x0:
3031
type: buffer
3132
size_bytes: 512

dev/aarch64_clean/src/poly_mulcache_compute_aarch64_asm.S

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
ABI:
1111
Architecture: aarch64
1212
CallingConvention: AAPCS64
13+
Features: [NEON]
1314
x0:
1415
type: buffer
1516
size_bytes: 256

dev/aarch64_clean/src/poly_reduce_aarch64_asm.S

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
ABI:
1111
Architecture: aarch64
1212
CallingConvention: AAPCS64
13+
Features: [NEON]
1314
x0:
1415
type: buffer
1516
size_bytes: 512

dev/aarch64_clean/src/poly_tobytes_aarch64_asm.S

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
ABI:
1111
Architecture: aarch64
1212
CallingConvention: AAPCS64
13+
Features: [NEON]
1314
x0:
1415
type: buffer
1516
size_bytes: 384

dev/aarch64_clean/src/poly_tomont_aarch64_asm.S

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
ABI:
1111
Architecture: aarch64
1212
CallingConvention: AAPCS64
13+
Features: [NEON]
1314
x0:
1415
type: buffer
1516
size_bytes: 512

dev/aarch64_clean/src/polyvec_basemul_acc_montgomery_cached_k2_aarch64_asm.S

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
ABI:
2020
Architecture: aarch64
2121
CallingConvention: AAPCS64
22+
Features: [NEON]
2223
x0:
2324
type: buffer
2425
size_bytes: 512

dev/aarch64_clean/src/polyvec_basemul_acc_montgomery_cached_k3_aarch64_asm.S

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
ABI:
2020
Architecture: aarch64
2121
CallingConvention: AAPCS64
22+
Features: [NEON]
2223
x0:
2324
type: buffer
2425
size_bytes: 512

dev/aarch64_clean/src/polyvec_basemul_acc_montgomery_cached_k4_aarch64_asm.S

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
ABI:
2020
Architecture: aarch64
2121
CallingConvention: AAPCS64
22+
Features: [NEON]
2223
x0:
2324
type: buffer
2425
size_bytes: 512

0 commit comments

Comments
 (0)