Skip to content

Commit 962dbd2

Browse files
hanno-beckermkannwischer
hanno-becker
authored and
mkannwischer
committed
AArch64: Gate NEON use on runtime capability
Previously, the native backend and ABI checker assumed that NEON was present on every AArch64 system. Add MLD_SYS_CAP_AARCH64_NEON and gate every NEON-dependent native and FIPS202 entry point on it. This allows integrators to provide runtime capability detection and fall back to C. Mark NEON requirements in the assembly ABI metadata and teach generated ABI checks to skip unsupported kernels. Route ABI checks through C call-stub wrappers. On AArch64, pass the runtime capability into the assembly stub so it only saves, seeds, captures, and restores vector registers when NEON is available. Keep the GPR self-test unconditional and run vector corrupters only when NEON is supported. Extend the custom ID-register test configuration to detect AdvSIMD through ID_AA64PFR0_EL1. Signed-off-by: Hanno Becker <beckphan@amazon.co.uk>
1 parent d704b7b commit 962dbd2

145 files changed

Lines changed: 751 additions & 192 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎dev/aarch64_clean/meta.h‎

Lines changed: 62 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,10 @@
3838
MLD_MUST_CHECK_RETURN_VALUE
3939
static MLD_INLINE int mld_ntt_native(int32_t data[MLDSA_N])
4040
{
41+
if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
42+
{
43+
return MLD_NATIVE_FUNC_FALLBACK;
44+
}
4145
mld_ntt_aarch64_asm(data, mld_aarch64_ntt_zetas_layer123456,
4246
mld_aarch64_ntt_zetas_layer78);
4347
return MLD_NATIVE_FUNC_SUCCESS;
@@ -46,6 +50,10 @@ static MLD_INLINE int mld_ntt_native(int32_t data[MLDSA_N])
4650
MLD_MUST_CHECK_RETURN_VALUE
4751
static MLD_INLINE int mld_intt_native(int32_t data[MLDSA_N])
4852
{
53+
if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
54+
{
55+
return MLD_NATIVE_FUNC_FALLBACK;
56+
}
4957
mld_intt_aarch64_asm(data, mld_aarch64_intt_zetas_layer78,
5058
mld_aarch64_intt_zetas_layer123456);
5159
return MLD_NATIVE_FUNC_SUCCESS;
@@ -56,8 +64,8 @@ static MLD_INLINE int mld_rej_uniform_native(int32_t *r, unsigned len,
5664
const uint8_t *buf,
5765
unsigned buflen)
5866
{
59-
if (len != MLDSA_N ||
60-
buflen % 24 != 0) /* NEON support is mandatory for AArch64 */
67+
if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON) || len != MLDSA_N ||
68+
buflen % 24 != 0)
6169
{
6270
return MLD_NATIVE_FUNC_FALLBACK;
6371
}
@@ -76,7 +84,8 @@ static MLD_INLINE int mld_rej_uniform_eta2_native(int32_t *r, unsigned len,
7684
{
7785
uint64_t outlen;
7886
/* AArch64 implementation assumes specific buffer lengths */
79-
if (len != MLDSA_N || buflen != MLD_AARCH64_REJ_UNIFORM_ETA2_BUFLEN)
87+
if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON) || len != MLDSA_N ||
88+
buflen != MLD_AARCH64_REJ_UNIFORM_ETA2_BUFLEN)
8089
{
8190
return MLD_NATIVE_FUNC_FALLBACK;
8291
}
@@ -104,7 +113,8 @@ static MLD_INLINE int mld_rej_uniform_eta4_native(int32_t *r, unsigned len,
104113
{
105114
uint64_t outlen;
106115
/* AArch64 implementation assumes specific buffer lengths */
107-
if (len != MLDSA_N || buflen != MLD_AARCH64_REJ_UNIFORM_ETA4_BUFLEN)
116+
if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON) || len != MLDSA_N ||
117+
buflen != MLD_AARCH64_REJ_UNIFORM_ETA4_BUFLEN)
108118
{
109119
return MLD_NATIVE_FUNC_FALLBACK;
110120
}
@@ -131,6 +141,10 @@ static MLD_INLINE int mld_rej_uniform_eta4_native(int32_t *r, unsigned len,
131141
MLD_MUST_CHECK_RETURN_VALUE
132142
static MLD_INLINE int mld_poly_decompose_32_native(int32_t *a1, int32_t *a0)
133143
{
144+
if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
145+
{
146+
return MLD_NATIVE_FUNC_FALLBACK;
147+
}
134148
mld_poly_decompose_32_aarch64_asm(a1, a0);
135149
return MLD_NATIVE_FUNC_SUCCESS;
136150
}
@@ -141,6 +155,10 @@ static MLD_INLINE int mld_poly_decompose_32_native(int32_t *a1, int32_t *a0)
141155
MLD_MUST_CHECK_RETURN_VALUE
142156
static MLD_INLINE int mld_poly_decompose_88_native(int32_t *a1, int32_t *a0)
143157
{
158+
if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
159+
{
160+
return MLD_NATIVE_FUNC_FALLBACK;
161+
}
144162
mld_poly_decompose_88_aarch64_asm(a1, a0);
145163
return MLD_NATIVE_FUNC_SUCCESS;
146164
}
@@ -151,6 +169,10 @@ static MLD_INLINE int mld_poly_decompose_88_native(int32_t *a1, int32_t *a0)
151169
MLD_MUST_CHECK_RETURN_VALUE
152170
static MLD_INLINE int mld_poly_caddq_native(int32_t a[MLDSA_N])
153171
{
172+
if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
173+
{
174+
return MLD_NATIVE_FUNC_FALLBACK;
175+
}
154176
mld_poly_caddq_aarch64_asm(a);
155177
return MLD_NATIVE_FUNC_SUCCESS;
156178
}
@@ -161,6 +183,10 @@ static MLD_INLINE int mld_poly_caddq_native(int32_t a[MLDSA_N])
161183
MLD_MUST_CHECK_RETURN_VALUE
162184
static MLD_INLINE int mld_poly_use_hint_32_native(int32_t *a, const int32_t *h)
163185
{
186+
if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
187+
{
188+
return MLD_NATIVE_FUNC_FALLBACK;
189+
}
164190
mld_poly_use_hint_32_aarch64_asm(a, h);
165191
return MLD_NATIVE_FUNC_SUCCESS;
166192
}
@@ -171,6 +197,10 @@ static MLD_INLINE int mld_poly_use_hint_32_native(int32_t *a, const int32_t *h)
171197
MLD_MUST_CHECK_RETURN_VALUE
172198
static MLD_INLINE int mld_poly_use_hint_88_native(int32_t *a, const int32_t *h)
173199
{
200+
if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
201+
{
202+
return MLD_NATIVE_FUNC_FALLBACK;
203+
}
174204
mld_poly_use_hint_88_aarch64_asm(a, h);
175205
return MLD_NATIVE_FUNC_SUCCESS;
176206
}
@@ -181,6 +211,10 @@ static MLD_INLINE int mld_poly_use_hint_88_native(int32_t *a, const int32_t *h)
181211
MLD_MUST_CHECK_RETURN_VALUE
182212
static MLD_INLINE int mld_poly_chknorm_native(const int32_t *a, int32_t B)
183213
{
214+
if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
215+
{
216+
return MLD_NATIVE_FUNC_FALLBACK;
217+
}
184218
return mld_poly_chknorm_aarch64_asm(a, B);
185219
}
186220

@@ -189,6 +223,10 @@ static MLD_INLINE int mld_poly_chknorm_native(const int32_t *a, int32_t B)
189223
MLD_MUST_CHECK_RETURN_VALUE
190224
static MLD_INLINE int mld_polyz_unpack_17_native(int32_t *r, const uint8_t *buf)
191225
{
226+
if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
227+
{
228+
return MLD_NATIVE_FUNC_FALLBACK;
229+
}
192230
mld_polyz_unpack_17_aarch64_asm(r, buf, mld_polyz_unpack_17_indices);
193231
return MLD_NATIVE_FUNC_SUCCESS;
194232
}
@@ -200,6 +238,10 @@ static MLD_INLINE int mld_polyz_unpack_17_native(int32_t *r, const uint8_t *buf)
200238
MLD_MUST_CHECK_RETURN_VALUE
201239
static MLD_INLINE int mld_polyz_unpack_19_native(int32_t *r, const uint8_t *buf)
202240
{
241+
if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
242+
{
243+
return MLD_NATIVE_FUNC_FALLBACK;
244+
}
203245
mld_polyz_unpack_19_aarch64_asm(r, buf, mld_polyz_unpack_19_indices);
204246
return MLD_NATIVE_FUNC_SUCCESS;
205247
}
@@ -213,6 +255,10 @@ MLD_MUST_CHECK_RETURN_VALUE
213255
static MLD_INLINE int mld_poly_pointwise_montgomery_native(
214256
int32_t a[MLDSA_N], const int32_t b[MLDSA_N])
215257
{
258+
if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
259+
{
260+
return MLD_NATIVE_FUNC_FALLBACK;
261+
}
216262
mld_poly_pointwise_montgomery_aarch64_asm(a, b);
217263
return MLD_NATIVE_FUNC_SUCCESS;
218264
}
@@ -225,6 +271,10 @@ static MLD_INLINE int mld_polyvecl_pointwise_acc_montgomery_l4_native(
225271
int32_t w[MLDSA_N], const int32_t u[4][MLDSA_N],
226272
const int32_t v[4][MLDSA_N])
227273
{
274+
if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
275+
{
276+
return MLD_NATIVE_FUNC_FALLBACK;
277+
}
228278
mld_polyvecl_pointwise_acc_montgomery_l4_aarch64_asm(w, u, v);
229279
return MLD_NATIVE_FUNC_SUCCESS;
230280
}
@@ -236,6 +286,10 @@ static MLD_INLINE int mld_polyvecl_pointwise_acc_montgomery_l5_native(
236286
int32_t w[MLDSA_N], const int32_t u[5][MLDSA_N],
237287
const int32_t v[5][MLDSA_N])
238288
{
289+
if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
290+
{
291+
return MLD_NATIVE_FUNC_FALLBACK;
292+
}
239293
mld_polyvecl_pointwise_acc_montgomery_l5_aarch64_asm(w, u, v);
240294
return MLD_NATIVE_FUNC_SUCCESS;
241295
}
@@ -247,6 +301,10 @@ static MLD_INLINE int mld_polyvecl_pointwise_acc_montgomery_l7_native(
247301
int32_t w[MLDSA_N], const int32_t u[7][MLDSA_N],
248302
const int32_t v[7][MLDSA_N])
249303
{
304+
if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
305+
{
306+
return MLD_NATIVE_FUNC_FALLBACK;
307+
}
250308
mld_polyvecl_pointwise_acc_montgomery_l7_aarch64_asm(w, u, v);
251309
return MLD_NATIVE_FUNC_SUCCESS;
252310
}

‎dev/aarch64_clean/src/intt_aarch64_asm.S‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
ABI:
3535
Architecture: aarch64
3636
CallingConvention: AAPCS64
37+
Features: [NEON]
3738
x0:
3839
type: buffer
3940
size_bytes: 1024

‎dev/aarch64_clean/src/mld_polyvecl_pointwise_acc_montgomery_l4_aarch64_asm.S‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
ABI:
1010
Architecture: aarch64
1111
CallingConvention: AAPCS64
12+
Features: [NEON]
1213
x0:
1314
type: buffer
1415
size_bytes: 1024

‎dev/aarch64_clean/src/mld_polyvecl_pointwise_acc_montgomery_l5_aarch64_asm.S‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
ABI:
1010
Architecture: aarch64
1111
CallingConvention: AAPCS64
12+
Features: [NEON]
1213
x0:
1314
type: buffer
1415
size_bytes: 1024

‎dev/aarch64_clean/src/mld_polyvecl_pointwise_acc_montgomery_l7_aarch64_asm.S‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
ABI:
1010
Architecture: aarch64
1111
CallingConvention: AAPCS64
12+
Features: [NEON]
1213
x0:
1314
type: buffer
1415
size_bytes: 1024

‎dev/aarch64_clean/src/ntt_aarch64_asm.S‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
ABI:
3535
Architecture: aarch64
3636
CallingConvention: AAPCS64
37+
Features: [NEON]
3738
x0:
3839
type: buffer
3940
size_bytes: 1024

‎dev/aarch64_clean/src/pointwise_montgomery_aarch64_asm.S‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
ABI:
1010
Architecture: aarch64
1111
CallingConvention: AAPCS64
12+
Features: [NEON]
1213
x0:
1314
type: buffer
1415
size_bytes: 1024

‎dev/aarch64_clean/src/poly_caddq_aarch64_asm.S‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
ABI:
1010
Architecture: aarch64
1111
CallingConvention: AAPCS64
12+
Features: [NEON]
1213
x0:
1314
type: buffer
1415
size_bytes: 1024

‎dev/aarch64_clean/src/poly_chknorm_aarch64_asm.S‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
ABI:
1010
Architecture: aarch64
1111
CallingConvention: AAPCS64
12+
Features: [NEON]
1213
x0:
1314
type: buffer
1415
size_bytes: 1024

‎dev/aarch64_clean/src/poly_decompose_32_aarch64_asm.S‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
ABI:
1010
Architecture: aarch64
1111
CallingConvention: AAPCS64
12+
Features: [NEON]
1213
x0:
1314
type: buffer
1415
size_bytes: 1024

0 commit comments

Comments
 (0)