@@ -907,7 +907,7 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits
907907 /* Pick the widest available implementation at runtime. Callers must
908908 * already be inside a VECTOR_REGISTERS_PUSH / SAVE_VECTOR_REGISTERS
909909 * region (all bulk AES-NI call sites are). */
910- static WC_INLINE void AesEcbEncryptBlocks(const unsigned char* in,
910+ static WC_MAYBE_UNUSED WC_INLINE void AesEcbEncryptBlocks(const unsigned char* in,
911911 unsigned char* out, word32 sz, const unsigned char* key, int nr)
912912 {
913913 #ifdef HAVE_INTEL_AVX512
@@ -936,7 +936,7 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits
936936 }
937937
938938 #ifdef HAVE_AES_DECRYPT
939- static WC_INLINE void AesEcbDecryptBlocks(const unsigned char* in,
939+ static WC_MAYBE_UNUSED WC_INLINE void AesEcbDecryptBlocks(const unsigned char* in,
940940 unsigned char* out, word32 sz, const unsigned char* key, int nr)
941941 {
942942 #ifdef HAVE_INTEL_AVX512
@@ -966,7 +966,7 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits
966966 #endif
967967
968968 #ifdef HAVE_AES_CBC
969- static WC_INLINE void AesCbcEncryptBlocks(const unsigned char* in,
969+ static WC_MAYBE_UNUSED WC_INLINE void AesCbcEncryptBlocks(const unsigned char* in,
970970 unsigned char* out, unsigned char* iv, word32 sz,
971971 const unsigned char* key, int nr)
972972 {
@@ -997,7 +997,7 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits
997997 #endif /* HAVE_AES_CBC */
998998
999999 #ifdef HAVE_AES_DECRYPT
1000- static WC_INLINE void AesCbcDecryptBlocks(const unsigned char* in,
1000+ static WC_MAYBE_UNUSED WC_INLINE void AesCbcDecryptBlocks(const unsigned char* in,
10011001 unsigned char* out, unsigned char* iv, word32 sz,
10021002 const unsigned char* key, int nr)
10031003 {
@@ -1027,7 +1027,7 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits
10271027 }
10281028 #endif /* HAVE_AES_DECRYPT */
10291029
1030- static WC_INLINE void AesCtrEncryptBlocks(const unsigned char* in,
1030+ static WC_MAYBE_UNUSED WC_INLINE void AesCtrEncryptBlocks(const unsigned char* in,
10311031 unsigned char* out, word32 sz, const unsigned char* key, int nr,
10321032 unsigned char* ctr)
10331033 {
@@ -5928,6 +5928,14 @@ int wc_AesSetIV(Aes* aes, const byte* iv)
59285928 } \
59295929 WC_DO_NOTHING
59305930
5931+ #define VECTOR_REGISTERS_PUSH2(fail_clause) { \
5932+ int orig_use_aesni = aes->use_aesni; \
5933+ if (aes->use_aesni && (SAVE_VECTOR_REGISTERS2() != 0)) { \
5934+ aes->use_aesni = 0; \
5935+ } \
5936+ WC_DO_NOTHING
5937+
5938+
59315939#define VECTOR_REGISTERS_POP \
59325940 if (aes->use_aesni) \
59335941 RESTORE_VECTOR_REGISTERS(); \
@@ -5941,14 +5949,24 @@ int wc_AesSetIV(Aes* aes, const byte* iv)
59415949#define VECTOR_REGISTERS_PUSH { \
59425950 WC_DO_NOTHING
59435951
5952+ #define VECTOR_REGISTERS_PUSH2(fail_clause) { \
5953+ WC_DO_NOTHING
5954+
59445955#define VECTOR_REGISTERS_POP \
59455956 } \
59465957 WC_DO_NOTHING
59475958
59485959#else
59495960
5950- #define VECTOR_REGISTERS_PUSH { \
5961+ #define VECTOR_REGISTERS_PUSH { \
5962+ if (aes->use_aesni && ((ret = SAVE_VECTOR_REGISTERS2()) != 0)) { \
5963+ return ret; \
5964+ } \
5965+ WC_DO_NOTHING
5966+
5967+ #define VECTOR_REGISTERS_PUSH2(fail_clause) { \
59515968 if (aes->use_aesni && ((ret = SAVE_VECTOR_REGISTERS2()) != 0)) { \
5969+ { fail_clause } \
59525970 return ret; \
59535971 } \
59545972 WC_DO_NOTHING
@@ -5965,6 +5983,7 @@ int wc_AesSetIV(Aes* aes, const byte* iv)
59655983#else /* !WOLFSSL_AESNI */
59665984
59675985#define VECTOR_REGISTERS_PUSH WC_DO_NOTHING
5986+ #define VECTOR_REGISTERS_PUSH2(fail_clause) WC_DO_NOTHING
59685987#define VECTOR_REGISTERS_POP WC_DO_NOTHING
59695988
59705989#endif /* !WOLFSSL_AESNI */
@@ -15511,6 +15530,14 @@ static WARN_UNUSED_RESULT int AesCfbDecrypt_C(Aes* aes, byte* out,
1551115530#ifdef WC_AES_HAVE_PREFETCH_ARG
1551215531 int did_prefetches = 0;
1551315532#endif
15533+ #ifndef WC_AES_CFB_DEC_BUF_BLOCKS
15534+ #define WC_AES_CFB_DEC_BUF_BLOCKS 32
15535+ #elif WC_AES_CFB_DEC_BUF_BLOCKS < 2
15536+ #error Invalid WC_AES_CFB_DEC_BUF_BLOCKS
15537+ #endif
15538+ #ifdef WOLFSSL_SMALL_STACK
15539+ byte *tmp = NULL;
15540+ #endif
1551415541
1551515542 (void)mode;
1551615543
@@ -15534,18 +15561,32 @@ static WARN_UNUSED_RESULT int AesCfbDecrypt_C(Aes* aes, byte* out,
1553415561 sz -= processed;
1553515562 }
1553615563
15564+ #if defined(WOLFSSL_SMALL_STACK) && defined(HAVE_AES_ECB) && \
15565+ !defined(WOLFSSL_PIC32MZ_CRYPT) && \
15566+ (defined(USE_INTEL_SPEEDUP) || defined(WOLFSSL_ARMASM))
15567+ /* Only suffer the heap overhead if sz is enough to warrant it.
15568+ *
15569+ * Allocate the working buffer before suspending interrupts, so that we can
15570+ * allocate with regular GFP_KERNEL.
15571+ */
15572+ if (sz >= WC_AES_CFB_DEC_BUF_BLOCKS * WC_AES_BLOCK_SIZE)
15573+ tmp = (byte *)XMALLOC(WC_AES_CFB_DEC_BUF_BLOCKS * WC_AES_BLOCK_SIZE, NULL, DYNAMIC_TYPE_AES);
15574+
15575+ VECTOR_REGISTERS_PUSH2(XFREE(tmp, NULL, DYNAMIC_TYPE_AES););
15576+ #else
1553715577 VECTOR_REGISTERS_PUSH;
15578+ #endif
1553815579
15539- #if !defined(WOLFSSL_SMALL_STACK) && defined(HAVE_AES_ECB) && \
15580+ #if defined(HAVE_AES_ECB) && \
1554015581 !defined(WOLFSSL_PIC32MZ_CRYPT) && \
1554115582 (defined(USE_INTEL_SPEEDUP) || defined(WOLFSSL_ARMASM))
15583+ #ifdef WOLFSSL_SMALL_STACK
15584+ if (tmp != NULL)
15585+ #endif
1554215586 {
15543- #ifndef WC_AES_CFB_DEC_BUF_BLOCKS
15544- #define WC_AES_CFB_DEC_BUF_BLOCKS 32
15545- #elif WC_AES_CFB_DEC_BUF_BLOCKS < 2
15546- #error Invalid WC_AES_CFB_DEC_BUF_BLOCKS
15547- #endif
15587+ #ifndef WOLFSSL_SMALL_STACK
1554815588 ALIGN16 byte tmp[WC_AES_CFB_DEC_BUF_BLOCKS * WC_AES_BLOCK_SIZE];
15589+ #endif
1554915590 while (sz >= 2 * WC_AES_BLOCK_SIZE) {
1555015591 word32 blocks = sz / WC_AES_BLOCK_SIZE;
1555115592 word32 nbytes;
@@ -15592,6 +15633,11 @@ static WARN_UNUSED_RESULT int AesCfbDecrypt_C(Aes* aes, byte* out,
1559215633
1559315634 VECTOR_REGISTERS_POP;
1559415635
15636+ #ifdef WOLFSSL_SMALL_STACK
15637+ /* Free tmp after restoring interrupts, so that GFP_KERNEL is usable. */
15638+ XFREE(tmp, NULL, DYNAMIC_TYPE_AES);
15639+ #endif
15640+
1559515641 return ret;
1559615642}
1559715643#endif /* HAVE_AES_DECRYPT */
0 commit comments