Skip to content

Commit b560973

Browse files
dgarskedgarske
authored
Merge pull request #10825 from SparkiDev/aes_x64_perf_1
AES Intel x64: improve performance
2 parents 38a4143 + 3a6717e commit b560973

5 files changed

Lines changed: 16054 additions & 7608 deletions

File tree

‎wolfcrypt/src/aes.c‎

Lines changed: 56 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -839,9 +839,12 @@ block cipher mechanism that uses n-bit binary string parameter key with 128-bits
839839
#elif WC_VAES_MIN_BLOCKS < 1
840840
#error Invalid WC_VAES_MIN_BLOCKS
841841
#endif
842-
/* CFB/ECB: wide ECB setup (key broadcast) doesn't pay off below this. */
842+
/* ECB/CBC/CTR/XTS: the wide ladder handles 2+ blocks in parallel and
843+
* only caches round keys once it pays off (>= 32B), so the wide path
844+
* beats the single-block AES-NI fallback from 2 blocks up; a lone block
845+
* stays on AES-NI. (Measured +8..+58% at 2-6 blocks on Zen5.) */
843846
#ifndef WC_VAES_ECB_MIN_BLOCKS
844-
#define WC_VAES_ECB_MIN_BLOCKS WC_VAES_MIN_BLOCKS
847+
#define WC_VAES_ECB_MIN_BLOCKS 2
845848
#elif WC_VAES_ECB_MIN_BLOCKS < 1
846849
#error Invalid WC_VAES_ECB_MIN_BLOCKS
847850
#endif
@@ -15587,23 +15590,52 @@ static WARN_UNUSED_RESULT int AesCfbDecrypt_C(Aes* aes, byte* out,
1558715590
#ifndef WOLFSSL_SMALL_STACK
1558815591
ALIGN16 byte tmp[WC_AES_CFB_DEC_BUF_BLOCKS * WC_AES_BLOCK_SIZE];
1558915592
#endif
15590-
while (sz >= 2 * WC_AES_BLOCK_SIZE) {
15591-
word32 blocks = sz / WC_AES_BLOCK_SIZE;
15592-
word32 nbytes;
15593-
if (blocks > WC_AES_CFB_DEC_BUF_BLOCKS)
15594-
blocks = WC_AES_CFB_DEC_BUF_BLOCKS;
15595-
nbytes = blocks * WC_AES_BLOCK_SIZE;
15596-
XMEMCPY(tmp, aes->reg, WC_AES_BLOCK_SIZE);
15597-
XMEMCPY(tmp + WC_AES_BLOCK_SIZE, in, nbytes - WC_AES_BLOCK_SIZE);
15598-
XMEMCPY(aes->reg, in + nbytes - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
15599-
ret = wc_AesEcbEncrypt(aes, tmp, tmp, nbytes);
15600-
if (ret != 0) {
15601-
break;
15593+
if (sz >= 2 * WC_AES_BLOCK_SIZE) {
15594+
/* CFB-decrypt keystream block i is E(C_{i-1}): block 0 uses the
15595+
* feedback register, block i>=1 uses the previous cipher block. So
15596+
* ECB the ciphertext straight out of 'in' (no shift-copy) to get
15597+
* E(C_0..C_{n-1}), XOR block i with the (i-1)th ECB output, and
15598+
* carry E(C_{n-1}) as the next chunk's block-0 keystream - E(reg)
15599+
* is computed only once here. */
15600+
ALIGN16 byte ks[WC_AES_BLOCK_SIZE];
15601+
ret = AesEncrypt_preFetchOpt(aes, (byte*)aes->reg, ks,
15602+
&did_prefetches);
15603+
while ((ret == 0) && (sz >= 2 * WC_AES_BLOCK_SIZE)) {
15604+
word32 blocks = sz / WC_AES_BLOCK_SIZE;
15605+
word32 nbytes;
15606+
if (blocks > WC_AES_CFB_DEC_BUF_BLOCKS)
15607+
blocks = WC_AES_CFB_DEC_BUF_BLOCKS;
15608+
nbytes = blocks * WC_AES_BLOCK_SIZE;
15609+
/* tmp[i] = E(C_i), read directly from the input. Already inside
15610+
* VECTOR_REGISTERS_PUSH, so use the inner ECB (no nested
15611+
* save/restore or re-dispatch) where available. */
15612+
#if defined(WOLFSSL_AESNI) && defined(WOLFSSL_X86_64_BUILD)
15613+
if (aes->use_aesni) {
15614+
AesEcbEncryptBlocks(in, tmp, nbytes, (byte*)aes->key,
15615+
(int)aes->rounds);
15616+
}
15617+
else
15618+
#endif
15619+
{
15620+
ret = wc_AesEcbEncrypt(aes, tmp, in, nbytes);
15621+
if (ret != 0)
15622+
break;
15623+
}
15624+
/* Feedback for the tail = last cipher block; save it before the
15625+
* XOR can overwrite 'in' (in == out case). */
15626+
XMEMCPY((byte*)aes->reg, in + nbytes - WC_AES_BLOCK_SIZE,
15627+
WC_AES_BLOCK_SIZE);
15628+
/* P_0 = C_0 ^ E(feedback); P_i = C_i ^ E(C_{i-1}) =
15629+
* C_i ^ tmp[i-1]. */
15630+
xorbufout(out, in, ks, WC_AES_BLOCK_SIZE);
15631+
xorbufout(out + WC_AES_BLOCK_SIZE, in + WC_AES_BLOCK_SIZE, tmp,
15632+
nbytes - WC_AES_BLOCK_SIZE);
15633+
/* Carry E(last cipher block) as the next chunk's block-0 KS. */
15634+
XMEMCPY(ks, tmp + nbytes - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
15635+
out += nbytes;
15636+
in += nbytes;
15637+
sz -= nbytes;
1560215638
}
15603-
xorbufout(out, in, tmp, nbytes);
15604-
out += nbytes;
15605-
in += nbytes;
15606-
sz -= nbytes;
1560715639
}
1560815640
}
1560915641
#endif
@@ -16903,8 +16935,7 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
1690316935
if (aes->use_aesni) {
1690416936
SAVE_VECTOR_REGISTERS(return _svr_ret;);
1690516937
#if defined(HAVE_INTEL_AVX512)
16906-
if ((sz >= WC_VAES_ECB_MIN_BLOCKS * WC_AES_BLOCK_SIZE) &&
16907-
IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) {
16938+
if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) {
1690816939
AES_XTS_encrypt_avx512(in, out, sz, i,
1690916940
(const byte*)aes->key,
1691016941
(const byte*)xaes->tweak.key,
@@ -16914,8 +16945,7 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
1691416945
else
1691516946
#endif
1691616947
#if defined(HAVE_INTEL_VAES)
16917-
if ((sz >= WC_VAES_ECB_MIN_BLOCKS * WC_AES_BLOCK_SIZE) &&
16918-
IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) {
16948+
if (IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) {
1691916949
AES_XTS_encrypt_vaes(in, out, sz, i,
1692016950
(const byte*)aes->key,
1692116951
(const byte*)xaes->tweak.key,
@@ -17140,8 +17170,7 @@ static int AesXtsEncryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s
1714017170
if (aes->use_aesni) {
1714117171
SAVE_VECTOR_REGISTERS(return _svr_ret;);
1714217172
#if defined(HAVE_INTEL_AVX512)
17143-
if ((sz >= WC_VAES_ECB_MIN_BLOCKS * WC_AES_BLOCK_SIZE) &&
17144-
IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) {
17173+
if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) {
1714517174
AES_XTS_encrypt_update_avx512(in, out, sz,
1714617175
(const byte*)aes->key,
1714717176
stream->tweak_block,
@@ -17445,8 +17474,7 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
1744517474
if (aes->use_aesni) {
1744617475
SAVE_VECTOR_REGISTERS(return _svr_ret;);
1744717476
#if defined(HAVE_INTEL_AVX512)
17448-
if ((sz >= WC_VAES_ECB_MIN_BLOCKS * WC_AES_BLOCK_SIZE) &&
17449-
IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) {
17477+
if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) {
1745017478
AES_XTS_decrypt_avx512(in, out, sz, i,
1745117479
(const byte*)aes->key,
1745217480
(const byte*)xaes->tweak.key,
@@ -17676,8 +17704,7 @@ static int AesXtsDecryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s
1767617704
if (aes->use_aesni) {
1767717705
SAVE_VECTOR_REGISTERS(return _svr_ret;);
1767817706
#if defined(HAVE_INTEL_AVX512)
17679-
if ((sz >= WC_VAES_ECB_MIN_BLOCKS * WC_AES_BLOCK_SIZE) &&
17680-
IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) {
17707+
if (IS_INTEL_AVX512(intel_flags) && IS_INTEL_VAES(intel_flags)) {
1768117708
AES_XTS_decrypt_update_avx512(in, out, sz,
1768217709
(const byte*)aes->key,
1768317710
stream->tweak_block,
@@ -17687,8 +17714,7 @@ static int AesXtsDecryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s
1768717714
else
1768817715
#endif
1768917716
#if defined(HAVE_INTEL_VAES)
17690-
if ((sz >= WC_VAES_ECB_MIN_BLOCKS * WC_AES_BLOCK_SIZE) &&
17691-
IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) {
17717+
if (IS_INTEL_AVX2(intel_flags) && IS_INTEL_VAES(intel_flags)) {
1769217718
AES_XTS_decrypt_update_vaes(in, out, sz,
1769317719
(const byte*)aes->key,
1769417720
stream->tweak_block,

0 commit comments

Comments
 (0)