|
| 1 | +/* |
| 2 | + * Copyright (c) The mldsa-native project authors |
| 3 | + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT |
| 4 | + */ |
| 5 | + |
| 6 | +/* References |
| 7 | + * ========== |
| 8 | + * |
| 9 | + * - [REF_AVX2] |
| 10 | + * CRYSTALS-Dilithium optimized AVX2 implementation |
| 11 | + * Bai, Ducas, Kiltz, Lepoint, Lyubashevsky, Schwabe, Seiler, Stehlé |
| 12 | + * https://github.com/pq-crystals/dilithium/tree/master/avx2 |
| 13 | + */ |
| 14 | + |
| 15 | +/* |
| 16 | + * This file is derived from the public domain |
| 17 | + * AVX2 Dilithium implementation @[REF_AVX2]. |
| 18 | + */ |
| 19 | + |
| 20 | +#include "../../../common.h" |
| 21 | + |
| 22 | +#if defined(MLD_ARITH_BACKEND_X86_64_DEFAULT) && \ |
| 23 | + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) && \ |
| 24 | + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \ |
| 25 | + MLD_CONFIG_PARAMETER_SET == 44) |
| 26 | + |
| 27 | +#include <immintrin.h> |
| 28 | +#include "arith_native_x86_64.h" |
| 29 | + |
| 30 | +/* Pack w1 polynomial (coefficients in [0,43]) for GAMMA2 = (Q-1)/88. |
| 31 | + * 6-bit encoding, 4 coefficients per 3 bytes; 32 coefficients per iteration. */ |
| 32 | +void mld_polyw1_pack_88_avx2(uint8_t *r, const int32_t *a) |
| 33 | +{ |
| 34 | + unsigned int i; |
| 35 | + const __m256i shift1 = _mm256_set1_epi16((64 << 8) + 1); |
| 36 | + const __m256i shift2 = _mm256_set1_epi32(((1 << 12) << 16) + 1); |
| 37 | + const __m256i shufdidx1 = _mm256_set_epi32(7, 3, 6, 2, 5, 1, 4, 0); |
| 38 | + const __m256i shufdidx2 = _mm256_set_epi32(-1, -1, 6, 5, 4, 2, 1, 0); |
| 39 | + const __m256i shufbidx = |
| 40 | + _mm256_set_epi8(-1, -1, -1, -1, 14, 13, 12, 10, 9, 8, 6, 5, 4, 2, 1, 0, |
| 41 | + -1, -1, -1, -1, 14, 13, 12, 10, 9, 8, 6, 5, 4, 2, 1, 0); |
| 42 | + |
| 43 | + for (i = 0; i < MLDSA_N / 32; i++) |
| 44 | + { |
| 45 | + __m256i f0 = _mm256_load_si256((__m256i *)&a[32 * i + 0]); |
| 46 | + __m256i f1 = _mm256_load_si256((__m256i *)&a[32 * i + 8]); |
| 47 | + __m256i f2 = _mm256_load_si256((__m256i *)&a[32 * i + 16]); |
| 48 | + __m256i f3 = _mm256_load_si256((__m256i *)&a[32 * i + 24]); |
| 49 | + f0 = _mm256_packus_epi32(f0, f1); |
| 50 | + f1 = _mm256_packus_epi32(f2, f3); |
| 51 | + f0 = _mm256_packus_epi16(f0, f1); |
| 52 | + f0 = _mm256_maddubs_epi16(f0, shift1); |
| 53 | + f0 = _mm256_madd_epi16(f0, shift2); |
| 54 | + f0 = _mm256_permutevar8x32_epi32(f0, shufdidx1); |
| 55 | + f0 = _mm256_shuffle_epi8(f0, shufbidx); |
| 56 | + f0 = _mm256_permutevar8x32_epi32(f0, shufdidx2); |
| 57 | + |
| 58 | + /* Each iteration produces 24 valid bytes in the low 192 bits. |
| 59 | + * Store as 128-bit + 64-bit to avoid writing past the output buffer. */ |
| 60 | + { |
| 61 | + __m128i lo = _mm256_castsi256_si128(f0); |
| 62 | + __m128i hi = _mm256_extracti128_si256(f0, 1); |
| 63 | + _mm_storeu_si128((__m128i *)&r[24 * i], lo); |
| 64 | + _mm_storel_epi64((__m128i *)&r[24 * i + 16], hi); |
| 65 | + } |
| 66 | + } |
| 67 | +} |
| 68 | + |
| 69 | +#else /* MLD_ARITH_BACKEND_X86_64_DEFAULT && !MLD_CONFIG_MULTILEVEL_NO_SHARED \ |
| 70 | + && (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == \ |
| 71 | + 44) */ |
| 72 | + |
| 73 | +MLD_EMPTY_CU(avx2_polyw1_pack_88) |
| 74 | + |
| 75 | +#endif /* !(MLD_ARITH_BACKEND_X86_64_DEFAULT && \ |
| 76 | + !MLD_CONFIG_MULTILEVEL_NO_SHARED && \ |
| 77 | + (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == \ |
| 78 | + 44)) */ |
0 commit comments