|
| 1 | +/* |
| 2 | + * Copyright (c) The mldsa-native project authors |
| 3 | + * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT |
| 4 | + */ |
| 5 | + |
| 6 | +/* References |
| 7 | + * ========== |
| 8 | + * |
| 9 | + * - [REF_AVX2] |
| 10 | + * CRYSTALS-Dilithium optimized AVX2 implementation |
| 11 | + * Bai, Ducas, Kiltz, Lepoint, Lyubashevsky, Schwabe, Seiler, Stehlé |
| 12 | + * https://github.com/pq-crystals/dilithium/tree/master/avx2 |
| 13 | + */ |
| 14 | + |
| 15 | +/* |
| 16 | + * This file is derived from the public domain |
| 17 | + * AVX2 Dilithium implementation @[REF_AVX2]. |
| 18 | + */ |
| 19 | + |
| 20 | + |
| 21 | +/*yaml |
| 22 | + Name: poly_use_hint_32_avx2_asm |
| 23 | + Description: x86_64 AVX2 hint application (alpha = (Q-1)/32). |
| 24 | + Use the hint polynomial h to correct the high bits of the polynomial a, |
| 25 | + in place. Variant for parameter sets ML-DSA-65 and ML-DSA-87 |
| 26 | + (GAMMA2 = (Q-1)/32). |
| 27 | + Signature: void mld_poly_use_hint_32_avx2_asm(int32_t *a, const int32_t *h) |
| 28 | + ABI: |
| 29 | + Architecture: x86_64 |
| 30 | + CallingConvention: SysV |
| 31 | + Features: [AVX2] |
| 32 | + rdi: |
| 33 | + type: buffer |
| 34 | + size_bytes: 1024 |
| 35 | + permissions: read/write |
| 36 | + c_parameter: int32_t *a |
| 37 | + description: Input/output polynomial (256 x int32_t) |
| 38 | + rsi: |
| 39 | + type: buffer |
| 40 | + size_bytes: 1024 |
| 41 | + permissions: read-only |
| 42 | + c_parameter: const int32_t *h |
| 43 | + description: Hint polynomial (256 x int32_t) |
| 44 | +*/ |
| 45 | + |
| 46 | +#include "../../../common.h" |
| 47 | + |
| 48 | +#if defined(MLD_ARITH_BACKEND_X86_64_DEFAULT) && \ |
| 49 | + !defined(MLD_CONFIG_NO_VERIFY_API) && \ |
| 50 | + !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED) && \ |
| 51 | + (defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \ |
| 52 | + (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87)) |
| 53 | + |
| 54 | +/* simpasm: header-end */ |
| 55 | + |
| 56 | +/* Reference: |
| 57 | + * - @[REF_AVX2] calls poly_decompose to compute all a1, a0 before the loop. |
| 58 | + * - Our implementation of decompose() is slightly different from that in |
| 59 | + * @[REF_AVX2]. See poly_decompose_32_avx2.c for more information. |
| 60 | + */ |
| 61 | + |
| 62 | +// a aliased with a0 |
| 63 | +.macro decompose32_avx2 a1, a, temp1, temp2, temp3 |
| 64 | +// Compute a1 = round-(a / 523776), where 523776 = 2*GAMMA2 = 128*B |
| 65 | +// with B = 4092, and round-() denotes "round half down". This is |
| 66 | +// exact for 0 <= a < Q. It is computed as round-(ceil(a / 128) / B) |
| 67 | +// ≈ round(ceil(a / 128) * 1025 / 2^22), with 1025 = floor(2^22 / B): |
| 68 | +// temp1 = ceil(a / 128) = floor((a + 127) / 2^7) (vpaddd 127; vpsrld 7) |
| 69 | +// temp1 = floor(temp1 * 1025 / 2^16) (vpmulhuw by 1025) |
| 70 | +// temp1 = round(temp1 / 2^6) via mulhrs(temp1, 2^9) (vpmulhrsw) |
| 71 | +// See mld_poly_decompose_32_avx2() for the detailed argument. |
| 72 | +vpaddd \a, %ymm5, \temp1 |
| 73 | +vpsrld $7, \temp1, \temp1 |
| 74 | +vpmulhuw %ymm8, \temp1, \temp1 |
| 75 | +vpmulhrsw %ymm7, \temp1, \temp1 |
| 76 | + |
| 77 | +// If a1 = 16, i.e. a > 31*GAMMA2, proceed as if a' = a - Q was |
| 78 | +// given instead. (For a = 31*GAMMA2 + 1 thus a' = -GAMMA2, we |
| 79 | +// still round it to 0 like other "wrapped around" cases.) |
| 80 | + |
| 81 | +// Check for wrap-around |
| 82 | +vpcmpgtd %ymm4, \a, \temp2 |
| 83 | +vpandn \temp1, \temp2, \a1 |
| 84 | + |
| 85 | +// Compute remainder a0 = a - a1 * 2 * GAMMA2 = a - a1 * 523776 |
| 86 | +vpslld $10, \temp1, \temp3 |
| 87 | +vpsubd \temp1, \temp3, \temp1 |
| 88 | +vpslld $9, \temp1, \temp1 |
| 89 | +vpsubd \temp1, \a, \a |
| 90 | + |
| 91 | +// If wrap-around is required, adjust a0 by -1 |
| 92 | +vpaddd \temp2, \a, \a |
| 93 | +.endm |
| 94 | + |
| 95 | +/* We follow the specification and the reference C implementation and check |
| 96 | + * a0 > 0. The @[REF_AVX2] implementation used to check a0 >= 0 instead, but |
| 97 | + * this was corrected to match the specification in pq-crystals/dilithium |
| 98 | + * commit bba9534a3c5cbef66481cd6518d786e23ae601b5. |
| 99 | + */ |
| 100 | + |
| 101 | +// a aliased with delta |
| 102 | +.macro use_hint32_avx2 b, a, h, a1, temp1, temp2, temp3 |
| 103 | +decompose32_avx2 \a1, \a, \temp1, \temp2, \temp3 |
| 104 | + |
| 105 | +// delta = (a0 <= 0) ? -1 : 1 |
| 106 | +vpcmpgtd %ymm6, \a, \a |
| 107 | +vpandn \h, \a, \a |
| 108 | +vpslld $1, \a, \a |
| 109 | +vpsubd \a, \h, \h |
| 110 | + |
| 111 | +// b = (b + delta * h) % 16 |
| 112 | +vpaddd \a1, \h, \b |
| 113 | +vpand %ymm3, \b, \b |
| 114 | +.endm |
| 115 | + |
| 116 | +.text |
| 117 | +.balign 16 |
| 118 | +.global MLD_ASM_NAMESPACE(poly_use_hint_32_avx2_asm) |
| 119 | +MLD_ASM_FN_SYMBOL(poly_use_hint_32_avx2_asm) |
| 120 | + |
| 121 | +// Initialize constants |
| 122 | +movl $127, %ecx |
| 123 | + |
| 124 | +/* check-magic: 1025 == floor(2^22 / 4092) */ |
| 125 | +movl $1025, %r8d |
| 126 | +vmovd %r8d, %xmm8 |
| 127 | +vpbroadcastd %xmm8, %ymm8 |
| 128 | + |
| 129 | +xorl %eax, %eax |
| 130 | +vpxor %xmm6, %xmm6, %xmm6 |
| 131 | +vmovd %ecx, %xmm5 |
| 132 | + |
| 133 | +/* 31 * ((Q-1) / 32) == 31 * GAMMA2, wrap-around threshold */ |
| 134 | +movl $8118528, %ecx |
| 135 | + |
| 136 | +/* round(x * 2^9 / 2^15) => round(x / 2^6), for f1 = round(f1'' / 2^6) */ |
| 137 | +movl $512, %r9d |
| 138 | +vmovd %r9d, %xmm7 |
| 139 | +vpbroadcastd %xmm7, %ymm7 |
| 140 | + |
| 141 | +vmovd %ecx, %xmm4 |
| 142 | +movl $15, %ecx |
| 143 | +vpbroadcastd %xmm5, %ymm5 |
| 144 | +vmovd %ecx, %xmm3 |
| 145 | +vpbroadcastd %xmm4, %ymm4 |
| 146 | +vpbroadcastd %xmm3, %ymm3 |
| 147 | + |
| 148 | + |
| 149 | +poly_use_hint_32_avx2_asm_loop: |
| 150 | +vmovdqa (%rdi), %ymm0 |
| 151 | +vmovdqa (%rsi), %ymm2 |
| 152 | + |
| 153 | +use_hint32_avx2 %ymm2, %ymm0, %ymm2, %ymm9, %ymm1, %ymm11, %ymm10 |
| 154 | + |
| 155 | +vmovdqa %ymm2, (%rdi) |
| 156 | +addq $32, %rdi |
| 157 | +addq $32, %rsi |
| 158 | +addq $32, %rax |
| 159 | +cmpq $1024, %rax |
| 160 | +jne poly_use_hint_32_avx2_asm_loop |
| 161 | +ret |
| 162 | + |
| 163 | +/* simpasm: footer-start */ |
| 164 | + |
| 165 | +#endif /* MLD_ARITH_BACKEND_X86_64_DEFAULT && !MLD_CONFIG_NO_VERIFY_API && \ |
| 166 | + !MLD_CONFIG_MULTILEVEL_NO_SHARED && \ |
| 167 | + (MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \ |
| 168 | + || MLD_CONFIG_PARAMETER_SET == 87) */ |
0 commit comments