diff --git a/dev/fips202/armv81m/src/state_xor_bytes_x4_mve.S b/dev/fips202/armv81m/src/state_xor_bytes_x4_mve.S index ef8824349e..93af51d9a3 100644 --- a/dev/fips202/armv81m/src/state_xor_bytes_x4_mve.S +++ b/dev/fips202/armv81m/src/state_xor_bytes_x4_mve.S @@ -51,8 +51,9 @@ // --------------------------------------------------------------------------- // Prologue -- if offset is not 8-byte aligned, absorb // min(length, 8-(offset%8)) bytes via predicated byte loads. -// Main -- process full 8-byte groups via word-level gather loads, -// bit-interleave, then VEOR into even/odd state halves. +// Main -- process full 8-byte groups via vldrw.u32 gather loads when +// 32-bit aligned, otherwise byte loads; bit-interleave, then +// VEOR into even/odd state halves. // Tail -- absorb remaining <8 bytes via predicated byte loads. #include "../../../../common.h" @@ -303,13 +304,13 @@ MLK_ASM_FN_SYMBOL(keccak_f1600_x4_state_xor_bytes_asm) lsl mask, mask, off vmsr p0, mask - // Predicated byte loads of nB bytes per instance; post-increment of 4 - // positions qP for the main loop's [qP, #4]! loads. + // Predicated byte loads of nB bytes per instance; post-increment to the + // next lane start. vpstttt - vldrbt.u8 qd0, [dp0], #4 - vldrbt.u8 qd1, [dp1], #4 - vldrbt.u8 qd2, [dp2], #4 - vldrbt.u8 qd3, [dp3], #4 + vldrbt.u8 qd0, [dp0], #8 + vldrbt.u8 qd1, [dp1], #8 + vldrbt.u8 qd2, [dp2], #8 + vldrbt.u8 qd3, [dp3], #8 // Transpose from per-instance layout to per-lane layout transpose_streams_to_lanes @@ -320,12 +321,14 @@ MLK_ASM_FN_SYMBOL(keccak_f1600_x4_state_xor_bytes_asm) // XOR into state (post-increments rSE/rSO by 16) xor_lane_and_store_postinc q0, q1, qS0, qS1, rSE, rSO - // Build qP = {d0,d1,d2,d3} as u32 lanes for gather loads - vmov qP[2], qP[0], dp0, dp2 - vmov qP[3], qP[1], dp1, dp3 - b keccak_f1600_x4_state_xor_bytes_asm_main_body - keccak_f1600_x4_state_xor_bytes_asm_pre_main: + // dpN points at the next lane start; vldrw requires 32-bit alignment. + orr tmp, dp0, dp1 + orr tmp, tmp, dp2 + orr tmp, tmp, dp3 + tst tmp, #3 + bne keccak_f1600_x4_state_xor_bytes_asm_main_body_byte + vmov qP[2], qP[0], dp0, dp2 vmov qP[3], qP[1], dp1, dp3 mov tmp, #4 @@ -333,7 +336,7 @@ keccak_f1600_x4_state_xor_bytes_asm_pre_main: keccak_f1600_x4_state_xor_bytes_asm_main_body: // ----------------------------------------------------------------------- - // MAIN: process full 8-byte lanes + // MAIN: process full 8-byte lanes with aligned gather-word loads // ----------------------------------------------------------------------- lsr lr, length, #3 wls lr, lr, keccak_f1600_x4_state_xor_bytes_asm_main_loop_end @@ -351,9 +354,6 @@ keccak_f1600_x4_state_xor_bytes_asm_main_loop_start: le lr, keccak_f1600_x4_state_xor_bytes_asm_main_loop_start keccak_f1600_x4_state_xor_bytes_asm_main_loop_end: - // ----------------------------------------------------------------------- - // TAIL: absorb <8 remaining bytes at lane offset 0 - // ----------------------------------------------------------------------- ands length, length, #7 beq keccak_f1600_x4_state_xor_bytes_asm_exit @@ -363,12 +363,25 @@ keccak_f1600_x4_state_xor_bytes_asm_main_loop_end: vmov dp0, dp2, qP[2], qP[0] vmov dp1, dp3, qP[3], qP[1] - vctp.8 length +keccak_f1600_x4_state_xor_bytes_asm_main_body_byte: + // ----------------------------------------------------------------------- + // MAIN/TAIL: process remaining lanes using predicated byte loads + // ----------------------------------------------------------------------- + cmp length, #0 + beq keccak_f1600_x4_state_xor_bytes_asm_exit +keccak_f1600_x4_state_xor_bytes_asm_main_loop_byte_start: + // Predicate with min(length, 8): a plain MVE byte vector load would + // over-read the Keccak lane, and the last iteration may be a partial tail. + mov nB, #8 + cmp length, #8 + it lo + movlo nB, length + vctp.8 nB vpstttt - vldrbt.u8 qd0, [dp0] - vldrbt.u8 qd1, [dp1] - vldrbt.u8 qd2, [dp2] - vldrbt.u8 qd3, [dp3] + vldrbt.u8 qd0, [dp0], #8 + vldrbt.u8 qd1, [dp1], #8 + vldrbt.u8 qd2, [dp2], #8 + vldrbt.u8 qd3, [dp3], #8 // Transpose from per-instance layout to per-lane layout transpose_streams_to_lanes @@ -377,7 +390,10 @@ keccak_f1600_x4_state_xor_bytes_asm_main_loop_end: to_bit_interleaving_4x q0, q1, q2, q3, q4 // XOR into state (post-increments rSE/rSO by 16) - xor_lane_and_store_postinc qd0, qd1, qS0, qS1, rSE, rSO + xor_lane_and_store_postinc q0, q1, qS0, qS1, rSE, rSO + + subs length, length, nB + bne keccak_f1600_x4_state_xor_bytes_asm_main_loop_byte_start keccak_f1600_x4_state_xor_bytes_asm_exit: vpop {d8-d15} diff --git a/mlkem/src/fips202/native/armv81m/src/state_xor_bytes_x4_mve.S b/mlkem/src/fips202/native/armv81m/src/state_xor_bytes_x4_mve.S index c14ee76401..4e8cd3a180 100644 --- a/mlkem/src/fips202/native/armv81m/src/state_xor_bytes_x4_mve.S +++ b/mlkem/src/fips202/native/armv81m/src/state_xor_bytes_x4_mve.S @@ -51,8 +51,9 @@ // --------------------------------------------------------------------------- // Prologue -- if offset is not 8-byte aligned, absorb // min(length, 8-(offset%8)) bytes via predicated byte loads. -// Main -- process full 8-byte groups via word-level gather loads, -// bit-interleave, then VEOR into even/odd state halves. +// Main -- process full 8-byte groups via vldrw.u32 gather loads when +// 32-bit aligned, otherwise byte loads; bit-interleave, then +// VEOR into even/odd state halves. // Tail -- absorb remaining <8 bytes via predicated byte loads. #include "../../../../common.h" @@ -98,13 +99,13 @@ MLK_ASM_FN_SYMBOL(keccak_f1600_x4_state_xor_bytes_asm) ldr.w r10, [sp, #0x6c] ldr r6, [sp, #0x70] cmp r6, #0x0 - beq.w Lkeccak_f1600_x4_state_xor_bytes_asm_exit @ imm = #0x346 + beq.w Lkeccak_f1600_x4_state_xor_bytes_asm_exit @ imm = #0x368 and r5, r10, #0x7 bic r9, r10, #0x7 add.w r8, r0, r9, lsl #1 add.w r7, r8, #0x190 cmp r5, #0x0 - beq.w Lkeccak_f1600_x4_state_xor_bytes_asm_pre_main @ imm = #0x12c + beq.w Lkeccak_f1600_x4_state_xor_bytes_asm_pre_main @ imm = #0x122 subs r1, r1, r5 subs r2, r2, r5 subs r3, r3, r5 @@ -119,10 +120,10 @@ MLK_ASM_FN_SYMBOL(keccak_f1600_x4_state_xor_bytes_asm) lsl.w r11, r11, r5 vmsr p0, r11 vpstttt - vldrbt.u8 q0, [r1], #4 - vldrbt.u8 q1, [r2], #4 - vldrbt.u8 q2, [r3], #4 - vldrbt.u8 q3, [r4], #4 + vldrbt.u8 q0, [r1], #8 + vldrbt.u8 q1, [r2], #8 + vldrbt.u8 q2, [r3], #8 + vldrbt.u8 q3, [r4], #8 vmov.f64 d1, d4 vmov.f64 d3, d6 vrev64.32 q2, q0 @@ -181,11 +182,13 @@ MLK_ASM_FN_SYMBOL(keccak_f1600_x4_state_xor_bytes_asm) veor q5, q5, q1 vstrw.32 q4, [r8], #16 vstrw.32 q5, [r7], #16 - vmov q7[2], q7[0], r1, r3 - vmov q7[3], q7[1], r2, r4 - b Lkeccak_f1600_x4_state_xor_bytes_asm_main_body @ imm = #0xe Lkeccak_f1600_x4_state_xor_bytes_asm_pre_main: + orr.w r0, r1, r2 + orr.w r0, r0, r3 + orr.w r0, r0, r4 + tst.w r0, #0x3 + bne.w Lkeccak_f1600_x4_state_xor_bytes_asm_main_body_byte @ imm = #0x104 vmov q7[2], q7[0], r1, r3 vmov q7[3], q7[1], r2, r4 mov.w r0, #0x4 @@ -252,17 +255,27 @@ Lkeccak_f1600_x4_state_xor_bytes_asm_main_loop_start: Lkeccak_f1600_x4_state_xor_bytes_asm_main_loop_end: ands r6, r6, #0x7 - beq.w Lkeccak_f1600_x4_state_xor_bytes_asm_exit @ imm = #0x110 + beq.w Lkeccak_f1600_x4_state_xor_bytes_asm_exit @ imm = #0x128 mov.w r0, #0x4 vadd.i32 q7, q7, r0 vmov r1, r3, q7[2], q7[0] vmov r2, r4, q7[3], q7[1] - vctp.8 r6 + +Lkeccak_f1600_x4_state_xor_bytes_asm_main_body_byte: + cmp r6, #0x0 + beq.w Lkeccak_f1600_x4_state_xor_bytes_asm_exit @ imm = #0x112 + +Lkeccak_f1600_x4_state_xor_bytes_asm_main_loop_byte_start: + mov.w lr, #0x8 + cmp r6, #0x8 + it lo + movlo lr, r6 + vctp.8 lr vpstttt - vldrbt.u8 q0, [r1] - vldrbt.u8 q1, [r2] - vldrbt.u8 q2, [r3] - vldrbt.u8 q3, [r4] + vldrbt.u8 q0, [r1], #8 + vldrbt.u8 q1, [r2], #8 + vldrbt.u8 q2, [r3], #8 + vldrbt.u8 q3, [r4], #8 vmov.f64 d1, d4 vmov.f64 d3, d6 vrev64.32 q2, q0 @@ -321,6 +334,8 @@ Lkeccak_f1600_x4_state_xor_bytes_asm_main_loop_end: veor q5, q5, q1 vstrw.32 q4, [r8], #16 vstrw.32 q5, [r7], #16 + subs.w r6, r6, lr + bne.w Lkeccak_f1600_x4_state_xor_bytes_asm_main_loop_byte_start @ imm = #-0x112 Lkeccak_f1600_x4_state_xor_bytes_asm_exit: vpop {d8, d9, d10, d11, d12, d13, d14, d15} @@ -345,6 +360,7 @@ Lkeccak_f1600_x4_state_xor_bytes_asm_exit: .cfi_restore lr .cfi_adjust_cfa_offset -0x28 .cfi_endproc + nop MLK_ASM_FN_SIZE(keccak_f1600_x4_state_xor_bytes_asm) diff --git a/test/src/test_unit.c b/test/src/test_unit.c index 5a3ccb4f2b..6f2c7d80bc 100644 --- a/test/src/test_unit.c +++ b/test/src/test_unit.c @@ -836,10 +836,10 @@ static int test_keccakf1600x4_xor_permute_extract(void) MLK_ALLOC(output_x4_1, unsigned char, MAX_RATE, NULL); MLK_ALLOC(output_x4_2, unsigned char, MAX_RATE, NULL); MLK_ALLOC(output_x4_3, unsigned char, MAX_RATE, NULL); - MLK_ALLOC(input_0, unsigned char, MAX_RATE, NULL); - MLK_ALLOC(input_1, unsigned char, MAX_RATE, NULL); - MLK_ALLOC(input_2, unsigned char, MAX_RATE, NULL); - MLK_ALLOC(input_3, unsigned char, MAX_RATE, NULL); + MLK_ALLOC(input_0, unsigned char, MAX_RATE + 3, NULL); + MLK_ALLOC(input_1, unsigned char, MAX_RATE + 3, NULL); + MLK_ALLOC(input_2, unsigned char, MAX_RATE + 3, NULL); + MLK_ALLOC(input_3, unsigned char, MAX_RATE + 3, NULL); unsigned char *output_x4[MLK_KECCAK_WAY]; unsigned char *input[MLK_KECCAK_WAY]; @@ -875,6 +875,10 @@ static int test_keccakf1600x4_xor_permute_extract(void) ext_length = (uint8_t)(1 + (ext_length % (MAX_RATE - ext_offset))); /* Generate different random input for each lane */ + input[0] = input_0 + (((unsigned)i + 0u) & 3u); + input[1] = input_1 + (((unsigned)i + 1u) & 3u); + input[2] = input_2 + (((unsigned)i + 2u) & 3u); + input[3] = input_3 + (((unsigned)i + 3u) & 3u); for (j = 0; j < MLK_KECCAK_WAY; j++) { randombytes(input[j], xor_length); @@ -905,10 +909,10 @@ static int test_keccakf1600x4_xor_permute_extract(void) ret = 0; cleanup: - MLK_FREE(input_3, unsigned char, MAX_RATE, NULL); - MLK_FREE(input_2, unsigned char, MAX_RATE, NULL); - MLK_FREE(input_1, unsigned char, MAX_RATE, NULL); - MLK_FREE(input_0, unsigned char, MAX_RATE, NULL); + MLK_FREE(input_3, unsigned char, MAX_RATE + 3, NULL); + MLK_FREE(input_2, unsigned char, MAX_RATE + 3, NULL); + MLK_FREE(input_1, unsigned char, MAX_RATE + 3, NULL); + MLK_FREE(input_0, unsigned char, MAX_RATE + 3, NULL); MLK_FREE(output_x4_3, unsigned char, MAX_RATE, NULL); MLK_FREE(output_x4_2, unsigned char, MAX_RATE, NULL); MLK_FREE(output_x4_1, unsigned char, MAX_RATE, NULL);