Skip to content

Fix Armv8.1-M x4 xor_bytes alignment fallback #1766

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bremoran wants to merge 1 commit into
base: main
Choose a base branch
from
+84 −48
Open

Fix Armv8.1-M x4 xor_bytes alignment fallback #1766

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 39 additions & 23 deletions dev/fips202/armv81m/src/state_xor_bytes_x4_mve.S
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,9 @@
// ---------------------------------------------------------------------------
// Prologue -- if offset is not 8-byte aligned, absorb
// min(length, 8-(offset%8)) bytes via predicated byte loads.
// Main -- process full 8-byte groups via word-level gather loads,
// bit-interleave, then VEOR into even/odd state halves.
// Main -- process full 8-byte groups via vldrw.u32 gather loads when
// 32-bit aligned, otherwise byte loads; bit-interleave, then
// VEOR into even/odd state halves.
// Tail -- absorb remaining <8 bytes via predicated byte loads.

#include "../../../../common.h"
Expand Down Expand Up @@ -303,13 +304,13 @@ MLK_ASM_FN_SYMBOL(keccak_f1600_x4_state_xor_bytes_asm)
lsl mask, mask, off
vmsr p0, mask

// Predicated byte loads of nB bytes per instance; post-increment of 4
// positions qP for the main loop's [qP, #4]! loads.
// Predicated byte loads of nB bytes per instance; post-increment to the
// next lane start.
vpstttt
vldrbt.u8 qd0, [dp0], #4
vldrbt.u8 qd1, [dp1], #4
vldrbt.u8 qd2, [dp2], #4
vldrbt.u8 qd3, [dp3], #4
vldrbt.u8 qd0, [dp0], #8
vldrbt.u8 qd1, [dp1], #8
vldrbt.u8 qd2, [dp2], #8
vldrbt.u8 qd3, [dp3], #8

// Transpose from per-instance layout to per-lane layout
transpose_streams_to_lanes
Expand All @@ -320,20 +321,22 @@ MLK_ASM_FN_SYMBOL(keccak_f1600_x4_state_xor_bytes_asm)
// XOR into state (post-increments rSE/rSO by 16)
xor_lane_and_store_postinc q0, q1, qS0, qS1, rSE, rSO

// Build qP = {d0,d1,d2,d3} as u32 lanes for gather loads
vmov qP[2], qP[0], dp0, dp2
vmov qP[3], qP[1], dp1, dp3
b keccak_f1600_x4_state_xor_bytes_asm_main_body

keccak_f1600_x4_state_xor_bytes_asm_pre_main:
// dpN points at the next lane start; vldrw requires 32-bit alignment.
orr tmp, dp0, dp1
orr tmp, tmp, dp2
orr tmp, tmp, dp3
tst tmp, #3
bne keccak_f1600_x4_state_xor_bytes_asm_main_body_byte

vmov qP[2], qP[0], dp0, dp2
vmov qP[3], qP[1], dp1, dp3
mov tmp, #4
vsub.u32 qP, qP, tmp // pre-bias so first [qP,#4]! lands at original ptr

keccak_f1600_x4_state_xor_bytes_asm_main_body:
// -----------------------------------------------------------------------
// MAIN: process full 8-byte lanes
// MAIN: process full 8-byte lanes with aligned gather-word loads
// -----------------------------------------------------------------------
lsr lr, length, #3
wls lr, lr, keccak_f1600_x4_state_xor_bytes_asm_main_loop_end
Expand All @@ -351,9 +354,6 @@ keccak_f1600_x4_state_xor_bytes_asm_main_loop_start:
le lr, keccak_f1600_x4_state_xor_bytes_asm_main_loop_start
keccak_f1600_x4_state_xor_bytes_asm_main_loop_end:

// -----------------------------------------------------------------------
// TAIL: absorb <8 remaining bytes at lane offset 0
// -----------------------------------------------------------------------
ands length, length, #7
beq keccak_f1600_x4_state_xor_bytes_asm_exit

Expand All @@ -363,12 +363,25 @@ keccak_f1600_x4_state_xor_bytes_asm_main_loop_end:
vmov dp0, dp2, qP[2], qP[0]
vmov dp1, dp3, qP[3], qP[1]

vctp.8 length
keccak_f1600_x4_state_xor_bytes_asm_main_body_byte:
// -----------------------------------------------------------------------
// MAIN/TAIL: process remaining lanes using predicated byte loads
// -----------------------------------------------------------------------
cmp length, #0
beq keccak_f1600_x4_state_xor_bytes_asm_exit
keccak_f1600_x4_state_xor_bytes_asm_main_loop_byte_start:
// Predicate with min(length, 8): a plain MVE byte vector load would
// over-read the Keccak lane, and the last iteration may be a partial tail.
mov nB, #8
cmp length, #8
it lo
movlo nB, length
vctp.8 nB
vpstttt
vldrbt.u8 qd0, [dp0]
vldrbt.u8 qd1, [dp1]
vldrbt.u8 qd2, [dp2]
vldrbt.u8 qd3, [dp3]
vldrbt.u8 qd0, [dp0], #8
vldrbt.u8 qd1, [dp1], #8
vldrbt.u8 qd2, [dp2], #8
vldrbt.u8 qd3, [dp3], #8

// Transpose from per-instance layout to per-lane layout
transpose_streams_to_lanes
Expand All @@ -377,7 +390,10 @@ keccak_f1600_x4_state_xor_bytes_asm_main_loop_end:
to_bit_interleaving_4x q0, q1, q2, q3, q4

// XOR into state (post-increments rSE/rSO by 16)
xor_lane_and_store_postinc qd0, qd1, qS0, qS1, rSE, rSO
xor_lane_and_store_postinc q0, q1, qS0, qS1, rSE, rSO

subs length, length, nB
bne keccak_f1600_x4_state_xor_bytes_asm_main_loop_byte_start

keccak_f1600_x4_state_xor_bytes_asm_exit:
vpop {d8-d15}
Expand Down
50 changes: 33 additions & 17 deletions mlkem/src/fips202/native/armv81m/src/state_xor_bytes_x4_mve.S
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,9 @@
// ---------------------------------------------------------------------------
// Prologue -- if offset is not 8-byte aligned, absorb
// min(length, 8-(offset%8)) bytes via predicated byte loads.
// Main -- process full 8-byte groups via word-level gather loads,
// bit-interleave, then VEOR into even/odd state halves.
// Main -- process full 8-byte groups via vldrw.u32 gather loads when
// 32-bit aligned, otherwise byte loads; bit-interleave, then
// VEOR into even/odd state halves.
// Tail -- absorb remaining <8 bytes via predicated byte loads.

#include "../../../../common.h"
Expand Down Expand Up @@ -98,13 +99,13 @@ MLK_ASM_FN_SYMBOL(keccak_f1600_x4_state_xor_bytes_asm)
ldr.w r10, [sp, #0x6c]
ldr r6, [sp, #0x70]
cmp r6, #0x0
beq.w Lkeccak_f1600_x4_state_xor_bytes_asm_exit @ imm = #0x346
beq.w Lkeccak_f1600_x4_state_xor_bytes_asm_exit @ imm = #0x368
and r5, r10, #0x7
bic r9, r10, #0x7
add.w r8, r0, r9, lsl #1
add.w r7, r8, #0x190
cmp r5, #0x0
beq.w Lkeccak_f1600_x4_state_xor_bytes_asm_pre_main @ imm = #0x12c
beq.w Lkeccak_f1600_x4_state_xor_bytes_asm_pre_main @ imm = #0x122
subs r1, r1, r5
subs r2, r2, r5
subs r3, r3, r5
Expand All @@ -119,10 +120,10 @@ MLK_ASM_FN_SYMBOL(keccak_f1600_x4_state_xor_bytes_asm)
lsl.w r11, r11, r5
vmsr p0, r11
vpstttt
vldrbt.u8 q0, [r1], #4
vldrbt.u8 q1, [r2], #4
vldrbt.u8 q2, [r3], #4
vldrbt.u8 q3, [r4], #4
vldrbt.u8 q0, [r1], #8
vldrbt.u8 q1, [r2], #8
vldrbt.u8 q2, [r3], #8
vldrbt.u8 q3, [r4], #8
vmov.f64 d1, d4
vmov.f64 d3, d6
vrev64.32 q2, q0
Expand Down Expand Up @@ -181,11 +182,13 @@ MLK_ASM_FN_SYMBOL(keccak_f1600_x4_state_xor_bytes_asm)
veor q5, q5, q1
vstrw.32 q4, [r8], #16
vstrw.32 q5, [r7], #16
vmov q7[2], q7[0], r1, r3
vmov q7[3], q7[1], r2, r4
b Lkeccak_f1600_x4_state_xor_bytes_asm_main_body @ imm = #0xe

Lkeccak_f1600_x4_state_xor_bytes_asm_pre_main:
orr.w r0, r1, r2
orr.w r0, r0, r3
orr.w r0, r0, r4
tst.w r0, #0x3
bne.w Lkeccak_f1600_x4_state_xor_bytes_asm_main_body_byte @ imm = #0x104
vmov q7[2], q7[0], r1, r3
vmov q7[3], q7[1], r2, r4
mov.w r0, #0x4
Expand Down Expand Up @@ -252,17 +255,27 @@ Lkeccak_f1600_x4_state_xor_bytes_asm_main_loop_start:

Lkeccak_f1600_x4_state_xor_bytes_asm_main_loop_end:
ands r6, r6, #0x7
beq.w Lkeccak_f1600_x4_state_xor_bytes_asm_exit @ imm = #0x110
beq.w Lkeccak_f1600_x4_state_xor_bytes_asm_exit @ imm = #0x128
mov.w r0, #0x4
vadd.i32 q7, q7, r0
vmov r1, r3, q7[2], q7[0]
vmov r2, r4, q7[3], q7[1]
vctp.8 r6

Lkeccak_f1600_x4_state_xor_bytes_asm_main_body_byte:
cmp r6, #0x0
beq.w Lkeccak_f1600_x4_state_xor_bytes_asm_exit @ imm = #0x112

Lkeccak_f1600_x4_state_xor_bytes_asm_main_loop_byte_start:
mov.w lr, #0x8
cmp r6, #0x8
it lo
movlo lr, r6
vctp.8 lr
vpstttt
vldrbt.u8 q0, [r1]
vldrbt.u8 q1, [r2]
vldrbt.u8 q2, [r3]
vldrbt.u8 q3, [r4]
vldrbt.u8 q0, [r1], #8
vldrbt.u8 q1, [r2], #8
vldrbt.u8 q2, [r3], #8
vldrbt.u8 q3, [r4], #8
vmov.f64 d1, d4
vmov.f64 d3, d6
vrev64.32 q2, q0
Expand Down Expand Up @@ -321,6 +334,8 @@ Lkeccak_f1600_x4_state_xor_bytes_asm_main_loop_end:
veor q5, q5, q1
vstrw.32 q4, [r8], #16
vstrw.32 q5, [r7], #16
subs.w r6, r6, lr
bne.w Lkeccak_f1600_x4_state_xor_bytes_asm_main_loop_byte_start @ imm = #-0x112

Lkeccak_f1600_x4_state_xor_bytes_asm_exit:
vpop {d8, d9, d10, d11, d12, d13, d14, d15}
Expand All @@ -345,6 +360,7 @@ Lkeccak_f1600_x4_state_xor_bytes_asm_exit:
.cfi_restore lr
.cfi_adjust_cfa_offset -0x28
.cfi_endproc
nop

MLK_ASM_FN_SIZE(keccak_f1600_x4_state_xor_bytes_asm)

Expand Down
20 changes: 12 additions & 8 deletions test/src/test_unit.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -836,10 +836,10 @@ static int test_keccakf1600x4_xor_permute_extract(void)
MLK_ALLOC(output_x4_1, unsigned char, MAX_RATE, NULL);
MLK_ALLOC(output_x4_2, unsigned char, MAX_RATE, NULL);
MLK_ALLOC(output_x4_3, unsigned char, MAX_RATE, NULL);
MLK_ALLOC(input_0, unsigned char, MAX_RATE, NULL);
MLK_ALLOC(input_1, unsigned char, MAX_RATE, NULL);
MLK_ALLOC(input_2, unsigned char, MAX_RATE, NULL);
MLK_ALLOC(input_3, unsigned char, MAX_RATE, NULL);
MLK_ALLOC(input_0, unsigned char, MAX_RATE + 3, NULL);
MLK_ALLOC(input_1, unsigned char, MAX_RATE + 3, NULL);
MLK_ALLOC(input_2, unsigned char, MAX_RATE + 3, NULL);
MLK_ALLOC(input_3, unsigned char, MAX_RATE + 3, NULL);

unsigned char *output_x4[MLK_KECCAK_WAY];
unsigned char *input[MLK_KECCAK_WAY];
Expand Down Expand Up @@ -875,6 +875,10 @@ static int test_keccakf1600x4_xor_permute_extract(void)
ext_length = (uint8_t)(1 + (ext_length % (MAX_RATE - ext_offset)));

/* Generate different random input for each lane */
input[0] = input_0 + (((unsigned)i + 0u) & 3u);
input[1] = input_1 + (((unsigned)i + 1u) & 3u);
input[2] = input_2 + (((unsigned)i + 2u) & 3u);
input[3] = input_3 + (((unsigned)i + 3u) & 3u);
for (j = 0; j < MLK_KECCAK_WAY; j++)
{
randombytes(input[j], xor_length);
Expand Down Expand Up @@ -905,10 +909,10 @@ static int test_keccakf1600x4_xor_permute_extract(void)
ret = 0;

cleanup:
MLK_FREE(input_3, unsigned char, MAX_RATE, NULL);
MLK_FREE(input_2, unsigned char, MAX_RATE, NULL);
MLK_FREE(input_1, unsigned char, MAX_RATE, NULL);
MLK_FREE(input_0, unsigned char, MAX_RATE, NULL);
MLK_FREE(input_3, unsigned char, MAX_RATE + 3, NULL);
MLK_FREE(input_2, unsigned char, MAX_RATE + 3, NULL);
MLK_FREE(input_1, unsigned char, MAX_RATE + 3, NULL);
MLK_FREE(input_0, unsigned char, MAX_RATE + 3, NULL);
MLK_FREE(output_x4_3, unsigned char, MAX_RATE, NULL);
MLK_FREE(output_x4_2, unsigned char, MAX_RATE, NULL);
MLK_FREE(output_x4_1, unsigned char, MAX_RATE, NULL);
Expand Down
Loading