Skip to content

Commit a6618fe

Browse files
bjorn3bjorn3
authored
Merge pull request #1666 from CathalMullan/sha3
Implement remaining aarch64 SHA-3 LLVM intrinsics
2 parents 4b57ec9 + 2c81a77 commit a6618fe

2 files changed

Lines changed: 317 additions & 0 deletions

File tree

‎example/neon.rs‎

Lines changed: 235 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -470,6 +470,220 @@ unsafe fn test_vsha512su1q_u64() {
470470
assert_eq!(r, e);
471471
}
472472

473+
#[cfg(target_arch = "aarch64")]
474+
#[target_feature(enable = "sha3")]
475+
unsafe fn test_veor3q_s8() {
476+
// AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3s.v16i8
477+
let a = i8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]);
478+
let b = i8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]);
479+
let c = i8x16::from([32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47]);
480+
let e = i8x16::from([48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63]);
481+
let r: i8x16 = unsafe { transmute(veor3q_s8(transmute(a), transmute(b), transmute(c))) };
482+
assert_eq!(r, e);
483+
}
484+
485+
#[cfg(target_arch = "aarch64")]
486+
#[target_feature(enable = "sha3")]
487+
unsafe fn test_veor3q_s16() {
488+
// AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3s.v8i16
489+
let a = i16x8::from([0, 1, 2, 3, 4, 5, 6, 7]);
490+
let b = i16x8::from([8, 9, 10, 11, 12, 13, 14, 15]);
491+
let c = i16x8::from([16, 17, 18, 19, 20, 21, 22, 23]);
492+
let e = i16x8::from([24, 25, 26, 27, 28, 29, 30, 31]);
493+
let r: i16x8 = unsafe { transmute(veor3q_s16(transmute(a), transmute(b), transmute(c))) };
494+
assert_eq!(r, e);
495+
}
496+
497+
#[cfg(target_arch = "aarch64")]
498+
#[target_feature(enable = "sha3")]
499+
unsafe fn test_veor3q_s32() {
500+
// AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3s.v4i32
501+
let a = i32x4::from([0, 1, 2, 3]);
502+
let b = i32x4::from([4, 5, 6, 7]);
503+
let c = i32x4::from([8, 9, 10, 11]);
504+
let e = i32x4::from([12, 13, 14, 15]);
505+
let r: i32x4 = unsafe { transmute(veor3q_s32(transmute(a), transmute(b), transmute(c))) };
506+
assert_eq!(r, e);
507+
}
508+
509+
#[cfg(target_arch = "aarch64")]
510+
#[target_feature(enable = "sha3")]
511+
unsafe fn test_veor3q_s64() {
512+
// AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3s.v2i64
513+
let a = i64x2::from([0, 1]);
514+
let b = i64x2::from([2, 3]);
515+
let c = i64x2::from([4, 5]);
516+
let e = i64x2::from([6, 7]);
517+
let r: i64x2 = unsafe { transmute(veor3q_s64(transmute(a), transmute(b), transmute(c))) };
518+
assert_eq!(r, e);
519+
}
520+
521+
#[cfg(target_arch = "aarch64")]
522+
#[target_feature(enable = "sha3")]
523+
unsafe fn test_veor3q_u8() {
524+
// AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3u.v16i8
525+
let a = u8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]);
526+
let b = u8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]);
527+
let c = u8x16::from([32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47]);
528+
let e = u8x16::from([48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63]);
529+
let r: u8x16 = unsafe { transmute(veor3q_u8(transmute(a), transmute(b), transmute(c))) };
530+
assert_eq!(r, e);
531+
}
532+
533+
#[cfg(target_arch = "aarch64")]
534+
#[target_feature(enable = "sha3")]
535+
unsafe fn test_veor3q_u16() {
536+
// AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3u.v8i16
537+
let a = u16x8::from([0, 1, 2, 3, 4, 5, 6, 7]);
538+
let b = u16x8::from([8, 9, 10, 11, 12, 13, 14, 15]);
539+
let c = u16x8::from([16, 17, 18, 19, 20, 21, 22, 23]);
540+
let e = u16x8::from([24, 25, 26, 27, 28, 29, 30, 31]);
541+
let r: u16x8 = unsafe { transmute(veor3q_u16(transmute(a), transmute(b), transmute(c))) };
542+
assert_eq!(r, e);
543+
}
544+
545+
#[cfg(target_arch = "aarch64")]
546+
#[target_feature(enable = "sha3")]
547+
unsafe fn test_veor3q_u32() {
548+
// AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3u.v4i32
549+
let a = u32x4::from([0, 1, 2, 3]);
550+
let b = u32x4::from([4, 5, 6, 7]);
551+
let c = u32x4::from([8, 9, 10, 11]);
552+
let e = u32x4::from([12, 13, 14, 15]);
553+
let r: u32x4 = unsafe { transmute(veor3q_u32(transmute(a), transmute(b), transmute(c))) };
554+
assert_eq!(r, e);
555+
}
556+
557+
#[cfg(target_arch = "aarch64")]
558+
#[target_feature(enable = "sha3")]
559+
unsafe fn test_veor3q_u64() {
560+
// AArch64 llvm intrinsic: llvm.aarch64.crypto.eor3u.v2i64
561+
let a = u64x2::from([0, 1]);
562+
let b = u64x2::from([2, 3]);
563+
let c = u64x2::from([4, 5]);
564+
let e = u64x2::from([6, 7]);
565+
let r: u64x2 = unsafe { transmute(veor3q_u64(transmute(a), transmute(b), transmute(c))) };
566+
assert_eq!(r, e);
567+
}
568+
569+
#[cfg(target_arch = "aarch64")]
570+
#[target_feature(enable = "sha3")]
571+
unsafe fn test_vbcaxq_s8() {
572+
// AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxs.v16i8
573+
let a = i8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]);
574+
let b = i8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]);
575+
let c = i8x16::from([32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47]);
576+
let e = i8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]);
577+
let r: i8x16 = unsafe { transmute(vbcaxq_s8(transmute(a), transmute(b), transmute(c))) };
578+
assert_eq!(r, e);
579+
}
580+
581+
#[cfg(target_arch = "aarch64")]
582+
#[target_feature(enable = "sha3")]
583+
unsafe fn test_vbcaxq_s16() {
584+
// AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxs.v8i16
585+
let a = i16x8::from([0, 1, 2, 3, 4, 5, 6, 7]);
586+
let b = i16x8::from([8, 9, 10, 11, 12, 13, 14, 15]);
587+
let c = i16x8::from([16, 17, 18, 19, 20, 21, 22, 23]);
588+
let e = i16x8::from([8, 9, 10, 11, 12, 13, 14, 15]);
589+
let r: i16x8 = unsafe { transmute(vbcaxq_s16(transmute(a), transmute(b), transmute(c))) };
590+
assert_eq!(r, e);
591+
}
592+
593+
#[cfg(target_arch = "aarch64")]
594+
#[target_feature(enable = "sha3")]
595+
unsafe fn test_vbcaxq_s32() {
596+
// AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxs.v4i32
597+
let a = i32x4::from([0, 1, 2, 3]);
598+
let b = i32x4::from([4, 5, 6, 7]);
599+
let c = i32x4::from([8, 9, 10, 11]);
600+
let e = i32x4::from([4, 5, 6, 7]);
601+
let r: i32x4 = unsafe { transmute(vbcaxq_s32(transmute(a), transmute(b), transmute(c))) };
602+
assert_eq!(r, e);
603+
}
604+
605+
#[cfg(target_arch = "aarch64")]
606+
#[target_feature(enable = "sha3")]
607+
unsafe fn test_vbcaxq_s64() {
608+
// AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxs.v2i64
609+
let a = i64x2::from([0, 1]);
610+
let b = i64x2::from([2, 3]);
611+
let c = i64x2::from([4, 5]);
612+
let e = i64x2::from([2, 3]);
613+
let r: i64x2 = unsafe { transmute(vbcaxq_s64(transmute(a), transmute(b), transmute(c))) };
614+
assert_eq!(r, e);
615+
}
616+
617+
#[cfg(target_arch = "aarch64")]
618+
#[target_feature(enable = "sha3")]
619+
unsafe fn test_vbcaxq_u8() {
620+
// AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxu.v16i8
621+
let a = u8x16::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]);
622+
let b = u8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]);
623+
let c = u8x16::from([32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47]);
624+
let e = u8x16::from([16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]);
625+
let r: u8x16 = unsafe { transmute(vbcaxq_u8(transmute(a), transmute(b), transmute(c))) };
626+
assert_eq!(r, e);
627+
}
628+
629+
#[cfg(target_arch = "aarch64")]
630+
#[target_feature(enable = "sha3")]
631+
unsafe fn test_vbcaxq_u16() {
632+
// AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxu.v8i16
633+
let a = u16x8::from([0, 1, 2, 3, 4, 5, 6, 7]);
634+
let b = u16x8::from([8, 9, 10, 11, 12, 13, 14, 15]);
635+
let c = u16x8::from([16, 17, 18, 19, 20, 21, 22, 23]);
636+
let e = u16x8::from([8, 9, 10, 11, 12, 13, 14, 15]);
637+
let r: u16x8 = unsafe { transmute(vbcaxq_u16(transmute(a), transmute(b), transmute(c))) };
638+
assert_eq!(r, e);
639+
}
640+
641+
#[cfg(target_arch = "aarch64")]
642+
#[target_feature(enable = "sha3")]
643+
unsafe fn test_vbcaxq_u32() {
644+
// AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxu.v4i32
645+
let a = u32x4::from([0, 1, 2, 3]);
646+
let b = u32x4::from([4, 5, 6, 7]);
647+
let c = u32x4::from([8, 9, 10, 11]);
648+
let e = u32x4::from([4, 5, 6, 7]);
649+
let r: u32x4 = unsafe { transmute(vbcaxq_u32(transmute(a), transmute(b), transmute(c))) };
650+
assert_eq!(r, e);
651+
}
652+
653+
#[cfg(target_arch = "aarch64")]
654+
#[target_feature(enable = "sha3")]
655+
unsafe fn test_vbcaxq_u64() {
656+
// AArch64 llvm intrinsic: llvm.aarch64.crypto.bcaxu.v2i64
657+
let a = u64x2::from([0, 1]);
658+
let b = u64x2::from([2, 3]);
659+
let c = u64x2::from([4, 5]);
660+
let e = u64x2::from([2, 3]);
661+
let r: u64x2 = unsafe { transmute(vbcaxq_u64(transmute(a), transmute(b), transmute(c))) };
662+
assert_eq!(r, e);
663+
}
664+
665+
#[cfg(target_arch = "aarch64")]
666+
#[target_feature(enable = "sha3")]
667+
unsafe fn test_vrax1q_u64() {
668+
// AArch64 llvm intrinsic: llvm.aarch64.crypto.rax1
669+
let a = u64x2::from([0, 1]);
670+
let b = u64x2::from([2, 3]);
671+
let e = u64x2::from([4, 7]);
672+
let r: u64x2 = unsafe { transmute(vrax1q_u64(transmute(a), transmute(b))) };
673+
assert_eq!(r, e);
674+
}
675+
676+
#[cfg(target_arch = "aarch64")]
677+
#[target_feature(enable = "sha3")]
678+
unsafe fn test_vxarq_u64() {
679+
// AArch64 llvm intrinsic: llvm.aarch64.crypto.xar
680+
let a = u64x2::from([0, 1]);
681+
let b = u64x2::from([2, 3]);
682+
let e = u64x2::from([4, 4]);
683+
let r: u64x2 = unsafe { transmute(vxarq_u64::<63>(transmute(a), transmute(b))) };
684+
assert_eq!(r, e);
685+
}
686+
473687
#[cfg(target_arch = "aarch64")]
474688
#[target_feature(enable = "aes")]
475689
fn test_vmull_p64() {
@@ -698,6 +912,27 @@ fn main() {
698912
test_vsha512h2q_u64();
699913
test_vsha512su0q_u64();
700914
test_vsha512su1q_u64();
915+
916+
test_veor3q_s8();
917+
test_veor3q_s16();
918+
test_veor3q_s32();
919+
test_veor3q_s64();
920+
test_veor3q_u8();
921+
test_veor3q_u16();
922+
test_veor3q_u32();
923+
test_veor3q_u64();
924+
925+
test_vbcaxq_s8();
926+
test_vbcaxq_s16();
927+
test_vbcaxq_s32();
928+
test_vbcaxq_s64();
929+
test_vbcaxq_u8();
930+
test_vbcaxq_u16();
931+
test_vbcaxq_u32();
932+
test_vbcaxq_u64();
933+
934+
test_vrax1q_u64();
935+
test_vxarq_u64();
701936
}
702937

703938
test_vmull_p64();

‎src/intrinsics/llvm_aarch64.rs‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -977,6 +977,88 @@ pub(super) fn codegen_aarch64_llvm_intrinsic_call<'tcx>(
977977
);
978978
}
979979

980+
"llvm.aarch64.crypto.eor3s.v2i64"
981+
| "llvm.aarch64.crypto.eor3s.v4i32"
982+
| "llvm.aarch64.crypto.eor3s.v8i16"
983+
| "llvm.aarch64.crypto.eor3s.v16i8"
984+
| "llvm.aarch64.crypto.eor3u.v2i64"
985+
| "llvm.aarch64.crypto.eor3u.v4i32"
986+
| "llvm.aarch64.crypto.eor3u.v8i16"
987+
| "llvm.aarch64.crypto.eor3u.v16i8" => {
988+
// https://developer.arm.com/documentation/ddi0602/2026-03/SIMD-FP-Instructions/EOR3--Three-way-exclusive-OR-
989+
intrinsic_args!(fx, args => (a, b, c); intrinsic);
990+
991+
simd_trio_for_each_lane(
992+
fx,
993+
a,
994+
b,
995+
c,
996+
ret,
997+
&|fx, _lane_ty, _res_lane_ty, a_lane, b_lane, c_lane| {
998+
let xor = fx.bcx.ins().bxor(a_lane, b_lane);
999+
fx.bcx.ins().bxor(xor, c_lane)
1000+
},
1001+
);
1002+
}
1003+
1004+
"llvm.aarch64.crypto.bcaxs.v2i64"
1005+
| "llvm.aarch64.crypto.bcaxs.v4i32"
1006+
| "llvm.aarch64.crypto.bcaxs.v8i16"
1007+
| "llvm.aarch64.crypto.bcaxs.v16i8"
1008+
| "llvm.aarch64.crypto.bcaxu.v2i64"
1009+
| "llvm.aarch64.crypto.bcaxu.v4i32"
1010+
| "llvm.aarch64.crypto.bcaxu.v8i16"
1011+
| "llvm.aarch64.crypto.bcaxu.v16i8" => {
1012+
// https://developer.arm.com/documentation/ddi0602/2026-03/SIMD-FP-Instructions/BCAX--Bit-clear-and-exclusive-OR-
1013+
intrinsic_args!(fx, args => (a, b, c); intrinsic);
1014+
1015+
simd_trio_for_each_lane(
1016+
fx,
1017+
a,
1018+
b,
1019+
c,
1020+
ret,
1021+
&|fx, _lane_ty, _res_lane_ty, a_lane, b_lane, c_lane| {
1022+
let band_not = fx.bcx.ins().band_not(b_lane, c_lane);
1023+
fx.bcx.ins().bxor(a_lane, band_not)
1024+
},
1025+
);
1026+
}
1027+
1028+
"llvm.aarch64.crypto.rax1" => {
1029+
// https://developer.arm.com/documentation/ddi0602/2026-03/SIMD-FP-Instructions/RAX1--Rotate-and-exclusive-OR-
1030+
intrinsic_args!(fx, args => (a, b); intrinsic);
1031+
1032+
simd_pair_for_each_lane(
1033+
fx,
1034+
a,
1035+
b,
1036+
ret,
1037+
&|fx, _lane_ty, _res_lane_ty, a_lane, b_lane| {
1038+
let rot = fx.bcx.ins().rotl_imm(b_lane, 1);
1039+
fx.bcx.ins().bxor(a_lane, rot)
1040+
},
1041+
);
1042+
}
1043+
1044+
"llvm.aarch64.crypto.xar" => {
1045+
// https://developer.arm.com/documentation/ddi0602/2026-03/SIMD-FP-Instructions/XAR--Exclusive-OR-and-rotate-
1046+
intrinsic_args!(fx, args => (a, b, c); intrinsic);
1047+
1048+
let c = c.load_scalar(fx);
1049+
1050+
simd_pair_for_each_lane(
1051+
fx,
1052+
a,
1053+
b,
1054+
ret,
1055+
&|fx, _lane_ty, _res_lane_ty, a_lane, b_lane| {
1056+
let xor = fx.bcx.ins().bxor(a_lane, b_lane);
1057+
fx.bcx.ins().rotr(xor, c)
1058+
},
1059+
);
1060+
}
1061+
9801062
"llvm.aarch64.neon.pmull64" => {
9811063
intrinsic_args!(fx, args => (a, b); intrinsic);
9821064

0 commit comments

Comments
 (0)