Implement aarch64 PMULL LLVM intrinsics

Author: CathalMullan

example/neon.rs

@@ -354,6 +354,27 @@ unsafe fn test_vsha256su1q_u32() {
     assert_eq!(r, e);
 }
 
+#[cfg(target_arch = "aarch64")]
+#[target_feature(enable = "aes")]
+fn test_vmull_p64() {
+    // AArch64 llvm intrinsic: llvm.aarch64.neon.pmull64
+    let a: u64 = 3;
+    let b: u64 = 6;
+    let e: u128 = 10;
+    let r: u128 = vmull_p64(a, b);
+    assert_eq!(r, e);
+}
+
+#[cfg(target_arch = "aarch64")]
+unsafe fn test_vmull_p8() {
+    // AArch64 llvm intrinsic: llvm.aarch64.neon.pmull.v8i16
+    let a = u8x8::from([0, 1, 2, 3, 4, 5, 6, 7]);
+    let b = u8x8::from([8, 9, 10, 11, 12, 13, 14, 15]);
+    let e = u16x8::from([0x0000, 0x0009, 0x0014, 0x001d, 0x0030, 0x0039, 0x0024, 0x002d]);
+    let r: u16x8 = unsafe { transmute(vmull_p8(transmute(a), transmute(b))) };
+    assert_eq!(r, e);
+}
+
 #[cfg(target_arch = "aarch64")]
 fn main() {
     unsafe {
@@ -398,6 +419,9 @@ fn main() {
         test_vsha256h2q_u32();
         test_vsha256su0q_u32();
         test_vsha256su1q_u32();
+
+        test_vmull_p64();
+        test_vmull_p8();
     }
 }
 

src/intrinsics/llvm_aarch64.rs

@@ -712,6 +712,81 @@ pub(super) fn codegen_aarch64_llvm_intrinsic_call<'tcx>(
             );
         }
 
+        "llvm.aarch64.neon.pmull64" => {
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            let a = a.load_scalar(fx);
+            let b = b.load_scalar(fx);
+
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String(
+                    "fmov    d0, x0
+                     fmov    d1, x1
+                     pmull   v0.1q, v0.1d, v1.1d"
+                        .into(),
+                )],
+                &[
+                    CInlineAsmOperand::Out {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64(
+                            AArch64InlineAsmReg::v0,
+                        )),
+                        late: true,
+                        place: Some(ret),
+                    },
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64(
+                            AArch64InlineAsmReg::x0,
+                        )),
+                        value: a,
+                    },
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64(
+                            AArch64InlineAsmReg::x1,
+                        )),
+                        value: b,
+                    },
+                    CInlineAsmOperand::Out {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64(
+                            AArch64InlineAsmReg::v1,
+                        )),
+                        late: true,
+                        place: None,
+                    },
+                ],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
+        }
+
+        "llvm.aarch64.neon.pmull.v8i16" => {
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            let a = a.load_scalar(fx);
+            let b = b.load_scalar(fx);
+
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String("pmull v0.8h, v0.8b, v1.8b".into())],
+                &[
+                    CInlineAsmOperand::InOut {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64(
+                            AArch64InlineAsmReg::v0,
+                        )),
+                        _late: true,
+                        in_value: a,
+                        out_place: Some(ret),
+                    },
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::AArch64(
+                            AArch64InlineAsmReg::v1,
+                        )),
+                        value: b,
+                    },
+                ],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
+        }
+
         _ => {
             fx.tcx.dcx().warn(format!(
                 "unsupported AArch64 llvm intrinsic {}; replacing with trap",