Merge remote-tracking branch 'upstream/main'

Author: alexcrichton

cranelift/assembler-x64/meta/src/generate/format.rs

@@ -208,31 +208,41 @@ impl dsl::Format {
                     rm: *rm,
                 }
             }
-            [Reg(reg_or_vvvv), RegMem(rm)] | [Reg(reg_or_vvvv), RegMem(rm), Imm(_)] => {
-                match vex.unwrap_digit() {
-                    Some(digit) => {
-                        let vvvv = reg_or_vvvv;
-                        fmtln!(f, "let reg = {digit:#x};");
-                        fmtln!(f, "let vvvv = self.{vvvv}.enc();");
-                        fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");
-                        fmtln!(f, "let vex = VexPrefix::three_op(reg, vvvv, rm, {bits});");
-                        ModRmStyle::RegMem {
-                            reg: ModRmReg::Digit(digit),
-                            rm: *rm,
-                        }
+            [Reg(reg_or_vvvv), RegMem(rm)]
+            | [RegMem(rm), Reg(reg_or_vvvv)]
+            | [Reg(reg_or_vvvv), RegMem(rm), Imm(_)] => match vex.unwrap_digit() {
+                Some(digit) => {
+                    let vvvv = reg_or_vvvv;
+                    fmtln!(f, "let reg = {digit:#x};");
+                    fmtln!(f, "let vvvv = self.{vvvv}.enc();");
+                    fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");
+                    fmtln!(f, "let vex = VexPrefix::three_op(reg, vvvv, rm, {bits});");
+                    ModRmStyle::RegMem {
+                        reg: ModRmReg::Digit(digit),
+                        rm: *rm,
                     }
-                    None => {
-                        let reg = reg_or_vvvv;
-                        fmtln!(f, "let reg = self.{reg}.enc();");
-                        fmtln!(f, "let vvvv = {};", "0b0");
-                        fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");
-                        fmtln!(f, "let vex = VexPrefix::three_op(reg, vvvv, rm, {bits});");
-                        ModRmStyle::RegMem {
-                            reg: ModRmReg::Reg(*reg),
-                            rm: *rm,
-                        }
+                }
+                None => {
+                    let reg = reg_or_vvvv;
+                    fmtln!(f, "let reg = self.{reg}.enc();");
+                    fmtln!(f, "let vvvv = {};", "0b0");
+                    fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");
+                    fmtln!(f, "let vex = VexPrefix::three_op(reg, vvvv, rm, {bits});");
+                    ModRmStyle::RegMem {
+                        reg: ModRmReg::Reg(*reg),
+                        rm: *rm,
                     }
                 }
+            },
+            [Reg(reg), Reg(rm)] => {
+                fmtln!(f, "let reg = self.{reg}.enc();");
+                fmtln!(f, "let vvvv = 0;");
+                fmtln!(f, "let rm = (Some(self.{rm}.enc()), None);");
+                fmtln!(f, "let vex = VexPrefix::three_op(reg, vvvv, rm, {bits});");
+                ModRmStyle::Reg {
+                    reg: ModRmReg::Reg(*reg),
+                    rm: *rm,
+                }
             }
             unknown => unimplemented!("unknown pattern: {unknown:?}"),
         };

cranelift/assembler-x64/meta/src/instructions/lanes.rs

@@ -25,6 +25,9 @@ pub fn list() -> Vec<Inst> {
         inst("movmskps", fmt("RM", [w(r32), r(xmm2)]), rex([0x0F, 0x50]).r(), _64b | compat | sse),
         inst("movmskpd", fmt("RM", [w(r32), r(xmm2)]), rex([0x66, 0x0F, 0x50]).r(), _64b | compat | sse2),
         inst("pmovmskb", fmt("RM", [w(r32), r(xmm2)]), rex([0x66, 0x0F, 0xD7]).r(), _64b | compat | sse2),
+        inst("vmovmskps", fmt("RM", [w(r32), r(xmm2)]), vex(L128)._0f().op(0x50).r(), _64b | compat | avx),
+        inst("vmovmskpd", fmt("RM", [w(r32), r(xmm2)]), vex(L128)._66()._0f().op(0x50).r(), _64b | compat | avx),
+        inst("vpmovmskb", fmt("RM", [w(r32), r(xmm2)]), vex(L128)._66()._0f().op(0xD7).r(), _64b | compat | avx),
 
         inst("vpinsrb", fmt("B", [w(xmm1), r(xmm2), r(r32m8), r(imm8)]), vex(L128)._66()._0f3a().w0().op(0x20).r().ib(), _64b | compat | avx),
         inst("vpinsrw", fmt("B", [w(xmm1), r(xmm2), r(r32m16), r(imm8)]), vex(L128)._66()._0f().w0().op(0xC4).r().ib(), _64b | compat | avx),

cranelift/assembler-x64/meta/src/instructions/mov.rs

@@ -1,5 +1,5 @@
-use crate::dsl::{Feature::*, Inst, Location::*};
-use crate::dsl::{align, fmt, inst, r, rex, rw, sxl, sxq, sxw, w};
+use crate::dsl::{Feature::*, Inst, Location::*, VexLength::*};
+use crate::dsl::{align, fmt, inst, r, rex, rw, sxl, sxq, sxw, vex, w};
 
 #[rustfmt::skip] // Keeps instructions on a single line.
 pub fn list() -> Vec<Inst> {
@@ -56,6 +56,10 @@ pub fn list() -> Vec<Inst> {
         inst("movq", fmt("A", [w(xmm1), r(rm64)]), rex([0x66, 0x0F, 0x6E]).r().w(), _64b | sse2),
         inst("movd", fmt("B", [w(rm32), r(xmm2)]), rex([0x66, 0x0F, 0x7E]).r(), _64b | compat | sse2),
         inst("movq", fmt("B", [w(rm64), r(xmm2)]), rex([0x66, 0x0F, 0x7E]).r().w(), _64b | sse2),
+        inst("vmovd", fmt("A", [w(xmm1), r(rm32)]), vex(L128)._66()._0f().w0().op(0x6E).r(), _64b | compat | avx),
+        inst("vmovq", fmt("A", [w(xmm1), r(rm64)]), vex(L128)._66()._0f().w1().op(0x6E).r(), _64b | avx),
+        inst("vmovd", fmt("B", [w(rm32), r(xmm2)]), vex(L128)._66()._0f().w0().op(0x7E).r(), _64b | compat | avx),
+        inst("vmovq", fmt("B", [w(rm64), r(xmm2)]), vex(L128)._66()._0f().w1().op(0x7E).r(), _64b | avx),
 
         // Move floating-point values to and from XMM locations. Some
         // memory-loading versions of `movs*` clear the upper bits of the XMM

cranelift/codegen/src/isa/x64/inst.isle

@@ -194,20 +194,6 @@
                        (dst WritableGpr)
                        (imm u8))
 
-       ;; XMM (scalar) unary op (from integer to float reg): vmovd, vmovq,
-       ;; vcvtsi2s{s,d}
-       (GprToXmmVex (op AvxOpcode)
-                    (src GprMem)
-                    (dst WritableXmm)
-                    (src_size OperandSize))
-
-       ;; XMM (scalar) unary op (from xmm to integer reg): vmovd, vmovq,
-       ;; vcvtts{s,d}2si
-       (XmmToGprVex (op AvxOpcode)
-                    (src Xmm)
-                    (dst WritableGpr)
-                    (dst_size OperandSize))
-
        ;; Float comparisons/tests: cmp (b w l q) (reg addr imm) reg.
        (XmmCmpRmRVex (op AvxOpcode)
                      (src1 Xmm)
@@ -1144,11 +1130,6 @@
             Vpbroadcastw
             Vpbroadcastd
             Vbroadcastss
-            Vmovd
-            Vmovq
-            Vmovmskps
-            Vmovmskpd
-            Vpmovmskb
             Vsqrtss
             Vsqrtsd
             Vroundss
@@ -1728,20 +1709,6 @@
             (_ Unit (emit (MInst.XmmToGprImmVex op src dst imm))))
         dst))
 
-;; Helper for creating `MInst.GprToXmmVex` instructions.
-(decl gpr_to_xmm_vex (AvxOpcode GprMem OperandSize) Xmm)
-(rule (gpr_to_xmm_vex op src size)
-      (let ((dst WritableXmm (temp_writable_xmm))
-            (_ Unit (emit (MInst.GprToXmmVex op src dst size))))
-        dst))
-
-;; Helper for creating `MInst.XmmToGprVex` instructions.
-(decl xmm_to_gpr_vex (AvxOpcode Xmm OperandSize) Gpr)
-(rule (xmm_to_gpr_vex op src size)
-      (let ((dst WritableGpr (temp_writable_gpr))
-            (_ Unit (emit (MInst.XmmToGprVex op src dst size))))
-        dst))
-
 ;; Helper for creating `xmm_min_max_seq` pseudo-instructions.
 (decl xmm_min_max_seq (Type bool Xmm Xmm) Xmm)
 (rule (xmm_min_max_seq ty is_min lhs rhs)
@@ -2126,28 +2093,28 @@
 (rule (x64_movd_to_gpr from) (x64_movd_b from))
 (rule 1 (x64_movd_to_gpr from)
         (if-let true (use_avx))
-        (xmm_to_gpr_vex (AvxOpcode.Vmovd) from (OperandSize.Size32)))
+        (x64_vmovd_b from))
 
 ;; Helper for creating `movd` instructions.
 (decl x64_movd_to_xmm (GprMem) Xmm)
 (rule (x64_movd_to_xmm from) (x64_movd_a from))
 (rule 1 (x64_movd_to_xmm from)
         (if-let true (use_avx))
-        (gpr_to_xmm_vex (AvxOpcode.Vmovd) from (OperandSize.Size32)))
+        (x64_vmovd_a from))
 
 ;; Helper for creating `movq` instructions.
 (decl x64_movq_to_xmm (GprMem) Xmm)
 (rule (x64_movq_to_xmm src) (x64_movq_a src))
 (rule 1 (x64_movq_to_xmm from)
         (if-let true (use_avx))
-        (gpr_to_xmm_vex (AvxOpcode.Vmovq) from (OperandSize.Size64)))
+        (x64_vmovq_a from))
 
 ;; Helper for creating `movq` instructions.
 (decl x64_movq_to_gpr (Xmm) Gpr)
 (rule (x64_movq_to_gpr src) (x64_movq_b src))
 (rule 1 (x64_movq_to_gpr from)
         (if-let true (use_avx))
-        (xmm_to_gpr_vex (AvxOpcode.Vmovq) from (OperandSize.Size64)))
+        (x64_vmovq_b from))
 
 (decl x64_movdqu_load (XmmMem) Xmm)
 (rule (x64_movdqu_load from) (x64_movdqu_a from))
@@ -3986,21 +3953,21 @@
 (rule (x64_pmovmskb src) (x64_pmovmskb_rm src))
 (rule 1 (x64_pmovmskb src)
         (if-let true (use_avx))
-        (xmm_to_gpr_vex (AvxOpcode.Vpmovmskb) src (OperandSize.Size32)))
+        (x64_vpmovmskb_rm src))
 
 ;; Helper for creating `movmskps` instructions.
 (decl x64_movmskps (Xmm) Gpr)
 (rule (x64_movmskps src) (x64_movmskps_rm src))
 (rule 1 (x64_movmskps src)
         (if-let true (use_avx))
-        (xmm_to_gpr_vex (AvxOpcode.Vmovmskps) src (OperandSize.Size32)))
+        (x64_vmovmskps_rm src))
 
 ;; Helper for creating `movmskpd` instructions.
 (decl x64_movmskpd (Xmm) Gpr)
 (rule (x64_movmskpd src) (x64_movmskpd_rm src))
 (rule 1 (x64_movmskpd src)
         (if-let true (use_avx))
-        (xmm_to_gpr_vex (AvxOpcode.Vmovmskpd) src (OperandSize.Size32)))
+        (x64_vmovmskpd_rm src))
 
 ;; Helper for creating `not` instructions.
 (decl x64_not (Type Gpr) Gpr)

cranelift/codegen/src/isa/x64/inst/args.rs

@@ -1129,11 +1129,6 @@ impl AvxOpcode {
             | AvxOpcode::Vpextrq
             | AvxOpcode::Vpblendw
             | AvxOpcode::Vbroadcastss
-            | AvxOpcode::Vmovd
-            | AvxOpcode::Vmovq
-            | AvxOpcode::Vmovmskps
-            | AvxOpcode::Vmovmskpd
-            | AvxOpcode::Vpmovmskb
             | AvxOpcode::Vsqrtss
             | AvxOpcode::Vsqrtsd
             | AvxOpcode::Vroundss

cranelift/codegen/src/isa/x64/inst/emit.rs

@@ -1996,82 +1996,6 @@ pub(crate) fn emit(
                 .encode(sink);
         }
 
-        Inst::XmmToGprVex {
-            op,
-            src,
-            dst,
-            dst_size,
-        } => {
-            let src = src.to_reg();
-            let dst = dst.to_reg().to_reg();
-
-            let (prefix, map, opcode) = match op {
-                // vmovd/vmovq are differentiated by `w`
-                AvxOpcode::Vmovd | AvxOpcode::Vmovq => (LegacyPrefixes::_66, OpcodeMap::_0F, 0x7E),
-                AvxOpcode::Vmovmskps => (LegacyPrefixes::None, OpcodeMap::_0F, 0x50),
-                AvxOpcode::Vmovmskpd => (LegacyPrefixes::_66, OpcodeMap::_0F, 0x50),
-                AvxOpcode::Vpmovmskb => (LegacyPrefixes::_66, OpcodeMap::_0F, 0xD7),
-                _ => unimplemented!("Opcode {:?} not implemented", op),
-            };
-            let w = match dst_size {
-                OperandSize::Size64 => true,
-                _ => false,
-            };
-            let mut vex = VexInstruction::new()
-                .length(VexVectorLength::V128)
-                .w(w)
-                .prefix(prefix)
-                .map(map)
-                .opcode(opcode);
-            vex = match op {
-                // The `vmovq/vmovd` reverse the order of the destination/source
-                // relative to other opcodes using this shape of instruction.
-                AvxOpcode::Vmovd | AvxOpcode::Vmovq => vex
-                    .rm(dst.to_real_reg().unwrap().hw_enc())
-                    .reg(src.to_real_reg().unwrap().hw_enc()),
-                _ => vex
-                    .rm(src.to_real_reg().unwrap().hw_enc())
-                    .reg(dst.to_real_reg().unwrap().hw_enc()),
-            };
-            vex.encode(sink);
-        }
-
-        Inst::GprToXmmVex {
-            op,
-            src,
-            dst,
-            src_size,
-        } => {
-            let dst = dst.to_reg().to_reg();
-            let src = match src.clone().to_reg_mem().clone() {
-                RegMem::Reg { reg } => {
-                    RegisterOrAmode::Register(reg.to_real_reg().unwrap().hw_enc().into())
-                }
-                RegMem::Mem { addr } => {
-                    RegisterOrAmode::Amode(addr.finalize(state.frame_layout(), sink))
-                }
-            };
-
-            let (prefix, map, opcode) = match op {
-                // vmovd/vmovq are differentiated by `w`
-                AvxOpcode::Vmovd | AvxOpcode::Vmovq => (LegacyPrefixes::_66, OpcodeMap::_0F, 0x6E),
-                _ => unimplemented!("Opcode {:?} not implemented", op),
-            };
-            let w = match src_size {
-                OperandSize::Size64 => true,
-                _ => false,
-            };
-            VexInstruction::new()
-                .length(VexVectorLength::V128)
-                .w(w)
-                .prefix(prefix)
-                .map(map)
-                .opcode(opcode)
-                .rm(src)
-                .reg(dst.to_real_reg().unwrap().hw_enc())
-                .encode(sink);
-        }
-
         Inst::XmmCmpRmRVex { op, src1, src2 } => {
             let src1 = src1.to_reg();
             let src2 = match src2.clone().to_reg_mem().clone() {

cranelift/codegen/src/isa/x64/inst/mod.rs

@@ -157,8 +157,6 @@ impl Inst {
             | Inst::XmmMovRMVex { op, .. }
             | Inst::XmmMovRMImmVex { op, .. }
             | Inst::XmmToGprImmVex { op, .. }
-            | Inst::XmmToGprVex { op, .. }
-            | Inst::GprToXmmVex { op, .. }
             | Inst::XmmCmpRmRVex { op, .. } => op.available_from(),
 
             Inst::External { inst } => {
@@ -876,38 +874,13 @@ impl PrettyPrint for Inst {
                 format!("{op} {dst}")
             }
 
-            Inst::XmmToGprVex {
-                op,
-                src,
-                dst,
-                dst_size,
-            } => {
-                let dst_size = dst_size.to_bytes();
-                let src = pretty_print_reg(src.to_reg(), 8);
-                let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size);
-                let op = ljustify(op.to_string());
-                format!("{op} {src}, {dst}")
-            }
-
             Inst::XmmToGprImmVex { op, src, dst, imm } => {
                 let src = pretty_print_reg(src.to_reg(), 8);
                 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
                 let op = ljustify(op.to_string());
                 format!("{op} ${imm}, {src}, {dst}")
             }
 
-            Inst::GprToXmmVex {
-                op,
-                src,
-                src_size,
-                dst,
-            } => {
-                let dst = pretty_print_reg(dst.to_reg().to_reg(), 8);
-                let src = src.pretty_print(src_size.to_bytes());
-                let op = ljustify(op.to_string());
-                format!("{op} {src}, {dst}")
-            }
-
             Inst::XmmCmpRmR { op, src1, src2 } => {
                 let src1 = pretty_print_reg(src1.to_reg(), 8);
                 let src2 = src2.pretty_print(8);
@@ -1688,14 +1661,10 @@ fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
             collector.reg_use(src);
             collector.reg_fixed_nonallocatable(*dst);
         }
-        Inst::XmmToGprVex { src, dst, .. } | Inst::XmmToGprImmVex { src, dst, .. } => {
+        Inst::XmmToGprImmVex { src, dst, .. } => {
             collector.reg_use(src);
             collector.reg_def(dst);
         }
-        Inst::GprToXmmVex { src, dst, .. } => {
-            collector.reg_def(dst);
-            src.get_operands(collector);
-        }
         Inst::CvtUint64ToFloatSeq {
             src,
             dst,

cranelift/codegen/src/isa/x64/lower.isle

@@ -1900,6 +1900,9 @@
 (rule 1 (lower (trapz (icmp cc a b) code))
         (side_effect (trap_if_icmp (emit_cmp (intcc_complement cc) a b) code)))
 
+(rule 1 (lower (trapz (fcmp cc a b) code))
+        (side_effect (trap_if_fcmp (emit_fcmp (floatcc_complement cc) a b) code)))
+
 ;;;; Rules for `trapnz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (rule 0 (lower (trapnz val code))
@@ -1908,6 +1911,9 @@
 (rule 1 (lower (trapnz (icmp cc a b) code))
         (side_effect (trap_if_icmp (emit_cmp cc a b) code)))
 
+(rule 1 (lower (trapnz (fcmp cc a b) code))
+        (side_effect (trap_if_fcmp (emit_fcmp cc a b) code)))
+
 ;;;; Rules for `uadd_overflow_trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (rule (lower (has_type (fits_in_64 ty) (uadd_overflow_trap a b tc)))