x64: Migrate cmov* to the new assembler (#11095)

This commit adds all `cmov*` variants from the Intel manual to the new
assembler. This then additionally removes the `Cmove` pseudo-inst in
favor of these new instructions. One difference from before is that the
naming in the `CC` enum does not exactly match what mnemonics Capstone
uses to disassemble. For example `CC.NB` in ISLE corresponds to the
Intel instruction `CMOVNB`. This instruction, however, has the same
encoding as `CMOVAE` and Capstone disassembles as `CMOVAE`. This means
that the instruction selection in ISLE isn't a 1:1 match with mnemonics.

This additionally adds support in the assembler ISLE generation to
understand that instructions which read EFLAGS generate a
`ConsumesFlags` variant in their instruction helpers.

Author: alexcrichton

cranelift/assembler-x64/meta/src/dsl/format.rs

@@ -601,6 +601,15 @@ pub enum Eflags {
 }
 
 impl Eflags {
+    /// Returns whether this represents a read of any bit in the EFLAGS
+    /// register.
+    pub fn is_read(&self) -> bool {
+        match self {
+            Eflags::None | Eflags::W => false,
+            Eflags::R | Eflags::RW => true,
+        }
+    }
+
     /// Returns whether this represents a writes to any bit in the EFLAGS
     /// register.
     pub fn is_write(&self) -> bool {

cranelift/assembler-x64/meta/src/instructions.rs

@@ -7,6 +7,7 @@ mod and;
 mod atomic;
 mod avg;
 mod bitmanip;
+mod cmov;
 mod cmp;
 mod cvt;
 mod div;
@@ -41,6 +42,7 @@ pub fn list() -> Vec<Inst> {
     all.extend(atomic::list());
     all.extend(avg::list());
     all.extend(bitmanip::list());
+    all.extend(cmov::list());
     all.extend(cmp::list());
     all.extend(cvt::list());
     all.extend(div::list());

cranelift/assembler-x64/meta/src/instructions/cmov.rs

@@ -0,0 +1,73 @@
+use crate::dsl::{Eflags::*, Feature::*, Inst, Location::*};
+use crate::dsl::{fmt, inst, r, rex, rw};
+
+#[rustfmt::skip] // Keeps instructions on a single line.
+pub fn list() -> Vec<Inst> {
+    vec![
+        // Note that the Intel manual lists many mnemonics for this family of
+        // instructions which are duplicates of other mnemonics. The order here
+        // matches the order in the manual and comments are left when variants
+        // are omitted due to the instructions being duplicates of another.
+        inst("cmovaw", fmt("RM", [rw(r16), r(rm16)]).flags(R), rex([0x66, 0x0f, 0x47]).r(), _64b | compat),
+        inst("cmoval", fmt("RM", [rw(r32), r(rm32)]).flags(R), rex([0x0f, 0x47]).r(), _64b | compat),
+        inst("cmovaq", fmt("RM", [rw(r64), r(rm64)]).flags(R), rex([0x0f, 0x47]).w().r(), _64b),
+        inst("cmovaew", fmt("RM", [rw(r16), r(rm16)]).flags(R), rex([0x66, 0x0f, 0x43]).r(), _64b | compat),
+        inst("cmovael", fmt("RM", [rw(r32), r(rm32)]).flags(R), rex([0x0f, 0x43]).r(), _64b | compat),
+        inst("cmovaeq", fmt("RM", [rw(r64), r(rm64)]).flags(R), rex([0x0f, 0x43]).w().r(), _64b),
+        inst("cmovbw", fmt("RM", [rw(r16), r(rm16)]).flags(R), rex([0x66, 0x0f, 0x42]).r(), _64b | compat),
+        inst("cmovbl", fmt("RM", [rw(r32), r(rm32)]).flags(R), rex([0x0f, 0x42]).r(), _64b | compat),
+        inst("cmovbq", fmt("RM", [rw(r64), r(rm64)]).flags(R), rex([0x0f, 0x42]).w().r(), _64b),
+        inst("cmovbew", fmt("RM", [rw(r16), r(rm16)]).flags(R), rex([0x66, 0x0f, 0x46]).r(), _64b | compat),
+        inst("cmovbel", fmt("RM", [rw(r32), r(rm32)]).flags(R), rex([0x0f, 0x46]).r(), _64b | compat),
+        inst("cmovbeq", fmt("RM", [rw(r64), r(rm64)]).flags(R), rex([0x0f, 0x46]).w().r(), _64b),
+        // NB: cmovc* is omitted here as it has the same encoding as cmovb*
+        inst("cmovew", fmt("RM", [rw(r16), r(rm16)]).flags(R), rex([0x66, 0x0f, 0x44]).r(), _64b | compat),
+        inst("cmovel", fmt("RM", [rw(r32), r(rm32)]).flags(R), rex([0x0f, 0x44]).r(), _64b | compat),
+        inst("cmoveq", fmt("RM", [rw(r64), r(rm64)]).flags(R), rex([0x0f, 0x44]).w().r(), _64b),
+        inst("cmovgw", fmt("RM", [rw(r16), r(rm16)]).flags(R), rex([0x66, 0x0f, 0x4f]).r(), _64b | compat),
+        inst("cmovgl", fmt("RM", [rw(r32), r(rm32)]).flags(R), rex([0x0f, 0x4f]).r(), _64b | compat),
+        inst("cmovgq", fmt("RM", [rw(r64), r(rm64)]).flags(R), rex([0x0f, 0x4f]).w().r(), _64b),
+        inst("cmovgew", fmt("RM", [rw(r16), r(rm16)]).flags(R), rex([0x66, 0x0f, 0x4d]).r(), _64b | compat),
+        inst("cmovgel", fmt("RM", [rw(r32), r(rm32)]).flags(R), rex([0x0f, 0x4d]).r(), _64b | compat),
+        inst("cmovgeq", fmt("RM", [rw(r64), r(rm64)]).flags(R), rex([0x0f, 0x4d]).w().r(), _64b),
+        inst("cmovlw", fmt("RM", [rw(r16), r(rm16)]).flags(R), rex([0x66, 0x0f, 0x4c]).r(), _64b | compat),
+        inst("cmovll", fmt("RM", [rw(r32), r(rm32)]).flags(R), rex([0x0f, 0x4c]).r(), _64b | compat),
+        inst("cmovlq", fmt("RM", [rw(r64), r(rm64)]).flags(R), rex([0x0f, 0x4c]).w().r(), _64b),
+        inst("cmovlew", fmt("RM", [rw(r16), r(rm16)]).flags(R), rex([0x66, 0x0f, 0x4e]).r(), _64b | compat),
+        inst("cmovlel", fmt("RM", [rw(r32), r(rm32)]).flags(R), rex([0x0f, 0x4e]).r(), _64b | compat),
+        inst("cmovleq", fmt("RM", [rw(r64), r(rm64)]).flags(R), rex([0x0f, 0x4e]).w().r(), _64b),
+        // NB: cmovna* is omitted here as it has the same encoding as cmovbe*
+        // NB: cmovnb* is omitted here as it has the same encoding as cmovae*
+        // NB: cmovnbe* is omitted here as it has the same encoding as cmova*
+        // NB: cmovnc* is omitted here as it has the same encoding as cmovae*
+        inst("cmovnew", fmt("RM", [rw(r16), r(rm16)]).flags(R), rex([0x66, 0x0f, 0x45]).r(), _64b | compat),
+        inst("cmovnel", fmt("RM", [rw(r32), r(rm32)]).flags(R), rex([0x0f, 0x45]).r(), _64b | compat),
+        inst("cmovneq", fmt("RM", [rw(r64), r(rm64)]).flags(R), rex([0x0f, 0x45]).w().r(), _64b),
+        // NB: cmovng* is omitted here as it has the same encoding as cmovle*
+        // NB: cmovnge* is omitted here as it has the same encoding as cmovl*
+        // NB: cmovnl* is omitted here as it has the same encoding as cmovge*
+        // NB: cmovnle* is omitted here as it has the same encoding as cmovg*
+        inst("cmovnow", fmt("RM", [rw(r16), r(rm16)]).flags(R), rex([0x66, 0x0f, 0x41]).r(), _64b | compat),
+        inst("cmovnol", fmt("RM", [rw(r32), r(rm32)]).flags(R), rex([0x0f, 0x41]).r(), _64b | compat),
+        inst("cmovnoq", fmt("RM", [rw(r64), r(rm64)]).flags(R), rex([0x0f, 0x41]).w().r(), _64b),
+        inst("cmovnpw", fmt("RM", [rw(r16), r(rm16)]).flags(R), rex([0x66, 0x0f, 0x4b]).r(), _64b | compat),
+        inst("cmovnpl", fmt("RM", [rw(r32), r(rm32)]).flags(R), rex([0x0f, 0x4b]).r(), _64b | compat),
+        inst("cmovnpq", fmt("RM", [rw(r64), r(rm64)]).flags(R), rex([0x0f, 0x4b]).w().r(), _64b),
+        inst("cmovnsw", fmt("RM", [rw(r16), r(rm16)]).flags(R), rex([0x66, 0x0f, 0x49]).r(), _64b | compat),
+        inst("cmovnsl", fmt("RM", [rw(r32), r(rm32)]).flags(R), rex([0x0f, 0x49]).r(), _64b | compat),
+        inst("cmovnsq", fmt("RM", [rw(r64), r(rm64)]).flags(R), rex([0x0f, 0x49]).w().r(), _64b),
+        // NB: cmovnz* is omitted here as it has the same encoding as cmovne*
+        inst("cmovow", fmt("RM", [rw(r16), r(rm16)]).flags(R), rex([0x66, 0x0f, 0x40]).r(), _64b | compat),
+        inst("cmovol", fmt("RM", [rw(r32), r(rm32)]).flags(R), rex([0x0f, 0x40]).r(), _64b | compat),
+        inst("cmovoq", fmt("RM", [rw(r64), r(rm64)]).flags(R), rex([0x0f, 0x40]).w().r(), _64b),
+        inst("cmovpw", fmt("RM", [rw(r16), r(rm16)]).flags(R), rex([0x66, 0x0f, 0x4a]).r(), _64b | compat),
+        inst("cmovpl", fmt("RM", [rw(r32), r(rm32)]).flags(R), rex([0x0f, 0x4a]).r(), _64b | compat),
+        inst("cmovpq", fmt("RM", [rw(r64), r(rm64)]).flags(R), rex([0x0f, 0x4a]).w().r(), _64b),
+        // NB: cmovpe* is omitted here as it has the same encoding as cmovp*
+        // NB: cmovpo* is omitted here as it has the same encoding as cmovnp*
+        inst("cmovsw", fmt("RM", [rw(r16), r(rm16)]).flags(R), rex([0x66, 0x0f, 0x48]).r(), _64b | compat),
+        inst("cmovsl", fmt("RM", [rw(r32), r(rm32)]).flags(R), rex([0x0f, 0x48]).r(), _64b | compat),
+        inst("cmovsq", fmt("RM", [rw(r64), r(rm64)]).flags(R), rex([0x0f, 0x48]).w().r(), _64b),
+        // NB: cmovz* is omitted here as it has the same encoding as cmove*
+    ]
+}

cranelift/codegen/meta/src/gen_asm.rs

@@ -301,6 +301,10 @@ enum IsleConstructor {
     /// This constructor produces no results, but the flags register is written,
     /// so a `ProducesFlags` value is returned with a side effect.
     ProducesFlagsSideEffect,
+
+    /// This instructions reads EFLAGS, and returns a single gpr, so this
+    /// creates `ConsumesFlags.ConsumesFlagsReturnsReg`.
+    ConsumesFlagsReturnsGpr,
 }
 
 impl IsleConstructor {
@@ -314,6 +318,7 @@ impl IsleConstructor {
                 "SideEffectNoResult"
             }
             IsleConstructor::ProducesFlagsSideEffect => "ProducesFlags",
+            IsleConstructor::ConsumesFlagsReturnsGpr => "ConsumesFlags",
         }
     }
 
@@ -328,6 +333,7 @@ impl IsleConstructor {
             IsleConstructor::RetXmm => "emit_ret_xmm",
             IsleConstructor::RetValueRegs => "emit_ret_value_regs",
             IsleConstructor::ProducesFlagsSideEffect => "asm_produce_flags_side_effect",
+            IsleConstructor::ConsumesFlagsReturnsGpr => "asm_consumes_flags_returns_gpr",
         }
     }
 
@@ -339,7 +345,8 @@ impl IsleConstructor {
             | IsleConstructor::RetXmm
             | IsleConstructor::RetValueRegs
             | IsleConstructor::NoReturnSideEffect
-            | IsleConstructor::ProducesFlagsSideEffect => "",
+            | IsleConstructor::ProducesFlagsSideEffect
+            | IsleConstructor::ConsumesFlagsReturnsGpr => "",
         }
     }
 
@@ -355,7 +362,8 @@ impl IsleConstructor {
             | IsleConstructor::RetXmm
             | IsleConstructor::RetValueRegs
             | IsleConstructor::NoReturnSideEffect
-            | IsleConstructor::ProducesFlagsSideEffect => false,
+            | IsleConstructor::ProducesFlagsSideEffect
+            | IsleConstructor::ConsumesFlagsReturnsGpr => false,
         }
     }
 }
@@ -371,7 +379,7 @@ fn isle_param_for_ctor(op: &Operand, ctor: IsleConstructor) -> String {
         OperandKind::RegMem(_) if op.mutability.is_write() => match ctor {
             IsleConstructor::RetMemorySideEffect => "SyntheticAmode".to_string(),
             IsleConstructor::NoReturnSideEffect => "".to_string(),
-            IsleConstructor::RetGpr => "Gpr".to_string(),
+            IsleConstructor::RetGpr | IsleConstructor::ConsumesFlagsReturnsGpr => "Gpr".to_string(),
             IsleConstructor::RetXmm => "Xmm".to_string(),
             IsleConstructor::RetValueRegs => "ValueRegs".to_string(),
             IsleConstructor::ProducesFlagsSideEffect => todo!(),
@@ -411,45 +419,66 @@ fn isle_constructors(format: &Format) -> Vec<IsleConstructor> {
                 // One read/write register output? Output the instruction
                 // and that register.
                 Reg(r) | FixedReg(r) => match r.reg_class().unwrap() {
-                    RegClass::Xmm => vec![IsleConstructor::RetXmm],
-                    RegClass::Gpr => vec![IsleConstructor::RetGpr],
+                    RegClass::Xmm => {
+                        assert!(!format.eflags.is_read());
+                        vec![IsleConstructor::RetXmm]
+                    }
+                    RegClass::Gpr => {
+                        if format.eflags.is_read() {
+                            vec![IsleConstructor::ConsumesFlagsReturnsGpr]
+                        } else {
+                            vec![IsleConstructor::RetGpr]
+                        }
+                    }
                 },
                 // One read/write memory operand? Output a side effect.
-                Mem(_) => vec![IsleConstructor::RetMemorySideEffect],
+                Mem(_) => {
+                    assert!(!format.eflags.is_read());
+                    vec![IsleConstructor::RetMemorySideEffect]
+                }
                 // One read/write reg-mem output? We need constructors for
                 // both variants.
-                RegMem(rm) => match rm.reg_class().unwrap() {
-                    RegClass::Xmm => vec![
-                        IsleConstructor::RetXmm,
-                        IsleConstructor::RetMemorySideEffect,
-                    ],
-                    RegClass::Gpr => vec![
-                        IsleConstructor::RetGpr,
-                        IsleConstructor::RetMemorySideEffect,
-                    ],
-                },
+                RegMem(rm) => {
+                    assert!(!format.eflags.is_read());
+                    match rm.reg_class().unwrap() {
+                        RegClass::Xmm => vec![
+                            IsleConstructor::RetXmm,
+                            IsleConstructor::RetMemorySideEffect,
+                        ],
+                        RegClass::Gpr => vec![
+                            IsleConstructor::RetGpr,
+                            IsleConstructor::RetMemorySideEffect,
+                        ],
+                    }
+                }
             },
         },
-        [one, two] => match (one.location.kind(), two.location.kind()) {
-            (FixedReg(_) | Reg(_), FixedReg(_) | Reg(_)) => {
-                vec![IsleConstructor::RetValueRegs]
-            }
-            (Reg(r), Mem(_)) | (Mem(_) | RegMem(_), Reg(r) | FixedReg(r)) => {
-                assert!(matches!(r.reg_class().unwrap(), RegClass::Gpr));
-                vec![IsleConstructor::RetGpr]
+        [one, two] => {
+            assert!(!format.eflags.is_read());
+            match (one.location.kind(), two.location.kind()) {
+                (FixedReg(_) | Reg(_), FixedReg(_) | Reg(_)) => {
+                    vec![IsleConstructor::RetValueRegs]
+                }
+                (Reg(r), Mem(_)) | (Mem(_) | RegMem(_), Reg(r) | FixedReg(r)) => {
+                    assert!(matches!(r.reg_class().unwrap(), RegClass::Gpr));
+                    vec![IsleConstructor::RetGpr]
+                }
+                other => panic!("unsupported number of write operands {other:?}"),
             }
-            other => panic!("unsupported number of write operands {other:?}"),
-        },
-        [one, two, three] => match (
-            one.location.kind(),
-            two.location.kind(),
-            three.location.kind(),
-        ) {
-            (FixedReg(_), FixedReg(_), Mem(_)) => {
-                vec![IsleConstructor::RetValueRegs]
+        }
+        [one, two, three] => {
+            assert!(!format.eflags.is_read());
+            match (
+                one.location.kind(),
+                two.location.kind(),
+                three.location.kind(),
+            ) {
+                (FixedReg(_), FixedReg(_), Mem(_)) => {
+                    vec![IsleConstructor::RetValueRegs]
+                }
+                other => panic!("unsupported number of write operands {other:?}"),
             }
-            other => panic!("unsupported number of write operands {other:?}"),
-        },
+        }
 
         other => panic!("unsupported number of write operands {other:?}"),
     }
@@ -563,7 +592,9 @@ fn generate_isle_inst_decls(f: &mut Formatter, inst: &Inst) {
                     IsleConstructor::RetMemorySideEffect | IsleConstructor::NoReturnSideEffect => {
                         unreachable!()
                     }
-                    IsleConstructor::RetGpr => "(temp_writable_gpr)",
+                    IsleConstructor::RetGpr | IsleConstructor::ConsumesFlagsReturnsGpr => {
+                        "(temp_writable_gpr)"
+                    }
                     IsleConstructor::RetXmm => "(temp_writable_xmm)",
                     IsleConstructor::RetValueRegs | IsleConstructor::ProducesFlagsSideEffect => {
                         todo!()