upstream merge

.github/workflows/main.yml

@@ -778,7 +778,7 @@ jobs:
   test_wasi_nn:
     strategy:
       matrix:
-        feature: ["openvino"]
+        feature: ["openvino", "onnx-download"]
         os: ["ubuntu-24.04", "windows-2025"]
         include:
           - os: windows-2025
@@ -794,7 +794,7 @@ jobs:
     - uses: ./.github/actions/install-rust
 
     # Install OpenVINO
-    - uses: abrown/install-openvino-action@v9
+    - uses: abrown/install-openvino-action@v10
       if: runner.arch == 'X64'
 
     # Install WinML for testing wasi-nn WinML backend. WinML is only available
@@ -835,7 +835,7 @@ jobs:
     needs: determine
     if: needs.determine.outputs.run-dwarf
     name: Test DWARF debugging
-    runs-on: ubuntu-22.04 # FIXME: fails on `ubuntu-24.04` right now
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@v4
       with:
@@ -848,14 +848,14 @@ jobs:
         tar -xzf wasi-sdk-25.0-x86_64-linux.tar.gz
         mv wasi-sdk-25.0-x86_64-linux wasi-sdk
     - run: |
-        sudo apt-get update && sudo apt-get install -y gdb lldb-15 llvm
+        sudo apt-get update && sudo apt-get install -y gdb lldb-18 llvm
         # workaround for https://bugs.launchpad.net/ubuntu/+source/llvm-defaults/+bug/1972855
         sudo mkdir -p /usr/lib/local/lib/python3.10/dist-packages/lldb
         sudo ln -s /usr/lib/llvm-15/lib/python3.10/dist-packages/lldb/* /usr/lib/python3/dist-packages/lldb/
         # Only testing release since it is more likely to expose issues with our low-level symbol handling.
         cargo test --release --test all -- --ignored --test-threads 1 debug::
       env:
-        LLDB: lldb-15 # override default version, 14
+        LLDB: lldb-18
         WASI_SDK_PATH: /tmp/wasi-sdk
 
   build-preview1-component-adapter:

cranelift/assembler-x64/meta/src/dsl/features.rs

@@ -38,12 +38,10 @@ impl Features {
     }
 
     pub(crate) fn is_sse(&self) -> bool {
-        self.0.iter().any(|f| {
-            matches!(
-                f,
-                Feature::sse | Feature::sse2 | Feature::ssse3 | Feature::sse41
-            )
-        })
+        use Feature::*;
+        self.0
+            .iter()
+            .any(|f| matches!(f, sse | sse2 | sse3 | ssse3 | sse41 | sse42))
     }
 }
 
@@ -86,6 +84,7 @@ pub enum Feature {
     lzcnt,
     popcnt,
     avx,
+    avx2,
     cmpxchg16b,
 }
 
@@ -110,6 +109,7 @@ pub const ALL_FEATURES: &[Feature] = &[
     Feature::lzcnt,
     Feature::popcnt,
     Feature::avx,
+    Feature::avx2,
     Feature::cmpxchg16b,
 ];
 

cranelift/assembler-x64/meta/src/dsl/format.rs

@@ -333,6 +333,7 @@ pub enum Location {
     xmm1,
     xmm2,
     xmm3,
+    xmm_m8,
     xmm_m16,
     xmm_m32,
     xmm_m64,
@@ -352,7 +353,7 @@ impl Location {
     pub fn bits(&self) -> u16 {
         use Location::*;
         match self {
-            al | cl | imm8 | r8 | rm8 | m8 => 8,
+            al | cl | imm8 | r8 | rm8 | m8 | xmm_m8 => 8,
             ax | dx | imm16 | r16 | rm16 | m16 | xmm_m16 => 16,
             eax | edx | imm32 | r32 | r32a | r32b | rm32 | m32 | xmm_m32 => 32,
             rax | rbx | rcx | rdx | imm64 | r64 | r64a | r64b | rm64 | m64 | xmm_m64 => 64,
@@ -399,7 +400,7 @@ impl Location {
             r8 | r16 | r32 | r32a | r32b | r64 | r64a | r64b | xmm1 | xmm2 | xmm3 => {
                 OperandKind::Reg(*self)
             }
-            rm8 | rm16 | rm32 | rm64 | xmm_m16 | xmm_m32 | xmm_m64 | xmm_m128 => {
+            rm8 | rm16 | rm32 | rm64 | xmm_m8 | xmm_m16 | xmm_m32 | xmm_m64 | xmm_m128 => {
                 OperandKind::RegMem(*self)
             }
             m8 | m16 | m32 | m64 | m128 => OperandKind::Mem(*self),
@@ -417,7 +418,7 @@ impl Location {
             imm8 | imm16 | imm32 | imm64 | m8 | m16 | m32 | m64 | m128 => None,
             al | ax | eax | rax | rbx | cl | rcx | dx | edx | rdx | r8 | r16 | r32 | r32a
             | r32b | r64 | r64a | r64b | rm8 | rm16 | rm32 | rm64 => Some(RegClass::Gpr),
-            xmm1 | xmm2 | xmm3 | xmm_m16 | xmm_m32 | xmm_m64 | xmm_m128 | xmm0 => {
+            xmm1 | xmm2 | xmm3 | xmm_m8 | xmm_m16 | xmm_m32 | xmm_m64 | xmm_m128 | xmm0 => {
                 Some(RegClass::Xmm)
             }
         }
@@ -461,6 +462,7 @@ impl core::fmt::Display for Location {
             xmm1 => write!(f, "xmm1"),
             xmm2 => write!(f, "xmm2"),
             xmm3 => write!(f, "xmm3"),
+            xmm_m8 => write!(f, "xmm_m8"),
             xmm_m16 => write!(f, "xmm_m16"),
             xmm_m32 => write!(f, "xmm_m32"),
             xmm_m64 => write!(f, "xmm_m64"),
@@ -598,6 +600,26 @@ pub enum Eflags {
     RW,
 }
 
+impl Eflags {
+    /// Returns whether this represents a read of any bit in the EFLAGS
+    /// register.
+    pub fn is_read(&self) -> bool {
+        match self {
+            Eflags::None | Eflags::W => false,
+            Eflags::R | Eflags::RW => true,
+        }
+    }
+
+    /// Returns whether this represents a writes to any bit in the EFLAGS
+    /// register.
+    pub fn is_write(&self) -> bool {
+        match self {
+            Eflags::None | Eflags::R => false,
+            Eflags::W | Eflags::RW => true,
+        }
+    }
+}
+
 impl Default for Eflags {
     fn default() -> Self {
         Self::None

cranelift/assembler-x64/meta/src/generate/format.rs

@@ -274,26 +274,34 @@ impl dsl::Format {
                     assert!(!vex.is4);
                     let reg = reg_or_vvvv;
                     fmtln!(f, "let reg = self.{reg}.enc();");
-                    fmtln!(f, "let vvvv = {};", "0b0");
                     fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");
-                    fmtln!(f, "let vex = VexPrefix::three_op(reg, vvvv, rm, {bits});");
+                    fmtln!(f, "let vex = VexPrefix::two_op(reg, rm, {bits});");
                     ModRmStyle::RegMem {
                         reg: ModRmReg::Reg(*reg),
                         rm: *rm,
                     }
                 }
             },
-            [Reg(reg), Reg(rm)] => {
+            [Reg(reg), Reg(rm)] | [Reg(reg), Reg(rm), Imm(_)] => {
                 assert!(!vex.is4);
                 fmtln!(f, "let reg = self.{reg}.enc();");
-                fmtln!(f, "let vvvv = 0;");
-                fmtln!(f, "let rm = (Some(self.{rm}.enc()), None);");
-                fmtln!(f, "let vex = VexPrefix::three_op(reg, vvvv, rm, {bits});");
+                fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");
+                fmtln!(f, "let vex = VexPrefix::two_op(reg, rm, {bits});");
                 ModRmStyle::Reg {
                     reg: ModRmReg::Reg(*reg),
                     rm: *rm,
                 }
             }
+            [Reg(reg), Mem(rm)] | [Mem(rm), Reg(reg)] | [RegMem(rm), Reg(reg), Imm(_)] => {
+                assert!(!vex.is4);
+                fmtln!(f, "let reg = self.{reg}.enc();");
+                fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");
+                fmtln!(f, "let vex = VexPrefix::two_op(reg, rm, {bits});");
+                ModRmStyle::RegMem {
+                    reg: ModRmReg::Reg(*reg),
+                    rm: *rm,
+                }
+            }
             unknown => unimplemented!("unknown pattern: {unknown:?}"),
         };
 

cranelift/assembler-x64/meta/src/generate/operand.rs

@@ -29,7 +29,7 @@ impl dsl::Operand {
             xmm1 | xmm2 | xmm3 => {
                 format!("Xmm<R::{mut_}Xmm>")
             }
-            xmm_m16 | xmm_m32 | xmm_m64 | xmm_m128 => {
+            xmm_m8 | xmm_m16 | xmm_m32 | xmm_m64 | xmm_m128 => {
                 format!("XmmMem<R::{mut_}Xmm, R::ReadGpr>")
             }
             m8 | m16 | m32 | m64 | m128 => format!("Amode<R::ReadGpr>"),
@@ -64,8 +64,8 @@ impl dsl::Location {
                     None => format!("self.{self}.to_string(None)"),
                 }
             }
-            xmm_m16 | xmm_m32 | xmm_m64 | xmm1 | xmm2 | xmm3 | xmm_m128 | m8 | m16 | m32 | m64
-            | m128 => {
+            xmm1 | xmm2 | xmm3 | xmm_m8 | xmm_m16 | xmm_m32 | xmm_m64 | xmm_m128 | m8 | m16
+            | m32 | m64 | m128 => {
                 format!("self.{self}.to_string()")
             }
         }
@@ -84,7 +84,7 @@ impl dsl::Location {
             m8 | m16 | m32 | m64 | m128 => {
                 panic!("no need to generate a size for memory-only access")
             }
-            xmm1 | xmm2 | xmm3 | xmm_m16 | xmm_m32 | xmm_m64 | xmm_m128 | xmm0 => None,
+            xmm1 | xmm2 | xmm3 | xmm_m8 | xmm_m16 | xmm_m32 | xmm_m64 | xmm_m128 | xmm0 => None,
         }
     }
 }

cranelift/assembler-x64/meta/src/instructions.rs

@@ -1,10 +1,13 @@
 //! Defines x64 instructions using the DSL.
 
+mod abs;
 mod add;
+mod align;
 mod and;
 mod atomic;
 mod avg;
 mod bitmanip;
+mod cmov;
 mod cmp;
 mod cvt;
 mod div;
@@ -17,6 +20,7 @@ mod mul;
 mod neg;
 mod nop;
 mod or;
+mod pack;
 mod round;
 mod shift;
 mod sqrt;
@@ -31,11 +35,14 @@ use std::collections::HashMap;
 #[must_use]
 pub fn list() -> Vec<Inst> {
     let mut all = vec![];
+    all.extend(abs::list());
     all.extend(add::list());
+    all.extend(align::list());
     all.extend(and::list());
     all.extend(atomic::list());
     all.extend(avg::list());
     all.extend(bitmanip::list());
+    all.extend(cmov::list());
     all.extend(cmp::list());
     all.extend(cvt::list());
     all.extend(div::list());
@@ -48,6 +55,7 @@ pub fn list() -> Vec<Inst> {
     all.extend(neg::list());
     all.extend(nop::list());
     all.extend(or::list());
+    all.extend(pack::list());
     all.extend(round::list());
     all.extend(shift::list());
     all.extend(sqrt::list());
@@ -74,7 +82,10 @@ fn check_avx_alternates(all: &mut [Inst]) {
         .map(|(index, inst)| (inst.name().clone(), index))
         .collect();
     for inst in all.iter().filter(|inst| inst.alternate.is_some()) {
-        assert!(inst.features.is_sse());
+        assert!(
+            inst.features.is_sse(),
+            "expected an SSE instruction: {inst}"
+        );
         let alternate = inst.alternate.as_ref().unwrap();
         assert_eq!(alternate.feature, Feature::avx);
         let avx_index = name_to_index.get(&alternate.name).expect(&format!(
@@ -116,16 +127,54 @@ fn check_sse_matches_avx(sse_inst: &Inst, avx_inst: &Inst) {
         // may have slightly different operand semantics (e.g., `roundss` ->
         // `vroundss`) and we want to be careful about matching too freely.
         (
-            [(ReadWrite, Reg(_)), (Read, Reg(_) | RegMem(_) | Mem(_))],
+            [
+                (ReadWrite | Write, Reg(_)),
+                (Read, Reg(_) | RegMem(_) | Mem(_)),
+            ],
             [
                 (Write, Reg(_)),
                 (Read, Reg(_)),
                 (Read, Reg(_) | RegMem(_) | Mem(_)),
             ],
         ) => {}
+        (
+            [
+                (Write, Reg(_) | RegMem(_) | Mem(_)),
+                (Read, Reg(_) | RegMem(_) | Mem(_)),
+            ],
+            [
+                (Write, Reg(_) | RegMem(_) | Mem(_)),
+                (Read, Reg(_) | RegMem(_) | Mem(_)),
+            ],
+        ) => {}
+        (
+            [
+                (Write, Reg(_) | RegMem(_)),
+                (Read, Reg(_) | RegMem(_)),
+                (Read, Imm(_)),
+            ],
+            [
+                (Write, Reg(_) | RegMem(_)),
+                (Read, Reg(_) | RegMem(_)),
+                (Read, Imm(_)),
+            ],
+        ) => {}
+        (
+            [(ReadWrite, Reg(_)), (Read, RegMem(_)), (Read, Imm(_))],
+            [
+                (Write, Reg(_)),
+                (Read, Reg(_)),
+                (Read, RegMem(_)),
+                (Read, Imm(_)),
+            ],
+        ) => {}
         // We panic on other formats for now; feel free to add more patterns to
         // avoid this.
-        _ => panic!("unmatched formats for SSE-to-AVX alternate:\n{sse_inst}\n{avx_inst}"),
+        _ => panic!(
+            "unmatched formats for SSE-to-AVX alternate:\n{sse_inst}\n{avx_inst}. {:?}, {:?}",
+            list_ops(sse_inst),
+            list_ops(avx_inst)
+        ),
     }
 }
 

cranelift/assembler-x64/meta/src/instructions/abs.rs

@@ -0,0 +1,14 @@
+use crate::dsl::{Feature::*, Inst, Location::*, VexLength::*};
+use crate::dsl::{align, fmt, inst, r, rex, vex, w};
+
+#[rustfmt::skip] // Keeps instructions on a single line.
+pub fn list() -> Vec<Inst> {
+    vec![
+        inst("pabsb", fmt("A", [w(xmm1), r(align(xmm_m128))]), rex([0x66, 0x0F, 0x38, 0x1C]), _64b | compat | ssse3).alt(avx, "vpabsb_a"),
+        inst("vpabsb", fmt("A", [w(xmm1), r(xmm_m128)]), vex(L128)._66()._0f38().op(0x1C), _64b | compat | avx),
+        inst("pabsw", fmt("A", [w(xmm1), r(align(xmm_m128))]), rex([0x66, 0x0F, 0x38, 0x1D]), _64b | compat | ssse3).alt(avx, "vpabsw_a"),
+        inst("vpabsw", fmt("A", [w(xmm1), r(xmm_m128)]), vex(L128)._66()._0f38().op(0x1D), _64b | compat | avx),
+        inst("pabsd", fmt("A", [w(xmm1), r(align(xmm_m128))]), rex([0x66, 0x0F, 0x38, 0x1E]), _64b | compat | ssse3).alt(avx, "vpabsd_a"),
+        inst("vpabsd", fmt("A", [w(xmm1), r(xmm_m128)]), vex(L128)._66()._0f38().op(0x1E), _64b | compat | avx),
+    ]
+}

cranelift/assembler-x64/meta/src/instructions/align.rs

@@ -0,0 +1,10 @@
+use crate::dsl::{Feature::*, Inst, Location::*, VexLength::*};
+use crate::dsl::{align, fmt, inst, r, rex, rw, vex, w};
+
+#[rustfmt::skip] // Keeps instructions on a single line.
+pub fn list() -> Vec<Inst> {
+    vec![
+        inst("palignr", fmt("A", [rw(xmm1), r(align(xmm_m128)), r(imm8)]), rex([0x66, 0x0F, 0x3A, 0x0F]).ib(), _64b | compat | ssse3).alt(avx, "vpalignr_b"),
+        inst("vpalignr", fmt("B", [w(xmm1), r(xmm2), r(xmm_m128), r(imm8)]), vex(L128)._66()._0f3a().ib().op(0x0F), _64b | compat | avx),
+     ]
+}