cmov: add asm! optimized maskgen32 for ARM32

In #1332 we ran into LLVM inserting branches in this routine for
`thumbv6m-none-eabi` targets. It was "fixed" by fiddling around with
`black_box` but that seems brittle.

In #1334 we attempted a simple portable `asm!` optimization barrier
approach but it did not work as expected.

This instead opts to implement one of the fiddliest bits, mask
generation, using ARM assembly instead. The resulting assembly is
actually more efficient than what rustc/LLVM outputs and avoids touching
the stack pointer.

It's a simple enough function to implement in assembly on other
platforms with stable `asm!` too, but this is a start.

Author: tarcieri

cmov/src/portable.rs

@@ -100,15 +100,40 @@ impl CmovEq for u64 {
 }
 
 /// Return a [`u32::MAX`] mask if `condition` is non-zero, otherwise return zero for a zero input.
-pub fn nzmask32(condition: Condition) -> u32 {
+#[cfg(not(target_arch = "arm"))]
+fn nzmask32(condition: Condition) -> u32 {
     bitnz!(condition as u32, u32::BITS).wrapping_neg()
 }
 
 /// Return a [`u64::MAX`] mask if `condition` is non-zero, otherwise return zero for a zero input.
-pub fn nzmask64(condition: Condition) -> u64 {
+#[cfg(not(target_arch = "arm"))]
+fn nzmask64(condition: Condition) -> u64 {
     bitnz!(condition as u64, u64::BITS).wrapping_neg()
 }
 
+/// Optimized mask generation for ARM32 targets.
+#[cfg(target_arch = "arm")]
+fn nzmask32(condition: u8) -> u32 {
+    let mut out = condition as u32;
+    unsafe {
+        core::arch::asm!(
+        "uxtb  {0}, {0}",      // Extend 8-bit value to 32-bit
+        "rsbs  {0}, {0}, #0",  // Reverse subtract
+        "sbcs  {0}, {0}, {0}", // Subtract with carry, setting flags
+        inout(reg) out,
+        options(nostack, nomem),
+        );
+    }
+    out
+}
+
+/// 64-bit wrapper for targets that implement 32-bit mask generation in assembly.
+#[cfg(target_arch = "arm")]
+fn nzmask64(condition: u8) -> u64 {
+    let mask = nzmask32(condition) as u64;
+    mask | mask << 32
+}
+
 #[cfg(test)]
 mod tests {
     #[test]