cmov: impl CmovEq for [u8; N] (#1353)

Similar to the `Cmov` implementation added in #1350, this adds a
`CmovEq` implementation for `[u8; N]` which breaks the byte slice up
into word-sized chunks, interprets them as a native word integer type
(either `u32` or `u64` ala #1350), then calls into the `CmovEq` for that
type, using a similar strategy for slices where we just iterate over the
array calling `CmovEq::cmovne` on the elements, and if any are non-equal
the condition will be moved.

This also simplifes the implementation of `Cmov` for `[u8; N]` by
vendoring the MSRV 1.88 core functions `[T]::as_chunks(_mut)` as
`utils::slice_as_chunks(_mut)`, and using those to write a simpler
implementation which can also benefit from receiving a word-sized byte
array instead of a slice, which can be passed directly to
`Word::from_ne_bytes`, avoiding the previous usage of potentially
panicking slice conversions.

As noted in the TODO, we can get rid of the vendored functions when we
bump MSRV to 1.88.

Author: tarcieri

cmov/src/array.rs

@@ -0,0 +1,55 @@
+use crate::{
+    Cmov, CmovEq, Condition,
+    utils::{slice_as_chunks, slice_as_chunks_mut},
+};
+
+// Uses 64-bit words on 64-bit targets, 32-bit everywhere else
+#[cfg(not(target_pointer_width = "64"))]
+type Word = u32;
+#[cfg(target_pointer_width = "64")]
+type Word = u64;
+const WORD_SIZE: usize = size_of::<Word>();
+
+/// Optimized implementation for byte arrays which coalesces them into word-sized chunks first,
+/// then performs [`Cmov`] at the word-level to cut down on the total number of instructions.
+///
+/// With compile-time knowledge of `N`, the compiler should also be able to unroll the loops in
+/// cases where efficiency would benefit, reducing the implementation to a sequence of word-sized
+/// [`Cmov`] ops (and if `N` isn't word-aligned, followed by a series of 1-byte ops).
+impl<const N: usize> Cmov for [u8; N] {
+    #[inline]
+    fn cmovnz(&mut self, value: &Self, condition: Condition) {
+        let (self_chunks, self_remainder) = slice_as_chunks_mut::<u8, WORD_SIZE>(self);
+        let (value_chunks, value_remainder) = slice_as_chunks::<u8, WORD_SIZE>(value);
+
+        for (self_chunk, value_chunk) in self_chunks.iter_mut().zip(value_chunks.iter()) {
+            let mut a = Word::from_ne_bytes(*self_chunk);
+            let b = Word::from_ne_bytes(*value_chunk);
+            a.cmovnz(&b, condition);
+            self_chunk.copy_from_slice(&a.to_ne_bytes());
+        }
+
+        // Process the remainder a byte-at-a-time.
+        for (a, b) in self_remainder.iter_mut().zip(value_remainder.iter()) {
+            a.cmovnz(b, condition);
+        }
+    }
+}
+
+impl<const N: usize> CmovEq for [u8; N] {
+    fn cmovne(&self, rhs: &Self, input: Condition, output: &mut Condition) {
+        let (self_chunks, self_remainder) = slice_as_chunks::<u8, WORD_SIZE>(self);
+        let (rhs_chunks, rhs_remainder) = slice_as_chunks::<u8, WORD_SIZE>(rhs);
+
+        for (self_chunk, rhs_chunk) in self_chunks.iter().zip(rhs_chunks.iter()) {
+            let a = Word::from_ne_bytes(*self_chunk);
+            let b = Word::from_ne_bytes(*rhs_chunk);
+            a.cmovne(&b, input, output);
+        }
+
+        // Process the remainder a byte-at-a-time.
+        for (a, b) in self_remainder.iter().zip(rhs_remainder.iter()) {
+            a.cmovne(b, input, output);
+        }
+    }
+}

cmov/src/lib.rs

@@ -28,11 +28,12 @@
 )]
 
 #[macro_use]
-mod macros;
+mod utils;
 
 #[cfg(not(miri))]
 #[cfg(target_arch = "aarch64")]
 mod aarch64;
+mod array;
 #[cfg(any(
     not(any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")),
     miri
@@ -49,16 +50,12 @@ pub type Condition = u8;
 pub trait Cmov {
     /// Move if non-zero.
     ///
-    /// Uses a `test` instruction to check if the given `condition` value is
-    /// equal to zero, conditionally moves `value` to `self` when `condition` is
-    /// not equal to zero.
+    /// Moves `value` to `self` in constant-time if `condition` is non-zero.
     fn cmovnz(&mut self, value: &Self, condition: Condition);
 
     /// Move if zero.
     ///
-    /// Uses a `cmp` instruction to check if the given `condition` value is
-    /// equal to zero, and if so, conditionally moves `value` to `self`
-    /// when `condition` is equal to zero.
+    /// Moves `value` to `self` in constant-time if `condition` is equal to zero.
     fn cmovz(&mut self, value: &Self, condition: Condition) {
         let nz = masknz!(condition: Condition);
         self.cmovnz(value, !nz)
@@ -67,21 +64,23 @@ pub trait Cmov {
 
 /// Conditional move with equality comparison
 pub trait CmovEq {
-    /// Move if both inputs are equal.
-    ///
-    /// Uses a `xor` instruction to compare the two values, and
-    /// conditionally moves `input` to `output` when they are equal.
-    fn cmoveq(&self, rhs: &Self, input: Condition, output: &mut Condition);
-
     /// Move if both inputs are not equal.
     ///
-    /// Uses a `xor` instruction to compare the two values, and
-    /// conditionally moves `input` to `output` when they are not equal.
+    /// Moves `input` to `output` in constant-time if `self` and `rhs` are NOT equal.
     fn cmovne(&self, rhs: &Self, input: Condition, output: &mut Condition) {
         let mut tmp = 1u8;
         self.cmoveq(rhs, 0u8, &mut tmp);
         tmp.cmoveq(&1u8, input, output);
     }
+
+    /// Move if both inputs are equal.
+    ///
+    /// Moves `input` to `output` in constant-time if `self` and `rhs` are equal.
+    fn cmoveq(&self, rhs: &Self, input: Condition, output: &mut Condition) {
+        let mut tmp = 1u8;
+        self.cmovne(rhs, 0u8, &mut tmp);
+        tmp.cmoveq(&1, input, output);
+    }
 }
 
 impl Cmov for u8 {
@@ -200,58 +199,7 @@ macro_rules! impl_cmov_traits_for_signed_ints {
 
 impl_cmov_traits_for_signed_ints!(i8 => u8, i16 => u16, i32 => u32, i64 => u64, i128 => u128);
 
-/// Optimized implementation for byte arrays which coalesces them into word-sized chunks first,
-/// then performs [`Cmov`] at the word-level to cut down on the total number of instructions.
-///
-/// With compile-time knowledge of `N`, the compiler should also be able to unroll the loops in
-/// cases where efficiency would benefit, reducing the implementation to a sequence of word-sized
-/// [`Cmov`] ops (and if `N` isn't word-aligned, followed by a series of 1-byte ops).
-impl<const N: usize> Cmov for [u8; N] {
-    #[inline]
-    fn cmovnz(&mut self, value: &Self, condition: Condition) {
-        // Uses 64-bit words on 64-bit targets, 32-bit everywhere else
-        #[cfg(not(target_pointer_width = "64"))]
-        type Chunk = u32;
-        #[cfg(target_pointer_width = "64")]
-        type Chunk = u64;
-        const CHUNK_SIZE: usize = size_of::<Chunk>();
-
-        // Load a chunk from a byte slice
-        // TODO(tarcieri): use `array_chunks` when stable (rust-lang/rust##100450)
-        #[inline]
-        fn load_chunk(slice: &[u8]) -> Chunk {
-            Chunk::from_ne_bytes(slice.try_into().expect("should be the right size"))
-        }
-
-        let mut self_chunks = self.chunks_exact_mut(CHUNK_SIZE);
-        let mut value_chunks = value.chunks_exact(CHUNK_SIZE);
-
-        // Process as much input as we can a `Chunk`-at-a-time.
-        for (self_chunk, value_chunk) in self_chunks.by_ref().zip(value_chunks.by_ref()) {
-            let mut a = load_chunk(self_chunk);
-            let b = load_chunk(value_chunk);
-            a.cmovnz(&b, condition);
-            self_chunk.copy_from_slice(&a.to_ne_bytes());
-        }
-
-        // Process the remainder a byte-at-a-time.
-        for (a, b) in self_chunks
-            .into_remainder()
-            .iter_mut()
-            .zip(value_chunks.remainder().iter())
-        {
-            a.cmovnz(b, condition);
-        }
-    }
-}
-
 impl<T: CmovEq> CmovEq for [T] {
-    fn cmoveq(&self, rhs: &Self, input: Condition, output: &mut Condition) {
-        let mut tmp = 1u8;
-        self.cmovne(rhs, 0u8, &mut tmp);
-        tmp.cmoveq(&1, input, output);
-    }
-
     fn cmovne(&self, rhs: &Self, input: Condition, output: &mut Condition) {
         // Short-circuit the comparison if the slices are of different lengths, and set the output
         // condition to the input condition.

cmov/src/macros.rs

@@ -0,0 +1,130 @@
+//! Macro definitions.
+
+use core::slice;
+
+/// Generates a mask the width of the input type if the input value is non-zero.
+///
+/// Uses `core::hint::black_box` to coerce our desired codegen based on real-world observations
+/// of the assembly generated by Rust/LLVM.
+///
+/// See also:
+/// - CVE-2026-23519
+/// - RustCrypto/utils#1332
+macro_rules! masknz {
+    ($value:tt : $int:ident) => {{
+        let mut value: $int = $value;
+        value |= value.wrapping_neg(); // has MSB `1` if non-zero, `0` if zero
+
+        // use `black_box` to obscure we're computing a 1-bit value
+        core::hint::black_box(
+            value >> ($int::BITS - 1), // Extract MSB
+        )
+        .wrapping_neg() // Generate $int::MAX mask if `black_box` outputs `1`
+    }};
+}
+
+/// Rust core `[T]::as_chunks` vendored because of its 1.88 MSRV.
+/// TODO(tarcieri): use upstream function when we bump MSRV
+#[inline]
+#[track_caller]
+#[must_use]
+#[allow(clippy::integer_division_remainder_used)]
+pub(crate) fn slice_as_chunks<T, const N: usize>(slice: &[T]) -> (&[[T; N]], &[T]) {
+    assert!(N != 0, "chunk size must be non-zero");
+    let len_rounded_down = slice.len() / N * N;
+    // SAFETY: The rounded-down value is always the same or smaller than the
+    // original length, and thus must be in-bounds of the slice.
+    let (multiple_of_n, remainder) = unsafe { slice.split_at_unchecked(len_rounded_down) };
+    // SAFETY: We already panicked for zero, and ensured by construction
+    // that the length of the subslice is a multiple of N.
+    let array_slice = unsafe { slice_as_chunks_unchecked(multiple_of_n) };
+    (array_slice, remainder)
+}
+
+/// Rust core `[T]::as_chunks_mut` vendored because of its 1.88 MSRV.
+/// TODO(tarcieri): use upstream function when we bump MSRV
+#[inline]
+#[track_caller]
+#[must_use]
+#[allow(clippy::integer_division_remainder_used)]
+pub(crate) fn slice_as_chunks_mut<T, const N: usize>(slice: &mut [T]) -> (&mut [[T; N]], &mut [T]) {
+    assert!(N != 0, "chunk size must be non-zero");
+    let len_rounded_down = slice.len() / N * N;
+    // SAFETY: The rounded-down value is always the same or smaller than the
+    // original length, and thus must be in-bounds of the slice.
+    let (multiple_of_n, remainder) = unsafe { slice.split_at_mut_unchecked(len_rounded_down) };
+    // SAFETY: We already panicked for zero, and ensured by construction
+    // that the length of the subslice is a multiple of N.
+    let array_slice = unsafe { slice_as_chunks_unchecked_mut(multiple_of_n) };
+    (array_slice, remainder)
+}
+
+/// Rust core `[T]::as_chunks_unchecked` vendored because of its 1.88 MSRV.
+/// TODO(tarcieri): use upstream function when we bump MSRV
+#[inline]
+#[must_use]
+#[track_caller]
+#[allow(clippy::integer_division_remainder_used)]
+unsafe fn slice_as_chunks_unchecked<T, const N: usize>(slice: &[T]) -> &[[T; N]] {
+    // SAFETY: Caller must guarantee that `N` is nonzero and exactly divides the slice length
+    const { debug_assert!(N != 0) };
+    debug_assert_eq!(slice.len() % N, 0);
+    let new_len = slice.len() / N;
+
+    // SAFETY: We cast a slice of `new_len * N` elements into
+    // a slice of `new_len` many `N` elements chunks.
+    unsafe { slice::from_raw_parts(slice.as_ptr().cast(), new_len) }
+}
+
+/// Rust core `[T]::as_chunks_unchecked_mut` vendored because of its 1.88 MSRV.
+/// TODO(tarcieri): use upstream function when we bump MSRV
+#[inline]
+#[must_use]
+#[track_caller]
+#[allow(clippy::integer_division_remainder_used)]
+unsafe fn slice_as_chunks_unchecked_mut<T, const N: usize>(slice: &mut [T]) -> &mut [[T; N]] {
+    // SAFETY: Caller must guarantee that `N` is nonzero and exactly divides the slice length
+    const { debug_assert!(N != 0) };
+    debug_assert_eq!(slice.len() % N, 0);
+    let new_len = slice.len() / N;
+
+    // SAFETY: We cast a slice of `new_len * N` elements into
+    // a slice of `new_len` many `N` elements chunks.
+    unsafe { slice::from_raw_parts_mut(slice.as_mut_ptr().cast(), new_len) }
+}
+
+#[cfg(test)]
+mod tests {
+    // Spot check up to a given limit
+    const TEST_LIMIT: u8 = 128;
+
+    macro_rules! masknz_test {
+        ( $($name:ident : $int:ident),+ ) => {
+            $(
+                #[test]
+                fn $name() {
+                    assert_eq!(masknz!(0: $int), 0);
+
+                    // Test lower values
+                    for i in 1..=$int::from(TEST_LIMIT) {
+                        assert_eq!(masknz!(i: $int), $int::MAX);
+                    }
+
+                    // Test upper values
+                    for i in ($int::MAX - $int::from(TEST_LIMIT))..=$int::MAX {
+                        assert_eq!(masknz!(i: $int), $int::MAX);
+                    }
+                }
+            )+
+        }
+    }
+
+    // Ensure the macro works with any types we might use it with (we only use u8, u32, and u64)
+    masknz_test!(
+        masknz_u8: u8,
+        masknz_u16: u16,
+        masknz_u32: u32,
+        masknz_u64: u64,
+        masknz_u128: u128
+    );
+}