fix: correct constant-mask load/store kernels and add shape predicates

The sse2 and neon constant-mask load_masked kernels were unreachable: they
lacked the convert<T> overload parameter the dispatcher passes, so calls fell
through to the runtime-mask path. Add the parameter and fix the mask
conditions, which used ambiguous count-based tests (e.g. countr_one()==2 also
matches 0b1011, silently dropping a lane).

Add batch_bool_constant::is_prefix/is_suffix shape predicates plus prefix()/
suffix() returning the set-run length (or size+1 when the mask is not that
shape, keeping == exact). The sse2 kernels now dispatch on these instead of
raw bit patterns, so each condition reads as the lane shape it handles.

avx_128 delegates prefix/upper-half masks to sse2: those lower to plain moves
(vmovss/movlps/…) which store-forward, whereas vmaskmov never does on Intel
and its stores are microcoded on AMD.

Tests cover the shape predicates including off-by-one boundaries (size-1 vs
the size+1 sentinel) and near-full mask patterns through the load/store
kernels.

Author: DiamonDinoia

include/xsimd/arch/xsimd_avx_128.hpp

@@ -103,11 +103,19 @@ namespace xsimd
             return _mm_cmp_pd(self, other, _CMP_NEQ_UQ);
         }
 
-        // constant masks gain nothing on a single register; forward to runtime
+        // Masks that lower to plain moves go to sse2; the rest gain nothing on a
+        // single register, so take the runtime path.
         template <class A, class T, bool... Values, class Mode, class = std::enable_if_t<std::is_floating_point<T>::value>>
         XSIMD_INLINE batch<T, A> load_masked(T const* mem, batch_bool_constant<T, A, Values...> mask, convert<T>, Mode, requires_arch<avx_128>) noexcept
         {
-            return load_masked(mem, mask.as_batch_bool(), convert<T> {}, Mode {}, avx_128 {});
+            XSIMD_IF_CONSTEXPR(detail::lowers_to_plain_moves(mask))
+            {
+                return load_masked(mem, mask, convert<T> {}, Mode {}, sse2 {});
+            }
+            else
+            {
+                return load_masked(mem, mask.as_batch_bool(), convert<T> {}, Mode {}, avx_128 {});
+            }
         }
 
         // Runtime-mask load (float/double).
@@ -142,7 +150,14 @@ namespace xsimd
         template <class A, class T, bool... Values, class Mode, class = std::enable_if_t<std::is_floating_point<T>::value>>
         XSIMD_INLINE void store_masked(T* mem, batch<T, A> const& src, batch_bool_constant<T, A, Values...> mask, Mode, requires_arch<avx_128>) noexcept
         {
-            store_masked(mem, src, mask.as_batch_bool(), Mode {}, avx_128 {});
+            XSIMD_IF_CONSTEXPR(detail::lowers_to_plain_moves(mask))
+            {
+                store_masked(mem, src, mask, Mode {}, sse2 {});
+            }
+            else
+            {
+                store_masked(mem, src, mask.as_batch_bool(), Mode {}, avx_128 {});
+            }
         }
 
         // Runtime-mask store (float/double).

include/xsimd/arch/xsimd_neon.hpp

@@ -539,8 +539,8 @@ namespace xsimd
             template <>
             struct load_masked<>
             {
-                template <size_t I, class A, class T, bool Use>
-                static XSIMD_INLINE batch<T, A> apply(T const* /* mem */, batch<T, A> acc, std::integral_constant<bool, Use>) noexcept
+                template <size_t I, class A, class T>
+                static XSIMD_INLINE batch<T, A> apply(T const* /* mem */, batch<T, A> acc) noexcept
                 {
                     return acc;
                 }
@@ -549,23 +549,21 @@ namespace xsimd
             struct load_masked<Value, Values...>
             {
                 template <size_t I, class A, class T>
-                static XSIMD_INLINE batch<T, A> apply(T const* mem, batch<T, A> acc, std::true_type) noexcept
-                {
-                    return load_masked<Values...>::template apply<I + 1>(mem, insert(acc, mem[I], index<I> {}), std::integral_constant<bool, Value> {});
-                }
-                template <size_t I, class A, class T>
-                static XSIMD_INLINE batch<T, A> apply(T const* mem, batch<T, A> acc, std::false_type) noexcept
+                static XSIMD_INLINE batch<T, A> apply(T const* mem, batch<T, A> acc) noexcept
                 {
-                    return load_masked<Values...>::template apply<I + 1>(mem, acc, std::integral_constant<bool, Value> {});
+                    XSIMD_IF_CONSTEXPR(Value)
+                    {
+                        acc = insert(acc, mem[I], index<I> {});
+                    }
+                    return load_masked<Values...>::template apply<I + 1>(mem, acc);
                 }
             };
         }
 
         template <class A, class T, bool Value, bool... Values, class Mode>
-        XSIMD_INLINE batch<T, A> load_masked(T const* mem, batch_bool_constant<T, A, Value, Values...> /* mask */, Mode, requires_arch<neon>) noexcept
+        XSIMD_INLINE batch<T, A> load_masked(T const* mem, batch_bool_constant<T, A, Value, Values...> /* mask */, convert<T>, Mode, requires_arch<neon>) noexcept
         {
-            // Call insert whenever Values... are true
-            return detail::load_masked<Values...>::template apply<0>(mem, broadcast(T(0), A {}), std::integral_constant<bool, Value> {});
+            return detail::load_masked<Value, Values...>::template apply<0>(mem, batch<T, A>(T(0)));
         }
 
         /*********

include/xsimd/arch/xsimd_sse2.hpp

@@ -1100,54 +1100,64 @@ namespace xsimd
             return { load_unaligned(mem, batch_bool<char, A> {}, r).data };
         }
 
+        namespace detail
+        {
+            // Plain moves store-forward; vmaskmov never does on Intel and its
+            // stores are microcoded on AMD. So wider archs delegate the masks
+            // that lower to plain moves here rather than taking their runtime path.
+            template <class T, class A, bool... Values>
+            constexpr bool lowers_to_plain_moves(batch_bool_constant<T, A, Values...> mask) noexcept
+            {
+                return (mask.is_prefix() && mask.any() && !mask.all())
+                    || mask.suffix() == mask.size / 2;
+            }
+        }
+
         // load_masked
         template <class A, class T, bool... Values, class Mode, class = std::enable_if_t<std::is_integral<T>::value>>
-        XSIMD_INLINE batch<T, A> load_masked(T const* mem, batch_bool_constant<T, A, Values...> mask, Mode, requires_arch<sse2>) noexcept
+        XSIMD_INLINE batch<T, A> load_masked(T const* mem, batch_bool_constant<T, A, Values...> mask, convert<T>, Mode, requires_arch<sse2>) noexcept
         {
-            XSIMD_IF_CONSTEXPR(mask.mask() == 0x1)
+            XSIMD_IF_CONSTEXPR(sizeof(T) == 2 && mask.prefix() == 1)
             {
-                XSIMD_IF_CONSTEXPR(sizeof(T) == 2)
-                {
-                    return mm_loadu_si16(mem);
-                }
-                XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
-                {
-                    return mm_loadu_si32(mem);
-                }
-                XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
-                {
-                    return mm_loadu_si64(mem);
-                }
+                return _mm_loadu_si16(mem);
+            }
+            else XSIMD_IF_CONSTEXPR(sizeof(T) == 4 && mask.prefix() == 1)
+            {
+                return _mm_loadu_si32(mem);
+            }
+            else XSIMD_IF_CONSTEXPR(sizeof(T) == 8 && mask.prefix() == 1)
+            {
+                return _mm_loadu_si64(mem);
             }
-            else XSIMD_IF_CONSTEXPR(sizeof(T) == 2 && mask.mask() == 0x3)
+            else XSIMD_IF_CONSTEXPR(sizeof(T) == 2 && mask.prefix() == 2)
             {
-                return mm_loadu_si32(mem);
+                return _mm_loadu_si32(mem);
             }
-            else XSIMD_IF_CONSTEXPR(sizeof(T) == 4 && mask.mask() == 0x3)
+            else XSIMD_IF_CONSTEXPR(sizeof(T) == 4 && mask.prefix() == 2)
             {
-                return mm_loadu_si64(mem);
+                return _mm_loadu_si64(mem);
             }
             else
             {
                 return load_masked<A>(mem, mask, convert<T> {}, Mode {}, common {});
             }
         }
         template <class A, bool... Values, class Mode>
-        XSIMD_INLINE batch<float, A> load_masked(float const* mem, batch_bool_constant<float, A, Values...> mask, Mode, requires_arch<sse2>) noexcept
+        XSIMD_INLINE batch<float, A> load_masked(float const* mem, batch_bool_constant<float, A, Values...> mask, convert<float>, Mode, requires_arch<sse2>) noexcept
         {
-            XSIMD_IF_CONSTEXPR(mask.mask() == 0x1)
+            XSIMD_IF_CONSTEXPR(mask.prefix() == 1)
             {
                 return _mm_load_ss(mem);
             }
-            else XSIMD_IF_CONSTEXPR(mask.countr_one() == 2)
+            else XSIMD_IF_CONSTEXPR(mask.prefix() == 2)
             {
                 return _mm_loadl_pi(_mm_setzero_ps(), reinterpret_cast<__m64 const*>(mem));
             }
-            else XSIMD_IF_CONSTEXPR(mask.countl_one() == 2)
+            else XSIMD_IF_CONSTEXPR(mask.suffix() == 2)
             {
                 return _mm_loadh_pi(_mm_setzero_ps(), reinterpret_cast<__m64 const*>(mem + 2));
             }
-            else XSIMD_IF_CONSTEXPR(mask.countr_one() == 3)
+            else XSIMD_IF_CONSTEXPR(mask.prefix() == 3)
             {
                 __m128 const lo2 = _mm_castsi128_ps(_mm_loadl_epi64(reinterpret_cast<__m128i const*>(mem)));
                 return _mm_shuffle_ps(lo2, _mm_load_ss(mem + 2), _MM_SHUFFLE(3, 0, 1, 0));
@@ -1158,13 +1168,13 @@ namespace xsimd
             }
         }
         template <class A, bool... Values, class Mode>
-        XSIMD_INLINE batch<double, A> load_masked(double const* mem, batch_bool_constant<double, A, Values...> mask, Mode, requires_arch<sse2>) noexcept
+        XSIMD_INLINE batch<double, A> load_masked(double const* mem, batch_bool_constant<double, A, Values...> mask, convert<double>, Mode, requires_arch<sse2>) noexcept
         {
-            XSIMD_IF_CONSTEXPR(mask.countr_one() == 1)
+            XSIMD_IF_CONSTEXPR(mask.prefix() == 1)
             {
                 return _mm_load_sd(mem);
             }
-            else XSIMD_IF_CONSTEXPR(mask.countl_one() == 1)
+            else XSIMD_IF_CONSTEXPR(mask.suffix() == 1)
             {
                 return _mm_loadh_pd(_mm_setzero_pd(), mem + 1);
             }
@@ -1178,19 +1188,19 @@ namespace xsimd
         template <class A, bool... Values, class Mode>
         XSIMD_INLINE void store_masked(float* mem, batch<float, A> const& src, batch_bool_constant<float, A, Values...> mask, Mode, requires_arch<sse2>) noexcept
         {
-            XSIMD_IF_CONSTEXPR(mask.mask() == 0x1)
+            XSIMD_IF_CONSTEXPR(mask.prefix() == 1)
             {
                 _mm_store_ss(mem, src);
             }
-            else XSIMD_IF_CONSTEXPR(mask.countr_one() == 2)
+            else XSIMD_IF_CONSTEXPR(mask.prefix() == 2)
             {
                 _mm_storel_pi(reinterpret_cast<__m64*>(mem), src);
             }
-            else XSIMD_IF_CONSTEXPR(mask.countl_one() == 2)
+            else XSIMD_IF_CONSTEXPR(mask.suffix() == 2)
             {
                 _mm_storeh_pi(reinterpret_cast<__m64*>(mem + 2), src);
             }
-            else XSIMD_IF_CONSTEXPR(mask.countr_one() == 3)
+            else XSIMD_IF_CONSTEXPR(mask.prefix() == 3)
             {
                 _mm_storel_pi(reinterpret_cast<__m64*>(mem), src);
                 _mm_store_ss(mem + 2, _mm_movehl_ps(src, src));
@@ -1204,11 +1214,11 @@ namespace xsimd
         template <class A, bool... Values, class Mode>
         XSIMD_INLINE void store_masked(double* mem, batch<double, A> const& src, batch_bool_constant<double, A, Values...> mask, Mode, requires_arch<sse2>) noexcept
         {
-            XSIMD_IF_CONSTEXPR(mask.countr_one() == 1)
+            XSIMD_IF_CONSTEXPR(mask.prefix() == 1)
             {
                 _mm_store_sd(mem, src);
             }
-            else XSIMD_IF_CONSTEXPR(mask.countl_one() == 1)
+            else XSIMD_IF_CONSTEXPR(mask.suffix() == 1)
             {
                 _mm_storeh_pd(mem + 1, src);
             }
@@ -2331,11 +2341,11 @@ namespace xsimd
                                        aligned_mode,
                                        requires_arch<sse2>) noexcept
         {
-            XSIMD_IF_CONSTEXPR(mask.countr_one() == 2)
+            XSIMD_IF_CONSTEXPR(mask.prefix() == 2)
             {
                 _mm_storel_pi(reinterpret_cast<__m64*>(mem), src);
             }
-            else XSIMD_IF_CONSTEXPR(mask.countl_one() == 2)
+            else XSIMD_IF_CONSTEXPR(mask.suffix() == 2)
             {
                 _mm_storeh_pi(reinterpret_cast<__m64*>(mem + 2), src);
             }

include/xsimd/types/xsimd_batch_constant.hpp

@@ -99,6 +99,34 @@ namespace xsimd
             return countl_one_impl(truncated_mask(), size);
         }
 
+        // true when the set lanes form one contiguous run starting at lane 0
+        // (the empty and full masks are prefixes)
+        static constexpr bool is_prefix() noexcept
+        {
+            return (truncated_mask() & (truncated_mask() + 1u)) == 0u;
+        }
+
+        // true when the set lanes form one contiguous run ending at the last
+        // lane (the empty and full masks are suffixes)
+        static constexpr bool is_suffix() noexcept
+        {
+            return ((truncated_mask() ^ low_mask(size)) & ((truncated_mask() ^ low_mask(size)) + 1u)) == 0u;
+        }
+
+        // length of the set run when the mask is a pure prefix (first k lanes
+        // set, rest clear), else size + 1 so `prefix() == k` stays exact
+        static constexpr std::size_t prefix() noexcept
+        {
+            return is_prefix() ? countr_one() : size + 1;
+        }
+
+        // length of the set run when the mask is a pure suffix (last k lanes
+        // set, rest clear), else size + 1 so `suffix() == k` stays exact
+        static constexpr std::size_t suffix() noexcept
+        {
+            return is_suffix() ? countl_one() : size + 1;
+        }
+
     private:
         static constexpr int mask_helper(int acc) noexcept { return acc; }
 

test/test_batch_constant.cpp

@@ -389,6 +389,76 @@ struct constant_bool_batch_test
         constexpr auto inv_x = ~x;
         static_assert(std::is_same<decltype(inv_x), decltype(y)>::value, "~x == y");
     }
+
+    struct first_half
+    {
+        static constexpr bool get(size_t index, size_t size) { return index < size / 2; }
+    };
+    struct second_half
+    {
+        static constexpr bool get(size_t index, size_t size) { return index >= size / 2; }
+    };
+    struct ends
+    {
+        static constexpr bool get(size_t index, size_t size) { return index == 0 || index + 1 == size; }
+    };
+    struct all_but_last
+    {
+        static constexpr bool get(size_t index, size_t size) { return index + 1 < size; }
+    };
+    struct all_but_first
+    {
+        static constexpr bool get(size_t index, size_t) { return index != 0; }
+    };
+    struct first_only
+    {
+        static constexpr bool get(size_t index, size_t) { return index == 0; }
+    };
+    struct last_only
+    {
+        static constexpr bool get(size_t index, size_t size) { return index + 1 == size; }
+    };
+
+    void test_shape() const
+    {
+        constexpr auto all_true = xsimd::make_batch_bool_constant<value_type, constant<true>, arch_type>();
+        constexpr auto all_false = xsimd::make_batch_bool_constant<value_type, constant<false>, arch_type>();
+        constexpr auto lo = xsimd::make_batch_bool_constant<value_type, first_half, arch_type>();
+        constexpr auto hi = xsimd::make_batch_bool_constant<value_type, second_half, arch_type>();
+        constexpr auto edges = xsimd::make_batch_bool_constant<value_type, ends, arch_type>();
+        constexpr auto size = decltype(all_true)::size;
+
+        static_assert(all_true.is_prefix() && all_true.is_suffix(), "full mask is prefix and suffix");
+        static_assert(all_false.is_prefix() && all_false.is_suffix(), "empty mask is prefix and suffix");
+        static_assert(lo.is_prefix() && !lo.is_suffix(), "first half is a prefix only");
+        static_assert(hi.is_suffix() && !hi.is_prefix(), "second half is a suffix only");
+        // {first, last} lanes: contiguous only when that covers the whole batch
+        static_assert(edges.is_prefix() == (size <= 2), "non-contiguous mask is not a prefix");
+        static_assert(edges.is_suffix() == (size <= 2), "non-contiguous mask is not a suffix");
+
+        // prefix()/suffix() return the set-run length, or size+1 when not that shape
+        static_assert(lo.prefix() == size / 2 && lo.suffix() == size + 1, "lo is the size/2 prefix, no suffix");
+        static_assert(hi.suffix() == size / 2 && hi.prefix() == size + 1, "hi is the size/2 suffix, no prefix");
+        static_assert(all_true.prefix() == size && all_true.suffix() == size, "full mask runs the whole width");
+        static_assert(all_false.prefix() == 0 && all_false.suffix() == 0, "empty mask has zero-length runs");
+        static_assert(edges.prefix() == (size <= 2 ? size : size + 1), "non-contiguous mask has no prefix length");
+
+        // off-by-one boundary: the length adjacent to the sentinel (size-1) must
+        // count exactly, and the sentinel must be exactly size+1 (never size, size+2)
+        constexpr auto pre1 = xsimd::make_batch_bool_constant<value_type, all_but_last, arch_type>();
+        constexpr auto suf1 = xsimd::make_batch_bool_constant<value_type, all_but_first, arch_type>();
+        constexpr auto f1 = xsimd::make_batch_bool_constant<value_type, first_only, arch_type>();
+        constexpr auto l1 = xsimd::make_batch_bool_constant<value_type, last_only, arch_type>();
+        static_assert(size < 2 || pre1.prefix() == size - 1, "size-1 prefix counts exactly");
+        static_assert(size < 2 || pre1.suffix() == size + 1, "a size-1 prefix is not a suffix");
+        static_assert(size < 2 || suf1.suffix() == size - 1, "size-1 suffix counts exactly");
+        static_assert(size < 2 || suf1.prefix() == size + 1, "a size-1 suffix is not a prefix");
+        static_assert(size < 2 || (pre1.suffix() != size && pre1.suffix() != size + 2), "sentinel is exactly size+1");
+        static_assert(f1.prefix() == 1, "single first lane is the 1-prefix");
+        static_assert(size < 2 || f1.suffix() == size + 1, "single first lane is not a suffix");
+        static_assert(l1.suffix() == 1, "single last lane is the 1-suffix");
+        static_assert(size < 2 || l1.prefix() == size + 1, "single last lane is not a prefix");
+    }
 };
 
 TEST_CASE_TEMPLATE("[constant bool batch]", B, BATCH_INT_TYPES)
@@ -410,5 +480,9 @@ TEST_CASE_TEMPLATE("[constant bool batch]", B, BATCH_INT_TYPES)
     {
         Test.test_ops();
     }
+    SUBCASE("shape")
+    {
+        Test.test_shape();
+    }
 }
 #endif