Introduce count{l,r}_{zero,one} for batch_bool

In #1236, it was mentioned that variable-sized bit groups for certain
`batch_bool` reductions would be slightly more efficient than extracting
a proper bitmask.  To achieve this, the xsimd API is extended with the
functions `xsimd::count{l,r}_{zero,one}`, and `count` is revised to
allow per-platform kernels.  The default implementations for each
function simply apply the corresponding scalar operation (for which
`__cpp_lib_bitops == 201907L` is partially backported) on
`batch_bool::mask`.  This is specialized for NEON(64) by instead
applying the scalar operation to the narrowed batch, then scaling the
result by the bit group size.

Author: onalante-ebay

include/xsimd/arch/common/xsimd_common_bit.hpp

@@ -0,0 +1,226 @@
+/****************************************************************
+ * Partial backport of `__cpp_lib_bitops == 201907L` from C++20 *
+ ****************************************************************/
+
+#ifndef XSIMD_BIT_HPP
+#define XSIMD_BIT_HPP
+
+#include <version>
+
+#if __cpp_lib_bitops >= 201907L
+
+#include <bit>
+
+namespace xsimd
+{
+    namespace detail
+    {
+        using std::countl_one;
+        using std::countl_zero;
+        using std::countr_one;
+        using std::countr_zero;
+        using std::popcount;
+    }
+}
+
+#else
+
+#include <climits>
+#include <type_traits>
+
+#ifdef __has_builtin
+#define XSIMD_HAS_BUILTIN(x) __has_builtin(x)
+#else
+#define XSIMD_HAS_BUILTIN(x) 0
+#endif
+
+#ifdef _MSC_VER
+#include <intrin.h>
+#endif
+
+namespace xsimd
+{
+    namespace detail
+    {
+        // FIXME: We could do better by dispatching to the appropriate popcount instruction
+        // depending on the arch.
+
+        template <class T, class = std::enable_if_t<std::is_unsigned<T>::value>>
+        XSIMD_INLINE int popcount(T x) noexcept
+        {
+#if XSIMD_HAS_BUILTIN(__builtin_popcountg)
+            return __builtin_popcountg(x);
+#else
+            XSIMD_IF_CONSTEXPR(sizeof(T) == 1)
+            {
+#if XSIMD_HAS_BUILTIN(__builtin_popcount)
+                return __builtin_popcount(x);
+#elif defined(_MSC_VER)
+                return __popcnt(x);
+#else
+                // https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSet64
+                return ((uint64_t)x * 0x200040008001ULL & 0x111111111111111ULL) % 0xf;
+#endif
+            }
+            else XSIMD_IF_CONSTEXPR(sizeof(T) == 2)
+            {
+#if XSIMD_HAS_BUILTIN(__builtin_popcount)
+                return __builtin_popcount(x);
+#elif defined(_MSC_VER)
+                return __popcnt16(x);
+#else
+                // https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSet64
+                constexpr unsigned long long msb12 = 0x1001001001001ULL;
+                constexpr unsigned long long mask5 = 0x84210842108421ULL;
+
+                unsigned int v = (unsigned int)x;
+
+                return ((v & 0xfff) * msb12 & mask5) % 0x1f
+                    + (((v & 0xfff000) >> 12) * msb12 & mask5) % 0x1f;
+#endif
+            }
+            else XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
+            {
+#if XSIMD_HAS_BUILTIN(__builtin_popcount)
+                return __builtin_popcount(x);
+#elif defined(_MSC_VER)
+                return __popcnt(x);
+#else
+                // https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
+                x = x - ((x >> 1) & (T) ~(T)0 / 3);
+                x = (x & (T) ~(T)0 / 15 * 3) + ((x >> 2) & (T) ~(T)0 / 15 * 3);
+                x = (x + (x >> 4)) & (T) ~(T)0 / 255 * 15;
+                return (x * ((T) ~(T)0 / 255)) >> (sizeof(T) - 1) * CHAR_BIT;
+#endif
+            }
+            else
+            {
+                // sizeof(T) == 8
+#if XSIMD_HAS_BUILTIN(__builtin_popcountll)
+                return __builtin_popcountll(x);
+#elif XSIMD_HAS_BUILTIN(__builtin_popcount)
+                return __builtin_popcount((unsigned int)x) + __builtin_popcount((unsigned int)(x >> 32));
+#elif defined(_MSC_VER)
+#ifdef _M_X64
+                return (int)__popcnt64(x);
+#else
+                return (int)(__popcnt((unsigned int)x) + __popcnt((unsigned int)(x >> 32)));
+#endif
+#else
+                // https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
+                x = x - ((x >> 1) & (T) ~(T)0 / 3);
+                x = (x & (T) ~(T)0 / 15 * 3) + ((x >> 2) & (T) ~(T)0 / 15 * 3);
+                x = (x + (x >> 4)) & (T) ~(T)0 / 255 * 15;
+                return (x * ((T) ~(T)0 / 255)) >> (sizeof(T) - 1) * CHAR_BIT;
+#endif
+            }
+#endif
+        }
+
+        template <class T, class = std::enable_if_t<std::is_unsigned<T>::value>>
+        XSIMD_INLINE int countl_zero(T x) noexcept
+        {
+#if XSIMD_HAS_BUILTIN(__builtin_clzg)
+            return __builtin_clzg(x, (int)(sizeof(T) * CHAR_BIT));
+#else
+            if (x == 0)
+                return sizeof(T) * CHAR_BIT;
+
+            XSIMD_IF_CONSTEXPR(sizeof(T) <= 4)
+            {
+#if XSIMD_HAS_BUILTIN(__builtin_clz)
+                return __builtin_clz((unsigned int)x) - (4 - sizeof(T)) * CHAR_BIT;
+#elif defined(_MSC_VER)
+                unsigned long index;
+                _BitScanReverse(&index, (unsigned long)x);
+                return sizeof(T) * CHAR_BIT - index - 1;
+#else
+                x |= x >> 1;
+                x |= x >> 2;
+                x |= x >> 4;
+                XSIMD_IF_CONSTEXPR(sizeof(T) >= 2)
+                {
+                    x |= x >> 8;
+                }
+                XSIMD_IF_CONSTEXPR(sizeof(T) >= 4)
+                {
+                    x |= x >> 16;
+                }
+                return sizeof(T) * CHAR_BIT - popcount(x);
+#endif
+            }
+            else
+            {
+                // sizeof(T) == 8
+#if XSIMD_HAS_BUILTIN(__builtin_clzll)
+                return __builtin_clzll((unsigned long long)x);
+#elif defined(_MSC_VER) && defined(_M_X64)
+                unsigned long index;
+                _BitScanReverse64(&index, (unsigned long long)x);
+                return sizeof(T) * CHAR_BIT - index - 1;
+#else
+                x |= x >> 1;
+                x |= x >> 2;
+                x |= x >> 4;
+                x |= x >> 8;
+                x |= x >> 16;
+                x |= x >> 32;
+                return sizeof(T) * CHAR_BIT - popcount(x);
+#endif
+            }
+#endif
+        }
+
+        template <class T, class = std::enable_if_t<std::is_unsigned<T>::value>>
+        XSIMD_INLINE int countl_one(T x) noexcept
+        {
+            return countl_zero(T(~x));
+        }
+
+        template <class T, class = std::enable_if_t<std::is_unsigned<T>::value>>
+        XSIMD_INLINE int countr_zero(T x) noexcept
+        {
+#if XSIMD_HAS_BUILTIN(__builtin_ctzg)
+            return __builtin_ctzg(x, (int)(sizeof(T) * CHAR_BIT));
+#else
+            if (x == 0)
+                return sizeof(T) * CHAR_BIT;
+
+            XSIMD_IF_CONSTEXPR(sizeof(T) <= 4)
+            {
+#if XSIMD_HAS_BUILTIN(__builtin_ctz)
+                return __builtin_ctz((unsigned int)x);
+#elif defined(_MSC_VER)
+                unsigned long index;
+                _BitScanForward(&index, (unsigned long)x);
+                return index;
+#endif
+            }
+            else
+            {
+                // sizeof(T) == 8
+#if XSIMD_HAS_BUILTIN(__builtin_ctzll)
+                return __builtin_ctzll((unsigned long long)x);
+#elif defined(_MSC_VER) && defined(_M_X64)
+                unsigned long index;
+                _BitScanForward64(&index, (unsigned long long)x);
+                return index;
+#endif
+            }
+
+            // https://graphics.stanford.edu/~seander/bithacks.html#ZerosOnRightMultLookup
+            return popcount((T)((x & -x) - 1));
+#endif
+        }
+
+        template <class T, class = std::enable_if_t<std::is_unsigned<T>::value>>
+        XSIMD_INLINE int countr_one(T x) noexcept
+        {
+            return countr_zero(T(~x));
+        }
+
+    }
+}
+
+#endif
+#endif

include/xsimd/arch/common/xsimd_common_logical.hpp

@@ -12,6 +12,7 @@
 #ifndef XSIMD_COMMON_LOGICAL_HPP
 #define XSIMD_COMMON_LOGICAL_HPP
 
+#include "./xsimd_common_bit.hpp"
 #include "./xsimd_common_details.hpp"
 
 #include <climits>
@@ -28,43 +29,37 @@ namespace xsimd
         template <class A, class T>
         XSIMD_INLINE size_t count(batch_bool<T, A> const& self, requires_arch<common>) noexcept
         {
-            uint64_t m = self.mask();
-            XSIMD_IF_CONSTEXPR(batch_bool<T, A>::size < 14)
-            {
-                // https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSet64
-                return (m * 0x200040008001ULL & 0x111111111111111ULL) % 0xf;
-            }
-            else
-            {
-#if defined __has_builtin
-#if __has_builtin(__builtin_popcountg)
-#define builtin_popcount(v) __builtin_popcountg(v)
-#endif
-#endif
+            return xsimd::detail::popcount(self.mask());
+        }
 
-#ifdef builtin_popcount
-                return builtin_popcount(m);
-#else
-                // FIXME: we could do better by dispatching to the appropriate
-                // popcount instruction depending on the arch...
-                XSIMD_IF_CONSTEXPR(batch_bool<T, A>::size <= 32)
-                {
-                    uint32_t m32 = static_cast<uint32_t>(m);
-                    // https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
-                    m32 = m32 - ((m32 >> 1) & 0x55555555); // reuse input as temporary
-                    m32 = (m32 & 0x33333333) + ((m32 >> 2) & 0x33333333); // temp
-                    return (((m32 + (m32 >> 4)) & 0xF0F0F0F) * 0x1010101) >> 24; // count
-                }
-                else
-                {
-                    // https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
-                    m = m - ((m >> 1) & (uint64_t) ~(uint64_t)0 / 3); // temp
-                    m = (m & (uint64_t) ~(uint64_t)0 / 15 * 3) + ((m >> 2) & (uint64_t) ~(uint64_t)0 / 15 * 3); // temp
-                    m = (m + (m >> 4)) & (uint64_t) ~(uint64_t)0 / 255 * 15; // temp
-                    return (m * ((uint64_t) ~(uint64_t)0 / 255)) >> (sizeof(uint64_t) - 1) * CHAR_BIT; // count
-                }
-#endif
-            }
+        template <class A, class T>
+        XSIMD_INLINE size_t countl_zero(batch_bool<T, A> const& self, requires_arch<common>) noexcept
+        {
+            constexpr size_t unused_bits = 64 - batch_bool<T, A>::size;
+            constexpr uint64_t lower_mask = batch_bool<T, A>::size < 64 ? ((uint64_t)1 << (batch_bool<T, A>::size % 64)) - 1 : (uint64_t)-1;
+            return xsimd::detail::countl_zero(self.mask() & lower_mask) - unused_bits;
+        }
+
+        template <class A, class T>
+        XSIMD_INLINE size_t countl_one(batch_bool<T, A> const& self, requires_arch<common>) noexcept
+        {
+            constexpr size_t unused_bits = 64 - batch_bool<T, A>::size;
+            constexpr uint64_t upper_mask = batch_bool<T, A>::size < 64 ? ~(((uint64_t)1 << (batch_bool<T, A>::size % 64)) - 1) : (uint64_t)0;
+            return xsimd::detail::countl_one(self.mask() | upper_mask) - unused_bits;
+        }
+
+        template <class A, class T>
+        XSIMD_INLINE size_t countr_zero(batch_bool<T, A> const& self, requires_arch<common>) noexcept
+        {
+            constexpr uint64_t stop = batch_bool<T, A>::size < 64 ? (uint64_t)1 << (batch_bool<T, A>::size % 64) : 0;
+            return xsimd::detail::countr_zero(self.mask() | stop);
+        }
+
+        template <class A, class T>
+        XSIMD_INLINE size_t countr_one(batch_bool<T, A> const& self, requires_arch<common>) noexcept
+        {
+            constexpr uint64_t stop = batch_bool<T, A>::size < 64 ? ~((uint64_t)1 << (batch_bool<T, A>::size % 64)) : (uint64_t)-1;
+            return xsimd::detail::countr_one(self.mask() & stop);
         }
 
         // from  mask

include/xsimd/arch/xsimd_common.hpp

@@ -13,6 +13,7 @@
 #define XSIMD_COMMON_HPP
 
 #include "./common/xsimd_common_arithmetic.hpp"
+#include "./common/xsimd_common_bit.hpp"
 #include "./common/xsimd_common_cast.hpp"
 #include "./common/xsimd_common_complex.hpp"
 #include "./common/xsimd_common_logical.hpp"