Add x86 optimizations

Author: AntoinePrv

include/xsimd/arch/utils/shifts.hpp

@@ -0,0 +1,72 @@
+/***************************************************************************
+ * Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and         *
+ * Martin Renou                                                             *
+ * Copyright (c) QuantStack                                                 *
+ * Copyright (c) Serge Guelton                                              *
+ * Copyright (c) Marco Barbone                                              *
+ *                                                                          *
+ * Distributed under the terms of the BSD 3-Clause License.                 *
+ *                                                                          *
+ * The full license is in the file LICENSE, distributed with this software. *
+ ****************************************************************************/
+
+#ifndef XSIMD_UTILS_SHIFTS_HPP
+#define XSIMD_UTILS_SHIFTS_HPP
+
+#include "../../config/xsimd_inline.hpp"
+#include "../../types/xsimd_batch.hpp"
+#include "../../types/xsimd_batch_constant.hpp"
+
+namespace xsimd
+{
+    namespace kernel
+    {
+        namespace utils
+        {
+            template <typename I, I offset, I length, I... Vs>
+            struct select_stride
+            {
+                static constexpr I values_array[] = { Vs... };
+
+                template <typename K>
+                static constexpr K get(K i, K)
+                {
+                    return static_cast<K>(values_array[length * i + offset]);
+                }
+            };
+
+            template <typename I>
+            constexpr I lsb_mask(I bit_index)
+            {
+                return static_cast<I>((I { 1 } << bit_index) - I { 1 });
+            }
+
+            template <class T, class T2, class A, T... Vs>
+            XSIMD_INLINE batch<T, A> bitwise_lshift_as_twice_larger(
+                batch<T, A> const& self, batch_constant<T, A, Vs...>) noexcept
+            {
+                static_assert(sizeof(T2) == 2 * sizeof(T), "One size must be twice the other");
+
+                const auto self2 = bitwise_cast<T2>(self);
+
+                // Lower byte: shift as twice the size and mask bits flowing to higher byte.
+                constexpr auto shifts_lo = make_batch_constant<T2, select_stride<T, 0, 2, Vs...>, A>();
+                constexpr auto mask_lo = lsb_mask<T2>(8 * sizeof(T));
+                const auto shifted_lo = bitwise_lshift(self2, shifts_lo);
+                constexpr auto batch_mask_lo = make_batch_constant<T2, mask_lo, A>();
+                const auto masked_lo = bitwise_and(shifted_lo, batch_mask_lo.as_batch());
+
+                // Higher byte: mask bits that would flow from lower byte and shift as twice the size.
+                constexpr auto shifts_hi = make_batch_constant<T2, select_stride<T, 1, 2, Vs...>, A>();
+                constexpr auto mask_hi = mask_lo << (8 * sizeof(T));
+                constexpr auto batch_mask_hi = make_batch_constant<T2, mask_hi, A>();
+                const auto masked_hi = bitwise_and(self2, batch_mask_hi.as_batch());
+                const auto shifted_hi = bitwise_lshift(masked_hi, shifts_hi);
+
+                return bitwise_cast<T>(bitwise_or(masked_lo, shifted_hi));
+            }
+        }
+    }
+}
+
+#endif

include/xsimd/arch/xsimd_avx2.hpp

@@ -17,6 +17,7 @@
 
 #include "../types/xsimd_avx2_register.hpp"
 #include "../types/xsimd_batch_constant.hpp"
+#include "./utils/shifts.hpp"
 
 #include <limits>
 
@@ -332,6 +333,30 @@ namespace xsimd
             }
         }
 
+        // bitwise_lshift multiple (constant) specific implementations.
+        // Missing implementations are dispacthed to the `batch` overload in xsimd_api.
+        template <class T, class A, T... Vs, detail::enable_sized_integral_t<T, 2> = 0>
+        XSIMD_INLINE batch<T, A> bitwise_lshift(
+            batch<T, A> const& self, batch_constant<T, A, Vs...>, requires_arch<avx2>) noexcept
+        {
+            using uint_t = typename std::make_unsigned<T>::type;
+            return bitwise_cast<T>(
+                utils::bitwise_lshift_as_twice_larger<uint_t, uint32_t>(
+                    bitwise_cast<uint_t>(self),
+                    batch_constant<uint_t, A, static_cast<uint_t>(Vs)...> {}));
+        }
+
+        template <class T, class A, T... Vs, detail::enable_sized_integral_t<T, 1> = 0>
+        XSIMD_INLINE batch<T, A> bitwise_lshift(
+            batch<T, A> const& self, batch_constant<T, A, Vs...>, requires_arch<avx2>) noexcept
+        {
+            using uint_t = typename std::make_unsigned<T>::type;
+            return bitwise_cast<T>(
+                utils::bitwise_lshift_as_twice_larger<uint_t, uint16_t>(
+                    bitwise_cast<uint_t>(self),
+                    batch_constant<uint_t, A, static_cast<uint_t>(Vs)...> {}));
+        }
+
         // bitwise_or
         template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
         XSIMD_INLINE batch<T, A> bitwise_or(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx2>) noexcept

include/xsimd/arch/xsimd_sse2.hpp

@@ -18,6 +18,7 @@
 
 #include "../types/xsimd_batch_constant.hpp"
 #include "../types/xsimd_sse2_register.hpp"
+#include "./utils/shifts.hpp"
 
 namespace xsimd
 {
@@ -326,6 +327,22 @@ namespace xsimd
             return bitwise_lshift<shift>(self, common {});
         }
 
+        // bitwise_lshift multiple (constant)
+        template <class A, uint16_t... Vs>
+        XSIMD_INLINE batch<uint16_t, A> bitwise_lshift(
+            batch<uint16_t, A> const& self, batch_constant<uint16_t, A, Vs...>, requires_arch<sse2>) noexcept
+        {
+            constexpr auto mults = batch_constant<uint16_t, A, static_cast<uint16_t>(1u << Vs)...>();
+            return _mm_mullo_epi16(self, mults.as_batch());
+        }
+
+        template <class A, uint8_t... Vs>
+        XSIMD_INLINE batch<uint8_t, A> bitwise_lshift(
+            batch<uint8_t, A> const& self, batch_constant<uint8_t, A, Vs...> shifts, requires_arch<sse2>) noexcept
+        {
+            return utils::bitwise_lshift_as_twice_larger<uint8_t, uint16_t>(self, shifts);
+        }
+
         // bitwise_not
         template <class A>
         XSIMD_INLINE batch<float, A> bitwise_not(batch<float, A> const& self, requires_arch<sse2>) noexcept

include/xsimd/arch/xsimd_sse4_1.hpp

@@ -41,6 +41,15 @@ namespace xsimd
             return _mm_ceil_pd(self);
         }
 
+        // bitwise_lshift multiple (constant)
+        template <class A, uint32_t... Vs>
+        XSIMD_INLINE batch<uint32_t, A> bitwise_lshift(
+            batch<uint32_t, A> const& self, batch_constant<uint32_t, A, Vs...>, requires_arch<sse4_1>) noexcept
+        {
+            constexpr auto mults = batch_constant<uint32_t, A, static_cast<uint32_t>(1u << Vs)...>();
+            return _mm_mullo_epi32(self, mults.as_batch());
+        }
+
         // fast_cast
         namespace detail
         {