diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 3fae59208..f6cf9545b 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -94,7 +94,7 @@ jobs: - name: Setup compiler uses: ilammy/msvc-dev-cmd@v1 with: - arch: amd64 + arch: arm64 - name: Setup Ninja run: | python3 -m pip install --upgrade pip setuptools wheel @@ -107,3 +107,52 @@ jobs: run: cmake --build _build - name: Testing xsimd run: ./_build/test/test_xsimd + - name: Run benchmark + run: ./_build/benchmark/benchmark_xsimd + - name: Run example + run: ./_build/examples/mandelbrot + + build-windows-arm64-clang: + name: 'MSYS2 CLANG64 arm64' + runs-on: windows-11-arm + defaults: + run: + shell: msys2 {0} + steps: + - name: Setup MSYS2 with Clang (ARM64) + uses: msys2/setup-msys2@v2 + with: + # CLANG64 environment: uses clang/clang++ targeting Windows ARM64 natively. + # This is the ARM64-native MSYS2 Clang toolchain — not a cross-compiler. + msystem: CLANG64 + update: true + path-type: minimal + pacboy: >- + cc:p + cmake:p + ninja:p + + - name: Checkout xsimd + uses: actions/checkout@v4 + + - name: Configure + run: | + cmake -B _build \ + -DBUILD_TESTS=ON \ + -DDOWNLOAD_DOCTEST=ON \ + -DBUILD_BENCHMARK=ON \ + -DBUILD_EXAMPLES=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -G Ninja + + - name: Build + run: cmake --build _build + + - name: Test xsimd + run: ./_build/test/test_xsimd + + - name: Run benchmark + run: ./_build/benchmark/benchmark_xsimd + + - name: Run mandelbrot example + run: ./_build/examples/mandelbrot diff --git a/include/xsimd/arch/common/xsimd_common_memory.hpp b/include/xsimd/arch/common/xsimd_common_memory.hpp index 6a301dd44..14fb3364e 100644 --- a/include/xsimd/arch/common/xsimd_common_memory.hpp +++ b/include/xsimd/arch/common/xsimd_common_memory.hpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include "../../types/xsimd_batch_constant.hpp" @@ -71,6 +72,9 @@ namespace xsimd for (size_t i = 0; i < sizeof...(Is); ++i) if ((bitmask >> i) & 1u) std::swap(mask_buffer[inserted++], mask_buffer[i]); + // Fill remaining positions with the last valid index to avoid undefined behavior + for (size_t i = inserted; i < sizeof...(Is); ++i) + mask_buffer[i] = mask_buffer[inserted > 0 ? inserted - 1 : 0]; return batch::load_aligned(&mask_buffer[0]); } } @@ -85,7 +89,12 @@ namespace xsimd auto bitmask = mask.mask(); auto z = select(mask, x, batch((T)0)); auto compress_mask = detail::create_compress_swizzle_mask(bitmask, std::make_index_sequence()); - return swizzle(z, compress_mask); + alignas(A::alignment()) IT mask_out[size]; + compress_mask.store_aligned(&mask_out[0]); + alignas(A::alignment()) T z_out[size]; + z.store_aligned(&z_out[0]); + auto res = swizzle(z, compress_mask); + return res; } // expand diff --git a/include/xsimd/arch/xsimd_neon.hpp b/include/xsimd/arch/xsimd_neon.hpp index 4af19a650..ff7b08bba 100644 --- a/include/xsimd/arch/xsimd_neon.hpp +++ b/include/xsimd/arch/xsimd_neon.hpp @@ -222,51 +222,33 @@ namespace xsimd * comparison dispatchers * **************************/ + // On MSVC ARM64, all NEON types are the same __n128 type, so we can't specialize + // We use a function-based approach instead template - struct comp_return_type_impl; - - template <> - struct comp_return_type_impl + struct comp_return_type_impl { - using type = uint8x16_t; + using type = T; }; +#if !defined(_MSC_VER) || !defined(_M_ARM64) template <> struct comp_return_type_impl { using type = uint8x16_t; }; - template <> - struct comp_return_type_impl - { - using type = uint16x8_t; - }; - template <> struct comp_return_type_impl { using type = uint16x8_t; }; - template <> - struct comp_return_type_impl - { - using type = uint32x4_t; - }; - template <> struct comp_return_type_impl { using type = uint32x4_t; }; - template <> - struct comp_return_type_impl - { - using type = uint64x2_t; - }; - template <> struct comp_return_type_impl { @@ -278,6 +260,7 @@ namespace xsimd { using type = uint32x4_t; }; +#endif template using comp_return_type = typename comp_return_type_impl::type; @@ -370,21 +353,82 @@ namespace xsimd template = 0> XSIMD_INLINE batch set(batch const&, requires_arch, Args... args) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + // On MSVC ARM64, use load from array instead of brace initialization + alignas(16) T data[] = { static_cast(args)... }; + if constexpr (sizeof(T) == 1) + { + if constexpr (std::is_unsigned::value) + return vld1q_u8(reinterpret_cast(data)); + else + return vld1q_s8(reinterpret_cast(data)); + } + else if constexpr (sizeof(T) == 2) + { + if constexpr (std::is_unsigned::value) + return vld1q_u16(reinterpret_cast(data)); + else + return vld1q_s16(reinterpret_cast(data)); + } + else if constexpr (sizeof(T) == 4) + { + if constexpr (std::is_unsigned::value) + return vld1q_u32(reinterpret_cast(data)); + else + return vld1q_s32(reinterpret_cast(data)); + } + else if constexpr (sizeof(T) == 8) + { + if constexpr (std::is_unsigned::value) + return vld1q_u64(reinterpret_cast(data)); + else + return vld1q_s64(reinterpret_cast(data)); + } +#else return xsimd::types::detail::neon_vector_type { args... }; +#endif } template = 0> XSIMD_INLINE batch_bool set(batch_bool const&, requires_arch, Args... args) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + // On MSVC ARM64, use load from array instead of brace initialization + using unsigned_type = as_unsigned_integer_t; + alignas(16) unsigned_type data[] = { static_cast(args ? -1LL : 0LL)... }; + if constexpr (sizeof(T) == 1) + { + return vld1q_u8(reinterpret_cast(data)); + } + else if constexpr (sizeof(T) == 2) + { + return vld1q_u16(reinterpret_cast(data)); + } + else if constexpr (sizeof(T) == 4) + { + return vld1q_u32(reinterpret_cast(data)); + } + else if constexpr (sizeof(T) == 8) + { + return vld1q_u64(reinterpret_cast(data)); + } +#else using register_type = typename batch_bool::register_type; using unsigned_type = as_unsigned_integer_t; return register_type { static_cast(args ? -1LL : 0LL)... }; +#endif } template XSIMD_INLINE batch set(batch const&, requires_arch, float f0, float f1, float f2, float f3) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + // On MSVC ARM64, use load from array instead of brace initialization + alignas(16) float data[] = { f0, f1, f2, f3 }; + return vld1q_f32(data); +#else return float32x4_t { f0, f1, f2, f3 }; +#endif } template @@ -392,16 +436,30 @@ namespace xsimd std::complex c0, std::complex c1, std::complex c2, std::complex c3) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + // On MSVC ARM64, use load from array instead of brace initialization + alignas(16) float real_data[] = { c0.real(), c1.real(), c2.real(), c3.real() }; + alignas(16) float imag_data[] = { c0.imag(), c1.imag(), c2.imag(), c3.imag() }; + return batch, A>(vld1q_f32(real_data), vld1q_f32(imag_data)); +#else return batch, A>(float32x4_t { c0.real(), c1.real(), c2.real(), c3.real() }, float32x4_t { c0.imag(), c1.imag(), c2.imag(), c3.imag() }); +#endif } template XSIMD_INLINE batch_bool set(batch_bool const&, requires_arch, Args... args) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + // On MSVC ARM64, use load from array instead of brace initialization + using unsigned_type = as_unsigned_integer_t; + alignas(16) unsigned_type data[] = { static_cast(args ? -1LL : 0LL)... }; + return vld1q_u32(data); +#else using register_type = typename batch_bool::register_type; using unsigned_type = as_unsigned_integer_t; return register_type { static_cast(args ? -1LL : 0LL)... }; +#endif } /************* @@ -417,7 +475,7 @@ namespace xsimd template = 0> XSIMD_INLINE batch from_bool(batch_bool const& arg, requires_arch) noexcept { - return vandq_s8(reinterpret_cast(arg.data), vdupq_n_s8(1)); + return vandq_s8(vreinterpretq_s8_u8(arg.data), vdupq_n_s8(1)); } template = 0> @@ -429,7 +487,7 @@ namespace xsimd template = 0> XSIMD_INLINE batch from_bool(batch_bool const& arg, requires_arch) noexcept { - return vandq_s16(reinterpret_cast(arg.data), vdupq_n_s16(1)); + return vandq_s16(vreinterpretq_s16_u16(arg.data), vdupq_n_s16(1)); } template = 0> @@ -441,7 +499,7 @@ namespace xsimd template = 0> XSIMD_INLINE batch from_bool(batch_bool const& arg, requires_arch) noexcept { - return vandq_s32(reinterpret_cast(arg.data), vdupq_n_s32(1)); + return vandq_s32(vreinterpretq_s32_u32(arg.data), vdupq_n_s32(1)); } template = 0> @@ -453,7 +511,7 @@ namespace xsimd template = 0> XSIMD_INLINE batch from_bool(batch_bool const& arg, requires_arch) noexcept { - return vandq_s64(reinterpret_cast(arg.data), vdupq_n_s64(1)); + return vandq_s64(vreinterpretq_s64_u64(arg.data), vdupq_n_s64(1)); } template @@ -581,7 +639,7 @@ namespace xsimd XSIMD_INLINE batch_bool load_unaligned(bool const* mem, batch_bool, requires_arch) noexcept { auto vmem = load_unaligned((unsigned char const*)mem, convert {}, A {}); - return { 0 - vmem.data }; + return { vsubq_u8(vdupq_n_u8(0), vmem.data) }; } template = 0> XSIMD_INLINE batch_bool load_aligned(bool const* mem, batch_bool t, requires_arch r) noexcept @@ -593,7 +651,7 @@ namespace xsimd XSIMD_INLINE batch_bool load_unaligned(bool const* mem, batch_bool, requires_arch) noexcept { uint16x8_t vmem = vmovl_u8(vld1_u8((unsigned char const*)mem)); - return { 0 - vmem }; + return { vsubq_u16(vdupq_n_u16(0), vmem) }; } template = 0> @@ -606,7 +664,7 @@ namespace xsimd XSIMD_INLINE batch_bool load_unaligned(bool const* mem, batch_bool, requires_arch) noexcept { uint8x8_t tmp = vreinterpret_u8_u32(vset_lane_u32(*(unsigned int*)mem, vdup_n_u32(0), 0)); - return { 0 - vmovl_u16(vget_low_u16(vmovl_u8(tmp))) }; + return { vsubq_u32(vdupq_n_u32(0), vmovl_u16(vget_low_u16(vmovl_u8(tmp)))) }; } template = 0> @@ -851,12 +909,46 @@ namespace xsimd * add * *******/ +#if !defined(_MSC_VER) || !defined(_M_ARM64) WRAP_BINARY_INT(vaddq, detail::identity_return_type) WRAP_BINARY_FLOAT(vaddq, detail::identity_return_type) +#endif template = 0> XSIMD_INLINE batch add(batch const& lhs, batch const& rhs, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + { + if constexpr (std::is_unsigned::value) + return vaddq_u8(lhs, rhs); + else + return vaddq_s8(lhs, rhs); + } + else if constexpr (sizeof(T) == 2) + { + if constexpr (std::is_unsigned::value) + return vaddq_u16(lhs, rhs); + else + return vaddq_s16(lhs, rhs); + } + else if constexpr (sizeof(T) == 4) + { + if constexpr (std::is_same::value) + return vaddq_f32(lhs, rhs); + else if constexpr (std::is_unsigned::value) + return vaddq_u32(lhs, rhs); + else + return vaddq_s32(lhs, rhs); + } + else if constexpr (sizeof(T) == 8) + { + if constexpr (std::is_unsigned::value) + return vaddq_u64(lhs, rhs); + else + return vaddq_s64(lhs, rhs); + } +#else using register_type = typename batch::register_type; const detail::neon_dispatcher::binary dispatcher = { std::make_tuple(wrap::vaddq_u8, wrap::vaddq_s8, wrap::vaddq_u16, wrap::vaddq_s16, @@ -864,49 +956,106 @@ namespace xsimd wrap::vaddq_f32) }; return dispatcher.apply(register_type(lhs), register_type(rhs)); +#endif } /******* * avg * *******/ +#if !defined(_MSC_VER) || !defined(_M_ARM64) WRAP_BINARY_UINT_EXCLUDING_64(vhaddq, detail::identity_return_type) +#endif template ::value && sizeof(T) != 8)>> XSIMD_INLINE batch avg(batch const& lhs, batch const& rhs, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + return vhaddq_u8(lhs, rhs); + else if constexpr (sizeof(T) == 2) + return vhaddq_u16(lhs, rhs); + else if constexpr (sizeof(T) == 4) + return vhaddq_u32(lhs, rhs); +#else using register_type = typename batch::register_type; const detail::neon_dispatcher_impl::binary dispatcher = { std::make_tuple(wrap::vhaddq_u8, wrap::vhaddq_u16, wrap::vhaddq_u32) }; return dispatcher.apply(register_type(lhs), register_type(rhs)); +#endif } /******** * avgr * ********/ +#if !defined(_MSC_VER) || !defined(_M_ARM64) WRAP_BINARY_UINT_EXCLUDING_64(vrhaddq, detail::identity_return_type) +#endif template ::value && sizeof(T) != 8)>> XSIMD_INLINE batch avgr(batch const& lhs, batch const& rhs, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + return vrhaddq_u8(lhs, rhs); + else if constexpr (sizeof(T) == 2) + return vrhaddq_u16(lhs, rhs); + else if constexpr (sizeof(T) == 4) + return vrhaddq_u32(lhs, rhs); +#else using register_type = typename batch::register_type; const detail::neon_dispatcher_impl::binary dispatcher = { std::make_tuple(wrap::vrhaddq_u8, wrap::vrhaddq_u16, wrap::vrhaddq_u32) }; return dispatcher.apply(register_type(lhs), register_type(rhs)); +#endif } /******** * sadd * ********/ +#if !defined(_MSC_VER) || !defined(_M_ARM64) WRAP_BINARY_INT(vqaddq, detail::identity_return_type) +#endif template = 0> XSIMD_INLINE batch sadd(batch const& lhs, batch const& rhs, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + { + if constexpr (std::is_unsigned::value) + return vqaddq_u8(lhs, rhs); + else + return vqaddq_s8(lhs, rhs); + } + else if constexpr (sizeof(T) == 2) + { + if constexpr (std::is_unsigned::value) + return vqaddq_u16(lhs, rhs); + else + return vqaddq_s16(lhs, rhs); + } + else if constexpr (sizeof(T) == 4) + { + if constexpr (std::is_same::value) + return vaddq_f32(lhs, rhs); + else if constexpr (std::is_unsigned::value) + return vqaddq_u32(lhs, rhs); + else + return vqaddq_s32(lhs, rhs); + } + else if constexpr (sizeof(T) == 8) + { + if constexpr (std::is_unsigned::value) + return vqaddq_u64(lhs, rhs); + else + return vqaddq_s64(lhs, rhs); + } +#else using register_type = typename batch::register_type; const detail::neon_dispatcher::binary dispatcher = { std::make_tuple(wrap::vqaddq_u8, wrap::vqaddq_s8, wrap::vqaddq_u16, wrap::vqaddq_s16, @@ -914,18 +1063,53 @@ namespace xsimd wrap::vaddq_f32) }; return dispatcher.apply(register_type(lhs), register_type(rhs)); +#endif } /******* * sub * *******/ +#if !defined(_MSC_VER) || !defined(_M_ARM64) WRAP_BINARY_INT(vsubq, detail::identity_return_type) WRAP_BINARY_FLOAT(vsubq, detail::identity_return_type) +#endif template = 0> XSIMD_INLINE batch sub(batch const& lhs, batch const& rhs, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + { + if constexpr (std::is_unsigned::value) + return vsubq_u8(lhs, rhs); + else + return vsubq_s8(lhs, rhs); + } + else if constexpr (sizeof(T) == 2) + { + if constexpr (std::is_unsigned::value) + return vsubq_u16(lhs, rhs); + else + return vsubq_s16(lhs, rhs); + } + else if constexpr (sizeof(T) == 4) + { + if constexpr (std::is_same::value) + return vsubq_f32(lhs, rhs); + else if constexpr (std::is_unsigned::value) + return vsubq_u32(lhs, rhs); + else + return vsubq_s32(lhs, rhs); + } + else if constexpr (sizeof(T) == 8) + { + if constexpr (std::is_unsigned::value) + return vsubq_u64(lhs, rhs); + else + return vsubq_s64(lhs, rhs); + } +#else using register_type = typename batch::register_type; const detail::neon_dispatcher::binary dispatcher = { std::make_tuple(wrap::vsubq_u8, wrap::vsubq_s8, wrap::vsubq_u16, wrap::vsubq_s16, @@ -933,17 +1117,52 @@ namespace xsimd wrap::vsubq_f32) }; return dispatcher.apply(register_type(lhs), register_type(rhs)); +#endif } /******** * ssub * ********/ +#if !defined(_MSC_VER) || !defined(_M_ARM64) WRAP_BINARY_INT(vqsubq, detail::identity_return_type) +#endif template = 0> XSIMD_INLINE batch ssub(batch const& lhs, batch const& rhs, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + { + if constexpr (std::is_unsigned::value) + return vqsubq_u8(lhs, rhs); + else + return vqsubq_s8(lhs, rhs); + } + else if constexpr (sizeof(T) == 2) + { + if constexpr (std::is_unsigned::value) + return vqsubq_u16(lhs, rhs); + else + return vqsubq_s16(lhs, rhs); + } + else if constexpr (sizeof(T) == 4) + { + if constexpr (std::is_same::value) + return vsubq_f32(lhs, rhs); + else if constexpr (std::is_unsigned::value) + return vqsubq_u32(lhs, rhs); + else + return vqsubq_s32(lhs, rhs); + } + else if constexpr (sizeof(T) == 8) + { + if constexpr (std::is_unsigned::value) + return vqsubq_u64(lhs, rhs); + else + return vqsubq_s64(lhs, rhs); + } +#else using register_type = typename batch::register_type; const detail::neon_dispatcher::binary dispatcher = { std::make_tuple(wrap::vqsubq_u8, wrap::vqsubq_s8, wrap::vqsubq_u16, wrap::vqsubq_s16, @@ -951,24 +1170,53 @@ namespace xsimd wrap::vsubq_f32) }; return dispatcher.apply(register_type(lhs), register_type(rhs)); +#endif } /******* * mul * *******/ +#if !defined(_MSC_VER) || !defined(_M_ARM64) WRAP_BINARY_INT_EXCLUDING_64(vmulq, detail::identity_return_type) WRAP_BINARY_FLOAT(vmulq, detail::identity_return_type) +#endif template = 0> XSIMD_INLINE batch mul(batch const& lhs, batch const& rhs, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + { + if constexpr (std::is_unsigned::value) + return vmulq_u8(lhs, rhs); + else + return vmulq_s8(lhs, rhs); + } + else if constexpr (sizeof(T) == 2) + { + if constexpr (std::is_unsigned::value) + return vmulq_u16(lhs, rhs); + else + return vmulq_s16(lhs, rhs); + } + else if constexpr (sizeof(T) == 4) + { + if constexpr (std::is_same::value) + return vmulq_f32(lhs, rhs); + else if constexpr (std::is_unsigned::value) + return vmulq_u32(lhs, rhs); + else + return vmulq_s32(lhs, rhs); + } +#else using register_type = typename batch::register_type; const detail::excluding_int64_dispatcher::binary dispatcher = { std::make_tuple(wrap::vmulq_u8, wrap::vmulq_s8, wrap::vmulq_u16, wrap::vmulq_s16, wrap::vmulq_u32, wrap::vmulq_s32, wrap::vmulq_f32) }; return dispatcher.apply(register_type(lhs), register_type(rhs)); +#endif } /******* @@ -1010,29 +1258,66 @@ namespace xsimd * eq * ******/ +#if !defined(_MSC_VER) || !defined(_M_ARM64) WRAP_BINARY_INT_EXCLUDING_64(vceqq, detail::comp_return_type) WRAP_BINARY_FLOAT(vceqq, detail::comp_return_type) +#endif template = 0> XSIMD_INLINE batch_bool eq(batch const& lhs, batch const& rhs, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + { + if constexpr (std::is_unsigned::value) + return vceqq_u8(lhs, rhs); + else + return vceqq_s8(lhs, rhs); + } + else if constexpr (sizeof(T) == 2) + { + if constexpr (std::is_unsigned::value) + return vceqq_u16(lhs, rhs); + else + return vceqq_s16(lhs, rhs); + } + else if constexpr (sizeof(T) == 4) + { + if constexpr (std::is_same::value) + return vceqq_f32(lhs, rhs); + else if constexpr (std::is_unsigned::value) + return vceqq_u32(lhs, rhs); + else + return vceqq_s32(lhs, rhs); + } +#else using register_type = typename batch::register_type; const detail::excluding_int64_comp_dispatcher::binary dispatcher = { std::make_tuple(wrap::vceqq_u8, wrap::vceqq_s8, wrap::vceqq_u16, wrap::vceqq_s16, wrap::vceqq_u32, wrap::vceqq_s32, wrap::vceqq_f32) }; return dispatcher.apply(register_type(lhs), register_type(rhs)); +#endif } template = 0> XSIMD_INLINE batch_bool eq(batch_bool const& lhs, batch_bool const& rhs, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + return vceqq_u8(lhs, rhs); + else if constexpr (sizeof(T) == 2) + return vceqq_u16(lhs, rhs); + else if constexpr (sizeof(T) == 4) + return vceqq_u32(lhs, rhs); +#else using register_type = typename batch_bool::register_type; using dispatcher_type = detail::neon_comp_dispatcher_impl::binary; const dispatcher_type dispatcher = { std::make_tuple(wrap::vceqq_u8, wrap::vceqq_u16, wrap::vceqq_u32) }; return dispatcher.apply(register_type(lhs), register_type(rhs)); +#endif } template = 0> @@ -1095,18 +1380,46 @@ namespace xsimd * lt * ******/ +#if !defined(_MSC_VER) || !defined(_M_ARM64) WRAP_BINARY_INT_EXCLUDING_64(vcltq, detail::comp_return_type) WRAP_BINARY_FLOAT(vcltq, detail::comp_return_type) +#endif template = 0> XSIMD_INLINE batch_bool lt(batch const& lhs, batch const& rhs, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + { + if constexpr (std::is_unsigned::value) + return vcltq_u8(lhs, rhs); + else + return vcltq_s8(lhs, rhs); + } + else if constexpr (sizeof(T) == 2) + { + if constexpr (std::is_unsigned::value) + return vcltq_u16(lhs, rhs); + else + return vcltq_s16(lhs, rhs); + } + else if constexpr (sizeof(T) == 4) + { + if constexpr (std::is_same::value) + return vcltq_f32(lhs, rhs); + else if constexpr (std::is_unsigned::value) + return vcltq_u32(lhs, rhs); + else + return vcltq_s32(lhs, rhs); + } +#else using register_type = typename batch::register_type; const detail::excluding_int64_comp_dispatcher::binary dispatcher = { std::make_tuple(wrap::vcltq_u8, wrap::vcltq_s8, wrap::vcltq_u16, wrap::vcltq_s16, wrap::vcltq_u32, wrap::vcltq_s32, wrap::vcltq_f32) }; return dispatcher.apply(register_type(lhs), register_type(rhs)); +#endif } template = 0> @@ -1128,18 +1441,46 @@ namespace xsimd * le * ******/ +#if !defined(_MSC_VER) || !defined(_M_ARM64) WRAP_BINARY_INT_EXCLUDING_64(vcleq, detail::comp_return_type) WRAP_BINARY_FLOAT(vcleq, detail::comp_return_type) +#endif template = 0> XSIMD_INLINE batch_bool le(batch const& lhs, batch const& rhs, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + { + if constexpr (std::is_unsigned::value) + return vcleq_u8(lhs, rhs); + else + return vcleq_s8(lhs, rhs); + } + else if constexpr (sizeof(T) == 2) + { + if constexpr (std::is_unsigned::value) + return vcleq_u16(lhs, rhs); + else + return vcleq_s16(lhs, rhs); + } + else if constexpr (sizeof(T) == 4) + { + if constexpr (std::is_same::value) + return vcleq_f32(lhs, rhs); + else if constexpr (std::is_unsigned::value) + return vcleq_u32(lhs, rhs); + else + return vcleq_s32(lhs, rhs); + } +#else using register_type = typename batch::register_type; const detail::excluding_int64_comp_dispatcher::binary dispatcher = { std::make_tuple(wrap::vcleq_u8, wrap::vcleq_s8, wrap::vcleq_u16, wrap::vcleq_s16, wrap::vcleq_u32, wrap::vcleq_s32, wrap::vcleq_f32) }; return dispatcher.apply(register_type(lhs), register_type(rhs)); +#endif } template = 0> @@ -1164,18 +1505,46 @@ namespace xsimd } } +#if !defined(_MSC_VER) || !defined(_M_ARM64) WRAP_BINARY_INT_EXCLUDING_64(vcgtq, detail::comp_return_type) WRAP_BINARY_FLOAT(vcgtq, detail::comp_return_type) +#endif template = 0> XSIMD_INLINE batch_bool gt(batch const& lhs, batch const& rhs, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + { + if constexpr (std::is_unsigned::value) + return vcgtq_u8(lhs, rhs); + else + return vcgtq_s8(lhs, rhs); + } + else if constexpr (sizeof(T) == 2) + { + if constexpr (std::is_unsigned::value) + return vcgtq_u16(lhs, rhs); + else + return vcgtq_s16(lhs, rhs); + } + else if constexpr (sizeof(T) == 4) + { + if constexpr (std::is_same::value) + return vcgtq_f32(lhs, rhs); + else if constexpr (std::is_unsigned::value) + return vcgtq_u32(lhs, rhs); + else + return vcgtq_s32(lhs, rhs); + } +#else using register_type = typename batch::register_type; const detail::excluding_int64_comp_dispatcher::binary dispatcher = { std::make_tuple(wrap::vcgtq_u8, wrap::vcgtq_s8, wrap::vcgtq_u16, wrap::vcgtq_s16, wrap::vcgtq_u32, wrap::vcgtq_s32, wrap::vcgtq_f32) }; return dispatcher.apply(register_type(lhs), register_type(rhs)); +#endif } template = 0> @@ -1197,18 +1566,46 @@ namespace xsimd * ge * ******/ +#if !defined(_MSC_VER) || !defined(_M_ARM64) WRAP_BINARY_INT_EXCLUDING_64(vcgeq, detail::comp_return_type) WRAP_BINARY_FLOAT(vcgeq, detail::comp_return_type) +#endif template = 0> XSIMD_INLINE batch_bool ge(batch const& lhs, batch const& rhs, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + { + if constexpr (std::is_unsigned::value) + return vcgeq_u8(lhs, rhs); + else + return vcgeq_s8(lhs, rhs); + } + else if constexpr (sizeof(T) == 2) + { + if constexpr (std::is_unsigned::value) + return vcgeq_u16(lhs, rhs); + else + return vcgeq_s16(lhs, rhs); + } + else if constexpr (sizeof(T) == 4) + { + if constexpr (std::is_same::value) + return vcgeq_f32(lhs, rhs); + else if constexpr (std::is_unsigned::value) + return vcgeq_u32(lhs, rhs); + else + return vcgeq_s32(lhs, rhs); + } +#else using register_type = typename batch::register_type; const detail::excluding_int64_comp_dispatcher::binary dispatcher = { std::make_tuple(wrap::vcgeq_u8, wrap::vcgeq_s8, wrap::vcgeq_u16, wrap::vcgeq_s16, wrap::vcgeq_u32, wrap::vcgeq_s32, wrap::vcgeq_f32) }; return dispatcher.apply(register_type(lhs), register_type(rhs)); +#endif } template = 0> @@ -1232,7 +1629,9 @@ namespace xsimd * bitwise_and * ***************/ +#if !defined(_MSC_VER) || !defined(_M_ARM64) WRAP_BINARY_INT(vandq, detail::identity_return_type) +#endif namespace detail { @@ -1242,6 +1641,7 @@ namespace xsimd vreinterpretq_u32_f32(rhs))); } +#if !defined(_MSC_VER) || !defined(_M_ARM64) template V bitwise_and_neon(V const& lhs, V const& rhs) { @@ -1252,27 +1652,54 @@ namespace xsimd }; return dispatcher.apply(lhs, rhs); } +#endif } template = 0> XSIMD_INLINE batch bitwise_and(batch const& lhs, batch const& rhs, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (std::is_same::value) + return detail::bitwise_and_f32(lhs, rhs); + else if constexpr (sizeof(T) == 1) + return vandq_u8(lhs, rhs); + else if constexpr (sizeof(T) == 2) + return vandq_u16(lhs, rhs); + else if constexpr (sizeof(T) == 4) + return vandq_u32(lhs, rhs); + else if constexpr (sizeof(T) == 8) + return vandq_u64(lhs, rhs); +#else using register_type = typename batch::register_type; return detail::bitwise_and_neon(register_type(lhs), register_type(rhs)); +#endif } template = 0> XSIMD_INLINE batch_bool bitwise_and(batch_bool const& lhs, batch_bool const& rhs, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + return vandq_u8(lhs, rhs); + else if constexpr (sizeof(T) == 2) + return vandq_u16(lhs, rhs); + else if constexpr (sizeof(T) == 4) + return vandq_u32(lhs, rhs); + else if constexpr (sizeof(T) == 8) + return vandq_u64(lhs, rhs); +#else using register_type = typename batch_bool::register_type; return detail::bitwise_and_neon(register_type(lhs), register_type(rhs)); +#endif } /************** * bitwise_or * **************/ +#if !defined(_MSC_VER) || !defined(_M_ARM64) WRAP_BINARY_INT(vorrq, detail::identity_return_type) +#endif namespace detail { @@ -1282,6 +1709,7 @@ namespace xsimd vreinterpretq_u32_f32(rhs))); } +#if !defined(_MSC_VER) || !defined(_M_ARM64) template XSIMD_INLINE V bitwise_or_neon(V const& lhs, V const& rhs) noexcept { @@ -1292,27 +1720,54 @@ namespace xsimd }; return dispatcher.apply(lhs, rhs); } +#endif } template = 0> XSIMD_INLINE batch bitwise_or(batch const& lhs, batch const& rhs, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (std::is_same::value) + return detail::bitwise_or_f32(lhs, rhs); + else if constexpr (sizeof(T) == 1) + return vorrq_u8(lhs, rhs); + else if constexpr (sizeof(T) == 2) + return vorrq_u16(lhs, rhs); + else if constexpr (sizeof(T) == 4) + return vorrq_u32(lhs, rhs); + else if constexpr (sizeof(T) == 8) + return vorrq_u64(lhs, rhs); +#else using register_type = typename batch::register_type; return detail::bitwise_or_neon(register_type(lhs), register_type(rhs)); +#endif } template = 0> XSIMD_INLINE batch_bool bitwise_or(batch_bool const& lhs, batch_bool const& rhs, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + return vorrq_u8(lhs, rhs); + else if constexpr (sizeof(T) == 2) + return vorrq_u16(lhs, rhs); + else if constexpr (sizeof(T) == 4) + return vorrq_u32(lhs, rhs); + else if constexpr (sizeof(T) == 8) + return vorrq_u64(lhs, rhs); +#else using register_type = typename batch_bool::register_type; return detail::bitwise_or_neon(register_type(lhs), register_type(rhs)); +#endif } /*************** * bitwise_xor * ***************/ +#if !defined(_MSC_VER) || !defined(_M_ARM64) WRAP_BINARY_INT(veorq, detail::identity_return_type) +#endif namespace detail { @@ -1322,6 +1777,7 @@ namespace xsimd vreinterpretq_u32_f32(rhs))); } +#if !defined(_MSC_VER) || !defined(_M_ARM64) template XSIMD_INLINE V bitwise_xor_neon(V const& lhs, V const& rhs) noexcept { @@ -1332,20 +1788,45 @@ namespace xsimd }; return dispatcher.apply(lhs, rhs); } +#endif } template = 0> XSIMD_INLINE batch bitwise_xor(batch const& lhs, batch const& rhs, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (std::is_same::value) + return detail::bitwise_xor_f32(lhs, rhs); + else if constexpr (sizeof(T) == 1) + return veorq_u8(lhs, rhs); + else if constexpr (sizeof(T) == 2) + return veorq_u16(lhs, rhs); + else if constexpr (sizeof(T) == 4) + return veorq_u32(lhs, rhs); + else if constexpr (sizeof(T) == 8) + return veorq_u64(lhs, rhs); +#else using register_type = typename batch::register_type; return detail::bitwise_xor_neon(register_type(lhs), register_type(rhs)); +#endif } template = 0> XSIMD_INLINE batch_bool bitwise_xor(batch_bool const& lhs, batch_bool const& rhs, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + return veorq_u8(lhs, rhs); + else if constexpr (sizeof(T) == 2) + return veorq_u16(lhs, rhs); + else if constexpr (sizeof(T) == 4) + return veorq_u32(lhs, rhs); + else if constexpr (sizeof(T) == 8) + return veorq_u64(lhs, rhs); +#else using register_type = typename batch_bool::register_type; return detail::bitwise_xor_neon(register_type(lhs), register_type(rhs)); +#endif } /******* @@ -1362,7 +1843,9 @@ namespace xsimd * bitwise_not * ***************/ +#if !defined(_MSC_VER) || !defined(_M_ARM64) WRAP_UNARY_INT_EXCLUDING_64(vmvnq) +#endif namespace detail { @@ -1371,6 +1854,7 @@ namespace xsimd return vreinterpretq_f32_u32(vmvnq_u32(vreinterpretq_u32_f32(arg))); } +#if !defined(_MSC_VER) || !defined(_M_ARM64) template XSIMD_INLINE V bitwise_not_neon(V const& arg) noexcept { @@ -1382,27 +1866,54 @@ namespace xsimd }; return dispatcher.apply(arg); } +#endif } template = 0> XSIMD_INLINE batch bitwise_not(batch const& arg, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (std::is_same::value) + return detail::bitwise_not_f32(arg); + else if constexpr (sizeof(T) == 1) + return vmvnq_u8(arg); + else if constexpr (sizeof(T) == 2) + return vmvnq_u16(arg); + else if constexpr (sizeof(T) == 4) + return vmvnq_u32(arg); + else if constexpr (sizeof(T) == 8) + return detail::bitwise_not_u64(arg); +#else using register_type = typename batch::register_type; return detail::bitwise_not_neon(register_type(arg)); +#endif } template = 0> XSIMD_INLINE batch_bool bitwise_not(batch_bool const& arg, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + return vmvnq_u8(arg); + else if constexpr (sizeof(T) == 2) + return vmvnq_u16(arg); + else if constexpr (sizeof(T) == 4) + return vmvnq_u32(arg); + else if constexpr (sizeof(T) == 8) + return detail::bitwise_not_u64(arg); +#else using register_type = typename batch_bool::register_type; return detail::bitwise_not_neon(register_type(arg)); +#endif } /****************** * bitwise_andnot * ******************/ +#if !defined(_MSC_VER) || !defined(_M_ARM64) WRAP_BINARY_INT(vbicq, detail::identity_return_type) +#endif namespace detail { @@ -1411,6 +1922,7 @@ namespace xsimd return vreinterpretq_f32_u32(vbicq_u32(vreinterpretq_u32_f32(lhs), vreinterpretq_u32_f32(rhs))); } +#if !defined(_MSC_VER) || !defined(_M_ARM64) template XSIMD_INLINE V bitwise_andnot_neon(V const& lhs, V const& rhs) noexcept { @@ -1421,38 +1933,91 @@ namespace xsimd }; return dispatcher.apply(lhs, rhs); } +#endif } template = 0> XSIMD_INLINE batch bitwise_andnot(batch const& lhs, batch const& rhs, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (std::is_same::value) + return detail::bitwise_andnot_f32(lhs, rhs); + else if constexpr (sizeof(T) == 1) + return vbicq_u8(lhs, rhs); + else if constexpr (sizeof(T) == 2) + return vbicq_u16(lhs, rhs); + else if constexpr (sizeof(T) == 4) + return vbicq_u32(lhs, rhs); + else if constexpr (sizeof(T) == 8) + return vbicq_u64(lhs, rhs); +#else using register_type = typename batch::register_type; return detail::bitwise_andnot_neon(register_type(lhs), register_type(rhs)); +#endif } template = 0> XSIMD_INLINE batch_bool bitwise_andnot(batch_bool const& lhs, batch_bool const& rhs, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + return vbicq_u8(lhs, rhs); + else if constexpr (sizeof(T) == 2) + return vbicq_u16(lhs, rhs); + else if constexpr (sizeof(T) == 4) + return vbicq_u32(lhs, rhs); + else if constexpr (sizeof(T) == 8) + return vbicq_u64(lhs, rhs); +#else using register_type = typename batch_bool::register_type; return detail::bitwise_andnot_neon(register_type(lhs), register_type(rhs)); +#endif } /******* * min * *******/ +#if !defined(_MSC_VER) || !defined(_M_ARM64) WRAP_BINARY_INT_EXCLUDING_64(vminq, detail::identity_return_type) WRAP_BINARY_FLOAT(vminq, detail::identity_return_type) +#endif template = 0> XSIMD_INLINE batch min(batch const& lhs, batch const& rhs, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + { + if constexpr (std::is_unsigned::value) + return vminq_u8(lhs, rhs); + else + return vminq_s8(lhs, rhs); + } + else if constexpr (sizeof(T) == 2) + { + if constexpr (std::is_unsigned::value) + return vminq_u16(lhs, rhs); + else + return vminq_s16(lhs, rhs); + } + else if constexpr (sizeof(T) == 4) + { + if constexpr (std::is_same::value) + return vminq_f32(lhs, rhs); + else if constexpr (std::is_unsigned::value) + return vminq_u32(lhs, rhs); + else + return vminq_s32(lhs, rhs); + } +#else using register_type = typename batch::register_type; const detail::excluding_int64_dispatcher::binary dispatcher = { std::make_tuple(wrap::vminq_u8, wrap::vminq_s8, wrap::vminq_u16, wrap::vminq_s16, wrap::vminq_u32, wrap::vminq_s32, wrap::vminq_f32) }; return dispatcher.apply(register_type(lhs), register_type(rhs)); +#endif } template = 0> @@ -1465,18 +2030,46 @@ namespace xsimd * max * *******/ +#if !defined(_MSC_VER) || !defined(_M_ARM64) WRAP_BINARY_INT_EXCLUDING_64(vmaxq, detail::identity_return_type) WRAP_BINARY_FLOAT(vmaxq, detail::identity_return_type) +#endif template = 0> XSIMD_INLINE batch max(batch const& lhs, batch const& rhs, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + { + if constexpr (std::is_unsigned::value) + return vmaxq_u8(lhs, rhs); + else + return vmaxq_s8(lhs, rhs); + } + else if constexpr (sizeof(T) == 2) + { + if constexpr (std::is_unsigned::value) + return vmaxq_u16(lhs, rhs); + else + return vmaxq_s16(lhs, rhs); + } + else if constexpr (sizeof(T) == 4) + { + if constexpr (std::is_same::value) + return vmaxq_f32(lhs, rhs); + else if constexpr (std::is_unsigned::value) + return vmaxq_u32(lhs, rhs); + else + return vmaxq_s32(lhs, rhs); + } +#else using register_type = typename batch::register_type; const detail::excluding_int64_dispatcher::binary dispatcher = { std::make_tuple(wrap::vmaxq_u8, wrap::vmaxq_s8, wrap::vmaxq_u16, wrap::vmaxq_s16, wrap::vmaxq_u32, wrap::vmaxq_s32, wrap::vmaxq_f32) }; return dispatcher.apply(register_type(lhs), register_type(rhs)); +#endif } template = 0> @@ -1489,6 +2082,7 @@ namespace xsimd * abs * *******/ +#if !defined(_MSC_VER) || !defined(_M_ARM64) namespace wrap { XSIMD_INLINE int8x16_t vabsq_s8(int8x16_t a) noexcept { return ::vabsq_s8(a); } @@ -1496,6 +2090,7 @@ namespace xsimd XSIMD_INLINE int32x4_t vabsq_s32(int32x4_t a) noexcept { return ::vabsq_s32(a); } } WRAP_UNARY_FLOAT(vabsq) +#endif namespace detail { @@ -1518,12 +2113,38 @@ namespace xsimd template = 0> XSIMD_INLINE batch abs(batch const& arg, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + { + if constexpr (std::is_unsigned::value) + return arg; + else + return vabsq_s8(arg); + } + else if constexpr (sizeof(T) == 2) + { + if constexpr (std::is_unsigned::value) + return arg; + else + return vabsq_s16(arg); + } + else if constexpr (sizeof(T) == 4) + { + if constexpr (std::is_same::value) + return vabsq_f32(arg); + else if constexpr (std::is_unsigned::value) + return arg; + else + return vabsq_s32(arg); + } +#else using register_type = typename batch::register_type; const detail::excluding_int64_dispatcher::unary dispatcher = { std::make_tuple(detail::abs_u8, wrap::vabsq_s8, detail::abs_u16, wrap::vabsq_s16, detail::abs_u32, wrap::vabsq_s32, wrap::vabsq_f32) }; return dispatcher.apply(register_type(arg)); +#endif } /******** @@ -1843,6 +2464,7 @@ namespace xsimd * select * **********/ +#if !defined(_MSC_VER) || !defined(_M_ARM64) namespace wrap { XSIMD_INLINE uint8x16_t vbslq_u8(uint8x16_t a, uint8x16_t b, uint8x16_t c) noexcept { return ::vbslq_u8(a, b, c); } @@ -1855,6 +2477,7 @@ namespace xsimd XSIMD_INLINE int64x2_t vbslq_s64(uint64x2_t a, int64x2_t b, int64x2_t c) noexcept { return ::vbslq_s64(a, b, c); } XSIMD_INLINE float32x4_t vbslq_f32(uint32x4_t a, float32x4_t b, float32x4_t c) noexcept { return ::vbslq_f32(a, b, c); } } +#endif namespace detail { @@ -1883,6 +2506,38 @@ namespace xsimd template = 0> XSIMD_INLINE batch select(batch_bool const& cond, batch const& a, batch const& b, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + { + if constexpr (std::is_unsigned::value) + return vbslq_u8(cond, a, b); + else + return vbslq_s8(cond, a, b); + } + else if constexpr (sizeof(T) == 2) + { + if constexpr (std::is_unsigned::value) + return vbslq_u16(cond, a, b); + else + return vbslq_s16(cond, a, b); + } + else if constexpr (sizeof(T) == 4) + { + if constexpr (std::is_same::value) + return vbslq_f32(cond, a, b); + else if constexpr (std::is_unsigned::value) + return vbslq_u32(cond, a, b); + else + return vbslq_s32(cond, a, b); + } + else if constexpr (sizeof(T) == 8) + { + if constexpr (std::is_unsigned::value) + return vbslq_u64(cond, a, b); + else + return vbslq_s64(cond, a, b); + } +#else using bool_register_type = typename batch_bool::register_type; using register_type = typename batch::register_type; const detail::neon_select_dispatcher dispatcher = { @@ -1891,6 +2546,7 @@ namespace xsimd wrap::vbslq_f32) }; return dispatcher.apply(bool_register_type(cond), register_type(a), register_type(b)); +#endif } template = 0> @@ -2861,6 +3517,7 @@ namespace xsimd * bitwise_cast * ****************/ +#if !defined(_MSC_VER) || !defined(_M_ARM64) #define WRAP_CAST(SUFFIX, TYPE) \ namespace wrap \ { \ @@ -2913,6 +3570,7 @@ namespace xsimd WRAP_CAST(f32, float32x4_t) #undef WRAP_CAST +#endif namespace detail { @@ -2973,6 +3631,10 @@ namespace xsimd template XSIMD_INLINE batch bitwise_cast(batch const& arg, batch const&, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + // On MSVC ARM64, all NEON types are __n128, so just return the argument + return arg.data; +#else const detail::neon_bitwise_caster caster = { std::make_tuple( detail::make_bitwise_caster_impl(wrap::vreinterpretq_u8_u8, wrap::vreinterpretq_u8_s8, wrap::vreinterpretq_u8_u16, wrap::vreinterpretq_u8_s16, @@ -3006,6 +3668,7 @@ namespace xsimd using src_register_type = typename batch::register_type; using dst_register_type = typename batch::register_type; return caster.apply(src_register_type(arg)); +#endif } /********* @@ -3087,6 +3750,7 @@ namespace xsimd /**************** * rotate_left * ****************/ +#if !defined(_MSC_VER) || !defined(_M_ARM64) namespace wrap { template @@ -3108,10 +3772,43 @@ namespace xsimd template XSIMD_INLINE float32x4_t rotate_left_f32(float32x4_t a, float32x4_t b) noexcept { return vextq_f32(a, b, N); } } +#endif template = 0> XSIMD_INLINE batch rotate_left(batch const& a, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + { + if constexpr (std::is_unsigned::value) + return vextq_u8(a, a, N); + else + return vextq_s8(a, a, N); + } + else if constexpr (sizeof(T) == 2) + { + if constexpr (std::is_unsigned::value) + return vextq_u16(a, a, N % 8); + else + return vextq_s16(a, a, N % 8); + } + else if constexpr (sizeof(T) == 4) + { + if constexpr (std::is_same::value) + return vextq_f32(a, a, N % 4); + else if constexpr (std::is_unsigned::value) + return vextq_u32(a, a, N % 4); + else + return vextq_s32(a, a, N % 4); + } + else if constexpr (sizeof(T) == 8) + { + if constexpr (std::is_unsigned::value) + return vextq_u64(a, a, N % 2); + else + return vextq_s64(a, a, N % 2); + } +#else using register_type = typename batch::register_type; // Adding modulo to avoid warning. const detail::neon_dispatcher::binary dispatcher = { @@ -3120,6 +3817,7 @@ namespace xsimd wrap::rotate_left_f32) }; return dispatcher.apply(register_type(a), register_type(a)); +#endif } } diff --git a/include/xsimd/arch/xsimd_neon64.hpp b/include/xsimd/arch/xsimd_neon64.hpp index 7a5263fb1..f896d957f 100644 --- a/include/xsimd/arch/xsimd_neon64.hpp +++ b/include/xsimd/arch/xsimd_neon64.hpp @@ -12,6 +12,8 @@ #ifndef XSIMD_NEON64_HPP #define XSIMD_NEON64_HPP +#include +#include #include #include #include @@ -117,16 +119,30 @@ namespace xsimd template XSIMD_INLINE batch set(batch const&, requires_arch, double d0, double d1) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + alignas(16) double data[] = { d0, d1 }; + return vld1q_f64(data); +#else return float64x2_t { d0, d1 }; +#endif } template XSIMD_INLINE batch_bool set(batch_bool const&, requires_arch, bool b0, bool b1) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + using unsigned_type = as_unsigned_integer_t; + alignas(16) unsigned_type data[] = { + static_cast(b0 ? -1LL : 0LL), + static_cast(b1 ? -1LL : 0LL) + }; + return vld1q_u64(data); +#else using register_type = typename batch_bool::register_type; using unsigned_type = as_unsigned_integer_t; return register_type { static_cast(b0 ? -1LL : 0LL), static_cast(b1 ? -1LL : 0LL) }; +#endif } /************* @@ -145,7 +161,11 @@ namespace xsimd #if defined(__clang__) || defined(__GNUC__) #define xsimd_aligned_load(inst, type, expr) inst((type)__builtin_assume_aligned(expr, 16)) #elif defined(_MSC_VER) +#if defined(_M_ARM64) +#define xsimd_aligned_load(inst, type, expr) inst((type)expr) +#else #define xsimd_aligned_load(inst, type, expr) inst##_ex((type)expr, 128) +#endif #else #define xsimd_aligned_load(inst, type, expr) inst((type)expr) #endif @@ -194,18 +214,18 @@ namespace xsimd ****************/ template - XSIMD_INLINE batch, A> load_complex_aligned(std::complex const* mem, convert>, requires_arch) noexcept + XSIMD_INLINE batch<::std::complex, A> load_complex_aligned(::std::complex const* mem, convert<::std::complex>, requires_arch) noexcept { using real_batch = batch; const double* buf = reinterpret_cast(mem); float64x2x2_t tmp = vld2q_f64(buf); real_batch real = tmp.val[0], imag = tmp.val[1]; - return batch, A> { real, imag }; + return batch<::std::complex, A> { real, imag }; } template - XSIMD_INLINE batch, A> load_complex_unaligned(std::complex const* mem, convert> cvt, requires_arch) noexcept + XSIMD_INLINE batch<::std::complex, A> load_complex_unaligned(::std::complex const* mem, convert<::std::complex> cvt, requires_arch) noexcept { return load_complex_aligned(mem, cvt, A {}); } @@ -215,7 +235,7 @@ namespace xsimd *****************/ template - XSIMD_INLINE void store_complex_aligned(std::complex* dst, batch, A> const& src, requires_arch) noexcept + XSIMD_INLINE void store_complex_aligned(::std::complex* dst, batch<::std::complex, A> const& src, requires_arch) noexcept { float64x2x2_t tmp; tmp.val[0] = src.real(); @@ -225,7 +245,7 @@ namespace xsimd } template - XSIMD_INLINE void store_complex_unaligned(std::complex* dst, batch, A> const& src, requires_arch) noexcept + XSIMD_INLINE void store_complex_unaligned(::std::complex* dst, batch<::std::complex, A> const& src, requires_arch) noexcept { store_complex_aligned(dst, src, A {}); } @@ -725,7 +745,7 @@ namespace xsimd template XSIMD_INLINE batch reciprocal(const batch& x, - kernel::requires_arch) noexcept + requires_arch) noexcept { return vrecpeq_f64(x); } @@ -794,7 +814,9 @@ namespace xsimd // Wrap reducer intrinsics so we can pass them as function pointers // - OP: intrinsics name prefix, e.g., vorrq + // On MSVC ARM64, skip these wrappers since all types are __n128 +#if !defined(_MSC_VER) || !defined(_M_ARM64) #define WRAP_REDUCER_INT_EXCLUDING_64(OP) \ namespace wrap \ { \ @@ -850,9 +872,16 @@ namespace xsimd return ::OP##_f64(a); \ } \ } +#else +// On MSVC ARM64, skip wrapper macros +#define WRAP_REDUCER_INT_EXCLUDING_64(OP) +#define WRAP_REDUCER_INT(OP) +#define WRAP_REDUCER_FLOAT(OP) +#endif namespace detail { +#if !defined(_MSC_VER) || !defined(_M_ARM64) template struct reducer_return_type_impl; @@ -929,6 +958,7 @@ namespace xsimd uint32x4_t, int32x4_t, uint64x2_t, int64x2_t, float32x4_t, float64x2_t>; +#endif template using enable_neon64_type_t = std::enable_if_t::value || std::is_same::value || std::is_same::value, int>; @@ -944,6 +974,40 @@ namespace xsimd template = 0> XSIMD_INLINE typename batch::value_type reduce_add(batch const& arg, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + { + if constexpr (std::is_unsigned::value) + return vaddvq_u8(arg); + else + return vaddvq_s8(arg); + } + else if constexpr (sizeof(T) == 2) + { + if constexpr (std::is_unsigned::value) + return vaddvq_u16(arg); + else + return vaddvq_s16(arg); + } + else if constexpr (sizeof(T) == 4) + { + if constexpr (std::is_same::value) + return vaddvq_f32(arg); + else if constexpr (std::is_unsigned::value) + return vaddvq_u32(arg); + else + return vaddvq_s32(arg); + } + else if constexpr (sizeof(T) == 8) + { + if constexpr (std::is_same::value) + return vaddvq_f64(arg); + else if constexpr (std::is_unsigned::value) + return vaddvq_u64(arg); + else + return vaddvq_s64(arg); + } +#else using register_type = typename batch::register_type; const detail::neon_reducer_dispatcher::unary dispatcher = { std::make_tuple(wrap::vaddvq_u8, wrap::vaddvq_s8, wrap::vaddvq_u16, wrap::vaddvq_s16, @@ -951,6 +1015,7 @@ namespace xsimd wrap::vaddvq_f32, wrap::vaddvq_f64) }; return dispatcher.apply(register_type(arg)); +#endif } /************** @@ -960,6 +1025,7 @@ namespace xsimd WRAP_REDUCER_INT_EXCLUDING_64(vmaxvq) WRAP_REDUCER_FLOAT(vmaxvq) +#if !defined(_MSC_VER) || !defined(_M_ARM64) namespace wrap { XSIMD_INLINE uint64_t vmaxvq_u64(uint64x2_t a) noexcept @@ -972,10 +1038,45 @@ namespace xsimd return std::max(vdupd_laneq_s64(a, 0), vdupd_laneq_s64(a, 1)); } } +#endif template = 0> XSIMD_INLINE typename batch::value_type reduce_max(batch const& arg, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + { + if constexpr (std::is_unsigned::value) + return vmaxvq_u8(arg); + else + return vmaxvq_s8(arg); + } + else if constexpr (sizeof(T) == 2) + { + if constexpr (std::is_unsigned::value) + return vmaxvq_u16(arg); + else + return vmaxvq_s16(arg); + } + else if constexpr (sizeof(T) == 4) + { + if constexpr (std::is_same::value) + return vmaxvq_f32(arg); + else if constexpr (std::is_unsigned::value) + return vmaxvq_u32(arg); + else + return vmaxvq_s32(arg); + } + else if constexpr (sizeof(T) == 8) + { + if constexpr (std::is_same::value) + return vmaxvq_f64(arg); + else if constexpr (std::is_unsigned::value) + return std::max(vdupd_laneq_u64(arg, 0), vdupd_laneq_u64(arg, 1)); + else + return std::max(vdupd_laneq_s64(arg, 0), vdupd_laneq_s64(arg, 1)); + } +#else using register_type = typename batch::register_type; const detail::neon_reducer_dispatcher::unary dispatcher = { std::make_tuple(wrap::vmaxvq_u8, wrap::vmaxvq_s8, wrap::vmaxvq_u16, wrap::vmaxvq_s16, @@ -983,6 +1084,7 @@ namespace xsimd wrap::vmaxvq_f32, wrap::vmaxvq_f64) }; return dispatcher.apply(register_type(arg)); +#endif } /************** @@ -992,6 +1094,7 @@ namespace xsimd WRAP_REDUCER_INT_EXCLUDING_64(vminvq) WRAP_REDUCER_FLOAT(vminvq) +#if !defined(_MSC_VER) || !defined(_M_ARM64) namespace wrap { XSIMD_INLINE uint64_t vminvq_u64(uint64x2_t a) noexcept @@ -1004,10 +1107,45 @@ namespace xsimd return std::min(vdupd_laneq_s64(a, 0), vdupd_laneq_s64(a, 1)); } } +#endif template = 0> XSIMD_INLINE typename batch::value_type reduce_min(batch const& arg, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + if constexpr (sizeof(T) == 1) + { + if constexpr (std::is_unsigned::value) + return vminvq_u8(arg); + else + return vminvq_s8(arg); + } + else if constexpr (sizeof(T) == 2) + { + if constexpr (std::is_unsigned::value) + return vminvq_u16(arg); + else + return vminvq_s16(arg); + } + else if constexpr (sizeof(T) == 4) + { + if constexpr (std::is_same::value) + return vminvq_f32(arg); + else if constexpr (std::is_unsigned::value) + return vminvq_u32(arg); + else + return vminvq_s32(arg); + } + else if constexpr (sizeof(T) == 8) + { + if constexpr (std::is_same::value) + return vminvq_f64(arg); + else if constexpr (std::is_unsigned::value) + return std::min(vdupd_laneq_u64(arg, 0), vdupd_laneq_u64(arg, 1)); + else + return std::min(vdupd_laneq_s64(arg, 0), vdupd_laneq_s64(arg, 1)); + } +#else using register_type = typename batch::register_type; const detail::neon_reducer_dispatcher::unary dispatcher = { std::make_tuple(wrap::vminvq_u8, wrap::vminvq_s8, wrap::vminvq_u16, wrap::vminvq_s16, @@ -1015,6 +1153,7 @@ namespace xsimd wrap::vminvq_f32, wrap::vminvq_f64) }; return dispatcher.apply(register_type(arg)); +#endif } #undef WRAP_REDUCER_INT_EXCLUDING_64 @@ -1260,6 +1399,7 @@ namespace xsimd * bitwise_cast * ****************/ +#if !defined(_MSC_VER) || !defined(_M_ARM64) #define WRAP_CAST(SUFFIX, TYPE) \ namespace wrap \ { \ @@ -1284,10 +1424,33 @@ namespace xsimd WRAP_CAST(f32, float32x4_t) #undef WRAP_CAST +#endif template XSIMD_INLINE batch bitwise_cast(batch const& arg, batch const&, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + using register_type = typename batch::register_type; + register_type reg = arg; + if constexpr (std::is_same::value) + return vreinterpretq_f64_u8(reg); + else if constexpr (std::is_same::value) + return vreinterpretq_f64_s8(reg); + else if constexpr (std::is_same::value) + return vreinterpretq_f64_u16(reg); + else if constexpr (std::is_same::value) + return vreinterpretq_f64_s16(reg); + else if constexpr (std::is_same::value) + return vreinterpretq_f64_u32(reg); + else if constexpr (std::is_same::value) + return vreinterpretq_f64_s32(reg); + else if constexpr (std::is_same::value) + return vreinterpretq_f64_u64(reg); + else if constexpr (std::is_same::value) + return vreinterpretq_f64_s64(reg); + else if constexpr (std::is_same::value) + return vreinterpretq_f64_f32(reg); +#else using caster_type = detail::bitwise_caster_impl::register_type; return caster.apply(register_type(arg)); +#endif } +#if !defined(_MSC_VER) || !defined(_M_ARM64) namespace detail { template @@ -1320,10 +1485,33 @@ namespace xsimd } }; } +#endif template XSIMD_INLINE batch bitwise_cast(batch const& arg, batch const&, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + using src_register_type = typename batch::register_type; + src_register_type reg = arg; + if constexpr (std::is_same::value) + return vreinterpretq_u8_f64(reg); + else if constexpr (std::is_same::value) + return vreinterpretq_s8_f64(reg); + else if constexpr (std::is_same::value) + return vreinterpretq_u16_f64(reg); + else if constexpr (std::is_same::value) + return vreinterpretq_s16_f64(reg); + else if constexpr (std::is_same::value) + return vreinterpretq_u32_f64(reg); + else if constexpr (std::is_same::value) + return vreinterpretq_s32_f64(reg); + else if constexpr (std::is_same::value) + return vreinterpretq_u64_f64(reg); + else if constexpr (std::is_same::value) + return vreinterpretq_s64_f64(reg); + else if constexpr (std::is_same::value) + return vreinterpretq_f32_f64(reg); +#else using caster_type = detail::bitwise_caster_neon64::register_type; using dst_register_type = typename batch::register_type; return caster.apply(src_register_type(arg)); +#endif } template @@ -1378,14 +1567,31 @@ namespace xsimd XSIMD_INLINE batch swizzle(batch const& self, batch idx, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + // MSVC ARM64: vqtbl1q_* are macro-based and conflict with our wrapper usage. + // Use the two-table lookup (vtbl2_u8) on low/high halves. + uint8x8x2_t tbl = { vget_low_u8(self), vget_high_u8(self) }; + uint8x8_t lo = vtbl2_u8(tbl, vget_low_u8(idx)); + uint8x8_t hi = vtbl2_u8(tbl, vget_high_u8(idx)); + return vcombine_u8(lo, hi); +#else return vqtbl1q_u8(self, idx); +#endif } template XSIMD_INLINE batch swizzle(batch const& self, batch idx, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + // Same approach as above but for signed payload. + uint8x8x2_t tbl = { vreinterpret_u8_s8(vget_low_s8(self)), vreinterpret_u8_s8(vget_high_s8(self)) }; + uint8x8_t lo = vtbl2_u8(tbl, vget_low_u8(idx)); + uint8x8_t hi = vtbl2_u8(tbl, vget_high_u8(idx)); + return vreinterpretq_s8_u8(vcombine_u8(lo, hi)); +#else return vqtbl1q_s8(self, idx); +#endif } template @@ -1395,9 +1601,26 @@ namespace xsimd { using batch_type = batch; using index_type = batch; +#if defined(_MSC_VER) && defined(_M_ARM64) + batch_type self_bytes = batch_type(vreinterpretq_u8_u16(self)); + constexpr std::size_t lanes = batch::size; + constexpr std::size_t elem_bytes = sizeof(uint16_t); + alignas(A::alignment()) uint16_t idx_in[lanes]; + idx.store_aligned(&idx_in[0]); + alignas(A::alignment()) uint8_t idx_out[batch_type::size]; + for (std::size_t j = 0; j < lanes; ++j) + { + std::size_t base = static_cast(idx_in[j]) * elem_bytes; + for (std::size_t k = 0; k < elem_bytes; ++k) + idx_out[j * elem_bytes + k] = static_cast(base + k); + } + index_type indices = index_type::load_aligned(&idx_out[0]); + return vreinterpretq_u16_u8(swizzle(self_bytes, indices, neon64 {})); +#else return vreinterpretq_u16_u8(swizzle(batch_type(vreinterpretq_u8_u16(self)), index_type(vreinterpretq_u8_u16(idx * 0x0202 + 0x0100)), neon64 {})); +#endif } template @@ -1415,9 +1638,26 @@ namespace xsimd { using batch_type = batch; using index_type = batch; +#if defined(_MSC_VER) && defined(_M_ARM64) + batch_type self_bytes = batch_type(vreinterpretq_u8_u32(self)); + constexpr std::size_t lanes = batch::size; + constexpr std::size_t elem_bytes = sizeof(uint32_t); + alignas(A::alignment()) uint32_t idx_in[lanes]; + idx.store_aligned(&idx_in[0]); + alignas(A::alignment()) uint8_t idx_out[batch_type::size]; + for (std::size_t j = 0; j < lanes; ++j) + { + std::size_t base = static_cast(idx_in[j]) * elem_bytes; + for (std::size_t k = 0; k < elem_bytes; ++k) + idx_out[j * elem_bytes + k] = static_cast(base + k); + } + index_type indices = index_type::load_aligned(&idx_out[0]); + return vreinterpretq_u32_u8(swizzle(self_bytes, indices, neon64 {})); +#else return vreinterpretq_u32_u8(swizzle(batch_type(vreinterpretq_u8_u32(self)), index_type(vreinterpretq_u8_u32(idx * 0x04040404 + 0x03020100)), neon64 {})); +#endif } template @@ -1435,9 +1675,26 @@ namespace xsimd { using batch_type = batch; using index_type = batch; +#if defined(_MSC_VER) && defined(_M_ARM64) + batch_type self_bytes = batch_type(vreinterpretq_u8_u64(self)); + constexpr std::size_t lanes = batch::size; + constexpr std::size_t elem_bytes = sizeof(uint64_t); + alignas(A::alignment()) uint64_t idx_in[lanes]; + idx.store_aligned(&idx_in[0]); + alignas(A::alignment()) uint8_t idx_out[batch_type::size]; + for (std::size_t j = 0; j < lanes; ++j) + { + std::size_t base = static_cast(idx_in[j]) * elem_bytes; + for (std::size_t k = 0; k < elem_bytes; ++k) + idx_out[j * elem_bytes + k] = static_cast(base + k); + } + index_type indices_batch = index_type::load_aligned(&idx_out[0]); + return vreinterpretq_u64_u8(swizzle(self_bytes, indices_batch, neon64 {})); +#else return vreinterpretq_u64_u8(swizzle(batch_type(vreinterpretq_u8_u64(self)), index_type(vreinterpretq_u8_u64(idx * 0x0808080808080808ull + 0x0706050403020100ull)), neon64 {})); +#endif } template @@ -1521,7 +1778,11 @@ namespace xsimd batch_constant idx, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + return swizzle(self, batch(idx), neon64 {}); +#else return vqtbl1q_u8(self, batch(idx)); +#endif } template idx, requires_arch) noexcept { +#if defined(_MSC_VER) && defined(_M_ARM64) + return swizzle(self, batch(idx), neon64 {}); +#else return vqtbl1q_s8(self, batch(idx)); +#endif } template @@ -1569,6 +1834,35 @@ namespace xsimd return vreinterpretq_s32_s8(swizzle(batch_type(vreinterpretq_s8_s32(self)), detail::burst_index(idx), A())); } +#if defined(_MSC_VER) && defined(_M_ARM64) + template + XSIMD_INLINE batch swizzle(batch const& self, + batch_constant, + requires_arch) noexcept + { + static_assert(batch::size == 2, "neon64 uint64 batch must have size 2"); + uint64_t in[2]; + uint64_t out[2]; + self.store_unaligned(in); + out[0] = in[V0]; + out[1] = in[V1]; + return batch::load_unaligned(out); + } + + template + XSIMD_INLINE batch swizzle(batch const& self, + batch_constant, + requires_arch) noexcept + { + static_assert(batch::size == 2, "neon64 int64 batch must have size 2"); + int64_t in[2]; + int64_t out[2]; + self.store_unaligned(in); + out[0] = in[V0]; + out[1] = in[V1]; + return batch::load_unaligned(out); + } +#else template XSIMD_INLINE batch swizzle(batch const& self, batch_constant idx, @@ -1586,6 +1880,7 @@ namespace xsimd using batch_type = batch; return vreinterpretq_s64_s8(swizzle(batch_type(vreinterpretq_s8_s64(self)), detail::burst_index(idx), A())); } +#endif template XSIMD_INLINE batch swizzle(batch const& self, @@ -1606,19 +1901,19 @@ namespace xsimd } template - XSIMD_INLINE batch, A> swizzle(batch, A> const& self, - batch_constant idx, - requires_arch) noexcept + XSIMD_INLINE batch<::std::complex, A> swizzle(batch<::std::complex, A> const& self, + batch_constant idx, + requires_arch) noexcept { - return batch>(swizzle(self.real(), idx, A()), swizzle(self.imag(), idx, A())); + return batch<::std::complex>(swizzle(self.real(), idx, A()), swizzle(self.imag(), idx, A())); } template - XSIMD_INLINE batch, A> swizzle(batch, A> const& self, - batch_constant idx, - requires_arch) noexcept + XSIMD_INLINE batch<::std::complex, A> swizzle(batch<::std::complex, A> const& self, + batch_constant idx, + requires_arch) noexcept { - return batch>(swizzle(self.real(), idx, A()), swizzle(self.imag(), idx, A())); + return batch<::std::complex>(swizzle(self.real(), idx, A()), swizzle(self.imag(), idx, A())); } /*********