Skip to content

Commit 19d9d27

Browse files
DiamonDinoiaserge-sans-paille
DiamonDinoia
authored and
serge-sans-paille
committed
perf(avx512vl): native EVEX for int64 sra, signed rotr, compress/expand
Add pure-VL overrides so these stop falling back to AVX2/SSE: - int64 signed >> -> vpsraq/vpsravq (unsigned/32-bit unchanged) - signed rotr -> vprorvq/vprolq (drop is_unsigned guard, mirror rotl) - compress/expand -> EVEX forms; 8/16-bit fall through to common{}
1 parent 122d083 commit 19d9d27

2 files changed

Lines changed: 254 additions & 40 deletions

File tree

‎include/xsimd/arch/xsimd_avx512vl_128.hpp‎

Lines changed: 127 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -404,6 +404,48 @@ namespace xsimd
404404
return (typename batch_bool<double, A>::register_type)_mm_cmp_pd_mask(self, self, _CMP_UNORD_Q);
405405
}
406406

407+
// bitwise_rshift — signed int64 uses the native EVEX arithmetic shift
408+
// (VPSRAQ / VPSRAVQ, lat 1 / CPI 0.5). Every other width/sign keeps the
409+
// inherited avx2_128 codegen (srai/srav for 32-bit, srli for unsigned 64).
410+
template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
411+
XSIMD_INLINE batch<T, A> bitwise_rshift(batch<T, A> const& self, int32_t other, requires_arch<avx512vl_128>) noexcept
412+
{
413+
XSIMD_IF_CONSTEXPR(std::is_signed<T>::value && sizeof(T) == 8)
414+
{
415+
return _mm_srai_epi64(self, other);
416+
}
417+
else
418+
{
419+
return bitwise_rshift(self, other, avx2_128 {});
420+
}
421+
}
422+
template <size_t shift, class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
423+
XSIMD_INLINE batch<T, A> bitwise_rshift(batch<T, A> const& self, requires_arch<avx512vl_128>) noexcept
424+
{
425+
constexpr auto bits = std::numeric_limits<T>::digits + std::numeric_limits<T>::is_signed;
426+
static_assert(shift < bits, "Shift amount must be less than the number of bits in T");
427+
XSIMD_IF_CONSTEXPR(std::is_signed<T>::value && sizeof(T) == 8)
428+
{
429+
return _mm_srai_epi64(self, shift);
430+
}
431+
else
432+
{
433+
return bitwise_rshift<shift>(self, avx2_128 {});
434+
}
435+
}
436+
template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
437+
XSIMD_INLINE batch<T, A> bitwise_rshift(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_128>) noexcept
438+
{
439+
XSIMD_IF_CONSTEXPR(std::is_signed<T>::value && sizeof(T) == 8)
440+
{
441+
return _mm_srav_epi64(self, other);
442+
}
443+
else
444+
{
445+
return bitwise_rshift(self, other, avx2_128 {});
446+
}
447+
}
448+
407449
// rotl
408450
template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
409451
XSIMD_INLINE batch<T, A> rotl(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_128>) noexcept
@@ -449,18 +491,18 @@ namespace xsimd
449491
template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
450492
XSIMD_INLINE batch<T, A> rotr(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_128>) noexcept
451493
{
452-
XSIMD_IF_CONSTEXPR(std::is_unsigned<T>::value)
494+
XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
453495
{
454-
XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
455-
{
456-
return _mm_rorv_epi32(self, other);
457-
}
458-
else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
459-
{
460-
return _mm_rorv_epi64(self, other);
461-
}
496+
return _mm_rorv_epi32(self, other);
497+
}
498+
else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
499+
{
500+
return _mm_rorv_epi64(self, other);
501+
}
502+
else
503+
{
504+
return rotr(self, other, avx2_128 {});
462505
}
463-
return rotr(self, other, avx2_128 {});
464506
}
465507
template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
466508
XSIMD_INLINE batch<T, A> rotr(batch<T, A> const& self, int32_t other, requires_arch<avx512vl_128>) noexcept
@@ -473,18 +515,83 @@ namespace xsimd
473515
{
474516
constexpr auto bits = std::numeric_limits<T>::digits + std::numeric_limits<T>::is_signed;
475517
static_assert(count < bits, "Count must be less than the number of bits in T");
476-
XSIMD_IF_CONSTEXPR(std::is_unsigned<T>::value)
518+
XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
477519
{
478-
XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
479-
{
480-
return _mm_ror_epi32(self, count);
481-
}
482-
else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
483-
{
484-
return _mm_ror_epi64(self, count);
485-
}
520+
return _mm_ror_epi32(self, count);
486521
}
487-
return rotr<count>(self, avx2_128 {});
522+
else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
523+
{
524+
return _mm_ror_epi64(self, count);
525+
}
526+
else
527+
{
528+
return rotr<count>(self, avx2_128 {});
529+
}
530+
}
531+
532+
// compress — native EVEX VPCOMPRESS{PS,PD,Q,D} for the widths with VL
533+
// forms. 8/16-bit need AVX512_VBMI2, so they fall through to common{}.
534+
template <class A>
535+
XSIMD_INLINE batch<float, A> compress(batch<float, A> const& self, batch_bool<float, A> const& mask, requires_arch<avx512vl_128>) noexcept
536+
{
537+
return _mm_maskz_compress_ps(mask.mask(), self);
538+
}
539+
template <class A>
540+
XSIMD_INLINE batch<double, A> compress(batch<double, A> const& self, batch_bool<double, A> const& mask, requires_arch<avx512vl_128>) noexcept
541+
{
542+
return _mm_maskz_compress_pd(mask.mask(), self);
543+
}
544+
template <class A>
545+
XSIMD_INLINE batch<int32_t, A> compress(batch<int32_t, A> const& self, batch_bool<int32_t, A> const& mask, requires_arch<avx512vl_128>) noexcept
546+
{
547+
return _mm_maskz_compress_epi32(mask.mask(), self);
548+
}
549+
template <class A>
550+
XSIMD_INLINE batch<uint32_t, A> compress(batch<uint32_t, A> const& self, batch_bool<uint32_t, A> const& mask, requires_arch<avx512vl_128>) noexcept
551+
{
552+
return _mm_maskz_compress_epi32(mask.mask(), self);
553+
}
554+
template <class A>
555+
XSIMD_INLINE batch<int64_t, A> compress(batch<int64_t, A> const& self, batch_bool<int64_t, A> const& mask, requires_arch<avx512vl_128>) noexcept
556+
{
557+
return _mm_maskz_compress_epi64(mask.mask(), self);
558+
}
559+
template <class A>
560+
XSIMD_INLINE batch<uint64_t, A> compress(batch<uint64_t, A> const& self, batch_bool<uint64_t, A> const& mask, requires_arch<avx512vl_128>) noexcept
561+
{
562+
return _mm_maskz_compress_epi64(mask.mask(), self);
563+
}
564+
565+
// expand
566+
template <class A>
567+
XSIMD_INLINE batch<float, A> expand(batch<float, A> const& self, batch_bool<float, A> const& mask, requires_arch<avx512vl_128>) noexcept
568+
{
569+
return _mm_maskz_expand_ps(mask.mask(), self);
570+
}
571+
template <class A>
572+
XSIMD_INLINE batch<double, A> expand(batch<double, A> const& self, batch_bool<double, A> const& mask, requires_arch<avx512vl_128>) noexcept
573+
{
574+
return _mm_maskz_expand_pd(mask.mask(), self);
575+
}
576+
template <class A>
577+
XSIMD_INLINE batch<int32_t, A> expand(batch<int32_t, A> const& self, batch_bool<int32_t, A> const& mask, requires_arch<avx512vl_128>) noexcept
578+
{
579+
return _mm_maskz_expand_epi32(mask.mask(), self);
580+
}
581+
template <class A>
582+
XSIMD_INLINE batch<uint32_t, A> expand(batch<uint32_t, A> const& self, batch_bool<uint32_t, A> const& mask, requires_arch<avx512vl_128>) noexcept
583+
{
584+
return _mm_maskz_expand_epi32(mask.mask(), self);
585+
}
586+
template <class A>
587+
XSIMD_INLINE batch<int64_t, A> expand(batch<int64_t, A> const& self, batch_bool<int64_t, A> const& mask, requires_arch<avx512vl_128>) noexcept
588+
{
589+
return _mm_maskz_expand_epi64(mask.mask(), self);
590+
}
591+
template <class A>
592+
XSIMD_INLINE batch<uint64_t, A> expand(batch<uint64_t, A> const& self, batch_bool<uint64_t, A> const& mask, requires_arch<avx512vl_128>) noexcept
593+
{
594+
return _mm_maskz_expand_epi64(mask.mask(), self);
488595
}
489596

490597
// all

‎include/xsimd/arch/xsimd_avx512vl_256.hpp‎

Lines changed: 127 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -494,6 +494,48 @@ namespace xsimd
494494
return (typename batch_bool<double, A>::register_type)_mm256_cmp_pd_mask(self, self, _CMP_UNORD_Q);
495495
}
496496

497+
// bitwise_rshift — signed int64 uses the native EVEX arithmetic shift
498+
// (VPSRAQ / VPSRAVQ, lat 1 / CPI 0.5). Every other width/sign keeps the
499+
// inherited avx2 codegen (srai/srav for 32-bit, srli for unsigned 64).
500+
template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
501+
XSIMD_INLINE batch<T, A> bitwise_rshift(batch<T, A> const& self, int32_t other, requires_arch<avx512vl_256>) noexcept
502+
{
503+
XSIMD_IF_CONSTEXPR(std::is_signed<T>::value && sizeof(T) == 8)
504+
{
505+
return _mm256_srai_epi64(self, other);
506+
}
507+
else
508+
{
509+
return bitwise_rshift(self, other, avx2 {});
510+
}
511+
}
512+
template <size_t shift, class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
513+
XSIMD_INLINE batch<T, A> bitwise_rshift(batch<T, A> const& self, requires_arch<avx512vl_256>) noexcept
514+
{
515+
constexpr auto bits = std::numeric_limits<T>::digits + std::numeric_limits<T>::is_signed;
516+
static_assert(shift < bits, "Shift amount must be less than the number of bits in T");
517+
XSIMD_IF_CONSTEXPR(std::is_signed<T>::value && sizeof(T) == 8)
518+
{
519+
return _mm256_srai_epi64(self, shift);
520+
}
521+
else
522+
{
523+
return bitwise_rshift<shift>(self, avx2 {});
524+
}
525+
}
526+
template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
527+
XSIMD_INLINE batch<T, A> bitwise_rshift(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_256>) noexcept
528+
{
529+
XSIMD_IF_CONSTEXPR(std::is_signed<T>::value && sizeof(T) == 8)
530+
{
531+
return _mm256_srav_epi64(self, other);
532+
}
533+
else
534+
{
535+
return bitwise_rshift(self, other, avx2 {});
536+
}
537+
}
538+
497539
// rotl
498540
template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
499541
XSIMD_INLINE batch<T, A> rotl(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_256>) noexcept
@@ -539,18 +581,18 @@ namespace xsimd
539581
template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
540582
XSIMD_INLINE batch<T, A> rotr(batch<T, A> const& self, batch<T, A> const& other, requires_arch<avx512vl_256>) noexcept
541583
{
542-
XSIMD_IF_CONSTEXPR(std::is_unsigned<T>::value)
584+
XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
543585
{
544-
XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
545-
{
546-
return _mm256_rorv_epi32(self, other);
547-
}
548-
else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
549-
{
550-
return _mm256_rorv_epi64(self, other);
551-
}
586+
return _mm256_rorv_epi32(self, other);
587+
}
588+
else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
589+
{
590+
return _mm256_rorv_epi64(self, other);
591+
}
592+
else
593+
{
594+
return rotr(self, other, avx2 {});
552595
}
553-
return rotr(self, other, avx2 {});
554596
}
555597
template <class A, class T, class = std::enable_if_t<std::is_integral<T>::value>>
556598
XSIMD_INLINE batch<T, A> rotr(batch<T, A> const& self, int32_t other, requires_arch<avx512vl_256>) noexcept
@@ -563,18 +605,83 @@ namespace xsimd
563605
{
564606
constexpr auto bits = std::numeric_limits<T>::digits + std::numeric_limits<T>::is_signed;
565607
static_assert(count < bits, "Count must be less than the number of bits in T");
566-
XSIMD_IF_CONSTEXPR(std::is_unsigned<T>::value)
608+
XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
567609
{
568-
XSIMD_IF_CONSTEXPR(sizeof(T) == 4)
569-
{
570-
return _mm256_ror_epi32(self, count);
571-
}
572-
else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
573-
{
574-
return _mm256_ror_epi64(self, count);
575-
}
610+
return _mm256_ror_epi32(self, count);
576611
}
577-
return rotr<count>(self, avx2 {});
612+
else XSIMD_IF_CONSTEXPR(sizeof(T) == 8)
613+
{
614+
return _mm256_ror_epi64(self, count);
615+
}
616+
else
617+
{
618+
return rotr<count>(self, avx2 {});
619+
}
620+
}
621+
622+
// compress — native EVEX VPCOMPRESS{PS,PD,Q,D} for the widths with VL
623+
// forms. 8/16-bit need AVX512_VBMI2, so they fall through to common{}.
624+
template <class A>
625+
XSIMD_INLINE batch<float, A> compress(batch<float, A> const& self, batch_bool<float, A> const& mask, requires_arch<avx512vl_256>) noexcept
626+
{
627+
return _mm256_maskz_compress_ps(mask.mask(), self);
628+
}
629+
template <class A>
630+
XSIMD_INLINE batch<double, A> compress(batch<double, A> const& self, batch_bool<double, A> const& mask, requires_arch<avx512vl_256>) noexcept
631+
{
632+
return _mm256_maskz_compress_pd(mask.mask(), self);
633+
}
634+
template <class A>
635+
XSIMD_INLINE batch<int32_t, A> compress(batch<int32_t, A> const& self, batch_bool<int32_t, A> const& mask, requires_arch<avx512vl_256>) noexcept
636+
{
637+
return _mm256_maskz_compress_epi32(mask.mask(), self);
638+
}
639+
template <class A>
640+
XSIMD_INLINE batch<uint32_t, A> compress(batch<uint32_t, A> const& self, batch_bool<uint32_t, A> const& mask, requires_arch<avx512vl_256>) noexcept
641+
{
642+
return _mm256_maskz_compress_epi32(mask.mask(), self);
643+
}
644+
template <class A>
645+
XSIMD_INLINE batch<int64_t, A> compress(batch<int64_t, A> const& self, batch_bool<int64_t, A> const& mask, requires_arch<avx512vl_256>) noexcept
646+
{
647+
return _mm256_maskz_compress_epi64(mask.mask(), self);
648+
}
649+
template <class A>
650+
XSIMD_INLINE batch<uint64_t, A> compress(batch<uint64_t, A> const& self, batch_bool<uint64_t, A> const& mask, requires_arch<avx512vl_256>) noexcept
651+
{
652+
return _mm256_maskz_compress_epi64(mask.mask(), self);
653+
}
654+
655+
// expand
656+
template <class A>
657+
XSIMD_INLINE batch<float, A> expand(batch<float, A> const& self, batch_bool<float, A> const& mask, requires_arch<avx512vl_256>) noexcept
658+
{
659+
return _mm256_maskz_expand_ps(mask.mask(), self);
660+
}
661+
template <class A>
662+
XSIMD_INLINE batch<double, A> expand(batch<double, A> const& self, batch_bool<double, A> const& mask, requires_arch<avx512vl_256>) noexcept
663+
{
664+
return _mm256_maskz_expand_pd(mask.mask(), self);
665+
}
666+
template <class A>
667+
XSIMD_INLINE batch<int32_t, A> expand(batch<int32_t, A> const& self, batch_bool<int32_t, A> const& mask, requires_arch<avx512vl_256>) noexcept
668+
{
669+
return _mm256_maskz_expand_epi32(mask.mask(), self);
670+
}
671+
template <class A>
672+
XSIMD_INLINE batch<uint32_t, A> expand(batch<uint32_t, A> const& self, batch_bool<uint32_t, A> const& mask, requires_arch<avx512vl_256>) noexcept
673+
{
674+
return _mm256_maskz_expand_epi32(mask.mask(), self);
675+
}
676+
template <class A>
677+
XSIMD_INLINE batch<int64_t, A> expand(batch<int64_t, A> const& self, batch_bool<int64_t, A> const& mask, requires_arch<avx512vl_256>) noexcept
678+
{
679+
return _mm256_maskz_expand_epi64(mask.mask(), self);
680+
}
681+
template <class A>
682+
XSIMD_INLINE batch<uint64_t, A> expand(batch<uint64_t, A> const& self, batch_bool<uint64_t, A> const& mask, requires_arch<avx512vl_256>) noexcept
683+
{
684+
return _mm256_maskz_expand_epi64(mask.mask(), self);
578685
}
579686

580687
// all

0 commit comments

Comments
 (0)