Skip to content

Commit 122d083

Browse files
DiamonDinoiaserge-sans-paille
DiamonDinoia
authored and
serge-sans-paille
committed
fix: route avx512vl_128 FMA ops through fma3<avx2_128>
avx512vl_128 derived from avx2_128, whose dispatch chain reaches no FMA3 kernel provider, so fnma/fnms fell back to the generic neg(x*y)+/-z form (an extra vxorpd ahead of vfmadd) even on a real AVX-512 build. Its 256-bit sibling avx512vl_256 already derives from fma3<avx2> and was unaffected. Apply the same fma3<> inheritance pattern as avxvnni and avx512vl_256: introduce fma3<avx2_128> (a strict superset of avx2_128 that adds the 128-bit FMA3 kernels, built by re-including xsimd_fma3_sse.hpp with sse4_2 -> avx2_128, exactly as fma3<avx2> is built from xsimd_fma3_avx.hpp), and derive avx512vl_128 from it. Because fma3<avx2_128> : avx2_128, all avx2_128 integer kernels remain reachable -- FMA is added on top, nothing lost. The tag is gated by XSIMD_WITH_FMA3_AVX2, so without FMA the kernels are absent and dispatch falls through to avx2_128/common unchanged. Validated under Intel SDE -skx: avx512vl_128 fnma/fnms/fma/fms/fmas emit 128-bit vf* (no vxorpd) and match scalar std::fma for float and double; avxvnni and fma3<avx2> codegen unchanged; all FMA-capable x86 arches compile across sse/avx2/avxvnni/avx512 flag sets.
1 parent 801631c commit 122d083

6 files changed

Lines changed: 106 additions & 1 deletion

File tree

‎include/xsimd/arch/xsimd_fma3_avx2_128.hpp‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/***************************************************************************
2+
* Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and *
3+
* Martin Renou *
4+
* Copyright (c) QuantStack *
5+
* Copyright (c) Serge Guelton *
6+
* Copyright (c) Marco Barbone *
7+
* *
8+
* Distributed under the terms of the BSD 3-Clause License. *
9+
* *
10+
* The full license is in the file LICENSE, distributed with this software. *
11+
****************************************************************************/
12+
13+
#ifndef XSIMD_FMA3_AVX2_128_HPP
14+
#define XSIMD_FMA3_AVX2_128_HPP
15+
16+
#include "../types/xsimd_fma3_avx2_128_register.hpp"
17+
18+
// Allow inclusion of xsimd_fma3_sse.hpp
19+
#ifdef XSIMD_FMA3_SSE_HPP
20+
#undef XSIMD_FMA3_SSE_HPP
21+
#define XSIMD_FORCE_FMA3_SSE_HPP
22+
#endif
23+
24+
// Disallow inclusion of ./xsimd_fma3_sse_register.hpp
25+
#ifndef XSIMD_FMA3_SSE_REGISTER_HPP
26+
#define XSIMD_FMA3_SSE_REGISTER_HPP
27+
#define XSIMD_FORCE_FMA3_SSE_REGISTER_HPP
28+
#endif
29+
30+
// Include ./xsimd_fma3_sse.hpp but s/sse4_2/avx2_128
31+
#define sse4_2 avx2_128
32+
#include "./xsimd_fma3_sse.hpp"
33+
#undef sse4_2
34+
#undef XSIMD_FMA3_SSE_HPP
35+
36+
// Carefully restore guards
37+
#ifdef XSIMD_FORCE_FMA3_SSE_HPP
38+
#define XSIMD_FMA3_SSE_HPP
39+
#undef XSIMD_FORCE_FMA3_SSE_HPP
40+
#endif
41+
42+
#ifdef XSIMD_FORCE_FMA3_SSE_REGISTER_HPP
43+
#undef XSIMD_FMA3_SSE_REGISTER_HPP
44+
#undef XSIMD_FORCE_FMA3_SSE_REGISTER_HPP
45+
#endif
46+
47+
#endif

‎include/xsimd/arch/xsimd_isa.hpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
* Martin Renou *
44
* Copyright (c) QuantStack *
55
* Copyright (c) Serge Guelton *
6+
* Copyright (c) Marco Barbone *
67
* *
78
* Distributed under the terms of the BSD 3-Clause License. *
89
* *
@@ -72,6 +73,7 @@
7273

7374
#if XSIMD_WITH_FMA3_AVX2
7475
#include "./xsimd_fma3_avx2.hpp"
76+
#include "./xsimd_fma3_avx2_128.hpp"
7577
#endif
7678

7779
#if XSIMD_WITH_AVX512VL

‎include/xsimd/types/xsimd_all_registers.hpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
* Martin Renou *
44
* Copyright (c) QuantStack *
55
* Copyright (c) Serge Guelton *
6+
* Copyright (c) Marco Barbone *
67
* *
78
* Distributed under the terms of the BSD 3-Clause License. *
89
* *
@@ -24,6 +25,7 @@
2425
#include "./xsimd_avx512vnni_avx512vbmi2_register.hpp"
2526
#include "./xsimd_avx_register.hpp"
2627
#include "./xsimd_avxvnni_register.hpp"
28+
#include "./xsimd_fma3_avx2_128_register.hpp"
2729
#include "./xsimd_fma3_avx2_register.hpp"
2830
#include "./xsimd_fma3_avx_register.hpp"
2931
#include "./xsimd_fma3_sse_register.hpp"

‎include/xsimd/types/xsimd_avx512vl_register.hpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
* Martin Renou *
44
* Copyright (c) QuantStack *
55
* Copyright (c) Serge Guelton *
6+
* Copyright (c) Marco Barbone *
67
* *
78
* Distributed under the terms of the BSD 3-Clause License. *
89
* *
@@ -13,6 +14,7 @@
1314
#define XSIMD_AVX512VL_REGISTER_HPP
1415

1516
#include "./xsimd_avx512cd_register.hpp"
17+
#include "./xsimd_fma3_avx2_128_register.hpp"
1618

1719
namespace xsimd
1820
{
@@ -34,7 +36,7 @@ namespace xsimd
3436
*
3537
* AVX512VL instructions extension for 128 bits registers
3638
*/
37-
struct avx512vl_128 : avx2_128
39+
struct avx512vl_128 : fma3<avx2_128>
3840
{
3941
static constexpr bool supported() noexcept { return XSIMD_WITH_AVX512VL; }
4042
static constexpr bool available() noexcept { return true; }

‎include/xsimd/types/xsimd_avxvnni_register.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
* Martin Renou *
44
* Copyright (c) QuantStack *
55
* Copyright (c) Serge Guelton *
6+
* Copyright (c) Marco Barbone *
67
* *
78
* Distributed under the terms of the BSD 3-Clause License. *
89
* *

‎include/xsimd/types/xsimd_fma3_avx2_128_register.hpp‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/***************************************************************************
2+
* Copyright (c) Johan Mabille, Sylvain Corlay, Wolf Vollprecht and *
3+
* Martin Renou *
4+
* Copyright (c) QuantStack *
5+
* Copyright (c) Serge Guelton *
6+
* Copyright (c) Marco Barbone *
7+
* *
8+
* Distributed under the terms of the BSD 3-Clause License. *
9+
* *
10+
* The full license is in the file LICENSE, distributed with this software. *
11+
****************************************************************************/
12+
13+
#ifndef XSIMD_FMA3_AVX2_128_REGISTER_HPP
14+
#define XSIMD_FMA3_AVX2_128_REGISTER_HPP
15+
16+
#include "./xsimd_avx2_register.hpp"
17+
18+
namespace xsimd
19+
{
20+
template <typename arch>
21+
struct fma3;
22+
23+
/**
24+
* @ingroup architectures
25+
*
26+
* AVX2 + FMA instructions, for 128 bits registers
27+
*/
28+
template <>
29+
struct fma3<avx2_128> : avx2_128
30+
{
31+
static constexpr bool supported() noexcept { return XSIMD_WITH_FMA3_AVX2; }
32+
static constexpr bool available() noexcept { return true; }
33+
static constexpr char const* name() noexcept { return "fma3+avx2/128"; }
34+
};
35+
36+
#if XSIMD_WITH_FMA3_AVX2
37+
38+
#if !XSIMD_WITH_AVX2
39+
#error "architecture inconsistency: fma3+avx2/128 requires avx2"
40+
#endif
41+
42+
namespace types
43+
{
44+
45+
XSIMD_DECLARE_SIMD_REGISTER_ALIAS(fma3<avx2_128>, avx2_128);
46+
47+
}
48+
#endif
49+
50+
}
51+
#endif

0 commit comments

Comments
 (0)