Skip to content

Commit f449a5f

Browse files
bartekxkbartekxk
authored
Disable fp32 atomic adds on gfx11 (#3510)
* Disable fp32 atomic adds on gfx11 * Fixes is supported
1 parent aad4cf0 commit f449a5f

4 files changed

Lines changed: 57 additions & 45 deletions

‎include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_data_multiple_d_wmma_cshuffle_v3.hpp‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -63,11 +63,7 @@ __launch_bounds__(CK_MAX_THREAD_PER_BLOCK, MinimumOccupancy)
6363
{
6464
#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx11__) || defined(__gfx12__))
6565
#if defined(__gfx11__)
66-
// gfx11 does not support *_atomic_pk_add_f16/bf16 instructions
67-
using e_data_type = remove_cvref_t<remove_pointer_t<decltype(karg.p_e_grid)>>;
68-
if constexpr(!(EGlobalMemoryDataOperation == InMemoryDataOperationEnum::AtomicAdd &&
69-
(std::is_same_v<e_data_type, ck::half_t> ||
70-
std::is_same_v<e_data_type, ck::bhalf_t>)))
66+
if constexpr(EGlobalMemoryDataOperation != InMemoryDataOperationEnum::AtomicAdd)
7167
{
7268
#endif
7369
__shared__ char p_shared[GridwiseGemm::template GetSharedMemoryNumberOfByte<

‎include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_multiple_d_wmma_cshuffle_v3.hpp‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,7 @@ __launch_bounds__(CK_MAX_THREAD_PER_BLOCK, MinimumOccupancy)
6262
#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx11__) || defined(__gfx12__))
6363
#if defined(__gfx11__)
6464
// gfx11 does not support *_atomic_pk_add_f16/bf16 instructions
65-
using c_data_type = remove_cvref_t<remove_pointer_t<decltype(karg.p_e_grid)>>;
66-
if constexpr(!(CGlobalMemoryDataOperation == InMemoryDataOperationEnum::AtomicAdd &&
67-
(std::is_same_v<c_data_type, ck::half_t> ||
68-
std::is_same_v<c_data_type, ck::bhalf_t>)))
65+
if constexpr(CGlobalMemoryDataOperation != InMemoryDataOperationEnum::AtomicAdd)
6966
{
7067
#endif
7168

@@ -1028,6 +1025,17 @@ struct DeviceGroupedConvBwdWeightMultipleD_Wmma_CShuffleV3
10281025
{
10291026
return false;
10301027
}
1028+
1029+
if(arg.k_batch_ > 1 && ck::is_gfx11_supported())
1030+
{
1031+
if(ck::EnvIsEnabled(CK_ENV(CK_LOGGING)))
1032+
{
1033+
std::cout << "Unsupported splitK on gfx11." << std::endl;
1034+
}
1035+
// gfx11 does not support *_atomic_pk_add_f16/bf16 instructions
1036+
return false;
1037+
}
1038+
10311039
if constexpr(std::is_same_v<ComputeTypeA, f8_t> || std::is_same_v<ComputeTypeA, bf8_t> ||
10321040
std::is_same_v<ComputeTypeB, f8_t> || std::is_same_v<ComputeTypeB, bf8_t>)
10331041
{

‎include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_two_stage_wmma_cshuffle_v3.hpp‎

Lines changed: 38 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -63,28 +63,34 @@ __launch_bounds__(CK_MAX_THREAD_PER_BLOCK, MinimumOccupancy)
6363
const index_t num_k_per_block)
6464
{
6565
#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx11__) || defined(__gfx12__))
66-
67-
constexpr index_t LDS_size = GridwiseGemm::template GetSharedMemoryNumberOfByte<
68-
typename GridwiseGemm::EpilogueCShuffle>();
69-
__shared__ char p_shared[LDS_size];
70-
71-
auto epilogue_args = typename GridwiseGemm::EpilogueCShuffle{};
72-
73-
GridwiseGemm::template Run<AGridDesc_AK0_M_K1,
74-
BGridDesc_BK0_N_K1,
75-
CGridDesc_MBlock_MPerBlock_NBlock_NPerBlock,
76-
ComputePtrOffsetOfBatch,
77-
NumGroupsToMerge,
78-
HasMainKBlockLoop,
79-
CGlobalMemoryDataOperation,
80-
TailNum>(p_shared,
81-
a_grid_desc_ak0_m_ak1,
82-
b_grid_desc_bk0_n_bk1,
83-
c_grid_desc_mblock_mperblock_nblock_nperblock,
84-
compute_ptr_offset_of_batch,
85-
num_k_per_block,
86-
karg,
87-
epilogue_args);
66+
#if defined(__gfx11__)
67+
if constexpr(CGlobalMemoryDataOperation != InMemoryDataOperationEnum::AtomicAdd)
68+
{
69+
#endif
70+
constexpr index_t LDS_size = GridwiseGemm::template GetSharedMemoryNumberOfByte<
71+
typename GridwiseGemm::EpilogueCShuffle>();
72+
__shared__ char p_shared[LDS_size];
73+
74+
auto epilogue_args = typename GridwiseGemm::EpilogueCShuffle{};
75+
76+
GridwiseGemm::template Run<AGridDesc_AK0_M_K1,
77+
BGridDesc_BK0_N_K1,
78+
CGridDesc_MBlock_MPerBlock_NBlock_NPerBlock,
79+
ComputePtrOffsetOfBatch,
80+
NumGroupsToMerge,
81+
HasMainKBlockLoop,
82+
CGlobalMemoryDataOperation,
83+
TailNum>(p_shared,
84+
a_grid_desc_ak0_m_ak1,
85+
b_grid_desc_bk0_n_bk1,
86+
c_grid_desc_mblock_mperblock_nblock_nperblock,
87+
compute_ptr_offset_of_batch,
88+
num_k_per_block,
89+
karg,
90+
epilogue_args);
91+
#if defined(__gfx11__)
92+
}
93+
#endif
8894
#else
8995
ignore = karg;
9096
ignore = a_grid_desc_ak0_m_ak1;
@@ -1179,6 +1185,16 @@ struct DeviceGroupedConvBwdWeightTwoStage_Wmma_CShuffleV3
11791185
return false;
11801186
}
11811187

1188+
if(arg.k_batch_ > 1 && ck::is_gfx11_supported())
1189+
{
1190+
if(ck::EnvIsEnabled(CK_ENV(CK_LOGGING)))
1191+
{
1192+
std::cout << "Unsupported splitK on gfx11." << std::endl;
1193+
}
1194+
// gfx11 does not support *_atomic_pk_add_f16/bf16 instructions
1195+
return false;
1196+
}
1197+
11821198
// Check this here, it allows to use other instances from factory even
11831199
// if workspace is not allocated
11841200
if(!arg.p_workspace_)

‎include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_wmma_cshuffle_v3.hpp‎

Lines changed: 6 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -64,11 +64,7 @@ __launch_bounds__(CK_MAX_THREAD_PER_BLOCK, MinimumOccupancy)
6464
{
6565
#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx11__) || defined(__gfx12__))
6666
#if defined(__gfx11__)
67-
// gfx11 does not support *_atomic_pk_add_f16/bf16 instructions
68-
using e_data_type = remove_cvref_t<remove_pointer_t<decltype(karg.p_e_grid)>>;
69-
if constexpr(!(CGlobalMemoryDataOperation == InMemoryDataOperationEnum::AtomicAdd &&
70-
(std::is_same_v<e_data_type, ck::half_t> ||
71-
std::is_same_v<e_data_type, ck::bhalf_t>)))
67+
if constexpr(CGlobalMemoryDataOperation != InMemoryDataOperationEnum::AtomicAdd)
7268
{
7369
#endif
7470
constexpr index_t LDS_size = GridwiseGemm::template GetSharedMemoryNumberOfByte<
@@ -1089,18 +1085,14 @@ struct DeviceGroupedConvBwdWeight_Wmma_CShuffleV3
10891085
return false;
10901086
}
10911087

1092-
if constexpr(std::is_same_v<CDataType, ck::half_t> ||
1093-
std::is_same_v<CDataType, ck::bhalf_t>)
1088+
if(gemm_arg.KBatch > 1 && ck::is_gfx11_supported())
10941089
{
1095-
if(gemm_arg.KBatch > 1 && ck::is_gfx11_supported())
1090+
if(ck::EnvIsEnabled(CK_ENV(CK_LOGGING)))
10961091
{
1097-
if(ck::EnvIsEnabled(CK_ENV(CK_LOGGING)))
1098-
{
1099-
std::cout << "Unsupported splitK on gfx11." << std::endl;
1100-
}
1101-
// gfx11 does not support *_atomic_pk_add_f16/bf16 instructions
1102-
return false;
1092+
std::cout << "Unsupported splitK on gfx11." << std::endl;
11031093
}
1094+
// gfx11 does not support *_atomic_pk_add_f16/bf16 instructions
1095+
return false;
11041096
}
11051097

11061098
if constexpr(std::is_same_v<ComputeTypeA, f8_t> || std::is_same_v<ComputeTypeA, bf8_t> ||

0 commit comments

Comments
 (0)