diff --git a/lib/SPIRV/SPIRVRegularizeLLVM.cpp b/lib/SPIRV/SPIRVRegularizeLLVM.cpp index 2bb72c83d..a33ec2475 100644 --- a/lib/SPIRV/SPIRVRegularizeLLVM.cpp +++ b/lib/SPIRV/SPIRVRegularizeLLVM.cpp @@ -870,6 +870,12 @@ bool SPIRVRegularizeLLVMBase::regularize() { auto *Res = addCallInstSPIRV(M, "__spirv_AtomicCompareExchange", MemType, Args, nullptr, {MemType}, &II, "cmpxchg.res"); + for (auto &&MD : {"amdgpu.no.fine.grained.memory", + "amdgpu.no.remote.memory", + "amdgpu.ignore.denormal.mode"}) { + if (auto *N = Cmpxchg->getMetadata(MD)) + cast(Res)->setMetadata(MD, N); + } IRBuilder<> Builder(Cmpxchg); auto *Cmp = Builder.CreateICmpEQ(Res, Comparator, "cmpxchg.success"); auto *V1 = Builder.CreateInsertValue( diff --git a/lib/SPIRV/SPIRVWriter.cpp b/lib/SPIRV/SPIRVWriter.cpp index 54df3ce96..164fee165 100644 --- a/lib/SPIRV/SPIRVWriter.cpp +++ b/lib/SPIRV/SPIRVWriter.cpp @@ -2057,6 +2057,18 @@ LLVMToSPIRVBase::getLoopControl(const Instruction *Branch, return static_cast(LoopControl); } +static void addAMDGPUAtomicDecorations(Instruction *Src, SPIRVValue *Dst, + const Module *M) { + if (M->getTargetTriple().getVendor() != Triple::VendorType::AMD) + return; + for (auto &&MD : {"amdgpu.no.fine.grained.memory", + "amdgpu.no.remote.memory", + "amdgpu.ignore.denormal.mode"}) { + if (Src->hasMetadata(MD)) + Dst->addDecorate(new SPIRVDecorateUserSemanticAttr(Dst, MD)); + } +} + static int transAtomicOrdering(llvm::AtomicOrdering Ordering) { return OCLMemOrderMap::map( static_cast(llvm::toCABI(Ordering))); @@ -2878,12 +2890,15 @@ LLVMToSPIRVBase::transValueWithoutDecoration(Value *V, SPIRVBasicBlock *BB, auto IncDec = mapValue(V, BM->addInstTemplate(OC, Ops, BB, Ty)); IncDec->addDecorate( new SPIRVDecorate(DecorationMaxByteOffsetId, IncDec, WrapV)); + addAMDGPUAtomicDecorations(ARMW, IncDec, M); return IncDec; // TODO: figure out handling of saturating val. } else OC = LLVMSPIRVAtomicRmwOpCodeMap::map(Op); - return mapValue(V, BM->addInstTemplate(OC, Ops, BB, Ty)); + auto Result = mapValue(V, BM->addInstTemplate(OC, Ops, BB, Ty)); + addAMDGPUAtomicDecorations(ARMW, Result, M); + return Result; } if (IntrinsicInst *II = dyn_cast(V)) { @@ -6674,6 +6689,8 @@ SPIRVInstruction *LLVMToSPIRVBase::transBuiltinToInst(StringRef DemangledName, auto *Inst = transBuiltinToInstWithoutDecoration(OC, CI, BB); addDecorations(Inst, Dec); + if (Inst && isAtomicOpCode(OC)) + addAMDGPUAtomicDecorations(CI, Inst, M); return Inst; } diff --git a/test/amdgpu-atomic-metadata-non-amd.ll b/test/amdgpu-atomic-metadata-non-amd.ll new file mode 100644 index 000000000..21c1cc485 --- /dev/null +++ b/test/amdgpu-atomic-metadata-non-amd.ll @@ -0,0 +1,24 @@ +; RUN: llvm-as < %s -o %t.bc +; RUN: llvm-spirv %t.bc -o %t.spv +; RUN: llvm-spirv -to-text %t.spv -o - | FileCheck %s + +; Check that atomic instructions with amdgpu atomic metadata DO NOT produce +; UserSemantic decorations when NOT using AMD triple. + +target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" +target triple = "spir64" + +@ui = common dso_local addrspace(1) global i32 0, align 4 + +; CHECK: AtomicIAdd +; CHECK-NOT: UserSemantic "amdgpu.no.fine.grained.memory" +; CHECK-NOT: UserSemantic "amdgpu.no.remote.memory" +; CHECK-NOT: UserSemantic "amdgpu.ignore.denormal.mode" + +define dso_local spir_func void @test_non_amd() { +entry: + %0 = atomicrmw add ptr addrspace(1) @ui, i32 42 monotonic, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0 + ret void +} + +!0 = !{} diff --git a/test/amdgpu-atomic-metadata.ll b/test/amdgpu-atomic-metadata.ll new file mode 100644 index 000000000..74cf04dbd --- /dev/null +++ b/test/amdgpu-atomic-metadata.ll @@ -0,0 +1,53 @@ +; RUN: llvm-as < %s -o %t.bc +; RUN: llvm-spirv %t.bc -o %t.spv +; RUN: llvm-spirv -to-text %t.spv -o - | FileCheck %s + +; Check that atomic instructions with amdgpu atomic metadata produce +; UserSemantic decorations. + +target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" +target triple = "spirv64-amd-amdhsa" + +@ui = common dso_local addrspace(1) global i32 0, align 4 +@f = common dso_local addrspace(1) global float 0.000000e+00, align 4 + +; CHECK: Decorate [[#ADD_RES:]] UserSemantic "amdgpu.no.fine.grained.memory" +; CHECK: Decorate [[#ADD_RES]] UserSemantic "amdgpu.no.remote.memory" +; CHECK-NOT: Decorate [[#ADD_RES]] UserSemantic "amdgpu.ignore.denormal.mode" + +; CHECK: Decorate [[#XCHG_RES:]] UserSemantic "amdgpu.no.fine.grained.memory" +; CHECK: Decorate [[#XCHG_RES]] UserSemantic "amdgpu.ignore.denormal.mode" +; CHECK-NOT: Decorate [[#XCHG_RES]] UserSemantic "amdgpu.no.remote.memory" + +; CHECK: Decorate [[#CAS_RES:]] UserSemantic "amdgpu.no.fine.grained.memory" +; CHECK: Decorate [[#CAS_RES]] UserSemantic "amdgpu.no.remote.memory" +; CHECK: Decorate [[#CAS_RES]] UserSemantic "amdgpu.ignore.denormal.mode" + +; CHECK: AtomicIAdd [[#]] [[#ADD_RES]] +; CHECK: AtomicExchange [[#]] [[#XCHG_RES]] +; CHECK: AtomicCompareExchange [[#]] [[#CAS_RES]] + +define dso_local spir_func void @test_atomicrmw_metadata() { +entry: + %0 = atomicrmw add ptr addrspace(1) @ui, i32 42 monotonic, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0 + %1 = atomicrmw xchg ptr addrspace(1) @f, float 42.0 seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0 + ret void +} + +define dso_local spir_func void @test_cmpxchg_metadata(ptr %ptr, ptr %value_ptr, i32 %comparator) { +entry: + %0 = load i32, ptr %value_ptr, align 4 + %1 = cmpxchg ptr %ptr, i32 %comparator, i32 %0 seq_cst acquire, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0 + %2 = extractvalue { i32, i1 } %1, 1 + br i1 %2, label %cmpxchg.continue, label %cmpxchg.store_expected + +cmpxchg.store_expected: + %3 = extractvalue { i32, i1 } %1, 0 + store i32 %3, ptr %value_ptr, align 4 + br label %cmpxchg.continue + +cmpxchg.continue: + ret void +} + +!0 = !{}