Encode atomic metadata as UserSemantic string decoration

lib/SPIRV/SPIRVRegularizeLLVM.cpp

@@ -870,6 +870,12 @@ bool SPIRVRegularizeLLVMBase::regularize() {
           auto *Res =
               addCallInstSPIRV(M, "__spirv_AtomicCompareExchange", MemType,
                                Args, nullptr, {MemType}, &II, "cmpxchg.res");
+          for (auto &&MD : {"amdgpu.no.fine.grained.memory",
+                            "amdgpu.no.remote.memory",
+                            "amdgpu.ignore.denormal.mode"}) {
+            if (auto *N = Cmpxchg->getMetadata(MD))
+              cast<Instruction>(Res)->setMetadata(MD, N);
+          }
           IRBuilder<> Builder(Cmpxchg);
           auto *Cmp = Builder.CreateICmpEQ(Res, Comparator, "cmpxchg.success");
           auto *V1 = Builder.CreateInsertValue(

lib/SPIRV/SPIRVWriter.cpp

@@ -2057,6 +2057,18 @@ LLVMToSPIRVBase::getLoopControl(const Instruction *Branch,
   return static_cast<spv::LoopControlMask>(LoopControl);
 }
 
+static void addAMDGPUAtomicDecorations(Instruction *Src, SPIRVValue *Dst,
+                                       const Module *M) {
+  if (M->getTargetTriple().getVendor() != Triple::VendorType::AMD)
+    return;
+  for (auto &&MD : {"amdgpu.no.fine.grained.memory",
+                    "amdgpu.no.remote.memory",
+                    "amdgpu.ignore.denormal.mode"}) {
+    if (Src->hasMetadata(MD))
+      Dst->addDecorate(new SPIRVDecorateUserSemanticAttr(Dst, MD));
+  }
+}
+
 static int transAtomicOrdering(llvm::AtomicOrdering Ordering) {
   return OCLMemOrderMap::map(
       static_cast<OCLMemOrderKind>(llvm::toCABI(Ordering)));
@@ -2878,12 +2890,15 @@ LLVMToSPIRVBase::transValueWithoutDecoration(Value *V, SPIRVBasicBlock *BB,
       auto IncDec = mapValue(V, BM->addInstTemplate(OC, Ops, BB, Ty));
       IncDec->addDecorate(
           new SPIRVDecorate(DecorationMaxByteOffsetId, IncDec, WrapV));
+      addAMDGPUAtomicDecorations(ARMW, IncDec, M);
       return IncDec;
       // TODO: figure out handling of saturating val.
     } else
       OC = LLVMSPIRVAtomicRmwOpCodeMap::map(Op);
 
-    return mapValue(V, BM->addInstTemplate(OC, Ops, BB, Ty));
+    auto Result = mapValue(V, BM->addInstTemplate(OC, Ops, BB, Ty));
+    addAMDGPUAtomicDecorations(ARMW, Result, M);
+    return Result;
   }
 
   if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(V)) {
@@ -6674,6 +6689,8 @@ SPIRVInstruction *LLVMToSPIRVBase::transBuiltinToInst(StringRef DemangledName,
 
   auto *Inst = transBuiltinToInstWithoutDecoration(OC, CI, BB);
   addDecorations(Inst, Dec);
+  if (Inst && isAtomicOpCode(OC))
+    addAMDGPUAtomicDecorations(CI, Inst, M);
   return Inst;
 }
 

test/amdgpu-atomic-metadata-non-amd.ll

@@ -0,0 +1,24 @@
+; RUN: llvm-as < %s -o %t.bc
+; RUN: llvm-spirv %t.bc -o %t.spv
+; RUN: llvm-spirv -to-text %t.spv -o - | FileCheck %s
+
+; Check that atomic instructions with amdgpu atomic metadata DO NOT produce
+; UserSemantic decorations when NOT using AMD triple.
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
+target triple = "spir64"
+
+@ui = common dso_local addrspace(1) global i32 0, align 4
+
+; CHECK: AtomicIAdd
+; CHECK-NOT: UserSemantic "amdgpu.no.fine.grained.memory"
+; CHECK-NOT: UserSemantic "amdgpu.no.remote.memory"
+; CHECK-NOT: UserSemantic "amdgpu.ignore.denormal.mode"
+
+define dso_local spir_func void @test_non_amd() {
+entry:
+  %0 = atomicrmw add ptr addrspace(1) @ui, i32 42 monotonic, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  ret void
+}
+
+!0 = !{}

test/amdgpu-atomic-metadata.ll

@@ -0,0 +1,53 @@
+; RUN: llvm-as < %s -o %t.bc
+; RUN: llvm-spirv %t.bc -o %t.spv
+; RUN: llvm-spirv -to-text %t.spv -o - | FileCheck %s
+
+; Check that atomic instructions with amdgpu atomic metadata produce
+; UserSemantic decorations.
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
+target triple = "spirv64-amd-amdhsa"
+
+@ui = common dso_local addrspace(1) global i32 0, align 4
+@f = common dso_local addrspace(1) global float 0.000000e+00, align 4
+
+; CHECK: Decorate [[#ADD_RES:]] UserSemantic "amdgpu.no.fine.grained.memory"
+; CHECK: Decorate [[#ADD_RES]] UserSemantic "amdgpu.no.remote.memory"
+; CHECK-NOT: Decorate [[#ADD_RES]] UserSemantic "amdgpu.ignore.denormal.mode"
+
+; CHECK: Decorate [[#XCHG_RES:]] UserSemantic "amdgpu.no.fine.grained.memory"
+; CHECK: Decorate [[#XCHG_RES]] UserSemantic "amdgpu.ignore.denormal.mode"
+; CHECK-NOT: Decorate [[#XCHG_RES]] UserSemantic "amdgpu.no.remote.memory"
+
+; CHECK: Decorate [[#CAS_RES:]] UserSemantic "amdgpu.no.fine.grained.memory"
+; CHECK: Decorate [[#CAS_RES]] UserSemantic "amdgpu.no.remote.memory"
+; CHECK: Decorate [[#CAS_RES]] UserSemantic "amdgpu.ignore.denormal.mode"
+
+; CHECK: AtomicIAdd [[#]] [[#ADD_RES]]
+; CHECK: AtomicExchange [[#]] [[#XCHG_RES]]
+; CHECK: AtomicCompareExchange [[#]] [[#CAS_RES]]
+
+define dso_local spir_func void @test_atomicrmw_metadata() {
+entry:
+  %0 = atomicrmw add ptr addrspace(1) @ui, i32 42 monotonic, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
+  %1 = atomicrmw xchg ptr addrspace(1) @f, float 42.0 seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0
+  ret void
+}
+
+define dso_local spir_func void @test_cmpxchg_metadata(ptr %ptr, ptr %value_ptr, i32 %comparator) {
+entry:
+  %0 = load i32, ptr %value_ptr, align 4
+  %1 = cmpxchg ptr %ptr, i32 %comparator, i32 %0 seq_cst acquire, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !0
+  %2 = extractvalue { i32, i1 } %1, 1
+  br i1 %2, label %cmpxchg.continue, label %cmpxchg.store_expected
+
+cmpxchg.store_expected:
+  %3 = extractvalue { i32, i1 } %1, 0
+  store i32 %3, ptr %value_ptr, align 4
+  br label %cmpxchg.continue
+
+cmpxchg.continue:
+  ret void
+}
+
+!0 = !{}