From bcc1e85cfbafd78b1a34af9a880817ede813fc0a Mon Sep 17 00:00:00 2001 From: Arseniy Obolenskiy Date: Wed, 1 Jul 2026 08:08:37 +0200 Subject: [PATCH] Add storage-class memory-semantics bit to LLVM IR atomic translation --- lib/SPIRV/OCLToSPIRV.cpp | 85 +------------------ lib/SPIRV/OCLUtil.cpp | 80 +++++++++++++++++ lib/SPIRV/OCLUtil.h | 6 ++ lib/SPIRV/SPIRVRegularizeLLVM.cpp | 8 +- lib/SPIRV/SPIRVWriter.cpp | 19 +++-- test/atomicrmw.ll | 56 ++++++++---- .../atomicrmw_fadd_double.ll | 2 +- .../atomicrmw_fadd_float.ll | 2 +- .../atomicrmw_fadd_half.ll | 2 +- .../atomicrmw_fsub_double.ll | 2 +- .../atomicrmw_fsub_float.ll | 2 +- .../atomicrmw_fsub_half.ll | 2 +- .../atomicrmw_fminfmax_double.ll | 2 +- .../atomicrmw_fminfmax_float.ll | 2 +- .../atomicrmw_fminfmax_half.ll | 2 +- .../SPV_INTEL_16bit_atomics/atomicrmw_fadd.ll | 2 +- .../atomicrmw_fminfmax.ll | 2 +- .../OpenCL/atomic_syncscope_test.ll | 39 +++++---- 18 files changed, 176 insertions(+), 139 deletions(-) diff --git a/lib/SPIRV/OCLToSPIRV.cpp b/lib/SPIRV/OCLToSPIRV.cpp index d958a54199..4cb3ce027a 100644 --- a/lib/SPIRV/OCLToSPIRV.cpp +++ b/lib/SPIRV/OCLToSPIRV.cpp @@ -69,87 +69,6 @@ using namespace OCLUtil; namespace SPIRV { -static unsigned getAddressSpaceFromType(const Type *Ty) { - assert(Ty && "Can't deduce pointer AS"); - if (auto *TypedPtr = dyn_cast(Ty)) - return TypedPtr->getAddressSpace(); - if (auto *Ptr = dyn_cast(Ty)) - return Ptr->getAddressSpace(); - llvm_unreachable("Can't deduce pointer AS"); -} - -// Performs an address space inference analysis. -static unsigned getAddressSpaceFromValue(const Value *Ptr) { - assert(Ptr && "Can't deduce pointer AS"); - - SmallPtrSet Visited; - SmallVector Worklist; - Worklist.push_back(Ptr); - unsigned AS = SPIRAS_Generic; - - while (!Worklist.empty()) { - const Value *Current = Worklist.pop_back_val(); - if (!Visited.insert(Current).second) - continue; - - unsigned DeducedAS = getAddressSpaceFromType(Current->getType()); - if (DeducedAS != SPIRAS_Generic) - return DeducedAS; - AS = DeducedAS; - - // Find origins of the pointer and add to the worklist. - if (auto *Op = dyn_cast(Current)) { - switch (Op->getOpcode()) { - case Instruction::AddrSpaceCast: - case Instruction::BitCast: - case Instruction::GetElementPtr: - Worklist.push_back(Op->getOperand(0)); - break; - case Instruction::Select: - Worklist.push_back(Op->getOperand(1)); - Worklist.push_back(Op->getOperand(2)); - break; - case Instruction::PHI: { - auto *Phi = cast(Op); - for (Value *Incoming : Phi->incoming_values()) - Worklist.push_back(Incoming); - break; - } - default: - break; - } - } - } - - return AS; -} - -// Sets memory semantic mask of an atomic depending on a pointer argument -// address space. -static unsigned -getAtomicPointerMemorySemanticsMemoryMask(const Value *Ptr, - const Type *RecordedType) { - assert((Ptr && RecordedType) && - "Can't evaluate atomic builtin's memory semantic"); - unsigned AddrSpace = getAddressSpaceFromType(RecordedType); - if (AddrSpace == SPIRAS_Generic) - AddrSpace = getAddressSpaceFromValue(Ptr); - - switch (AddrSpace) { - case SPIRAS_Global: - case SPIRAS_GlobalDevice: - case SPIRAS_GlobalHost: - return MemorySemanticsCrossWorkgroupMemoryMask; - case SPIRAS_Local: - return MemorySemanticsWorkgroupMemoryMask; - case SPIRAS_Generic: - return MemorySemanticsCrossWorkgroupMemoryMask | - MemorySemanticsWorkgroupMemoryMask; - default: - return MemorySemanticsMaskNone; - } -} - static size_t getOCLCpp11AtomicMaxNumOps(StringRef Name) { return StringSwitch(Name) .Cases({"load", "flag_test_and_set", "flag_clear"}, 3) @@ -790,8 +709,8 @@ void OCLToSPIRVBase::transAtomicBuiltin(CallInst *CI, unsigned PtrMemSemantics = MemorySemanticsMaskNone; if (Mutator.arg_size() > 0) - PtrMemSemantics = getAtomicPointerMemorySemanticsMemoryMask( - Mutator.getArg(0), Mutator.getType(0)); + PtrMemSemantics = getAtomicPointerMemorySemanticsMask(Mutator.getArg(0), + Mutator.getType(0)); if (NeedsNegate) { Mutator.mapArg(1, [=](Value *V) { diff --git a/lib/SPIRV/OCLUtil.cpp b/lib/SPIRV/OCLUtil.cpp index b3dc0f4bec..031dfa7c7a 100644 --- a/lib/SPIRV/OCLUtil.cpp +++ b/lib/SPIRV/OCLUtil.cpp @@ -41,10 +41,13 @@ #include "SPIRVFunction.h" #include "SPIRVInstruction.h" #include "SPIRVInternal.h" +#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/StringSwitch.h" +#include "llvm/IR/DerivedTypes.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstVisitor.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/Operator.h" #include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" @@ -670,6 +673,83 @@ template <> void LLVMSPIRVAtomicRmwOpCodeMap::init() { namespace OCLUtil { +static unsigned getAddressSpaceFromType(const Type *Ty) { + assert(Ty && "Can't deduce pointer AS"); + if (auto *TypedPtr = dyn_cast(Ty)) + return TypedPtr->getAddressSpace(); + if (auto *Ptr = dyn_cast(Ty)) + return Ptr->getAddressSpace(); + llvm_unreachable("Can't deduce pointer AS"); +} + +// Performs an address space inference analysis. +static unsigned getAddressSpaceFromValue(const Value *Ptr) { + assert(Ptr && "Can't deduce pointer AS"); + + SmallPtrSet Visited; + SmallVector Worklist; + Worklist.push_back(Ptr); + unsigned AS = SPIRAS_Generic; + + while (!Worklist.empty()) { + const Value *Current = Worklist.pop_back_val(); + if (!Visited.insert(Current).second) + continue; + + unsigned DeducedAS = getAddressSpaceFromType(Current->getType()); + if (DeducedAS != SPIRAS_Generic) + return DeducedAS; + + // Find origins of the pointer and add to the worklist. + if (auto *Op = dyn_cast(Current)) { + switch (Op->getOpcode()) { + case Instruction::AddrSpaceCast: + case Instruction::BitCast: + case Instruction::GetElementPtr: + Worklist.push_back(Op->getOperand(0)); + break; + case Instruction::Select: + Worklist.push_back(Op->getOperand(1)); + Worklist.push_back(Op->getOperand(2)); + break; + case Instruction::PHI: { + auto *Phi = cast(Op); + for (Value *Incoming : Phi->incoming_values()) + Worklist.push_back(Incoming); + break; + } + default: + break; + } + } + } + + return AS; +} + +unsigned getAtomicPointerMemorySemanticsMask(const Value *Ptr, + const Type *RecordedType) { + assert((Ptr && RecordedType) && + "Can't evaluate atomic builtin's memory semantic"); + unsigned AddrSpace = getAddressSpaceFromType(RecordedType); + if (AddrSpace == SPIRAS_Generic) + AddrSpace = getAddressSpaceFromValue(Ptr); + + switch (AddrSpace) { + case SPIRAS_Global: + case SPIRAS_GlobalDevice: + case SPIRAS_GlobalHost: + return MemorySemanticsCrossWorkgroupMemoryMask; + case SPIRAS_Local: + return MemorySemanticsWorkgroupMemoryMask; + case SPIRAS_Generic: + return MemorySemanticsCrossWorkgroupMemoryMask | + MemorySemanticsWorkgroupMemoryMask; + default: + return MemorySemanticsMaskNone; + } +} + AtomicWorkItemFenceLiterals getAtomicWorkItemFenceLiterals(CallInst *CI) { return std::make_tuple(getArgAsInt(CI, 0), static_cast(getArgAsInt(CI, 1)), diff --git a/lib/SPIRV/OCLUtil.h b/lib/SPIRV/OCLUtil.h index e7615263c9..9fa4c28ef7 100644 --- a/lib/SPIRV/OCLUtil.h +++ b/lib/SPIRV/OCLUtil.h @@ -508,6 +508,12 @@ bool isPipeOrAddressSpaceCastBI(const StringRef MangledName); bool isEnqueueKernelBI(const StringRef MangledName); bool isKernelQueryBI(const StringRef MangledName); +// Returns the storage-class memory-semantics bit mask derived from the pointer +// address space. RecordedType is checked first; if it resolves to Generic, Ptr +// is analyzed via use-def walk. +unsigned getAtomicPointerMemorySemanticsMask(const Value *Ptr, + const Type *RecordedType); + /// Check that the type is the sampler_t bool isSamplerTy(Type *Ty); diff --git a/lib/SPIRV/SPIRVRegularizeLLVM.cpp b/lib/SPIRV/SPIRVRegularizeLLVM.cpp index b5d1643542..5633850a34 100644 --- a/lib/SPIRV/SPIRVRegularizeLLVM.cpp +++ b/lib/SPIRV/SPIRVRegularizeLLVM.cpp @@ -771,8 +771,12 @@ bool SPIRVRegularizeLLVMBase::regularize() { llvm::toCABI(Cmpxchg->getSuccessOrdering())); auto FailureOrder = static_cast( llvm::toCABI(Cmpxchg->getFailureOrdering())); - Value *EqualSem = getInt32(M, OCLMemOrderMap::map(SuccessOrder)); - Value *UnequalSem = getInt32(M, OCLMemOrderMap::map(FailureOrder)); + unsigned SCMask = + getAtomicPointerMemorySemanticsMask(Ptr, Ptr->getType()); + Value *EqualSem = + getInt32(M, OCLMemOrderMap::map(SuccessOrder) | SCMask); + Value *UnequalSem = + getInt32(M, OCLMemOrderMap::map(FailureOrder) | SCMask); Value *Val = Cmpxchg->getNewValOperand(); Value *Comparator = Cmpxchg->getCompareOperand(); diff --git a/lib/SPIRV/SPIRVWriter.cpp b/lib/SPIRV/SPIRVWriter.cpp index c6f48667a2..6297a1691a 100644 --- a/lib/SPIRV/SPIRVWriter.cpp +++ b/lib/SPIRV/SPIRVWriter.cpp @@ -2014,8 +2014,10 @@ SPIRVValue *LLVMToSPIRVBase::transAtomicStore(StoreInst *ST, SPIRVBasicBlock *BB) { spv::Scope S = toSPIRVScope(ST->getContext(), ST->getSyncScopeID()); - std::vector Ops{ST->getPointerOperand(), getUInt32(M, S), - getUInt32(M, transAtomicOrdering(ST->getOrdering())), + Value *Ptr = ST->getPointerOperand(); + auto MemSem = transAtomicOrdering(ST->getOrdering()) | + getAtomicPointerMemorySemanticsMask(Ptr, Ptr->getType()); + std::vector Ops{Ptr, getUInt32(M, S), getUInt32(M, MemSem), ST->getValueOperand()}; std::vector SPIRVOps = transValue(Ops, BB); @@ -2027,9 +2029,10 @@ SPIRVValue *LLVMToSPIRVBase::transAtomicLoad(LoadInst *LD, SPIRVBasicBlock *BB) { spv::Scope S = toSPIRVScope(LD->getContext(), LD->getSyncScopeID()); - std::vector Ops{ - LD->getPointerOperand(), getUInt32(M, S), - getUInt32(M, transAtomicOrdering(LD->getOrdering()))}; + Value *Ptr = LD->getPointerOperand(); + auto MemSem = transAtomicOrdering(LD->getOrdering()) | + getAtomicPointerMemorySemanticsMask(Ptr, Ptr->getType()); + std::vector Ops{Ptr, getUInt32(M, S), getUInt32(M, MemSem)}; std::vector SPIRVOps = transValue(Ops, BB); return mapValue(LD, BM->addInstTemplate(OpAtomicLoad, BM->getIds(SPIRVOps), @@ -2793,9 +2796,11 @@ LLVMToSPIRVBase::transValueWithoutDecoration(Value *V, SPIRVBasicBlock *BB, return nullptr; AtomicOrderingCABI Ordering = llvm::toCABI(ARMW->getOrdering()); - auto MemSem = OCLMemOrderMap::map(static_cast(Ordering)); + Value *Ptr = ARMW->getPointerOperand(); + auto MemSem = OCLMemOrderMap::map(static_cast(Ordering)) | + getAtomicPointerMemorySemanticsMask(Ptr, Ptr->getType()); std::vector Operands(4); - Operands[0] = ARMW->getPointerOperand(); + Operands[0] = Ptr; spv::Scope S = toSPIRVScope(ARMW->getContext(), ARMW->getSyncScopeID()); Operands[1] = getUInt32(M, S); Operands[2] = getUInt32(M, MemSem); diff --git a/test/atomicrmw.ll b/test/atomicrmw.ll index 1e55ff4290..04055f1844 100644 --- a/test/atomicrmw.ll +++ b/test/atomicrmw.ll @@ -8,15 +8,19 @@ ; RUN: llvm-spirv -to-text %t.spv -o - | FileCheck %s ; CHECK: TypeInt [[Int:[0-9]+]] 32 0 -; CHECK-DAG: Constant [[Int]] [[MemSem_Relaxed:[0-9]+]] 0 -; CHECK-DAG: Constant [[Int]] [[MemSem_Acquire:[0-9]+]] 2 -; CHECK-DAG: Constant [[Int]] [[MemSem_Release:[0-9]+]] 4 {{$}} -; CHECK-DAG: Constant [[Int]] [[MemSem_AcquireRelease:[0-9]+]] 8 -; CHECK-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 16 +; CHECK-DAG: Constant [[Int]] [[MemSem_CW_Relaxed:[0-9]+]] 512 +; CHECK-DAG: Constant [[Int]] [[MemSem_CW_Acquire:[0-9]+]] 514 +; CHECK-DAG: Constant [[Int]] [[MemSem_CW_Release:[0-9]+]] 516 +; CHECK-DAG: Constant [[Int]] [[MemSem_CW_AcquireRelease:[0-9]+]] 520 +; CHECK-DAG: Constant [[Int]] [[MemSem_CW_SequentiallyConsistent:[0-9]+]] 528 +; CHECK-DAG: Constant [[Int]] [[MemSem_WG_Relaxed:[0-9]+]] 256 +; CHECK-DAG: Constant [[Int]] [[MemSem_WG_AcquireRelease:[0-9]+]] 264 +; CHECK-DAG: Constant [[Int]] [[MemSem_WG_SequentiallyConsistent:[0-9]+]] 272 ; CHECK-DAG: Constant [[Int]] [[Value:[0-9]+]] 42 ; CHECK: TypeFloat [[Float:[0-9]+]] 32 ; CHECK: {{(Variable|UntypedVariableKHR)}} {{[0-9]+}} [[Pointer:[0-9]+]] ; CHECK: {{(Variable|UntypedVariableKHR)}} {{[0-9]+}} [[FPPointer:[0-9]+]] +; CHECK: {{(Variable|UntypedVariableKHR)}} {{[0-9]+}} [[LocalPointer:[0-9]+]] ; CHECK: Constant [[Float]] [[FPValue:[0-9]+]] 1109917696 target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" @@ -24,42 +28,62 @@ target triple = "spir64" @ui = common dso_local addrspace(1) global i32 0, align 4 @f = common dso_local local_unnamed_addr addrspace(1) global float 0.000000e+00, align 4 +@li = common dso_local addrspace(3) global i32 0, align 4 ; Function Attrs: nounwind define dso_local spir_func void @test_atomicrmw() local_unnamed_addr #0 { entry: %0 = atomicrmw xchg ptr addrspace(1) @ui, i32 42 acq_rel -; CHECK: AtomicExchange [[Int]] {{[0-9]+}} [[Pointer]] {{.+}} [[MemSem_AcquireRelease]] [[Value]] +; CHECK: AtomicExchange [[Int]] {{[0-9]+}} [[Pointer]] {{.+}} [[MemSem_CW_AcquireRelease]] [[Value]] %1 = atomicrmw xchg ptr addrspace(1) @f, float 42.000000e+00 seq_cst -; CHECK: AtomicExchange [[Float]] {{[0-9]+}} [[FPPointer]] {{.+}} [[MemSem_SequentiallyConsistent]] [[FPValue]] +; CHECK: AtomicExchange [[Float]] {{[0-9]+}} [[FPPointer]] {{.+}} [[MemSem_CW_SequentiallyConsistent]] [[FPValue]] %2 = atomicrmw add ptr addrspace(1) @ui, i32 42 monotonic -; CHECK: AtomicIAdd [[Int]] {{[0-9]+}} [[Pointer]] {{.+}} [[MemSem_Relaxed]] [[Value]] +; CHECK: AtomicIAdd [[Int]] {{[0-9]+}} [[Pointer]] {{.+}} [[MemSem_CW_Relaxed]] [[Value]] %3 = atomicrmw sub ptr addrspace(1) @ui, i32 42 acquire -; CHECK: AtomicISub [[Int]] {{[0-9]+}} [[Pointer]] {{.+}} [[MemSem_Acquire]] [[Value]] +; CHECK: AtomicISub [[Int]] {{[0-9]+}} [[Pointer]] {{.+}} [[MemSem_CW_Acquire]] [[Value]] %4 = atomicrmw or ptr addrspace(1) @ui, i32 42 release -; CHECK: AtomicOr [[Int]] {{[0-9]+}} [[Pointer]] {{.+}} [[MemSem_Release]] [[Value]] +; CHECK: AtomicOr [[Int]] {{[0-9]+}} [[Pointer]] {{.+}} [[MemSem_CW_Release]] [[Value]] %5 = atomicrmw xor ptr addrspace(1) @ui, i32 42 acq_rel -; CHECK: AtomicXor [[Int]] {{[0-9]+}} [[Pointer]] {{.+}} [[MemSem_AcquireRelease]] [[Value]] +; CHECK: AtomicXor [[Int]] {{[0-9]+}} [[Pointer]] {{.+}} [[MemSem_CW_AcquireRelease]] [[Value]] %6 = atomicrmw and ptr addrspace(1) @ui, i32 42 seq_cst -; CHECK: AtomicAnd [[Int]] {{[0-9]+}} [[Pointer]] {{.+}} [[MemSem_SequentiallyConsistent]] [[Value]] +; CHECK: AtomicAnd [[Int]] {{[0-9]+}} [[Pointer]] {{.+}} [[MemSem_CW_SequentiallyConsistent]] [[Value]] %7 = atomicrmw max ptr addrspace(1) @ui, i32 42 monotonic -; CHECK: AtomicSMax [[Int]] {{[0-9]+}} [[Pointer]] {{.+}} [[MemSem_Relaxed]] [[Value]] +; CHECK: AtomicSMax [[Int]] {{[0-9]+}} [[Pointer]] {{.+}} [[MemSem_CW_Relaxed]] [[Value]] %8 = atomicrmw min ptr addrspace(1) @ui, i32 42 acquire -; CHECK: AtomicSMin [[Int]] {{[0-9]+}} [[Pointer]] {{.+}} [[MemSem_Acquire]] [[Value]] +; CHECK: AtomicSMin [[Int]] {{[0-9]+}} [[Pointer]] {{.+}} [[MemSem_CW_Acquire]] [[Value]] %9 = atomicrmw umax ptr addrspace(1) @ui, i32 42 release -; CHECK: AtomicUMax [[Int]] {{[0-9]+}} [[Pointer]] {{.+}} [[MemSem_Release]] [[Value]] +; CHECK: AtomicUMax [[Int]] {{[0-9]+}} [[Pointer]] {{.+}} [[MemSem_CW_Release]] [[Value]] %10 = atomicrmw umin ptr addrspace(1) @ui, i32 42 acq_rel -; CHECK: AtomicUMin [[Int]] {{[0-9]+}} [[Pointer]] {{.+}} [[MemSem_AcquireRelease]] [[Value]] +; CHECK: AtomicUMin [[Int]] {{[0-9]+}} [[Pointer]] {{.+}} [[MemSem_CW_AcquireRelease]] [[Value]] + + %11 = atomicrmw add ptr addrspace(3) @li, i32 42 acq_rel +; CHECK: AtomicIAdd [[Int]] {{[0-9]+}} [[LocalPointer]] {{.+}} [[MemSem_WG_AcquireRelease]] [[Value]] + + ret void +} + +; Function Attrs: nounwind +define dso_local spir_func void @test_atomic_load_store() local_unnamed_addr #0 { +entry: + %0 = load atomic i32, ptr addrspace(1) @ui seq_cst, align 4 +; CHECK: AtomicLoad [[Int]] {{[0-9]+}} [[Pointer]] {{.+}} [[MemSem_CW_SequentiallyConsistent]] + store atomic i32 42, ptr addrspace(1) @ui monotonic, align 4 +; CHECK: AtomicStore [[Pointer]] {{.+}} [[MemSem_CW_Relaxed]] [[Value]] + + %1 = load atomic i32, ptr addrspace(3) @li seq_cst, align 4 +; CHECK: AtomicLoad [[Int]] {{[0-9]+}} [[LocalPointer]] {{.+}} [[MemSem_WG_SequentiallyConsistent]] + store atomic i32 42, ptr addrspace(3) @li monotonic, align 4 +; CHECK: AtomicStore [[LocalPointer]] {{.+}} [[MemSem_WG_Relaxed]] [[Value]] ret void } diff --git a/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fadd_double.ll b/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fadd_double.ll index ffe6826c88..9549ac06f9 100644 --- a/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fadd_double.ll +++ b/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fadd_double.ll @@ -7,7 +7,7 @@ ; CHECK-DAG: Capability AtomicFloat64AddEXT ; CHECK: TypeInt [[Int:[0-9]+]] 32 0 ; CHECK-DAG: Constant [[Int]] [[Scope_CrossDevice:[0-9]+]] 0 {{$}} -; CHECK-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 16 +; CHECK-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 528 ; CHECK: TypeFloat [[Double:[0-9]+]] 64 ; CHECK: Variable {{[0-9]+}} [[DoublePointer:[0-9]+]] ; CHECK: Constant [[Double]] [[DoubleValue:[0-9]+]] 0 1078263808 diff --git a/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fadd_float.ll b/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fadd_float.ll index 7b1cc9a4d7..bee0a860bf 100644 --- a/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fadd_float.ll +++ b/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fadd_float.ll @@ -7,7 +7,7 @@ ; CHECK-DAG: Capability AtomicFloat32AddEXT ; CHECK: TypeInt [[Int:[0-9]+]] 32 0 ; CHECK-DAG: Constant [[Int]] [[Scope_CrossDevice:[0-9]+]] 0 {{$}} -; CHECK-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 16 +; CHECK-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 528 ; CHECK: TypeFloat [[Float:[0-9]+]] 32 ; CHECK: Variable {{[0-9]+}} [[FPPointer:[0-9]+]] ; CHECK: Constant [[Float]] [[FPValue:[0-9]+]] 1109917696 diff --git a/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fadd_half.ll b/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fadd_half.ll index 3a5dc2ed5a..d1e1389932 100644 --- a/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fadd_half.ll +++ b/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fadd_half.ll @@ -8,7 +8,7 @@ ; CHECK-DAG: Capability AtomicFloat16AddEXT ; CHECK: TypeInt [[TypeIntID:[0-9]+]] 32 0 ; CHECK-DAG: Constant [[TypeIntID]] [[ScopeCrossDevice:[0-9]+]] 0 {{$}} -; CHECK-DAG: Constant [[TypeIntID]] [[MemSem_SequentiallyConsistent:[0-9]+]] 16 +; CHECK-DAG: Constant [[TypeIntID]] [[MemSem_SequentiallyConsistent:[0-9]+]] 528 ; CHECK: TypeFloat [[TypeFloatHalfID:[0-9]+]] 16 ; CHECK: Variable {{[0-9]+}} [[HalfPointer:[0-9]+]] ; CHECK: Constant [[TypeFloatHalfID]] [[HalfValue:[0-9]+]] 20800 diff --git a/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fsub_double.ll b/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fsub_double.ll index 064628f8fb..58c7c5c74b 100644 --- a/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fsub_double.ll +++ b/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fsub_double.ll @@ -10,7 +10,7 @@ ; CHECK-SPIRV-DAG: Capability AtomicFloat64AddEXT ; CHECK-SPIRV: TypeInt [[Int:[0-9]+]] 32 0 ; CHECK-SPIRV-DAG: Constant [[Int]] [[Scope_CrossDevice:[0-9]+]] 0 {{$}} -; CHECK-SPIRV-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 16 +; CHECK-SPIRV-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 528 ; CHECK-SPIRV: TypeFloat [[Double:[0-9]+]] 64 ; CHECK-SPIRV: Variable {{[0-9]+}} [[DoublePointer:[0-9]+]] ; CHECK-SPIRV: Constant [[Double]] [[DoubleValue:[0-9]+]] 0 1078263808 diff --git a/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fsub_float.ll b/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fsub_float.ll index 70288cb2d1..190a234752 100644 --- a/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fsub_float.ll +++ b/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fsub_float.ll @@ -10,7 +10,7 @@ ; CHECK-SPIRV-DAG: Capability AtomicFloat32AddEXT ; CHECK-SPIRV: TypeInt [[Int:[0-9]+]] 32 0 ; CHECK-SPIRV-DAG: Constant [[Int]] [[Scope_CrossDevice:[0-9]+]] 0 {{$}} -; CHECK-SPIRV-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 16 +; CHECK-SPIRV-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 528 ; CHECK-SPIRV: TypeFloat [[Float:[0-9]+]] 32 ; CHECK-SPIRV: Variable {{[0-9]+}} [[FPPointer:[0-9]+]] ; CHECK-SPIRV: Constant [[Float]] [[FPValue:[0-9]+]] 1109917696 diff --git a/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fsub_half.ll b/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fsub_half.ll index c421df1577..bd013853e3 100644 --- a/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fsub_half.ll +++ b/test/extensions/EXT/SPV_EXT_shader_atomic_float_/atomicrmw_fsub_half.ll @@ -11,7 +11,7 @@ ; CHECK-SPIRV-DAG: Capability AtomicFloat16AddEXT ; CHECK-SPIRV: TypeInt [[Int:[0-9]+]] 32 0 ; CHECK-SPIRV-DAG: Constant [[Int]] [[ScopeCrossDevice:[0-9]+]] 0 {{$}} -; CHECK-SPIRV-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 16 +; CHECK-SPIRV-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 528 ; CHECK-SPIRV: TypeFloat [[Half:[0-9]+]] 16 ; CHECK-SPIRV: Variable {{[0-9]+}} [[HalfPointer:[0-9]+]] ; CHECK-SPIRV: Constant [[Half]] [[HalfValue:[0-9]+]] 15360 diff --git a/test/extensions/EXT/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_double.ll b/test/extensions/EXT/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_double.ll index ff8aa756b5..d9be0ffc8c 100644 --- a/test/extensions/EXT/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_double.ll +++ b/test/extensions/EXT/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_double.ll @@ -7,7 +7,7 @@ ; CHECK-DAG: Capability AtomicFloat64MinMaxEXT ; CHECK: TypeInt [[Int:[0-9]+]] 32 0 ; CHECK-DAG: Constant [[Int]] [[Scope_CrossDevice:[0-9]+]] 0 {{$}} -; CHECK-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 16 +; CHECK-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 528 ; CHECK: TypeFloat [[Double:[0-9]+]] 64 ; CHECK: Variable {{[0-9]+}} [[DoublePointer:[0-9]+]] ; CHECK: Constant [[Double]] [[DoubleValue:[0-9]+]] 0 1078263808 diff --git a/test/extensions/EXT/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_float.ll b/test/extensions/EXT/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_float.ll index 5ac35b6af1..0b7bd3a4a2 100644 --- a/test/extensions/EXT/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_float.ll +++ b/test/extensions/EXT/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_float.ll @@ -7,7 +7,7 @@ ; CHECK-DAG: Capability AtomicFloat32MinMaxEXT ; CHECK: TypeInt [[Int:[0-9]+]] 32 0 ; CHECK-DAG: Constant [[Int]] [[Scope_CrossDevice:[0-9]+]] 0 {{$}} -; CHECK-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 16 +; CHECK-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 528 ; CHECK: TypeFloat [[Float:[0-9]+]] 32 ; CHECK: Variable {{[0-9]+}} [[FPPointer:[0-9]+]] ; CHECK: Constant [[Float]] [[FPValue:[0-9]+]] 1109917696 diff --git a/test/extensions/EXT/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_half.ll b/test/extensions/EXT/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_half.ll index 6c530f0cee..4fd644493a 100644 --- a/test/extensions/EXT/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_half.ll +++ b/test/extensions/EXT/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_half.ll @@ -7,7 +7,7 @@ ; CHECK-DAG: Capability AtomicFloat16MinMaxEXT ; CHECK: TypeInt [[Int:[0-9]+]] 32 0 ; CHECK-DAG: Constant [[Int]] [[Scope_CrossDevice:[0-9]+]] 0 {{$}} -; CHECK-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 16 +; CHECK-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 528 ; CHECK: TypeFloat [[Half:[0-9]+]] 16 ; CHECK: Variable {{[0-9]+}} [[HalfPointer:[0-9]+]] ; CHECK: Constant [[Half]] [[HalfValue:[0-9]+]] 20800 diff --git a/test/extensions/INTEL/SPV_INTEL_16bit_atomics/atomicrmw_fadd.ll b/test/extensions/INTEL/SPV_INTEL_16bit_atomics/atomicrmw_fadd.ll index f42ce61bd3..4b634c3a3f 100644 --- a/test/extensions/INTEL/SPV_INTEL_16bit_atomics/atomicrmw_fadd.ll +++ b/test/extensions/INTEL/SPV_INTEL_16bit_atomics/atomicrmw_fadd.ll @@ -8,7 +8,7 @@ ; CHECK-DAG: Capability BFloat16TypeKHR ; CHECK: TypeInt [[Int:[0-9]+]] 32 0 ; CHECK-DAG: Constant [[Int]] [[Scope_CrossDevice:[0-9]+]] 0 {{$}} -; CHECK-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 16 +; CHECK-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 528 ; CHECK: TypeFloat [[BFloat:[0-9]+]] 16 0 ; CHECK: Variable {{[0-9]+}} [[BFloatPointer:[0-9]+]] ; CHECK: Constant [[BFloat]] [[BFloatValue:[0-9]+]] 16936 diff --git a/test/extensions/INTEL/SPV_INTEL_16bit_atomics/atomicrmw_fminfmax.ll b/test/extensions/INTEL/SPV_INTEL_16bit_atomics/atomicrmw_fminfmax.ll index 8877ab9160..bb5fa97477 100644 --- a/test/extensions/INTEL/SPV_INTEL_16bit_atomics/atomicrmw_fminfmax.ll +++ b/test/extensions/INTEL/SPV_INTEL_16bit_atomics/atomicrmw_fminfmax.ll @@ -8,7 +8,7 @@ ; CHECK-DAG: Capability BFloat16TypeKHR ; CHECK: TypeInt [[Int:[0-9]+]] 32 0 ; CHECK-DAG: Constant [[Int]] [[Scope_CrossDevice:[0-9]+]] 0 {{$}} -; CHECK-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 16 +; CHECK-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 528 ; CHECK: TypeFloat [[BFloat:[0-9]+]] 16 0 ; CHECK: Variable {{[0-9]+}} [[BFloatPointer:[0-9]+]] ; CHECK: Constant [[BFloat]] [[BFloatValue:[0-9]+]] 16936 diff --git a/test/transcoding/OpenCL/atomic_syncscope_test.ll b/test/transcoding/OpenCL/atomic_syncscope_test.ll index 8248a9b05f..c9b30fd308 100644 --- a/test/transcoding/OpenCL/atomic_syncscope_test.ll +++ b/test/transcoding/OpenCL/atomic_syncscope_test.ll @@ -33,23 +33,23 @@ target triple = "spir64" ; 4 - sub_group ; CHECK-SPIRV-DAG: Constant [[#]] [[#ConstInt0:]] 0 -; CHECK-SPIRV-DAG: Constant [[#]] [[#SCPrivate:]] 16 +; CHECK-SPIRV-DAG: Constant [[#]] [[#SeqCst:]] 16 ; CHECK-SPIRV-DAG: Constant [[#]] [[#ConstInt1:]] 1 ; CHECK-SPIRV-DAG: Constant [[#]] [[#ConstInt2:]] 2 ; CHECK-SPIRV-DAG: Constant [[#]] [[#ConstInt3:]] 3 ; CHECK-SPIRV-DAG: Constant [[#]] [[#ConstInt4:]] 4 +; CHECK-SPIRV-DAG: Constant [[#]] [[#SeqCst_Generic:]] 784 +; CHECK-SPIRV-DAG: Constant [[#]] [[#SeqCst_Global:]] 528 +; CHECK-SPIRV-DAG: Constant [[#]] [[#SeqCst_Local:]] 272 +; CHECK-SPIRV-DAG: Constant [[#]] [[#Relaxed_Global:]] 512 ; CHECK-SPIRV-DAG: Constant [[#]] [[#Const2Power30:]] 1073741824 ; CHECK-SPIRV-DAG: Constant [[#]] [[#ConstInt42:]] 42 -; Note: Storage class bits (SCGlobal, SCLocal, etc.) are not added for plain LLVM IR atomics -; Only OpenCL builtin atomics get the storage class memory semantics bits from the patch ; AtomicLoad ResTypeId ResId PtrId MemScopeId MemSemanticsId -; Note: Plain LLVM atomic loads don't get storage class bits added (only OpenCL builtins do) -; These use SCPrivate (16) which is SequentiallyConsistent without storage class bits -; CHECK-SPIRV: AtomicLoad [[#]] [[#]] [[#]] [[#ConstInt2]] [[#SCPrivate]] -; CHECK-SPIRV: AtomicLoad [[#]] [[#]] [[#]] [[#ConstInt1]] [[#SCPrivate]] -; CHECK-SPIRV: AtomicLoad [[#]] [[#]] [[#]] [[#ConstInt0]] [[#SCPrivate]] -; CHECK-SPIRV: AtomicLoad [[#]] [[#]] [[#]] [[#ConstInt3]] [[#SCPrivate]] +; CHECK-SPIRV: AtomicLoad [[#]] [[#]] [[#]] [[#ConstInt2]] [[#SeqCst_Generic]] +; CHECK-SPIRV: AtomicLoad [[#]] [[#]] [[#]] [[#ConstInt1]] [[#SeqCst_Generic]] +; CHECK-SPIRV: AtomicLoad [[#]] [[#]] [[#]] [[#ConstInt0]] [[#SeqCst_Generic]] +; CHECK-SPIRV: AtomicLoad [[#]] [[#]] [[#]] [[#ConstInt3]] [[#SeqCst_Generic]] ; CHECK-LLVM: call spir_func i32 @_Z20atomic_load_explicitPU3AS4VU7_Atomici12memory_order12memory_scope(ptr{{.*}}, i32 5, i32 1) ; CHECK-LLVM: call spir_func i32 @_Z20atomic_load_explicitPU3AS4VU7_Atomici12memory_order12memory_scope(ptr{{.*}}, i32 5, i32 2) @@ -66,9 +66,8 @@ entry: } ; AtomicStore PtrId MemScopeId MemSemanticsId ValueId -; Plain LLVM IR store atomic instructions don't get storage class bits -; CHECK-SPIRV: AtomicStore [[#]] [[#ConstInt3]] [[#SCPrivate]] [[#ConstInt1]] -; CHECK-SPIRV: AtomicStore [[#]] [[#ConstInt2]] [[#SCPrivate]] [[#ConstInt1]] +; CHECK-SPIRV: AtomicStore [[#]] [[#ConstInt3]] [[#SeqCst_Global]] [[#ConstInt1]] +; CHECK-SPIRV: AtomicStore [[#]] [[#ConstInt2]] [[#SeqCst_Local]] [[#ConstInt1]] ; CHECK-LLVM: call spir_func void @_Z21atomic_store_explicitPU3AS4VU7_Atomicii12memory_order12memory_scope(ptr{{.*}}, i32 5, i32 4) ; CHECK-LLVM: call spir_func void @_Z21atomic_store_explicitPU3AS4VU7_Atomicii12memory_order12memory_scope(ptr{{.*}}, i32 5, i32 1) @@ -80,11 +79,11 @@ entry: } ; Atomic* ResTypeId ResId PtrId MemScopeId MemSemanticsId ValueId -; CHECK-SPIRV: AtomicAnd [[#]] [[#]] [[#]] [[#ConstInt4]] [[#SCPrivate]] [[#ConstInt1]] -; CHECK-SPIRV: AtomicSMin [[#]] [[#]] [[#]] [[#ConstInt0]] [[#SCPrivate]] [[#ConstInt1]] -; CHECK-SPIRV: AtomicSMax [[#]] [[#]] [[#]] [[#ConstInt0]] [[#SCPrivate]] [[#ConstInt1]] -; CHECK-SPIRV: AtomicUMin [[#]] [[#]] [[#]] [[#ConstInt2]] [[#SCPrivate]] [[#ConstInt1]] -; CHECK-SPIRV: AtomicUMax [[#]] [[#]] [[#]] [[#ConstInt2]] [[#SCPrivate]] [[#ConstInt1]] +; CHECK-SPIRV: AtomicAnd [[#]] [[#]] [[#]] [[#ConstInt4]] [[#SeqCst]] [[#ConstInt1]] +; CHECK-SPIRV: AtomicSMin [[#]] [[#]] [[#]] [[#ConstInt0]] [[#SeqCst]] [[#ConstInt1]] +; CHECK-SPIRV: AtomicSMax [[#]] [[#]] [[#]] [[#ConstInt0]] [[#SeqCst]] [[#ConstInt1]] +; CHECK-SPIRV: AtomicUMin [[#]] [[#]] [[#]] [[#ConstInt2]] [[#SeqCst]] [[#ConstInt1]] +; CHECK-SPIRV: AtomicUMax [[#]] [[#]] [[#]] [[#ConstInt2]] [[#SeqCst]] [[#ConstInt1]] ; CHECK-LLVM: call spir_func i32 @_Z25atomic_fetch_and_explicitPU3AS4VU7_Atomicii12memory_order12memory_scope(ptr{{.*}}, i32 1, i32 5, i32 0) ; CHECK-LLVM: call spir_func i32 @_Z25atomic_fetch_min_explicitPU3AS4VU7_Atomicii12memory_order12memory_scope(ptr{{.*}}, i32 1, i32 5, i32 3) @@ -114,7 +113,7 @@ entry: } ; AtomicExchange ResTypeId ResId PtrId MemScopeId MemSemanticsId ValueId -; CHECK-SPIRV: AtomicExchange [[#]] [[#]] [[#]] [[#ConstInt2]] [[#SCPrivate]] [[#Const2Power30]] +; CHECK-SPIRV: AtomicExchange [[#]] [[#]] [[#]] [[#ConstInt2]] [[#SeqCst]] [[#Const2Power30]] ; CHECK-LLVM: call spir_func i32 @_Z24atomic_exchange_explicitPU3AS4VU7_Atomicii12memory_order12memory_scope(ptr{{.*}}, i32 1073741824, i32 5, i32 1) define dso_local float @ff3(ptr captures(none) noundef %d) local_unnamed_addr #0 { @@ -125,8 +124,8 @@ entry: } ; AtomicFAddEXT ResTypeId ResId PtrId MemScopeId MemSemanticsId ValueId -; Plain LLVM atomicrmw fadd doesn't get storage class bits -; CHECK-SPIRV: AtomicFAddEXT [[#]] [[#]] [[#]] [[#ConstInt2]] [[#ConstInt0]] [[#]] +; Global pointer + monotonic (0) combines with CrossWorkgroupMemory -> 512. +; CHECK-SPIRV: AtomicFAddEXT [[#]] [[#]] [[#]] [[#ConstInt2]] [[#Relaxed_Global]] [[#]] ; CHECK-LLVM: call spir_func float @_Z25atomic_fetch_add_explicitPU3AS4VU7_Atomicff12memory_order12memory_scope(ptr{{.*}}, i32 0, i32 1) define dso_local float @ff4(ptr addrspace(1) captures(none) noundef %d, float noundef %a) local_unnamed_addr #0 {