From 07884249117829a580ad8fb209ee9b36fe46a374 Mon Sep 17 00:00:00 2001 From: Dmitry Sidorov Date: Mon, 30 Mar 2026 08:29:54 -0500 Subject: [PATCH 1/2] Annotate every AMD customization in llvm-spirv --- CMakeLists.txt | 2 + README.md | 83 ++++++++++++++++++++++++++- include/LLVMSPIRVExtensions.inc | 2 + lib/SPIRV/CMakeLists.txt | 2 + lib/SPIRV/LLVMToSPIRVDbgTran.cpp | 6 ++ lib/SPIRV/LLVMToSPIRVDbgTran.h | 2 + lib/SPIRV/OCLToSPIRV.cpp | 2 + lib/SPIRV/OCLUtil.cpp | 4 ++ lib/SPIRV/OCLUtil.h | 4 ++ lib/SPIRV/SPIRVInternal.h | 2 + lib/SPIRV/SPIRVLowerLLVMIntrinsic.cpp | 2 + lib/SPIRV/SPIRVReader.cpp | 78 +++++++++++++++++++++++++ lib/SPIRV/SPIRVToLLVMDbgTran.cpp | 8 +++ lib/SPIRV/SPIRVToLLVMDbgTran.h | 2 + lib/SPIRV/SPIRVToOCL.cpp | 2 + lib/SPIRV/SPIRVToOCL20.cpp | 20 +++++++ lib/SPIRV/SPIRVUtil.cpp | 2 + lib/SPIRV/SPIRVWriter.cpp | 28 +++++++++ lib/SPIRV/libSPIRV/SPIRV.debug.h | 10 ++++ lib/SPIRV/libSPIRV/SPIRVDecorate.h | 2 + lib/SPIRV/libSPIRV/SPIRVInstruction.h | 10 ++++ lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h | 2 + lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h | 2 + lib/SPIRV/libSPIRV/spirv_internal.hpp | 2 + test/CMakeLists.txt | 4 ++ test/lit.cfg.py | 4 ++ tools/llvm-spirv/CMakeLists.txt | 4 ++ 27 files changed, 290 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 32b478727..610f35f5c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,12 +111,14 @@ if(LLVM_SPIRV_BUILD_EXTERNAL) endif() endif() +# AMD customization begin: Disable SPIR-V backend for AMD's fork is_llvm_target_library("SPIRV" spirv_present_result INCLUDED_TARGETS) if(spirv_present_result) message(STATUS "Found SPIR-V Backend, but it is currently disabled for AMD's fork.") # set(SPIRV_BACKEND_FOUND TRUE) # add_compile_definitions(LLVM_SPIRV_BACKEND_TARGET_PRESENT) endif() +# AMD customization end set(LLVM_SPIRV_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/include) diff --git a/README.md b/README.md index 31d80204a..989884860 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,88 @@ dependencies on anything but superficial semantics). ### Differences vs Upstream -- TBA +All AMD-specific code is marked with `// AMD customization begin:` and +`// AMD customization end` comments in the source files. + +#### Build System Changes + +| File | Description | +|------|-------------| +| `CMakeLists.txt` | Disable SPIR-V backend for AMD's fork | +| `lib/SPIRV/CMakeLists.txt` | Rename library to `LLVMSPIRVAMDLib` | +| `tools/llvm-spirv/CMakeLists.txt` | Rename tool to `amd-llvm-spirv`, use `SPIRVAMDLib` | +| `test/CMakeLists.txt` | Update target names for AMD | +| `test/lit.cfg.py` | Use `amd-llvm-spirv` tool name and `LLVMSPIRVAMDLib` library | + +#### New Extensions + +| Extension | Description | +|-----------|-------------| +| `SPV_NV_shader_atomic_fp16_vector` | FP16 vector atomic support | +| `SPV_AMD_weak_linkage` | Weak linkage support for AMDGPU | + +#### Weak Linkage Support + +| File | Description | +|------|-------------| +| `lib/SPIRV/libSPIRV/spirv_internal.hpp` | Define `LinkageTypeWeak` | +| `lib/SPIRV/libSPIRV/SPIRVDecorate.h` | Extension requirement for weak linkage | +| `lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h` | Validate weak linkage type | +| `lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h` | Name mapping for weak linkage | + +#### DIOp-based DIExpression Support (Debug Info) + +| File | Description | +|------|-------------| +| `lib/SPIRV/libSPIRV/SPIRV.debug.h` | DIOp operations, expression opcodes, operand counts | +| `lib/SPIRV/LLVMToSPIRVDbgTran.cpp` | DIOp operand translation, expression translation | +| `lib/SPIRV/LLVMToSPIRVDbgTran.h` | `transDIOpOperand` template declaration | +| `lib/SPIRV/SPIRVToLLVMDbgTran.cpp` | DIOp translation, expression poisoning | +| `lib/SPIRV/SPIRVToLLVMDbgTran.h` | `transDIOpOperand`, `tryTransDIOpDIExpression` declarations | + +#### AMDGPU Target Support (SPIRVReader.cpp) + +| Category | Description | +|----------|-------------| +| Address Space Mapping | SPIR-V to AMDGPU address space mapping functions | +| Calling Convention | Use `AMDGPU_KERNEL`/`C` instead of `SPIR_KERNEL`/`SPIR_FUNC` | +| Type Handling | Zero-sized arrays, pointer type relaxation | +| Intrinsics | OpenCL intrinsics to LLVM intrinsics translation | +| Module Setup | AMDGCN target triple, data layout, COV flags | +| Variable Handling | Workgroup variables, externally initialized globals | +| Attribute Handling | ByVal→ByRef, Captures, kernel attributes | + +#### AMDGPU Target Support (SPIRVWriter.cpp) + +| Category | Description | +|----------|-------------| +| Generator Version | Use `UINT16_MAX` for AMD to identify AMDGPU binaries | +| Zero-sized Arrays | Use `UINT64_MAX` sentinel | +| Variadic Functions | Support vararg functions | +| Atomics | UIncWrap/UDecWrap handling | +| Validation | Relax address space cast checks | + +#### Atomic Operations (SPIRVToOCL20.cpp) + +| Feature | Description | +|---------|-------------| +| Memory Scope | Map OpenCL scope to AMDGPU sync scope | +| Atomics | Translate SPIR-V atomics to LLVM atomic instructions | +| Fences | Translate memory barrier to LLVM fence | +| CmpXchg | Native LLVM atomicrmw/cmpxchg generation | + +#### Other Customizations + +| File | Description | +|------|-------------| +| `lib/SPIRV/libSPIRV/SPIRVInstruction.h` | Type validation relaxation for untyped pointers | +| `lib/SPIRV/OCLUtil.h` | AMDGCN↔SPIR-V address space mapping functions | +| `lib/SPIRV/OCLUtil.cpp` | Scope naming, atomic inc/dec wrap ops | +| `lib/SPIRV/SPIRVInternal.h` | Captures attribute mapping | +| `lib/SPIRV/SPIRVUtil.cpp` | C calling convention for AMDGPU | +| `lib/SPIRV/SPIRVToOCL.cpp` | Skip cast mutation for AMD | +| `lib/SPIRV/OCLToSPIRV.cpp` | Barrier argument validation | +| `lib/SPIRV/SPIRVLowerLLVMIntrinsic.cpp` | Extension name changes (INTEL→ALTERA) | ## Directory Structure diff --git a/include/LLVMSPIRVExtensions.inc b/include/LLVMSPIRVExtensions.inc index 0ec1224f4..87a01f43c 100644 --- a/include/LLVMSPIRVExtensions.inc +++ b/include/LLVMSPIRVExtensions.inc @@ -92,5 +92,7 @@ EXT(SPV_INTEL_sigmoid) EXT(SPV_INTEL_float4) EXT(SPV_INTEL_fp_conversions) EXT(SPV_KHR_float_controls2) +// AMD customization begin: AMD-specific extensions EXT(SPV_NV_shader_atomic_fp16_vector) EXT(SPV_AMD_weak_linkage) +// AMD customization end diff --git a/lib/SPIRV/CMakeLists.txt b/lib/SPIRV/CMakeLists.txt index 8b1278a7d..0274c841c 100644 --- a/lib/SPIRV/CMakeLists.txt +++ b/lib/SPIRV/CMakeLists.txt @@ -60,6 +60,7 @@ if(SPIRV_BACKEND_FOUND) list(APPEND SPIRVLIB_LINK_COMPONENTS "SPIRVCodeGen") endif() +# AMD customization begin: Rename library to LLVMSPIRVAMDLib add_llvm_library(LLVMSPIRVAMDLib ${SRC_LIST} LINK_COMPONENTS @@ -69,6 +70,7 @@ add_llvm_library(LLVMSPIRVAMDLib ) target_include_directories(LLVMSPIRVAMDLib +# AMD customization end PRIVATE ${LLVM_INCLUDE_DIRS} ${LLVM_SPIRV_INCLUDE_DIRS} diff --git a/lib/SPIRV/LLVMToSPIRVDbgTran.cpp b/lib/SPIRV/LLVMToSPIRVDbgTran.cpp index 916e518d5..c18719c75 100644 --- a/lib/SPIRV/LLVMToSPIRVDbgTran.cpp +++ b/lib/SPIRV/LLVMToSPIRVDbgTran.cpp @@ -1221,6 +1221,7 @@ LLVMToSPIRVDbgTran::transDbgGlobalVariable(const DIGlobalVariable *GV) { // DIExpression is non-empty GVE->getExpression()->getNumElements()) { if (Ops[VariableIdx] != getDebugInfoNoneId()) { +// AMD customization begin: DIOp-based global variable expression handling #ifdef SPIRV_HAS_DIOP_DIEXPRESSION if (GVE->getExpression()->holdsNewElements()) { Ops.resize(MaxOperandCount, getDebugInfoNoneId()); @@ -1228,6 +1229,7 @@ LLVMToSPIRVDbgTran::transDbgGlobalVariable(const DIGlobalVariable *GV) { transDbgExpression(GVE->getExpression())->getId(); } #endif +// AMD customization end break; } // Repurpose VariableIdx operand to hold the initial value held in the @@ -1613,6 +1615,7 @@ LLVMToSPIRVDbgTran::transDbgLocalVariable(const DILocalVariable *Var) { // DWARF Operations and expressions +// AMD customization begin: DIOp-based DIExpression operand translation template <> void LLVMToSPIRVDbgTran::transDIOpOperand(SPIRVWordVec &Vec, unsigned Idx, llvm::Type *Ty) { @@ -1632,10 +1635,12 @@ void LLVMToSPIRVDbgTran::transDIOpOperand(SPIRVWordVec &Vec, unsigned Idx, llvm::ConstantData *Data) { Vec[Idx] = SPIRVWriter->transConstant(Data)->getId(); } +// AMD customization end SPIRVEntry *LLVMToSPIRVDbgTran::transDbgExpression(const DIExpression *Expr) { SPIRVWordVec Operations; +// AMD customization begin: DIOp-based DIExpression translation #ifdef SPIRV_HAS_DIOP_DIEXPRESSION if (auto NewElems = Expr->getNewElementsRef()) { if (!(BM->allowExtraDIExpressions() || @@ -1690,6 +1695,7 @@ SPIRVEntry *LLVMToSPIRVDbgTran::transDbgExpression(const DIExpression *Expr) { return BM->addDebugInfo(SPIRVDebug::Expression, getVoidTy(), Operations); } #endif +// AMD customization end for (unsigned I = 0, N = Expr->getNumElements(); I < N; ++I) { using namespace SPIRVDebug::Operand::Operation; diff --git a/lib/SPIRV/LLVMToSPIRVDbgTran.h b/lib/SPIRV/LLVMToSPIRVDbgTran.h index 9c8e8548d..7503db7b6 100644 --- a/lib/SPIRV/LLVMToSPIRVDbgTran.h +++ b/lib/SPIRV/LLVMToSPIRVDbgTran.h @@ -159,8 +159,10 @@ class LLVMToSPIRVDbgTran { // DWARF expressions SPIRVEntry *transDbgExpression(const DIExpression *Expr); + // AMD customization begin: DIOp-based DIExpression support template void transDIOpOperand(SPIRVWordVec &Vec, unsigned Idx, OperandTy Operand); + // AMD customization end // Imported declarations and modules SPIRVEntry *transDbgImportedEntry(const DIImportedEntity *IE); diff --git a/lib/SPIRV/OCLToSPIRV.cpp b/lib/SPIRV/OCLToSPIRV.cpp index 823e3881c..55480b88e 100644 --- a/lib/SPIRV/OCLToSPIRV.cpp +++ b/lib/SPIRV/OCLToSPIRV.cpp @@ -403,10 +403,12 @@ void OCLToSPIRVBase::visitCallInst(CallInst &CI) { if (DemangledName == kOCLBuiltinName::WorkGroupBarrier || DemangledName == kOCLBuiltinName::Barrier || DemangledName == kOCLBuiltinName::SubGroupBarrier) { + // AMD customization begin: barrier argument validation for AMD if (F->arg_size() != 1 && F->arg_size() != 2 && F->getParent()->getTargetTriple().getVendor() == Triple::VendorType::AMD) return; // Somebody used the name. + // AMD customization end visitCallBarrier(&CI); return; } diff --git a/lib/SPIRV/OCLUtil.cpp b/lib/SPIRV/OCLUtil.cpp index 2e4bcae38..2ea7d2406 100644 --- a/lib/SPIRV/OCLUtil.cpp +++ b/lib/SPIRV/OCLUtil.cpp @@ -146,6 +146,7 @@ template <> void SPIRVMap::init() { add(OCLMS_sub_group, ScopeSubgroup); } +// AMD customization begin: scope name change ("work_item" -> "singlethread") template <> void SPIRVMap::init() { add("singlethread", ScopeInvocation); add("workgroup", ScopeWorkgroup); @@ -153,6 +154,7 @@ template <> void SPIRVMap::init() { add("all_svm_devices", ScopeCrossDevice); add("subgroup", ScopeSubgroup); } +// AMD customization end template <> void SPIRVMap::init() { add("reduce", GroupOperationReduce); @@ -658,8 +660,10 @@ template <> void LLVMSPIRVAtomicRmwOpCodeMap::init() { add(llvm::AtomicRMWInst::FAdd, OpAtomicFAddEXT); add(llvm::AtomicRMWInst::FMin, OpAtomicFMinEXT); add(llvm::AtomicRMWInst::FMax, OpAtomicFMaxEXT); + // AMD customization begin: atomic inc/dec wrap operation mapping add(llvm::AtomicRMWInst::UIncWrap, OpAtomicIIncrement); add(llvm::AtomicRMWInst::UDecWrap, OpAtomicIDecrement); + // AMD customization end } } // namespace SPIRV diff --git a/lib/SPIRV/OCLUtil.h b/lib/SPIRV/OCLUtil.h index 2bdf8f887..a3b689b27 100644 --- a/lib/SPIRV/OCLUtil.h +++ b/lib/SPIRV/OCLUtil.h @@ -500,6 +500,7 @@ inline OCLMemOrderKind mapSPIRVMemOrderToOCL(unsigned Sema) { return OCLMemOrderMap::rmap(extractSPIRVMemOrderSemantic(Sema)); } +// AMD customization begin: map AMDGCN address space to SPIR-V address space inline unsigned int mapAMDGCNAddrSpaceToSPIRV(unsigned int AS) { switch (AS) { case 0: @@ -517,7 +518,9 @@ inline unsigned int mapAMDGCNAddrSpaceToSPIRV(unsigned int AS) { return UINT_MAX; } } +// AMD customization end +// AMD customization begin: map SPIR-V storage class to AMDGPU address space inline SPIRAddressSpace mapSPIRVAddrSpaceToAMDGPU(SPIRVStorageClassKind SPVAS) { switch (SPVAS) { case StorageClassCrossWorkgroup: @@ -536,6 +539,7 @@ inline SPIRAddressSpace mapSPIRVAddrSpaceToAMDGPU(SPIRVStorageClassKind SPVAS) { return static_cast(UINT_MAX); } } +// AMD customization end bool isPipeOrAddressSpaceCastBI(const StringRef MangledName); bool isEnqueueKernelBI(const StringRef MangledName); diff --git a/lib/SPIRV/SPIRVInternal.h b/lib/SPIRV/SPIRVInternal.h index 968902605..fa259052b 100644 --- a/lib/SPIRV/SPIRVInternal.h +++ b/lib/SPIRV/SPIRVInternal.h @@ -239,7 +239,9 @@ inline void SPIRVMap::init() { add(Attribute::ByVal, FunctionParameterAttributeByVal); add(Attribute::StructRet, FunctionParameterAttributeSret); add(Attribute::NoAlias, FunctionParameterAttributeNoAlias); + // AMD customization begin: Captures attribute mapping add(Attribute::Captures, FunctionParameterAttributeNoCapture); + // AMD customization end add(Attribute::ReadOnly, FunctionParameterAttributeNoWrite); add(Attribute::ReadNone, FunctionParameterAttributeNoReadWrite); } diff --git a/lib/SPIRV/SPIRVLowerLLVMIntrinsic.cpp b/lib/SPIRV/SPIRVLowerLLVMIntrinsic.cpp index cf7987141..94ce20f4e 100644 --- a/lib/SPIRV/SPIRVLowerLLVMIntrinsic.cpp +++ b/lib/SPIRV/SPIRVLowerLLVMIntrinsic.cpp @@ -72,6 +72,7 @@ typedef struct { #define NO_REQUIRED_EXTENSION ExtensionID::Last #define NO_FORBIDDEN_EXTENSION ExtensionID::Last +// AMD customization begin: extension name changes (SPV_INTEL_arbitrary_precision_integers -> SPV_ALTERA_arbitrary_precision_integers) const std::map LLVMIntrinsicMapEntries = { // LLVM Intrinsic Name Required Extension Forbidden Extension Module with // emulation function @@ -121,6 +122,7 @@ const std::map LLVMIntrinsicMa { "llvm.sadd.with.overflow.i32", {NO_REQUIRED_EXTENSION, NO_FORBIDDEN_EXTENSION, LLVMSaddWithOverflow}}, { "llvm.sadd.with.overflow.i64", {NO_REQUIRED_EXTENSION, NO_FORBIDDEN_EXTENSION, LLVMSaddWithOverflow}}, }; +// AMD customization end // clang-format on } // namespace diff --git a/lib/SPIRV/SPIRVReader.cpp b/lib/SPIRV/SPIRVReader.cpp index 92e2c535c..6f4465599 100644 --- a/lib/SPIRV/SPIRVReader.cpp +++ b/lib/SPIRV/SPIRVReader.cpp @@ -75,7 +75,9 @@ #include "llvm/IR/PassInstrumentation.h" #include "llvm/IR/Type.h" #include "llvm/IR/TypedPointerType.h" +// AMD customization begin: Include for target triple handling #include "llvm/MC/TargetRegistry.h" +// AMD customization end #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" @@ -344,7 +346,9 @@ Type *SPIRVToLLVM::transType(SPIRVType *T, bool UseTPT) { SPIRVDBG(spvdbgs() << "[transType] " << *T << " -> ";) T->validate(); + // AMD customization begin: AMDGCN target detection for address space mapping auto IsAMDGCN = M->getTargetTriple().getVendor() == Triple::VendorType::AMD; + // AMD customization end switch (static_cast(T->getOpCode())) { case OpTypeVoid: return mapType(T, Type::getVoidTy(*Context)); @@ -359,14 +363,17 @@ Type *SPIRVToLLVM::transType(SPIRVType *T, bool UseTPT) { // and evaluated before the LLVM ArrayType can be constructed. auto *LenExpr = static_cast(T)->getLength(); auto *LenValue = cast(transValue(LenExpr, nullptr, nullptr)); + // AMD customization begin: Handle zero-sized arrays if (LenValue->getZExtValue() == UINT64_MAX && IsAMDGCN) return mapType(T, ArrayType::get(transType(T->getArrayElementType()), 0)); + // AMD customization end return mapType(T, ArrayType::get(transType(T->getArrayElementType()), LenValue->getZExtValue())); } case internal::OpTypeTokenINTEL: return mapType(T, Type::getTokenTy(*Context)); case OpTypePointer: { + // AMD customization begin: AMDGPU address space mapping unsigned AS = IsAMDGCN ? mapSPIRVAddrSpaceToAMDGPU(T->getPointerStorageClass()) : SPIRSPIRVAddrSpaceMap::rmap(T->getPointerStorageClass()); @@ -377,18 +384,21 @@ Type *SPIRVToLLVM::transType(SPIRVType *T, bool UseTPT) { T->getPointerElementType()->getOpCode() == OpTypeFunction) AS = IsAMDGCN ? M->getDataLayout().getProgramAddressSpace() :SPIRAS_CodeSectionINTEL; + // AMD customization end Type *ElementTy = transType(T->getPointerElementType(), UseTPT); if (UseTPT) return TypedPointerType::get(ElementTy, AS); return mapType(T, PointerType::get(*Context, AS)); } case OpTypeUntypedPointerKHR: { + // AMD customization begin: AMDGPU address space mapping unsigned AS = IsAMDGCN ? mapSPIRVAddrSpaceToAMDGPU(T->getPointerStorageClass()) : SPIRSPIRVAddrSpaceMap::rmap(T->getPointerStorageClass()); if (AS == SPIRAS_CodeSectionINTEL && !BM->shouldEmitFunctionPtrAddrSpace()) AS = IsAMDGCN ? M->getDataLayout().getProgramAddressSpace() : SPIRAS_Private; + // AMD customization end return mapType(T, PointerType::get(*Context, AS)); } case OpTypeVector: @@ -1171,6 +1181,7 @@ Value *SPIRVToLLVM::transConvertInst(SPIRVValue *BV, Function *F, } case OpBitcast: if (Src->getType()->isPointerTy() && Dst->isPointerTy()) { + // AMD customization begin: Handle pointer bitcast with address space conversion if (M->getTargetTriple().getVendor() == Triple::VendorType::AMD) { if (Src->getType()->getPointerAddressSpace() != Dst->getPointerAddressSpace()) @@ -1178,6 +1189,7 @@ Value *SPIRVToLLVM::transConvertInst(SPIRVValue *BV, Function *F, else return Src; // Spuriously inserted pointer BC. } + // AMD customization end } else if ((!Dst->isPointerTy() && Dst == Src->getType()) || (Src->getType() == Dst)) { // Spuriously inserted BC return Src; @@ -1243,9 +1255,11 @@ static void applyNoIntegerWrapDecorations(const SPIRVValue *BV, void SPIRVToLLVM::applyFPFastMathModeDecorations(const SPIRVValue *BV, Instruction *Inst) { + // AMD customization begin: Allow FP fast math on CallBase instructions for AMDGCN if (!isa(Inst) && (!M->getTargetTriple().isAMDGCN() || !isa(Inst))) return; + // AMD customization end SPIRVWord V{0}; if (BV->hasDecorate(DecorationFPFastMathMode, 0, &V)) { @@ -1725,11 +1739,13 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F, static_cast(BV); SPIRVFunction *F = BC->getFunction(); BV->setName(F->getName()); + // AMD customization begin: Use program address space for AMD targets const unsigned AS = M->getTargetTriple().getVendor() == Triple::VendorType::AMD ? M->getDataLayout().getProgramAddressSpace() : (BM->shouldEmitFunctionPtrAddrSpace() ? SPIRAS_CodeSectionINTEL : SPIRAS_Private); + // AMD customization end return mapValue(BV, transFunction(F, AS)); } @@ -1793,8 +1809,10 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F, AddrSpace = VectorComputeUtil::getVCGlobalVarAddressSpace(BS); Initializer = PoisonValue::get(Ty); } else + // AMD customization begin: AMDGPU address space mapping for variables AddrSpace = M->getTargetTriple().getVendor() == Triple::VendorType::AMD ? mapSPIRVAddrSpaceToAMDGPU(BS) : SPIRSPIRVAddrSpaceMap::rmap(BS); + // AMD customization end // Force SPIRV BuiltIn variable's name to be __spirv_BuiltInXXXX. // No matter what BV's linkage name is. SPIRVBuiltinVariableKind BVKind; @@ -1821,6 +1839,7 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F, ? GlobalValue::UnnamedAddr::Global : GlobalValue::UnnamedAddr::None); LVar->setInitializer(Initializer); + // AMD customization begin: Handle externally initialized global variables if (M->getTargetTriple().isAMDGCN()) { if (BVar->hasDecorate(DecorationUserTypeGOOGLE)) { // This is how the Translator stashes externally initialized @@ -1833,6 +1852,7 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F, LVar->setExternallyInitialized(true); } } + // AMD customization end if (IsVectorCompute) { LVar->addAttribute(kVCMetadata::VCGlobalVariable); SPIRVWord Offset; @@ -1845,11 +1865,13 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F, LVar->addAttribute(SEVAttr.value().getKindAsString(), SEVAttr.value().getValueAsString()); } + // AMD customization begin: Set section for llvm.used/llvm.compiler.used if (M->getTargetTriple().getVendor() == Triple::VendorType::AMD && LVar->hasAppendingLinkage() && (LVar->getName() == "llvm.compiler.used" || LVar->getName() == "llvm.used")) LVar->setSection("llvm.metadata"); + // AMD customization end return Res; } @@ -1966,6 +1988,7 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F, Phi->foreachPair([&](SPIRVValue *IncomingV, SPIRVBasicBlock *IncomingBB, size_t Index) { auto *Translated = transValue(IncomingV, F, BB); + // AMD customization begin: Handle address space mismatch in phi nodes if (LPhi->getType() != Translated->getType() && LPhi->getType()->isPointerTy() && F->getParent()->getTargetTriple().getVendor() == @@ -1974,6 +1997,7 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F, // these mismatches might occur; find a better way to handle it. Translated = ConstantExpr::getAddrSpaceCast(cast(Translated), LPhi->getType()); + // AMD customization end LPhi->addIncoming(Translated, dyn_cast(transValue(IncomingBB, F, BB))); }); @@ -2752,10 +2776,12 @@ Value *SPIRVToLLVM::transValueWithoutDecoration(SPIRVValue *BV, Function *F, BV, Call, static_cast(BC->getCalledValue()->getType()->getPointerElementType())); // Assuming we are calling a regular device function + // AMD customization begin: Use C calling convention for AMD targets Call->setCallingConv( M->getTargetTriple().getVendor() == Triple::VendorType::AMD ? CallingConv::C : CallingConv::SPIR_FUNC); + // AMD customization end // Don't set attributes, because at translation time we don't know which // function exactly we are calling. return mapValue(BV, Call); @@ -3209,11 +3235,13 @@ Value *SPIRVToLLVM::transFixedPointInst(SPIRVInstruction *BI, BasicBlock *BB) { std::vector Args; Args.reserve(8); if (RetTy->getIntegerBitWidth() > 64) { + // AMD customization begin: AMDGPU address space mapping for fixed point instructions llvm::PointerType *RetPtrTy = llvm::PointerType::get( *Context, M->getTargetTriple().getVendor() == Triple::VendorType::AMD ? mapSPIRVAddrSpaceToAMDGPU(StorageClassGeneric) : SPIRAS_Generic); + // AMD customization end Value *Alloca = new AllocaInst(RetTy, M->getDataLayout().getAllocaAddrSpace(), "", BB); Value *RetValPtr = new AddrSpaceCastInst(Alloca, RetPtrTy, "", BB); @@ -3241,10 +3269,12 @@ Value *SPIRVToLLVM::transFixedPointInst(SPIRVInstruction *BI, BasicBlock *BB) { FunctionCallee FCallee = M->getOrInsertFunction(FuncName, FT); auto *Func = cast(FCallee.getCallee()); + // AMD customization begin: Use C calling convention for AMD targets Func->setCallingConv( M->getTargetTriple().getVendor() == Triple::VendorType::AMD ? CallingConv::C : CallingConv::SPIR_FUNC); + // AMD customization end if (isFuncNoUnwind()) Func->addFnAttr(Attribute::NoUnwind); @@ -3340,11 +3370,13 @@ Value *SPIRVToLLVM::transArbFloatInst(SPIRVInstruction *BI, BasicBlock *BB, std::vector Args; if (RetTy->getIntegerBitWidth() > 64) { + // AMD customization begin: AMDGPU address space mapping for arbitrary float instructions llvm::PointerType *RetPtrTy = llvm::PointerType::get( *Context, M->getTargetTriple().getVendor() == Triple::VendorType::AMD ? mapSPIRVAddrSpaceToAMDGPU(StorageClassGeneric) : SPIRAS_Generic); + // AMD customization end ArgTys.push_back(RetPtrTy); Value *Alloca = new AllocaInst(RetTy, M->getDataLayout().getAllocaAddrSpace(), "", BB); @@ -3392,9 +3424,11 @@ Value *SPIRVToLLVM::transArbFloatInst(SPIRVInstruction *BI, BasicBlock *BB, FunctionCallee FCallee = M->getOrInsertFunction(FuncName, FT); auto *Func = cast(FCallee.getCallee()); + // AMD customization begin: Use C calling convention for AMD targets Func->setCallingConv( M->getTargetTriple().getVendor() == Triple::VendorType::AMD ? CallingConv::C : CallingConv::SPIR_FUNC); + // AMD customization end if (isFuncNoUnwind()) Func->addFnAttr(Attribute::NoUnwind); @@ -3457,9 +3491,11 @@ void SPIRVToLLVM::transFunctionAttrs(SPIRVFunction *BF, Function *F) { default: break; // do nothing } + // AMD customization begin: AMDGPU uses ByRef instead of ByVal // AMDGPU doesn't use ByVal, it is actually a masquerading ByRef. if (M->getTargetTriple().isAMDGCN() && LLVMKind == Attribute::ByVal) LLVMKind = Attribute::ByRef; + // AMD customization end // Make sure to use a correct constructor for a typed/typeless attribute auto A = AttrTy ? Attribute::get(*Context, LLVMKind, AttrTy) : (LLVMKind != Attribute::Captures) @@ -3637,6 +3673,7 @@ Function *SPIRVToLLVM::transFunction(SPIRVFunction *BF, unsigned AS) { auto Linkage = IsKernel ? GlobalValue::ExternalLinkage : transLinkageType(BF); FunctionType *FT = cast(transType(BF->getFunctionType())); std::string FuncName = BF->getName(); + // AMD customization begin: Handle variadic functions and intrinsic name fixes if (M->getTargetTriple().getVendor() == Triple::VendorType::AMD) { if (FuncName == "fprintf" || FuncName == "sprintf" || FuncName == "snprintf" || FuncName == "__isoc23_fscanf" || @@ -3652,6 +3689,7 @@ Function *SPIRVToLLVM::transFunction(SPIRVFunction *BF, unsigned AS) { if (FuncName == "llvm.va_end.p4") FuncName = "llvm.va_end.p0"; } + // AMD customization end StringRef FuncNameRef(FuncName); // Transform "@spirv.llvm_memset_p0i8_i32.volatile" to @llvm.memset.p0i8.i32 // assuming llvm.memset is supported by the device compiler. If this @@ -3666,6 +3704,7 @@ Function *SPIRVToLLVM::transFunction(SPIRVFunction *BF, unsigned AS) { ->getName(); } + // AMD customization begin: Fix memcpy intrinsic name mangling for AMDGCN // The name mangling here is broken, as it'd have used the SPIR-V AS Map, so // we have to fix it here and call the intrinsic. // TODO: maybe handle memcpy_inline and memcpy_atomic. @@ -3681,6 +3720,7 @@ Function *SPIRVToLLVM::transFunction(SPIRVFunction *BF, unsigned AS) { mapFunction(BF, F); return F; } + // AMD customization end // Special handling for spirv.llvm_umul_with_overflow_* functions // These were created during forward translation by lowering intrinsics. @@ -3712,11 +3752,13 @@ Function *SPIRVToLLVM::transFunction(SPIRVFunction *BF, unsigned AS) { mapFunction(BF, F); + // AMD customization begin: Set AMDGPU calling conventions if (M->getTargetTriple().getVendor() == Triple::VendorType::AMD) F->setCallingConv(IsKernel ? CallingConv::AMDGPU_KERNEL : CallingConv::C); else F->setCallingConv(IsKernel ? CallingConv::SPIR_KERNEL : CallingConv::SPIR_FUNC); + // AMD customization end transFunctionAttrs(BF, F); parseFloatControls2ExecutionModeId(BF, F); @@ -3802,10 +3844,12 @@ SPIRVToLLVM::transOCLBuiltinPostproc(SPIRVInstruction *BI, CallInst *CI, Value *SPIRVToLLVM::transBlockInvoke(SPIRVValue *Invoke, BasicBlock *BB) { auto *TranslatedInvoke = transFunction(static_cast(Invoke)); + // AMD customization begin: AMDGPU address space mapping for block invoke auto *Int8PtrTyGen = PointerType::get( *Context, M->getTargetTriple().getVendor() == Triple::VendorType::AMD ? mapSPIRVAddrSpaceToAMDGPU(StorageClassGeneric) : SPIRAS_Generic); + // AMD customization end return CastInst::CreatePointerBitCastOrAddrSpaceCast(TranslatedInvoke, Int8PtrTyGen, "", BB); } @@ -3819,10 +3863,12 @@ Instruction *SPIRVToLLVM::transWGSizeQueryBI(SPIRVInstruction *BI, Function *F = M->getFunction(FName); if (!F) { + // AMD customization begin: AMDGPU address space mapping for workgroup size query auto *Int8PtrTyGen = PointerType::get( *Context, M->getTargetTriple().getVendor() == Triple::VendorType::AMD ? mapSPIRVAddrSpaceToAMDGPU(StorageClassGeneric) : SPIRAS_Generic); + // AMD customization end FunctionType *FT = FunctionType::get(Type::getInt32Ty(*Context), {Int8PtrTyGen, Int8PtrTyGen}, false); F = Function::Create(FT, GlobalValue::ExternalLinkage, FName, M); @@ -3847,10 +3893,12 @@ Instruction *SPIRVToLLVM::transSGSizeQueryBI(SPIRVInstruction *BI, auto Ops = BI->getOperands(); Function *F = M->getFunction(FName); if (!F) { + // AMD customization begin: AMDGPU address space mapping for subgroup size query auto *Int8PtrTyGen = PointerType::get( *Context, M->getTargetTriple().getVendor() == Triple::VendorType::AMD ? mapSPIRVAddrSpaceToAMDGPU(StorageClassGeneric) : SPIRAS_Generic); + // AMD customization end SmallVector Tys = { transType(Ops[0]->getType()), // ndrange Int8PtrTyGen, // block_invoke @@ -3877,11 +3925,13 @@ Instruction *SPIRVToLLVM::transBuiltinFromInst(const std::string &FuncName, BasicBlock *BB) { std::string MangledName; auto Ops = BI->getOperands(); + // AMD customization begin: Add value operand for atomic increment/decrement if ((FuncName == "__spirv_AtomicIIncrement" || FuncName == "__spirv_AtomicIDecrement") && M->getTargetTriple().getVendor() == Triple::VendorType::AMD) Ops.insert(Ops.end(), BM->getValue(*BI->getDecorate(DecorationMaxByteOffsetId).cbegin())); + // AMD customization end Op OC = BI->getOpCode(); if (isUntypedAccessChainOpCode(OC)) { auto *AC = static_cast(BI); @@ -3901,6 +3951,7 @@ Instruction *SPIRVToLLVM::transBuiltinFromInst(const std::string &FuncName, // builtin mangling of atomic and matrix operations. if (isAtomicOpCodeUntypedPtrSupported(OC)) { auto *AI = static_cast(BI); + // AMD customization begin: AMDGPU address space mapping for atomic operations ArgTys[PtrIdx] = TypedPointerType::get( transType(AI->getSemanticType()), M->getTargetTriple().getVendor() == Triple::VendorType::AMD @@ -3908,6 +3959,7 @@ Instruction *SPIRVToLLVM::transBuiltinFromInst(const std::string &FuncName, BI->getValueType(Ops[PtrIdx]->getId())->getPointerStorageClass()) : SPIRSPIRVAddrSpaceMap::rmap( BI->getValueType(Ops[PtrIdx]->getId())->getPointerStorageClass())); + // AMD customization end } } @@ -3928,6 +3980,7 @@ Instruction *SPIRVToLLVM::transBuiltinFromInst(const std::string &FuncName, } } + // AMD customization begin: Use program address space for function types for (auto &I : ArgTys) { if (isa(I)) { I = TypedPointerType::get( @@ -3936,7 +3989,9 @@ Instruction *SPIRVToLLVM::transBuiltinFromInst(const std::string &FuncName, M->getDataLayout().getProgramAddressSpace() : SPIRAS_Private); } } + // AMD customization end + // AMD customization begin: Map AMDGCN address spaces to SPIRV for mangling if (BM->getDesiredBIsRepresentation() != BIsRepresentation::SPIRVFriendlyIR) if (M->getTargetTriple().getVendor() == Triple::VendorType::AMD) { auto TmpTys = ArgTys; @@ -3952,6 +4007,7 @@ Instruction *SPIRVToLLVM::transBuiltinFromInst(const std::string &FuncName, } else { mangleOpenClBuiltin(FuncName, ArgTys, MangledName); } + // AMD customization end else MangledName = getSPIRVFriendlyIRFunctionName(FuncName, OC, ArgTys, Ops); @@ -3973,9 +4029,11 @@ Instruction *SPIRVToLLVM::transBuiltinFromInst(const std::string &FuncName, if (!Func || Func->getFunctionType() != FT) { LLVM_DEBUG(for (auto &I : ArgTys) { dbgs() << *I << '\n'; }); Func = Function::Create(FT, GlobalValue::ExternalLinkage, MangledName, M); + // AMD customization begin: Use C calling convention for AMD targets Func->setCallingConv( M->getTargetTriple().getVendor() == Triple::VendorType::AMD ? CallingConv::C : CallingConv::SPIR_FUNC); + // AMD customization end if (isFuncNoUnwind()) Func->addFnAttr(Attribute::NoUnwind); if (isGroupOpCode(OC) || isGroupNonUniformOpcode(OC) || @@ -4042,9 +4100,11 @@ Type *SPIRVToLLVM::getTypedPtrFromUntypedOperand(SPIRVValue *Val, Type *RetTy) { } StorageClass SC = Val->getType()->getPointerStorageClass(); + // AMD customization begin: AMDGPU address space mapping for typed pointers unsigned AddrSpace = (M->getTargetTriple().getVendor() == Triple::VendorType::AMD) ? mapSPIRVAddrSpaceToAMDGPU(SC) : SPIRSPIRVAddrSpaceMap::rmap(SC); + // AMD customization end if (Ty) return TypedPointerType::get(Ty, AddrSpace); @@ -4284,6 +4344,7 @@ bool SPIRVToLLVM::translate() { DbgTran->addDbgInfoVersion(); DbgTran->finalize(); + // AMD customization begin: Add AMD module flags and ABI version global if (M->getTargetTriple().getVendor() != Triple::VendorType::AMD) return true; // TODO: this is temporary hardcoding, but will ultimately get handled in the @@ -4314,9 +4375,11 @@ bool SPIRVToLLVM::translate() { } return true; + // AMD customization end } bool SPIRVToLLVM::transAddressingModel() { + // AMD customization begin: Set AMD target triple and data layout if (BM->getGeneratorVer() == UINT16_MAX) { // TODO: we should use the Target registry here instead of hardcoding M->setTargetTriple(Triple("amdgcn-amd-amdhsa")); @@ -4327,6 +4390,7 @@ bool SPIRVToLLVM::transAddressingModel() { "v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"); return true; } + // AMD customization end switch (BM->getAddressingModel()) { case AddressingModelPhysical64: M->setTargetTriple(Triple(SPIR_TARGETTRIPLE64)); @@ -4720,10 +4784,12 @@ void SPIRVToLLVM::transUserSemantic(SPIRV::SPIRVFunction *Fun) { Constant *C = ConstantExpr::getPointerBitCastOrAddrSpaceCast(TransFun, ResType); + // AMD customization begin: AMDGPU address space mapping for annotations Type *Int8PtrTyPrivate = PointerType::get( *Context, M->getTargetTriple().getVendor() == Triple::VendorType::AMD ? mapSPIRVAddrSpaceToAMDGPU(StorageClassFunction) : SPIRAS_Private); + // AMD customization end IntegerType *Int32Ty = Type::getInt32Ty(*Context); llvm::Constant *Fields[5] = { @@ -5043,6 +5109,7 @@ bool SPIRVToLLVM::transMetadata() { if (BM->getDesiredBIsRepresentation() == BIsRepresentation::SPIRVFriendlyIR) transFunctionDecorationsToMetadata(BF, F); + // AMD customization begin: Check for AMDGPU_KERNEL calling convention if (F->getCallingConv() != CallingConv::SPIR_KERNEL && F->getCallingConv() != CallingConv::AMDGPU_KERNEL) continue; @@ -5052,6 +5119,7 @@ bool SPIRVToLLVM::transMetadata() { F->addFnAttr(Attribute::Convergent); F->addFnAttr(Attribute::MustProgress); } + // AMD customization end // Generate metadata for reqd_work_group_size if (auto *EM = BF->getExecutionMode(ExecutionModeLocalSize)) { @@ -5134,12 +5202,14 @@ bool SPIRVToLLVM::transMetadata() { } // Generate metadata for max_work_group_size if (auto *EM = BF->getExecutionMode(ExecutionModeMaxWorkgroupSizeINTEL)) { + // AMD customization begin: Use amdgpu-flat-work-group-size attribute if (M->getTargetTriple().getVendor() == Triple::VendorType::AMD) F->addFnAttr("amdgpu-flat-work-group-size", "1," + llvm::utostr(EM->getLiterals().front())); else F->setMetadata(kSPIR2MD::MaxWGSize, getMDNodeStringIntVec(Context, EM->getLiterals())); + // AMD customization end } // Generate metadata for no_global_work_offset if (BF->getExecutionMode(ExecutionModeNoGlobalOffsetINTEL)) { @@ -5257,9 +5327,11 @@ bool SPIRVToLLVM::transMetadata() { bool SPIRVToLLVM::transOCLMetadata(SPIRVFunction *BF) { Function *F = static_cast(getTranslatedValue(BF)); assert(F && "Invalid translated function"); + // AMD customization begin: Check for AMDGPU_KERNEL calling convention if (F->getCallingConv() != CallingConv::SPIR_KERNEL && F->getCallingConv() != CallingConv::AMDGPU_KERNEL) return true; + // AMD customization end if (BF->hasDecorate(DecorationVectorComputeFunctionINTEL)) return true; @@ -5269,6 +5341,7 @@ bool SPIRVToLLVM::transOCLMetadata(SPIRVFunction *BF) { Context, SPIR_MD_KERNEL_ARG_ADDR_SPACE, BF, F, [=](SPIRVFunctionParameter *Arg) { SPIRVType *ArgTy = Arg->getType(); + // AMD customization begin: AMDGPU address space mapping for kernel arguments SPIRAddressSpace AS = M->getTargetTriple().getVendor() == Triple::VendorType::AMD ? mapSPIRVAddrSpaceToAMDGPU(StorageClassFunction) : SPIRAS_Private; @@ -5276,6 +5349,7 @@ bool SPIRVToLLVM::transOCLMetadata(SPIRVFunction *BF) { AS = M->getTargetTriple().getVendor() == Triple::VendorType::AMD ? mapSPIRVAddrSpaceToAMDGPU(ArgTy->getPointerStorageClass()) : SPIRSPIRVAddrSpaceMap::rmap(ArgTy->getPointerStorageClass()); + // AMD customization end else if (ArgTy->isTypeOCLImage() || ArgTy->isTypePipe()) AS = SPIRAS_Global; return ConstantAsMetadata::get( @@ -5589,6 +5663,7 @@ bool SPIRVToLLVM::transAlign(SPIRVValue *BV, Value *V) { return true; } +// AMD customization begin: Translate OpenCL extended instructions to LLVM intrinsics for AMDGCN Instruction *SPIRVToLLVM::transLLVMFromExtInst(SPIRVExtInst *BC, Type *RetTy, std::vector ArgTys, BasicBlock *BB) { @@ -5800,6 +5875,7 @@ Instruction *SPIRVToLLVM::transLLVMFromExtInst(SPIRVExtInst *BC, Type *RetTy, return CI; } +// AMD customization end Instruction *SPIRVToLLVM::transOCLBuiltinFromExtInst(SPIRVExtInst *BC, BasicBlock *BB) { @@ -5864,8 +5940,10 @@ Instruction *SPIRVToLLVM::transOCLBuiltinFromExtInst(SPIRVExtInst *BC, } } } + // AMD customization begin: Use LLVM intrinsics for AMDGCN if (M->getTargetTriple().isAMDGCN()) return transLLVMFromExtInst(BC, RetTy, std::move(ArgTypes), BB); + // AMD customization end std::string MangledName = getSPIRVFriendlyIRFunctionName(ExtOp, ArgTypes, RetTy); diff --git a/lib/SPIRV/SPIRVToLLVMDbgTran.cpp b/lib/SPIRV/SPIRVToLLVMDbgTran.cpp index 55f014ad1..f0b7f5407 100644 --- a/lib/SPIRV/SPIRVToLLVMDbgTran.cpp +++ b/lib/SPIRV/SPIRVToLLVMDbgTran.cpp @@ -1151,10 +1151,12 @@ MDNode *SPIRVToLLVMDbgTran::transGlobalVariable(const SPIRVExtInst *DebugInst) { bool IsDefinition = Flags & SPIRVDebug::FlagIsDefinition; MDNode *VarDecl = nullptr; if (IsDefinition) { +// AMD customization begin: expression poisoning for invalid DIOp expressions #ifdef SPIRV_HAS_DIOP_DIEXPRESSION if (DIExpr && DIExpr->holdsNewElements() && !DIExpr->isValid()) DIExpr = DIExpr->getPoisoned(); #endif +// AMD customization end VarDecl = getDIBuilder(DebugInst).createGlobalVariableExpression( Parent, Name, LinkageName, File, LineNo, Ty, IsLocal, IsDefinition, DIExpr, StaticMemberDecl); @@ -1419,6 +1421,7 @@ DINode *SPIRVToLLVMDbgTran::transModule(const SPIRVExtInst *DebugInst) { Scope, Name, ConfigMacros, IncludePath, ApiNotes, File, Line, IsDecl); } +// AMD customization begin: DIOp-based DIExpression operand translation template <> Type *SPIRVToLLVMDbgTran::transDIOpOperand(const SPIRVExtInst *DbgOpInst, unsigned Idx) { @@ -1442,7 +1445,9 @@ SPIRVToLLVMDbgTran::transDIOpOperand(const SPIRVExtInst *DbgOpInst, SPIRVValue *Val = BM->get(Operands[Idx]); return cast(SPIRVReader->transValue(Val, nullptr, nullptr)); } +// AMD customization end +// AMD customization begin: DIOp-based DIExpression translation MDNode * SPIRVToLLVMDbgTran::tryTransDIOpDIExpression(const SPIRVExtInst *DebugInst) { using namespace SPIRVDebug::Operand::Operation; @@ -1500,6 +1505,7 @@ SPIRVToLLVMDbgTran::tryTransDIOpDIExpression(const SPIRVExtInst *DebugInst) { return DIExpression::get(M->getContext(), bool(), DIOps); #endif } +// AMD customization end DIMacroFile * SPIRVToLLVMDbgTran::getOrCreateMacroFile(DIFile *File, @@ -1737,6 +1743,7 @@ SPIRVToLLVMDbgTran::transDebugIntrinsic(const SPIRVExtInst *DebugInst, auto GetExpression = [&](SPIRVId Id) -> DIExpression * { return transDebugInst(BM->get(Id)); }; + // AMD customization begin: expression poisoning for invalid DIOp expressions auto PoisonInvalidExpr = [&](DIExpression *Expr, DILocalVariable *Var, const Value *Op) { #ifdef SPIRV_HAS_DIOP_DIEXPRESSION @@ -1750,6 +1757,7 @@ SPIRVToLLVMDbgTran::transDebugIntrinsic(const SPIRVExtInst *DebugInst, #endif return Expr; }; + // AMD customization end SPIRVWordVec Ops = DebugInst->getArguments(); switch (DebugInst->getExtOp()) { case SPIRVDebug::Scope: diff --git a/lib/SPIRV/SPIRVToLLVMDbgTran.h b/lib/SPIRV/SPIRVToLLVMDbgTran.h index 65c99aa81..72943b47b 100644 --- a/lib/SPIRV/SPIRVToLLVMDbgTran.h +++ b/lib/SPIRV/SPIRVToLLVMDbgTran.h @@ -178,9 +178,11 @@ class SPIRVToLLVMDbgTran { DINode *transModule(const SPIRVExtInst *DebugInst); + // AMD customization begin: DIOp-based DIExpression support template OperTy transDIOpOperand(const SPIRVExtInst *DbgOpInst, unsigned Idx); MDNode *tryTransDIOpDIExpression(const SPIRVExtInst *DebugInst); + // AMD customization end DIMacro *transMacroDef(const SPIRVExtInst *DebugInst); DIMacro *transMacroUndef(const SPIRVExtInst *DebugInst); diff --git a/lib/SPIRV/SPIRVToOCL.cpp b/lib/SPIRV/SPIRVToOCL.cpp index c04292421..ab2df72c5 100644 --- a/lib/SPIRV/SPIRVToOCL.cpp +++ b/lib/SPIRV/SPIRVToOCL.cpp @@ -239,8 +239,10 @@ void SPIRVToOCLBase::visitCastInst(CastInst &Cast) { !isa(Cast) && !isa(Cast)) return; + // AMD customization begin: skip cast mutation for AMD target if (M->getTargetTriple().getVendor() == Triple::VendorType::AMD) return; + // AMD customization end Type const *SrcTy = Cast.getSrcTy(); Type *DstVecTy = Cast.getDestTy(); diff --git a/lib/SPIRV/SPIRVToOCL20.cpp b/lib/SPIRV/SPIRVToOCL20.cpp index ea780980e..4f778976f 100644 --- a/lib/SPIRV/SPIRVToOCL20.cpp +++ b/lib/SPIRV/SPIRVToOCL20.cpp @@ -74,6 +74,7 @@ bool SPIRVToOCL20Base::runSPIRVToOCL(Module &Module) { return true; } +// AMD customization begin: map OpenCL memory scope to AMDGPU sync scope static SyncScope::ID mapOpenCLScopeToAMDGPU(LLVMContext &Ctx, uint64_t S) { if (S == OCLMS_work_item) return SyncScope::SingleThread; @@ -85,7 +86,9 @@ static SyncScope::ID mapOpenCLScopeToAMDGPU(LLVMContext &Ctx, uint64_t S) { return Ctx.getOrInsertSyncScopeID("wavefront"); return SyncScope::System; } +// AMD customization end +// AMD customization begin: translate SPIR-V atomic compare exchange to LLVM atomic cmpxchg static void translateSPIRVCmpXchgToLLVM(CallInst *CI, Op OC) { auto Ptr = CI->getOperand(0); auto Cmp = CI->getOperand(1); @@ -114,7 +117,9 @@ static void translateSPIRVCmpXchgToLLVM(CallInst *CI, Op OC) { CI->dropAllReferences(); CI->eraseFromParent(); } +// AMD customization end +// AMD customization begin: map SPIR-V atomic op to LLVM AtomicRMW BinOp static AtomicRMWInst::BinOp getAtomicRMWInstForOp(Op op) { switch (op) { case OpAtomicAnd: @@ -151,7 +156,9 @@ static AtomicRMWInst::BinOp getAtomicRMWInstForOp(Op op) { llvm_unreachable("Undefined operation"); } } +// AMD customization end +// AMD customization begin: translate SPIR-V atomic builtin to LLVM atomic instruction static void translateSPIRVAtomicBuiltinToLLVMAtomicOp(CallInst *CI, Op OC) { if (OC == OpAtomicCompareExchange || OC == OpAtomicCompareExchangeWeak) return translateSPIRVCmpXchgToLLVM(CI, OC); @@ -183,7 +190,9 @@ static void translateSPIRVAtomicBuiltinToLLVMAtomicOp(CallInst *CI, Op OC) { CI->dropAllReferences(); CI->eraseFromParent(); } +// AMD customization end +// AMD customization begin: translate SPIR-V memory barrier to LLVM fence static void visitCallLLVMFence(CallInst *CI) { // TODO: AMDSPV JANK, this is incorrect auto MS = transSPIRVMemoryScopeIntoOCLMemoryScope(CI->getArgOperand(0), CI); auto MO = transSPIRVMemorySemanticsIntoOCLMemoryOrder(CI->getArgOperand(1), @@ -201,10 +210,13 @@ static void visitCallLLVMFence(CallInst *CI) { // TODO: AMDSPV JANK, this is inc CI->dropAllReferences(); CI->eraseFromParent(); } +// AMD customization end void SPIRVToOCL20Base::visitCallSPIRVMemoryBarrier(CallInst *CI) { + // AMD customization begin: use LLVM fence for AMD target if (M->getTargetTriple().getVendor() == Triple::VendorType::AMD) return visitCallLLVMFence(CI); + // AMD customization end Value *MemScope = SPIRV::transSPIRVMemoryScopeIntoOCLMemoryScope(CI->getArgOperand(0), CI); @@ -264,8 +276,10 @@ void SPIRVToOCL20Base::mutateAtomicName(CallInst *CI, Op OC) { void SPIRVToOCL20Base::visitCallSPIRVAtomicBuiltin(CallInst *CI, Op OC) { CallInst *CIG = mutateCommonAtomicArguments(CI, OC); + // AMD customization begin: translate to LLVM atomic ops for AMD target if (M->getTargetTriple().getVendor() == Triple::VendorType::AMD) return translateSPIRVAtomicBuiltinToLLVMAtomicOp(CIG, OC); + // AMD customization end switch (OC) { case OpAtomicIIncrement: @@ -306,6 +320,7 @@ CallInst *SPIRVToOCL20Base::mutateCommonAtomicArguments(CallInst *CI, Op OC) { auto OrderIdx = Ptr + 2; auto Mutator = mutateCallInst(CI, Name); + // AMD customization begin: use AMD address space mapping for atomics Mutator.mapArgs([=](IRBuilder<> &Builder, Value *PtrArg, Type *PtrArgTy) { if (auto *TypedPtrTy = dyn_cast(PtrArgTy)) { unsigned AS = M->getTargetTriple().getVendor() == Triple::VendorType::AMD @@ -321,6 +336,7 @@ CallInst *SPIRVToOCL20Base::mutateCommonAtomicArguments(CallInst *CI, Op OC) { } return std::make_pair(PtrArg, PtrArgTy); }); + // AMD customization end Mutator.mapArg(ScopeIdx, [=](Value *Arg) { return SPIRV::transSPIRVMemoryScopeIntoOCLMemoryScope(Arg, CI); }); @@ -354,6 +370,7 @@ void SPIRVToOCL20Base::visitCallSPIRVAtomicCmpExchg(CallInst *CI) { // OpAtomicCompareExchangeWeak is not "weak" at all, but instead has the same // semantics as OpAtomicCompareExchange. + // AMD customization begin: use AMD address space mapping for cmpxchg mutateCallInst(CI, "atomic_compare_exchange_strong_explicit") .mapArg(1, [=](IRBuilder<> &Builder, Value *Expected) { @@ -368,6 +385,7 @@ void SPIRVToOCL20Base::visitCallSPIRVAtomicCmpExchg(CallInst *CI) { PExpected, PtrTyAS, PExpected->getName() + ".as"); return std::make_pair(V, TypedPointerType::get(MemTy, AddrSpc)); }) + // AMD customization end .moveArg(4, 2) .changeReturnType(Type::getInt1Ty(*Ctx), [=](IRBuilder<> &Builder, CallInst *NewCI) { @@ -403,6 +421,7 @@ void SPIRVToOCL20Base::visitCallSPIRVEnqueueKernel(CallInst *CI, Op OC) { FName = "__enqueue_kernel_events_varargs"; auto Mutator = mutateCallInst(CI, FName.str()); + // AMD customization begin: use AMD address space for enqueue kernel Mutator.mapArg(6, [=](IRBuilder<> &Builder, Value *Invoke) { unsigned AS = M->getTargetTriple().getVendor() == Triple::VendorType::AMD ? mapSPIRVAddrSpaceToAMDGPU(StorageClassGeneric) : SPIRAS_Generic; @@ -411,6 +430,7 @@ void SPIRVToOCL20Base::visitCallSPIRVEnqueueKernel(CallInst *CI, Op OC) { return std::make_pair( Replace, TypedPointerType::get(Builder.getInt8Ty(), AS)); }); + // AMD customization end if (!HasVaargs) { // Remove arguments at indices 8 (Param Size), 9 (Param Align) diff --git a/lib/SPIRV/SPIRVUtil.cpp b/lib/SPIRV/SPIRVUtil.cpp index 1e3b1c7ae..adbf91aeb 100644 --- a/lib/SPIRV/SPIRVUtil.cpp +++ b/lib/SPIRV/SPIRVUtil.cpp @@ -311,10 +311,12 @@ Function *getOrCreateFunction(Module *M, Type *RetTy, ArrayRef ArgTypes, if (F) NewF->setDSOLocal(F->isDSOLocal()); F = NewF; + // AMD customization begin: use C calling convention for AMDGPU if (M->getTargetTriple().getVendor() == Triple::VendorType::AMD) F->setCallingConv(CallingConv::C); else F->setCallingConv(CallingConv::SPIR_FUNC); + // AMD customization end if (Attrs) F->setAttributes(*Attrs); } diff --git a/lib/SPIRV/SPIRVWriter.cpp b/lib/SPIRV/SPIRVWriter.cpp index 5721d40c8..555741e04 100644 --- a/lib/SPIRV/SPIRVWriter.cpp +++ b/lib/SPIRV/SPIRVWriter.cpp @@ -454,10 +454,12 @@ SPIRVType *LLVMToSPIRVBase::transType(Type *T) { if (T->isArrayTy()) { // SPIR-V 1.3 s3.32.6: Length is the number of elements in the array. // It must be at least 1. + // AMD customization begin: zero-sized array handling const auto ArraySize = T->getArrayNumElements() ? T->getArrayNumElements() : (M->getTargetTriple().getVendor() == Triple::VendorType::AMD ? UINT64_MAX : 1); + // AMD customization end Type *ElTy = T->getArrayElementType(); SPIRVType *TransType = BM->addArrayType( @@ -848,14 +850,18 @@ SPIRVType *LLVMToSPIRVBase::transScavengedType(Value *V) { // error. To be on the safe side, an assertion is added to check printf // never reaches this point. assert(F->getName() != "printf"); + // AMD customization begin: variadic function support if (M->getTargetTriple().getVendor() != Triple::VendorType::AMD) BM->getErrorLog().checkError(!FnTy->isVarArg(), SPIRVEC_UnsupportedVarArgFunction); + // AMD customization end SPIRVType *RT = transType(FnTy->getReturnType()); + // AMD customization begin: dispatch pointer return type handling if (M->getTargetTriple().getVendor() == Triple::VendorType::AMD && F->hasName() && F->getName().contains("dispatch.ptr")) RT = transType(PointerType::get(F->getContext(), SPIRAS_Constant)); + // AMD customization end std::vector PT; for (Argument &Arg : F->args()) { @@ -1639,21 +1645,25 @@ SPIRVValue *LLVMToSPIRVBase::transUnaryInst(UnaryInstruction *U, return BM->addUndef(ExpectedTy); } } + // AMD customization begin: variadic function support if (isa(U) && M->getTargetTriple().getVendor() == Triple::VendorType::AMD) { SPIRVType *ExpectedTy = transScavengedType(U); return BM->addUndef(ExpectedTy); } + // AMD customization end Op BOC = OpNop; if (auto *Cast = dyn_cast(U)) { const auto SrcAddrSpace = Cast->getSrcTy()->getPointerAddressSpace(); const auto DestAddrSpace = Cast->getDestTy()->getPointerAddressSpace(); if (DestAddrSpace == SPIRAS_Generic) { + // AMD customization begin: address space cast validation relaxation if (M->getTargetTriple().getVendor() != Triple::VendorType::AMD) getErrorLog().checkError( SrcAddrSpace != SPIRAS_Constant, SPIRVEC_InvalidModule, U, "Casts from constant address space to generic are illegal\n"); + // AMD customization end BOC = OpPtrCastToGeneric; // In SPIR-V only casts to/from generic are allowed. But with // SPV_INTEL_usm_storage_classes we can also have casts from global_device @@ -1697,10 +1707,12 @@ SPIRVValue *LLVMToSPIRVBase::transUnaryInst(UnaryInstruction *U, SrcAddrSpace == SPIRAS_Generic, SPIRVEC_InvalidModule, U, "Casts from private/local/global address space are allowed only to " "generic\n"); + // AMD customization begin: address space cast validation relaxation if (M->getTargetTriple().getVendor() != Triple::VendorType::AMD) getErrorLog().checkError( DestAddrSpace != SPIRAS_Constant, SPIRVEC_InvalidModule, U, "Casts from generic address space to constant are illegal\n"); + // AMD customization end BOC = OpGenericCastToPtr; } } else { @@ -2186,10 +2198,12 @@ LLVMToSPIRVBase::transValueWithoutDecoration(Value *V, SPIRVBasicBlock *BB, : nullptr, GV->isConstant(), transLinkageType(GV), BVarInit, GV->getName().str(), StorageClass, nullptr)); + // AMD customization begin: externally initialized global variable decoration if (GV->isExternallyInitialized() && M->getTargetTriple().getVendor() == Triple::VendorType::AMD) BVar->addDecorate(DecorationUserTypeGOOGLE, BM->getString("externally_initialized")->getId()); + // AMD customization end if (IsVectorCompute) { BVar->addDecorate(DecorationVectorComputeVariableINTEL); @@ -2409,11 +2423,13 @@ LLVMToSPIRVBase::transValueWithoutDecoration(Value *V, SPIRVBasicBlock *BB, BB)); if (AllocaInst *Alc = dyn_cast(V)) { + // AMD customization begin: alloca type handling SPIRVType *TranslatedTy = M->getTargetTriple().getVendor() != Triple::VendorType::AMD ? transScavengedType(V) : BM->addPointerType(StorageClassFunction, transType(Alc->getAllocatedType())); + // AMD customization end if (Alc->isArrayAllocation()) { SPIRVValue *Length = transValue(Alc->getArraySize(), BB); assert(Length && "Couldn't translate array size!"); @@ -2782,6 +2798,7 @@ LLVMToSPIRVBase::transValueWithoutDecoration(Value *V, SPIRVBasicBlock *BB, // Implement FSub through FNegate and AtomicFAddExt Ops[3] = BM->addUnaryInst(OpFNegate, Ty, OpVals[3], BB)->getId(); OC = OpAtomicFAddEXT; + // AMD customization begin: atomic inc/dec wrap handling } else if (Op == AtomicRMWInst::UIncWrap || Op == AtomicRMWInst::UDecWrap) { OC = LLVMSPIRVAtomicRmwOpCodeMap::map(Op); auto WrapV = Ops.back(); @@ -2791,6 +2808,7 @@ LLVMToSPIRVBase::transValueWithoutDecoration(Value *V, SPIRVBasicBlock *BB, WrapV)); return IncDec; // TODO: figure out handling of saturating val. + // AMD customization end } else OC = LLVMSPIRVAtomicRmwOpCodeMap::map(Op); @@ -2885,12 +2903,14 @@ void addFuncPointerCallArgumentAttributes(CallInst *CI, for (const auto &I : CI->getAttributes().getParamAttrs(ArgNo)) { spv::FunctionParameterAttribute Attr = spv::FunctionParameterAttributeMax; SPIRSPIRVFuncParamAttrMap::find(I.getKindAsEnum(), &Attr); + // AMD customization begin: function pointer call argument attribute handling (Captures) if (Attr != spv::FunctionParameterAttributeMax && (I.getKindAsEnum() != Attribute::Captures || capturesNothing(I.getCaptureInfo()))) FuncPtrCall->addDecorate( new SPIRVDecorate(spv::internal::DecorationArgumentAttributeINTEL, FuncPtrCall, ArgNo, Attr)); + // AMD customization end } } } @@ -3405,9 +3425,11 @@ void LLVMToSPIRVBase::transMemAliasingINTELDecorations(Instruction *Inst, if (!BM->isAllowedToUseExtension( ExtensionID::SPV_INTEL_memory_access_aliasing)) return; + // AMD customization begin: memory aliasing decoration handling for fences if (!BV->hasId() && M->getTargetTriple().getVendor() == Triple::VendorType::AMD) // Fences return; + // AMD customization end if (MDNode *AliasingListMD = Inst->getMetadata(LLVMContext::MD_alias_scope)) { auto *MemAliasList = addMemAliasingINTELInstructions(BM, AliasingListMD); if (!MemAliasList) @@ -6388,10 +6410,12 @@ bool isEmptyLLVMModule(Module *M) { } bool LLVMToSPIRVBase::translate() { + // AMD customization begin: generator version marking if (M->getTargetTriple().getVendor() == Triple::VendorType::AMD) BM->setGeneratorVer(UINT16_MAX); else BM->setGeneratorVer(KTranslatorVer); + // AMD customization end if (isEmptyLLVMModule(M)) BM->addCapability(CapabilityLinkage); @@ -7480,9 +7504,11 @@ LLVMToSPIRVBase::transLinkageType(const GlobalValue *GV) { if (GV->hasLinkOnceODRLinkage()) if (BM->isAllowedToUseExtension(ExtensionID::SPV_KHR_linkonce_odr)) return SPIRVLinkageTypeKind::LinkageTypeLinkOnceODR; + // AMD customization begin: weak linkage translation if (GV->hasWeakAnyLinkage()) if (BM->isAllowedToUseExtension(ExtensionID::SPV_AMD_weak_linkage)) return spv::internal::LinkageTypeWeak; + // AMD customization end return SPIRVLinkageTypeKind::LinkageTypeExport; } @@ -7630,7 +7656,9 @@ bool runSpirvBackend(Module *M, std::string &Result, std::string &ErrMsg, SPIRV::ExtensionID::SPV_KHR_expect_assume, SPIRV::ExtensionID::SPV_KHR_bit_instructions, SPIRV::ExtensionID::SPV_KHR_linkonce_odr, + // AMD customization begin: SPV_AMD_weak_linkage extension support SPIRV::ExtensionID::SPV_AMD_weak_linkage, + // AMD customization end SPIRV::ExtensionID::SPV_INTEL_inline_assembly, SPIRV::ExtensionID::SPV_INTEL_bfloat16_conversion, SPIRV::ExtensionID::SPV_KHR_subgroup_rotate, diff --git a/lib/SPIRV/libSPIRV/SPIRV.debug.h b/lib/SPIRV/libSPIRV/SPIRV.debug.h index 5d66aa3d2..d0f03e296 100644 --- a/lib/SPIRV/libSPIRV/SPIRV.debug.h +++ b/lib/SPIRV/libSPIRV/SPIRV.debug.h @@ -7,11 +7,13 @@ #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/IR/DebugInfoMetadata.h" +// AMD customization begin: DIOp-based DIExpression support #ifdef __has_include #if __has_include("llvm/IR/DIExprOps.def") #define SPIRV_HAS_DIOP_DIEXPRESSION #endif #endif +// AMD customization end namespace SPIRVDebug { @@ -294,6 +296,7 @@ enum ExpressionOpCode { ImplicitPointerTag = 166, TagOffset = 167, +// AMD customization begin: DIOp-based debug operations #ifdef SPIRV_HAS_DIOP_DIEXPRESSION // AMD-specific debug operations, padded. Poisoned = 10000, @@ -306,6 +309,7 @@ enum ExpressionOpCode { #else AMDExtensions_Begin = 10000, #endif +// AMD customization end }; enum ImportedEntityTag { @@ -607,8 +611,10 @@ enum { FlagsIdx = 8, StaticMemberDeclarationIdx = 9, MinOperandCount = 9, + // AMD customization begin: DIOp-based expression support DIOpBasedExprIdx = 10, MaxOperandCount = 11, + // AMD customization end }; } @@ -961,6 +967,7 @@ static std::unordered_map OpCountMap { { ImplicitPointerTag, 2 }, { TagOffset, 2 }, +// AMD customization begin: DIOp-based operation counts #ifdef SPIRV_HAS_DIOP_DIEXPRESSION { Poisoned, 1 }, @@ -969,6 +976,7 @@ static std::unordered_map OpCountMap { #define HANDLE_OP2(NAME, T1, N1, T2, N2) { DIOp##NAME, 3 }, #include "llvm/IR/DIExprOps.def" #endif +// AMD customization end }; } @@ -1481,6 +1489,7 @@ inline void DbgExpressionOpCodeMap::init() { add(dwarf::DW_OP_LLVM_implicit_pointer, SPIRVDebug::ImplicitPointerTag); add(dwarf::DW_OP_LLVM_tag_offset, SPIRVDebug::TagOffset); +// AMD customization begin: DIOp-based expression mapping #ifdef SPIRV_HAS_DIOP_DIEXPRESSION add(dwarf::DW_OP_LLVM_poisoned, SPIRVDebug::Poisoned); #endif @@ -1496,6 +1505,7 @@ inline void DbgExpressionDIOpBasedOpCodeMap::init() { #include "llvm/IR/DIExprOps.def" #endif } +// AMD customization end typedef SPIRVMap DbgImportedEntityMap; diff --git a/lib/SPIRV/libSPIRV/SPIRVDecorate.h b/lib/SPIRV/libSPIRV/SPIRVDecorate.h index a8c415663..84544084d 100644 --- a/lib/SPIRV/libSPIRV/SPIRVDecorate.h +++ b/lib/SPIRV/libSPIRV/SPIRVDecorate.h @@ -314,9 +314,11 @@ class SPIRVDecorateLinkageAttr : public SPIRVDecorate { std::optional getRequiredExtension() const override { if (getLinkageType() == SPIRVLinkageTypeKind::LinkageTypeLinkOnceODR) return ExtensionID::SPV_KHR_linkonce_odr; + // AMD customization begin: Weak linkage support if (getLinkageType() == static_cast(spv::internal::LinkageTypeWeak)) return ExtensionID::SPV_AMD_weak_linkage; + // AMD customization end return {}; } }; diff --git a/lib/SPIRV/libSPIRV/SPIRVInstruction.h b/lib/SPIRV/libSPIRV/SPIRVInstruction.h index 0a22dcf5e..510632aba 100644 --- a/lib/SPIRV/libSPIRV/SPIRVInstruction.h +++ b/lib/SPIRV/libSPIRV/SPIRVInstruction.h @@ -647,6 +647,7 @@ class SPIRVStore : public SPIRVInstruction, public SPIRVMemoryAccess { SPIRVInstruction::validate(); if (getSrc()->isForward() || getDst()->isForward()) return; + // AMD customization begin: Relax type check for untyped pointers assert( (getValueType(PtrId) ->getPointerElementType() @@ -655,6 +656,7 @@ class SPIRVStore : public SPIRVInstruction, public SPIRVMemoryAccess { getValueType(ValId)->isTypeUntypedPointerKHR()) || getValueType(PtrId)->getPointerElementType() == getValueType(ValId)) && "Inconsistent operand types"); + // AMD customization end } private: @@ -764,8 +766,10 @@ class SPIRVBinary : public SPIRVInstTemplateBase { } else if (isBinaryPtrOpCode(OpCode)) { assert((Op1Ty->isTypePointer() && Op2Ty->isTypePointer()) && "Invalid types for PtrEqual, PtrNotEqual, or PtrDiff instruction"); + // AMD customization begin: Fix untyped pointer check (|| -> &&) if (!Op1Ty->isTypeUntypedPointerKHR() && !Op2Ty->isTypeUntypedPointerKHR()) + // AMD customization end assert( static_cast(Op1Ty)->getElementType() == static_cast(Op2Ty)->getElementType() && @@ -1023,6 +1027,7 @@ class SPIRVPhi : public SPIRVInstruction { assert(WordCount == Pairs.size() + FixedWordCount); assert(OpCode == OC); assert(Pairs.size() % 2 == 0); + // AMD customization begin: Relax PHI type validation for untyped pointers foreachPair([=](SPIRVValue *IncomingV, SPIRVBasicBlock *IncomingBB) { assert(IncomingV->isForward() || IncomingV->getType() == Type || (IncomingV->getType()->isTypePointer() && @@ -1031,6 +1036,7 @@ class SPIRVPhi : public SPIRVInstruction { Type->isTypePointer())); assert(IncomingBB->isBasicBlock() || IncomingBB->isForward()); }); + // AMD customization end SPIRVInstruction::validate(); } @@ -1150,9 +1156,11 @@ class SPIRVSelectBase : public SPIRVInstTemplateBase { : getValueType(Condition); (void)ConTy; assert(ConTy->isTypeBool() && "Invalid type"); + // AMD customization begin: Skip type check for untyped pointers if (getType()->getOpCode() != OpTypeUntypedPointerKHR && getValueType(Op1)->getOpCode() != OpTypeUntypedPointerKHR && getValueType(Op2)->getOpCode() != OpTypeUntypedPointerKHR) + // AMD customization end assert(getType() == getValueType(Op1) && getType() == getValueType(Op2) && "Inconsistent type"); } @@ -1271,8 +1279,10 @@ class SPIRVSwitch : public SPIRVInstruction { return static_cast(getValue(Default)); } size_t getLiteralSize() const { + // AMD customization begin: Ensure minimum byte width of 1 unsigned ByteWidth = std::max(getSelect()->getType()->getBitWidth() / 8, 1u); + // AMD customization end unsigned Remainder = (ByteWidth % sizeof(SPIRVWord)) != 0; return (ByteWidth / sizeof(SPIRVWord)) + Remainder; } diff --git a/lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h b/lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h index 40b0e1ee8..8ff531988 100644 --- a/lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h +++ b/lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h @@ -143,7 +143,9 @@ inline bool isValid(spv::LinkageType V) { case LinkageTypeExport: case LinkageTypeImport: case LinkageTypeLinkOnceODR: + // AMD customization begin: Weak linkage support case internal::LinkageTypeWeak: + // AMD customization end case internal::LinkageTypeInternal: return true; default: diff --git a/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h b/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h index 331778389..85c26f3a8 100644 --- a/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h +++ b/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h @@ -58,7 +58,9 @@ template <> inline void SPIRVMap::init() { add(LinkageTypeExport, "Export"); add(LinkageTypeImport, "Import"); add(LinkageTypeLinkOnceODR, "LinkOnceODR"); + // AMD customization begin: Weak linkage support add(internal::LinkageTypeWeak, "Weak"); + // AMD customization end add(internal::LinkageTypeInternal, "Internal"); add(LinkageTypeMax, "Max"); } diff --git a/lib/SPIRV/libSPIRV/spirv_internal.hpp b/lib/SPIRV/libSPIRV/spirv_internal.hpp index fe88295c1..689189c83 100644 --- a/lib/SPIRV/libSPIRV/spirv_internal.hpp +++ b/lib/SPIRV/libSPIRV/spirv_internal.hpp @@ -133,7 +133,9 @@ enum InternalExecutionMode { constexpr LinkageType LinkageTypeInternal = static_cast(ILTInternal); +// AMD customization begin: Weak linkage support for SPV_AMD_weak_linkage constexpr LinkageType LinkageTypeWeak = static_cast(3); +// AMD customization end enum InternalJointMatrixLayout { RowMajor = 0, diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index bacf36b8a..ef5c1b22b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -5,8 +5,10 @@ llvm_canonicalize_cmake_booleans(LLVM_SPIRV_BUILD_EXTERNAL) llvm_canonicalize_cmake_booleans(SPIRV_BACKEND_FOUND) # required by lit.site.cfg.py.in +# AMD customization begin: Use amd-llvm-spirv and LLVMSPIRVAMDLib get_target_property(LLVM_SPIRV_DIR amd-llvm-spirv BINARY_DIR) get_target_property(LLVM_SPIRV_LIB_DIR LLVMSPIRVAMDLib BINARY_DIR) +# AMD customization end set(LLVM_SPIRV_TEST_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) if(SPIRV_TOOLS_FOUND AND NOT SPIRV-Tools-tools_FOUND) @@ -129,6 +131,7 @@ if(NOT LLVM_SPIRV_BUILD_EXTERNAL) endif(NOT LLVM_SPIRV_BUILD_EXTERNAL) +# AMD customization begin: Rename test target to check-amd-llvm-spirv add_lit_testsuite(check-amd-llvm-spirv "Running the amd-llvm-spirv regression tests" ${CMAKE_CURRENT_BINARY_DIR} ARGS @@ -149,5 +152,6 @@ if(LLVM_SPIRV_BUILD_EXTERNAL) DEPENDS check-amd-llvm-spirv ) +# AMD customization end cmake_policy(POP) endif(LLVM_SPIRV_BUILD_EXTERNAL) diff --git a/test/lit.cfg.py b/test/lit.cfg.py index 5b7e82075..17ee3cb79 100644 --- a/test/lit.cfg.py +++ b/test/lit.cfg.py @@ -52,6 +52,7 @@ tool_dirs = [config.llvm_spirv_dir, config.llvm_tools_dir] +# AMD customization begin: Use amd-llvm-spirv tool name tools = ['llvm-as', 'llvm-dis', 'amd-llvm-spirv', 'not'] if not config.spirv_skip_debug_info_tests: tools.extend(['llc', 'llvm-dwarfdump', 'llvm-objdump', 'llvm-readelf', 'llvm-readobj']) @@ -61,6 +62,7 @@ tools.extend(['llc']) config.substitutions.append(('llvm-spirv', 'amd-llvm-spirv')) +# AMD customization end llvm_config.add_tool_substitutions(tools, tool_dirs) using_spirv_tools = False @@ -88,6 +90,7 @@ else: config.substitutions.append(('spirv-val', ':')) +# AMD customization begin: Use LLVMSPIRVAMDLib library name if not config.llvm_spirv_build_external and config.llvm_build_shared_libs: config.available_features.add('pass-plugin') config.substitutions.append( @@ -98,6 +101,7 @@ ), ) ) +# AMD customization end llvm_config.with_system_environment('LD_LIBRARY_PATH') if using_spirv_tools: diff --git a/tools/llvm-spirv/CMakeLists.txt b/tools/llvm-spirv/CMakeLists.txt index efd57b486..0f6e19c5a 100644 --- a/tools/llvm-spirv/CMakeLists.txt +++ b/tools/llvm-spirv/CMakeLists.txt @@ -1,3 +1,4 @@ +# AMD customization begin: Rename library to SPIRVAMDLib set(LLVM_LINK_COMPONENTS SPIRVAMDLib Analysis @@ -10,6 +11,7 @@ set(LLVM_LINK_COMPONENTS TargetParser TransformUtils ) +# AMD customization end # llvm_setup_rpath messes with the rpath making llvm-spirv not # executable from the build directory in out-of-tree builds @@ -20,6 +22,7 @@ if(LLVM_SPIRV_BUILD_EXTERNAL) ) endif() +# AMD customization begin: Rename tool to amd-llvm-spirv add_llvm_tool(amd-llvm-spirv llvm-spirv.cpp ${add_llvm_tool_options} @@ -32,6 +35,7 @@ if (LLVM_SPIRV_BUILD_EXTERNAL OR LLVM_LINK_LLVM_DYLIB) endif() target_include_directories(amd-llvm-spirv +# AMD customization end PRIVATE ${LLVM_INCLUDE_DIRS} ${LLVM_SPIRV_INCLUDE_DIRS} From 7f47b1d707993030760ba0c74526ed8d37db9fed Mon Sep 17 00:00:00 2001 From: Dmitry Sidorov Date: Tue, 31 Mar 2026 12:43:16 +0200 Subject: [PATCH 2/2] Apply suggestion from @MrSidims --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 989884860..a6f3720b6 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ All AMD-specific code is marked with `// AMD customization begin:` and | `lib/SPIRV/SPIRVUtil.cpp` | C calling convention for AMDGPU | | `lib/SPIRV/SPIRVToOCL.cpp` | Skip cast mutation for AMD | | `lib/SPIRV/OCLToSPIRV.cpp` | Barrier argument validation | -| `lib/SPIRV/SPIRVLowerLLVMIntrinsic.cpp` | Extension name changes (INTEL→ALTERA) | +| `lib/SPIRV/SPIRVLowerLLVMIntrinsic.cpp` | Extension name changes (INTEL->ALTERA) | ## Directory Structure