[NFC][SPIRV] Move SPIRVStripConvergenceIntrinsics to Utils (#188537)

inbelic · web-flow · commit ffde06f4c119 · 2026-04-16T09:24:33.000-07:00
The `SPIRVStripConvergenceIntrinsic` pass was written as a spirv pass as it is the currently the only target that emits convergence tokens during codegen. There is nothing target specific to the pass, and, we plan to emit convergence tokens when targeting DirectX (and all targets in general), so move the pass to a common place. The previous pass used temporary `Undef`s, as part of moving the pass we can simply reverse the traverse order to remove the use of `Undef` as it is deprecated. Enables the pass for targeting DirectX and is a pre-req for: #188792. Assisted by: Github Copilot
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
@@ -322,6 +322,8 @@ LLVM_ABI void initializeStackSafetyInfoWrapperPassPass(PassRegistry &);
 LLVM_ABI void initializeStackSlotColoringLegacyPass(PassRegistry &);
 LLVM_ABI void
 initializeStraightLineStrengthReduceLegacyPassPass(PassRegistry &);
+LLVM_ABI void
+initializeStripConvergenceIntrinsicsLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeStripDebugMachineModulePass(PassRegistry &);
 LLVM_ABI void initializeStructurizeCFGLegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeTailCallElimPass(PassRegistry &);
diff --git a/llvm/include/llvm/Transforms/Utils.h b/llvm/include/llvm/Transforms/Utils.h
@@ -129,6 +129,13 @@ LLVM_ABI Pass *createCanonicalizeFreezeInLoopsPass();
 // functions that are registered in @llvm.global_ctors and which contain a call
 // to `__cxa_atexit` to register their destructor functions.
 LLVM_ABI ModulePass *createLowerGlobalDtorsLegacyPass();
+
+//===----------------------------------------------------------------------===//
+//
+// createStripConvergenceIntrinsicsPass - Strip convergence intrinsics and
+// convergencectrl operand bundles.
+//
+LLVM_ABI FunctionPass *createStripConvergenceIntrinsicsPass();
 } // namespace llvm
 
 #endif
diff --git a/llvm/include/llvm/Transforms/Utils/StripConvergenceIntrinsics.h b/llvm/include/llvm/Transforms/Utils/StripConvergenceIntrinsics.h
@@ -0,0 +1,29 @@
+//===- StripConvergenceIntrinsics.h -----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// This pass strips convergence intrinsics and operand bundles as those are
+/// only useful when modifying the CFG during IR passes.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_UTILS_STRIPCONVERGENCEINTRINSICS_H
+#define LLVM_TRANSFORMS_UTILS_STRIPCONVERGENCEINTRINSICS_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class StripConvergenceIntrinsicsPass
+    : public PassInfoMixin<StripConvergenceIntrinsicsPass> {
+public:
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &);
+};
+
+} // namespace llvm
+
+#endif // LLVM_TRANSFORMS_UTILS_STRIPCONVERGENCEINTRINSICS_H
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
@@ -374,6 +374,7 @@
 #include "llvm/Transforms/Utils/PredicateInfo.h"
 #include "llvm/Transforms/Utils/ProfileVerify.h"
 #include "llvm/Transforms/Utils/RelLookupTableConverter.h"
+#include "llvm/Transforms/Utils/StripConvergenceIntrinsics.h"
 #include "llvm/Transforms/Utils/StripGCRelocates.h"
 #include "llvm/Transforms/Utils/StripNonLineTableDebugInfo.h"
 #include "llvm/Transforms/Utils/SymbolRewriter.h"
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
@@ -542,6 +542,7 @@ FUNCTION_PASS("sjlj-eh-prepare", SjLjEHPreparePass(TM))
 FUNCTION_PASS("slp-vectorizer", SLPVectorizerPass())
 FUNCTION_PASS("slsr", StraightLineStrengthReducePass())
 FUNCTION_PASS("stack-protector", StackProtectorPass(*TM))
+FUNCTION_PASS("strip-convergence-intrinsics", StripConvergenceIntrinsicsPass())
 FUNCTION_PASS("strip-gc-relocates", StripGCRelocates())
 FUNCTION_PASS("tailcallelim", TailCallElimPass())
 FUNCTION_PASS("transform-warning", WarnMissedTransformationsPass())
diff --git a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
@@ -50,6 +50,7 @@
 #include "llvm/Transforms/IPO/GlobalDCE.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Scalar/Scalarizer.h"
+#include "llvm/Transforms/Utils.h"
 #include <optional>
 
 using namespace llvm;
@@ -81,6 +82,7 @@ LLVMInitializeDirectXTarget() {
   initializeDXILForwardHandleAccessesLegacyPass(*PR);
   initializeDSELegacyPassPass(*PR);
   initializeDXILCBufferAccessLegacyPass(*PR);
+  initializeStripConvergenceIntrinsicsLegacyPassPass(*PR);
 }
 
 class DXILTargetObjectFile : public TargetLoweringObjectFile {
@@ -110,6 +112,7 @@ class DirectXPassConfig : public TargetPassConfig {
 
   FunctionPass *createTargetRegisterAllocator(bool) override { return nullptr; }
   void addCodeGenPrepare() override {
+    addPass(createStripConvergenceIntrinsicsPass());
     addPass(createDXILFinalizeLinkageLegacyPass());
     addPass(createGlobalDCEPass());
     addPass(createDXILMemIntrinsicsLegacyPass());
diff --git a/llvm/lib/Target/SPIRV/CMakeLists.txt b/llvm/lib/Target/SPIRV/CMakeLists.txt
@@ -29,7 +29,6 @@ add_llvm_target(SPIRVCodeGen
   SPIRVInstructionSelector.cpp
   SPIRVLegalizeImplicitBinding.cpp
   SPIRVLegalizeZeroSizeArrays.cpp
-  SPIRVStripConvergentIntrinsics.cpp
   SPIRVLegalizePointerCast.cpp
   SPIRVMergeRegionExitTargets.cpp
   SPIRVISelLowering.cpp
diff --git a/llvm/lib/Target/SPIRV/SPIRV.h b/llvm/lib/Target/SPIRV/SPIRV.h
@@ -24,7 +24,6 @@ FunctionPass *createSPIRVStructurizerPass();
 ModulePass *createSPIRVCBufferAccessLegacyPass();
 ModulePass *createSPIRVPushConstantAccessLegacyPass(SPIRVTargetMachine *TM);
 FunctionPass *createSPIRVMergeRegionExitTargetsPass();
-FunctionPass *createSPIRVStripConvergenceIntrinsicsPass();
 ModulePass *createSPIRVLegalizeImplicitBindingPass();
 ModulePass *createSPIRVLegalizeZeroSizeArraysPass(const SPIRVTargetMachine &TM);
 FunctionPass *createSPIRVLegalizePointerCastPass(SPIRVTargetMachine *TM);
@@ -57,7 +56,6 @@ void initializeSPIRVRegularizerPass(PassRegistry &);
 void initializeSPIRVMergeRegionExitTargetsPass(PassRegistry &);
 void initializeSPIRVPrepareFunctionsPass(PassRegistry &);
 void initializeSPIRVPrepareGlobalsPass(PassRegistry &);
-void initializeSPIRVStripConvergentIntrinsicsPass(PassRegistry &);
 void initializeSPIRVLegalizeImplicitBindingPass(PassRegistry &);
 void initializeSPIRVLegalizeZeroSizeArraysLegacyPass(PassRegistry &);
 void initializeSPIRVCtorDtorLoweringLegacyPass(PassRegistry &);
diff --git a/llvm/lib/Target/SPIRV/SPIRVStripConvergentIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVStripConvergentIntrinsics.cpp
diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
@@ -66,7 +66,6 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeSPIRVTarget() {
   initializeSPIRVEmitNonSemanticDIPass(PR);
   initializeSPIRVPrepareFunctionsPass(PR);
   initializeSPIRVPrepareGlobalsPass(PR);
-  initializeSPIRVStripConvergentIntrinsicsPass(PR);
   initializeSPIRVCtorDtorLoweringLegacyPass(PR);
 }
 
@@ -229,7 +228,7 @@ void SPIRVPassConfig::addISelPrepare() {
     addPass(createLoopSimplifyPass());
   }
   SPIRVTargetMachine &TM = getTM<SPIRVTargetMachine>();
-  addPass(createSPIRVStripConvergenceIntrinsicsPass());
+  addPass(createStripConvergenceIntrinsicsPass());
   addPass(createSPIRVLegalizeImplicitBindingPass());
   addPass(createSPIRVLegalizeZeroSizeArraysPass(TM));
   addPass(createSPIRVCBufferAccessLegacyPass());
diff --git a/llvm/lib/Transforms/Utils/CMakeLists.txt b/llvm/lib/Transforms/Utils/CMakeLists.txt
@@ -74,6 +74,7 @@ add_llvm_component_library(LLVMTransformUtils
   RelLookupTableConverter.cpp
   ScalarEvolutionExpander.cpp
   SCCPSolver.cpp
+  StripConvergenceIntrinsics.cpp
   StripGCRelocates.cpp
   SSAUpdater.cpp
   SSAUpdaterBulk.cpp
diff --git a/llvm/lib/Transforms/Utils/StripConvergenceIntrinsics.cpp b/llvm/lib/Transforms/Utils/StripConvergenceIntrinsics.cpp
@@ -0,0 +1,99 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass strips convergence intrinsics and convergencectrl operand bundles,
+// as those are only useful when modifying the CFG during IR passes.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Utils/StripConvergenceIntrinsics.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Transforms/Utils.h"
+
+using namespace llvm;
+
+static bool stripConvergenceIntrinsics(Function &F) {
+  SmallVector<IntrinsicInst *> ConvergenceIntrinsics;
+  bool Changed = false;
+
+  for (BasicBlock &BB : F) {
+    for (Instruction &I : make_early_inc_range(BB)) {
+      auto *CI = dyn_cast<CallInst>(&I);
+      if (!CI)
+        continue;
+
+      // Strip a convergencectrl operand bundle if present. Note that
+      // convergence intrinsics (e.g. convergence.loop) may use a
+      // convergencectrl bundle.
+      if (CI->getOperandBundle(LLVMContext::OB_convergencectrl)) {
+        auto *NewCall = CallBase::removeOperandBundle(
+            CI, LLVMContext::OB_convergencectrl, CI->getIterator());
+        NewCall->copyMetadata(*CI);
+        CI->replaceAllUsesWith(NewCall);
+        CI->eraseFromParent();
+        CI = cast<CallInst>(NewCall);
+        Changed = true;
+      }
+
+      // Collect convergence intrinsics for deferred removal.
+      if (auto *II = dyn_cast<IntrinsicInst>(CI))
+        if (II->getIntrinsicID() == Intrinsic::experimental_convergence_entry ||
+            II->getIntrinsicID() == Intrinsic::experimental_convergence_loop ||
+            II->getIntrinsicID() == Intrinsic::experimental_convergence_anchor)
+          ConvergenceIntrinsics.push_back(II);
+    }
+  }
+
+  // Erase all convergence intrinsics now that convergence tokens are no
+  // longer in use.
+  for (IntrinsicInst *II : ConvergenceIntrinsics)
+    II->eraseFromParent();
+  Changed |= !ConvergenceIntrinsics.empty();
+
+  return Changed;
+}
+
+PreservedAnalyses
+StripConvergenceIntrinsicsPass::run(Function &F, FunctionAnalysisManager &) {
+  if (!stripConvergenceIntrinsics(F))
+    return PreservedAnalyses::all();
+  PreservedAnalyses PA;
+  PA.preserveSet<CFGAnalyses>();
+  return PA;
+}
+
+namespace {
+class StripConvergenceIntrinsicsLegacyPass : public FunctionPass {
+public:
+  static char ID;
+
+  StripConvergenceIntrinsicsLegacyPass() : FunctionPass(ID) {
+    initializeStripConvergenceIntrinsicsLegacyPassPass(
+        *PassRegistry::getPassRegistry());
+  }
+
+  bool runOnFunction(Function &F) override {
+    return stripConvergenceIntrinsics(F);
+  }
+};
+} // namespace
+
+char StripConvergenceIntrinsicsLegacyPass::ID = 0;
+INITIALIZE_PASS(StripConvergenceIntrinsicsLegacyPass,
+                "strip-convergence-intrinsics",
+                "Strip convergence intrinsics and operand bundles", false,
+                false)
+
+FunctionPass *llvm::createStripConvergenceIntrinsicsPass() {
+  return new StripConvergenceIntrinsicsLegacyPass();
+}
diff --git a/llvm/test/CodeGen/DirectX/llc-pipeline.ll b/llvm/test/CodeGen/DirectX/llc-pipeline.ll
@@ -14,6 +14,8 @@
 ; CHECK-OBJ-NEXT: Create Garbage Collector Module Metadata
 
 ; CHECK-NEXT: ModulePass Manager
+; CHECK-NEXT:   FunctionPass Manager
+; CHECK-NEXT:     Strip convergence intrinsics and operand bundles
 ; CHECK-NEXT:   DXIL Finalize Linkage
 ; CHECK-NEXT:   Dead Global Elimination
 ; CHECK-NEXT:   DXIL Memory Intrinsic Elimination
diff --git a/llvm/test/CodeGen/DirectX/strip-convergence-intrinsics.ll b/llvm/test/CodeGen/DirectX/strip-convergence-intrinsics.ll
@@ -0,0 +1,38 @@
+; RUN: llc %s -mtriple=dxil-pc-shadermodel6.3-library -o - | FileCheck %s
+
+; Verify that convergence intrinsics and operand bundles are stripped
+; during DXIL lowering pipeline.
+
+; CHECK-LABEL: define float @test(
+; CHECK-NOT: convergence
+; CHECK: ret float
+
+; CHECK-LABEL: define void @test_loop(
+; CHECK-NOT: convergence
+; CHECK: ret void
+
+define float @test(float %a) convergent {
+entry:
+  %0 = call token @llvm.experimental.convergence.entry()
+  %1 = call float @llvm.dx.dot(float %a) [ "convergencectrl"(token %0) ]
+  ret float %1
+}
+
+define void @test_loop(float %a) convergent {
+entry:
+  %0 = call token @llvm.experimental.convergence.entry()
+  br label %loop
+
+loop:
+  %1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %0) ]
+  call void @convergent_callee() [ "convergencectrl"(token %1) ]
+  br i1 true, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+declare void @convergent_callee() convergent
+declare float @llvm.dx.dot(float)
+declare token @llvm.experimental.convergence.entry()
+declare token @llvm.experimental.convergence.loop()
diff --git a/llvm/test/CodeGen/SPIRV/llc-pipeline.ll b/llvm/test/CodeGen/SPIRV/llc-pipeline.ll
@@ -41,7 +41,7 @@
 ; SPIRV-O0-NEXT:      Dominator Tree Construction
 ; SPIRV-O0-NEXT:      Natural Loop Information
 ; SPIRV-O0-NEXT:      Canonicalize natural loops
-; SPIRV-O0-NEXT:      SPIRV strip convergent intrinsics
+; SPIRV-O0-NEXT:      Strip convergence intrinsics and operand bundles
 ; SPIRV-O0-NEXT:    SPIRV Legalize Implicit Binding
 ; SPIRV-O0-NEXT:    SPIRV Legalize Zero-Size Arrays
 ; SPIRV-O0-NEXT:    SPIRV CBuffer Access
@@ -156,7 +156,7 @@
 ; SPIRV-Opt-NEXT:      Dominator Tree Construction
 ; SPIRV-Opt-NEXT:      Natural Loop Information
 ; SPIRV-Opt-NEXT:      Canonicalize natural loops
-; SPIRV-Opt-NEXT:      SPIRV strip convergent intrinsics
+; SPIRV-Opt-NEXT:      Strip convergence intrinsics and operand bundles
 ; SPIRV-Opt-NEXT:    SPIRV Legalize Implicit Binding
 ; SPIRV-Opt-NEXT:    SPIRV Legalize Zero-Size Arrays
 ; SPIRV-Opt-NEXT:    SPIRV CBuffer Access
diff --git a/llvm/test/Transforms/StripConvergenceIntrinsics/basic.ll b/llvm/test/Transforms/StripConvergenceIntrinsics/basic.ll
