diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp index 259b6c040706b..383f752a3a417 100644 --- a/clang/lib/CodeGen/CGCUDANV.cpp +++ b/clang/lib/CodeGen/CGCUDANV.cpp @@ -25,6 +25,7 @@ #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/GlobalValue.h" #include "llvm/IR/ReplaceConstant.h" #include "llvm/Support/Format.h" #include "llvm/Support/VirtualFileSystem.h" @@ -975,7 +976,10 @@ llvm::Function *CGNVCUDARuntime::makeModuleCtorFunction() { // Generate a unique module ID. SmallString<64> ModuleID; llvm::raw_svector_ostream OS(ModuleID); - OS << ModuleIDPrefix << llvm::format("%" PRIx64, FatbinWrapper->getGUID()); + OS << ModuleIDPrefix + << llvm::format("%" PRIx64, + llvm::GlobalValue::getGUIDAssumingExternalLinkage( + FatbinWrapper->getName())); llvm::Constant *ModuleIDConstant = makeConstantArray( std::string(ModuleID), "", ModuleIDSectionName, 32, /*AddNull=*/true); diff --git a/clang/test/CodeGen/cfi-icall-trap-recover-runtime.c b/clang/test/CodeGen/cfi-icall-trap-recover-runtime.c index 2c44842f9d28e..5717fc66488af 100644 --- a/clang/test/CodeGen/cfi-icall-trap-recover-runtime.c +++ b/clang/test/CodeGen/cfi-icall-trap-recover-runtime.c @@ -15,32 +15,32 @@ // TRAP-LABEL: define hidden void @f( -// TRAP-SAME: ) #[[ATTR0:[0-9]+]] !type [[META6:![0-9]+]] !type [[META7:![0-9]+]] { +// TRAP-SAME: ) #[[ATTR0:[0-9]+]] !type [[META6:![0-9]+]] !type [[META7:![0-9]+]] // TRAP-NEXT: [[ENTRY:.*:]] // TRAP-NEXT: ret void // // ABORT-LABEL: define hidden void @f( -// ABORT-SAME: ) #[[ATTR0:[0-9]+]] !type [[META6:![0-9]+]] !type [[META7:![0-9]+]] { +// ABORT-SAME: ) #[[ATTR0:[0-9]+]] !type [[META6:![0-9]+]] !type [[META7:![0-9]+]] // ABORT-NEXT: [[ENTRY:.*:]] // ABORT-NEXT: ret void // // RECOVER-LABEL: define hidden void @f( -// RECOVER-SAME: ) #[[ATTR0:[0-9]+]] !type [[META6:![0-9]+]] !type [[META7:![0-9]+]] { +// RECOVER-SAME: ) #[[ATTR0:[0-9]+]] !type [[META6:![0-9]+]] !type [[META7:![0-9]+]] // RECOVER-NEXT: [[ENTRY:.*:]] // RECOVER-NEXT: ret void // // ABORT_MIN-LABEL: define hidden void @f( -// ABORT_MIN-SAME: ) #[[ATTR0:[0-9]+]] !type [[META6:![0-9]+]] !type [[META7:![0-9]+]] { +// ABORT_MIN-SAME: ) #[[ATTR0:[0-9]+]] !type [[META6:![0-9]+]] !type [[META7:![0-9]+]] // ABORT_MIN-NEXT: [[ENTRY:.*:]] // ABORT_MIN-NEXT: ret void // // RECOVER_MIN-LABEL: define hidden void @f( -// RECOVER_MIN-SAME: ) #[[ATTR0:[0-9]+]] !type [[META6:![0-9]+]] !type [[META7:![0-9]+]] { +// RECOVER_MIN-SAME: ) #[[ATTR0:[0-9]+]] !type [[META6:![0-9]+]] !type [[META7:![0-9]+]] // RECOVER_MIN-NEXT: [[ENTRY:.*:]] // RECOVER_MIN-NEXT: ret void // // PRESERVE_MIN-LABEL: define hidden void @f( -// PRESERVE_MIN-SAME: ) #[[ATTR0:[0-9]+]] !type [[META6:![0-9]+]] !type [[META7:![0-9]+]] { +// PRESERVE_MIN-SAME: ) #[[ATTR0:[0-9]+]] !type [[META6:![0-9]+]] !type [[META7:![0-9]+]] // PRESERVE_MIN-NEXT: [[ENTRY:.*:]] // PRESERVE_MIN-NEXT: ret void // @@ -50,7 +50,7 @@ void f() { void xf(); // TRAP-LABEL: define hidden void @g( -// TRAP-SAME: i32 noundef [[B:%.*]]) #[[ATTR0]] !type [[META8:![0-9]+]] !type [[META9:![0-9]+]] { +// TRAP-SAME: i32 noundef [[B:%.*]]) #[[ATTR0]] !type [[META8:![0-9]+]] !type [[META9:![0-9]+]] // TRAP-NEXT: [[ENTRY:.*:]] // TRAP-NEXT: [[B_ADDR:%.*]] = alloca i32, align 4 // TRAP-NEXT: [[FP:%.*]] = alloca ptr, align 8 @@ -71,7 +71,7 @@ void xf(); // TRAP-NEXT: ret void // // ABORT-LABEL: define hidden void @g( -// ABORT-SAME: i32 noundef [[B:%.*]]) #[[ATTR0]] !type [[META8:![0-9]+]] !type [[META9:![0-9]+]] { +// ABORT-SAME: i32 noundef [[B:%.*]]) #[[ATTR0]] !type [[META8:![0-9]+]] !type [[META9:![0-9]+]] // ABORT-NEXT: [[ENTRY:.*:]] // ABORT-NEXT: [[B_ADDR:%.*]] = alloca i32, align 4 // ABORT-NEXT: [[FP:%.*]] = alloca ptr, align 8 @@ -93,7 +93,7 @@ void xf(); // ABORT-NEXT: ret void // // RECOVER-LABEL: define hidden void @g( -// RECOVER-SAME: i32 noundef [[B:%.*]]) #[[ATTR0]] !type [[META8:![0-9]+]] !type [[META9:![0-9]+]] { +// RECOVER-SAME: i32 noundef [[B:%.*]]) #[[ATTR0]] !type [[META8:![0-9]+]] !type [[META9:![0-9]+]] // RECOVER-NEXT: [[ENTRY:.*:]] // RECOVER-NEXT: [[B_ADDR:%.*]] = alloca i32, align 4 // RECOVER-NEXT: [[FP:%.*]] = alloca ptr, align 8 @@ -115,7 +115,7 @@ void xf(); // RECOVER-NEXT: ret void // // ABORT_MIN-LABEL: define hidden void @g( -// ABORT_MIN-SAME: i32 noundef [[B:%.*]]) #[[ATTR0]] !type [[META8:![0-9]+]] !type [[META9:![0-9]+]] { +// ABORT_MIN-SAME: i32 noundef [[B:%.*]]) #[[ATTR0]] !type [[META8:![0-9]+]] !type [[META9:![0-9]+]] // ABORT_MIN-NEXT: [[ENTRY:.*:]] // ABORT_MIN-NEXT: [[B_ADDR:%.*]] = alloca i32, align 4 // ABORT_MIN-NEXT: [[FP:%.*]] = alloca ptr, align 8 @@ -136,7 +136,7 @@ void xf(); // ABORT_MIN-NEXT: ret void // // RECOVER_MIN-LABEL: define hidden void @g( -// RECOVER_MIN-SAME: i32 noundef [[B:%.*]]) #[[ATTR0]] !type [[META8:![0-9]+]] !type [[META9:![0-9]+]] { +// RECOVER_MIN-SAME: i32 noundef [[B:%.*]]) #[[ATTR0]] !type [[META8:![0-9]+]] !type [[META9:![0-9]+]] // RECOVER_MIN-NEXT: [[ENTRY:.*:]] // RECOVER_MIN-NEXT: [[B_ADDR:%.*]] = alloca i32, align 4 // RECOVER_MIN-NEXT: [[FP:%.*]] = alloca ptr, align 8 @@ -157,7 +157,7 @@ void xf(); // RECOVER_MIN-NEXT: ret void // // PRESERVE_MIN-LABEL: define hidden void @g( -// PRESERVE_MIN-SAME: i32 noundef [[B:%.*]]) #[[ATTR0]] !type [[META8:![0-9]+]] !type [[META9:![0-9]+]] { +// PRESERVE_MIN-SAME: i32 noundef [[B:%.*]]) #[[ATTR0]] !type [[META8:![0-9]+]] !type [[META9:![0-9]+]] // PRESERVE_MIN-NEXT: [[ENTRY:.*:]] // PRESERVE_MIN-NEXT: [[B_ADDR:%.*]] = alloca i32, align 4 // PRESERVE_MIN-NEXT: [[FP:%.*]] = alloca ptr, align 8 diff --git a/clang/test/CodeGen/lto-newpm-pipeline.c b/clang/test/CodeGen/lto-newpm-pipeline.c index ea9784a76f923..c8ee5e949ae87 100644 --- a/clang/test/CodeGen/lto-newpm-pipeline.c +++ b/clang/test/CodeGen/lto-newpm-pipeline.c @@ -34,6 +34,7 @@ // CHECK-FULL-O0-NEXT: Running pass: CoroConditionalWrapper // CHECK-FULL-O0-NEXT: Running pass: CanonicalizeAliasesPass // CHECK-FULL-O0-NEXT: Running pass: NameAnonGlobalPass +// CHECK-FULL-O0-NEXT: Running pass: AssignGUIDPass // CHECK-FULL-O0-NEXT: Running pass: AnnotationRemarksPass // CHECK-FULL-O0-NEXT: Running analysis: TargetLibraryAnalysis // CHECK-FULL-O0-NEXT: Running pass: VerifierPass @@ -48,6 +49,7 @@ // CHECK-THIN-O0-NEXT: Running pass: CoroConditionalWrapper // CHECK-THIN-O0-NEXT: Running pass: CanonicalizeAliasesPass // CHECK-THIN-O0-NEXT: Running pass: NameAnonGlobalPass +// CHECK-THIN-O0-NEXT: Running pass: AssignGUIDPass // CHECK-THIN-O0-NEXT: Running pass: AnnotationRemarksPass // CHECK-THIN-O0-NEXT: Running analysis: TargetLibraryAnalysis // CHECK-THIN-O0-NEXT: Running pass: VerifierPass @@ -64,6 +66,7 @@ // CHECK-THIN-OPTIMIZED-NOT: Running pass: LoopVectorizePass // CHECK-THIN-OPTIMIZED: Running pass: CanonicalizeAliasesPass // CHECK-THIN-OPTIMIZED: Running pass: NameAnonGlobalPass +// CHECK-THIN-OPTIMIZED: Running pass: AssignGUIDPass // CHECK-THIN-OPTIMIZED: Running pass: ThinLTOBitcodeWriterPass void Foo(void) {} diff --git a/clang/test/CodeGenCXX/cfi-vcall-trap-recover-runtime.cpp b/clang/test/CodeGenCXX/cfi-vcall-trap-recover-runtime.cpp index 2451d31e9a489..a1e1563a4f38d 100644 --- a/clang/test/CodeGenCXX/cfi-vcall-trap-recover-runtime.cpp +++ b/clang/test/CodeGenCXX/cfi-vcall-trap-recover-runtime.cpp @@ -19,7 +19,7 @@ struct S1 { }; // TRAP-LABEL: define hidden void @_Z3s1fP2S1( -// TRAP-SAME: ptr noundef [[S1:%.*]]) #[[ATTR0:[0-9]+]] { +// TRAP-SAME: ptr noundef [[S1:%.*]]) #[[ATTR0:[0-9]+]] // TRAP-NEXT: [[ENTRY:.*:]] // TRAP-NEXT: [[S1_ADDR:%.*]] = alloca ptr, align 8 // TRAP-NEXT: store ptr [[S1]], ptr [[S1_ADDR]], align 8 @@ -37,7 +37,7 @@ struct S1 { // TRAP-NEXT: ret void // // ABORT-LABEL: define hidden void @_Z3s1fP2S1( -// ABORT-SAME: ptr noundef [[S1:%.*]]) #[[ATTR0:[0-9]+]] { +// ABORT-SAME: ptr noundef [[S1:%.*]]) #[[ATTR0:[0-9]+]] // ABORT-NEXT: [[ENTRY:.*:]] // ABORT-NEXT: [[S1_ADDR:%.*]] = alloca ptr, align 8 // ABORT-NEXT: store ptr [[S1]], ptr [[S1_ADDR]], align 8 @@ -58,7 +58,7 @@ struct S1 { // ABORT-NEXT: ret void // // RECOVER-LABEL: define hidden void @_Z3s1fP2S1( -// RECOVER-SAME: ptr noundef [[S1:%.*]]) #[[ATTR0:[0-9]+]] { +// RECOVER-SAME: ptr noundef [[S1:%.*]]) #[[ATTR0:[0-9]+]] // RECOVER-NEXT: [[ENTRY:.*:]] // RECOVER-NEXT: [[S1_ADDR:%.*]] = alloca ptr, align 8 // RECOVER-NEXT: store ptr [[S1]], ptr [[S1_ADDR]], align 8 @@ -79,7 +79,7 @@ struct S1 { // RECOVER-NEXT: ret void // // ABORT_MIN-LABEL: define hidden void @_Z3s1fP2S1( -// ABORT_MIN-SAME: ptr noundef [[S1:%.*]]) #[[ATTR0:[0-9]+]] { +// ABORT_MIN-SAME: ptr noundef [[S1:%.*]]) #[[ATTR0:[0-9]+]] // ABORT_MIN-NEXT: [[ENTRY:.*:]] // ABORT_MIN-NEXT: [[S1_ADDR:%.*]] = alloca ptr, align 8 // ABORT_MIN-NEXT: store ptr [[S1]], ptr [[S1_ADDR]], align 8 @@ -98,7 +98,7 @@ struct S1 { // ABORT_MIN-NEXT: ret void // // RECOVER_MIN-LABEL: define hidden void @_Z3s1fP2S1( -// RECOVER_MIN-SAME: ptr noundef [[S1:%.*]]) #[[ATTR0:[0-9]+]] { +// RECOVER_MIN-SAME: ptr noundef [[S1:%.*]]) #[[ATTR0:[0-9]+]] // RECOVER_MIN-NEXT: [[ENTRY:.*:]] // RECOVER_MIN-NEXT: [[S1_ADDR:%.*]] = alloca ptr, align 8 // RECOVER_MIN-NEXT: store ptr [[S1]], ptr [[S1_ADDR]], align 8 @@ -117,7 +117,7 @@ struct S1 { // RECOVER_MIN-NEXT: ret void // // PRESERVE_MIN-LABEL: define hidden void @_Z3s1fP2S1( -// PRESERVE_MIN-SAME: ptr noundef [[S1:%.*]]) #[[ATTR0:[0-9]+]] { +// PRESERVE_MIN-SAME: ptr noundef [[S1:%.*]]) #[[ATTR0:[0-9]+]] // PRESERVE_MIN-NEXT: [[ENTRY:.*:]] // PRESERVE_MIN-NEXT: [[S1_ADDR:%.*]] = alloca ptr, align 8 // PRESERVE_MIN-NEXT: store ptr [[S1]], ptr [[S1_ADDR]], align 8 diff --git a/lld/test/ELF/lto/devirt_vcall_vis_export_dynamic.ll b/lld/test/ELF/lto/devirt_vcall_vis_export_dynamic.ll index bcb92a1beb17b..9b9c7891a6da6 100644 --- a/lld/test/ELF/lto/devirt_vcall_vis_export_dynamic.ll +++ b/lld/test/ELF/lto/devirt_vcall_vis_export_dynamic.ll @@ -5,7 +5,7 @@ ;; Index based WPD ;; Generate unsplit module with summary for ThinLTO index-based WPD. -; RUN: opt --thinlto-bc -o %t2.o %s +; RUN: opt --passes=assign-guid --thinlto-bc -o %t2.o %s ; RUN: ld.lld %t2.o -o %t3 -save-temps --lto-whole-program-visibility \ ; RUN: -mllvm -pass-remarks=. 2>&1 | FileCheck %s --check-prefix=REMARK ; RUN: llvm-dis %t2.o.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR @@ -16,13 +16,13 @@ ;; Hybrid WPD ;; Generate split module with summary for hybrid Thin/Regular LTO WPD. -; RUN: opt --thinlto-bc --thinlto-split-lto-unit -o %t.o %s +; RUN: opt --passes=assign-guid --thinlto-bc --thinlto-split-lto-unit -o %t.o %s ; RUN: ld.lld %t.o -o %t3 -save-temps --lto-whole-program-visibility \ ; RUN: -mllvm -pass-remarks=. 2>&1 | FileCheck %s --check-prefix=REMARK ; RUN: llvm-dis %t.o.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR ;; Regular LTO WPD -; RUN: opt -o %t4.o %s +; RUN: opt --passes=assign-guid -o %t4.o %s ; RUN: ld.lld %t4.o -o %t3 -save-temps --lto-whole-program-visibility \ ; RUN: -mllvm -pass-remarks=. 2>&1 | FileCheck %s --check-prefix=REMARK ; RUN: llvm-dis %t3.0.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR @@ -107,19 +107,19 @@ ;; preemption, even without any options. ;; Index based WPD -; RUN: opt -relocation-model=pic --thinlto-bc -o %t5.o %s +; RUN: opt --passes=assign-guid -relocation-model=pic --thinlto-bc -o %t5.o %s ; RUN: ld.lld %t5.o -o %t5.so -shared ; RUN: ld.lld %t5.o %t5.so -o %t5 -save-temps --lto-whole-program-visibility \ ; RUN: -mllvm -pass-remarks=. 2>&1 | FileCheck /dev/null --implicit-check-not single-impl --allow-empty ;; Hybrid WPD -; RUN: opt -relocation-model=pic --thinlto-bc --thinlto-split-lto-unit -o %t5.o %s +; RUN: opt --passes=assign-guid -relocation-model=pic --thinlto-bc --thinlto-split-lto-unit -o %t5.o %s ; RUN: ld.lld %t5.o -o %t5.so -shared ; RUN: ld.lld %t5.o %t5.so -o %t5 -save-temps --lto-whole-program-visibility \ ; RUN: -mllvm -pass-remarks=. 2>&1 | FileCheck /dev/null --implicit-check-not single-impl --allow-empty ;; Regular LTO WPD -; RUN: opt -relocation-model=pic -o %t5.o %s +; RUN: opt --passes=assign-guid -relocation-model=pic -o %t5.o %s ; RUN: ld.lld %t5.o -o %t5.so -shared ; RUN: ld.lld %t5.o %t5.so -o %t5 -save-temps --lto-whole-program-visibility \ ; RUN: -mllvm -pass-remarks=. 2>&1 | FileCheck /dev/null --implicit-check-not single-impl --allow-empty diff --git a/lld/test/ELF/lto/devirt_vcall_vis_public.ll b/lld/test/ELF/lto/devirt_vcall_vis_public.ll index a827fea465fd7..0030e5804af81 100644 --- a/lld/test/ELF/lto/devirt_vcall_vis_public.ll +++ b/lld/test/ELF/lto/devirt_vcall_vis_public.ll @@ -3,20 +3,20 @@ ;; Index based WPD ;; Generate unsplit module with summary for ThinLTO index-based WPD. -; RUN: opt --thinlto-bc -o %t2.o %s +; RUN: opt --passes=assign-guid --thinlto-bc -o %t2.o %s ; RUN: ld.lld %t2.o -o %t3 -save-temps --lto-whole-program-visibility \ ; RUN: -mllvm -pass-remarks=. 2>&1 | FileCheck %s --check-prefix=REMARK ; RUN: llvm-dis %t2.o.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR ;; Hybrid WPD ;; Generate split module with summary for hybrid Thin/Regular LTO WPD. -; RUN: opt --thinlto-bc --thinlto-split-lto-unit -o %t.o %s +; RUN: opt --passes=assign-guid --thinlto-bc --thinlto-split-lto-unit -o %t.o %s ; RUN: ld.lld %t.o -o %t3 -save-temps --lto-whole-program-visibility \ ; RUN: -mllvm -pass-remarks=. 2>&1 | FileCheck %s --check-prefix=REMARK ; RUN: llvm-dis %t.o.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR ;; Regular LTO WPD -; RUN: opt -o %t4.o %s +; RUN: opt --passes=assign-guid -o %t4.o %s ; RUN: ld.lld %t4.o -o %t3 -save-temps --lto-whole-program-visibility \ ; RUN: -mllvm -pass-remarks=. 2>&1 | FileCheck %s --check-prefix=REMARK ; RUN: llvm-dis %t3.0.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR diff --git a/lld/test/ELF/lto/devirt_vcall_vis_shared_def.ll b/lld/test/ELF/lto/devirt_vcall_vis_shared_def.ll index a61e290bb0eb1..b77dde97a2c05 100644 --- a/lld/test/ELF/lto/devirt_vcall_vis_shared_def.ll +++ b/lld/test/ELF/lto/devirt_vcall_vis_shared_def.ll @@ -6,23 +6,23 @@ ;; Index based WPD ;; Generate unsplit module with summary for ThinLTO index-based WPD. -; RUN: opt --thinlto-bc -o %t1a.o %s -; RUN: opt --thinlto-bc -o %t2a.o %S/Inputs/devirt_vcall_vis_shared_def.ll +; RUN: opt --passes=assign-guid --thinlto-bc -o %t1a.o %s +; RUN: opt --passes=assign-guid --thinlto-bc -o %t2a.o %S/Inputs/devirt_vcall_vis_shared_def.ll ; RUN: ld.lld %t1a.o %t2a.o -o %t3a -save-temps --lto-whole-program-visibility \ ; RUN: -mllvm -pass-remarks=. 2>&1 | FileCheck %s --check-prefix=REMARK ; RUN: llvm-dis %t1a.o.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR ;; Hybrid WPD ;; Generate split module with summary for hybrid Thin/Regular LTO WPD. -; RUN: opt --thinlto-bc --thinlto-split-lto-unit -o %t1b.o %s -; RUN: opt --thinlto-bc --thinlto-split-lto-unit -o %t2b.o %S/Inputs/devirt_vcall_vis_shared_def.ll +; RUN: opt --passes=assign-guid --thinlto-bc --thinlto-split-lto-unit -o %t1b.o %s +; RUN: opt --passes=assign-guid --thinlto-bc --thinlto-split-lto-unit -o %t2b.o %S/Inputs/devirt_vcall_vis_shared_def.ll ; RUN: ld.lld %t1b.o %t2b.o -o %t3b -save-temps --lto-whole-program-visibility \ ; RUN: -mllvm -pass-remarks=. 2>&1 | FileCheck %s --check-prefix=REMARK ; RUN: llvm-dis %t1b.o.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR ;; Regular LTO WPD -; RUN: opt -o %t1c.o %s -; RUN: opt -o %t2c.o %S/Inputs/devirt_vcall_vis_shared_def.ll +; RUN: opt --passes=assign-guid -o %t1c.o %s +; RUN: opt --passes=assign-guid -o %t2c.o %S/Inputs/devirt_vcall_vis_shared_def.ll ; RUN: ld.lld %t1c.o %t2c.o -o %t3c -save-temps --lto-whole-program-visibility \ ; RUN: -mllvm -pass-remarks=. 2>&1 | FileCheck %s --check-prefix=REMARK ; RUN: llvm-dis %t3c.0.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR diff --git a/llvm/include/llvm/Analysis/CtxProfAnalysis.h b/llvm/include/llvm/Analysis/CtxProfAnalysis.h index 8260b95026ad2..d2a1c07bd58e4 100644 --- a/llvm/include/llvm/Analysis/CtxProfAnalysis.h +++ b/llvm/include/llvm/Analysis/CtxProfAnalysis.h @@ -47,9 +47,6 @@ class PGOContextualProfile { // we'll need when we maintain the profiles during IPO transformations. std::map FuncInfo; - /// Get the GUID of this Function if it's defined in this module. - LLVM_ABI GlobalValue::GUID getDefinedFunctionGUID(const Function &F) const; - // This is meant to be constructed from CtxProfAnalysis, which will also set // its state piecemeal. PGOContextualProfile() = default; @@ -68,9 +65,7 @@ class PGOContextualProfile { LLVM_ABI bool isInSpecializedModule() const; - bool isFunctionKnown(const Function &F) const { - return getDefinedFunctionGUID(F) != 0; - } + bool isFunctionKnown(const Function &F) const { return !F.isDeclaration(); } StringRef getFunctionName(GlobalValue::GUID GUID) const { auto It = FuncInfo.find(GUID); @@ -81,22 +76,22 @@ class PGOContextualProfile { uint32_t getNumCounters(const Function &F) const { assert(isFunctionKnown(F)); - return FuncInfo.find(getDefinedFunctionGUID(F))->second.NextCounterIndex; + return FuncInfo.find(F.getGUID())->second.NextCounterIndex; } uint32_t getNumCallsites(const Function &F) const { assert(isFunctionKnown(F)); - return FuncInfo.find(getDefinedFunctionGUID(F))->second.NextCallsiteIndex; + return FuncInfo.find(F.getGUID())->second.NextCallsiteIndex; } uint32_t allocateNextCounterIndex(const Function &F) { assert(isFunctionKnown(F)); - return FuncInfo.find(getDefinedFunctionGUID(F))->second.NextCounterIndex++; + return FuncInfo.find(F.getGUID())->second.NextCounterIndex++; } uint32_t allocateNextCallsiteIndex(const Function &F) { assert(isFunctionKnown(F)); - return FuncInfo.find(getDefinedFunctionGUID(F))->second.NextCallsiteIndex++; + return FuncInfo.find(F.getGUID())->second.NextCallsiteIndex++; } using ConstVisitor = function_ref; @@ -187,26 +182,5 @@ class ProfileAnnotator { LLVM_ABI ~ProfileAnnotator(); }; -/// Assign a GUID to functions as metadata. GUID calculation takes linkage into -/// account, which may change especially through and after thinlto. By -/// pre-computing and assigning as metadata, this mechanism is resilient to such -/// changes (as well as name changes e.g. suffix ".llvm." additions). - -// FIXME(mtrofin): we can generalize this mechanism to calculate a GUID early in -// the pass pipeline, associate it with any Global Value, and then use it for -// PGO and ThinLTO. -// At that point, this should be moved elsewhere. -class AssignGUIDPass : public OptionalPassInfoMixin { -public: - explicit AssignGUIDPass() = default; - - /// Assign a GUID *if* one is not already assign, as a function metadata named - /// `GUIDMetadataName`. - LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); - LLVM_ABI static const char *GUIDMetadataName; - // This should become GlobalValue::getGUID - LLVM_ABI static uint64_t getGUID(const Function &F); -}; - } // namespace llvm #endif // LLVM_ANALYSIS_CTXPROFANALYSIS_H diff --git a/llvm/include/llvm/Bitcode/BitcodeReader.h b/llvm/include/llvm/Bitcode/BitcodeReader.h index 772ca82019278..b7cab51857f0b 100644 --- a/llvm/include/llvm/Bitcode/BitcodeReader.h +++ b/llvm/include/llvm/Bitcode/BitcodeReader.h @@ -17,6 +17,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/Bitstream/BitCodeEnums.h" #include "llvm/IR/GlobalValue.h" +#include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" @@ -165,7 +166,8 @@ struct ParserCallbacks { /// into CombinedIndex. LLVM_ABI Error readSummary(ModuleSummaryIndex &CombinedIndex, StringRef ModulePath, - std::function IsPrevailing = nullptr); + std::function IsPrevailing = nullptr, + std::function OnValueInfo = nullptr); }; struct BitcodeFileContents { diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h index 95787c595dff7..3f9727782b131 100644 --- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -120,6 +120,9 @@ enum ModuleCodes { // IFUNC: [ifunc value type, addrspace, resolver val#, linkage, visibility] MODULE_CODE_IFUNC = 18, + + // GUIDLIST: [n x i64] + MODULE_CODE_GUIDLIST = 19, }; /// PARAMATTR blocks have code for defining a parameter attribute set. diff --git a/llvm/include/llvm/IR/FixedMetadataKinds.def b/llvm/include/llvm/IR/FixedMetadataKinds.def index 47c221fd60fc5..5c8eb2079f857 100644 --- a/llvm/include/llvm/IR/FixedMetadataKinds.def +++ b/llvm/include/llvm/IR/FixedMetadataKinds.def @@ -62,3 +62,4 @@ LLVM_FIXED_MD_KIND(MD_nofpclass, "nofpclass", 47) LLVM_FIXED_MD_KIND(MD_call_target, "call_target", 48) LLVM_FIXED_MD_KIND(MD_inline_history, "inline_history", 49) LLVM_FIXED_MD_KIND(MD_elf_section_properties, "elf_section_properties", 50) +LLVM_FIXED_MD_KIND(MD_unique_id, "guid", 51) diff --git a/llvm/include/llvm/IR/GlobalValue.h b/llvm/include/llvm/IR/GlobalValue.h index 83e695cdd27d9..ef037a25fac2e 100644 --- a/llvm/include/llvm/IR/GlobalValue.h +++ b/llvm/include/llvm/IR/GlobalValue.h @@ -589,17 +589,54 @@ class GlobalValue : public Constant { /// used as the key for a global lookup (e.g. profile or ThinLTO). LLVM_ABI std::string getGlobalIdentifier() const; + /// Assign a GUID to this value based on its current name and linkage. + /// This GUID will remain the same even if those change. This method is + /// idempotent -- if a GUID has already been assigned, calling it again + /// will do nothing. + /// + /// This is private (exposed only to \c AssignGUIDPass), as users don't need + /// to call it. GUIDs are assigned only by \c AssignGUIDPass. The pass + /// pipeline should be set up such that GUIDs are always available when + /// needed. If not, the GUID assignment pass should be moved (or run again) + /// such that they are. + void assignGUID(); + + // assignGUID needs to be accessible from AssignGUIDPass, which is called + // early in the pipeline to make GUIDs available to later passes. But we'd + // rather not expose it publicly, as no-one else should call it. + friend class AssignGUIDPass; + + MDNode *getGUIDMetadata() const; + public: /// Return a 64-bit global unique ID constructed from the name of a global /// symbol. Since this call doesn't supply the linkage or defining filename, /// the GUID computation will assume that the global has external linkage. LLVM_ABI static GUID getGUIDAssumingExternalLinkage(StringRef GlobalName); - /// Return a 64-bit global unique ID constructed from global value name - /// (i.e. returned by getGlobalIdentifier()). - GUID getGUID() const { - return getGUIDAssumingExternalLinkage(getGlobalIdentifier()); - } + /// Return a 64-bit global unique ID for this value. It is based on the + /// "original" name and linkage of this value (i.e. whenever its GUID was + /// assigned). This might not match the current name and linkage. + /// + /// The \c AssignGUIDPass must be run before this is called, otherwise + /// GUIDs won't be available. This pass can be run multiple times as it does + /// nothing if GUID metadata is already present. + GUID getGUID() const; + + /// Return the GUID for this value if it has been assigned, nullopt + /// otherwise. This should only need to be used in some exceptional + /// situations. If possible whatever code needs access to a GUID should + /// be set up to run after AssignGUIDPass, in which case it will always + /// be available. + std::optional getGUIDIfAssigned() const; + + /// Return the GUID for this value if it has been assigned, otherwise fall + /// back to computing it based on its current name and linkage. + /// + /// This is to be used in situations where we need a GUID but can't guarantee + /// that it's been computed. Notably, if we're reading from bitcode files + /// that might pre-date the storage of GUIDs in metadata. + GUID getGUIDOrFallback() const; /// @name Materialization /// Materialization is used to construct functions only as they're needed. diff --git a/llvm/include/llvm/IR/Module.h b/llvm/include/llvm/IR/Module.h index 3a5166d92d89d..e71dd36ba5155 100644 --- a/llvm/include/llvm/IR/Module.h +++ b/llvm/include/llvm/IR/Module.h @@ -32,6 +32,7 @@ #include "llvm/Support/CBindingWrapping.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/Compiler.h" +#include "llvm/Support/Debug.h" #include "llvm/TargetParser/Triple.h" #include #include @@ -578,10 +579,34 @@ class LLVM_ABI Module { // Use global_size() to get the total number of global variables. // Use globals() to get the range of all global variables. + std::optional getGUID(const Value *V) const { + const auto It = ValueToGUIDMap.find(V); + if (It == ValueToGUIDMap.end()) + return std::nullopt; + + return It->getSecond(); + } + + void insertGUID(const Value *V, GlobalValue::GUID GUID) { + const auto [It, WasInserted] = ValueToGUIDMap.insert({V, GUID}); + + (void)It, (void)WasInserted; +#ifndef NDEBUG + if (!WasInserted) { + assert((It->second == GUID) && "insertGUID called with different value"); + } +#endif + } + private: -/// @} -/// @name Direct access to the globals list, functions list, and symbol table -/// @{ + /// A mapping directly from Value to GUID. Populated from bitcode + /// (MODULE_CODE_GUIDLIST). Necessary for lazy-loading modules, where we + /// don't load metadata. + DenseMap ValueToGUIDMap; + + /// @} + /// @name Direct access to the globals list, functions list, and symbol table + /// @{ /// Get the Module's list of global variables (constant). const GlobalListType &getGlobalList() const { return GlobalList; } diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h index 6859b9a2f46e6..ed39eb99a9196 100644 --- a/llvm/include/llvm/IR/ModuleSummaryIndex.h +++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h @@ -29,6 +29,7 @@ #include "llvm/IR/Module.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Compiler.h" +#include "llvm/Support/Debug.h" #include "llvm/Support/InterleavedRange.h" #include "llvm/Support/ScaledNumber.h" #include "llvm/Support/StringSaver.h" @@ -1769,14 +1770,21 @@ class ModuleSummaryIndex { return ValueInfo(HaveGVs, VP); } - /// Return a ValueInfo for \p GV and mark it as belonging to GV. - ValueInfo getOrInsertValueInfo(const GlobalValue *GV) { + /// Return a ValueInfo for \p GV with GUID \p GUID and mark it as belonging to + /// GV. + ValueInfo getOrInsertValueInfo(const GlobalValue *GV, + GlobalValue::GUID GUID) { assert(HaveGVs); - auto VP = getOrInsertValuePtr(GV->getGUID()); + auto VP = getOrInsertValuePtr(GUID); VP->second.U.GV = GV; return ValueInfo(HaveGVs, VP); } + /// Return a ValueInfo for \p GV and mark it as belonging to GV. + ValueInfo getOrInsertValueInfo(const GlobalValue *GV) { + return getOrInsertValueInfo(GV, GV->getGUID()); + } + /// Return the GUID for \p OriginalId in the OidGuidMap. GlobalValue::GUID getGUIDFromOriginalID(GlobalValue::GUID OriginalID) const { const auto I = OidGuidMap.find(OriginalID); diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index aba2661e81c47..b7f904aebaa66 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -554,6 +554,8 @@ class LTO { // been added and the client has called run(). During run() we apply // internalization decisions either directly to the module (for regular LTO) // or to the combined index (for ThinLTO). + // FIXME: Make this GlobalResolution a class, it has been becoming more than + // just a data bag. struct GlobalResolution { /// The unmangled name of the global. std::string IRName; @@ -590,6 +592,24 @@ class LTO { /// LTO backend. unsigned Partition = Unknown; + private: + GlobalValue::GUID GUID = 0; + + public: + void setGUID(GlobalValue::GUID G) { + assert(G); + assert(!GUID || GUID == G); + GUID = G; + } + + GlobalValue::GUID getGUID() const { + return GUID ? GUID + : GlobalValue::getGUIDAssumingExternalLinkage( + GlobalValue::getGlobalIdentifier( + IRName, GlobalValue::LinkageTypes::ExternalLinkage, + "")); + } + /// Special partition numbers. enum : unsigned { /// A partition number has not yet been assigned to this global. diff --git a/llvm/include/llvm/Transforms/Utils/AssignGUID.h b/llvm/include/llvm/Transforms/Utils/AssignGUID.h new file mode 100644 index 0000000000000..0abd450d5a522 --- /dev/null +++ b/llvm/include/llvm/Transforms/Utils/AssignGUID.h @@ -0,0 +1,42 @@ +//===-- AssignGUID.h - Unique identifier assignment pass --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides a pass which assigns a a GUID (globally unique identifier) +// to every GlobalValue in the module, according to its current name, linkage, +// and originating file. This way we have a consistent identifier even when +// these inputs to the GUID change (for instance, after externalising a global +// in ThinLTO). +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_UTILS_ASSIGNGUID_H +#define LLVM_TRANSFORMS_UTILS_ASSIGNGUID_H + +#include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Support/Debug.h" + +namespace llvm { + +class AssignGUIDPass : public PassInfoMixin { +public: + AssignGUIDPass() = default; + + static void runOnModule(Module &M); + + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM) { + AssignGUIDPass::runOnModule(M); + return PreservedAnalyses::all(); + } + + static bool isRequired() { return true; } +}; + +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_UTILS_ASSIGNGUID_H \ No newline at end of file diff --git a/llvm/lib/Analysis/CtxProfAnalysis.cpp b/llvm/lib/Analysis/CtxProfAnalysis.cpp index c4abec02e765a..9df6ab048d382 100644 --- a/llvm/lib/Analysis/CtxProfAnalysis.cpp +++ b/llvm/lib/Analysis/CtxProfAnalysis.cpp @@ -51,8 +51,6 @@ static cl::opt ForceIsInSpecializedModule( cl::desc("Treat the given module as-if it were containing the " "post-thinlink module containing the root")); -const char *AssignGUIDPass::GUIDMetadataName = "guid"; - class ProfileAnnotatorImpl final { friend class ProfileAnnotator; class BBInfo; @@ -420,33 +418,6 @@ bool ProfileAnnotator::getOutgoingBranchWeights( return MaxCount > 0; } -PreservedAnalyses AssignGUIDPass::run(Module &M, ModuleAnalysisManager &MAM) { - for (auto &F : M.functions()) { - if (F.isDeclaration()) - continue; - if (F.getMetadata(GUIDMetadataName)) - continue; - const GlobalValue::GUID GUID = F.getGUID(); - F.setMetadata(GUIDMetadataName, - MDNode::get(M.getContext(), - {ConstantAsMetadata::get(ConstantInt::get( - Type::getInt64Ty(M.getContext()), GUID))})); - } - return PreservedAnalyses::none(); -} - -GlobalValue::GUID AssignGUIDPass::getGUID(const Function &F) { - if (F.isDeclaration()) { - assert(GlobalValue::isExternalLinkage(F.getLinkage())); - return F.getGUID(); - } - auto *MD = F.getMetadata(GUIDMetadataName); - assert(MD && "guid not found for defined function"); - return cast(cast(MD->getOperand(0)) - ->getValue() - ->stripPointerCasts()) - ->getZExtValue(); -} AnalysisKey CtxProfAnalysis::Key; CtxProfAnalysis::CtxProfAnalysis(std::optional Profile) @@ -515,7 +486,7 @@ PGOContextualProfile CtxProfAnalysis::run(Module &M, for (const auto &F : M) { if (F.isDeclaration()) continue; - auto GUID = AssignGUIDPass::getGUID(F); + auto GUID = F.getGUID(); assert(GUID && "guid not found for defined function"); const auto &Entry = F.begin(); uint32_t MaxCounters = 0; // we expect at least a counter. @@ -549,13 +520,6 @@ PGOContextualProfile CtxProfAnalysis::run(Module &M, return Result; } -GlobalValue::GUID -PGOContextualProfile::getDefinedFunctionGUID(const Function &F) const { - if (auto It = FuncInfo.find(AssignGUIDPass::getGUID(F)); It != FuncInfo.end()) - return It->first; - return 0; -} - CtxProfAnalysisPrinterPass::CtxProfAnalysisPrinterPass(raw_ostream &OS) : OS(OS), Mode(PrintLevel) {} @@ -671,7 +635,7 @@ bool PGOContextualProfile::isInSpecializedModule() const { void PGOContextualProfile::update(Visitor V, const Function &F) { assert(isFunctionKnown(F)); - GlobalValue::GUID G = getDefinedFunctionGUID(F); + GlobalValue::GUID G = F.getGUID(); for (auto *Node = FuncInfo.find(G)->second.Index.Next; Node; Node = Node->Next) V(*reinterpret_cast(Node)); @@ -682,7 +646,7 @@ void PGOContextualProfile::visit(ConstVisitor V, const Function *F) const { return preorderVisit(Profiles.Contexts, V); assert(isFunctionKnown(*F)); - GlobalValue::GUID G = getDefinedFunctionGUID(*F); + GlobalValue::GUID G = F->getGUID(); for (const auto *Node = FuncInfo.find(G)->second.Index.Next; Node; Node = Node->Next) V(*reinterpret_cast(Node)); diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp index 990febaacbe48..46fb87cc41aa1 100644 --- a/llvm/lib/AsmParser/LLParser.cpp +++ b/llvm/lib/AsmParser/LLParser.cpp @@ -9931,7 +9931,11 @@ bool LLParser::addGlobalValueToIndex( if (!GV) return error(Loc, "Reference to undefined global \"" + Name + "\""); - VI = Index->getOrInsertValueInfo(GV); + // Be a little lenient here, to accomodate older files without GUIDs + // already computed and assigned as metadata. + GUID = GV->getGUIDOrFallback(); + + VI = Index->getOrInsertValueInfo(GV, GUID); } else { assert( (!GlobalValue::isLocalLinkage(Linkage) || !SourceFileName.empty()) && diff --git a/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp b/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp index 911ec7501eb8b..6574ab7a93c58 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp @@ -142,6 +142,7 @@ GetCodeName(unsigned CodeID, unsigned BlockID, STRINGIFY_CODE(MODULE_CODE, METADATA_VALUES_UNUSED) STRINGIFY_CODE(MODULE_CODE, SOURCE_FILENAME) STRINGIFY_CODE(MODULE_CODE, HASH) + STRINGIFY_CODE(MODULE_CODE, GUIDLIST) } case bitc::IDENTIFICATION_BLOCK_ID: switch (CodeID) { diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index 3e863f4786e1a..f4324aa37f2e8 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -697,6 +697,9 @@ class BitcodeReader : public BitcodeReaderBase, public GVMaterializer { std::optional ValueTypeCallback; + /// A list of GUIDs defined by this module. Indexed by ValueID. + std::vector GUIDList; + public: BitcodeReader(BitstreamCursor Stream, StringRef Strtab, StringRef ProducerIdentification, LLVMContext &Context); @@ -975,9 +978,12 @@ class ModuleSummaryIndexBitcodeReader : public BitcodeReaderBase { /// Callback to ask whether a symbol is the prevailing copy when invoked /// during combined index building. - std::function IsPrevailing; + std::function IsPrevailing = nullptr; + + /// Callback invoked whenever a new ValueInfo is generated. + std::function OnValueInfo = nullptr; - /// Saves the stack ids from the STACK_IDS record to consult when adding stack + /// Saves the stack ids from the STACK_IDS record to consult when adding /// ids from the lists in the callsite and alloc entries to the index. std::vector StackIds; @@ -990,11 +996,15 @@ class ModuleSummaryIndexBitcodeReader : public BitcodeReaderBase { /// list and used to avoid repeated hash lookups. std::vector StackIdToIndex; + /// A list of GUIDs defined by this module. Indexed by ValueID. + std::vector DefinedGUIDs; + public: ModuleSummaryIndexBitcodeReader( BitstreamCursor Stream, StringRef Strtab, ModuleSummaryIndex &TheIndex, StringRef ModulePath, - std::function IsPrevailing = nullptr); + std::function IsPrevailing = nullptr, + std::function OnValueInfo = nullptr); Error parseModule(); @@ -4036,6 +4046,15 @@ Error BitcodeReader::globalCleanup() { TheModule->insertGlobalVariable(Pair.second); } + for (size_t ValueID = 0; ValueID < GUIDList.size(); ValueID++) { + const auto GUID = GUIDList[ValueID]; + if (GUID == 0) + continue; + + const auto *Value = ValueList[ValueID]; + TheModule->insertGUID(Value, GUID); + } + // Force deallocation of memory for these vectors to favor the client that // want lazy deserialization. std::vector>().swap(GlobalInits); @@ -4839,6 +4858,13 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit, // historically always the start of the regular bitcode header. VSTOffset = Record[0] - 1; break; + // MODULE_CODE_GUIDLIST: [i64 x N] + case bitc::MODULE_CODE_GUIDLIST: + assert(Record.size() % 2 == 0); + GUIDList.reserve(GUIDList.size() + Record.size() / 2); + for (size_t i = 0; i < Record.size(); i += 2) + GUIDList.push_back(Record[i] << 32 | Record[i + 1]); + break; /// MODULE_CODE_SOURCE_FILENAME: [namechar x N] case bitc::MODULE_CODE_SOURCE_FILENAME: SmallString<128> ValueName; @@ -4849,6 +4875,7 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit, } Record.clear(); } + this->ValueTypeCallback = std::nullopt; return Error::success(); } @@ -7222,9 +7249,11 @@ std::vector BitcodeReader::getIdentifiedStructTypes() const { ModuleSummaryIndexBitcodeReader::ModuleSummaryIndexBitcodeReader( BitstreamCursor Cursor, StringRef Strtab, ModuleSummaryIndex &TheIndex, - StringRef ModulePath, std::function IsPrevailing) + StringRef ModulePath, std::function IsPrevailing, + std::function OnValueInfo) : BitcodeReaderBase(std::move(Cursor), Strtab), TheIndex(TheIndex), - ModulePath(ModulePath), IsPrevailing(IsPrevailing) {} + ModulePath(ModulePath), IsPrevailing(IsPrevailing), + OnValueInfo(OnValueInfo) {} void ModuleSummaryIndexBitcodeReader::addThisModule() { TheIndex.addModule(ModulePath); @@ -7252,9 +7281,15 @@ ModuleSummaryIndexBitcodeReader::getValueInfoFromValueId(unsigned ValueId) { void ModuleSummaryIndexBitcodeReader::setValueGUID( uint64_t ValueID, StringRef ValueName, GlobalValue::LinkageTypes Linkage, StringRef SourceFileName) { - std::string GlobalId = - GlobalValue::getGlobalIdentifier(ValueName, Linkage, SourceFileName); - auto ValueGUID = GlobalValue::getGUIDAssumingExternalLinkage(GlobalId); + GlobalValue::GUID ValueGUID = 0; + if (ValueID < DefinedGUIDs.size()) + ValueGUID = DefinedGUIDs[ValueID]; + if (ValueGUID == 0) + // DefinedGUIDs is a sparse array and can contain zero entries, so this + // can't just be an `else`. + ValueGUID = GlobalValue::getGUIDAssumingExternalLinkage( + GlobalValue::getGlobalIdentifier(ValueName, Linkage, SourceFileName)); + auto OriginalNameID = ValueGUID; if (GlobalValue::isLocalLinkage(Linkage)) OriginalNameID = GlobalValue::getGUIDAssumingExternalLinkage(ValueName); @@ -7265,10 +7300,11 @@ void ModuleSummaryIndexBitcodeReader::setValueGUID( // UseStrtab is false for legacy summary formats and value names are // created on stack. In that case we save the name in a string saver in // the index so that the value name can be recorded. - ValueIdToValueInfoMap[ValueID] = std::make_pair( - TheIndex.getOrInsertValueInfo( - ValueGUID, UseStrtab ? ValueName : TheIndex.saveString(ValueName)), - OriginalNameID); + auto VI = TheIndex.getOrInsertValueInfo( + ValueGUID, UseStrtab ? ValueName : TheIndex.saveString(ValueName)); + ValueIdToValueInfoMap[ValueID] = std::make_pair(VI, OriginalNameID); + if (OnValueInfo) + OnValueInfo(VI); } // Specialized value symbol table parser used when reading module index @@ -7477,6 +7513,13 @@ Error ModuleSummaryIndexBitcodeReader::parseModule() { // was historically always the start of the regular bitcode header. VSTOffset = Record[0] - 1; break; + // MODULE_CODE_GUIDLIST: [i64 x N] + case bitc::MODULE_CODE_GUIDLIST: + assert(Record.size() % 2 == 0); + DefinedGUIDs.reserve(DefinedGUIDs.size() + Record.size() / 2); + for (size_t i = 0; i < Record.size(); i += 2) + DefinedGUIDs.push_back(Record[i] << 32 | Record[i + 1]); + break; // v1 GLOBALVAR: [pointer type, isconst, initid, linkage, ...] // v1 FUNCTION: [type, callingconv, isproto, linkage, ...] // v1 ALIAS: [alias type, addrspace, aliasee val#, linkage, ...] @@ -7873,7 +7916,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { // and kept). auto LT = (GlobalValue::LinkageTypes)Flags.Linkage; bool IsPrevailingSym = !IsPrevailing || GlobalValue::isLocalLinkage(LT) || - IsPrevailing(VI.getGUID()); + IsPrevailing(VI.name()); // If this is not the prevailing copy, and the records are in the "old" // order (preceding), clear them now. They should already be empty in @@ -8687,15 +8730,16 @@ BitcodeModule::getLazyModule(LLVMContext &Context, bool ShouldLazyLoadMetadata, // We don't use ModuleIdentifier here because the client may need to control the // module path used in the combined summary (e.g. when reading summaries for // regular LTO modules). -Error BitcodeModule::readSummary( - ModuleSummaryIndex &CombinedIndex, StringRef ModulePath, - std::function IsPrevailing) { +Error BitcodeModule::readSummary(ModuleSummaryIndex &CombinedIndex, + StringRef ModulePath, + std::function IsPrevailing, + std::function OnValueInfo) { BitstreamCursor Stream(Buffer); if (Error JumpFailed = Stream.JumpToBit(ModuleBit)) return JumpFailed; ModuleSummaryIndexBitcodeReader R(std::move(Stream), Strtab, CombinedIndex, - ModulePath, IsPrevailing); + ModulePath, IsPrevailing, OnValueInfo); return R.parseModule(); } diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index ed7f95701ea65..3f878be4cd59a 100644 --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -254,6 +254,7 @@ class ModuleBitcodeWriterBase : public BitcodeWriterBase { protected: void writePerModuleGlobalValueSummary(); + void writeGUIDList(); private: void writePerModuleFunctionSummaryRecord( @@ -1647,6 +1648,8 @@ void ModuleBitcodeWriter::writeModuleInfo() { Vals.clear(); } + writeGUIDList(); + // Emit the global variable information. for (const GlobalVariable &GV : M.globals()) { unsigned AbbrevToUse = 0; @@ -4673,7 +4676,11 @@ void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord( void ModuleBitcodeWriterBase::writeModuleLevelReferences( const GlobalVariable &V, SmallVector &NameVals, unsigned FSModRefsAbbrev, unsigned FSModVTableRefsAbbrev) { - auto VI = Index->getValueInfo(V.getGUID()); + // Be a little lenient here, to accomodate older files without GUIDs + // already computed and assigned as metadata. + GlobalValue::GUID GUID = V.getGUIDOrFallback(); + + auto VI = Index->getValueInfo(GUID); if (!VI || VI.getSummaryList().empty()) { // Only declarations should not have a summary (a declaration might however // have a summary if the def was in module level asm). @@ -4886,7 +4893,11 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() { if (!F.hasName()) report_fatal_error("Unexpected anonymous function when writing summary"); - ValueInfo VI = Index->getValueInfo(F.getGUID()); + // Be a little lenient here, to accomodate older files without GUIDs + // already computed and assigned as metadata. + GlobalValue::GUID GUID = F.getGUIDOrFallback(); + + ValueInfo VI = Index->getValueInfo(GUID); if (!VI || VI.getSummaryList().empty()) { // Only declarations should not have a summary (a declaration might // however have a summary if the def was in module level asm). @@ -4918,7 +4929,9 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() { if (!F.hasName()) report_fatal_error("Unexpected anonymous function when writing summary"); - ValueInfo VI = Index->getValueInfo(F.getGUID()); + GlobalValue::GUID GUID = F.getGUIDOrFallback(); + + ValueInfo VI = Index->getValueInfo(GUID); if (!VI || VI.getSummaryList().empty()) { // Only declarations should not have a summary (a declaration might // however have a summary if the def was in module level asm). @@ -4970,6 +4983,37 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() { Stream.ExitBlock(); } +void ModuleBitcodeWriterBase::writeGUIDList() { + const ValueEnumerator::ValueList &Vals = VE.getValues(); + const size_t Max = Vals.size(); + + std::vector GUIDs(Max, 0); + for (const GlobalValue &GV : M.global_values()) { + auto MaybeGUID = GV.getGUIDIfAssigned(); + if (!MaybeGUID) + continue; + auto GUID = *MaybeGUID; + + const auto ValueID = VE.getValueID(&GV); + GUIDs[ValueID] = GUID; + } + + auto Abbv = std::make_shared(); + Abbv->Add(BitCodeAbbrevOp(bitc::MODULE_CODE_GUIDLIST)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); + unsigned GUIDListAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + + SmallVector RecordVals; + RecordVals.reserve(Max * 2); + for (auto GUID : GUIDs) { + RecordVals.push_back(static_cast(GUID >> 32)); + RecordVals.push_back(static_cast(GUID)); + } + + Stream.EmitRecord(bitc::MODULE_CODE_GUIDLIST, RecordVals, GUIDListAbbrev); +} + /// Emit the combined summary section into the combined index file. void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 4); @@ -5762,6 +5806,8 @@ void ThinLinkBitcodeWriter::writeSimplifiedModuleInfo() { Vals.clear(); } + writeGUIDList(); + // Emit the global variable information. for (const GlobalVariable &GV : M.globals()) { // GLOBALVAR: [strtab offset, strtab size, 0, 0, 0, linkage] diff --git a/llvm/lib/IR/Globals.cpp b/llvm/lib/IR/Globals.cpp index 1bd7e9605ffc7..27e934369b285 100644 --- a/llvm/lib/IR/Globals.cpp +++ b/llvm/lib/IR/Globals.cpp @@ -81,6 +81,61 @@ GlobalValue::getGUIDAssumingExternalLinkage(StringRef GlobalIdentifier) { return MD5Hash(GlobalIdentifier); } +void GlobalValue::assignGUID() { + if (getGUIDMetadata() != nullptr) + return; + + const GUID G = + GlobalValue::getGUIDAssumingExternalLinkage(getGlobalIdentifier()); + setMetadata( + LLVMContext::MD_unique_id, + MDNode::get(getContext(), {ConstantAsMetadata::get(ConstantInt::get( + Type::getInt64Ty(getContext()), G))})); +} + +GlobalValue::GUID GlobalValue::getGUID() const { + auto MaybeGUID = getGUIDIfAssigned(); + assert(MaybeGUID.has_value() && + "GUID was not assigned before calling GetGUID()"); + return *MaybeGUID; +} + +GlobalValue::GUID GlobalValue::getGUIDOrFallback() const { + if (auto MaybeGUID = getGUIDIfAssigned(); MaybeGUID) + return *MaybeGUID; + return getGUIDAssumingExternalLinkage(getGlobalIdentifier()); +} + +std::optional GlobalValue::getGUIDIfAssigned() const { + // First check the metadata. + auto *MD = getGUIDMetadata(); + if (MD != nullptr) + return cast(cast(MD->getOperand(0)) + ->getValue() + ->stripPointerCasts()) + ->getZExtValue(); + + // Handle a few special cases where we just want to compute it based on the + // current properties. + // TODO: Maybe we should use a more robust check for intrinsics than just + // matching on the name? + if (isDeclaration() || isa(this) || + getName().starts_with("llvm.")) { + return GlobalValue::getGUIDAssumingExternalLinkage(getGlobalIdentifier()); + } + + // Otherwise we try to look it up in the module, for cases where we've read + // the GUID table but not the metadata. This happens when lazy-loading a + // module. + return getParent()->getGUID(this); +} + +MDNode *GlobalValue::getGUIDMetadata() const { + if (auto *GO = dyn_cast(this)) + return GO->getMetadata(LLVMContext::MD_unique_id); + return nullptr; +} + void GlobalValue::removeFromParent() { switch (getValueID()) { #define HANDLE_GLOBAL_VALUE(NAME) \ diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index 95faf3484c456..a811cddfb2348 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -27,6 +27,7 @@ #include "llvm/Config/llvm-config.h" #include "llvm/IR/AutoUpgrade.h" #include "llvm/IR/DiagnosticPrinter.h" +#include "llvm/IR/GlobalValue.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/LLVMRemarkStreamer.h" #include "llvm/IR/LegacyPassManager.h" @@ -758,6 +759,7 @@ void LTO::addModuleToGlobalRes(ArrayRef Syms, // FIXME: instead of this check, it would be desirable to compute GUIDs // based on mangled name, but this requires an access to the Target Triple // and would be relatively invasive on the codebase. + // FIXME: use the GUID member of GlobalRes. if (GlobalRes.IRName != Sym.getIRName()) { GlobalRes.Partition = GlobalResolution::External; GlobalRes.VisibleOutsideSummary = true; @@ -1136,17 +1138,20 @@ Error LTO::linkRegularLTO(RegularLTOState::AddedModule Mod, llvm::TimeTraceScope timeScope("LTO link regular LTO"); std::vector Keep; for (GlobalValue *GV : Mod.Keep) { - if (LivenessFromIndex && !ThinLTO.CombinedIndex.isGUIDLive(GV->getGUID())) { - if (Function *F = dyn_cast(GV)) { - if (DiagnosticOutputFile) { - if (Error Err = F->materialize()) - return Err; - auto R = OptimizationRemark(DEBUG_TYPE, "deadfunction", F); - R << ore::NV("Function", F) << " not added to the combined module "; - emitRemark(R); + if (LivenessFromIndex) { + const auto GUID = GV->getGUIDOrFallback(); + if (!ThinLTO.CombinedIndex.isGUIDLive(GUID)) { + if (Function *F = dyn_cast(GV)) { + if (DiagnosticOutputFile) { + if (Error Err = F->materialize()) + return Err; + auto R = OptimizationRemark(DEBUG_TYPE, "deadfunction", F); + R << ore::NV("Function", F) << " not added to the combined module "; + emitRemark(R); + } } + continue; } - continue; } if (!GV->hasAvailableExternallyLinkage()) { @@ -1175,21 +1180,28 @@ LTO::addThinLTO(BitcodeModule BM, ArrayRef Syms, llvm::TimeTraceScope timeScope("LTO add thin LTO"); const auto BMID = BM.getModuleIdentifier(); ArrayRef ResTmp = Res; + DenseSet Prevailing; for (const InputFile::Symbol &Sym : Syms) { assert(!ResTmp.empty()); const SymbolResolution &R = ResTmp.consume_front(); - - if (!Sym.getIRName().empty() && R.Prevailing) { - auto GUID = GlobalValue::getGUIDAssumingExternalLinkage( - GlobalValue::getGlobalIdentifier(Sym.getIRName(), - GlobalValue::ExternalLinkage, "")); - ThinLTO.setPrevailingModuleForGUID(GUID, BMID); - } + if (!Sym.getIRName().empty() && R.Prevailing) + Prevailing.insert(Sym.getIRName()); } + // Track the GUIDs stored in the bitcode GUID table. + StringMap IRSpecifiedGUIDs; if (Error Err = BM.readSummary( - ThinLTO.CombinedIndex, BMID, [&](GlobalValue::GUID GUID) { - return ThinLTO.isPrevailingModuleForGUID(GUID, BMID); + ThinLTO.CombinedIndex, BMID, + [&](StringRef Name) { return (Prevailing.count(Name) > 0); }, + [&](ValueInfo VI) { + auto IT = IRSpecifiedGUIDs.insert({VI.name(), VI.getGUID()}); + (void)IT; + assert(IT.second); + if (auto GRIt = GlobalResolutions->find(VI.name()); + GRIt != GlobalResolutions->end() && + Prevailing.count(VI.name())) { + GRIt->second.setGUID(VI.getGUID()); + } })) return Err; LLVM_DEBUG(dbgs() << "Module " << BMID << "\n"); @@ -1197,28 +1209,33 @@ LTO::addThinLTO(BitcodeModule BM, ArrayRef Syms, for (const InputFile::Symbol &Sym : Syms) { assert(!Res.empty()); const SymbolResolution &R = Res.consume_front(); - + auto GUIDIter = IRSpecifiedGUIDs.find(Sym.getIRName()); + // The bitcode GUID table might not be present if this is an old bitcode + // file. For backwards-compatibility, just compute the GUID now in that + // case. + auto GUID = + GUIDIter == IRSpecifiedGUIDs.end() + ? GlobalValue::getGUIDAssumingExternalLinkage( + GlobalValue::getGlobalIdentifier( + Sym.getIRName(), GlobalValue::ExternalLinkage, "")) + : GUIDIter->second; if (!Sym.getIRName().empty() && (R.Prevailing || R.FinalDefinitionInLinkageUnit)) { - auto GUID = GlobalValue::getGUIDAssumingExternalLinkage( - GlobalValue::getGlobalIdentifier(Sym.getIRName(), - GlobalValue::ExternalLinkage, "")); if (R.Prevailing) { - assert(ThinLTO.isPrevailingModuleForGUID(GUID, BMID)); - + ThinLTO.setPrevailingModuleForGUID(GUID, BMID); // For linker redefined symbols (via --wrap or --defsym) we want to // switch the linkage to `weak` to prevent IPOs from happening. // Find the summary in the module for this very GV and record the new // linkage so that we can switch it when we import the GV. if (R.LinkerRedefined) - if (auto S = ThinLTO.CombinedIndex.findSummaryInModule(GUID, BMID)) + if (auto *S = ThinLTO.CombinedIndex.findSummaryInModule(GUID, BMID)) S->setLinkage(GlobalValue::WeakAnyLinkage); } // If the linker resolved the symbol to a local definition then mark it // as local in the summary for the module we are adding. if (R.FinalDefinitionInLinkageUnit) { - if (auto S = ThinLTO.CombinedIndex.findSummaryInModule(GUID, BMID)) { + if (auto *S = ThinLTO.CombinedIndex.findSummaryInModule(GUID, BMID)) { S->setDSOLocal(true); } } @@ -1313,8 +1330,7 @@ Error LTO::run(AddStreamFn AddStream, FileCache Cache) { if (Res.second.IRName.empty()) continue; - GlobalValue::GUID GUID = GlobalValue::getGUIDAssumingExternalLinkage( - GlobalValue::dropLLVMManglingEscape(Res.second.IRName)); + GlobalValue::GUID GUID = Res.second.getGUID(); if (Res.second.VisibleOutsideSummary && Res.second.Prevailing) GUIDPreservedSymbols.insert(GUID); @@ -2136,8 +2152,7 @@ Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache, if (Res.second.Partition != GlobalResolution::External || !Res.second.isPrevailingIRSymbol()) continue; - auto GUID = GlobalValue::getGUIDAssumingExternalLinkage( - GlobalValue::dropLLVMManglingEscape(Res.second.IRName)); + auto GUID = Res.second.getGUID(); // Mark exported unless index-based analysis determined it to be dead. if (ThinLTO.CombinedIndex.isGUIDLive(GUID)) ExportedGUIDs.insert(GUID); diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp index 16a42e526eb8a..73697a9d0d446 100644 --- a/llvm/lib/LTO/LTOBackend.cpp +++ b/llvm/lib/LTO/LTOBackend.cpp @@ -625,12 +625,18 @@ static void dropDeadSymbols(Module &Mod, const GVSummaryMapTy &DefinedGlobals, const ModuleSummaryIndex &Index) { llvm::TimeTraceScope timeScope("Drop dead symbols"); std::vector DeadGVs; - for (auto &GV : Mod.global_values()) - if (GlobalValueSummary *GVS = DefinedGlobals.lookup(GV.getGUID())) + + for (auto &GV : Mod.global_values()) { + auto GUID = GV.getGUIDIfAssigned(); + if (!GUID) + continue; + + if (GlobalValueSummary *GVS = DefinedGlobals.lookup(*GUID)) if (!Index.isGlobalValueLive(GVS)) { DeadGVs.push_back(&GV); convertToDeclaration(GV); } + } // Now that all dead bodies have been dropped, delete the actual objects // themselves when possible. diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 0ada0f7e766dc..438d7352f3697 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -352,6 +352,7 @@ #include "llvm/Transforms/Scalar/TailRecursionElimination.h" #include "llvm/Transforms/Scalar/WarnMissedTransforms.h" #include "llvm/Transforms/Utils/AddDiscriminators.h" +#include "llvm/Transforms/Utils/AssignGUID.h" #include "llvm/Transforms/Utils/AssumeBundleBuilder.h" #include "llvm/Transforms/Utils/BreakCriticalEdges.h" #include "llvm/Transforms/Utils/CanonicalizeAliases.h" diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index a78a5afe05ae9..7ed5e2f168435 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -138,6 +138,7 @@ #include "llvm/Transforms/Scalar/TailRecursionElimination.h" #include "llvm/Transforms/Scalar/WarnMissedTransforms.h" #include "llvm/Transforms/Utils/AddDiscriminators.h" +#include "llvm/Transforms/Utils/AssignGUID.h" #include "llvm/Transforms/Utils/AssumeBundleBuilder.h" #include "llvm/Transforms/Utils/CanonicalizeAliases.h" #include "llvm/Transforms/Utils/CountVisits.h" @@ -834,6 +835,7 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level, void PassBuilder::addRequiredLTOPreLinkPasses(ModulePassManager &MPM) { MPM.addPass(CanonicalizeAliasesPass()); MPM.addPass(NameAnonGlobalPass()); + MPM.addPass(AssignGUIDPass()); } void PassBuilder::addPreInlinerPasses(ModulePassManager &MPM, @@ -1091,6 +1093,7 @@ PassBuilder::buildModuleInlinerPipeline(OptimizationLevel Level, if (!UseCtxProfile.empty() && Phase == ThinOrFullLTOPhase::ThinLTOPostLink) { MPM.addPass(GlobalOptPass()); MPM.addPass(GlobalDCEPass()); + MPM.addPass(AssignGUIDPass()); MPM.addPass(PGOCtxProfFlatteningPass(/*IsPreThinlink=*/false)); } @@ -1276,10 +1279,8 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, // In pre-link, we just want the instrumented IR. We use the contextual // profile in the post-thinlink phase. // The instrumentation will be removed in post-thinlink after IPO. - // FIXME(mtrofin): move AssignGUIDPass if there is agreement to use this - // mechanism for GUIDs. - MPM.addPass(AssignGUIDPass()); if (IsCtxProfUse) { + MPM.addPass(AssignGUIDPass()); MPM.addPass(PGOCtxProfFlatteningPass(/*IsPreThinlink=*/true)); return MPM; } @@ -1291,6 +1292,7 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, // unnecessary to collect profiles for non-prevailing copies. MPM.addPass(NoinlineNonPrevailing()); addPostPGOLoopRotation(MPM, Level); + MPM.addPass(AssignGUIDPass()); MPM.addPass(PGOCtxProfLoweringPass()); } else if (IsColdFuncOnlyInstrGen) { addPGOInstrPasses(MPM, Level, /* RunProfileGen */ true, /* IsCS */ false, @@ -1707,6 +1709,8 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level, if (PTO.DevirtualizeSpeculatively && LTOPhase == ThinOrFullLTOPhase::None) { // TODO: explore a better pipeline configuration that can improve // compilation time overhead. + // FIXME: move this earlier (lots of pass ordering tests will need fixing) + MPM.addPass(AssignGUIDPass()); MPM.addPass(WholeProgramDevirtPass( /*ExportSummary*/ nullptr, /*ImportSummary*/ nullptr, diff --git a/llvm/lib/Transforms/IPO/FunctionImport.cpp b/llvm/lib/Transforms/IPO/FunctionImport.cpp index d305eadc12f35..04da71822906e 100644 --- a/llvm/lib/Transforms/IPO/FunctionImport.cpp +++ b/llvm/lib/Transforms/IPO/FunctionImport.cpp @@ -12,6 +12,7 @@ #include "llvm/Transforms/IPO/FunctionImport.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallVector.h" @@ -1666,14 +1667,22 @@ void llvm::processImportsFiles( bool llvm::convertToDeclaration(GlobalValue &GV) { LLVM_DEBUG(dbgs() << "Converting to a declaration: `" << GV.getName() << "\n"); + MDNode *UniqueID = nullptr; + if (auto *GO = dyn_cast(&GV)) + UniqueID = GO->getMetadata(LLVMContext::MD_unique_id); + if (Function *F = dyn_cast(&GV)) { F->deleteBody(); F->clearMetadata(); + if (UniqueID) + F->setMetadata(LLVMContext::MD_unique_id, UniqueID); F->setComdat(nullptr); } else if (GlobalVariable *V = dyn_cast(&GV)) { V->setInitializer(nullptr); V->setLinkage(GlobalValue::ExternalLinkage); V->clearMetadata(); + if (UniqueID) + V->setMetadata(LLVMContext::MD_unique_id, UniqueID); V->setComdat(nullptr); } else { GlobalValue *NewGV; @@ -1705,7 +1714,7 @@ void llvm::thinLTOFinalizeInModule(Module &TheModule, DenseSet NonPrevailingComdats; auto FinalizeInModule = [&](GlobalValue &GV, bool Propagate = false) { // See if the global summary analysis computed a new resolved linkage. - const auto &GS = DefinedGlobals.find(GV.getGUID()); + const auto &GS = DefinedGlobals.find(GV.getGUIDOrFallback()); if (GS == DefinedGlobals.end()) return; @@ -1842,7 +1851,7 @@ void llvm::thinLTOInternalizeModule(Module &TheModule, return true; // Lookup the linkage recorded in the summaries during global analysis. - auto GS = DefinedGlobals.find(GV.getGUID()); + auto GS = DefinedGlobals.find(GV.getGUIDOrFallback()); if (GS == DefinedGlobals.end()) { // Must have been promoted (possibly conservatively). Find original // name so that we can access the correct summary and see if it can @@ -1917,7 +1926,7 @@ Expected FunctionImporter::importFunctions( DenseSet MoveSymbolGUIDSet; MoveSymbolGUIDSet.insert_range(MoveSymbolGUID); for (auto &F : DestModule) - if (!F.isDeclaration() && MoveSymbolGUIDSet.contains(F.getGUID())) + if (!F.isDeclaration() && MoveSymbolGUIDSet.contains(F.getGUIDOrFallback())) F.deleteBody(); IRMover Mover(DestModule); @@ -1945,7 +1954,7 @@ Expected FunctionImporter::importFunctions( for (Function &F : *SrcModule) { if (!F.hasName()) continue; - auto GUID = F.getGUID(); + auto GUID = F.getGUIDOrFallback(); auto MaybeImportType = ImportList.getImportType(ModName, GUID); bool ImportDefinition = MaybeImportType == GlobalValueSummary::Definition; @@ -1985,7 +1994,7 @@ Expected FunctionImporter::importFunctions( for (GlobalVariable &GV : SrcModule->globals()) { if (!GV.hasName()) continue; - auto GUID = GV.getGUID(); + auto GUID = GV.getGUIDOrFallback(); auto MaybeImportType = ImportList.getImportType(ModName, GUID); bool ImportDefinition = MaybeImportType == GlobalValueSummary::Definition; @@ -2009,7 +2018,7 @@ Expected FunctionImporter::importFunctions( for (GlobalAlias &GA : SrcModule->aliases()) { if (!GA.hasName() || isa(GA.getAliaseeObject())) continue; - auto GUID = GA.getGUID(); + auto GUID = GA.getGUIDOrFallback(); auto MaybeImportType = ImportList.getImportType(ModName, GUID); bool ImportDefinition = MaybeImportType == GlobalValueSummary::Definition; @@ -2029,9 +2038,12 @@ Expected FunctionImporter::importFunctions( if (Error Err = GO->materialize()) return std::move(Err); auto *Fn = replaceAliasWithAliasee(SrcModule.get(), &GA); - LLVM_DEBUG(dbgs() << "Is importing aliasee fn " << GO->getGUID() - << " " << GO->getName() << " from " - << SrcModule->getSourceFileName() << "\n"); + assert(Fn); + (void)Fn; + LLVM_DEBUG(dbgs() + << "Is importing aliasee fn " << GO->getGUIDOrFallback() + << " " << GO->getName() << " from " + << SrcModule->getSourceFileName() << "\n"); if (EnableImportMetadata || EnableMemProfContextDisambiguation) { // Add 'thinlto_src_module' and 'thinlto_src_file' metadata for // statistics and debugging. diff --git a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp index 3a6851eb60272..8ceefae72d9bf 100644 --- a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp +++ b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp @@ -198,6 +198,8 @@ void simplifyExternals(Module &M) { AttributeList::FunctionIndex, F.getAttributes().getFnAttrs())); NewF->takeName(&F); + NewF->setMetadata(LLVMContext::MD_unique_id, + F.getMetadata(LLVMContext::MD_unique_id)); F.replaceAllUsesWith(NewF); F.eraseFromParent(); } diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp index 5b2391a6a7b67..3f3e0654dec94 100644 --- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp +++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp @@ -886,6 +886,7 @@ void llvm::updateVCallVisibilityInModule( function_ref IsVisibleToRegularObj) { if (!hasWholeProgramVisibility(WholeProgramVisibilityEnabledInLTO)) return; + for (GlobalVariable &GV : M.globals()) { // Add linkage unit visibility to any variable with type metadata, which are // the vtable definitions. We won't have an existing vcall_visibility diff --git a/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp b/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp index 32ae14b45764b..42609f29df625 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp @@ -126,7 +126,7 @@ void annotateIndirectCalls(Module &M, const CtxProfAnalysis::Result &CtxProf) { for (auto &F : M) { if (F.isDeclaration()) continue; - auto FlatProfIter = FlatIndCalls.find(AssignGUIDPass::getGUID(F)); + auto FlatProfIter = FlatIndCalls.find(F.getGUID()); if (FlatProfIter == FlatIndCalls.end()) continue; const auto &FlatProf = FlatProfIter->second; @@ -179,7 +179,7 @@ PreservedAnalyses PGOCtxProfFlatteningPass::run(Module &M, "Function has unreachable basic blocks. The expectation was that " "DCE was run before."); - auto It = FlattenedProfile.find(AssignGUIDPass::getGUID(F)); + auto It = FlattenedProfile.find(F.getGUID()); // If this function didn't appear in the contextual profile, it's cold. if (It == FlattenedProfile.end()) clearColdFunctionProfile(F); diff --git a/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp b/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp index de7c1696078db..6a9d10ef5cca0 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp @@ -280,8 +280,7 @@ bool CtxInstrumentationLowerer::lowerFunction(Function &F) { assert(Mark->getIndex()->isZero()); IRBuilder<> Builder(Mark); - Guid = Builder.getInt64( - AssignGUIDPass::getGUID(cast(*Mark->getNameValue()))); + Guid = Builder.getInt64(cast(*Mark->getNameValue()).getGUID()); // The type of the context of this function is now knowable since we have // NumCallsites and NumCounters. We declare it here because it's more // convenient - we have the Builder. diff --git a/llvm/lib/Transforms/Scalar/JumpTableToSwitch.cpp b/llvm/lib/Transforms/Scalar/JumpTableToSwitch.cpp index d5696bc9ea956..12001218801c8 100644 --- a/llvm/lib/Transforms/Scalar/JumpTableToSwitch.cpp +++ b/llvm/lib/Transforms/Scalar/JumpTableToSwitch.cpp @@ -12,7 +12,6 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/ConstantFolding.h" -#include "llvm/Analysis/CtxProfAnalysis.h" #include "llvm/Analysis/DomTreeUpdater.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/PostDominators.h" @@ -215,9 +214,9 @@ PreservedAnalyses JumpTableToSwitchPass::run(Function &F, PostDominatorTree *PDT = AM.getCachedResult(F); DomTreeUpdater DTU(DT, PDT, DomTreeUpdater::UpdateStrategy::Lazy); bool Changed = false; - auto FuncToGuid = [InLTO = this->InLTO](const Function &Fct) { - if (Fct.getMetadata(AssignGUIDPass::GUIDMetadataName)) - return AssignGUIDPass::getGUID(Fct); + auto FuncToGuid = [&](const Function &Fct) { + if (const auto MaybeGUID = Fct.getGUIDIfAssigned(); MaybeGUID) + return *MaybeGUID; return Function::getGUIDAssumingExternalLinkage( getIRPGOFuncName(Fct, InLTO)); diff --git a/llvm/lib/Transforms/Utils/AssignGUID.cpp b/llvm/lib/Transforms/Utils/AssignGUID.cpp new file mode 100644 index 0000000000000..eb2c4844c1c87 --- /dev/null +++ b/llvm/lib/Transforms/Utils/AssignGUID.cpp @@ -0,0 +1,32 @@ +//===-- AssignGUID.cpp - Unique identifier assignment pass ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides a pass which assigns GUID (globally unique identifier) +// metadata to every GlobalValue in the module, according to its current name, +// linkage, and originating file. It is idempotent -- if GUID metadata is +// already present, it does nothing. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Utils/AssignGUID.h" +#include "llvm/Support/Debug.h" + +using namespace llvm; + +void AssignGUIDPass::runOnModule(Module &M) { + for (auto &GV : M.globals()) { + if (GV.isDeclaration()) + continue; + GV.assignGUID(); + } + for (auto &F : M.functions()) { + if (F.isDeclaration()) + continue; + F.assignGUID(); + } +} \ No newline at end of file diff --git a/llvm/lib/Transforms/Utils/CMakeLists.txt b/llvm/lib/Transforms/Utils/CMakeLists.txt index 933e204081ad2..8fe0476ab1a32 100644 --- a/llvm/lib/Transforms/Utils/CMakeLists.txt +++ b/llvm/lib/Transforms/Utils/CMakeLists.txt @@ -2,6 +2,7 @@ add_llvm_component_library(LLVMTransformUtils AddDiscriminators.cpp AMDGPUEmitPrintf.cpp ASanStackFrameLayout.cpp + AssignGUID.cpp AssumeBundleBuilder.cpp BasicBlockUtils.cpp BreakCriticalEdges.cpp diff --git a/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp b/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp index 1bfeea3b711f9..9656d99f6d473 100644 --- a/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp +++ b/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp @@ -637,11 +637,11 @@ CallBase *llvm::promoteCallWithIfThenElse(CallBase &CB, Function &Callee, IndirectBBIns->setIndex(IndirectID); IndirectBBIns->insertInto(&IndirectBB, IndirectBB.getFirstInsertionPt()); - const GlobalValue::GUID CalleeGUID = AssignGUIDPass::getGUID(Callee); + const GlobalValue::GUID CalleeGUID = Callee.getGUID(); const uint32_t NewCountersSize = IndirectID + 1; auto ProfileUpdater = [&](PGOCtxProfContext &Ctx) { - assert(Ctx.guid() == AssignGUIDPass::getGUID(Caller)); + assert(Ctx.guid() == Caller.getGUID()); assert(NewCountersSize - 2 == Ctx.counters().size()); // All the ctx-es belonging to a function must have the same size counters. Ctx.resizeCounters(NewCountersSize); @@ -676,7 +676,6 @@ CallBase *llvm::promoteCallWithIfThenElse(CallBase &CB, Function &Callee, // times, and the indirect BB, IndirectCount times Ctx.counters()[DirectID] = DirectCount; Ctx.counters()[IndirectID] = IndirectCount; - }; CtxProf.update(ProfileUpdater, Caller); return &DirectCall; diff --git a/llvm/lib/Transforms/Utils/CloneModule.cpp b/llvm/lib/Transforms/Utils/CloneModule.cpp index bb638f180bfbf..7ffb4f1da662d 100644 --- a/llvm/lib/Transforms/Utils/CloneModule.cpp +++ b/llvm/lib/Transforms/Utils/CloneModule.cpp @@ -129,17 +129,21 @@ std::unique_ptr llvm::CloneModule( for (const Function &I : M) { Function *F = cast(VMap[&I]); - if (I.isDeclaration()) { + auto CopyMD = [&]() { // Copy over metadata for declarations since we're not doing it below in // CloneFunctionInto(). SmallVector, 1> MDs; I.getAllMetadata(MDs); for (auto MD : MDs) F->addMetadata(MD.first, *MapMetadata(MD.second, VMap)); + }; + if (I.isDeclaration()) { + CopyMD(); continue; } if (!ShouldCloneDefinition(&I)) { + CopyMD(); // Skip after setting the correct linkage for an external reference. F->setLinkage(GlobalValue::ExternalLinkage); // Personality function is not valid on a declaration. diff --git a/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp b/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp index d641b33c50133..4f71239cda786 100644 --- a/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp +++ b/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp @@ -185,7 +185,7 @@ FunctionImportGlobalProcessing::getLinkage(const GlobalValue *SGV, // and/or optimization, but are turned into declarations later // during the EliminateAvailableExternally pass. if (doImportAsDefinition(SGV) && !isa(SGV)) - return SymbolsToMove.contains(SGV->getGUID()) + return SymbolsToMove.contains(SGV->getGUIDOrFallback()) ? GlobalValue::ExternalLinkage : GlobalValue::AvailableExternallyLinkage; // An imported external declaration stays external. @@ -261,7 +261,7 @@ void FunctionImportGlobalProcessing::processGlobalForThinLTO(GlobalValue &GV) { ValueInfo VI; if (GV.hasName()) - VI = ImportIndex.getValueInfo(GV.getGUID()); + VI = ImportIndex.getValueInfo(GV.getGUIDOrFallback()); // We should always have a ValueInfo (i.e. GV in index) for definitions when // we are exporting, and also when importing that value. diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp index 1d3f66509b1c5..60dd1cd7929a3 100644 --- a/llvm/lib/Transforms/Utils/InlineFunction.cpp +++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp @@ -2491,7 +2491,7 @@ llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, // Get some preliminary data about the callsite before it might get inlined. // Inlining shouldn't delete the callee, but it's cleaner (and low-cost) to // get this data upfront and rely less on InlineFunction's behavior. - const auto CalleeGUID = AssignGUIDPass::getGUID(Callee); + const auto CalleeGUID = Callee.getGUID(); auto *CallsiteIDIns = CtxProfAnalysis::getCallsiteInstrumentation(CB); const auto CallsiteID = static_cast(CallsiteIDIns->getIndex()->getZExtValue()); @@ -2516,7 +2516,7 @@ llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, const uint32_t NewCountersSize = CtxProf.getNumCounters(Caller); auto Updater = [&](PGOCtxProfContext &Ctx) { - assert(Ctx.guid() == AssignGUIDPass::getGUID(Caller)); + assert(Ctx.guid() == Caller.getGUID()); const auto &[CalleeCounterMap, CalleeCallsiteMap] = IndicesMaps; assert( (Ctx.counters().size() + diff --git a/llvm/test/Assembler/index-value-order.ll b/llvm/test/Assembler/index-value-order.ll index 9df6d4a46fef2..15ace18647de6 100644 --- a/llvm/test/Assembler/index-value-order.ll +++ b/llvm/test/Assembler/index-value-order.ll @@ -11,13 +11,16 @@ target triple = "x86_64-unknown-linux-gnu" $_ZTSN3FooE = comdat any -@_ZTSN3FooE = linkonce_odr constant [7 x i8] c"N3FooE\00", comdat, align 1 -@"_ZTVN3FooE" = internal unnamed_addr constant { [2 x ptr] } { [2 x ptr] [ptr null, ptr @"_Z3barv"] }, align 8 +@_ZTSN3FooE = linkonce_odr constant [7 x i8] c"N3FooE\00", comdat, align 1, !guid !0 +@"_ZTVN3FooE" = internal unnamed_addr constant { [2 x ptr] } { [2 x ptr] [ptr null, ptr @"_Z3barv"] }, align 8, !guid !1 -define internal i32 @"_Z3barv"() { +define internal i32 @"_Z3barv"() !guid !2 { ret i32 0 } +!0 = !{i64 0} +!1 = !{i64 1} +!2 = !{i64 2} ^0 = module: (path: "index-value-order.ll", hash: (0, 0, 0, 0, 0)) ^9 = gv: (name: "_ZTVN3FooE", summaries: (variable: (module: ^0, flags: (linkage: internal, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), varFlags: (readonly: 1, writeonly: 0, constant: 1, vcall_visibility: 0), vTableFuncs: ((virtFunc: ^3, offset: 16))))) ^4 = gv: (name: "_ZTSN3FooE", summaries: (variable: (module: ^0, flags: (linkage: linkonce_odr, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 0, canAutoHide: 0), varFlags: (readonly: 0, writeonly: 0, constant: 1)))) diff --git a/llvm/test/Bitcode/thinlto-alias.ll b/llvm/test/Bitcode/thinlto-alias.ll index 4191983055b2c..e6213f61ca8ed 100644 --- a/llvm/test/Bitcode/thinlto-alias.ll +++ b/llvm/test/Bitcode/thinlto-alias.ll @@ -13,6 +13,7 @@ ; CHECK: +; CHECK-NEXT: ; op4=none1 op6=hot1 op8=cold1 op10=none2 op12=hot2 op14=cold2 op16=none3 op18=hot3 op20=cold3 op22=123 -; CHECK-NEXT: +; CHECK-NEXT: ; CHECK-NEXT: ; CHECK-NEXT: diff --git a/llvm/test/Bitcode/thinlto-function-summary-callgraph-pgo.ll b/llvm/test/Bitcode/thinlto-function-summary-callgraph-pgo.ll index ed3c716288d6f..7f5f5ca39c544 100644 --- a/llvm/test/Bitcode/thinlto-function-summary-callgraph-pgo.ll +++ b/llvm/test/Bitcode/thinlto-function-summary-callgraph-pgo.ll @@ -11,6 +11,7 @@ ; RUN: llvm-lto -thinlto-index-stats %p/Inputs/thinlto-function-summary-callgraph-pgo-combined.1.bc | FileCheck %s --check-prefix=OLD-COMBINED ; CHECK: +; CHECK-NEXT: ; op4=hot1 op6=cold op8=hot2 op10=hot4 op12=none1 op14=hot3 op16=none2 op18=none3 op20=123 -; CHECK-NEXT: +; CHECK-NEXT: ; CHECK-NEXT: ; CHECK: +; CHECK-NEXT: ; op4=none1 op6=hot1 op8=cold1 op10=none2 op12=hot2 op14=cold2 op16=none3 op18=hot3 op20=cold3 op22=123 -; CHECK-NEXT: +; CHECK-NEXT: ; CHECK-NEXT: ; CHECK: