diff --git a/llvm/include/llvm/Analysis/CtxProfAnalysis.h b/llvm/include/llvm/Analysis/CtxProfAnalysis.h index 5c9823b0f6cc1..90b1d3fc0a688 100644 --- a/llvm/include/llvm/Analysis/CtxProfAnalysis.h +++ b/llvm/include/llvm/Analysis/CtxProfAnalysis.h @@ -47,9 +47,6 @@ class PGOContextualProfile { // we'll need when we maintain the profiles during IPO transformations. std::map FuncInfo; - /// Get the GUID of this Function if it's defined in this module. - LLVM_ABI GlobalValue::GUID getDefinedFunctionGUID(const Function &F) const; - // This is meant to be constructed from CtxProfAnalysis, which will also set // its state piecemeal. PGOContextualProfile() = default; @@ -68,9 +65,7 @@ class PGOContextualProfile { LLVM_ABI bool isInSpecializedModule() const; - bool isFunctionKnown(const Function &F) const { - return getDefinedFunctionGUID(F) != 0; - } + bool isFunctionKnown(const Function &F) const { return !F.isDeclaration(); } StringRef getFunctionName(GlobalValue::GUID GUID) const { auto It = FuncInfo.find(GUID); @@ -81,22 +76,22 @@ class PGOContextualProfile { uint32_t getNumCounters(const Function &F) const { assert(isFunctionKnown(F)); - return FuncInfo.find(getDefinedFunctionGUID(F))->second.NextCounterIndex; + return FuncInfo.find(F.getGUID())->second.NextCounterIndex; } uint32_t getNumCallsites(const Function &F) const { assert(isFunctionKnown(F)); - return FuncInfo.find(getDefinedFunctionGUID(F))->second.NextCallsiteIndex; + return FuncInfo.find(F.getGUID())->second.NextCallsiteIndex; } uint32_t allocateNextCounterIndex(const Function &F) { assert(isFunctionKnown(F)); - return FuncInfo.find(getDefinedFunctionGUID(F))->second.NextCounterIndex++; + return FuncInfo.find(F.getGUID())->second.NextCounterIndex++; } uint32_t allocateNextCallsiteIndex(const Function &F) { assert(isFunctionKnown(F)); - return FuncInfo.find(getDefinedFunctionGUID(F))->second.NextCallsiteIndex++; + return FuncInfo.find(F.getGUID())->second.NextCallsiteIndex++; } using ConstVisitor = function_ref; @@ -188,26 +183,5 @@ class ProfileAnnotator { LLVM_ABI ~ProfileAnnotator(); }; -/// Assign a GUID to functions as metadata. GUID calculation takes linkage into -/// account, which may change especially through and after thinlto. By -/// pre-computing and assigning as metadata, this mechanism is resilient to such -/// changes (as well as name changes e.g. suffix ".llvm." additions). - -// FIXME(mtrofin): we can generalize this mechanism to calculate a GUID early in -// the pass pipeline, associate it with any Global Value, and then use it for -// PGO and ThinLTO. -// At that point, this should be moved elsewhere. -class AssignGUIDPass : public PassInfoMixin { -public: - explicit AssignGUIDPass() = default; - - /// Assign a GUID *if* one is not already assign, as a function metadata named - /// `GUIDMetadataName`. - LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); - LLVM_ABI static const char *GUIDMetadataName; - // This should become GlobalValue::getGUID - LLVM_ABI static uint64_t getGUID(const Function &F); -}; - } // namespace llvm #endif // LLVM_ANALYSIS_CTXPROFANALYSIS_H diff --git a/llvm/include/llvm/IR/FixedMetadataKinds.def b/llvm/include/llvm/IR/FixedMetadataKinds.def index df572e8791e13..e7e3081271093 100644 --- a/llvm/include/llvm/IR/FixedMetadataKinds.def +++ b/llvm/include/llvm/IR/FixedMetadataKinds.def @@ -53,3 +53,4 @@ LLVM_FIXED_MD_KIND(MD_DIAssignID, "DIAssignID", 38) LLVM_FIXED_MD_KIND(MD_coro_outside_frame, "coro.outside.frame", 39) LLVM_FIXED_MD_KIND(MD_mmra, "mmra", 40) LLVM_FIXED_MD_KIND(MD_noalias_addrspace, "noalias.addrspace", 41) +LLVM_FIXED_MD_KIND(MD_unique_id, "unique_id", 42) diff --git a/llvm/include/llvm/IR/GlobalValue.h b/llvm/include/llvm/IR/GlobalValue.h index 83e695cdd27d9..b77bdc4deff69 100644 --- a/llvm/include/llvm/IR/GlobalValue.h +++ b/llvm/include/llvm/IR/GlobalValue.h @@ -589,17 +589,33 @@ class GlobalValue : public Constant { /// used as the key for a global lookup (e.g. profile or ThinLTO). LLVM_ABI std::string getGlobalIdentifier() const; + /// Assign a GUID to this value based on its current name and linkage. + /// This GUID will remain the same even if those change. This method is + /// idempotent -- if a GUID has already been assigned, calling it again + /// will do nothing. + /// + /// This is private (exposed only to \c AssignGUIDPass), as users don't need + /// to call it. GUIDs are assigned only by \c AssignGUIDPass. The pass + /// pipeline should be set up such that GUIDs are always available when + /// needed. If not, the GUID assignment pass should be moved (or run again) + /// such that they are. + void assignGUID(); + + // assignGUID needs to be accessible from AssignGUIDPass, which is called + // early in the pipeline to make GUIDs available to later passes. But we'd + // rather not expose it publicly, as no-one else should call it. + friend class AssignGUIDPass; + public: /// Return a 64-bit global unique ID constructed from the name of a global /// symbol. Since this call doesn't supply the linkage or defining filename, /// the GUID computation will assume that the global has external linkage. LLVM_ABI static GUID getGUIDAssumingExternalLinkage(StringRef GlobalName); - /// Return a 64-bit global unique ID constructed from global value name - /// (i.e. returned by getGlobalIdentifier()). - GUID getGUID() const { - return getGUIDAssumingExternalLinkage(getGlobalIdentifier()); - } + /// Return a 64-bit global unique ID for this value. It is based on the + /// "original" name and linkage of this value (i.e. whenever its GUID was + /// assigned). This might not match the current name and linkage. + GUID getGUID() const; /// @name Materialization /// Materialization is used to construct functions only as they're needed. diff --git a/llvm/include/llvm/Transforms/Utils/AssignGUID.h b/llvm/include/llvm/Transforms/Utils/AssignGUID.h new file mode 100644 index 0000000000000..923a1f92cbb7c --- /dev/null +++ b/llvm/include/llvm/Transforms/Utils/AssignGUID.h @@ -0,0 +1,34 @@ +//===-- AssignGUID.h - Unique identifier assignment pass --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides a pass which assigns a a GUID (globally unique identifier) +// to every GlobalValue in the module, according to its current name, linkage, +// and originating file. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_UTILS_ASSIGNGUID_H +#define LLVM_TRANSFORMS_UTILS_ASSIGNGUID_H + +#include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class AssignGUIDPass : public PassInfoMixin { +public: + AssignGUIDPass() = default; + + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); + + static bool isRequired() { return true; } +}; + +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_UTILS_ASSIGNGUID_H \ No newline at end of file diff --git a/llvm/lib/Analysis/CtxProfAnalysis.cpp b/llvm/lib/Analysis/CtxProfAnalysis.cpp index a363bce0570e7..7e8cb8e67a6d1 100644 --- a/llvm/lib/Analysis/CtxProfAnalysis.cpp +++ b/llvm/lib/Analysis/CtxProfAnalysis.cpp @@ -48,8 +48,6 @@ static cl::opt ForceIsInSpecializedModule( cl::desc("Treat the given module as-if it were containing the " "post-thinlink module containing the root")); -const char *AssignGUIDPass::GUIDMetadataName = "guid"; - namespace llvm { class ProfileAnnotatorImpl final { friend class ProfileAnnotator; @@ -418,33 +416,6 @@ bool ProfileAnnotator::getOutgoingBranchWeights( return MaxCount > 0; } -PreservedAnalyses AssignGUIDPass::run(Module &M, ModuleAnalysisManager &MAM) { - for (auto &F : M.functions()) { - if (F.isDeclaration()) - continue; - if (F.getMetadata(GUIDMetadataName)) - continue; - const GlobalValue::GUID GUID = F.getGUID(); - F.setMetadata(GUIDMetadataName, - MDNode::get(M.getContext(), - {ConstantAsMetadata::get(ConstantInt::get( - Type::getInt64Ty(M.getContext()), GUID))})); - } - return PreservedAnalyses::none(); -} - -GlobalValue::GUID AssignGUIDPass::getGUID(const Function &F) { - if (F.isDeclaration()) { - assert(GlobalValue::isExternalLinkage(F.getLinkage())); - return F.getGUID(); - } - auto *MD = F.getMetadata(GUIDMetadataName); - assert(MD && "guid not found for defined function"); - return cast(cast(MD->getOperand(0)) - ->getValue() - ->stripPointerCasts()) - ->getZExtValue(); -} AnalysisKey CtxProfAnalysis::Key; CtxProfAnalysis::CtxProfAnalysis(std::optional Profile) @@ -513,7 +484,7 @@ PGOContextualProfile CtxProfAnalysis::run(Module &M, for (const auto &F : M) { if (F.isDeclaration()) continue; - auto GUID = AssignGUIDPass::getGUID(F); + auto GUID = F.getGUID(); assert(GUID && "guid not found for defined function"); const auto &Entry = F.begin(); uint32_t MaxCounters = 0; // we expect at least a counter. @@ -547,13 +518,6 @@ PGOContextualProfile CtxProfAnalysis::run(Module &M, return Result; } -GlobalValue::GUID -PGOContextualProfile::getDefinedFunctionGUID(const Function &F) const { - if (auto It = FuncInfo.find(AssignGUIDPass::getGUID(F)); It != FuncInfo.end()) - return It->first; - return 0; -} - CtxProfAnalysisPrinterPass::CtxProfAnalysisPrinterPass(raw_ostream &OS) : OS(OS), Mode(PrintLevel) {} @@ -669,7 +633,7 @@ bool PGOContextualProfile::isInSpecializedModule() const { void PGOContextualProfile::update(Visitor V, const Function &F) { assert(isFunctionKnown(F)); - GlobalValue::GUID G = getDefinedFunctionGUID(F); + GlobalValue::GUID G = F.getGUID(); for (auto *Node = FuncInfo.find(G)->second.Index.Next; Node; Node = Node->Next) V(*reinterpret_cast(Node)); @@ -680,7 +644,7 @@ void PGOContextualProfile::visit(ConstVisitor V, const Function *F) const { return preorderVisit(Profiles.Contexts, V); assert(isFunctionKnown(*F)); - GlobalValue::GUID G = getDefinedFunctionGUID(*F); + GlobalValue::GUID G = F->getGUID(); for (const auto *Node = FuncInfo.find(G)->second.Index.Next; Node; Node = Node->Next) V(*reinterpret_cast(Node)); diff --git a/llvm/lib/IR/Globals.cpp b/llvm/lib/IR/Globals.cpp index 7b799c70a3318..068a78034b912 100644 --- a/llvm/lib/IR/Globals.cpp +++ b/llvm/lib/IR/Globals.cpp @@ -78,6 +78,27 @@ GlobalValue::getGUIDAssumingExternalLinkage(StringRef GlobalIdentifier) { return MD5Hash(GlobalIdentifier); } +void GlobalValue::assignGUID() { + if (getMetadata(LLVMContext::MD_unique_id) != nullptr) + return; + + const GUID G = + GlobalValue::getGUIDAssumingExternalLinkage(getGlobalIdentifier()); + setMetadata( + LLVMContext::MD_unique_id, + MDNode::get(getContext(), {ConstantAsMetadata::get(ConstantInt::get( + Type::getInt64Ty(getContext()), G))})); +} + +GlobalValue::GUID GlobalValue::getGUID() const { + auto *MD = getMetadata(LLVMContext::MD_unique_id); + assert(MD != nullptr && "GUID was not assigned before calling GetGUID()"); + return cast(cast(MD->getOperand(0)) + ->getValue() + ->stripPointerCasts()) + ->getZExtValue(); +} + void GlobalValue::removeFromParent() { switch (getValueID()) { #define HANDLE_GLOBAL_VALUE(NAME) \ diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 4603eaff8ade9..f25420babafdb 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -332,6 +332,7 @@ #include "llvm/Transforms/Scalar/TailRecursionElimination.h" #include "llvm/Transforms/Scalar/WarnMissedTransforms.h" #include "llvm/Transforms/Utils/AddDiscriminators.h" +#include "llvm/Transforms/Utils/AssignGUID.h" #include "llvm/Transforms/Utils/AssumeBundleBuilder.h" #include "llvm/Transforms/Utils/BreakCriticalEdges.h" #include "llvm/Transforms/Utils/CanonicalizeAliases.h" diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index a99146d5eaa34..ae2c12f2790ed 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -130,6 +130,7 @@ #include "llvm/Transforms/Scalar/TailRecursionElimination.h" #include "llvm/Transforms/Scalar/WarnMissedTransforms.h" #include "llvm/Transforms/Utils/AddDiscriminators.h" +#include "llvm/Transforms/Utils/AssignGUID.h" #include "llvm/Transforms/Utils/AssumeBundleBuilder.h" #include "llvm/Transforms/Utils/CanonicalizeAliases.h" #include "llvm/Transforms/Utils/CountVisits.h" @@ -1239,9 +1240,6 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, // In pre-link, we just want the instrumented IR. We use the contextual // profile in the post-thinlink phase. // The instrumentation will be removed in post-thinlink after IPO. - // FIXME(mtrofin): move AssignGUIDPass if there is agreement to use this - // mechanism for GUIDs. - MPM.addPass(AssignGUIDPass()); if (IsCtxProfUse) { MPM.addPass(PGOCtxProfFlatteningPass(/*IsPreThinlink=*/true)); return MPM; @@ -1640,6 +1638,9 @@ PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level, ModulePassManager MPM; + MPM.addPass(NameAnonGlobalPass()); + MPM.addPass(AssignGUIDPass()); + // Convert @llvm.global.annotations to !annotation metadata. MPM.addPass(Annotation2MetadataPass()); @@ -2207,6 +2208,9 @@ PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level, ModulePassManager MPM; + MPM.addPass(NameAnonGlobalPass()); + MPM.addPass(AssignGUIDPass()); + // Perform pseudo probe instrumentation in O0 mode. This is for the // consistency between different build modes. For example, a LTO build can be // mixed with an O0 prelink and an O2 postlink. Loading a sample profile in diff --git a/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp b/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp index 61285810cc529..d71079e3e4aa1 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp @@ -126,7 +126,7 @@ void annotateIndirectCalls(Module &M, const CtxProfAnalysis::Result &CtxProf) { for (auto &F : M) { if (F.isDeclaration()) continue; - auto FlatProfIter = FlatIndCalls.find(AssignGUIDPass::getGUID(F)); + auto FlatProfIter = FlatIndCalls.find(F.getGUID()); if (FlatProfIter == FlatIndCalls.end()) continue; const auto &FlatProf = FlatProfIter->second; @@ -179,7 +179,7 @@ PreservedAnalyses PGOCtxProfFlatteningPass::run(Module &M, "Function has unreacheable basic blocks. The expectation was that " "DCE was run before."); - auto It = FlattenedProfile.find(AssignGUIDPass::getGUID(F)); + auto It = FlattenedProfile.find(F.getGUID()); // If this function didn't appear in the contextual profile, it's cold. if (It == FlattenedProfile.end()) clearColdFunctionProfile(F); diff --git a/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp b/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp index d741695d4e53c..142db9179a46e 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp @@ -280,8 +280,7 @@ bool CtxInstrumentationLowerer::lowerFunction(Function &F) { assert(Mark->getIndex()->isZero()); IRBuilder<> Builder(Mark); - Guid = Builder.getInt64( - AssignGUIDPass::getGUID(cast(*Mark->getNameValue()))); + Guid = Builder.getInt64(cast(*Mark->getNameValue()).getGUID()); // The type of the context of this function is now knowable since we have // NumCallsites and NumCounters. We delcare it here because it's more // convenient - we have the Builder. diff --git a/llvm/lib/Transforms/Utils/AssignGUID.cpp b/llvm/lib/Transforms/Utils/AssignGUID.cpp new file mode 100644 index 0000000000000..8c33ff4b63e91 --- /dev/null +++ b/llvm/lib/Transforms/Utils/AssignGUID.cpp @@ -0,0 +1,34 @@ +//===-- AssignGUID.cpp - Unique identifier assignment pass ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides a pass which assigns a a GUID (globally unique identifier) +// to every GlobalValue in the module, according to its current name, linkage, +// and originating file. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Utils/AssignGUID.h" +#include "llvm/Support/Debug.h" + +using namespace llvm; + +PreservedAnalyses AssignGUIDPass::run(Module &M, ModuleAnalysisManager &AM) { + for (auto &GV : M.globals()) { + if (GV.isDeclaration()) + continue; + GV.assignGUID(); + dbgs() << "[Added GUID to GV:] " << GV.getName() << "\n"; + } + for (auto &F : M.functions()) { + if (F.isDeclaration()) + continue; + F.assignGUID(); + dbgs() << "[Added GUID to F:] " << F.getName() << "\n"; + } + return PreservedAnalyses::none(); +} \ No newline at end of file diff --git a/llvm/lib/Transforms/Utils/CMakeLists.txt b/llvm/lib/Transforms/Utils/CMakeLists.txt index 78cad0d253be8..9de8ce03ad31f 100644 --- a/llvm/lib/Transforms/Utils/CMakeLists.txt +++ b/llvm/lib/Transforms/Utils/CMakeLists.txt @@ -2,6 +2,7 @@ add_llvm_component_library(LLVMTransformUtils AddDiscriminators.cpp AMDGPUEmitPrintf.cpp ASanStackFrameLayout.cpp + AssignGUID.cpp AssumeBundleBuilder.cpp BasicBlockUtils.cpp BreakCriticalEdges.cpp diff --git a/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp b/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp index f0f9add09bf82..6912338ce5ffd 100644 --- a/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp +++ b/llvm/lib/Transforms/Utils/CallPromotionUtils.cpp @@ -616,11 +616,11 @@ CallBase *llvm::promoteCallWithIfThenElse(CallBase &CB, Function &Callee, IndirectBBIns->setIndex(IndirectID); IndirectBBIns->insertInto(&IndirectBB, IndirectBB.getFirstInsertionPt()); - const GlobalValue::GUID CalleeGUID = AssignGUIDPass::getGUID(Callee); + const GlobalValue::GUID CalleeGUID = Callee.getGUID(); const uint32_t NewCountersSize = IndirectID + 1; auto ProfileUpdater = [&](PGOCtxProfContext &Ctx) { - assert(Ctx.guid() == AssignGUIDPass::getGUID(Caller)); + assert(Ctx.guid() == Caller.getGUID()); assert(NewCountersSize - 2 == Ctx.counters().size()); // All the ctx-es belonging to a function must have the same size counters. Ctx.resizeCounters(NewCountersSize); @@ -655,7 +655,6 @@ CallBase *llvm::promoteCallWithIfThenElse(CallBase &CB, Function &Callee, // times, and the indirect BB, IndirectCount times Ctx.counters()[DirectID] = DirectCount; Ctx.counters()[IndirectID] = IndirectCount; - }; CtxProf.update(ProfileUpdater, Caller); return &DirectCall; diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp index 7df5e9958182c..1fb61e1f308e8 100644 --- a/llvm/lib/Transforms/Utils/InlineFunction.cpp +++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp @@ -2381,7 +2381,7 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, // Get some preliminary data about the callsite before it might get inlined. // Inlining shouldn't delete the callee, but it's cleaner (and low-cost) to // get this data upfront and rely less on InlineFunction's behavior. - const auto CalleeGUID = AssignGUIDPass::getGUID(Callee); + const auto CalleeGUID = Callee.getGUID(); auto *CallsiteIDIns = CtxProfAnalysis::getCallsiteInstrumentation(CB); const auto CallsiteID = static_cast(CallsiteIDIns->getIndex()->getZExtValue()); @@ -2406,7 +2406,7 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, const uint32_t NewCountersSize = CtxProf.getNumCounters(Caller); auto Updater = [&](PGOCtxProfContext &Ctx) { - assert(Ctx.guid() == AssignGUIDPass::getGUID(Caller)); + assert(Ctx.guid() == Caller.getGUID()); const auto &[CalleeCounterMap, CalleeCallsiteMap] = IndicesMaps; assert( (Ctx.counters().size() + diff --git a/llvm/test/Transforms/AssignGUID/assign_guid.ll b/llvm/test/Transforms/AssignGUID/assign_guid.ll new file mode 100644 index 0000000000000..2570b1ff40a7e --- /dev/null +++ b/llvm/test/Transforms/AssignGUID/assign_guid.ll @@ -0,0 +1,18 @@ +; RUN: opt -S -passes=assign-guid %s | FileCheck %s + +@G = global i32 0 +; CHECK: @G = global i32 0, !unique_id !0 +@G_EXT = external global i32 + +declare external void @f_ext() + +@A = alias i32, ptr @G +@A_EXT = external alias i32, ptr @G + +define void @f() { +; CHECK: define void @f() !unique_id !1 { + ret void +} + +; CHECK: !0 = !{i64 -6455552227143004193} +; CHECK: !1 = !{i64 -3706093650706652785}