diff --git a/llvm/include/llvm/Analysis/DomTreeUpdater.h b/llvm/include/llvm/Analysis/DomTreeUpdater.h index 0386262ba2b65..c120a6cc6ce5a 100644 --- a/llvm/include/llvm/Analysis/DomTreeUpdater.h +++ b/llvm/include/llvm/Analysis/DomTreeUpdater.h @@ -81,9 +81,6 @@ class DomTreeUpdater ///@} - /// Debug method to help view the internal state of this class. - LLVM_DUMP_METHOD void dump() const; - private: class CallBackOnDeletion final : public CallbackVH { public: @@ -112,6 +109,9 @@ class DomTreeUpdater /// Returns true if at least one BasicBlock is deleted. bool forceFlushDeletedBB(); + + /// Debug method to help view the internal state of this class. + LLVM_DUMP_METHOD void dump() const; }; extern template class GenericDomTreeUpdater::recalculate(Function &F); - -extern template void -GenericDomTreeUpdater:: - applyUpdatesImpl(); -extern template void -GenericDomTreeUpdater:: - applyUpdatesImpl(); } // namespace llvm #endif // LLVM_ANALYSIS_DOMTREEUPDATER_H diff --git a/llvm/include/llvm/Analysis/GenericDomTreeUpdater.h b/llvm/include/llvm/Analysis/GenericDomTreeUpdater.h index 4a03f548823ee..ca4ce68b85cbc 100644 --- a/llvm/include/llvm/Analysis/GenericDomTreeUpdater.h +++ b/llvm/include/llvm/Analysis/GenericDomTreeUpdater.h @@ -30,7 +30,6 @@ class GenericDomTreeUpdater { public: enum class UpdateStrategy : unsigned char { Eager = 0, Lazy = 1 }; using BasicBlockT = typename DomTreeT::NodeType; - using UpdateT = typename DomTreeT::UpdateType; explicit GenericDomTreeUpdater(UpdateStrategy Strategy_) : Strategy(Strategy_) {} @@ -147,12 +146,7 @@ class GenericDomTreeUpdater { /// 2. It is illegal to submit any update that has already been submitted, /// i.e., you are supposed not to insert an existent edge or delete a /// nonexistent edge. - void applyUpdates(ArrayRef Updates); - - /// Apply updates that the critical edge (FromBB, ToBB) has been - /// split with NewBB. - void splitCriticalEdge(BasicBlockT *FromBB, BasicBlockT *ToBB, - BasicBlockT *NewBB); + void applyUpdates(ArrayRef Updates); /// Submit updates to all available trees. It will also /// 1. discard duplicated updates, @@ -175,7 +169,7 @@ class GenericDomTreeUpdater { /// 3. It is only legal to submit updates to an edge in the order CFG changes /// are made. The order you submit updates on different edges is not /// restricted. - void applyUpdatesPermissive(ArrayRef Updates); + void applyUpdatesPermissive(ArrayRef Updates); ///@} @@ -211,25 +205,7 @@ class GenericDomTreeUpdater { LLVM_DUMP_METHOD void dump() const; protected: - /// Helper structure used to hold all the basic blocks - /// involved in the split of a critical edge. - struct CriticalEdge { - BasicBlockT *FromBB; - BasicBlockT *ToBB; - BasicBlockT *NewBB; - }; - - struct DomTreeUpdate { - bool IsCriticalEdgeSplit = false; - union { - UpdateT Update; - CriticalEdge EdgeSplit; - }; - DomTreeUpdate(UpdateT Update) : Update(Update) {} - DomTreeUpdate(CriticalEdge E) : IsCriticalEdgeSplit(true), EdgeSplit(E) {} - }; - - SmallVector PendUpdates; + SmallVector PendUpdates; size_t PendDTUpdateIndex = 0; size_t PendPDTUpdateIndex = 0; DomTreeT *DT = nullptr; @@ -240,21 +216,21 @@ class GenericDomTreeUpdater { bool IsRecalculatingPostDomTree = false; /// Returns true if the update is self dominance. - bool isSelfDominance(UpdateT Update) const { + bool isSelfDominance(typename DomTreeT::UpdateType Update) const { // Won't affect DomTree and PostDomTree. return Update.getFrom() == Update.getTo(); } /// Helper function to apply all pending DomTree updates. - void applyDomTreeUpdates() { applyUpdatesImpl(); } + void applyDomTreeUpdates(); /// Helper function to apply all pending PostDomTree updates. - void applyPostDomTreeUpdates() { applyUpdatesImpl(); } + void applyPostDomTreeUpdates(); /// Returns true if the update appears in the LLVM IR. /// It is used to check whether an update is valid in /// insertEdge/deleteEdge or is unnecessary in the batch update. - bool isUpdateValid(UpdateT Update) const; + bool isUpdateValid(typename DomTreeT::UpdateType Update) const; /// Erase Basic Block node before it is unlinked from Function /// in the DomTree and PostDomTree. @@ -267,11 +243,6 @@ class GenericDomTreeUpdater { /// Drop all updates applied by all available trees and delete BasicBlocks if /// all available trees are up-to-date. void dropOutOfDateUpdates(); - -private: - void splitDTCriticalEdges(ArrayRef Updates); - void splitPDTCriticalEdges(ArrayRef Updates); - template void applyUpdatesImpl(); }; } // namespace llvm diff --git a/llvm/include/llvm/Analysis/GenericDomTreeUpdaterImpl.h b/llvm/include/llvm/Analysis/GenericDomTreeUpdaterImpl.h index 70ae72626c6a2..b79eaef57710f 100644 --- a/llvm/include/llvm/Analysis/GenericDomTreeUpdaterImpl.h +++ b/llvm/include/llvm/Analysis/GenericDomTreeUpdaterImpl.h @@ -16,7 +16,6 @@ #ifndef LLVM_ANALYSIS_GENERICDOMTREEUPDATERIMPL_H #define LLVM_ANALYSIS_GENERICDOMTREEUPDATERIMPL_H -#include "llvm/ADT/SmallBitVector.h" #include "llvm/Analysis/GenericDomTreeUpdater.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" @@ -57,7 +56,7 @@ void GenericDomTreeUpdater::recalculate( template void GenericDomTreeUpdater::applyUpdates( - ArrayRef Updates) { + ArrayRef Updates) { if (!DT && !PDT) return; @@ -78,12 +77,12 @@ void GenericDomTreeUpdater::applyUpdates( template void GenericDomTreeUpdater:: - applyUpdatesPermissive(ArrayRef Updates) { + applyUpdatesPermissive(ArrayRef Updates) { if (!DT && !PDT) return; SmallSet, 8> Seen; - SmallVector DeduplicatedUpdates; + SmallVector DeduplicatedUpdates; for (const auto &U : Updates) { auto Edge = std::make_pair(U.getFrom(), U.getTo()); // Because it is illegal to submit updates that have already been applied @@ -130,24 +129,6 @@ void GenericDomTreeUpdater:: PDT->applyUpdates(DeduplicatedUpdates); } -template -void GenericDomTreeUpdater::splitCriticalEdge( - BasicBlockT *FromBB, BasicBlockT *ToBB, BasicBlockT *NewBB) { - if (!DT && !PDT) - return; - - CriticalEdge E = {FromBB, ToBB, NewBB}; - if (Strategy == UpdateStrategy::Lazy) { - PendUpdates.push_back(E); - return; - } - - if (DT) - splitDTCriticalEdges(E); - if (PDT) - splitPDTCriticalEdges(E); -} - template DomTreeT & GenericDomTreeUpdater::getDomTree() { @@ -188,39 +169,41 @@ GenericDomTreeUpdater::dump() const { return; } else OS << "Lazy\n"; - - auto printBlockInfo = [&](BasicBlockT *BB, StringRef Ending) { - if (BB) { - auto S = BB->getName(); - if (!BB->hasName()) - S = "(no name)"; - OS << S << "(" << BB << ")" << Ending; - } else { - OS << "(badref)" << Ending; - } - }; + int Index = 0; auto printUpdates = - [&](typename ArrayRef::const_iterator begin, - typename ArrayRef::const_iterator end) { + [&](typename ArrayRef::const_iterator + begin, + typename ArrayRef::const_iterator + end) { if (begin == end) OS << " None\n"; - for (auto [Index, DTUpdate] : enumerate(make_range(begin, end))) { - if (!DTUpdate.IsCriticalEdgeSplit) { - auto U = DTUpdate.Update; - OS << " " << Index << " : "; - if (U.getKind() == DomTreeT::Insert) - OS << "Insert, "; - else - OS << "Delete, "; - printBlockInfo(U.getFrom(), ", "); - printBlockInfo(U.getTo(), "\n"); + Index = 0; + for (auto It = begin, ItEnd = end; It != ItEnd; ++It) { + auto U = *It; + OS << " " << Index << " : "; + ++Index; + if (U.getKind() == DomTreeT::Insert) + OS << "Insert, "; + else + OS << "Delete, "; + BasicBlockT *From = U.getFrom(); + if (From) { + auto S = From->getName(); + if (!From->hasName()) + S = "(no name)"; + OS << S << "(" << From << "), "; } else { - const auto &Edge = DTUpdate.EdgeSplit; - OS << " " << Index << " : Split critical edge, "; - printBlockInfo(Edge.FromBB, ", "); - printBlockInfo(Edge.ToBB, ", "); - printBlockInfo(Edge.NewBB, "\n"); + OS << "(badref), "; + } + BasicBlockT *To = U.getTo(); + if (To) { + auto S = To->getName(); + if (!To->hasName()) + S = "(no_name)"; + OS << S << "(" << To << ")\n"; + } else { + OS << "(badref)\n"; } } }; @@ -246,58 +229,57 @@ GenericDomTreeUpdater::dump() const { } OS << "Pending DeletedBBs:\n"; - for (auto [Index, BB] : enumerate(DeletedBBs)) { + Index = 0; + for (const auto *BB : DeletedBBs) { OS << " " << Index << " : "; + ++Index; if (BB->hasName()) OS << BB->getName() << "("; else - OS << "(no name)("; + OS << "(no_name)("; OS << BB << ")\n"; } #endif } template -template void GenericDomTreeUpdater::applyUpdatesImpl() { - auto *DomTree = [&]() { - if constexpr (IsForward) - return DT; - else - return PDT; - }(); + PostDomTreeT>::applyDomTreeUpdates() { // No pending DomTreeUpdates. - if (Strategy != UpdateStrategy::Lazy || !DomTree) + if (Strategy != UpdateStrategy::Lazy || !DT) return; - size_t &PendUpdateIndex = IsForward ? PendDTUpdateIndex : PendPDTUpdateIndex; - // Only apply updates not are applied by (Post)DomTree. - while (IsForward ? hasPendingDomTreeUpdates() - : hasPendingPostDomTreeUpdates()) { - auto I = PendUpdates.begin() + PendUpdateIndex; + // Only apply updates not are applied by DomTree. + if (hasPendingDomTreeUpdates()) { + const auto I = PendUpdates.begin() + PendDTUpdateIndex; const auto E = PendUpdates.end(); assert(I < E && "Iterator range invalid; there should be DomTree updates."); - if (!I->IsCriticalEdgeSplit) { - SmallVector NormalUpdates; - for (; I != E && !I->IsCriticalEdgeSplit; ++I) - NormalUpdates.push_back(I->Update); - DomTree->applyUpdates(NormalUpdates); - PendUpdateIndex += NormalUpdates.size(); - } else { - SmallVector CriticalEdges; - for (; I != E && I->IsCriticalEdgeSplit; ++I) - CriticalEdges.push_back(I->EdgeSplit); - IsForward ? splitDTCriticalEdges(CriticalEdges) - : splitPDTCriticalEdges(CriticalEdges); - PendUpdateIndex += CriticalEdges.size(); - } + DT->applyUpdates(ArrayRef(I, E)); + PendDTUpdateIndex = PendUpdates.size(); + } +} + +template +void GenericDomTreeUpdater::applyPostDomTreeUpdates() { + // No pending PostDomTreeUpdates. + if (Strategy != UpdateStrategy::Lazy || !PDT) + return; + + // Only apply updates not are applied by PostDomTree. + if (hasPendingPostDomTreeUpdates()) { + const auto I = PendUpdates.begin() + PendPDTUpdateIndex; + const auto E = PendUpdates.end(); + assert(I < E && + "Iterator range invalid; there should be PostDomTree updates."); + PDT->applyUpdates(ArrayRef(I, E)); + PendPDTUpdateIndex = PendUpdates.size(); } } template bool GenericDomTreeUpdater::isUpdateValid( - UpdateT Update) const { + typename DomTreeT::UpdateType Update) const { const auto *From = Update.getFrom(); const auto *To = Update.getTo(); const auto Kind = Update.getKind(); @@ -365,116 +347,6 @@ void GenericDomTreeUpdater -void GenericDomTreeUpdater:: - splitDTCriticalEdges(ArrayRef Edges) { - // Bail out early if there is nothing to do. - if (!DT || Edges.empty()) - return; - - // Remember all the basic blocks that are inserted during - // edge splitting. - // Invariant: NewBBs == all the basic blocks contained in the NewBB - // field of all the elements of Edges. - // I.e., forall elt in Edges, it exists BB in NewBBs - // such as BB == elt.NewBB. - SmallSet NewBBs; - for (auto &Edge : Edges) - NewBBs.insert(Edge.NewBB); - // For each element in Edges, remember whether or not element - // is the new immediate domminator of its successor. The mapping is done by - // index, i.e., the information for the ith element of Edges is - // the ith element of IsNewIDom. - SmallBitVector IsNewIDom(Edges.size(), true); - - // Collect all the dominance properties info, before invalidating - // the underlying DT. - for (const auto &[Idx, Edge] : enumerate(Edges)) { - // Update dominator information. - BasicBlockT *Succ = Edge.ToBB; - auto *SuccDTNode = DT->getNode(Succ); - - for (BasicBlockT *PredBB : predecessors(Succ)) { - if (PredBB == Edge.NewBB) - continue; - // If we are in this situation: - // FromBB1 FromBB2 - // + + - // + + + + - // + + + + - // ... Split1 Split2 ... - // + + - // + + - // + - // Succ - // Instead of checking the domiance property with Split2, we check it - // with FromBB2 since Split2 is still unknown of the underlying DT - // structure. - if (NewBBs.contains(PredBB)) { - assert(pred_size(PredBB) == 1 && "A basic block resulting from a " - "critical edge split has more " - "than one predecessor!"); - PredBB = *pred_begin(PredBB); - } - if (!DT->dominates(SuccDTNode, DT->getNode(PredBB))) { - IsNewIDom[Idx] = false; - break; - } - } - } - - // Now, update DT with the collected dominance properties info. - for (const auto &[Idx, Edge] : enumerate(Edges)) { - // We know FromBB dominates NewBB. - auto *NewDTNode = DT->addNewBlock(Edge.NewBB, Edge.FromBB); - - // If all the other predecessors of "Succ" are dominated by "Succ" itself - // then the new block is the new immediate dominator of "Succ". Otherwise, - // the new block doesn't dominate anything. - if (IsNewIDom[Idx]) - DT->changeImmediateDominator(DT->getNode(Edge.ToBB), NewDTNode); - } -} - -template -void GenericDomTreeUpdater:: - splitPDTCriticalEdges(ArrayRef Edges) { - // Bail out early if there is nothing to do. - if (!PDT || Edges.empty()) - return; - - SmallBitVector IsNewIPDom(Edges.size(), true); - SmallSet NewBBs; - for (const auto &Edge : Edges) - NewBBs.insert(Edge.NewBB); - - for (const auto &[Idx, Edge] : enumerate(Edges)) { - // Same as DT version but from another direction. - BasicBlockT *Pred = Edge.FromBB; - auto *PredDTNode = PDT->getNode(Pred); - for (BasicBlockT *SuccBB : successors(Pred)) { - if (SuccBB == Edge.NewBB) - continue; - if (NewBBs.count(SuccBB)) { - assert(succ_size(SuccBB) == 1 && "A basic block resulting from a " - "critical edge split has more " - "than one successor!"); - SuccBB = *succ_begin(SuccBB); - } - if (!PDT->dominates(PredDTNode, PDT->getNode(SuccBB))) { - IsNewIPDom[Idx] = false; - break; - } - } - } - - for (const auto &[Idx, Edge] : enumerate(Edges)) { - auto *NewPDTNode = PDT->addNewBlock(Edge.NewBB, Edge.ToBB); - if (IsNewIPDom[Idx]) - PDT->changeImmediateDominator(PDT->getNode(Edge.FromBB), NewPDTNode); - } -} - } // namespace llvm #endif // LLVM_ANALYSIS_GENERICDOMTREEUPDATERIMPL_H diff --git a/llvm/include/llvm/CodeGen/MachineBasicBlock.h b/llvm/include/llvm/CodeGen/MachineBasicBlock.h index 7fe33c3913f2d..6cf151c951b19 100644 --- a/llvm/include/llvm/CodeGen/MachineBasicBlock.h +++ b/llvm/include/llvm/CodeGen/MachineBasicBlock.h @@ -32,7 +32,6 @@ namespace llvm { class BasicBlock; -class MachineDomTreeUpdater; class MachineFunction; class MCSymbol; class ModuleSlotTracker; @@ -973,23 +972,22 @@ class MachineBasicBlock /// MachineLoopInfo, as applicable. MachineBasicBlock * SplitCriticalEdge(MachineBasicBlock *Succ, Pass &P, - std::vector> *LiveInSets = nullptr, - MachineDomTreeUpdater *MDTU = nullptr) { - return SplitCriticalEdge(Succ, &P, nullptr, LiveInSets, MDTU); + std::vector> *LiveInSets = nullptr) { + return SplitCriticalEdge(Succ, &P, nullptr, LiveInSets); } MachineBasicBlock * SplitCriticalEdge(MachineBasicBlock *Succ, MachineFunctionAnalysisManager &MFAM, - std::vector> *LiveInSets = nullptr, - MachineDomTreeUpdater *MDTU = nullptr) { - return SplitCriticalEdge(Succ, nullptr, &MFAM, LiveInSets, MDTU); + std::vector> *LiveInSets = nullptr) { + return SplitCriticalEdge(Succ, nullptr, &MFAM, LiveInSets); } // Helper method for new pass manager migration. - MachineBasicBlock *SplitCriticalEdge( - MachineBasicBlock *Succ, Pass *P, MachineFunctionAnalysisManager *MFAM, - std::vector> *LiveInSets, MachineDomTreeUpdater *MDTU); + MachineBasicBlock * + SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P, + MachineFunctionAnalysisManager *MFAM, + std::vector> *LiveInSets); /// Check if the edge between this block and the given successor \p /// Succ, can be split. If this returns true a subsequent call to @@ -1377,12 +1375,6 @@ inline auto successors(const MachineBasicBlock *BB) { return BB->successors(); } inline auto predecessors(const MachineBasicBlock *BB) { return BB->predecessors(); } -inline auto succ_size(const MachineBasicBlock *BB) { return BB->succ_size(); } -inline auto pred_size(const MachineBasicBlock *BB) { return BB->pred_size(); } -inline auto succ_begin(const MachineBasicBlock *BB) { return BB->succ_begin(); } -inline auto pred_begin(const MachineBasicBlock *BB) { return BB->pred_begin(); } -inline auto succ_end(const MachineBasicBlock *BB) { return BB->succ_end(); } -inline auto pred_end(const MachineBasicBlock *BB) { return BB->pred_end(); } /// MachineInstrSpan provides an interface to get an iteration range /// containing the instruction it was initialized with, along with all diff --git a/llvm/include/llvm/CodeGen/MachineDomTreeUpdater.h b/llvm/include/llvm/CodeGen/MachineDomTreeUpdater.h index fcdc0becf31c1..9e3971f0b9fce 100644 --- a/llvm/include/llvm/CodeGen/MachineDomTreeUpdater.h +++ b/llvm/include/llvm/CodeGen/MachineDomTreeUpdater.h @@ -69,12 +69,5 @@ extern template void GenericDomTreeUpdater::recalculate(MachineFunction &MF); - -extern template void GenericDomTreeUpdater< - MachineDomTreeUpdater, MachineDominatorTree, - MachinePostDominatorTree>::applyUpdatesImpl(); -extern template void GenericDomTreeUpdater< - MachineDomTreeUpdater, MachineDominatorTree, - MachinePostDominatorTree>::applyUpdatesImpl(); } // namespace llvm #endif // LLVM_CODEGEN_MACHINEDOMTREEUPDATER_H diff --git a/llvm/include/llvm/CodeGen/MachineDominators.h b/llvm/include/llvm/CodeGen/MachineDominators.h index 61635ff64502d..74cf94398736d 100644 --- a/llvm/include/llvm/CodeGen/MachineDominators.h +++ b/llvm/include/llvm/CodeGen/MachineDominators.h @@ -73,22 +73,86 @@ extern template bool Verify(const MBBDomTree &DT, /// compute a normal dominator tree. /// class MachineDominatorTree : public DomTreeBase { + /// Helper structure used to hold all the basic blocks + /// involved in the split of a critical edge. + struct CriticalEdge { + MachineBasicBlock *FromBB; + MachineBasicBlock *ToBB; + MachineBasicBlock *NewBB; + }; + + /// Pile up all the critical edges to be split. + /// The splitting of a critical edge is local and thus, it is possible + /// to apply several of those changes at the same time. + mutable SmallVector CriticalEdgesToSplit; + + /// Remember all the basic blocks that are inserted during + /// edge splitting. + /// Invariant: NewBBs == all the basic blocks contained in the NewBB + /// field of all the elements of CriticalEdgesToSplit. + /// I.e., forall elt in CriticalEdgesToSplit, it exists BB in NewBBs + /// such as BB == elt.NewBB. + mutable SmallSet NewBBs; + + /// Apply all the recorded critical edges to the DT. + /// This updates the underlying DT information in a way that uses + /// the fast query path of DT as much as possible. + /// FIXME: This method should not be a const member! + /// + /// \post CriticalEdgesToSplit.empty(). + void applySplitCriticalEdges() const; public: using Base = DomTreeBase; MachineDominatorTree() = default; - explicit MachineDominatorTree(MachineFunction &MF) { recalculate(MF); } + explicit MachineDominatorTree(MachineFunction &MF) { calculate(MF); } /// Handle invalidation explicitly. bool invalidate(MachineFunction &, const PreservedAnalyses &PA, MachineFunctionAnalysisManager::Invalidator &); - using Base::dominates; + // FIXME: If there is an updater for MachineDominatorTree, + // migrate to this updater and remove these wrappers. + + MachineDominatorTree &getBase() { + applySplitCriticalEdges(); + return *this; + } + + MachineBasicBlock *getRoot() const { + applySplitCriticalEdges(); + return Base::getRoot(); + } + + MachineDomTreeNode *getRootNode() const { + applySplitCriticalEdges(); + return const_cast(Base::getRootNode()); + } + + void calculate(MachineFunction &F); + + bool dominates(const MachineDomTreeNode *A, + const MachineDomTreeNode *B) const { + applySplitCriticalEdges(); + return Base::dominates(A, B); + } + + void getDescendants(MachineBasicBlock *A, + SmallVectorImpl &Result) { + applySplitCriticalEdges(); + Base::getDescendants(A, Result); + } + + bool dominates(const MachineBasicBlock *A, const MachineBasicBlock *B) const { + applySplitCriticalEdges(); + return Base::dominates(A, B); + } // dominates - Return true if A dominates B. This performs the // special checks necessary if A and B are in the same basic block. bool dominates(const MachineInstr *A, const MachineInstr *B) const { + applySplitCriticalEdges(); const MachineBasicBlock *BBA = A->getParent(), *BBB = B->getParent(); if (BBA != BBB) return Base::dominates(BBA, BBB); @@ -100,6 +164,107 @@ class MachineDominatorTree : public DomTreeBase { return &*I == A; } + + bool properlyDominates(const MachineDomTreeNode *A, + const MachineDomTreeNode *B) const { + applySplitCriticalEdges(); + return Base::properlyDominates(A, B); + } + + bool properlyDominates(const MachineBasicBlock *A, + const MachineBasicBlock *B) const { + applySplitCriticalEdges(); + return Base::properlyDominates(A, B); + } + + /// findNearestCommonDominator - Find nearest common dominator basic block + /// for basic block A and B. If there is no such block then return NULL. + MachineBasicBlock *findNearestCommonDominator(MachineBasicBlock *A, + MachineBasicBlock *B) { + applySplitCriticalEdges(); + return Base::findNearestCommonDominator(A, B); + } + + MachineDomTreeNode *operator[](MachineBasicBlock *BB) const { + applySplitCriticalEdges(); + return Base::getNode(BB); + } + + /// getNode - return the (Post)DominatorTree node for the specified basic + /// block. This is the same as using operator[] on this class. + /// + MachineDomTreeNode *getNode(MachineBasicBlock *BB) const { + applySplitCriticalEdges(); + return Base::getNode(BB); + } + + /// addNewBlock - Add a new node to the dominator tree information. This + /// creates a new node as a child of DomBB dominator node,linking it into + /// the children list of the immediate dominator. + MachineDomTreeNode *addNewBlock(MachineBasicBlock *BB, + MachineBasicBlock *DomBB) { + applySplitCriticalEdges(); + return Base::addNewBlock(BB, DomBB); + } + + /// changeImmediateDominator - This method is used to update the dominator + /// tree information when a node's immediate dominator changes. + /// + void changeImmediateDominator(MachineBasicBlock *N, + MachineBasicBlock *NewIDom) { + applySplitCriticalEdges(); + Base::changeImmediateDominator(N, NewIDom); + } + + void changeImmediateDominator(MachineDomTreeNode *N, + MachineDomTreeNode *NewIDom) { + applySplitCriticalEdges(); + Base::changeImmediateDominator(N, NewIDom); + } + + /// eraseNode - Removes a node from the dominator tree. Block must not + /// dominate any other blocks. Removes node from its immediate dominator's + /// children list. Deletes dominator node associated with basic block BB. + void eraseNode(MachineBasicBlock *BB) { + applySplitCriticalEdges(); + Base::eraseNode(BB); + } + + /// splitBlock - BB is split and now it has one successor. Update dominator + /// tree to reflect this change. + void splitBlock(MachineBasicBlock* NewBB) { + applySplitCriticalEdges(); + Base::splitBlock(NewBB); + } + + /// isReachableFromEntry - Return true if A is dominated by the entry + /// block of the function containing it. + bool isReachableFromEntry(const MachineBasicBlock *A) { + applySplitCriticalEdges(); + return Base::isReachableFromEntry(A); + } + + /// Record that the critical edge (FromBB, ToBB) has been + /// split with NewBB. + /// This is best to use this method instead of directly update the + /// underlying information, because this helps mitigating the + /// number of time the DT information is invalidated. + /// + /// \note Do not use this method with regular edges. + /// + /// \note To benefit from the compile time improvement incurred by this + /// method, the users of this method have to limit the queries to the DT + /// interface between two edges splitting. In other words, they have to + /// pack the splitting of critical edges as much as possible. + void recordSplitCriticalEdge(MachineBasicBlock *FromBB, + MachineBasicBlock *ToBB, + MachineBasicBlock *NewBB) { + bool Inserted = NewBBs.insert(NewBB).second; + (void)Inserted; + assert(Inserted && + "A basic block inserted via edge splitting cannot appear twice"); + CriticalEdgesToSplit.push_back({FromBB, ToBB, NewBB}); + } }; /// \brief Analysis pass which computes a \c MachineDominatorTree. diff --git a/llvm/include/llvm/CodeGen/MachineSSAContext.h b/llvm/include/llvm/CodeGen/MachineSSAContext.h index 0e4304f69380f..b70450c19f283 100644 --- a/llvm/include/llvm/CodeGen/MachineSSAContext.h +++ b/llvm/include/llvm/CodeGen/MachineSSAContext.h @@ -24,6 +24,12 @@ class MachineInstr; class MachineFunction; class Register; +inline unsigned succ_size(const MachineBasicBlock *BB) { + return BB->succ_size(); +} +inline unsigned pred_size(const MachineBasicBlock *BB) { + return BB->pred_size(); +} inline auto instrs(const MachineBasicBlock &BB) { return BB.instrs(); } template <> struct GenericSSATraits { diff --git a/llvm/lib/Analysis/DomTreeUpdater.cpp b/llvm/lib/Analysis/DomTreeUpdater.cpp index 588944428616c..8731c3faf21e5 100644 --- a/llvm/lib/Analysis/DomTreeUpdater.cpp +++ b/llvm/lib/Analysis/DomTreeUpdater.cpp @@ -28,13 +28,6 @@ template void GenericDomTreeUpdater::recalculate(Function &F); -template void -GenericDomTreeUpdater:: - applyUpdatesImpl(); -template void -GenericDomTreeUpdater:: - applyUpdatesImpl(); - bool DomTreeUpdater::forceFlushDeletedBB() { if (DeletedBBs.empty()) return false; diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 575ef10d9bbde..3072edc5088e2 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -1791,7 +1791,7 @@ void AsmPrinter::emitFunctionBody() { MDT = MDTWrapper ? &MDTWrapper->getDomTree() : nullptr; if (!MDT) { OwnedMDT = std::make_unique(); - OwnedMDT->recalculate(*MF); + OwnedMDT->getBase().recalculate(*MF); MDT = OwnedMDT.get(); } @@ -1800,7 +1800,7 @@ void AsmPrinter::emitFunctionBody() { MLI = MLIWrapper ? &MLIWrapper->getLI() : nullptr; if (!MLI) { OwnedMLI = std::make_unique(); - OwnedMLI->analyze(*MDT); + OwnedMLI->analyze(MDT->getBase()); MLI = OwnedMLI.get(); } } diff --git a/llvm/lib/CodeGen/LazyMachineBlockFrequencyInfo.cpp b/llvm/lib/CodeGen/LazyMachineBlockFrequencyInfo.cpp index 6fd84646009be..2561f2e5c9bbb 100644 --- a/llvm/lib/CodeGen/LazyMachineBlockFrequencyInfo.cpp +++ b/llvm/lib/CodeGen/LazyMachineBlockFrequencyInfo.cpp @@ -74,13 +74,13 @@ LazyMachineBlockFrequencyInfoPass::calculateIfNotAvailable() const { if (!MDT) { LLVM_DEBUG(dbgs() << "Building DominatorTree on the fly\n"); OwnedMDT = std::make_unique(); - OwnedMDT->recalculate(*MF); + OwnedMDT->getBase().recalculate(*MF); MDT = OwnedMDT.get(); } // Generate LoopInfo from it. OwnedMLI = std::make_unique(); - OwnedMLI->analyze(*MDT); + OwnedMLI->analyze(MDT->getBase()); MLI = OwnedMLI.get(); } diff --git a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp index ade67bb545d16..748dd0ca9858e 100644 --- a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp +++ b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp @@ -2782,7 +2782,7 @@ void InstrRefBasedLDV::BlockPHIPlacement( // Apply IDF calculator to the designated set of location defs, storing // required PHIs into PHIBlocks. Uses the dominator tree stored in the // InstrRefBasedLDV object. - IDFCalculatorBase IDF(*DomTree); + IDFCalculatorBase IDF(DomTree->getBase()); IDF.setLiveInBlocks(AllBlocks); IDF.setDefiningBlocks(DefBlocks); diff --git a/llvm/lib/CodeGen/LiveDebugValues/LiveDebugValues.cpp b/llvm/lib/CodeGen/LiveDebugValues/LiveDebugValues.cpp index a7f089928f84d..a2b1662271940 100644 --- a/llvm/lib/CodeGen/LiveDebugValues/LiveDebugValues.cpp +++ b/llvm/lib/CodeGen/LiveDebugValues/LiveDebugValues.cpp @@ -112,7 +112,7 @@ bool LiveDebugValues::runOnMachineFunction(MachineFunction &MF) { MachineDominatorTree *DomTree = nullptr; if (InstrRefBased) { DomTree = &MDT; - MDT.recalculate(MF); + MDT.calculate(MF); TheImpl = &*InstrRefImpl; } diff --git a/llvm/lib/CodeGen/MachineBasicBlock.cpp b/llvm/lib/CodeGen/MachineBasicBlock.cpp index 5ac6472a01e9f..5d06af3ebf336 100644 --- a/llvm/lib/CodeGen/MachineBasicBlock.cpp +++ b/llvm/lib/CodeGen/MachineBasicBlock.cpp @@ -16,13 +16,11 @@ #include "llvm/CodeGen/LiveIntervals.h" #include "llvm/CodeGen/LivePhysRegs.h" #include "llvm/CodeGen/LiveVariables.h" -#include "llvm/CodeGen/MachineDomTreeUpdater.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineLoopInfo.h" -#include "llvm/CodeGen/MachinePostDominators.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/SlotIndexes.h" #include "llvm/CodeGen/TargetInstrInfo.h" @@ -1148,7 +1146,7 @@ class SlotIndexUpdateDelegate : public MachineFunction::Delegate { MachineBasicBlock *MachineBasicBlock::SplitCriticalEdge( MachineBasicBlock *Succ, Pass *P, MachineFunctionAnalysisManager *MFAM, - std::vector> *LiveInSets, MachineDomTreeUpdater *MDTU) { + std::vector> *LiveInSets) { assert((P || MFAM) && "Need a way to get analysis results!"); if (!canSplitCriticalEdge(Succ)) return nullptr; @@ -1348,8 +1346,8 @@ MachineBasicBlock *MachineBasicBlock::SplitCriticalEdge( LIS->repairIntervalsInRange(this, getFirstTerminator(), end(), UsedRegs); } - if (MDTU) - MDTU->splitCriticalEdge(this, Succ, NMBB); + if (auto *MDT = GET_RESULT(MachineDominatorTree, getDomTree, )) + MDT->recordSplitCriticalEdge(this, Succ, NMBB); if (MachineLoopInfo *MLI = GET_RESULT(MachineLoop, getLI, Info)) if (MachineLoop *TIL = MLI->getLoopFor(this)) { diff --git a/llvm/lib/CodeGen/MachineDomTreeUpdater.cpp b/llvm/lib/CodeGen/MachineDomTreeUpdater.cpp index 72e4be0165bf8..a640fc0079038 100644 --- a/llvm/lib/CodeGen/MachineDomTreeUpdater.cpp +++ b/llvm/lib/CodeGen/MachineDomTreeUpdater.cpp @@ -25,13 +25,6 @@ GenericDomTreeUpdater::recalculate(MachineFunction &MF); -template void GenericDomTreeUpdater< - MachineDomTreeUpdater, MachineDominatorTree, - MachinePostDominatorTree>::applyUpdatesImpl(); -template void GenericDomTreeUpdater< - MachineDomTreeUpdater, MachineDominatorTree, - MachinePostDominatorTree>::applyUpdatesImpl(); - bool MachineDomTreeUpdater::forceFlushDeletedBB() { if (DeletedBBs.empty()) return false; diff --git a/llvm/lib/CodeGen/MachineDominanceFrontier.cpp b/llvm/lib/CodeGen/MachineDominanceFrontier.cpp index ed69ed931c5cb..6a8ede4feb937 100644 --- a/llvm/lib/CodeGen/MachineDominanceFrontier.cpp +++ b/llvm/lib/CodeGen/MachineDominanceFrontier.cpp @@ -38,7 +38,8 @@ char &llvm::MachineDominanceFrontierID = MachineDominanceFrontier::ID; bool MachineDominanceFrontier::runOnMachineFunction(MachineFunction &) { releaseMemory(); - Base.analyze(getAnalysis().getDomTree()); + Base.analyze( + getAnalysis().getDomTree().getBase()); return false; } diff --git a/llvm/lib/CodeGen/MachineDominators.cpp b/llvm/lib/CodeGen/MachineDominators.cpp index 67a91c87bb1bc..a2cc8fdfa7c9f 100644 --- a/llvm/lib/CodeGen/MachineDominators.cpp +++ b/llvm/lib/CodeGen/MachineDominators.cpp @@ -95,6 +95,12 @@ MachineDominatorTreeWrapperPass::MachineDominatorTreeWrapperPass() *PassRegistry::getPassRegistry()); } +void MachineDominatorTree::calculate(MachineFunction &F) { + CriticalEdgesToSplit.clear(); + NewBBs.clear(); + recalculate(F); +} + char &llvm::MachineDominatorsID = MachineDominatorTreeWrapperPass::ID; bool MachineDominatorTreeWrapperPass::runOnMachineFunction(MachineFunction &F) { @@ -115,3 +121,71 @@ void MachineDominatorTreeWrapperPass::print(raw_ostream &OS, if (DT) DT->print(OS); } + +void MachineDominatorTree::applySplitCriticalEdges() const { + // Bail out early if there is nothing to do. + if (CriticalEdgesToSplit.empty()) + return; + + // For each element in CriticalEdgesToSplit, remember whether or not element + // is the new immediate domminator of its successor. The mapping is done by + // index, i.e., the information for the ith element of CriticalEdgesToSplit is + // the ith element of IsNewIDom. + SmallBitVector IsNewIDom(CriticalEdgesToSplit.size(), true); + size_t Idx = 0; + + // Collect all the dominance properties info, before invalidating + // the underlying DT. + for (CriticalEdge &Edge : CriticalEdgesToSplit) { + // Update dominator information. + MachineBasicBlock *Succ = Edge.ToBB; + MachineDomTreeNode *SuccDTNode = Base::getNode(Succ); + + for (MachineBasicBlock *PredBB : Succ->predecessors()) { + if (PredBB == Edge.NewBB) + continue; + // If we are in this situation: + // FromBB1 FromBB2 + // + + + // + + + + + // + + + + + // ... Split1 Split2 ... + // + + + // + + + // + + // Succ + // Instead of checking the domiance property with Split2, we check it with + // FromBB2 since Split2 is still unknown of the underlying DT structure. + if (NewBBs.count(PredBB)) { + assert(PredBB->pred_size() == 1 && "A basic block resulting from a " + "critical edge split has more " + "than one predecessor!"); + PredBB = *PredBB->pred_begin(); + } + if (!Base::dominates(SuccDTNode, Base::getNode(PredBB))) { + IsNewIDom[Idx] = false; + break; + } + } + ++Idx; + } + + // Now, update DT with the collected dominance properties info. + Idx = 0; + for (CriticalEdge &Edge : CriticalEdgesToSplit) { + // We know FromBB dominates NewBB. + MachineDomTreeNode *NewDTNode = + const_cast(this)->Base::addNewBlock( + Edge.NewBB, Edge.FromBB); + + // If all the other predecessors of "Succ" are dominated by "Succ" itself + // then the new block is the new immediate dominator of "Succ". Otherwise, + // the new block doesn't dominate anything. + if (IsNewIDom[Idx]) + const_cast(this)->Base::changeImmediateDominator( + Base::getNode(Edge.ToBB), NewDTNode); + ++Idx; + } + NewBBs.clear(); + CriticalEdgesToSplit.clear(); +} diff --git a/llvm/lib/CodeGen/MachineLICM.cpp b/llvm/lib/CodeGen/MachineLICM.cpp index d1d5509dc482a..d21059189b184 100644 --- a/llvm/lib/CodeGen/MachineLICM.cpp +++ b/llvm/lib/CodeGen/MachineLICM.cpp @@ -24,7 +24,6 @@ #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" -#include "llvm/CodeGen/MachineDomTreeUpdater.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" @@ -134,7 +133,7 @@ namespace { AliasAnalysis *AA = nullptr; // Alias analysis info. MachineBlockFrequencyInfo *MBFI = nullptr; // Machine block frequncy info MachineLoopInfo *MLI = nullptr; // Current MachineLoopInfo - MachineDomTreeUpdater *MDTU = nullptr; // Wraps current dominator tree + MachineDominatorTree *DT = nullptr; // Machine dominator tree for the cur loop // State that is updated as we process loops bool Changed = false; // True if a loop is changed. @@ -376,9 +375,7 @@ bool MachineLICMImpl::run(MachineFunction &MF) { .getManager() .getResult(MF.getFunction()) : &LegacyPass->getAnalysis().getAAResults(); - MachineDomTreeUpdater DTU(GET_RESULT(MachineDominatorTree, getDomTree, ), - MachineDomTreeUpdater::UpdateStrategy::Lazy); - MDTU = &DTU; + DT = GET_RESULT(MachineDominatorTree, getDomTree, ); MLI = GET_RESULT(MachineLoop, getLI, Info); MBFI = DisableHoistingToHotterBlocks != UseBFI::None ? GET_RESULT(MachineBlockFrequency, getMBFI, Info) @@ -424,7 +421,7 @@ bool MachineLICMImpl::run(MachineFunction &MF) { else { // CSEMap is initialized for loop header when the first instruction is // being hoisted. - MachineDomTreeNode *N = MDTU->getDomTree().getNode(CurLoop->getHeader()); + MachineDomTreeNode *N = DT->getNode(CurLoop->getHeader()); FirstInLoop = true; HoistOutOfLoop(N, CurLoop, CurPreheader); CSEMap.clear(); @@ -767,7 +764,7 @@ bool MachineLICMImpl::IsGuaranteedToExecute(MachineBasicBlock *BB, SmallVector CurrentLoopExitingBlocks; CurLoop->getExitingBlocks(CurrentLoopExitingBlocks); for (MachineBasicBlock *CurrentLoopExitingBlock : CurrentLoopExitingBlocks) - if (!MDTU->getDomTree().dominates(BB, CurrentLoopExitingBlock)) { + if (!DT->dominates(BB, CurrentLoopExitingBlock)) { SpeculationState = SpeculateTrue; return false; } @@ -1603,7 +1600,7 @@ bool MachineLICMImpl::MayCSE(MachineInstr *MI) { unsigned Opcode = MI->getOpcode(); for (auto &Map : CSEMap) { // Check this CSEMap's preheader dominates MI's basic block. - if (MDTU->getDomTree().dominates(Map.first, MI->getParent())) { + if (DT->dominates(Map.first, MI->getParent())) { DenseMap>::iterator CI = Map.second.find(Opcode); // Do not CSE implicit_def so ProcessImplicitDefs can properly propagate @@ -1671,7 +1668,7 @@ unsigned MachineLICMImpl::Hoist(MachineInstr *MI, MachineBasicBlock *Preheader, bool HasCSEDone = false; for (auto &Map : CSEMap) { // Check this CSEMap's preheader dominates MI's basic block. - if (MDTU->getDomTree().dominates(Map.first, MI->getParent())) { + if (DT->dominates(Map.first, MI->getParent())) { DenseMap>::iterator CI = Map.second.find(Opcode); if (CI != Map.second.end()) { @@ -1735,7 +1732,7 @@ MachineLICMImpl::getCurPreheader(MachineLoop *CurLoop, } CurPreheader = Pred->SplitCriticalEdge(CurLoop->getHeader(), LegacyPass, - MFAM, nullptr, MDTU); + MFAM, nullptr); if (!CurPreheader) { CurPreheader = reinterpret_cast(-1); return nullptr; diff --git a/llvm/lib/CodeGen/MachineLoopInfo.cpp b/llvm/lib/CodeGen/MachineLoopInfo.cpp index d6906bacde0e2..b0d74ecd6a854 100644 --- a/llvm/lib/CodeGen/MachineLoopInfo.cpp +++ b/llvm/lib/CodeGen/MachineLoopInfo.cpp @@ -77,7 +77,7 @@ bool MachineLoopInfo::invalidate( void MachineLoopInfo::calculate(MachineDominatorTree &MDT) { releaseMemory(); - analyze(MDT); + analyze(MDT.getBase()); } void MachineLoopInfoWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const { diff --git a/llvm/lib/CodeGen/MachineSink.cpp b/llvm/lib/CodeGen/MachineSink.cpp index bdf1ef68109d3..7d0bedab7cdab 100644 --- a/llvm/lib/CodeGen/MachineSink.cpp +++ b/llvm/lib/CodeGen/MachineSink.cpp @@ -30,7 +30,6 @@ #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" #include "llvm/CodeGen/MachineBranchProbabilityInfo.h" #include "llvm/CodeGen/MachineCycleAnalysis.h" -#include "llvm/CodeGen/MachineDomTreeUpdater.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" @@ -748,11 +747,8 @@ bool MachineSinking::runOnMachineFunction(MachineFunction &MF) { MadeChange |= ProcessBlock(MBB); // If we have anything we marked as toSplit, split it now. - MachineDomTreeUpdater MDTU(DT, PDT, - MachineDomTreeUpdater::UpdateStrategy::Lazy); for (const auto &Pair : ToSplit) { - auto NewSucc = - Pair.first->SplitCriticalEdge(Pair.second, *this, nullptr, &MDTU); + auto NewSucc = Pair.first->SplitCriticalEdge(Pair.second, *this); if (NewSucc != nullptr) { LLVM_DEBUG(dbgs() << " *** Splitting critical edge: " << printMBBReference(*Pair.first) << " -- " diff --git a/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp b/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp index a4b78c1c75ceb..7548fc8141ec5 100644 --- a/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp +++ b/llvm/lib/CodeGen/MachineUniformityAnalysis.cpp @@ -199,7 +199,8 @@ void MachineUniformityAnalysisPass::getAnalysisUsage(AnalysisUsage &AU) const { } bool MachineUniformityAnalysisPass::runOnMachineFunction(MachineFunction &MF) { - auto &DomTree = getAnalysis().getDomTree(); + auto &DomTree = + getAnalysis().getDomTree().getBase(); auto &CI = getAnalysis().getCycleInfo(); // FIXME: Query TTI::hasBranchDivergence. -run-pass seems to end up with a // default NoTTI diff --git a/llvm/lib/CodeGen/PHIElimination.cpp b/llvm/lib/CodeGen/PHIElimination.cpp index b71e5b8538689..e5f40771eda86 100644 --- a/llvm/lib/CodeGen/PHIElimination.cpp +++ b/llvm/lib/CodeGen/PHIElimination.cpp @@ -22,7 +22,6 @@ #include "llvm/CodeGen/LiveIntervals.h" #include "llvm/CodeGen/LiveVariables.h" #include "llvm/CodeGen/MachineBasicBlock.h" -#include "llvm/CodeGen/MachineDomTreeUpdater.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" @@ -91,8 +90,7 @@ class PHIEliminationImpl { /// Split critical edges where necessary for good coalescer performance. bool SplitPHIEdges(MachineFunction &MF, MachineBasicBlock &MBB, MachineLoopInfo *MLI, - std::vector> *LiveInSets, - MachineDomTreeUpdater &MDTU); + std::vector> *LiveInSets); // These functions are temporary abstractions around LiveVariables and // LiveIntervals, so they can go away when LiveVariables does. @@ -205,16 +203,6 @@ void PHIElimination::getAnalysisUsage(AnalysisUsage &AU) const { bool PHIEliminationImpl::run(MachineFunction &MF) { MRI = &MF.getRegInfo(); - MachineDominatorTree *MDT = nullptr; - if (P) { - auto *MDTWrapper = - P->getAnalysisIfAvailable(); - MDT = MDTWrapper ? &MDTWrapper->getDomTree() : nullptr; - } else { - MDT = MFAM->getCachedResult(MF); - } - MachineDomTreeUpdater MDTU(MDT, MachineDomTreeUpdater::UpdateStrategy::Lazy); - bool Changed = false; // Split critical edges to help the coalescer. @@ -249,8 +237,7 @@ bool PHIEliminationImpl::run(MachineFunction &MF) { } for (auto &MBB : MF) - Changed |= - SplitPHIEdges(MF, MBB, MLI, (LV ? &LiveInSets : nullptr), MDTU); + Changed |= SplitPHIEdges(MF, MBB, MLI, (LV ? &LiveInSets : nullptr)); } // This pass takes the function out of SSA form. @@ -281,6 +268,10 @@ bool PHIEliminationImpl::run(MachineFunction &MF) { MF.deleteMachineInstr(I.first); } + // TODO: we should use the incremental DomTree updater here. + if (Changed && MDT) + MDT->getBase().recalculate(MF); + LoweredPHIs.clear(); ImpDefs.clear(); VRegPHIUseCount.clear(); @@ -761,7 +752,7 @@ void PHIEliminationImpl::analyzePHINodes(const MachineFunction &MF) { bool PHIEliminationImpl::SplitPHIEdges( MachineFunction &MF, MachineBasicBlock &MBB, MachineLoopInfo *MLI, - std::vector> *LiveInSets, MachineDomTreeUpdater &MDTU) { + std::vector> *LiveInSets) { if (MBB.empty() || !MBB.front().isPHI() || MBB.isEHPad()) return false; // Quick exit for basic blocks without PHIs. @@ -828,8 +819,8 @@ bool PHIEliminationImpl::SplitPHIEdges( } if (!ShouldSplit && !SplitAllCriticalEdges) continue; - if (!(P ? PreMBB->SplitCriticalEdge(&MBB, *P, LiveInSets, &MDTU) - : PreMBB->SplitCriticalEdge(&MBB, *MFAM, LiveInSets, &MDTU))) { + if (!(P ? PreMBB->SplitCriticalEdge(&MBB, *P, LiveInSets) + : PreMBB->SplitCriticalEdge(&MBB, *MFAM, LiveInSets))) { LLVM_DEBUG(dbgs() << "Failed to split critical edge.\n"); continue; } diff --git a/llvm/lib/CodeGen/XRayInstrumentation.cpp b/llvm/lib/CodeGen/XRayInstrumentation.cpp index fa0ce6db1d6f7..8af16fa6249f4 100644 --- a/llvm/lib/CodeGen/XRayInstrumentation.cpp +++ b/llvm/lib/CodeGen/XRayInstrumentation.cpp @@ -175,7 +175,7 @@ bool XRayInstrumentation::runOnMachineFunction(MachineFunction &MF) { auto *MDT = MDTWrapper ? &MDTWrapper->getDomTree() : nullptr; MachineDominatorTree ComputedMDT; if (!MDT) { - ComputedMDT.recalculate(MF); + ComputedMDT.getBase().recalculate(MF); MDT = &ComputedMDT; } @@ -184,7 +184,7 @@ bool XRayInstrumentation::runOnMachineFunction(MachineFunction &MF) { auto *MLI = MLIWrapper ? &MLIWrapper->getLI() : nullptr; MachineLoopInfo ComputedMLI; if (!MLI) { - ComputedMLI.analyze(*MDT); + ComputedMLI.analyze(MDT->getBase()); MLI = &ComputedMLI; } diff --git a/llvm/lib/Target/AMDGPU/SILateBranchLowering.cpp b/llvm/lib/Target/AMDGPU/SILateBranchLowering.cpp index d02173f57ee37..8dec5ded848c6 100644 --- a/llvm/lib/Target/AMDGPU/SILateBranchLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SILateBranchLowering.cpp @@ -113,7 +113,7 @@ static void splitBlock(MachineBasicBlock &MBB, MachineInstr &MI, DTUpdates.push_back({DomTreeT::Delete, &MBB, Succ}); } DTUpdates.push_back({DomTreeT::Insert, &MBB, SplitBB}); - MDT->applyUpdates(DTUpdates); + MDT->getBase().applyUpdates(DTUpdates); } void SILateBranchLowering::expandChainCall(MachineInstr &MI) { @@ -141,7 +141,7 @@ void SILateBranchLowering::earlyTerm(MachineInstr &MI, splitBlock(MBB, *BranchMI, MDT); MBB.addSuccessor(EarlyExitBlock); - MDT->insertEdge(&MBB, EarlyExitBlock); + MDT->getBase().insertEdge(&MBB, EarlyExitBlock); } bool SILateBranchLowering::runOnMachineFunction(MachineFunction &MF) { @@ -237,7 +237,7 @@ bool SILateBranchLowering::runOnMachineFunction(MachineFunction &MF) { } MBB->addSuccessor(EmptyMBBAtEnd); - MDT->insertEdge(MBB, EmptyMBBAtEnd); + MDT->getBase().insertEdge(MBB, EmptyMBBAtEnd); BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(AMDGPU::S_BRANCH)) .addMBB(EmptyMBBAtEnd); MI->eraseFromParent(); diff --git a/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp b/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp index 5805ec658fd61..3c76da4085275 100644 --- a/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp +++ b/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp @@ -486,7 +486,7 @@ bool PhiLoweringHelper::lowerPhis() { if (Vreg1Phis.empty()) return false; - DT->updateDFSNumbers(); + DT->getBase().updateDFSNumbers(); MachineBasicBlock *PrevMBB = nullptr; for (MachineInstr *MI : Vreg1Phis) { MachineBasicBlock &MBB = *MI->getParent(); diff --git a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp index 9fbb847da2af1..fba1d0c026958 100644 --- a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp +++ b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp @@ -786,7 +786,7 @@ MachineBasicBlock *SIWholeQuadMode::splitBlock(MachineBasicBlock *BB, } DTUpdates.push_back({DomTreeT::Insert, BB, SplitBB}); if (MDT) - MDT->applyUpdates(DTUpdates); + MDT->getBase().applyUpdates(DTUpdates); if (PDT) PDT->applyUpdates(DTUpdates); diff --git a/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp b/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp index a35f7a3350f8c..48acd9da9587f 100644 --- a/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp +++ b/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp @@ -413,7 +413,7 @@ void HexagonFrameLowering::findShrunkPrologEpilog(MachineFunction &MF, auto &HRI = *MF.getSubtarget().getRegisterInfo(); MachineDominatorTree MDT; - MDT.recalculate(MF); + MDT.calculate(MF); MachinePostDominatorTree MPT; MPT.recalculate(MF); diff --git a/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp b/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp index ab4c70aee486a..ea1b6e97aa32d 100644 --- a/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp +++ b/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp @@ -277,7 +277,8 @@ bool X86FlagsCopyLoweringPass::runOnMachineFunction(MachineFunction &MF) { if (MDTWrapper) { MDT = &MDTWrapper->getDomTree(); } else { - OwnedMDT = std::make_unique(MF); + OwnedMDT = std::make_unique(); + OwnedMDT->getBase().recalculate(MF); MDT = OwnedMDT.get(); } diff --git a/llvm/tools/llvm-reduce/deltas/ReduceInstructionsMIR.cpp b/llvm/tools/llvm-reduce/deltas/ReduceInstructionsMIR.cpp index 40bc6b180fb88..5f0697f5aaad7 100644 --- a/llvm/tools/llvm-reduce/deltas/ReduceInstructionsMIR.cpp +++ b/llvm/tools/llvm-reduce/deltas/ReduceInstructionsMIR.cpp @@ -65,7 +65,7 @@ static bool shouldNotRemoveInstruction(const TargetInstrInfo &TII, static void extractInstrFromFunction(Oracle &O, MachineFunction &MF) { MachineDominatorTree MDT; - MDT.recalculate(MF); + MDT.calculate(MF); auto MRI = &MF.getRegInfo(); SetVector ToDelete; diff --git a/llvm/unittests/Analysis/DomTreeUpdaterTest.cpp b/llvm/unittests/Analysis/DomTreeUpdaterTest.cpp index cabfc2aba57cf..0777bbe3887bc 100644 --- a/llvm/unittests/Analysis/DomTreeUpdaterTest.cpp +++ b/llvm/unittests/Analysis/DomTreeUpdaterTest.cpp @@ -15,7 +15,6 @@ #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/Support/SourceMgr.h" -#include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "gtest/gtest.h" #include @@ -788,45 +787,3 @@ TEST(DomTreeUpdater, LazyUpdateDeduplicationTest) { DTU.applyUpdates({{DominatorTree::Insert, BB0, BB2}}); ASSERT_TRUE(DTU.getDomTree().verify()); } - -TEST(DomTreeUpdater, CriticalEdgeSplitTest) { - StringRef FuncName = "f"; - StringRef ModuleString = R"( -declare void @use1(i32) - -define void @f(i32 %i, i1 %c) { -entry: - %A = icmp eq i32 %i, 0 ; [#uses=1] - br i1 %A, label %brtrue, label %brfalse - -brtrue: ; preds = %entry - call void @use1( i32 %i ) - br label %brfalse - -brfalse: ; preds = %brtrue, %entry - call void @use1( i32 %i ) - ret void -} - )"; - // Make the module. - LLVMContext Context; - std::unique_ptr M = makeLLVMModule(Context, ModuleString); - Function *F = M->getFunction(FuncName); - - // Make the DTU. - DominatorTree DT(*F); - DomTreeUpdater DTU(&DT, nullptr, DomTreeUpdater::UpdateStrategy::Lazy); - ASSERT_TRUE(DTU.getDomTree().verify()); - - CriticalEdgeSplittingOptions Opts; - SplitAllCriticalEdges(*F, Opts); - - Function::iterator FI = F->begin(); - BasicBlock *BBEntry = &*FI++; - BasicBlock *SplitB = &*FI++; - [[maybe_unused]] BasicBlock *BBTrue = &*FI++; - BasicBlock *BBFalse = &*FI++; - DTU.splitCriticalEdge(BBEntry, BBFalse, SplitB); - DominatorTree NewDT(*F); - ASSERT_FALSE(NewDT.compare(DTU.getDomTree())); -} diff --git a/llvm/unittests/Target/WebAssembly/WebAssemblyExceptionInfoTest.cpp b/llvm/unittests/Target/WebAssembly/WebAssemblyExceptionInfoTest.cpp index 223a5714f62f2..9bbbb352efdce 100644 --- a/llvm/unittests/Target/WebAssembly/WebAssemblyExceptionInfoTest.cpp +++ b/llvm/unittests/Target/WebAssembly/WebAssemblyExceptionInfoTest.cpp @@ -167,8 +167,8 @@ body: | WebAssemblyExceptionInfo WEI; MachineDominatorTree MDT; MachineDominanceFrontier MDF; - MDT.recalculate(*MF); - MDF.getBase().analyze(MDT); + MDT.calculate(*MF); + MDF.getBase().analyze(MDT.getBase()); WEI.recalculate(*MF, MDT, MDF); // Exception info structure: @@ -341,8 +341,8 @@ body: | WebAssemblyExceptionInfo WEI; MachineDominatorTree MDT; MachineDominanceFrontier MDF; - MDT.recalculate(*MF); - MDF.getBase().analyze(MDT); + MDT.calculate(*MF); + MDF.getBase().analyze(MDT.getBase()); WEI.recalculate(*MF, MDT, MDF); // Exception info structure: