Skip to content

[VPlan] Thread plan to VPBuilder (NFC) #125364

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

[VPlan] Thread plan to VPBuilder (NFC) #125364

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 10 additions & 15 deletions llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
#define LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONPLANNER_H

#include "VPlan.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/Support/InstructionCost.h"

namespace llvm {
Expand All @@ -46,6 +45,7 @@ struct VFRange;
class VPBuilder {
VPBasicBlock *BB = nullptr;
VPBasicBlock::iterator InsertPt = VPBasicBlock::iterator();
[[maybe_unused]] VPlan &Plan;

/// Insert \p VPI in BB at InsertPt if BB is set.
template <typename T> T *tryInsertInstruction(T *R) {
Expand All @@ -67,10 +67,15 @@ class VPBuilder {
}

public:
VPBuilder() = default;
VPBuilder(VPBasicBlock *InsertBB) { setInsertPoint(InsertBB); }
VPBuilder(VPRecipeBase *InsertPt) { setInsertPoint(InsertPt); }
VPBuilder(VPBasicBlock *TheBB, VPBasicBlock::iterator IP) {
VPBuilder(VPlan &Plan) : Plan(Plan) {}
VPBuilder(VPlan &Plan, VPBasicBlock *InsertBB) : Plan(Plan) {
setInsertPoint(InsertBB);
}
VPBuilder(VPlan &Plan, VPRecipeBase *InsertPt) : Plan(Plan) {
setInsertPoint(InsertPt);
}
VPBuilder(VPlan &Plan, VPBasicBlock *TheBB, VPBasicBlock::iterator IP)
: Plan(Plan) {
setInsertPoint(TheBB, IP);
}

Expand All @@ -84,13 +89,6 @@ class VPBuilder {
VPBasicBlock *getInsertBlock() const { return BB; }
VPBasicBlock::iterator getInsertPoint() const { return InsertPt; }

/// Create a VPBuilder to insert after \p R.
static VPBuilder getToInsertAfter(VPRecipeBase *R) {
VPBuilder B;
B.setInsertPoint(R->getParent(), std::next(R->getIterator()));
return B;
}

/// InsertPoint - A saved insertion point.
class VPInsertPoint {
VPBasicBlock *Block = nullptr;
Expand Down Expand Up @@ -394,9 +392,6 @@ class LoopVectorizationPlanner {
/// Profitable vector factors.
SmallVector<VectorizationFactor, 8> ProfitableVFs;

/// A builder used to construct the current plan.
VPBuilder Builder;

/// Computes the cost of \p Plan for vectorization factor \p VF.
///
/// The current implementation requires access to the
Expand Down
27 changes: 13 additions & 14 deletions llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8986,7 +8986,7 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW,
VPBasicBlock *Header = TopRegion->getEntryBasicBlock();
Header->insert(CanonicalIVPHI, Header->begin());

VPBuilder Builder(TopRegion->getExitingBasicBlock());
VPBuilder Builder(Plan, TopRegion->getExitingBasicBlock());
// Add a VPInstruction to increment the scalar canonical IV by VF * UF.
auto *CanonicalIVIncrement = Builder.createOverflowingOp(
Instruction::Add, {CanonicalIVPHI, &Plan.getVFxUF()}, {HasNUW, false}, DL,
Expand Down Expand Up @@ -9046,9 +9046,9 @@ static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan,
auto *MiddleVPBB = cast<VPBasicBlock>(ScalarPH->getSinglePredecessor());
VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
VPBuilder VectorPHBuilder(
cast<VPBasicBlock>(VectorRegion->getSinglePredecessor()));
VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
VPBuilder ScalarPHBuilder(ScalarPH);
Plan, cast<VPBasicBlock>(VectorRegion->getSinglePredecessor()));
VPBuilder MiddleBuilder(Plan, MiddleVPBB, MiddleVPBB->getFirstNonPhi());
VPBuilder ScalarPHBuilder(Plan, ScalarPH);
VPValue *OneVPV = Plan.getOrAddLiveIn(
ConstantInt::get(Plan.getCanonicalIV()->getScalarType(), 1));
for (VPRecipeBase &ScalarPhiR : *Plan.getScalarHeader()) {
Expand Down Expand Up @@ -9140,7 +9140,7 @@ addUsersInExitBlocks(VPlan &Plan,
return;

auto *MiddleVPBB = Plan.getMiddleBlock();
VPBuilder B(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
VPBuilder B(Plan, MiddleVPBB, MiddleVPBB->getFirstNonPhi());

// Introduce extract for exiting values and update the VPIRInstructions
// modeling the corresponding LCSSA phis.
Expand All @@ -9162,8 +9162,8 @@ static void addExitUsersForFirstOrderRecurrences(
VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
auto *ScalarPHVPBB = Plan.getScalarPreheader();
auto *MiddleVPBB = Plan.getMiddleBlock();
VPBuilder ScalarPHBuilder(ScalarPHVPBB);
VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
VPBuilder ScalarPHBuilder(Plan, ScalarPHVPBB);
VPBuilder MiddleBuilder(Plan, MiddleVPBB, MiddleVPBB->getFirstNonPhi());
VPValue *TwoVPV = Plan.getOrAddLiveIn(
ConstantInt::get(Plan.getCanonicalIV()->getScalarType(), 2));

Expand Down Expand Up @@ -9300,8 +9300,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
bool HasNUW = !IVUpdateMayOverflow || Style == TailFoldingStyle::None;
addCanonicalIVRecipes(*Plan, Legal->getWidestInductionType(), HasNUW, DL);

VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
Builder);
VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE);

// ---------------------------------------------------------------------------
// Pre-construction: record ingredients whose recipes we'll need to further
Expand Down Expand Up @@ -9357,7 +9356,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
// ingredients and fill a new VPBasicBlock.
if (VPBB != HeaderVPBB)
VPBB->setName(BB->getName());
Builder.setInsertPoint(VPBB);
RecipeBuilder.setInsertPoint(VPBB);

if (VPBB == HeaderVPBB)
RecipeBuilder.createHeaderMask();
Expand Down Expand Up @@ -9521,7 +9520,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
// Sink users of fixed-order recurrence past the recipe defining the previous
// value and introduce FirstOrderRecurrenceSplice VPInstructions.
if (!VPlanTransforms::runPass(VPlanTransforms::adjustFixedOrderRecurrences,
*Plan, Builder))
*Plan, RecipeBuilder.getIRBuilder()))
return nullptr;

if (useActiveLaneMask(Style)) {
Expand Down Expand Up @@ -9571,8 +9570,7 @@ VPlanPtr LoopVectorizationPlanner::buildVPlan(VFRange &Range) {

// Collect mapping of IR header phis to header phi recipes, to be used in
// addScalarResumePhis.
VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
Builder);
VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE);
for (auto &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
if (isa<VPCanonicalIVPHIRecipe>(&R))
continue;
Expand Down Expand Up @@ -9737,6 +9735,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
}
}
VPBasicBlock *LatchVPBB = VectorLoopRegion->getExitingBasicBlock();
VPBuilder Builder(*Plan);
Builder.setInsertPoint(&*LatchVPBB->begin());
VPBasicBlock::iterator IP = MiddleVPBB->getFirstNonPhi();
for (VPRecipeBase &R :
Expand Down Expand Up @@ -10248,7 +10247,7 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
m_Specific(VectorTC), m_SpecificInt(0)));
}))
return;
VPBuilder ScalarPHBuilder(MainScalarPH, MainScalarPH->begin());
VPBuilder ScalarPHBuilder(MainPlan, MainScalarPH, MainScalarPH->begin());
ScalarPHBuilder.createNaryOp(
VPInstruction::ResumePhi,
{VectorTC, MainPlan.getCanonicalIV()->getStartValue()}, {},
Expand Down
11 changes: 7 additions & 4 deletions llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
#include "LoopVectorizationPlanner.h"
#include "VPlan.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/PointerUnion.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/IR/IRBuilder.h"

Expand Down Expand Up @@ -66,7 +65,7 @@ class VPRecipeBuilder {

PredicatedScalarEvolution &PSE;

VPBuilder &Builder;
VPBuilder Builder;

/// When we if-convert we need to create edge masks. We have to cache values
/// so that we don't end up with exponential recursion/IR. Note that
Expand Down Expand Up @@ -155,9 +154,13 @@ class VPRecipeBuilder {
const TargetTransformInfo *TTI,
LoopVectorizationLegality *Legal,
LoopVectorizationCostModel &CM,
PredicatedScalarEvolution &PSE, VPBuilder &Builder)
PredicatedScalarEvolution &PSE)
: Plan(Plan), OrigLoop(OrigLoop), TLI(TLI), TTI(TTI), Legal(Legal),
CM(CM), PSE(PSE), Builder(Builder) {}
CM(CM), PSE(PSE), Builder(Plan) {}

void setInsertPoint(VPBasicBlock *VPBB) { Builder.setInsertPoint(VPBB); }

VPBuilder &getIRBuilder() { return Builder; }
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a bit confusing for me personally, since there is a class called IRBuilder that really does refer to LLVM IR. I'd prefer a different name here, something like getVPBuilder


std::optional<unsigned> getScalingForReduction(const Instruction *ExitInst) {
auto It = ScaledReductionMap.find(ExitInst);
Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/Transforms/Vectorize/VPlan.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -922,7 +922,7 @@ VPlanPtr VPlan::createInitialVPlan(Type *InductionTy,
// of the corresponding compare because they may have ended up with
// different line numbers and we want to avoid awkward line stepping while
// debugging. Eg. if the compare has got a line number inside the loop.
VPBuilder Builder(MiddleVPBB);
VPBuilder Builder(*Plan, MiddleVPBB);
VPValue *Cmp =
TailFolded
? Plan->getOrAddLiveIn(ConstantInt::getTrue(
Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ class PlainCFGBuilder {

public:
PlainCFGBuilder(Loop *Lp, LoopInfo *LI, VPlan &P)
: TheLoop(Lp), LI(LI), Plan(P) {}
: TheLoop(Lp), LI(LI), Plan(P), VPIRBuilder(Plan) {}

/// Build plain CFG for TheLoop and connects it to Plan's entry.
void buildPlainCFG();
Expand Down
21 changes: 11 additions & 10 deletions llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -592,7 +592,7 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) {
using namespace llvm::VPlanPatternMatch;
VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
bool HasOnlyVectorVFs = !Plan.hasScalarVFOnly();
VPBuilder Builder(HeaderVPBB, HeaderVPBB->getFirstNonPhi());
VPBuilder Builder(Plan, HeaderVPBB, HeaderVPBB->getFirstNonPhi());
for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
auto *PhiR = dyn_cast<VPWidenInductionRecipe>(&Phi);
if (!PhiR)
Expand Down Expand Up @@ -745,7 +745,7 @@ void VPlanTransforms::optimizeInductionExitUsers(
"predecessor must be the middle block");

VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
VPBuilder B(Plan.getMiddleBlock()->getTerminator());
VPBuilder B(Plan, Plan.getMiddleBlock()->getTerminator());
for (VPRecipeBase &R : *ExitVPBB) {
auto *ExitIRI = cast<VPIRInstruction>(&R);
if (!isa<PHINode>(ExitIRI->getInstruction()))
Expand Down Expand Up @@ -1503,7 +1503,7 @@ static VPActiveLaneMaskPHIRecipe *addVPLaneMaskPhiAndUpdateExitBranch(
// we have to take unrolling into account. Each part needs to start at
// Part * VF
auto *VecPreheader = Plan.getVectorPreheader();
VPBuilder Builder(VecPreheader);
VPBuilder Builder(Plan, VecPreheader);

// Create the ActiveLaneMask instruction using the correct start values.
VPValue *TC = Plan.getTripCount();
Expand Down Expand Up @@ -1622,7 +1622,8 @@ void VPlanTransforms::addActiveLaneMask(
LaneMask = addVPLaneMaskPhiAndUpdateExitBranch(
Plan, DataAndControlFlowWithoutRuntimeCheck);
} else {
VPBuilder B = VPBuilder::getToInsertAfter(WideCanonicalIV);
VPBuilder B(Plan, WideCanonicalIV->getParent(),
std::next(WideCanonicalIV->getIterator()));
LaneMask = B.createNaryOp(VPInstruction::ActiveLaneMask,
{WideCanonicalIV, Plan.getTripCount()}, nullptr,
"active.lane.mask");
Expand Down Expand Up @@ -1826,7 +1827,7 @@ bool VPlanTransforms::tryAddExplicitVectorLength(
// Create the ExplicitVectorLengthPhi recipe in the main loop.
auto *EVLPhi = new VPEVLBasedIVPHIRecipe(StartV, DebugLoc());
EVLPhi->insertAfter(CanonicalIVPHI);
VPBuilder Builder(Header, Header->getFirstNonPhi());
VPBuilder Builder(Plan, Header, Header->getFirstNonPhi());
// Compute original TC - IV as the AVL (application vector length).
VPValue *AVL = Builder.createNaryOp(
Instruction::Sub, {Plan.getTripCount(), EVLPhi}, DebugLoc(), "avl");
Expand Down Expand Up @@ -1907,7 +1908,7 @@ void VPlanTransforms::dropPoisonGeneratingRecipes(
// where the operands are disjoint or poison otherwise.
if (match(RecWithFlags, m_BinaryOr(m_VPValue(A), m_VPValue(B))) &&
RecWithFlags->isDisjoint()) {
VPBuilder Builder(RecWithFlags);
VPBuilder Builder(Plan, RecWithFlags);
VPInstruction *New = Builder.createOverflowingOp(
Instruction::Add, {A, B}, {false, false},
RecWithFlags->getDebugLoc());
Expand Down Expand Up @@ -2021,7 +2022,7 @@ void VPlanTransforms::createInterleaveGroups(
/*IsSigned=*/true);
VPValue *OffsetVPV = Plan.getOrAddLiveIn(
ConstantInt::get(IRInsertPos->getParent()->getContext(), -Offset));
VPBuilder B(InsertPos);
VPBuilder B(Plan, InsertPos);
Addr = InBounds ? B.createInBoundsPtrAdd(InsertPos->getAddr(), OffsetVPV)
: B.createPtrAdd(InsertPos->getAddr(), OffsetVPV);
}
Expand Down Expand Up @@ -2067,7 +2068,7 @@ void VPlanTransforms::handleUncountableEarlyExit(
BasicBlock *UncountableExitingBlock, VPRecipeBuilder &RecipeBuilder) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
auto *LatchVPBB = cast<VPBasicBlock>(LoopRegion->getExiting());
VPBuilder Builder(LatchVPBB->getTerminator());
VPBuilder Builder(Plan, LatchVPBB->getTerminator());
auto *MiddleVPBB = Plan.getMiddleBlock();
VPValue *IsEarlyExitTaken = nullptr;

Expand Down Expand Up @@ -2108,8 +2109,8 @@ void VPlanTransforms::handleUncountableEarlyExit(
VPBlockUtils::connectBlocks(VectorEarlyExitVPBB, VPEarlyExitBlock);

// Update the exit phis in the early exit block.
VPBuilder MiddleBuilder(NewMiddle);
VPBuilder EarlyExitB(VectorEarlyExitVPBB);
VPBuilder MiddleBuilder(Plan, NewMiddle);
VPBuilder EarlyExitB(Plan, VectorEarlyExitVPBB);
for (VPRecipeBase &R : *VPEarlyExitBlock) {
auto *ExitIRI = cast<VPIRInstruction>(&R);
auto *ExitPhi = dyn_cast<PHINode>(&ExitIRI->getInstruction());
Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ void UnrollState::unrollWidenInductionByUF(
FMFs = ID.getInductionBinOp()->getFastMathFlags();

VPValue *VectorStep = &Plan.getVF();
VPBuilder Builder(PH);
VPBuilder Builder(Plan, PH);
if (TypeInfo.inferScalarType(VectorStep) != IVTy) {
Instruction::CastOps CastOp =
IVTy->isFloatingPointTy() ? Instruction::UIToFP : Instruction::Trunc;
Expand Down