diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index d9f53c4146c28..07d8dbf213f86 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7826,7 +7826,7 @@ VPHeaderPHIRecipe *VPRecipeBuilder::tryToOptimizeInductionPHI( VPValue *Step = vputils::getOrCreateVPValueForSCEVExpr(Plan, II->getStep(), *PSE.getSE()); return new VPWidenPointerInductionRecipe( - Phi, Operands[0], Step, *II, + Phi, Operands[0], Step, &Plan.getVFxUF(), *II, LoopVectorizationPlanner::getDecisionAndClampRange( [&](ElementCount VF) { return CM.isScalarAfterVectorization(Phi, VF); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index ab5ff82a77208..f4163b0743a9a 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2034,25 +2034,30 @@ class VPWidenIntOrFpInductionRecipe : public VPWidenInductionRecipe { }; class VPWidenPointerInductionRecipe : public VPWidenInductionRecipe, - public VPUnrollPartAccessor<3> { + public VPUnrollPartAccessor<4> { bool IsScalarAfterVectorization; public: /// Create a new VPWidenPointerInductionRecipe for \p Phi with start value \p - /// Start. + /// Start and the number of elements unrolled \p NumUnrolledElems, typically + /// VF*UF. VPWidenPointerInductionRecipe(PHINode *Phi, VPValue *Start, VPValue *Step, + VPValue *NumUnrolledElems, const InductionDescriptor &IndDesc, bool IsScalarAfterVectorization, DebugLoc DL) : VPWidenInductionRecipe(VPDef::VPWidenPointerInductionSC, Phi, Start, Step, IndDesc, DL), - IsScalarAfterVectorization(IsScalarAfterVectorization) {} + IsScalarAfterVectorization(IsScalarAfterVectorization) { + addOperand(NumUnrolledElems); + } ~VPWidenPointerInductionRecipe() override = default; VPWidenPointerInductionRecipe *clone() override { return new VPWidenPointerInductionRecipe( cast(getUnderlyingInstr()), getOperand(0), getOperand(1), - getInductionDescriptor(), IsScalarAfterVectorization, getDebugLoc()); + getOperand(2), getInductionDescriptor(), IsScalarAfterVectorization, + getDebugLoc()); } VP_CLASSOF_IMPL(VPDef::VPWidenPointerInductionSC) @@ -2067,7 +2072,7 @@ class VPWidenPointerInductionRecipe : public VPWidenInductionRecipe, /// the first unrolled part, if it exists. Returns itself if unrolling did not /// take place. VPValue *getFirstUnrolledPartOperand() { - return getUnrollPart(*this) == 0 ? this : getOperand(2); + return getUnrollPart(*this) == 0 ? this : getOperand(3); } #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 22861eb1c7dfc..f3ce92911da3a 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -3545,8 +3545,7 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) { if (CurrentPart == 0) { // The recipe represents the first part of the pointer induction. Create the // GEP to increment the phi across all unrolled parts. - Value *NumUnrolledElems = - State.get(&getParent()->getPlan()->getVFxUF(), true); + Value *NumUnrolledElems = State.get(getOperand(2), true); Value *InductionGEP = GetElementPtrInst::Create( State.Builder.getInt8Ty(), NewPointerPhi, @@ -3582,7 +3581,7 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) { #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void VPWidenPointerInductionRecipe::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { - assert((getNumOperands() == 2 || getNumOperands() == 4) && + assert((getNumOperands() == 3 || getNumOperands() == 5) && "unexpected number of operands"); O << Indent << "EMIT "; printAsOperand(O, SlotTracker); @@ -3590,11 +3589,13 @@ void VPWidenPointerInductionRecipe::print(raw_ostream &O, const Twine &Indent, getStartValue()->printAsOperand(O, SlotTracker); O << ", "; getStepValue()->printAsOperand(O, SlotTracker); - if (getNumOperands() == 4) { - O << ", "; - getOperand(2)->printAsOperand(O, SlotTracker); + O << ", "; + getOperand(2)->printAsOperand(O, SlotTracker); + if (getNumOperands() == 5) { O << ", "; getOperand(3)->printAsOperand(O, SlotTracker); + O << ", "; + getOperand(4)->printAsOperand(O, SlotTracker); } } #endif