From bc7024ce7709f018a3390b5a1b8ad38f93d7b6e5 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 30 May 2025 10:02:17 +0100 Subject: [PATCH 1/3] [VPlan] Replace RdxDesc with RecurKind in VPReductionPHIRecipe (NFC). Replace VPReductionPHIRecipe with RecurKind in VPReductionPHIRecipe, as all VPlan analyses and codegen only require the recurrence kind. This enables creating new VPReductionPHIRecipe directly in LV, without needing to construction a whole RecurrenceDescriptor object. Depends on https://github.com/llvm/llvm-project/pull/141860 https://github.com/llvm/llvm-project/pull/141932 https://github.com/llvm/llvm-project/pull/142290 https://github.com/llvm/llvm-project/pull/142291 --- .../Transforms/Vectorize/LoopVectorize.cpp | 27 +++++++------------ llvm/lib/Transforms/Vectorize/VPlan.h | 20 +++++++++----- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 12 ++++----- .../Transforms/Vectorize/VPlanTransforms.cpp | 3 +-- 4 files changed, 28 insertions(+), 34 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 907839711a39c..1c8651a05697c 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7267,8 +7267,7 @@ static void fixReductionScalarResumeWhenVectorizingEpilog( auto *EpiRedHeaderPhi = cast(EpiRedResult->getOperand(0)); - const RecurrenceDescriptor &RdxDesc = - EpiRedHeaderPhi->getRecurrenceDescriptor(); + RecurKind Kind = EpiRedHeaderPhi->getRecurrenceKind(); Value *MainResumeValue; if (auto *VPI = dyn_cast(EpiRedHeaderPhi->getStartValue())) { assert((VPI->getOpcode() == VPInstruction::Broadcast || @@ -7277,8 +7276,7 @@ static void fixReductionScalarResumeWhenVectorizingEpilog( MainResumeValue = VPI->getOperand(0)->getUnderlyingValue(); } else MainResumeValue = EpiRedHeaderPhi->getStartValue()->getUnderlyingValue(); - if (RecurrenceDescriptor::isAnyOfRecurrenceKind( - RdxDesc.getRecurrenceKind())) { + if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind)) { [[maybe_unused]] Value *StartV = EpiRedResult->getOperand(1)->getLiveInIRValue(); auto *Cmp = cast(MainResumeValue); @@ -7288,8 +7286,7 @@ static void fixReductionScalarResumeWhenVectorizingEpilog( "AnyOf expected to start by comparing main resume value to original " "start value"); MainResumeValue = Cmp->getOperand(0); - } else if (RecurrenceDescriptor::isFindIVRecurrenceKind( - RdxDesc.getRecurrenceKind())) { + } else if (RecurrenceDescriptor::isFindIVRecurrenceKind(Kind)) { Value *StartV = getStartValueFromReductionResult(EpiRedResult); Value *SentinelV = EpiRedResult->getOperand(2)->getLiveInIRValue(); using namespace llvm::PatternMatch; @@ -9070,8 +9067,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( if (!PhiR || !PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered())) continue; - const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor(); - RecurKind Kind = RdxDesc.getRecurrenceKind(); + RecurKind Kind = PhiR->getRecurrenceKind(); assert( !RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind) && !RecurrenceDescriptor::isFindIVRecurrenceKind(Kind) && @@ -9177,6 +9173,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( if (CM.blockNeedsPredicationForAnyReason(CurrentLinkI->getParent())) CondOp = RecipeBuilder.getBlockInMask(CurrentLink->getParent()); + const RecurrenceDescriptor &RdxDesc = Legal->getReductionVars().lookup( + cast(PhiR->getUnderlyingInstr())); // Non-FP RdxDescs will have all fast math flags set, so clear them. FastMathFlags FMFs = isa(CurrentLinkI) ? RdxDesc.getFastMathFlags() @@ -9207,7 +9205,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( if (!PhiR) continue; - const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor(); + const RecurrenceDescriptor &RdxDesc = Legal->getReductionVars().lookup( + cast(PhiR->getUnderlyingInstr())); Type *PhiTy = PhiR->getUnderlyingValue()->getType(); // If tail is folded by masking, introduce selects between the phi // and the users outside the vector region of each reduction, at the @@ -9853,14 +9852,9 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L, })); ResumeV = cast(ReductionPhi->getUnderlyingInstr()) ->getIncomingValueForBlock(L->getLoopPreheader()); - const RecurrenceDescriptor &RdxDesc = - ReductionPhi->getRecurrenceDescriptor(); - RecurKind RK = RdxDesc.getRecurrenceKind(); + RecurKind RK = ReductionPhi->getRecurrenceKind(); if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) { Value *StartV = RdxResult->getOperand(1)->getLiveInIRValue(); - assert(RdxDesc.getRecurrenceStartValue() == StartV && - "start value from ComputeAnyOfResult must match"); - // VPReductionPHIRecipes for AnyOf reductions expect a boolean as // start value; compare the final value from the main vector loop // to the start value. @@ -9869,9 +9863,6 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L, ResumeV = Builder.CreateICmpNE(ResumeV, StartV); } else if (RecurrenceDescriptor::isFindIVRecurrenceKind(RK)) { Value *StartV = getStartValueFromReductionResult(RdxResult); - assert(RdxDesc.getRecurrenceStartValue() == StartV && - "start value from ComputeFinIVResult must match"); - ToFrozen[StartV] = cast(ResumeV)->getIncomingValueForBlock( EPI.MainLoopIterationCountCheck); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index c5b214b355545..c922e71fe56d7 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2192,7 +2192,7 @@ struct VPFirstOrderRecurrencePHIRecipe : public VPHeaderPHIRecipe { class VPReductionPHIRecipe : public VPHeaderPHIRecipe, public VPUnrollPartAccessor<2> { /// Descriptor for the reduction. - const RecurrenceDescriptor &RdxDesc; + const RecurKind Kind; /// The phi is part of an in-loop reduction. bool IsInLoop; @@ -2211,16 +2211,24 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe, VPValue &Start, bool IsInLoop = false, bool IsOrdered = false, unsigned VFScaleFactor = 1) : VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start), - RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered), - VFScaleFactor(VFScaleFactor) { + Kind(RdxDesc.getRecurrenceKind()), IsInLoop(IsInLoop), + IsOrdered(IsOrdered), VFScaleFactor(VFScaleFactor) { + assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop"); + } + VPReductionPHIRecipe(PHINode *Phi, RecurKind Kind, VPValue &Start, + bool IsInLoop = false, bool IsOrdered = false, + unsigned VFScaleFactor = 1) + : VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start), Kind(Kind), + IsInLoop(IsInLoop), IsOrdered(IsOrdered), VFScaleFactor(VFScaleFactor) { assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop"); } ~VPReductionPHIRecipe() override = default; VPReductionPHIRecipe *clone() override { + auto *R = new VPReductionPHIRecipe( - dyn_cast_or_null(getUnderlyingValue()), RdxDesc, + dyn_cast_or_null(getUnderlyingValue()), getRecurrenceKind(), *getOperand(0), IsInLoop, IsOrdered, VFScaleFactor); R->addOperand(getBackedgeValue()); return R; @@ -2240,9 +2248,7 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe, VPSlotTracker &SlotTracker) const override; #endif - const RecurrenceDescriptor &getRecurrenceDescriptor() const { - return RdxDesc; - } + RecurKind getRecurrenceKind() const { return Kind; } /// Returns true, if the phi is part of an ordered reduction. bool isOrdered() const { return IsOrdered; } diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 06511b61a67c3..59932f832f766 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -751,8 +751,7 @@ Value *VPInstruction::generate(VPTransformState &State) { // and will be removed by breaking up the recipe further. auto *PhiR = cast(getOperand(0)); // Get its reduction variable descriptor. - const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor(); - RecurKind RK = RdxDesc.getRecurrenceKind(); + [[maybe_unused]] RecurKind RK = PhiR->getRecurrenceKind(); assert(RecurrenceDescriptor::isFindIVRecurrenceKind(RK) && "Unexpected reduction kind"); assert(!PhiR->isInLoop() && @@ -786,9 +785,8 @@ Value *VPInstruction::generate(VPTransformState &State) { // and will be removed by breaking up the recipe further. auto *PhiR = cast(getOperand(0)); // Get its reduction variable descriptor. - const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor(); - RecurKind RK = RdxDesc.getRecurrenceKind(); + RecurKind RK = PhiR->getRecurrenceKind(); assert(!RecurrenceDescriptor::isFindIVRecurrenceKind(RK) && "should be handled by ComputeFindIVResult"); @@ -814,9 +812,9 @@ Value *VPInstruction::generate(VPTransformState &State) { if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK)) ReducedPartRdx = createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart); else - ReducedPartRdx = - Builder.CreateBinOp((Instruction::BinaryOps)RdxDesc.getOpcode(), - RdxPart, ReducedPartRdx, "bin.rdx"); + ReducedPartRdx = Builder.CreateBinOp( + (Instruction::BinaryOps)RecurrenceDescriptor::getOpcode(RK), + RdxPart, ReducedPartRdx, "bin.rdx"); } } diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 0bceb70d8661f..3d12f4b530b63 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1735,8 +1735,7 @@ void VPlanTransforms::clearReductionWrapFlags(VPlan &Plan) { auto *PhiR = dyn_cast(&R); if (!PhiR) continue; - const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor(); - RecurKind RK = RdxDesc.getRecurrenceKind(); + RecurKind RK = PhiR->getRecurrenceKind(); if (RK != RecurKind::Add && RK != RecurKind::Mul) continue; From 5d0c744a3dcb51c80c0f03df1c497372b67861bb Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 2 Jul 2025 15:44:21 +0100 Subject: [PATCH 2/3] !fixup use static cast, add helper. --- .../llvm/Transforms/Vectorize/LoopVectorizationLegality.h | 5 +++++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 +- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h index d654ac3ec9273..8889faf94a17d 100644 --- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h +++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h @@ -301,6 +301,11 @@ class LoopVectorizationLegality { /// Returns the reduction variables found in the loop. const ReductionList &getReductionVars() const { return Reductions; } + RecurrenceDescriptor getRecurrenceDescriptor(PHINode *PN) const { + assert(Reductions.contains(PN) && "no recurrence descriptor for phi"); + return Reductions.lookup(PN); + } + /// Returns the induction variables found in the loop. const InductionList &getInductionVars() const { return Inductions; } diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 1c8651a05697c..9950c28ef9e25 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -9173,7 +9173,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( if (CM.blockNeedsPredicationForAnyReason(CurrentLinkI->getParent())) CondOp = RecipeBuilder.getBlockInMask(CurrentLink->getParent()); - const RecurrenceDescriptor &RdxDesc = Legal->getReductionVars().lookup( + RecurrenceDescriptor RdxDesc = Legal->getRecurrenceDescriptor( cast(PhiR->getUnderlyingInstr())); // Non-FP RdxDescs will have all fast math flags set, so clear them. FastMathFlags FMFs = isa(CurrentLinkI) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 59932f832f766..e619f947c6174 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -740,8 +740,8 @@ Value *VPInstruction::generate(VPTransformState &State) { Value *ReducedPartRdx = State.get(getOperand(2)); for (unsigned Idx = 3; Idx < getNumOperands(); ++Idx) ReducedPartRdx = Builder.CreateBinOp( - (Instruction::BinaryOps)RecurrenceDescriptor::getOpcode( - RecurKind::AnyOf), + static_cast( + RecurrenceDescriptor::getOpcode(RecurKind::AnyOf)), State.get(getOperand(Idx)), ReducedPartRdx, "bin.rdx"); return createAnyOfReduction(Builder, ReducedPartRdx, State.get(getOperand(1), VPLane(0)), OrigPhi); From dca0d4b0ec47ea83f49f44794874c9af4b3192d4 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 3 Jul 2025 18:41:13 +0100 Subject: [PATCH 3/3] !fixup unify constructors, use helper --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 4 ++-- llvm/lib/Transforms/Vectorize/VPlan.h | 10 +--------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 9950c28ef9e25..8445357b1ea92 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8307,7 +8307,7 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R, unsigned ScaleFactor = getScalingForReduction(RdxDesc.getLoopExitInstr()).value_or(1); PhiRecipe = new VPReductionPHIRecipe( - Phi, RdxDesc, *StartV, CM.isInLoopReduction(Phi), + Phi, RdxDesc.getRecurrenceKind(), *StartV, CM.isInLoopReduction(Phi), CM.useOrderedReductions(RdxDesc), ScaleFactor); } else { // TODO: Currently fixed-order recurrences are modeled as chains of @@ -9205,7 +9205,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( if (!PhiR) continue; - const RecurrenceDescriptor &RdxDesc = Legal->getReductionVars().lookup( + const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor( cast(PhiR->getUnderlyingInstr())); Type *PhiTy = PhiR->getUnderlyingValue()->getType(); // If tail is folded by masking, introduce selects between the phi diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index c922e71fe56d7..7b747f3167aed 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2206,15 +2206,7 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe, public: /// Create a new VPReductionPHIRecipe for the reduction \p Phi described by \p - /// RdxDesc. - VPReductionPHIRecipe(PHINode *Phi, const RecurrenceDescriptor &RdxDesc, - VPValue &Start, bool IsInLoop = false, - bool IsOrdered = false, unsigned VFScaleFactor = 1) - : VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start), - Kind(RdxDesc.getRecurrenceKind()), IsInLoop(IsInLoop), - IsOrdered(IsOrdered), VFScaleFactor(VFScaleFactor) { - assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop"); - } + /// Kind. VPReductionPHIRecipe(PHINode *Phi, RecurKind Kind, VPValue &Start, bool IsInLoop = false, bool IsOrdered = false, unsigned VFScaleFactor = 1)