diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 05d10f0b84fac..e7bae17dd2ceb 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7266,8 +7266,7 @@ static void fixReductionScalarResumeWhenVectorizingEpilog( auto *EpiRedHeaderPhi = cast(EpiRedResult->getOperand(0)); - const RecurrenceDescriptor &RdxDesc = - EpiRedHeaderPhi->getRecurrenceDescriptor(); + RecurKind Kind = EpiRedHeaderPhi->getRecurrenceKind(); Value *MainResumeValue; if (auto *VPI = dyn_cast(EpiRedHeaderPhi->getStartValue())) { assert((VPI->getOpcode() == VPInstruction::Broadcast || @@ -7276,8 +7275,7 @@ static void fixReductionScalarResumeWhenVectorizingEpilog( MainResumeValue = VPI->getOperand(0)->getUnderlyingValue(); } else MainResumeValue = EpiRedHeaderPhi->getStartValue()->getUnderlyingValue(); - if (RecurrenceDescriptor::isAnyOfRecurrenceKind( - RdxDesc.getRecurrenceKind())) { + if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind)) { [[maybe_unused]] Value *StartV = EpiRedResult->getOperand(1)->getLiveInIRValue(); auto *Cmp = cast(MainResumeValue); @@ -7287,8 +7285,7 @@ static void fixReductionScalarResumeWhenVectorizingEpilog( "AnyOf expected to start by comparing main resume value to original " "start value"); MainResumeValue = Cmp->getOperand(0); - } else if (RecurrenceDescriptor::isFindIVRecurrenceKind( - RdxDesc.getRecurrenceKind())) { + } else if (RecurrenceDescriptor::isFindIVRecurrenceKind(Kind)) { Value *StartV = getStartValueFromReductionResult(EpiRedResult); Value *SentinelV = EpiRedResult->getOperand(2)->getLiveInIRValue(); using namespace llvm::PatternMatch; @@ -8308,7 +8305,7 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R, unsigned ScaleFactor = getScalingForReduction(RdxDesc.getLoopExitInstr()).value_or(1); PhiRecipe = new VPReductionPHIRecipe( - Phi, RdxDesc, *StartV, CM.isInLoopReduction(Phi), + Phi, RdxDesc.getRecurrenceKind(), *StartV, CM.isInLoopReduction(Phi), CM.useOrderedReductions(RdxDesc), ScaleFactor); } else { // TODO: Currently fixed-order recurrences are modeled as chains of @@ -9068,8 +9065,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( if (!PhiR || !PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered())) continue; - const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor(); - RecurKind Kind = RdxDesc.getRecurrenceKind(); + RecurKind Kind = PhiR->getRecurrenceKind(); assert( !RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind) && !RecurrenceDescriptor::isFindIVRecurrenceKind(Kind) && @@ -9175,6 +9171,9 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( if (CM.blockNeedsPredicationForAnyReason(CurrentLinkI->getParent())) CondOp = RecipeBuilder.getBlockInMask(CurrentLink->getParent()); + // TODO: Retrieve FMFs from recipes directly. + RecurrenceDescriptor RdxDesc = Legal->getRecurrenceDescriptor( + cast(PhiR->getUnderlyingInstr())); // Non-FP RdxDescs will have all fast math flags set, so clear them. FastMathFlags FMFs = isa(CurrentLinkI) ? RdxDesc.getFastMathFlags() @@ -9205,7 +9204,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( if (!PhiR) continue; - const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor(); + const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor( + cast(PhiR->getUnderlyingInstr())); Type *PhiTy = PhiR->getUnderlyingValue()->getType(); // If tail is folded by masking, introduce selects between the phi // and the users outside the vector region of each reduction, at the @@ -9253,24 +9253,23 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( VPInstruction *FinalReductionResult; VPBuilder::InsertPointGuard Guard(Builder); Builder.setInsertPoint(MiddleVPBB, IP); - if (RecurrenceDescriptor::isFindIVRecurrenceKind( - RdxDesc.getRecurrenceKind())) { + RecurKind RecurrenceKind = PhiR->getRecurrenceKind(); + if (RecurrenceDescriptor::isFindIVRecurrenceKind(RecurrenceKind)) { VPValue *Start = PhiR->getStartValue(); VPValue *Sentinel = Plan->getOrAddLiveIn(RdxDesc.getSentinelValue()); FinalReductionResult = Builder.createNaryOp(VPInstruction::ComputeFindIVResult, {PhiR, Start, Sentinel, NewExitingVPV}, ExitDL); - } else if (RecurrenceDescriptor::isAnyOfRecurrenceKind( - RdxDesc.getRecurrenceKind())) { + } else if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RecurrenceKind)) { VPValue *Start = PhiR->getStartValue(); FinalReductionResult = Builder.createNaryOp(VPInstruction::ComputeAnyOfResult, {PhiR, Start, NewExitingVPV}, ExitDL); } else { - VPIRFlags Flags = RecurrenceDescriptor::isFloatingPointRecurrenceKind( - RdxDesc.getRecurrenceKind()) - ? VPIRFlags(RdxDesc.getFastMathFlags()) - : VPIRFlags(); + VPIRFlags Flags = + RecurrenceDescriptor::isFloatingPointRecurrenceKind(RecurrenceKind) + ? VPIRFlags(RdxDesc.getFastMathFlags()) + : VPIRFlags(); FinalReductionResult = Builder.createNaryOp(VPInstruction::ComputeReductionResult, {PhiR, NewExitingVPV}, Flags, ExitDL); @@ -9279,11 +9278,9 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( // then extend the loop exit value to enable InstCombine to evaluate the // entire expression in the smaller type. if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType() && - !RecurrenceDescriptor::isAnyOfRecurrenceKind( - RdxDesc.getRecurrenceKind())) { + !RecurrenceDescriptor::isAnyOfRecurrenceKind(RecurrenceKind)) { assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!"); - assert(!RecurrenceDescriptor::isMinMaxRecurrenceKind( - RdxDesc.getRecurrenceKind()) && + assert(!RecurrenceDescriptor::isMinMaxRecurrenceKind(RecurrenceKind) && "Unexpected truncated min-max recurrence!"); Type *RdxTy = RdxDesc.getRecurrenceType(); auto *Trunc = @@ -9319,8 +9316,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( // with a boolean reduction phi node to check if the condition is true in // any iteration. The final value is selected by the final // ComputeReductionResult. - if (RecurrenceDescriptor::isAnyOfRecurrenceKind( - RdxDesc.getRecurrenceKind())) { + if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RecurrenceKind)) { auto *Select = cast(*find_if(PhiR->users(), [](VPUser *U) { return isa(U) || (isa(U) && @@ -9851,14 +9847,9 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L, })); ResumeV = cast(ReductionPhi->getUnderlyingInstr()) ->getIncomingValueForBlock(L->getLoopPreheader()); - const RecurrenceDescriptor &RdxDesc = - ReductionPhi->getRecurrenceDescriptor(); - RecurKind RK = RdxDesc.getRecurrenceKind(); + RecurKind RK = ReductionPhi->getRecurrenceKind(); if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) { Value *StartV = RdxResult->getOperand(1)->getLiveInIRValue(); - assert(RdxDesc.getRecurrenceStartValue() == StartV && - "start value from ComputeAnyOfResult must match"); - // VPReductionPHIRecipes for AnyOf reductions expect a boolean as // start value; compare the final value from the main vector loop // to the start value. @@ -9867,9 +9858,6 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L, ResumeV = Builder.CreateICmpNE(ResumeV, StartV); } else if (RecurrenceDescriptor::isFindIVRecurrenceKind(RK)) { Value *StartV = getStartValueFromReductionResult(RdxResult); - assert(RdxDesc.getRecurrenceStartValue() == StartV && - "start value from ComputeFinIVResult must match"); - ToFrozen[StartV] = cast(ResumeV)->getIncomingValueForBlock( EPI.MainLoopIterationCountCheck); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index c5b214b355545..356af4a0e74e4 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2191,8 +2191,8 @@ struct VPFirstOrderRecurrencePHIRecipe : public VPHeaderPHIRecipe { /// operand. class VPReductionPHIRecipe : public VPHeaderPHIRecipe, public VPUnrollPartAccessor<2> { - /// Descriptor for the reduction. - const RecurrenceDescriptor &RdxDesc; + /// The recurrence kind of the reduction. + const RecurKind Kind; /// The phi is part of an in-loop reduction. bool IsInLoop; @@ -2205,14 +2205,12 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe, unsigned VFScaleFactor = 1; public: - /// Create a new VPReductionPHIRecipe for the reduction \p Phi described by \p - /// RdxDesc. - VPReductionPHIRecipe(PHINode *Phi, const RecurrenceDescriptor &RdxDesc, - VPValue &Start, bool IsInLoop = false, - bool IsOrdered = false, unsigned VFScaleFactor = 1) - : VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start), - RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered), - VFScaleFactor(VFScaleFactor) { + /// Create a new VPReductionPHIRecipe for the reduction \p Phi. + VPReductionPHIRecipe(PHINode *Phi, RecurKind Kind, VPValue &Start, + bool IsInLoop = false, bool IsOrdered = false, + unsigned VFScaleFactor = 1) + : VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start), Kind(Kind), + IsInLoop(IsInLoop), IsOrdered(IsOrdered), VFScaleFactor(VFScaleFactor) { assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop"); } @@ -2220,7 +2218,7 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe, VPReductionPHIRecipe *clone() override { auto *R = new VPReductionPHIRecipe( - dyn_cast_or_null(getUnderlyingValue()), RdxDesc, + dyn_cast_or_null(getUnderlyingValue()), getRecurrenceKind(), *getOperand(0), IsInLoop, IsOrdered, VFScaleFactor); R->addOperand(getBackedgeValue()); return R; @@ -2240,9 +2238,8 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe, VPSlotTracker &SlotTracker) const override; #endif - const RecurrenceDescriptor &getRecurrenceDescriptor() const { - return RdxDesc; - } + /// Returns the recurrence kind of the reduction. + RecurKind getRecurrenceKind() const { return Kind; } /// Returns true, if the phi is part of an ordered reduction. bool isOrdered() const { return IsOrdered; } diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 06511b61a67c3..ccb7512051d77 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -751,8 +751,7 @@ Value *VPInstruction::generate(VPTransformState &State) { // and will be removed by breaking up the recipe further. auto *PhiR = cast(getOperand(0)); // Get its reduction variable descriptor. - const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor(); - RecurKind RK = RdxDesc.getRecurrenceKind(); + RecurKind RK = PhiR->getRecurrenceKind(); assert(RecurrenceDescriptor::isFindIVRecurrenceKind(RK) && "Unexpected reduction kind"); assert(!PhiR->isInLoop() && @@ -786,9 +785,8 @@ Value *VPInstruction::generate(VPTransformState &State) { // and will be removed by breaking up the recipe further. auto *PhiR = cast(getOperand(0)); // Get its reduction variable descriptor. - const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor(); - RecurKind RK = RdxDesc.getRecurrenceKind(); + RecurKind RK = PhiR->getRecurrenceKind(); assert(!RecurrenceDescriptor::isFindIVRecurrenceKind(RK) && "should be handled by ComputeFindIVResult"); @@ -814,9 +812,9 @@ Value *VPInstruction::generate(VPTransformState &State) { if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK)) ReducedPartRdx = createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart); else - ReducedPartRdx = - Builder.CreateBinOp((Instruction::BinaryOps)RdxDesc.getOpcode(), - RdxPart, ReducedPartRdx, "bin.rdx"); + ReducedPartRdx = Builder.CreateBinOp( + (Instruction::BinaryOps)RecurrenceDescriptor::getOpcode(RK), + RdxPart, ReducedPartRdx, "bin.rdx"); } } diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 75c7b20250846..90137b72c83fb 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1735,8 +1735,7 @@ void VPlanTransforms::clearReductionWrapFlags(VPlan &Plan) { auto *PhiR = dyn_cast(&R); if (!PhiR) continue; - const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor(); - RecurKind RK = RdxDesc.getRecurrenceKind(); + RecurKind RK = PhiR->getRecurrenceKind(); if (RK != RecurKind::Add && RK != RecurKind::Mul) continue;