diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h index bae53c600c18c..c751f053cb65a 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -164,25 +164,19 @@ class VPBuilder { DebugLoc DL, const Twine &Name = "") { return createInstruction(Opcode, Operands, DL, Name); } - VPInstruction *createNaryOp(unsigned Opcode, - std::initializer_list Operands, - std::optional FMFs = {}, - DebugLoc DL = {}, const Twine &Name = "") { - if (FMFs) - return tryInsertInstruction( - new VPInstruction(Opcode, Operands, *FMFs, DL, Name)); - return createInstruction(Opcode, Operands, DL, Name); + VPInstruction *createNaryOp(unsigned Opcode, ArrayRef Operands, + const VPIRFlags &Flags, DebugLoc DL = {}, + const Twine &Name = "") { + return tryInsertInstruction( + new VPInstruction(Opcode, Operands, Flags, DL, Name)); } + VPInstruction *createNaryOp(unsigned Opcode, std::initializer_list Operands, - Type *ResultTy, - std::optional FMFs = {}, + Type *ResultTy, const VPIRFlags &Flags = {}, DebugLoc DL = {}, const Twine &Name = "") { - if (FMFs) - return tryInsertInstruction(new VPInstructionWithType( - Opcode, Operands, ResultTy, *FMFs, DL, Name)); return tryInsertInstruction( - new VPInstructionWithType(Opcode, Operands, ResultTy, DL, Name)); + new VPInstructionWithType(Opcode, Operands, ResultTy, Flags, DL, Name)); } VPInstruction *createOverflowingOp(unsigned Opcode, @@ -236,18 +230,20 @@ class VPBuilder { assert(Pred >= CmpInst::FIRST_ICMP_PREDICATE && Pred <= CmpInst::LAST_ICMP_PREDICATE && "invalid predicate"); return tryInsertInstruction( - new VPInstruction(Instruction::ICmp, Pred, A, B, DL, Name)); + new VPInstruction(Instruction::ICmp, {A, B}, Pred, DL, Name)); } VPInstruction *createPtrAdd(VPValue *Ptr, VPValue *Offset, DebugLoc DL = {}, const Twine &Name = "") { return tryInsertInstruction( - new VPInstruction(Ptr, Offset, GEPNoWrapFlags::none(), DL, Name)); + new VPInstruction(VPInstruction::PtrAdd, {Ptr, Offset}, + GEPNoWrapFlags::none(), DL, Name)); } VPValue *createInBoundsPtrAdd(VPValue *Ptr, VPValue *Offset, DebugLoc DL = {}, const Twine &Name = "") { return tryInsertInstruction( - new VPInstruction(Ptr, Offset, GEPNoWrapFlags::inBounds(), DL, Name)); + new VPInstruction(VPInstruction::PtrAdd, {Ptr, Offset}, + GEPNoWrapFlags::inBounds(), DL, Name)); } VPInstruction *createScalarPhi(ArrayRef IncomingValues, @@ -269,7 +265,7 @@ class VPBuilder { VPInstruction *createScalarCast(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, DebugLoc DL) { return tryInsertInstruction( - new VPInstructionWithType(Opcode, Op, ResultTy, DL)); + new VPInstructionWithType(Opcode, Op, ResultTy, {}, DL)); } VPWidenCastRecipe *createWidenCast(Instruction::CastOps Opcode, VPValue *Op, diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index c4e66cd89e69c..28fad085b4b7b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -577,8 +577,8 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPValue { #endif }; -/// Class to record LLVM IR flag for a recipe along with it. -class VPRecipeWithIRFlags : public VPSingleDefRecipe { +/// Class to record and manage LLVM IR flags. +class VPIRFlags { enum class OperationType : unsigned char { Cmp, OverflowingBinOp, @@ -637,23 +637,10 @@ class VPRecipeWithIRFlags : public VPSingleDefRecipe { unsigned AllFlags; }; -protected: - void transferFlags(VPRecipeWithIRFlags &Other) { - OpType = Other.OpType; - AllFlags = Other.AllFlags; - } - public: - VPRecipeWithIRFlags(const unsigned char SC, ArrayRef Operands, - DebugLoc DL = {}) - : VPSingleDefRecipe(SC, Operands, DL) { - OpType = OperationType::Other; - AllFlags = 0; - } + VPIRFlags() : OpType(OperationType::Other), AllFlags(0) {} - VPRecipeWithIRFlags(const unsigned char SC, ArrayRef Operands, - Instruction &I) - : VPSingleDefRecipe(SC, Operands, &I, I.getDebugLoc()) { + VPIRFlags(Instruction &I) { if (auto *Op = dyn_cast(&I)) { OpType = OperationType::Cmp; CmpPredicate = Op->getPredicate(); @@ -681,63 +668,27 @@ class VPRecipeWithIRFlags : public VPSingleDefRecipe { } } - VPRecipeWithIRFlags(const unsigned char SC, ArrayRef Operands, - CmpInst::Predicate Pred, DebugLoc DL = {}) - : VPSingleDefRecipe(SC, Operands, DL), OpType(OperationType::Cmp), - CmpPredicate(Pred) {} + VPIRFlags(CmpInst::Predicate Pred) + : OpType(OperationType::Cmp), CmpPredicate(Pred) {} - VPRecipeWithIRFlags(const unsigned char SC, ArrayRef Operands, - WrapFlagsTy WrapFlags, DebugLoc DL = {}) - : VPSingleDefRecipe(SC, Operands, DL), - OpType(OperationType::OverflowingBinOp), WrapFlags(WrapFlags) {} + VPIRFlags(WrapFlagsTy WrapFlags) + : OpType(OperationType::OverflowingBinOp), WrapFlags(WrapFlags) {} - VPRecipeWithIRFlags(const unsigned char SC, ArrayRef Operands, - FastMathFlags FMFs, DebugLoc DL = {}) - : VPSingleDefRecipe(SC, Operands, DL), OpType(OperationType::FPMathOp), - FMFs(FMFs) {} + VPIRFlags(FastMathFlags FMFs) : OpType(OperationType::FPMathOp), FMFs(FMFs) {} - VPRecipeWithIRFlags(const unsigned char SC, ArrayRef Operands, - DisjointFlagsTy DisjointFlags, DebugLoc DL = {}) - : VPSingleDefRecipe(SC, Operands, DL), OpType(OperationType::DisjointOp), - DisjointFlags(DisjointFlags) {} + VPIRFlags(DisjointFlagsTy DisjointFlags) + : OpType(OperationType::DisjointOp), DisjointFlags(DisjointFlags) {} - template - VPRecipeWithIRFlags(const unsigned char SC, IterT Operands, - NonNegFlagsTy NonNegFlags, DebugLoc DL = {}) - : VPSingleDefRecipe(SC, Operands, DL), OpType(OperationType::NonNegOp), - NonNegFlags(NonNegFlags) {} + VPIRFlags(NonNegFlagsTy NonNegFlags) + : OpType(OperationType::NonNegOp), NonNegFlags(NonNegFlags) {} -protected: - VPRecipeWithIRFlags(const unsigned char SC, ArrayRef Operands, - GEPNoWrapFlags GEPFlags, DebugLoc DL = {}) - : VPSingleDefRecipe(SC, Operands, DL), OpType(OperationType::GEPOp), - GEPFlags(GEPFlags) {} + VPIRFlags(GEPNoWrapFlags GEPFlags) + : OpType(OperationType::GEPOp), GEPFlags(GEPFlags) {} public: - static inline bool classof(const VPRecipeBase *R) { - return R->getVPDefID() == VPRecipeBase::VPInstructionSC || - R->getVPDefID() == VPRecipeBase::VPWidenSC || - R->getVPDefID() == VPRecipeBase::VPWidenGEPSC || - R->getVPDefID() == VPRecipeBase::VPWidenCallSC || - R->getVPDefID() == VPRecipeBase::VPWidenCastSC || - R->getVPDefID() == VPRecipeBase::VPWidenIntrinsicSC || - R->getVPDefID() == VPRecipeBase::VPReductionSC || - R->getVPDefID() == VPRecipeBase::VPReductionEVLSC || - R->getVPDefID() == VPRecipeBase::VPReplicateSC || - R->getVPDefID() == VPRecipeBase::VPVectorEndPointerSC || - R->getVPDefID() == VPRecipeBase::VPVectorPointerSC || - R->getVPDefID() == VPRecipeBase::VPExtendedReductionSC || - R->getVPDefID() == VPRecipeBase::VPMulAccumulateReductionSC; - } - - static inline bool classof(const VPUser *U) { - auto *R = dyn_cast(U); - return R && classof(R); - } - - static inline bool classof(const VPValue *V) { - auto *R = dyn_cast_or_null(V->getDefiningRecipe()); - return R && classof(R); + void transferFlags(VPIRFlags &Other) { + OpType = Other.OpType; + AllFlags = Other.AllFlags; } /// Drop all poison-generating flags. @@ -851,11 +802,60 @@ class VPRecipeWithIRFlags : public VPSingleDefRecipe { return DisjointFlags.IsDisjoint; } +#if !defined(NDEBUG) + /// Returns true if the set flags are valid for \p Opcode. + bool flagsValidForOpcode(unsigned Opcode) const; +#endif + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void printFlags(raw_ostream &O) const; #endif }; +/// A pure-virtual common base class for recipes defining a single VPValue and +/// using IR flags. +struct VPRecipeWithIRFlags : public VPSingleDefRecipe, public VPIRFlags { + VPRecipeWithIRFlags(const unsigned char SC, ArrayRef Operands, + DebugLoc DL = {}) + : VPSingleDefRecipe(SC, Operands, DL), VPIRFlags() {} + + VPRecipeWithIRFlags(const unsigned char SC, ArrayRef Operands, + Instruction &I) + : VPSingleDefRecipe(SC, Operands, &I, I.getDebugLoc()), VPIRFlags(I) {} + + VPRecipeWithIRFlags(const unsigned char SC, ArrayRef Operands, + const VPIRFlags &Flags, DebugLoc DL = {}) + : VPSingleDefRecipe(SC, Operands, DL), VPIRFlags(Flags) {} + + static inline bool classof(const VPRecipeBase *R) { + return R->getVPDefID() == VPRecipeBase::VPInstructionSC || + R->getVPDefID() == VPRecipeBase::VPWidenSC || + R->getVPDefID() == VPRecipeBase::VPWidenGEPSC || + R->getVPDefID() == VPRecipeBase::VPWidenCallSC || + R->getVPDefID() == VPRecipeBase::VPWidenCastSC || + R->getVPDefID() == VPRecipeBase::VPWidenIntrinsicSC || + R->getVPDefID() == VPRecipeBase::VPReductionSC || + R->getVPDefID() == VPRecipeBase::VPReductionEVLSC || + R->getVPDefID() == VPRecipeBase::VPReplicateSC || + R->getVPDefID() == VPRecipeBase::VPVectorEndPointerSC || + R->getVPDefID() == VPRecipeBase::VPVectorPointerSC || + R->getVPDefID() == VPRecipeBase::VPExtendedReductionSC || + R->getVPDefID() == VPRecipeBase::VPMulAccumulateReductionSC; + } + + static inline bool classof(const VPUser *U) { + auto *R = dyn_cast(U); + return R && classof(R); + } + + static inline bool classof(const VPValue *V) { + auto *R = dyn_cast_or_null(V->getDefiningRecipe()); + return R && classof(R); + } + + void execute(VPTransformState &State) override = 0; +}; + /// Helper to access the operand that contains the unroll part for this recipe /// after unrolling. template class VPUnrollPartAccessor { @@ -958,54 +958,21 @@ class VPInstruction : public VPRecipeWithIRFlags, /// value for lane \p Lane. Value *generatePerLane(VPTransformState &State, const VPLane &Lane); -#if !defined(NDEBUG) - /// Return true if the VPInstruction is a floating point math operation, i.e. - /// has fast-math flags. - bool isFPMathOp() const; -#endif - public: - VPInstruction(unsigned Opcode, ArrayRef Operands, DebugLoc DL, + VPInstruction(unsigned Opcode, ArrayRef Operands, DebugLoc DL = {}, const Twine &Name = "") : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, DL), Opcode(Opcode), Name(Name.str()) {} - VPInstruction(unsigned Opcode, std::initializer_list Operands, - DebugLoc DL = {}, const Twine &Name = "") - : VPInstruction(Opcode, ArrayRef(Operands), DL, Name) {} - - VPInstruction(unsigned Opcode, CmpInst::Predicate Pred, VPValue *A, - VPValue *B, DebugLoc DL = {}, const Twine &Name = ""); - - VPInstruction(unsigned Opcode, std::initializer_list Operands, - WrapFlagsTy WrapFlags, DebugLoc DL = {}, const Twine &Name = "") - : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, WrapFlags, DL), - Opcode(Opcode), Name(Name.str()) {} - - VPInstruction(unsigned Opcode, std::initializer_list Operands, - DisjointFlagsTy DisjointFlag, DebugLoc DL = {}, - const Twine &Name = "") - : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, DisjointFlag, DL), - Opcode(Opcode), Name(Name.str()) { - assert(Opcode == Instruction::Or && "only OR opcodes can be disjoint"); - } - - VPInstruction(VPValue *Ptr, VPValue *Offset, GEPNoWrapFlags Flags, - DebugLoc DL = {}, const Twine &Name = "") - : VPRecipeWithIRFlags(VPDef::VPInstructionSC, - ArrayRef({Ptr, Offset}), Flags, DL), - Opcode(VPInstruction::PtrAdd), Name(Name.str()) {} - - VPInstruction(unsigned Opcode, std::initializer_list Operands, - FastMathFlags FMFs, DebugLoc DL = {}, const Twine &Name = ""); + VPInstruction(unsigned Opcode, ArrayRef Operands, + const VPIRFlags &Flags, DebugLoc DL = {}, + const Twine &Name = ""); VP_CLASSOF_IMPL(VPDef::VPInstructionSC) VPInstruction *clone() override { SmallVector Operands(operands()); - auto *New = new VPInstruction(Opcode, Operands, getDebugLoc(), Name); - New->transferFlags(*this); - return New; + return new VPInstruction(Opcode, Operands, *this, getDebugLoc(), Name); } unsigned getOpcode() const { return Opcode; } @@ -1082,13 +1049,9 @@ class VPInstructionWithType : public VPInstruction { public: VPInstructionWithType(unsigned Opcode, ArrayRef Operands, - Type *ResultTy, DebugLoc DL, const Twine &Name = "") - : VPInstruction(Opcode, Operands, DL, Name), ResultTy(ResultTy) {} - VPInstructionWithType(unsigned Opcode, - std::initializer_list Operands, - Type *ResultTy, FastMathFlags FMFs, DebugLoc DL = {}, + Type *ResultTy, const VPIRFlags &Flags, DebugLoc DL, const Twine &Name = "") - : VPInstruction(Opcode, Operands, FMFs, DL, Name), ResultTy(ResultTy) {} + : VPInstruction(Opcode, Operands, Flags, DL, Name), ResultTy(ResultTy) {} static inline bool classof(const VPRecipeBase *R) { // VPInstructionWithType are VPInstructions with specific opcodes requiring @@ -1113,8 +1076,9 @@ class VPInstructionWithType : public VPInstruction { VPInstruction *clone() override { SmallVector Operands(operands()); - auto *New = new VPInstructionWithType( - getOpcode(), Operands, getResultType(), getDebugLoc(), getName()); + auto *New = + new VPInstructionWithType(getOpcode(), Operands, getResultType(), *this, + getDebugLoc(), getName()); New->setUnderlyingValue(getUnderlyingValue()); return New; } @@ -1373,15 +1337,12 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata { } VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, - DebugLoc DL = {}) - : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, DL), VPIRMetadata(), - Opcode(Opcode), ResultTy(ResultTy) {} - - VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, - bool IsNonNeg, DebugLoc DL = {}) - : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, NonNegFlagsTy(IsNonNeg), - DL), - Opcode(Opcode), ResultTy(ResultTy) {} + const VPIRFlags &Flags = {}, DebugLoc DL = {}) + : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, Flags, DL), + VPIRMetadata(), Opcode(Opcode), ResultTy(ResultTy) { + assert(flagsValidForOpcode(Opcode) && + "Set flags not supported for the provided opcode"); + } ~VPWidenCastRecipe() override = default; diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 5c2ddb62c7155..26176e7859d74 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -368,7 +368,7 @@ void VPPartialReductionRecipe::print(raw_ostream &O, const Twine &Indent, } #endif -FastMathFlags VPRecipeWithIRFlags::getFastMathFlags() const { +FastMathFlags VPIRFlags::getFastMathFlags() const { assert(OpType == OperationType::FPMathOp && "recipe doesn't have fast math flags"); FastMathFlags Res; @@ -406,23 +406,13 @@ template class VPUnrollPartAccessor<2>; template class VPUnrollPartAccessor<3>; } -VPInstruction::VPInstruction(unsigned Opcode, CmpInst::Predicate Pred, - VPValue *A, VPValue *B, DebugLoc DL, +VPInstruction::VPInstruction(unsigned Opcode, ArrayRef Operands, + const VPIRFlags &Flags, DebugLoc DL, const Twine &Name) - : VPRecipeWithIRFlags(VPDef::VPInstructionSC, ArrayRef({A, B}), - Pred, DL), + : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, Flags, DL), Opcode(Opcode), Name(Name.str()) { - assert(Opcode == Instruction::ICmp && - "only ICmp predicates supported at the moment"); -} - -VPInstruction::VPInstruction(unsigned Opcode, - std::initializer_list Operands, - FastMathFlags FMFs, DebugLoc DL, const Twine &Name) - : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, FMFs, DL), - Opcode(Opcode), Name(Name.str()) { - // Make sure the VPInstruction is a floating-point operation. - assert(isFPMathOp() && "this op can't take fast-math flags"); + assert(flagsValidForOpcode(getOpcode()) && + "Set flags not supported for the provided opcode"); } bool VPInstruction::doesGeneratePerAllLanes() const { @@ -855,24 +845,11 @@ bool VPInstruction::isSingleScalar() const { getOpcode() == Instruction::PHI; } -#if !defined(NDEBUG) -bool VPInstruction::isFPMathOp() const { - // Inspired by FPMathOperator::classof. Notable differences are that we don't - // support Call, PHI and Select opcodes here yet. - return Opcode == Instruction::FAdd || Opcode == Instruction::FMul || - Opcode == Instruction::FNeg || Opcode == Instruction::FSub || - Opcode == Instruction::FDiv || Opcode == Instruction::FRem || - Opcode == Instruction::FCmp || Opcode == Instruction::Select || - Opcode == VPInstruction::WideIVStep; -} -#endif - void VPInstruction::execute(VPTransformState &State) { assert(!State.Lane && "VPInstruction executing an Lane"); IRBuilderBase::FastMathFlagGuard FMFGuard(State.Builder); - assert((hasFastMathFlags() == isFPMathOp() || - getOpcode() == Instruction::Select) && - "Recipe not a FPMathOp but has fast-math flags?"); + assert(flagsValidForOpcode(getOpcode()) && + "Set flags not supported for the provided opcode"); if (hasFastMathFlags()) State.Builder.setFastMathFlags(getFastMathFlags()); bool GeneratesPerFirstLaneOnly = canGenerateScalarForFirstLane() && @@ -1593,8 +1570,7 @@ InstructionCost VPWidenSelectRecipe::computeCost(ElementCount VF, {TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None}, SI); } -VPRecipeWithIRFlags::FastMathFlagsTy::FastMathFlagsTy( - const FastMathFlags &FMF) { +VPIRFlags::FastMathFlagsTy::FastMathFlagsTy(const FastMathFlags &FMF) { AllowReassoc = FMF.allowReassoc(); NoNaNs = FMF.noNaNs(); NoInfs = FMF.noInfs(); @@ -1604,8 +1580,39 @@ VPRecipeWithIRFlags::FastMathFlagsTy::FastMathFlagsTy( ApproxFunc = FMF.approxFunc(); } +#if !defined(NDEBUG) +bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const { + switch (OpType) { + case OperationType::OverflowingBinOp: + return Opcode == Instruction::Add || Opcode == Instruction::Sub || + Opcode == Instruction::Mul || + Opcode == VPInstruction::VPInstruction::CanonicalIVIncrementForPart; + case OperationType::DisjointOp: + return Opcode == Instruction::Or; + case OperationType::PossiblyExactOp: + return Opcode == Instruction::AShr; + case OperationType::GEPOp: + return Opcode == Instruction::GetElementPtr || + Opcode == VPInstruction::PtrAdd; + case OperationType::FPMathOp: + return Opcode == Instruction::FAdd || Opcode == Instruction::FMul || + Opcode == Instruction::FSub || Opcode == Instruction::FNeg || + Opcode == Instruction::FDiv || Opcode == Instruction::FRem || + Opcode == Instruction::FCmp || Opcode == Instruction::Select || + Opcode == VPInstruction::WideIVStep; + case OperationType::NonNegOp: + return Opcode == Instruction::ZExt; + break; + case OperationType::Cmp: + return Opcode == Instruction::ICmp; + case OperationType::Other: + return true; + } +} +#endif + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -void VPRecipeWithIRFlags::printFlags(raw_ostream &O) const { +void VPIRFlags::printFlags(raw_ostream &O) const { switch (OpType) { case OperationType::Cmp: O << " " << CmpInst::getPredicateName(getPredicate()); @@ -1642,8 +1649,7 @@ void VPRecipeWithIRFlags::printFlags(raw_ostream &O) const { case OperationType::Other: break; } - if (getNumOperands() > 0) - O << " "; + O << " "; } #endif diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 52d61d96c8083..d3f15eafb0214 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1343,8 +1343,9 @@ static bool optimizeVectorInductionWidthForTCAndVFUF(VPlan &Plan, assert(OldStepVector->getNumUsers() == 1 && "step vector should only be used by single " "VPWidenIntOrFpInductionRecipe"); - auto *NewStepVector = new VPInstructionWithType( - VPInstruction::StepVector, {}, NewIVTy, OldStepVector->getDebugLoc()); + auto *NewStepVector = + new VPInstructionWithType(VPInstruction::StepVector, {}, NewIVTy, {}, + OldStepVector->getDebugLoc()); NewStepVector->insertAfter(OldStepVector->getDefiningRecipe()); OldStepVector->replaceAllUsesWith(NewStepVector); OldStepVector->eraseFromParent(); @@ -2524,11 +2525,12 @@ static void expandVPExtendedReduction(VPExtendedReductionRecipe *ExtRed) { // Only ZExt contains non-neg flags. if (ExtRed->isZExt()) Ext = new VPWidenCastRecipe(ExtRed->getExtOpcode(), ExtRed->getVecOp(), - ExtRed->getResultType(), ExtRed->isNonNeg(), + ExtRed->getResultType(), *ExtRed, ExtRed->getDebugLoc()); else Ext = new VPWidenCastRecipe(ExtRed->getExtOpcode(), ExtRed->getVecOp(), - ExtRed->getResultType(), ExtRed->getDebugLoc()); + ExtRed->getResultType(), {}, + ExtRed->getDebugLoc()); auto *Red = new VPReductionRecipe( ExtRed->getRecurrenceKind(), FastMathFlags(), ExtRed->getChainOp(), Ext, @@ -2551,12 +2553,12 @@ expandVPMulAccumulateReduction(VPMulAccumulateReductionRecipe *MulAcc) { if (MulAcc->isExtended()) { Type *RedTy = MulAcc->getResultType(); if (MulAcc->isZExt()) - Op0 = new VPWidenCastRecipe(MulAcc->getExtOpcode(), MulAcc->getVecOp0(), - RedTy, MulAcc->isNonNeg(), - MulAcc->getDebugLoc()); + Op0 = new VPWidenCastRecipe( + MulAcc->getExtOpcode(), MulAcc->getVecOp0(), RedTy, + VPIRFlags::NonNegFlagsTy(MulAcc->isNonNeg()), MulAcc->getDebugLoc()); else Op0 = new VPWidenCastRecipe(MulAcc->getExtOpcode(), MulAcc->getVecOp0(), - RedTy, MulAcc->getDebugLoc()); + RedTy, {}, MulAcc->getDebugLoc()); Op0->getDefiningRecipe()->insertBefore(MulAcc); // Prevent reduce.add(mul(ext(A), ext(A))) generate duplicate // VPWidenCastRecipe. @@ -2564,12 +2566,13 @@ expandVPMulAccumulateReduction(VPMulAccumulateReductionRecipe *MulAcc) { Op1 = Op0; } else { if (MulAcc->isZExt()) - Op1 = new VPWidenCastRecipe(MulAcc->getExtOpcode(), MulAcc->getVecOp1(), - RedTy, MulAcc->isNonNeg(), - MulAcc->getDebugLoc()); + Op1 = new VPWidenCastRecipe( + MulAcc->getExtOpcode(), MulAcc->getVecOp1(), RedTy, + VPIRFlags::NonNegFlagsTy(MulAcc->isNonNeg()), + MulAcc->getDebugLoc()); else Op1 = new VPWidenCastRecipe(MulAcc->getExtOpcode(), MulAcc->getVecOp1(), - RedTy, MulAcc->getDebugLoc()); + RedTy, {}, MulAcc->getDebugLoc()); Op1->getDefiningRecipe()->insertBefore(MulAcc); } } else { @@ -2643,14 +2646,14 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan, Builder.createWidenCast(Instruction::Trunc, ScalarStep, IVTy); } - std::optional FMFs; + VPIRFlags Flags; if (IVTy->isFloatingPointTy()) - FMFs = VPI->getFastMathFlags(); + Flags = {VPI->getFastMathFlags()}; unsigned MulOpc = IVTy->isFloatingPointTy() ? Instruction::FMul : Instruction::Mul; VPInstruction *Mul = Builder.createNaryOp( - MulOpc, {VectorStep, ScalarStep}, FMFs, R.getDebugLoc()); + MulOpc, {VectorStep, ScalarStep}, Flags, R.getDebugLoc()); VectorStep = Mul; VPI->replaceAllUsesWith(VectorStep); ToRemove.push_back(VPI); diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp index ce83c276297c0..e1fb3d476c58d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp @@ -151,14 +151,14 @@ void UnrollState::unrollWidenInductionByUF( IV->getParent()->getEnclosingLoopRegion()->getSinglePredecessor()); Type *IVTy = TypeInfo.inferScalarType(IV); auto &ID = IV->getInductionDescriptor(); - std::optional FMFs; + VPIRFlags Flags; if (isa_and_present(ID.getInductionBinOp())) - FMFs = ID.getInductionBinOp()->getFastMathFlags(); + Flags = ID.getInductionBinOp()->getFastMathFlags(); VPValue *ScalarStep = IV->getStepValue(); VPBuilder Builder(PH); VPInstruction *VectorStep = Builder.createNaryOp( - VPInstruction::WideIVStep, {&Plan.getVF(), ScalarStep}, IVTy, FMFs, + VPInstruction::WideIVStep, {&Plan.getVF(), ScalarStep}, IVTy, Flags, IV->getDebugLoc()); ToSkip.insert(VectorStep); @@ -188,7 +188,7 @@ void UnrollState::unrollWidenInductionByUF( Prev, VectorStep, }, - FMFs, IV->getDebugLoc(), Name); + Flags, IV->getDebugLoc(), Name); ToSkip.insert(Add); addRecipeForPart(IV, Add, Part); Prev = Add; diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp index 7218311518594..8b4fd066aaa26 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -752,13 +752,13 @@ edge [fontname=Courier, fontsize=30] compound=true N0 [label = "preheader:\l" + - " EMIT vp\<%1\> = add\l" + + " EMIT vp\<%1\> = add \l" + "Successor(s): bb1\l" ] N0 -> N1 [ label=""] N1 [label = "bb1:\l" + - " EMIT vp\<%2\> = add\l" + + " EMIT vp\<%2\> = add \l" + " EMIT vp\<%3\> = sub vp\<%2\>\l" + " EMIT br vp\<%2\>, vp\<%3\>\l" + "Successor(s): bb2\l" @@ -780,7 +780,7 @@ compound=true EXPECT_EQ(ExpectedStr, FullDump); const char *ExpectedBlock1Str = R"(bb1: - EMIT vp<%2> = add + EMIT vp<%2> = add EMIT vp<%3> = sub vp<%2> EMIT br vp<%2>, vp<%3> Successor(s): bb2 @@ -842,11 +842,11 @@ TEST_F(VPBasicBlockTest, printPlanWithVFsAndUFs) { vp<%1> = original trip-count preheader: - EMIT vp<%1> = sub + EMIT vp<%1> = sub Successor(s): bb1 bb1: - EMIT vp<%2> = add + EMIT vp<%2> = add Successor(s): ir-bb ir-bb: @@ -866,11 +866,11 @@ No successors vp<%1> = original trip-count preheader: - EMIT vp<%1> = sub + EMIT vp<%1> = sub Successor(s): bb1 bb1: - EMIT vp<%2> = add + EMIT vp<%2> = add Successor(s): ir-bb ir-bb: @@ -890,11 +890,11 @@ No successors vp<%1> = original trip-count preheader: - EMIT vp<%1> = sub + EMIT vp<%1> = sub Successor(s): bb1 bb1: - EMIT vp<%2> = add + EMIT vp<%2> = add Successor(s): ir-bb ir-bb: @@ -932,7 +932,7 @@ compound=true N0 -> N1 [ label=""] N1 [label = "bb1:\l" + - " EMIT vp\<%1\> = add\l" + + " EMIT vp\<%1\> = add \l" + " EMIT vp\<%2\> = sub vp\<%1\>\l" + " EMIT br vp\<%1\>, vp\<%2\>\l" + "No successors\l"