From 5c96c8d51863fb6c2b96e06ffcd7b90280bb20ca Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sat, 10 Feb 2024 21:49:22 +0000 Subject: [PATCH 01/11] [VPlan] Track VPValue names in VPlan. This patch restructures the way names for printing VPValues are handled. It updates VPlan to track assigned names for VPValues (similar to how LLVMContext tracks names for LLVM IR Values). Tracking names in a single place allows to deduplicate names and ensure all names are unique. Now there are the following cases: * If a VPValue has a defining recipe, its name is determined by checking its associated VPlan's mapping. The name is wrapped in vp<>. If no VPlan is associated, is used. * If a VPValue is a live-in with an underlying IR value, the name of the IR value is used and wrapped in ir<>. * If none of the above applies, the number assigned by VPSlotTracker is used wrapped in vp<> if an associated VPlan is provided or otherwise. Note that contrary to LLMV's Value which always can retrieve LLVMContext, this is not possible for VPValues without defining recipe, so in some cases, the associated VPlan needs to be provided explicitly. The new handling subsumes VPInstruction's manual handling of its name. --- .../Vectorize/LoopVectorizationPlanner.h | 23 +- .../Transforms/Vectorize/LoopVectorize.cpp | 13 +- llvm/lib/Transforms/Vectorize/VPlan.cpp | 82 +++++- llvm/lib/Transforms/Vectorize/VPlan.h | 66 ++++- .../Transforms/Vectorize/VPlanHCFGBuilder.cpp | 9 +- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 12 +- .../Transforms/Vectorize/VPlanTransforms.cpp | 16 +- llvm/lib/Transforms/Vectorize/VPlanValue.h | 20 +- .../AArch64/sve-tail-folding-forced.ll | 14 +- .../LoopVectorize/AArch64/sve-widen-gep.ll | 20 +- .../AArch64/synthesize-mask-for-call.ll | 84 +++--- .../widen-call-with-intrinsic-or-libfunc.ll | 28 +- .../RISCV/riscv-vector-reverse.ll | 36 +-- .../first-order-recurrence-chains-vplan.ll | 46 ++-- ...-order-recurrence-sink-replicate-region.ll | 132 ++++----- .../Transforms/LoopVectorize/icmp-uniforms.ll | 10 +- .../interleave-and-scalarize-only.ll | 20 +- .../LoopVectorize/vplan-dot-printing.ll | 14 +- .../LoopVectorize/vplan-iv-transforms.ll | 6 +- .../vplan-printing-outer-loop.ll | 26 +- .../LoopVectorize/vplan-printing.ll | 252 +++++++++--------- .../vplan-sink-scalars-and-merge-vf1.ll | 8 +- .../vplan-sink-scalars-and-merge.ll | 250 ++++++++--------- .../vplan-unused-interleave-group.ll | 6 +- 24 files changed, 662 insertions(+), 531 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h index a7ebf78e54ceb..5aa4892bf0ffe 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -47,16 +47,19 @@ class VPBuilder { VPBasicBlock::iterator InsertPt = VPBasicBlock::iterator(); /// Insert \p VPI in BB at InsertPt if BB is set. - VPInstruction *tryInsertInstruction(VPInstruction *VPI) { - if (BB) + VPInstruction *tryInsertInstruction(VPInstruction *VPI, + const Twine &Name = "") { + if (BB) { BB->insert(VPI, InsertPt); + VPI->setName(Name); + } return VPI; } VPInstruction *createInstruction(unsigned Opcode, ArrayRef Operands, DebugLoc DL, const Twine &Name = "") { - return tryInsertInstruction(new VPInstruction(Opcode, Operands, DL, Name)); + return tryInsertInstruction(new VPInstruction(Opcode, Operands, DL), Name); } VPInstruction *createInstruction(unsigned Opcode, @@ -150,7 +153,7 @@ class VPBuilder { VPRecipeWithIRFlags::WrapFlagsTy WrapFlags, DebugLoc DL = {}, const Twine &Name = "") { return tryInsertInstruction( - new VPInstruction(Opcode, Operands, WrapFlags, DL, Name)); + new VPInstruction(Opcode, Operands, WrapFlags, DL), Name); } VPValue *createNot(VPValue *Operand, DebugLoc DL = {}, const Twine &Name = "") { @@ -170,12 +173,12 @@ class VPBuilder { VPValue *createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal, DebugLoc DL = {}, const Twine &Name = "", std::optional FMFs = std::nullopt) { - auto *Select = - FMFs ? new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal}, - *FMFs, DL, Name) - : new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal}, - DL, Name); - return tryInsertInstruction(Select); + auto *Select = FMFs + ? new VPInstruction(Instruction::Select, + {Cond, TrueVal, FalseVal}, *FMFs, DL) + : new VPInstruction(Instruction::Select, + {Cond, TrueVal, FalseVal}, DL); + return tryInsertInstruction(Select, Name); } /// Create a new ICmp VPInstruction with predicate \p Pred and operands \p A diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 1a7b301c35f2b..d08aa8fcb12c2 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7262,7 +7262,7 @@ VPValue *VPBuilder::createICmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B, assert(Pred >= CmpInst::FIRST_ICMP_PREDICATE && Pred <= CmpInst::LAST_ICMP_PREDICATE && "invalid predicate"); return tryInsertInstruction( - new VPInstruction(Instruction::ICmp, Pred, A, B, DL, Name)); + new VPInstruction(Instruction::ICmp, Pred, A, B, DL), Name); } // This function will select a scalable VF if the target supports scalable @@ -8610,11 +8610,11 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW, // IV by VF * UF. auto *CanonicalIVIncrement = new VPInstruction(Instruction::Add, {CanonicalIVPHI, &Plan.getVFxUF()}, - {HasNUW, false}, DL, "index.next"); + {HasNUW, false}, DL); CanonicalIVPHI->addOperand(CanonicalIVIncrement); VPBasicBlock *EB = TopRegion->getExitingBasicBlock(); - EB->appendRecipe(CanonicalIVIncrement); + EB->appendRecipe(CanonicalIVIncrement, "index.next"); // Add the BranchOnCount VPInstruction to the latch. VPInstruction *BranchBack = @@ -8781,8 +8781,9 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { CM.foldTailByMasking() || isa(Instr)) && "unexpected recipe needs moving"); Recipe->insertBefore(*HeaderVPBB, HeaderVPBB->getFirstNonPhi()); + Recipe->getVPSingleValue()->setName(Instr->getName()); } else - VPBB->appendRecipe(Recipe); + VPBB->appendRecipe(Recipe, Instr->getName()); } VPBlockUtils::insertBlockAfter(new VPBasicBlock(), VPBB); @@ -8842,6 +8843,8 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { J++; } MemberR->eraseFromParent(); + if (!Member->getType()->isVoidTy()) + VPIG->getVPValue(J - 1)->setName(Member->getName()); } } @@ -9097,7 +9100,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( // ensure that it comes after all of it's inputs, including CondOp. // Note that this transformation may leave over dead recipes (including // CurrentLink), which will be cleaned by a later VPlan transform. - LinkVPBB->appendRecipe(RedRecipe); + LinkVPBB->appendRecipe(RedRecipe, CurrentLinkI->getName()); CurrentLink->replaceAllUsesWith(RedRecipe); PreviousLink = RedRecipe; } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 2c0daa82afa59..bb603653f37dd 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -588,6 +588,15 @@ bool VPBasicBlock::isExiting() const { return getParent()->getExitingBasicBlock() == this; } +void VPBasicBlock::appendRecipe(VPRecipeBase *Recipe, const Twine &Name) { + insert(Recipe, end()); + if (Recipe->getNumDefinedValues() == 1) + Recipe->getVPSingleValue()->setName(Name); + else + assert(Name.str().empty() && "Provided a name for a recipe that does not " + "define exactly a single VPValue"); +} + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void VPBlockBase::printSuccessors(raw_ostream &O, const Twine &Indent) const { if (getSuccessors().empty()) { @@ -1053,8 +1062,10 @@ static void remapOperands(VPBlockBase *Entry, VPBlockBase *NewEntry, assert(OldR.getNumDefinedValues() == NewR.getNumDefinedValues() && "recipes must define the same number of operands"); for (const auto &[OldV, NewV] : - zip(OldR.definedValues(), NewR.definedValues())) + zip(OldR.definedValues(), NewR.definedValues())) { Old2NewVPValues[OldV] = NewV; + NewV->setName(OldV->getName()); + } } } @@ -1112,6 +1123,26 @@ VPlan *VPlan::duplicate() { return NewPlan; } +void VPlan::setName(const VPValue *V, const Twine &Name) { +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + std::string N = Name.str(); + if (N.empty()) + return; + assert(!VPValue2Name.contains(V)); + std::string CurrName = N; + + if (UsedNames.contains(N)) + CurrName = N + ".1"; + unsigned Cnt = 2; + while (UsedNames.contains(CurrName)) { + CurrName = N + "." + std::to_string(Cnt); + Cnt += 1; + } + VPValue2Name[V] = CurrName; + UsedNames.insert(CurrName); +#endif +} + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) Twine VPlanPrinter::getUID(const VPBlockBase *Block) { @@ -1298,18 +1329,12 @@ void VPValue::replaceUsesWithIf( #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void VPValue::printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const { - if (const Value *UV = getUnderlyingValue()) { + if (!getDefiningRecipe() && getUnderlyingValue()) { OS << "ir<"; - UV->printAsOperand(OS, false); + getUnderlyingValue()->printAsOperand(OS, false); OS << ">"; - return; - } - - unsigned Slot = Tracker.getSlot(this); - if (Slot == unsigned(-1)) - OS << ""; - else - OS << "vp<%" << Tracker.getSlot(this) << ">"; + } else + OS << Tracker.getName(this); } void VPUser::printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const { @@ -1319,6 +1344,31 @@ void VPUser::printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const { } #endif +std::string VPValue::getName() { +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + if (auto *Def = getDefiningRecipe()) + return Def->getParent()->getPlan()->getName(this); + assert(isLiveIn() && "called getName() on unsupported VPValue"); + return getLiveInIRValue()->getName().str(); +#else + return ""; +#endif +} + +void VPValue::setName(const Twine &Name) { +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + auto *Def = getDefiningRecipe(); + Def->getParent()->getPlan()->setName(this, Name); +#endif +} + +void VPValue::takeName(const VPValue *Src) { +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + auto *Def = getDefiningRecipe(); + Def->getParent()->getPlan()->takeName(Src, this); +#endif +} + void VPInterleavedAccessInfo::visitRegion(VPRegionBlock *Region, Old2NewTy &Old2New, InterleavedAccessInfo &IAI) { @@ -1397,6 +1447,16 @@ void VPSlotTracker::assignSlots(const VPBasicBlock *VPBB) { assignSlot(Def); } +std::string VPSlotTracker::getName(const VPValue *V) const { + if (!Plan) + return ""; + std::string N = Plan->getName(V); + if (!N.empty()) + return (Twine("vp<%") + N + ">").str(); + + return (Twine("vp<%") + std::to_string(getSlot(V)) + ">").str(); +} + bool vputils::onlyFirstLaneUsed(const VPValue *Def) { return all_of(Def->users(), [Def](const VPUser *U) { return U->onlyFirstLaneUsed(Def); }); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 162a3c4b195e5..08c0041f23ae0 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -32,6 +32,7 @@ #include "llvm/ADT/SmallBitVector.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringSet.h" #include "llvm/ADT/Twine.h" #include "llvm/ADT/ilist.h" #include "llvm/ADT/ilist_node.h" @@ -1162,9 +1163,6 @@ class VPInstruction : public VPRecipeWithIRFlags { typedef unsigned char OpcodeTy; OpcodeTy Opcode; - /// An optional name that can be used for the generated IR instruction. - const std::string Name; - /// Utility method serving execute(): generates a single instance of the /// modeled instruction. \returns the generated value for \p Part. /// In some cases an existing value is returned rather than a generated @@ -1181,31 +1179,30 @@ class VPInstruction : public VPRecipeWithIRFlags { void setUnderlyingInstr(Instruction *I) { setUnderlyingValue(I); } public: - VPInstruction(unsigned Opcode, ArrayRef Operands, DebugLoc DL, - const Twine &Name = "") + VPInstruction(unsigned Opcode, ArrayRef Operands, DebugLoc DL) : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, DL), - Opcode(Opcode), Name(Name.str()) {} + Opcode(Opcode) {} VPInstruction(unsigned Opcode, std::initializer_list Operands, - DebugLoc DL = {}, const Twine &Name = "") - : VPInstruction(Opcode, ArrayRef(Operands), DL, Name) {} + DebugLoc DL = {}) + : VPInstruction(Opcode, ArrayRef(Operands), DL) {} VPInstruction(unsigned Opcode, CmpInst::Predicate Pred, VPValue *A, - VPValue *B, DebugLoc DL = {}, const Twine &Name = ""); + VPValue *B, DebugLoc DL = {}); VPInstruction(unsigned Opcode, std::initializer_list Operands, - WrapFlagsTy WrapFlags, DebugLoc DL = {}, const Twine &Name = "") + WrapFlagsTy WrapFlags, DebugLoc DL = {}) : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, WrapFlags, DL), - Opcode(Opcode), Name(Name.str()) {} + Opcode(Opcode) {} VPInstruction(unsigned Opcode, std::initializer_list Operands, - FastMathFlags FMFs, DebugLoc DL = {}, const Twine &Name = ""); + FastMathFlags FMFs, DebugLoc DL = {}); VP_CLASSOF_IMPL(VPDef::VPInstructionSC) VPRecipeBase *clone() override { SmallVector Operands(operands()); - auto *New = new VPInstruction(Opcode, Operands, getDebugLoc(), Name); + auto *New = new VPInstruction(Opcode, Operands, getDebugLoc()); New->transferFlags(*this); return New; } @@ -2653,7 +2650,7 @@ class VPBasicBlock : public VPBlockBase { /// Augment the existing recipes of a VPBasicBlock with an additional /// \p Recipe as the last recipe. - void appendRecipe(VPRecipeBase *Recipe) { insert(Recipe, end()); } + void appendRecipe(VPRecipeBase *Recipe, const Twine &Name = ""); /// The method which generates the output IR instructions that correspond to /// this VPBasicBlock, thereby "executing" the VPlan. @@ -2876,6 +2873,11 @@ class VPlan { /// been modeled in VPlan directly. DenseMap SCEVToExpansion; +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + DenseMap VPValue2Name; + StringSet<> UsedNames; +#endif + public: /// Construct a VPlan with original preheader \p Preheader, trip count \p TC /// and \p Entry to the plan. At the moment, \p Preheader and \p Entry need to @@ -3016,6 +3018,42 @@ class VPlan { LLVM_DUMP_METHOD void dump() const; #endif + /// Set the name for \p V to \p Name if it is not empty. \p V must not have a + /// name assigned already. + void setName(const VPValue *V, const Twine &Name); + + /// Remove the assigned name for \p V, if there is one. + void removeName(const VPValue *V) { +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + if (VPValue2Name.contains(V)) { + UsedNames.erase(VPValue2Name[V]); + VPValue2Name.erase(V); + } +#endif + } + + /// Take the name for \p Src and move it to \p Dst. The name of \p Src will be + /// empty afterwards. + void takeName(const VPValue *Src, const VPValue *Dst) { +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + std::string N = VPValue2Name.lookup(Src); + if (N.empty()) + return; + VPValue2Name[Dst] = N; + VPValue2Name.erase(Src); +#endif + } + + /// Return the name assigned to \p V or an empty string if no name has been + /// assigned. + std::string getName(const VPValue *V) const { +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + return VPValue2Name.lookup(V); +#else + return ""; +#endif + } + /// Returns a range mapping the values the range \p Operands to their /// corresponding VPValues. iterator_range>> diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp index 94456bf858d9c..f5b34a740a21b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp @@ -297,7 +297,8 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB, if (Br->isConditional()) { VPValue *Cond = getOrCreateVPOperand(Br->getCondition()); VPBB->appendRecipe( - new VPInstruction(VPInstruction::BranchOnCond, {Cond})); + new VPInstruction(VPInstruction::BranchOnCond, {Cond}), + Inst->getName()); } // Skip the rest of the Instruction processing for Branch instructions. @@ -310,7 +311,7 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB, // an empty VPInstruction that we will fix once the whole plain CFG has // been built. NewVPV = new VPWidenPHIRecipe(Phi); - VPBB->appendRecipe(cast(NewVPV)); + VPBB->appendRecipe(cast(NewVPV), Inst->getName()); PhisToFix.push_back(Phi); } else { // Translate LLVM-IR operands into VPValue operands and set them in the @@ -321,8 +322,8 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB, // Build VPInstruction for any arbitrary Instruction without specific // representation in VPlan. - NewVPV = cast( - VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst)); + NewVPV = cast(VPIRBuilder.createNaryOp( + Inst->getOpcode(), VPOperands, Inst, Inst->getName())); } IRDef2VPValue[Inst] = NewVPV; diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 9ee0cb2bd6153..c24e0d72c222f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -226,6 +226,8 @@ void VPRecipeBase::removeFromParent() { iplist::iterator VPRecipeBase::eraseFromParent() { assert(getParent() && "Recipe not in any VPBasicBlock"); + for (auto *V : definedValues()) + getParent()->getPlan()->removeName(V); return getParent()->getRecipeList().erase(getIterator()); } @@ -255,20 +257,19 @@ FastMathFlags VPRecipeWithIRFlags::getFastMathFlags() const { } VPInstruction::VPInstruction(unsigned Opcode, CmpInst::Predicate Pred, - VPValue *A, VPValue *B, DebugLoc DL, - const Twine &Name) + VPValue *A, VPValue *B, DebugLoc DL) : VPRecipeWithIRFlags(VPDef::VPInstructionSC, ArrayRef({A, B}), Pred, DL), - Opcode(Opcode), Name(Name.str()) { + Opcode(Opcode) { assert(Opcode == Instruction::ICmp && "only ICmp predicates supported at the moment"); } VPInstruction::VPInstruction(unsigned Opcode, std::initializer_list Operands, - FastMathFlags FMFs, DebugLoc DL, const Twine &Name) + FastMathFlags FMFs, DebugLoc DL) : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, FMFs, DL), - Opcode(Opcode), Name(Name.str()) { + Opcode(Opcode) { // Make sure the VPInstruction is a floating-point operation. assert(isFPMathOp() && "this op can't take fast-math flags"); } @@ -277,6 +278,7 @@ Value *VPInstruction::generateInstruction(VPTransformState &State, unsigned Part) { IRBuilderBase &Builder = State.Builder; Builder.SetCurrentDebugLocation(getDebugLoc()); + std::string Name = getParent()->getPlan()->getName(this); if (Instruction::isBinaryOp(getOpcode())) { if (Part != 0 && vputils::onlyFirstPartUsed(this)) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 71f5285f90236..bb1db21970443 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -88,9 +88,10 @@ void VPlanTransforms::VPInstructionsToVPRecipes( } NewRecipe->insertBefore(&Ingredient); - if (NewRecipe->getNumDefinedValues() == 1) + if (NewRecipe->getNumDefinedValues() == 1) { VPV->replaceAllUsesWith(NewRecipe->getVPSingleValue()); - else + NewRecipe->getVPSingleValue()->takeName(VPV); + } else assert(NewRecipe->getNumDefinedValues() == 0 && "Only recpies with zero or one defined values expected"); Ingredient.eraseFromParent(); @@ -163,6 +164,8 @@ static bool sinkScalarOperands(VPlan &Plan) { // TODO: add ".cloned" suffix to name of Clone's VPValue. Clone->insertBefore(SinkCandidate); + if (!I->getName().empty()) + Clone->setName(I->getName() + ".cloned"); SinkCandidate->replaceUsesWithIf(Clone, [SinkTo](VPUser &U, unsigned) { return cast(&U)->getParent() != SinkTo; }); @@ -318,6 +321,8 @@ static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe, PHIRecipe->setOperand(0, RecipeWithoutMask); } PredRecipe->eraseFromParent(); + Plan.setName(RecipeWithoutMask, + RecipeWithoutMask->getUnderlyingValue()->getName()); auto *Exiting = new VPBasicBlock(Twine(RegionName) + ".continue", PHIRecipe); VPRegionBlock *Region = new VPRegionBlock(Entry, Exiting, RegionName, true); @@ -1177,10 +1182,11 @@ void VPlanTransforms::addActiveLaneMask( LaneMask = addVPLaneMaskPhiAndUpdateExitBranch( Plan, DataAndControlFlowWithoutRuntimeCheck); } else { - LaneMask = new VPInstruction(VPInstruction::ActiveLaneMask, - {WideCanonicalIV, Plan.getTripCount()}, - nullptr, "active.lane.mask"); + LaneMask = + new VPInstruction(VPInstruction::ActiveLaneMask, + {WideCanonicalIV, Plan.getTripCount()}, nullptr); LaneMask->insertAfter(WideCanonicalIV); + LaneMask->setName("active.lane.mask"); } // Walk users of WideCanonicalIV and replace all compares of the form diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h index c85f7715feaa2..e7ddc47248448 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -192,6 +192,21 @@ class VPValue { /// is a live-in value. /// TODO: Also handle recipes defined in pre-header blocks. bool isDefinedOutsideVectorRegions() const { return !hasDefiningRecipe(); } + + /// Returns the name associated with the VPValue. In case the VPValue has a + /// defining recipe, look up the name in the containing VPlan. In case it is a + /// live-in with an underlying IR value, return the name of the underlying + /// value. Calling getName() in other cases is invalid. + std::string getName(); + + /// Sets the name for this VPValue to \p Name. The VPValue must have a + /// defining recipe inserted in a VPlan. + void setName(const Twine &Name); + + /// Take the name for \p Src and move it to this VPValue. The name of \p Src + /// will be empty afterwards. The VPValue must have a defining recipe inserted + /// in a VPlan. + void takeName(const VPValue *Src); }; typedef DenseMap Value2VPValueTy; @@ -446,6 +461,7 @@ class VPBasicBlock; /// VPlan and allows querying the numbering for printing, similar to the /// ModuleSlotTracker for IR values. class VPSlotTracker { + const VPlan *Plan; DenseMap Slots; unsigned NextSlot = 0; @@ -454,7 +470,7 @@ class VPSlotTracker { void assignSlots(const VPBasicBlock *VPBB); public: - VPSlotTracker(const VPlan *Plan = nullptr) { + VPSlotTracker(const VPlan *Plan = nullptr) : Plan(Plan) { if (Plan) assignSlots(*Plan); } @@ -465,6 +481,8 @@ class VPSlotTracker { return -1; return I->second; } + + std::string getName(const VPValue *V) const; }; } // namespace llvm diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll index cc72dfa4ce639..33708bad471da 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll @@ -19,20 +19,20 @@ target triple = "aarch64-unknown-linux-gnu" ; VPLANS-EMPTY: ; VPLANS-NEXT: vector.ph: ; VPLANS-NEXT: EMIT vp<[[NEWTC:%[0-9]+]]> = TC > VF ? TC - VF : 0 vp<[[TC]]> -; VPLANS-NEXT: EMIT vp<[[VF:%[0-9]+]]> = VF * Part + ir<0> -; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY:%[0-9]+]]> = active lane mask vp<[[VF]]>, vp<[[TC]]> +; VPLANS-NEXT: EMIT vp<%index.part.next> = VF * Part + ir<0> +; VPLANS-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask vp<%index.part.next>, vp<[[TC]]> ; VPLANS-NEXT: Successor(s): vector loop ; VPLANS-EMPTY: ; VPLANS-NEXT: vector loop: { ; VPLANS-NEXT: vector.body: ; VPLANS-NEXT: EMIT vp<[[INDV:%[0-9]+]]> = CANONICAL-INDUCTION -; VPLANS-NEXT: ACTIVE-LANE-MASK-PHI vp<[[LANEMASK_PHI:%[0-9]+]]> = phi vp<[[LANEMASK_ENTRY]]>, vp<[[LANEMASK_LOOP:%[0-9]+]]> +; VPLANS-NEXT: ACTIVE-LANE-MASK-PHI vp<[[LANEMASK_PHI:%[0-9]+]]> = phi vp<%active.lane.mask.entry>, vp<[[LANEMASK_LOOP:%.+]]> ; VPLANS-NEXT: vp<[[STEP:%[0-9]+]]> = SCALAR-STEPS vp<[[INDV]]>, ir<1> -; VPLANS-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEP]]> -; VPLANS-NEXT: vp<[[VEC_PTR:%[0-9]+]]> = vector-pointer ir<%gep> +; VPLANS-NEXT: CLONE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEP]]> +; VPLANS-NEXT: vp<[[VEC_PTR:%[0-9]+]]> = vector-pointer vp<%gep> ; VPLANS-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%val>, vp<[[LANEMASK_PHI]]> -; VPLANS-NEXT: EMIT vp<[[INDV_UPDATE:%[0-9]+]]> = add vp<[[INDV]]>, vp<[[VFxUF]]> -; VPLANS-NEXT: EMIT vp<[[INC:%[0-9]+]]> = VF * Part + vp<[[INDV]]> +; VPLANS-NEXT: EMIT vp<[[INDV_UPDATE:%.+]]> = add vp<[[INDV]]>, vp<[[VFxUF]]> +; VPLANS-NEXT: EMIT vp<[[INC:%.+]]> = VF * Part + vp<[[INDV]]> ; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP]]> = active lane mask vp<[[INC]]>, vp<[[NEWTC]]> ; VPLANS-NEXT: EMIT vp<[[NOT:%[0-9]+]]> = not vp<[[LANEMASK_LOOP]]> ; VPLANS-NEXT: EMIT branch-on-cond vp<[[NOT]]> diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll index 899fcce5c02ab..eb7e5997b96c8 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll @@ -19,16 +19,16 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: EMIT ir<%ptr.iv.1> = WIDEN-POINTER-INDUCTION ir<%start.1>, 8 -; CHECK-NEXT: EMIT ir<%ptr.iv.2> = WIDEN-POINTER-INDUCTION ir<%start.2>, 1 -; CHECK-NEXT: WIDEN-GEP Var[Inv] ir<%ptr.iv.2.next> = getelementptr inbounds ir<%ptr.iv.2>, ir<1> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%ptr.iv.1> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%ptr.iv.2.next> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%ptr.iv.2> -; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR2]]> -; CHECK-NEXT: WIDEN ir<%add> = add ir<%lv>, ir<1> -; CHECK-NEXT: vp<[[VEC_PTR3:%.+]]> = vector-pointer ir<%ptr.iv.2> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR3]]>, ir<%add> +; CHECK-NEXT: EMIT vp<%ptr.iv.1> = WIDEN-POINTER-INDUCTION ir<%start.1>, 8 +; CHECK-NEXT: EMIT vp<%ptr.iv.2> = WIDEN-POINTER-INDUCTION ir<%start.2>, 1 +; CHECK-NEXT: WIDEN-GEP Var[Inv] vp<%ptr.iv.2.next> = getelementptr inbounds vp<%ptr.iv.2>, ir<1> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%ptr.iv.1> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%ptr.iv.2.next> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%ptr.iv.2> +; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR2]]> +; CHECK-NEXT: WIDEN vp<%add> = add vp<%lv>, ir<1> +; CHECK-NEXT: vp<[[VEC_PTR3:%.+]]> = vector-pointer vp<%ptr.iv.2> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR3]]>, vp<%add> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll index ad0d40ee7fbde..02e348a8e6f57 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll @@ -21,13 +21,13 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep> -; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: REPLICATE ir<%call> = call @foo(ir<%load>) -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call> +; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep> +; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: REPLICATE vp<%call> = call @foo(vp<%load>) +; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors @@ -50,13 +50,13 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[STEPS]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep> -; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed4_nomask) -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call> +; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep> +; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>) (using library function: foo_vector_fixed4_nomask) +; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors @@ -84,13 +84,13 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep> -; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed2_nomask) -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%call> +; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep> +; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>) (using library function: foo_vector_fixed2_nomask) +; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%call> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXST:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors @@ -113,13 +113,13 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep> -; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>, ir) (using library function: foo_vector_fixed4_mask) -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call> +; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep> +; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>, ir) (using library function: foo_vector_fixed4_mask) +; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors @@ -146,13 +146,13 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep> -; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed2_nomask) -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call> +; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep> +; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>) (using library function: foo_vector_fixed2_nomask) +; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors @@ -175,13 +175,13 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep> -; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed4_nomask) -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call> +; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep> +; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>) (using library function: foo_vector_fixed4_nomask) +; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll index f67b932b04a0d..9b8cbb37ea2de 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll @@ -19,13 +19,13 @@ target triple = "arm64-apple-ios" ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.src> -; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN-CAST ir<%conv> = fpext ir<%l> to double -; CHECK-NEXT: WIDEN-CALL ir<%s> = call @llvm.sin.f64(ir<%conv>) (using library function: __simd_sin_v2f64) -; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep.dst> +; CHECK-NEXT: CLONE vp<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.src> +; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN-CAST vp<%conv> = fpext vp<%l> to double +; CHECK-NEXT: WIDEN-CALL vp<%s> = call @llvm.sin.f64(vp<%conv>) (using library function: __simd_sin_v2f64) +; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE store vp<%s>, vp<%gep.dst> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors @@ -48,13 +48,13 @@ target triple = "arm64-apple-ios" ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.src> -; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN-CAST ir<%conv> = fpext ir<%l> to double -; CHECK-NEXT: WIDEN-CALL ir<%s> = call @llvm.sin.f64(ir<%conv>) (using vector intrinsic) -; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep.dst> +; CHECK-NEXT: CLONE vp<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.src> +; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN-CAST vp<%conv> = fpext vp<%l> to double +; CHECK-NEXT: WIDEN-CALL vp<%s> = call @llvm.sin.f64(vp<%conv>) (using vector intrinsic) +; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE store vp<%s>, vp<%gep.dst> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll index 1bcd7a2e009e0..ec70c0e3657fd 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll @@ -64,15 +64,15 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1> -; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1> -; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0> -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%arrayidx> -; CHECK-NEXT: WIDEN ir<%1> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN ir<%add9> = add ir<%1>, ir<1> -; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) ir<%arrayidx3> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add9> +; CHECK-NEXT: CLONE vp<%i.0> = add nsw vp<[[STEPS]]>, ir<-1> +; CHECK-NEXT: CLONE vp<%idxprom> = zext vp<%i.0> +; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%B>, vp<%idxprom> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) vp<%arrayidx> +; CHECK-NEXT: WIDEN vp<[[L:%.+]]> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN vp<%add9> = add vp<[[L]]>, ir<1> +; CHECK-NEXT: CLONE vp<%arrayidx3> = getelementptr inbounds ir<%A>, vp<%idxprom> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) vp<%arrayidx3> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add9> ; CHECK-NEXT: EMIT vp<[[IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -204,15 +204,15 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1> ; CHECK-NEXT: vp<[[STEPS]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1> -; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1> -; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0> -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%arrayidx> -; CHECK-NEXT: WIDEN ir<%1> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN ir<%conv1> = fadd ir<%1>, ir<1.000000e+00> -; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) ir<%arrayidx3> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%conv1> +; CHECK-NEXT: CLONE vp<%i.0> = add nsw vp<[[STEPS]]>, ir<-1> +; CHECK-NEXT: CLONE vp<%idxprom> = zext vp<%i.0> +; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%B>, vp<%idxprom> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) vp<%arrayidx> +; CHECK-NEXT: WIDEN vp<[[L:%.+]]> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN vp<%conv1> = fadd vp<[[L]]>, ir<1.000000e+00> +; CHECK-NEXT: CLONE vp<%arrayidx3> = getelementptr inbounds ir<%A>, vp<%idxprom> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) vp<%arrayidx3> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%conv1> ; CHECK-NEXT: EMIT vp<[[IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll index 15e77f3a48470..db85e9d8a34c9 100644 --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll @@ -15,17 +15,17 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) { ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.1> = phi ir<22>, ir<%for.1.next> -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.2> = phi ir<33>, vp<[[FOR1_SPLICE:%.+]]> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.1> = phi ir<22>, vp<%for.1.next> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.2> = phi ir<33>, vp<[[FOR1_SPLICE:%.+]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep.ptr> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.ptr> -; CHECK-NEXT: WIDEN ir<%for.1.next> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: EMIT vp<[[FOR1_SPLICE]]> = first-order splice ir<%for.1>, ir<%for.1.next> -; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE:%.+]]> = first-order splice ir<%for.2>, vp<[[FOR1_SPLICE]]> -; CHECK-NEXT: WIDEN ir<%add> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%gep.ptr> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add> +; CHECK-NEXT: CLONE vp<%gep.ptr> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.ptr> +; CHECK-NEXT: WIDEN vp<%for.1.next> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: EMIT vp<[[FOR1_SPLICE]]> = first-order splice vp<%for.1>, vp<%for.1.next> +; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE:%.+]]> = first-order splice vp<%for.2>, vp<[[FOR1_SPLICE]]> +; CHECK-NEXT: WIDEN vp<%add> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%gep.ptr> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors @@ -68,20 +68,20 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) { ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.1> = phi ir<22>, ir<%for.1.next> -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.2> = phi ir<33>, vp<[[FOR1_SPLICE:%.+]]> -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.3> = phi ir<33>, vp<[[FOR2_SPLICE:%.+]]> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.1> = phi ir<22>, vp<%for.1.next> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.2> = phi ir<33>, vp<[[FOR1_SPLICE:%.+]]> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.3> = phi ir<33>, vp<[[FOR2_SPLICE:%.+]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep.ptr> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.ptr> -; CHECK-NEXT: WIDEN ir<%for.1.next> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: EMIT vp<[[FOR1_SPLICE]]> = first-order splice ir<%for.1>, ir<%for.1.next> -; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE]]> = first-order splice ir<%for.2>, vp<[[FOR1_SPLICE]]> -; CHECK-NEXT: EMIT vp<[[FOR3_SPLICE:%.+]]> = first-order splice ir<%for.3>, vp<[[FOR2_SPLICE]]> -; CHECK-NEXT: WIDEN ir<%add.1> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]> -; CHECK-NEXT: WIDEN ir<%add.2> = add ir<%add.1>, vp<[[FOR3_SPLICE]]> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%gep.ptr> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add.2> +; CHECK-NEXT: CLONE vp<%gep.ptr> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.ptr> +; CHECK-NEXT: WIDEN vp<%for.1.next> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: EMIT vp<[[FOR1_SPLICE]]> = first-order splice vp<%for.1>, vp<%for.1.next> +; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE]]> = first-order splice vp<%for.2>, vp<[[FOR1_SPLICE]]> +; CHECK-NEXT: EMIT vp<[[FOR3_SPLICE:%.+]]> = first-order splice vp<%for.3>, vp<[[FOR2_SPLICE]]> +; CHECK-NEXT: WIDEN vp<%add.1> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]> +; CHECK-NEXT: WIDEN vp<%add.2> = add vp<%add.1>, vp<[[FOR3_SPLICE]]> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%gep.ptr> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add.2> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll index 833d55f09294e..06f7a7be352f4 100644 --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll @@ -20,10 +20,10 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<[[FOR:%.+]]> = phi ir<0>, vp<%conv> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> ; CHECK-NEXT: vp<[[STEPS:%.]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> +; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> ; CHECK-NEXT: Successor(s): pred.load ; CHECK-EMPTY: ; CHECK-NEXT: pred.load: { @@ -32,19 +32,19 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize ; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.if: -; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep> (S->V) +; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE vp<%lv> = load vp<%gep> (S->V) ; CHECK-NEXT: Successor(s): pred.load.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%lv> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = vp<%lv> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.0 ; CHECK-EMPTY: ; CHECK-NEXT: loop.0: -; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext vp<[[PRED1]]> to i32 -; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%0>, ir<%conv> +; CHECK-NEXT: WIDEN-CAST vp<%conv> = sext vp<[[PRED1]]> to i32 +; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<[[FOR]]>, vp<%conv> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -53,14 +53,14 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> -; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%add> = add ir<%conv>, ir<%rem> -; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.dst> +; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x> +; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE vp<%add> = add vp<%conv>, vp<%rem> +; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep.dst> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%rem> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%rem> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.2 @@ -111,11 +111,11 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%recur> = phi ir<0>, vp<%recur.next> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> -; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32 -; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur>, ir<%recur.next> +; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> +; CHECK-NEXT: WIDEN-CAST vp<%recur.next> = sext ir<%y> to i32 +; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%recur>, vp<%recur.next> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -124,15 +124,15 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> +; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%add> = add ir<%rem>, ir<%recur.next> -; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep> +; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE vp<%add> = add vp<%rem>, vp<%recur.next> +; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%rem> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%rem> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.1 @@ -181,12 +181,12 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next> -; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%and.red> = phi ir<1234>, ir<%and.red.next> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%recur> = phi ir<0>, vp<%recur.next> +; CHECK-NEXT: WIDEN-REDUCTION-PHI vp<%and.red> = phi ir<1234>, vp<%and.red.next> ; CHECK-NEXT: EMIT vp<[[WIDEN_CAN:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<[[WIDEN_CAN]]>, vp<[[BTC]]> -; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32 -; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur>, ir<%recur.next> +; CHECK-NEXT: WIDEN-CAST vp<%recur.next> = sext ir<%y> to i32 +; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%recur>, vp<%recur.next> ; CHECK-NEXT: Successor(s): pred.srem ; CHECK-EMPTY: ; CHECK-NEXT: pred.srem: { @@ -195,19 +195,19 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: Successor(s): pred.srem.if, pred.srem.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.srem.if: -; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> (S->V) +; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x> (S->V) ; CHECK-NEXT: Successor(s): pred.srem.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.srem.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%rem> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%rem> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.0 ; CHECK-EMPTY: ; CHECK-NEXT: loop.0: -; CHECK-NEXT: WIDEN ir<%add> = add vp<[[PRED]]>, ir<%recur.next> -; CHECK-NEXT: WIDEN ir<%and.red.next> = and ir<%and.red>, ir<%add> -; CHECK-NEXT: EMIT vp<[[SEL:%.+]]> = select vp<[[MASK]]>, ir<%and.red.next>, ir<%and.red> +; CHECK-NEXT: WIDEN vp<%add> = add vp<[[PRED]]>, vp<%recur.next> +; CHECK-NEXT: WIDEN vp<%and.red.next> = and vp<%and.red>, vp<%add> +; CHECK-NEXT: EMIT vp<[[SEL:%.+]]> = select vp<[[MASK]]>, vp<%and.red.next>, vp<%and.red> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -215,7 +215,7 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: ; CHECK-NEXT: middle.block: -; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%and.red>, vp<[[SEL]]> +; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result vp<%and.red>, vp<[[SEL]]> ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: Live-out i32 %res = vp<[[RED_RES]]> @@ -257,11 +257,11 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<[[FOR:%.+]]> = phi ir<0>, vp<%conv> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> -; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> +; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> +; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> ; CHECK-NEXT: Successor(s): pred.load ; CHECK-EMPTY: ; CHECK-NEXT: pred.load: { @@ -270,18 +270,18 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr ; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.if: -; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep> (S->V) +; CHECK-NEXT: REPLICATE vp<%lv> = load vp<%gep> (S->V) ; CHECK-NEXT: Successor(s): pred.load.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.0 ; CHECK-EMPTY: ; CHECK-NEXT: loop.0: -; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext vp<[[PRED]]> to i32 -; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%0>, ir<%conv> +; CHECK-NEXT: WIDEN-CAST vp<%conv> = sext vp<[[PRED]]> to i32 +; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<[[FOR]]>, vp<%conv> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK: pred.store: { @@ -290,18 +290,18 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> -; CHECK-NEXT: REPLICATE ir<%lv.2> = load ir<%gep> -; CHECK-NEXT: REPLICATE ir<%conv.lv.2> = sext ir<%lv.2> -; CHECK-NEXT: REPLICATE ir<%add.1> = add ir<%conv>, ir<%rem> -; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%add> = add ir<%add.1>, ir<%conv.lv.2> -; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.dst> +; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x> +; CHECK-NEXT: REPLICATE vp<%lv.2> = load vp<%gep> +; CHECK-NEXT: REPLICATE vp<%conv.lv.2> = sext vp<%lv.2> +; CHECK-NEXT: REPLICATE vp<%add.1> = add vp<%conv>, vp<%rem> +; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE vp<%add> = add vp<%add.1>, vp<%conv.lv.2> +; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep.dst> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%rem> -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%lv.2> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = vp<%rem> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%lv.2> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.3 @@ -360,11 +360,11 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%recur> = phi ir<0>, vp<%recur.next> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> -; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32 -; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur>, ir<%recur.next> +; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> +; CHECK-NEXT: WIDEN-CAST vp<%recur.next> = sext ir<%y> to i32 +; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%recur>, vp<%recur.next> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -374,17 +374,17 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> -; CHECK-NEXT: REPLICATE ir<%rem.div> = sdiv ir<20>, ir<%rem> -; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store ir<%rem.div>, ir<%gep> -; CHECK-NEXT: REPLICATE ir<%gep.2> = getelementptr ir<%dst.2>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store ir<%rem.div>, ir<%gep.2> +; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x> +; CHECK-NEXT: REPLICATE vp<%rem.div> = sdiv ir<20>, vp<%rem> +; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE store vp<%rem.div>, vp<%gep> +; CHECK-NEXT: REPLICATE vp<%gep.2> = getelementptr ir<%dst.2>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE store vp<%rem.div>, vp<%gep.2> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%rem> -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%rem.div> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%rem> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%rem.div> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.3 @@ -435,12 +435,12 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%.pn> = phi ir<0>, ir<[[L:%.+]]> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%.pn> = phi ir<0>, vp<[[L:%.+]]> ; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<2> + vp<[[CAN_IV]]> * ir<1> ; CHECK-NEXT: EMIT vp<[[WIDE_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]> ; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp ule vp<[[WIDE_IV]]>, vp<[[BTC]]> -; CHECK-NEXT: CLONE ir<[[L]]> = load ir<%src> -; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%.pn>, ir<[[L]]> +; CHECK-NEXT: CLONE vp<[[L]]> = load ir<%src> +; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%.pn>, vp<[[L]]> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -449,14 +449,14 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%val> = sdiv vp<[[SPLICE]]>, ir<%x> +; CHECK-NEXT: REPLICATE vp<%val> = sdiv vp<[[SPLICE]]>, ir<%x> ; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[SCALAR_STEPS]]> -; CHECK-NEXT: REPLICATE store ir<%val>, ir<%gep.dst> +; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr ir<%dst>, vp<[[SCALAR_STEPS]]> +; CHECK-NEXT: REPLICATE store vp<%val>, vp<%gep.dst> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_VAL:%.+]]> = ir<%val> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_VAL:%.+]]> = vp<%val> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.1 diff --git a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll index b4ab6f7e8ceb7..f0b8ded142210 100644 --- a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll +++ b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll @@ -49,9 +49,9 @@ for.end: ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[COND:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> -; CHECK-NEXT: WIDEN ir<%cond0> = icmp ult ir<%iv>, ir<13> -; CHECK-NEXT: WIDEN-SELECT ir<%s> = select ir<%cond0>, ir<10>, ir<20> +; CHECK-NEXT: EMIT vp<[[COND:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> +; CHECK-NEXT: WIDEN vp<%cond0> = icmp ult vp<%iv>, ir<13> +; CHECK-NEXT: WIDEN-SELECT vp<%s> = select vp<%cond0>, ir<10>, ir<20> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -61,8 +61,8 @@ for.end: ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep> +; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE store vp<%s>, vp<%gep> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: diff --git a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll index 81cc2024bb31a..54a66dd424fae 100644 --- a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll +++ b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll @@ -21,9 +21,9 @@ ; DBG-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; DBG-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%start> + vp<[[CAN_IV]]> * ir<1> ; DBG-NEXT: vp<[[IV_STEPS:%.]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1> -; DBG-NEXT: CLONE ir<%min> = call @llvm.smin.i32(vp<[[IV_STEPS]]>, ir<65535>) -; DBG-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%dst>, vp<[[IV_STEPS]]> -; DBG-NEXT: CLONE store ir<%min>, ir<%arrayidx> +; DBG-NEXT: CLONE vp<%min> = call @llvm.smin.i32(vp<[[IV_STEPS]]>, ir<65535>) +; DBG-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%dst>, vp<[[IV_STEPS]]> +; DBG-NEXT: CLONE store vp<%min>, vp<%arrayidx> ; DBG-NEXT: EMIT vp<[[INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; DBG-NEXT: EMIT branch-on-count vp<[[INC]]>, vp<[[VEC_TC]]> ; DBG-NEXT: No successors @@ -89,14 +89,14 @@ declare i32 @llvm.smin.i32(i32, i32) ; DBG-EMPTY: ; DBG-NEXT: pred.store.if: ; DBG-NEXT: vp<[[STEPS2:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; DBG-NEXT: CLONE ir<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS2]]> -; DBG-NEXT: CLONE ir<%l> = load ir<%gep.src> -; DBG-NEXT: CLONE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS2]]> -; DBG-NEXT: CLONE store ir<%l>, ir<%gep.dst> +; DBG-NEXT: CLONE vp<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS2]]> +; DBG-NEXT: CLONE vp<%l> = load vp<%gep.src> +; DBG-NEXT: CLONE vp<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS2]]> +; DBG-NEXT: CLONE store vp<%l>, vp<%gep.dst> ; DBG-NEXT: Successor(s): pred.store.continue ; DBG-EMPTY: ; DBG-NEXT: pred.store.continue: -; DBG-NEXT: PHI-PREDICATED-INSTRUCTION vp<{{.+}}> = ir<%l> +; DBG-NEXT: PHI-PREDICATED-INSTRUCTION vp<{{.+}}> = vp<%l> ; DBG-NEXT: No successors ; DBG-NEXT: } ; DBG-NEXT: Successor(s): cond.false.1 @@ -190,10 +190,10 @@ exit: ; DBG-NEXT: vector loop: { ; DBG-NEXT: vector.body: ; DBG-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; DBG-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, vp<[[SCALAR_STEPS:.+]]> +; DBG-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for> = phi ir<0>, vp<[[SCALAR_STEPS:.+]]> ; DBG-NEXT: SCALAR-CAST vp<[[TRUNC_IV:%.+]]> = trunc vp<[[CAN_IV]]> to i32 ; DBG-NEXT: vp<[[SCALAR_STEPS]]> = SCALAR-STEPS vp<[[TRUNC_IV]]>, vp<[[CAST]]> -; DBG-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%for>, vp<[[SCALAR_STEPS]]> +; DBG-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%for>, vp<[[SCALAR_STEPS]]> ; DBG-NEXT: CLONE store vp<[[SPLICE]]>, ir<%dst> ; DBG-NEXT: EMIT vp<[[IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; DBG-NEXT: EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VTC]]> diff --git a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll index 12b8e657aabf1..85d035f1ddaee 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll @@ -28,13 +28,13 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw ; CHECK-NEXT: "vector.body:\l" + ; CHECK-NEXT: " EMIT vp\<[[CAN_IV:%.+]]\> = CANONICAL-INDUCTION ir\<0\>, vp\<[[CAN_IV_NEXT:%.+]]\>\l" + ; CHECK-NEXT: " vp\<[[STEPS:%.+]]\> = SCALAR-STEPS vp\<[[CAN_IV]]\>, ir\<1\>\l" + -; CHECK-NEXT: " CLONE ir\<%arrayidx\> = getelementptr inbounds ir\<%y\>, vp\<[[STEPS]]\>\l" + -; CHECK-NEXT: " vp\<[[VEC_PTR:%.+]]\> = vector-pointer ir\<%arrayidx\>\l" + -; CHECK-NEXT: " WIDEN ir\<%lv\> = load vp\<[[VEC_PTR]]\>\l" + -; CHECK-NEXT: " WIDEN-CALL ir\<%call\> = call @llvm.sqrt.f32(ir\<%lv\>) (using vector intrinsic)\l" + -; CHECK-NEXT: " CLONE ir\<%arrayidx2\> = getelementptr inbounds ir\<%x\>, vp\<[[STEPS]]\>\l" + -; CHECK-NEXT: " vp\<[[VEC_PTR2:%.+]]\> = vector-pointer ir\<%arrayidx2\>\l" + -; CHECK-NEXT: " WIDEN store vp\<[[VEC_PTR2]]\>, ir\<%call\>\l" + +; CHECK-NEXT: " CLONE vp\<%arrayidx\> = getelementptr inbounds ir\<%y\>, vp\<[[STEPS]]\>\l" + +; CHECK-NEXT: " vp\<[[VEC_PTR:%.+]]\> = vector-pointer vp\<%arrayidx\>\l" + +; CHECK-NEXT: " WIDEN vp\<%lv\> = load vp\<[[VEC_PTR]]\>\l" + +; CHECK-NEXT: " WIDEN-CALL vp\<%call\> = call @llvm.sqrt.f32(vp\<%lv\>) (using vector intrinsic)\l" + +; CHECK-NEXT: " CLONE vp\<%arrayidx2\> = getelementptr inbounds ir\<%x\>, vp\<[[STEPS]]\>\l" + +; CHECK-NEXT: " vp\<[[VEC_PTR2:%.+]]\> = vector-pointer vp\<%arrayidx2\>\l" + +; CHECK-NEXT: " WIDEN store vp\<[[VEC_PTR2]]\>, vp\<%call\>\l" + ; CHECK-NEXT: " EMIT vp\<[[CAN_IV_NEXT]]\> = add nuw vp\<[[CAN_IV]]\>, vp\<[[VFxUF]]\>\l" + ; CHECK-NEXT: " EMIT branch-on-count vp\<[[CAN_IV_NEXT]]\>, vp\<[[VEC_TC]]\>\l" + ; CHECK-NEXT: "No successors\l" diff --git a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll index 7ab2459ada2ed..00996e7eee566 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll @@ -16,9 +16,9 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) { ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next.p, ir<1> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%dst>, vp<[[STEPS:%.+]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%iv> +; CHECK-NEXT: CLONE vp<%gep> = getelementptr inbounds ir<%dst>, vp<[[STEPS:%.+]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%iv> ; CHECK-NEXT: EMIT vp<[[CAN_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_INC]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll index 65df5973ac75b..1795c6a4fe33f 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll @@ -15,28 +15,28 @@ define void @foo(i64 %n) { ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: -; CHECK-NEXT: WIDEN-PHI ir<%outer.iv> = phi ir<0>, ir<%outer.iv.next> -; CHECK-NEXT: EMIT ir<%gep.1> = getelementptr ir<@arr2>, ir<0>, ir<%outer.iv> -; CHECK-NEXT: EMIT store ir<%outer.iv>, ir<%gep.1> -; CHECK-NEXT: EMIT ir<%add> = add ir<%outer.iv>, ir<%n> +; CHECK-NEXT: WIDEN-PHI vp<%outer.iv> = phi ir<0>, vp<%outer.iv.next> +; CHECK-NEXT: EMIT vp<%gep.1> = getelementptr ir<@arr2>, ir<0>, vp<%outer.iv> +; CHECK-NEXT: EMIT store vp<%outer.iv>, vp<%gep.1> +; CHECK-NEXT: EMIT vp<%add> = add vp<%outer.iv>, ir<%n> ; CHECK-NEXT: Successor(s): inner ; CHECK-EMPTY: ; CHECK-NEXT: inner: { ; CHECK-NEXT: inner: -; CHECK-NEXT: WIDEN-PHI ir<%inner.iv> = phi ir<0>, ir<%inner.iv.next> -; CHECK-NEXT: EMIT ir<%gep.2> = getelementptr ir<@arr>, ir<0>, ir<%inner.iv>, ir<%outer.iv> -; CHECK-NEXT: EMIT store ir<%add>, ir<%gep.2> -; CHECK-NEXT: EMIT ir<%inner.iv.next> = add ir<%inner.iv>, ir<1> -; CHECK-NEXT: EMIT ir<%inner.ec> = icmp ir<%inner.iv.next>, ir<8> -; CHECK-NEXT: EMIT branch-on-cond ir<%inner.ec> +; CHECK-NEXT: WIDEN-PHI vp<%inner.iv> = phi ir<0>, vp<%inner.iv.next> +; CHECK-NEXT: EMIT vp<%gep.2> = getelementptr ir<@arr>, ir<0>, vp<%inner.iv>, vp<%outer.iv> +; CHECK-NEXT: EMIT store vp<%add>, vp<%gep.2> +; CHECK-NEXT: EMIT vp<%inner.iv.next> = add vp<%inner.iv>, ir<1> +; CHECK-NEXT: EMIT vp<%inner.ec> = icmp vp<%inner.iv.next>, ir<8> +; CHECK-NEXT: EMIT branch-on-cond vp<%inner.ec> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): outer.latch ; CHECK-EMPTY: ; CHECK-NEXT: outer.latch: -; CHECK-NEXT: EMIT ir<%outer.iv.next> = add ir<%outer.iv>, ir<1> -; CHECK-NEXT: EMIT ir<%outer.ec> = icmp ir<%outer.iv.next>, ir<8> -; CHECK-NEXT: EMIT branch-on-cond ir<%outer.ec> +; CHECK-NEXT: EMIT vp<%outer.iv.next> = add vp<%outer.iv>, ir<1> +; CHECK-NEXT: EMIT vp<%outer.ec> = icmp vp<%outer.iv.next>, ir<8> +; CHECK-NEXT: EMIT branch-on-cond vp<%outer.ec> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): middle.block diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll index 89178953010fe..372e99583c406 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll @@ -20,13 +20,13 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx> -; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN-CALL ir<%call> = call @llvm.sqrt.f32(ir<%lv>) -; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx2> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call> +; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx> +; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN-CALL vp<%call> = call @llvm.sqrt.f32(vp<%lv>) +; CHECK-NEXT: CLONE vp<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx2> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -71,15 +71,15 @@ define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x, ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi %iv.next, 0, ir<1> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: WIDEN-GEP Inv[Var] ir<%arrayidx> = getelementptr inbounds ir<%y>, ir<%iv> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx> -; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN ir<%cmp> = icmp eq ir<%arrayidx>, ir<%z> -; CHECK-NEXT: WIDEN-SELECT ir<%sel> = select ir<%cmp>, ir<1.000000e+01>, ir<2.000000e+01> -; CHECK-NEXT: WIDEN ir<%add> = fadd ir<%lv>, ir<%sel> -; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx2> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add> +; CHECK-NEXT: WIDEN-GEP Inv[Var] vp<%arrayidx> = getelementptr inbounds ir<%y>, vp<%iv> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx> +; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN vp<%cmp> = icmp eq vp<%arrayidx>, ir<%z> +; CHECK-NEXT: WIDEN-SELECT vp<%sel> = select vp<%cmp>, ir<1.000000e+01>, ir<2.000000e+01> +; CHECK-NEXT: WIDEN vp<%add> = fadd vp<%lv>, vp<%sel> +; CHECK-NEXT: CLONE vp<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx2> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -124,12 +124,12 @@ define float @print_reduction(i64 %n, ptr noalias %y) { ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> -; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next> +; CHECK-NEXT: WIDEN-REDUCTION-PHI vp<%red> = phi ir<0.000000e+00>, vp<%red.next.1> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx> -; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>) +; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx> +; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: REDUCE vp<%red.next.1> = vp<%red> + fast reduce.fadd (vp<%lv>) ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -137,7 +137,7 @@ define float @print_reduction(i64 %n, ptr noalias %y) { ; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: ; CHECK-NEXT: middle.block: -; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%red>, ir<%red.next> +; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result vp<%red>, vp<%red.next.1> ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: Live-out float %red.next.lcssa = vp<[[RED_RES]]> @@ -173,12 +173,12 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> -; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next> +; CHECK-NEXT: WIDEN-REDUCTION-PHI vp<%red> = phi ir<0.000000e+00>, vp<%red.next.1> ; CHECK-NEXT: vp<[[IV:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[IV]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx> -; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>) (with final reduction value stored in invariant address sank outside of loop) +; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[IV]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx> +; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: REDUCE vp<%red.next.1> = vp<%red> + fast reduce.fadd (vp<%lv>) (with final reduction value stored in invariant address sank outside of loop) ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -186,7 +186,7 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no ; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: ; CHECK-NEXT: middle.block: -; CHECK-NEXT: EMIT vp<[[RED_RES:.+]]> = compute-reduction-result ir<%red>, ir<%red.next> +; CHECK-NEXT: EMIT vp<[[RED_RES:.+]]> = compute-reduction-result vp<%red>, vp<%red.next.1> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -227,30 +227,30 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) { ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: WIDEN-INDUCTION %i = phi 0, %i.next, ir<1> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: WIDEN ir<%cmp> = icmp ult ir<%i>, ir<5> +; CHECK-NEXT: WIDEN vp<%cmp> = icmp ult vp<%i>, ir<5> ; CHECK-NEXT: Successor(s): pred.udiv ; CHECK-EMPTY: ; CHECK-NEXT: pred.udiv: { ; CHECK-NEXT: pred.udiv.entry: -; CHECK-NEXT: BRANCH-ON-MASK ir<%cmp> +; CHECK-NEXT: BRANCH-ON-MASK vp<%cmp> ; CHECK-NEXT: Successor(s): pred.udiv.if, pred.udiv.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.udiv.if: -; CHECK-NEXT: REPLICATE ir<%tmp4> = udiv ir<%n>, vp<[[STEPS]]> (S->V) +; CHECK-NEXT: REPLICATE vp<%tmp4> = udiv ir<%n>, vp<[[STEPS]]> (S->V) ; CHECK-NEXT: Successor(s): pred.udiv.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.udiv.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%tmp4> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%tmp4> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): if.then.0 ; CHECK-EMPTY: ; CHECK-NEXT: if.then.0: -; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%cmp> -; CHECK-NEXT: BLEND ir<%d> = ir<0>/vp<[[NOT]]> vp<[[PRED]]>/ir<%cmp> -; CHECK-NEXT: CLONE ir<%idx> = getelementptr ir<%x>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%idx> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%d> +; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not vp<%cmp> +; CHECK-NEXT: BLEND vp<%d> = ir<0>/vp<[[NOT]]> vp<[[PRED]]>/vp<%cmp> +; CHECK-NEXT: CLONE vp<%idx> = getelementptr ir<%x>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%idx> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%d> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -303,19 +303,19 @@ define void @print_interleave_groups(i32 %C, i32 %D) { ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * ir<4> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<4> -; CHECK-NEXT: CLONE ir<%gep.AB.0> = getelementptr inbounds ir<@AB>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at %AB.0, ir<%gep.AB.0> -; CHECK-NEXT: ir<%AB.0> = load from index 0 -; CHECK-NEXT: ir<%AB.1> = load from index 1 -; CHECK-NEXT: ir<%AB.3> = load from index 3 -; CHECK-NEXT: CLONE ir<%iv.plus.3> = add vp<[[STEPS]]>, ir<3> -; CHECK-NEXT: WIDEN ir<%add> = add nsw ir<%AB.0>, ir<%AB.1> -; CHECK-NEXT: CLONE ir<%gep.CD.3> = getelementptr inbounds ir<@CD>, ir<0>, ir<%iv.plus.3> -; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at , ir<%gep.CD.3> -; CHECK-NEXT: store ir<%add> to index 0 +; CHECK-NEXT: CLONE vp<%gep.AB.0> = getelementptr inbounds ir<@AB>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at %AB.0, vp<%gep.AB.0> +; CHECK-NEXT: vp<%AB.0> = load from index 0 +; CHECK-NEXT: vp<%AB.1> = load from index 1 +; CHECK-NEXT: vp<%AB.3> = load from index 3 +; CHECK-NEXT: CLONE vp<%iv.plus.3> = add vp<[[STEPS]]>, ir<3> +; CHECK-NEXT: WIDEN vp<%add> = add nsw vp<%AB.0>, vp<%AB.1> +; CHECK-NEXT: CLONE vp<%gep.CD.3> = getelementptr inbounds ir<@CD>, ir<0>, vp<%iv.plus.3> +; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at , vp<%gep.CD.3> +; CHECK-NEXT: store vp<%add> to index 0 ; CHECK-NEXT: store ir<1> to index 1 ; CHECK-NEXT: store ir<2> to index 2 -; CHECK-NEXT: store ir<%AB.3> to index 3 +; CHECK-NEXT: store vp<%AB.3> to index 3 ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -370,16 +370,16 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) { ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> -; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.07> = phi ir<0.000000e+00>, ir<%muladd> +; CHECK-NEXT: WIDEN-REDUCTION-PHI vp<%sum.07> = phi ir<0.000000e+00>, vp<%muladd.1> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx> -; CHECK-NEXT: WIDEN ir<%l.a> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%b>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx2> -; CHECK-NEXT: WIDEN ir<%l.b> = load vp<[[VEC_PTR2]]> -; CHECK-NEXT: EMIT vp<[[FMUL:%.+]]> = fmul nnan ninf nsz ir<%l.a>, ir<%l.b> -; CHECK-NEXT: REDUCE ir<[[MULADD:%.+]]> = ir<%sum.07> + nnan ninf nsz reduce.fadd (vp<[[FMUL]]>) +; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx> +; CHECK-NEXT: WIDEN vp<%l.a> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: CLONE vp<%arrayidx2> = getelementptr inbounds ir<%b>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx2> +; CHECK-NEXT: WIDEN vp<%l.b> = load vp<[[VEC_PTR2]]> +; CHECK-NEXT: EMIT vp<[[FMUL:%.+]]> = fmul nnan ninf nsz vp<%l.a>, vp<%l.b> +; CHECK-NEXT: REDUCE vp<[[MULADD:%.+]]> = vp<%sum.07> + nnan ninf nsz reduce.fadd (vp<[[FMUL]]>) ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -387,7 +387,7 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) { ; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: ; CHECK-NEXT: middle.block: -; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%sum.07>, ir<[[MULADD]]> +; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result vp<%sum.07>, vp<[[MULADD]]> ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: Live-out float %muladd.lcssa = vp<[[RED_RES]]> @@ -426,15 +426,15 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%isd> = getelementptr inbounds ir<%asd>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%isd> -; CHECK-NEXT: WIDEN ir<%lsd> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN ir<%psd> = add nuw nsw ir<%lsd>, ir<23> -; CHECK-NEXT: WIDEN ir<%cmp1> = icmp slt ir<%lsd>, ir<100> -; CHECK-NEXT: EMIT vp<[[NOT1:%.+]]> = not ir<%cmp1>, !dbg /tmp/s.c:5:3 -; CHECK-NEXT: WIDEN ir<%cmp2> = icmp sge ir<%lsd>, ir<200> -; CHECK-NEXT: EMIT vp<[[SEL1:%.+]]> = select vp<[[NOT1]]>, ir<%cmp2>, ir, !dbg /tmp/s.c:5:21 -; CHECK-NEXT: EMIT vp<[[OR1:%.+]]> = or vp<[[SEL1]]>, ir<%cmp1> +; CHECK-NEXT: CLONE vp<%isd> = getelementptr inbounds ir<%asd>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%isd> +; CHECK-NEXT: WIDEN vp<%lsd> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN vp<%psd> = add nuw nsw vp<%lsd>, ir<23> +; CHECK-NEXT: WIDEN vp<%cmp1> = icmp slt vp<%lsd>, ir<100> +; CHECK-NEXT: EMIT vp<[[NOT1:%.+]]> = not vp<%cmp1>, !dbg /tmp/s.c:5:3 +; CHECK-NEXT: WIDEN vp<%cmp2> = icmp sge vp<%lsd>, ir<200> +; CHECK-NEXT: EMIT vp<[[SEL1:%.+]]> = select vp<[[NOT1]]>, vp<%cmp2>, ir, !dbg /tmp/s.c:5:21 +; CHECK-NEXT: EMIT vp<[[OR1:%.+]]> = or vp<[[SEL1]]>, vp<%cmp1> ; CHECK-NEXT: Successor(s): pred.sdiv ; CHECK-EMPTY: ; CHECK-NEXT: pred.sdiv: { @@ -443,21 +443,21 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db ; CHECK-NEXT: Successor(s): pred.sdiv.if, pred.sdiv.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.sdiv.if: -; CHECK-NEXT: REPLICATE ir<%sd1> = sdiv ir<%psd>, ir<%lsd> (S->V) +; CHECK-NEXT: REPLICATE vp<%sd1> = sdiv vp<%psd>, vp<%lsd> (S->V) ; CHECK-NEXT: Successor(s): pred.sdiv.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.sdiv.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI:%.+]]> = ir<%sd1> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI:%.+]]> = vp<%sd1> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): if.then.0 ; CHECK-EMPTY: ; CHECK-NEXT: if.then.0: -; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not ir<%cmp2> +; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not vp<%cmp2> ; CHECK-NEXT: EMIT vp<[[SEL2:%.+]]> = select vp<[[NOT1]]>, vp<[[NOT2]]>, ir -; CHECK-NEXT: BLEND ir<%ysd.0> = vp<[[PHI]]>/vp<[[OR1]]> ir<%psd>/vp<[[SEL2]]> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%isd> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%ysd.0> +; CHECK-NEXT: BLEND vp<%ysd.0> = vp<[[PHI]]>/vp<[[OR1]]> vp<%psd>/vp<[[SEL2]]> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%isd> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%ysd.0> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -521,12 +521,12 @@ define void @print_expand_scev(i64 %y, ptr %ptr) { ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: WIDEN-INDUCTION\l" + ; CHECK-NEXT: " %iv = phi %iv.next, 0\l" + -; CHECK-NEXT: " ir<%v2>, vp<[[EXP_SCEV]]> +; CHECK-NEXT: " vp<%v2>, vp<[[EXP_SCEV]]> ; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * vp<[[EXP_SCEV]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, vp<[[EXP_SCEV]]> -; CHECK-NEXT: WIDEN ir<%v3> = add nuw ir<%v2>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store ir<%v3>, ir<%gep> +; CHECK-NEXT: WIDEN vp<%v3> = add nuw vp<%v2>, ir<1> +; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE store vp<%v3>, vp<%gep> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors @@ -572,9 +572,9 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) { ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN ir<%add> = add ir<%iv>, ir<%off> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep> +; CHECK-NEXT: CLONE vp<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> +; CHECK-NEXT: WIDEN vp<%add> = add vp<%iv>, ir<%off> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep> ; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<0> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> @@ -585,7 +585,7 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) { ; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-EMPTY: -; CHECK-NEXT: Live-out i32 %lcssa = ir<%add> +; CHECK-NEXT: Live-out i32 %lcssa = vp<%add> ; CHECK-NEXT: } ; entry: @@ -619,15 +619,15 @@ define void @print_fast_math_flags(i64 %n, ptr noalias %y, ptr noalias %x, ptr % ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep.y> = getelementptr inbounds ir<%y>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.y> -; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN ir<%add> = fadd nnan ir<%lv>, ir<1.000000e+00> -; CHECK-NEXT: WIDEN ir<%mul> = fmul reassoc nnan ninf nsz arcp contract afn ir<%add>, ir<2.000000e+00> -; CHECK-NEXT: WIDEN ir<%div> = fdiv reassoc nsz contract ir<%mul>, ir<2.000000e+00> -; CHECK-NEXT: CLONE ir<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.x> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%div> +; CHECK-NEXT: CLONE vp<%gep.y> = getelementptr inbounds ir<%y>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.y> +; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN vp<%add> = fadd nnan vp<%lv>, ir<1.000000e+00> +; CHECK-NEXT: WIDEN vp<%mul> = fmul reassoc nnan ninf nsz arcp contract afn vp<%add>, ir<2.000000e+00> +; CHECK-NEXT: WIDEN vp<%div> = fdiv reassoc nsz contract vp<%mul>, ir<2.000000e+00> +; CHECK-NEXT: CLONE vp<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.x> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%div> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -672,14 +672,14 @@ define void @print_exact_flags(i64 %n, ptr noalias %x) { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.x> -; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN ir<%div.1> = udiv exact ir<%lv>, ir<20> -; CHECK-NEXT: WIDEN ir<%div.2> = udiv ir<%lv>, ir<60> -; CHECK-NEXT: WIDEN ir<%add> = add nuw nsw ir<%div.1>, ir<%div.2> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%gep.x> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add> +; CHECK-NEXT: CLONE vp<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.x> +; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN vp<%div.1> = udiv exact vp<%lv>, ir<20> +; CHECK-NEXT: WIDEN vp<%div.2> = udiv vp<%lv>, ir<60> +; CHECK-NEXT: WIDEN vp<%add> = add nuw nsw vp<%div.1>, vp<%div.2> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%gep.x> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -723,36 +723,36 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%ld.addr> = getelementptr inbounds ir<%src>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%ld.addr> -; CHECK-NEXT: WIDEN ir<%ld.value> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN ir<%ifcond> = fcmp oeq ir<%ld.value>, ir<5.000000e+00> +; CHECK-NEXT: CLONE vp<%ld.addr> = getelementptr inbounds ir<%src>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%ld.addr> +; CHECK-NEXT: WIDEN vp<%ld.value> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN vp<%ifcond> = fcmp oeq vp<%ld.value>, ir<5.000000e+00> ; CHECK-NEXT: Successor(s): pred.call ; CHECK-EMPTY: ; CHECK-NEXT: pred.call: { ; CHECK-NEXT: pred.call.entry: -; CHECK-NEXT: BRANCH-ON-MASK ir<%ifcond> +; CHECK-NEXT: BRANCH-ON-MASK vp<%ifcond> ; CHECK-NEXT: Successor(s): pred.call.if, pred.call.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.call.if: -; CHECK-NEXT: REPLICATE ir<%foo.ret.1> = call nnan ninf nsz @foo(ir<%ld.value>) (S->V) -; CHECK-NEXT: REPLICATE ir<%foo.ret.2> = call @foo(ir<%ld.value>) (S->V) +; CHECK-NEXT: REPLICATE vp<%foo.ret.1> = call nnan ninf nsz @foo(vp<%ld.value>) (S->V) +; CHECK-NEXT: REPLICATE vp<%foo.ret.2> = call @foo(vp<%ld.value>) (S->V) ; CHECK-NEXT: Successor(s): pred.call.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.call.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI1:%.+]]> = ir<%foo.ret.1> -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI2:%.+]]> = ir<%foo.ret.2> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI1:%.+]]> = vp<%foo.ret.1> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI2:%.+]]> = vp<%foo.ret.2> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): if.then.1 ; CHECK-EMPTY: ; CHECK-NEXT: if.then.1: -; CHECK-NEXT: WIDEN ir<%fadd> = fadd vp<[[PHI1]]>, vp<[[PHI2]]> -; CHECK-NEXT: EMIT vp<[[NOT_COND:%.+]]> = not ir<%ifcond> -; CHECK-NEXT: BLEND ir<%st.value> = ir<%ld.value>/vp<[[NOT_COND]]> ir<%fadd>/ir<%ifcond> -; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr inbounds ir<%dest>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%st.addr> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%st.value> +; CHECK-NEXT: WIDEN vp<%fadd> = fadd vp<[[PHI1]]>, vp<[[PHI2]]> +; CHECK-NEXT: EMIT vp<[[NOT_COND:%.+]]> = not vp<%ifcond> +; CHECK-NEXT: BLEND vp<%st.value> = vp<%ld.value>/vp<[[NOT_COND]]> vp<%fadd>/vp<%ifcond> +; CHECK-NEXT: CLONE vp<%st.addr> = getelementptr inbounds ir<%dest>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%st.addr> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%st.value> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -806,14 +806,14 @@ define void @print_disjoint_flags(i64 %n, ptr noalias %x) { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.x> -; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN ir<%or.1> = or disjoint ir<%lv>, ir<1> -; CHECK-NEXT: WIDEN ir<%or.2> = or ir<%lv>, ir<3> -; CHECK-NEXT: WIDEN ir<%add> = add nuw nsw ir<%or.1>, ir<%or.2> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.x> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%add> +; CHECK-NEXT: CLONE vp<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.x> +; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN vp<%or.1> = or disjoint vp<%lv>, ir<1> +; CHECK-NEXT: WIDEN vp<%or.2> = or vp<%lv>, ir<3> +; CHECK-NEXT: WIDEN vp<%add> = add nuw nsw vp<%or.1>, vp<%or.2> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.x> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%add> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -857,11 +857,11 @@ define void @zext_nneg(ptr noalias %p, ptr noalias %p1) { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%idx> = getelementptr ir<%p>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%idx> -; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN-CAST ir<%zext> = zext nneg ir<%l> -; CHECK-NEXT: REPLICATE store ir<%zext>, ir<%p1> +; CHECK-NEXT: CLONE vp<%idx> = getelementptr ir<%p>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%idx> +; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN-CAST vp<%zext> = zext nneg vp<%l> +; CHECK-NEXT: REPLICATE store vp<%zext>, ir<%p1> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll index 8c18c107b39fc..7bf5e6bd76bd3 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll @@ -19,9 +19,9 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%tmp2> = getelementptr ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: CLONE ir<%tmp3> = load ir<%tmp2> -; CHECK-NEXT: CLONE store ir<0>, ir<%tmp2> +; CHECK-NEXT: CLONE vp<%tmp2> = getelementptr ir<%ptr>, vp<[[STEPS]]> +; CHECK-NEXT: CLONE vp<%tmp3> = load vp<%tmp2> +; CHECK-NEXT: CLONE store ir<0>, vp<%tmp2> ; CHECK-NEXT: Successor(s): pred.store ; CHECK: pred.store: { @@ -30,7 +30,7 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK: pred.store.if: -; CHECK-NEXT: CLONE store ir<%tmp3>, ir<%tmp2> +; CHECK-NEXT: CLONE store vp<%tmp3>, vp<%tmp2> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK: pred.store.continue: diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll index 9b9c3e704852a..3019a7e29e4e3 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll @@ -27,7 +27,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> +; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> ; CHECK-NEXT: Successor(s): pred.store ; CHECK: pred.store: { @@ -37,16 +37,16 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 ; CHECK: pred.store.if: ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> -; CHECK-NEXT: REPLICATE ir<%add> = add ir<%lv.b>, ir<10> -; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]] -; CHECK-NEXT: REPLICATE ir<%mul> = mul ir<2>, ir<%add> -; CHECK-NEXT: REPLICATE store ir<%mul>, ir<%gep.a> +; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b> +; CHECK-NEXT: REPLICATE vp<%add> = add vp<%lv.b>, ir<10> +; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]] +; CHECK-NEXT: REPLICATE vp<%mul> = mul ir<2>, vp<%add> +; CHECK-NEXT: REPLICATE store vp<%mul>, vp<%gep.a> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -96,7 +96,7 @@ exit: ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> +; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> ; CHECK-NEXT: Successor(s): pred.load ; CHECK: pred.load: { @@ -106,17 +106,17 @@ exit: ; CHECK: pred.load.if: ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> +; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b> ; CHECK-NEXT: Successor(s): pred.load.continue ; CHECK: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK: loop.0: -; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<2> +; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, ir<2> ; CHECK-NEXT: Successor(s): pred.store ; CHECK: pred.store: { @@ -125,9 +125,9 @@ exit: ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul> -; CHECK-NEXT: REPLICATE ir<%add> = add vp<[[PRED]]>, ir<10> -; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.a> +; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul> +; CHECK-NEXT: REPLICATE vp<%add> = add vp<[[PRED]]>, ir<10> +; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep.a> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK: pred.store.continue: @@ -180,7 +180,7 @@ exit: ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> +; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> ; CHECK-NEXT: Successor(s): pred.load ; CHECK: pred.load: { @@ -190,18 +190,18 @@ exit: ; CHECK: pred.load.if: ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V) +; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b> (S->V) ; CHECK-NEXT: Successor(s): pred.load.continue ; CHECK: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK: loop.0: -; CHECK-NEXT: WIDEN ir<%add> = add vp<[[PRED]]>, ir<10> -; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<%add> +; CHECK-NEXT: WIDEN vp<%add> = add vp<[[PRED]]>, ir<10> +; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, vp<%add> ; CHECK-NEXT: Successor(s): pred.store ; CHECK: pred.store: { @@ -210,8 +210,8 @@ exit: ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul> -; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.a> +; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul> +; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep.a> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK: pred.store.continue: @@ -265,10 +265,10 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) { ; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<21> + vp<[[CAN_IV]]> * ir<1> ; CHECK-NEXT: EMIT vp<[[WIDE_CAN_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<[[WIDE_CAN_IV]]>, vp<[[BTC]]> -; CHECK-NEXT: CLONE ir<%gep.A.uniform> = getelementptr inbounds ir<%A>, ir<0> -; CHECK-NEXT: CLONE ir<%lv> = load ir<%gep.A.uniform> -; CHECK-NEXT: WIDEN ir<%cmp> = icmp ult ir<%iv>, ir<%k> -; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not ir<%cmp> +; CHECK-NEXT: CLONE vp<%gep.A.uniform> = getelementptr inbounds ir<%A>, ir<0> +; CHECK-NEXT: CLONE vp<%lv> = load vp<%gep.A.uniform> +; CHECK-NEXT: WIDEN vp<%cmp> = icmp ult vp<%iv>, ir<%k> +; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not vp<%cmp> ; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK]]>, vp<[[NOT2]]>, ir ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: @@ -279,8 +279,8 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) { ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep.B> = getelementptr inbounds ir<%B>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store ir<%lv>, ir<%gep.B> +; CHECK-NEXT: REPLICATE vp<%gep.B> = getelementptr inbounds ir<%B>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE store vp<%lv>, vp<%gep.B> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: @@ -337,10 +337,10 @@ define void @pred_cfg1(i32 %k, i32 %j) { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> -; CHECK-NEXT: WIDEN ir<%c.1> = icmp ult ir<%iv>, ir<%j> -; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10> -; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]>, ir<%c.1>, ir +; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> +; CHECK-NEXT: WIDEN vp<%c.1> = icmp ult vp<%iv>, ir<%j> +; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, ir<10> +; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]>, vp<%c.1>, ir ; CHECK-NEXT: Successor(s): pred.load ; CHECK-EMPTY: ; CHECK-NEXT: pred.load: { @@ -350,21 +350,21 @@ define void @pred_cfg1(i32 %k, i32 %j) { ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.if: ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V) +; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b> (S->V) ; CHECK-NEXT: Successor(s): pred.load.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): then.0.0 ; CHECK-EMPTY: ; CHECK-NEXT: then.0.0: -; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%c.1> +; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not vp<%c.1> ; CHECK-NEXT: EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]>, vp<[[NOT]]>, ir ; CHECK-NEXT: EMIT vp<[[OR:%.+]]> = or vp<[[MASK2]]>, vp<[[MASK3]]> -; CHECK-NEXT: BLEND ir<%p> = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]> +; CHECK-NEXT: BLEND vp<%p> = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -373,8 +373,8 @@ define void @pred_cfg1(i32 %k, i32 %j) { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul> -; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.a> +; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul> +; CHECK-NEXT: REPLICATE store vp<%p>, vp<%gep.a> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: @@ -437,11 +437,11 @@ define void @pred_cfg2(i32 %k, i32 %j) { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> -; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10> -; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j> -; CHECK-NEXT: WIDEN ir<%c.1> = icmp ugt ir<%iv>, ir<%j> -; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]>, ir<%c.0>, ir +; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> +; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, ir<10> +; CHECK-NEXT: WIDEN vp<%c.0> = icmp ult vp<%iv>, ir<%j> +; CHECK-NEXT: WIDEN vp<%c.1> = icmp ugt vp<%iv>, ir<%j> +; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]>, vp<%c.0>, ir ; CHECK-NEXT: Successor(s): pred.load ; CHECK-EMPTY: ; CHECK-NEXT: pred.load: { @@ -451,22 +451,22 @@ define void @pred_cfg2(i32 %k, i32 %j) { ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.if: ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V) +; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b> (S->V) ; CHECK-NEXT: Successor(s): pred.load.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): then.0.0 ; CHECK-EMPTY: ; CHECK-NEXT: then.0.0: -; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%c.0> +; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not vp<%c.0> ; CHECK-NEXT: EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]>, vp<[[NOT]]>, ir ; CHECK-NEXT: EMIT vp<[[OR:%.+]]> = or vp<[[MASK2]]>, vp<[[MASK3]]> -; CHECK-NEXT: BLEND ir<%p> = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]> -; CHECK-NEXT: EMIT vp<[[MASK4:%.+]]> = select vp<[[OR]]>, ir<%c.1>, ir +; CHECK-NEXT: BLEND vp<%p> = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]> +; CHECK-NEXT: EMIT vp<[[MASK4:%.+]]> = select vp<[[OR]]>, vp<%c.1>, ir ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -475,8 +475,8 @@ define void @pred_cfg2(i32 %k, i32 %j) { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul> -; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.a> +; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul> +; CHECK-NEXT: REPLICATE store vp<%p>, vp<%gep.a> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: @@ -546,10 +546,10 @@ define void @pred_cfg3(i32 %k, i32 %j) { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> -; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10> -; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j> -; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1:%.+]]>, ir<%c.0>, ir +; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> +; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, ir<10> +; CHECK-NEXT: WIDEN vp<%c.0> = icmp ult vp<%iv>, ir<%j> +; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1:%.+]]>, vp<%c.0>, ir ; CHECK-NEXT: Successor(s): pred.load ; CHECK-EMPTY: ; CHECK-NEXT: pred.load: { @@ -559,22 +559,22 @@ define void @pred_cfg3(i32 %k, i32 %j) { ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.if: ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> +; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b> ; CHECK-NEXT: Successor(s): pred.load.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): then.0.0 ; CHECK-EMPTY: ; CHECK-NEXT: then.0.0: -; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%c.0> +; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not vp<%c.0> ; CHECK-NEXT: EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]>, vp<[[NOT]]>, ir ; CHECK-NEXT: EMIT vp<[[MASK4:%.+]]> = or vp<[[MASK2]]>, vp<[[MASK3]]> -; CHECK-NEXT: BLEND ir<%p> = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]> -; CHECK-NEXT: EMIT vp<[[MASK5:%.+]]> = select vp<[[MASK4]]>, ir<%c.0>, ir +; CHECK-NEXT: BLEND vp<%p> = ir<0>/vp<[[MASK3]]> vp<[[PRED]]>/vp<[[MASK2]]> +; CHECK-NEXT: EMIT vp<[[MASK5:%.+]]> = select vp<[[MASK4]]>, vp<%c.0>, ir ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -583,10 +583,10 @@ define void @pred_cfg3(i32 %k, i32 %j) { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul> -; CHECK-NEXT: REPLICATE store ir<0>, ir<%gep.a> -; CHECK-NEXT: REPLICATE ir<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, ir<%mul> -; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.c> +; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul> +; CHECK-NEXT: REPLICATE store ir<0>, vp<%gep.a> +; CHECK-NEXT: REPLICATE vp<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, vp<%mul> +; CHECK-NEXT: REPLICATE store vp<%p>, vp<%gep.c> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: @@ -656,7 +656,7 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) { ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> +; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -665,26 +665,26 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a> -; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> -; CHECK-NEXT: REPLICATE ir<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store ir<%lv.a>, ir<%gep.c> -; CHECK-NEXT: REPLICATE store ir<%lv.b>, ir<%gep.a> +; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE vp<%lv.a> = load vp<%gep.a> +; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b> +; CHECK-NEXT: REPLICATE vp<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE store vp<%lv.a>, vp<%gep.c> +; CHECK-NEXT: REPLICATE store vp<%lv.b>, vp<%gep.a> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%lv.a> -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%lv.b> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = vp<%lv.a> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%lv.b> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.3 ; CHECK-EMPTY: ; CHECK-NEXT: loop.3: -; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j> -; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK]]>, ir<%c.0>, ir -; CHECK-NEXT: WIDEN ir<%mul> = mul vp<[[PRED1]]>, vp<[[PRED2]]> +; CHECK-NEXT: WIDEN vp<%c.0> = icmp ult vp<%iv>, ir<%j> +; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK]]>, vp<%c.0>, ir +; CHECK-NEXT: WIDEN vp<%mul> = mul vp<[[PRED1]]>, vp<[[PRED2]]> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -693,8 +693,8 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%gep.c.1> = getelementptr inbounds ir<@c>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store ir<%mul>, ir<%gep.c.1> +; CHECK-NEXT: REPLICATE vp<%gep.c.1> = getelementptr inbounds ir<@c>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE store vp<%mul>, vp<%gep.c.1> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: @@ -760,7 +760,7 @@ define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> +; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -770,15 +770,15 @@ define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) { ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a> -; CHECK-NEXT: REPLICATE ir<%div> = sdiv ir<%lv.a>, ir<%lv.a> -; CHECK-NEXT: REPLICATE store ir<%div>, ir<%gep.a> +; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE vp<%lv.a> = load vp<%gep.a> +; CHECK-NEXT: REPLICATE vp<%div> = sdiv vp<%lv.a>, vp<%lv.a> +; CHECK-NEXT: REPLICATE store vp<%div>, vp<%gep.a> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%lv.a> -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%div> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = vp<%lv.a> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%div> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.2 @@ -827,10 +827,10 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, vp<[[PRED:%.+]]> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for> = phi ir<0>, vp<[[PRED:%.+]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> -; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> +; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]> ; CHECK-NEXT: Successor(s): pred.load ; CHECK-EMPTY: ; CHECK-NEXT: pred.load: { @@ -839,17 +839,17 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) { ; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.if: -; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a> +; CHECK-NEXT: REPLICATE vp<%lv.a> = load vp<%gep.a> ; CHECK-NEXT: Successor(s): pred.load.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED]]> = ir<%lv.a> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED]]> = vp<%lv.a> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.0 ; CHECK-EMPTY: ; CHECK-NEXT: loop.0: -; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%for>, vp<[[PRED]]> +; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%for>, vp<[[PRED]]> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -858,12 +858,12 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%div> = sdiv vp<[[SPLICE]]>, vp<[[PRED]]> -; CHECK-NEXT: REPLICATE store ir<%div>, ir<%gep.a> +; CHECK-NEXT: REPLICATE vp<%div> = sdiv vp<[[SPLICE]]>, vp<[[PRED]]> +; CHECK-NEXT: REPLICATE store vp<%div>, vp<%gep.a> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%div> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%div> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.2 @@ -915,15 +915,15 @@ define void @update_multiple_users(ptr noalias %src, ptr noalias %dst, i1 %c) { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%l1> = load ir<%src> -; CHECK-NEXT: REPLICATE ir<%l2> = trunc ir<%l1> -; CHECK-NEXT: REPLICATE ir<%cmp> = icmp eq ir<%l1>, ir<0> -; CHECK-NEXT: REPLICATE ir<%sel> = select ir<%cmp>, ir<5>, ir<%l2> -; CHECK-NEXT: REPLICATE store ir<%sel>, ir<%dst> +; CHECK-NEXT: REPLICATE vp<%l1> = load ir<%src> +; CHECK-NEXT: REPLICATE vp<%l2> = trunc vp<%l1> +; CHECK-NEXT: REPLICATE vp<%cmp> = icmp eq vp<%l1>, ir<0> +; CHECK-NEXT: REPLICATE vp<%sel> = select vp<%cmp>, ir<5>, vp<%l2> +; CHECK-NEXT: REPLICATE store vp<%sel>, ir<%dst> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%l1> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%l1> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.then.1 @@ -973,11 +973,11 @@ define void @sinking_requires_duplication(ptr %addr) { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%addr>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep> -; CHECK-NEXT: WIDEN ir<%0> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN ir<%pred> = fcmp oeq ir<%0>, ir<0.000000e+00> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = not ir<%pred> +; CHECK-NEXT: CLONE vp<%gep.cloned> = getelementptr ir<%addr>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.cloned> +; CHECK-NEXT: WIDEN vp<[[L:%.+]]> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN vp<%pred> = fcmp oeq vp<[[L]]>, ir<0.000000e+00> +; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = not vp<%pred> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -986,8 +986,8 @@ define void @sinking_requires_duplication(ptr %addr) { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%addr>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store ir<1.000000e+01>, ir<%gep> +; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%addr>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE store ir<1.000000e+01>, vp<%gep> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: @@ -1055,14 +1055,14 @@ define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr n ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1> -; CHECK-NEXT: REPLICATE ir<%gep.src> = getelementptr inbounds ir<%src>, vp<[[SCALAR_STEPS]]> -; CHECK-NEXT: REPLICATE ir<%l> = load ir<%gep.src> -; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[SCALAR_STEPS]]> -; CHECK-NEXT: REPLICATE store ir<%l>, ir<%gep.dst> +; CHECK-NEXT: REPLICATE vp<%gep.src> = getelementptr inbounds ir<%src>, vp<[[SCALAR_STEPS]]> +; CHECK-NEXT: REPLICATE vp<%l> = load vp<%gep.src> +; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[SCALAR_STEPS]]> +; CHECK-NEXT: REPLICATE store vp<%l>, vp<%gep.dst> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_LOAD:%.+]]> = ir<%l> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_LOAD:%.+]]> = vp<%l> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.1 @@ -1113,12 +1113,12 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) { ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: EMIT ir<%ptr.iv> = WIDEN-POINTER-INDUCTION ir<%start>, -1 -; CHECK-NEXT: CLONE ir<%ptr.iv.next> = getelementptr inbounds ir<%ptr.iv>, ir<-1> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%ptr.iv.next> -; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN ir<%c.1> = icmp eq ir<%l>, ir<0> -; CHECK-NEXT: EMIT vp<[[NEG:%.+]]> = not ir<%c.1> +; CHECK-NEXT: EMIT vp<%ptr.iv> = WIDEN-POINTER-INDUCTION ir<%start>, -1 +; CHECK-NEXT: CLONE vp<%ptr.iv.next.cloned> = getelementptr inbounds vp<%ptr.iv>, ir<-1> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) vp<%ptr.iv.next.cloned> +; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN vp<%c.1> = icmp eq vp<%l>, ir<0> +; CHECK-NEXT: EMIT vp<[[NEG:%.+]]> = not vp<%c.1> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -1127,8 +1127,8 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%ptr.iv.next> = getelementptr inbounds ir<%ptr.iv>, ir<-1> -; CHECK-NEXT: REPLICATE store ir<95>, ir<%ptr.iv.next> +; CHECK-NEXT: REPLICATE vp<%ptr.iv.next> = getelementptr inbounds vp<%ptr.iv>, ir<-1> +; CHECK-NEXT: REPLICATE store ir<95>, vp<%ptr.iv.next> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: diff --git a/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll b/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll index 5ea27994b356d..27d81de260d3b 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll @@ -18,9 +18,9 @@ define void @test_unused_interleave(ptr %src, i32 %length) { ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%3> -; CHECK-NEXT: EMIT vp<%3> = add nuw vp<%2>, vp<%0> -; CHECK-NEXT: EMIT branch-on-count vp<%3>, vp<%1> +; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%index.next> +; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%2>, vp<%0> +; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<%1> ; CHECK-NEXT: No successors ; CHECK-NEXT: } entry: From d72a7b34e175d5ebb78f2300c37618fa2526bfcd Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 11 Mar 2024 09:11:28 +0000 Subject: [PATCH 02/11] Step --- .../Transforms/Vectorize/LoopVectorize.cpp | 3 +-- llvm/lib/Transforms/Vectorize/VPlan.cpp | 21 ++++++++++++------- llvm/lib/Transforms/Vectorize/VPlan.h | 19 ++++++++++------- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 9 +++----- .../Transforms/Vectorize/VPlanTransforms.cpp | 2 -- 5 files changed, 29 insertions(+), 25 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 850108fd80342..a16615c04730c 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8696,9 +8696,8 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { CM.foldTailByMasking() || isa(Instr)) && "unexpected recipe needs moving"); Recipe->insertBefore(*HeaderVPBB, HeaderVPBB->getFirstNonPhi()); - Recipe->getVPSingleValue()->setName(Instr->getName()); } else - VPBB->appendRecipe(Recipe, Instr->getName()); + VPBB->appendRecipe(Recipe); } VPBlockUtils::insertBlockAfter(new VPBasicBlock(), VPBB); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 6413c3f3aad01..459b3b9e3666f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -606,13 +606,19 @@ bool VPBasicBlock::isExiting() const { return getParent() && getParent()->getExitingBasicBlock() == this; } -void VPBasicBlock::appendRecipe(VPRecipeBase *Recipe, const Twine &Name) { - insert(Recipe, end()); - if (Recipe->getNumDefinedValues() == 1) - Recipe->getVPSingleValue()->setName(Name); - else - assert(Name.str().empty() && "Provided a name for a recipe that does not " - "define exactly a single VPValue"); +void VPBasicBlock::insert(VPRecipeBase *Recipe, iterator InsertPt) { + assert(Recipe && "No recipe to append."); + assert(!Recipe->Parent && "Recipe already in VPlan"); + Recipe->Parent = this; + Recipes.insert(InsertPt, Recipe); + + if (Recipe->getNumDefinedValues() == 1) { + VPValue *VPV = Recipe->getVPSingleValue(); + Value *UV = VPV->getUnderlyingValue(); + VPlan *ParentPlan = getPlan(); + if (UV && ParentPlan && !ParentPlan->hasName(VPV)) + VPV->setName(UV->getName()); + } } #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) @@ -1140,7 +1146,6 @@ void VPlan::setName(const VPValue *V, const Twine &Name) { std::string N = Name.str(); if (N.empty()) return; - assert(!VPValue2Name.contains(V)); std::string CurrName = N; if (UsedNames.contains(N)) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 6708c0f050b72..b99827321eb03 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2647,16 +2647,13 @@ class VPBasicBlock : public VPBlockBase { return V->getVPBlockID() == VPBlockBase::VPBasicBlockSC; } - void insert(VPRecipeBase *Recipe, iterator InsertPt) { - assert(Recipe && "No recipe to append."); - assert(!Recipe->Parent && "Recipe already in VPlan"); - Recipe->Parent = this; - Recipes.insert(InsertPt, Recipe); - } + void insert(VPRecipeBase *Recipe, iterator InsertPt); /// Augment the existing recipes of a VPBasicBlock with an additional /// \p Recipe as the last recipe. - void appendRecipe(VPRecipeBase *Recipe, const Twine &Name = ""); + void appendRecipe(VPRecipeBase *Recipe, const Twine &Name = "") { + insert(Recipe, end()); + } /// The method which generates the output IR instructions that correspond to /// this VPBasicBlock, thereby "executing" the VPlan. @@ -3066,6 +3063,14 @@ class VPlan { #endif } + bool hasName(const VPValue *V) const { +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + return VPValue2Name.contains(V); +#else + return false; +#endif + } + /// Returns a range mapping the values the range \p Operands to their /// corresponding VPValues. iterator_range>> diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 5b65b1959f9ad..3e63010a37b85 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -198,24 +198,21 @@ void VPRecipeBase::insertBefore(VPRecipeBase *InsertPos) { assert(!Parent && "Recipe already in some VPBasicBlock"); assert(InsertPos->getParent() && "Insertion position not in any VPBasicBlock"); - Parent = InsertPos->getParent(); - Parent->getRecipeList().insert(InsertPos->getIterator(), this); + InsertPos->getParent()->insert(this, InsertPos->getIterator()); } void VPRecipeBase::insertBefore(VPBasicBlock &BB, iplist::iterator I) { assert(!Parent && "Recipe already in some VPBasicBlock"); assert(I == BB.end() || I->getParent() == &BB); - Parent = &BB; - BB.getRecipeList().insert(I, this); + BB.insert(this, I); } void VPRecipeBase::insertAfter(VPRecipeBase *InsertPos) { assert(!Parent && "Recipe already in some VPBasicBlock"); assert(InsertPos->getParent() && "Insertion position not in any VPBasicBlock"); - Parent = InsertPos->getParent(); - Parent->getRecipeList().insertAfter(InsertPos->getIterator(), this); + InsertPos->getParent()->insert(this, std::next(InsertPos->getIterator())); } void VPRecipeBase::removeFromParent() { diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 6ed3e95d06142..58f3153640e62 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -163,8 +163,6 @@ static bool sinkScalarOperands(VPlan &Plan) { // TODO: add ".cloned" suffix to name of Clone's VPValue. Clone->insertBefore(SinkCandidate); - if (!I->getName().empty()) - Clone->setName(I->getName() + ".cloned"); SinkCandidate->replaceUsesWithIf(Clone, [SinkTo](VPUser &U, unsigned) { return cast(&U)->getParent() != SinkTo; }); From 180015691116ef22e8a69e6d12b02245313bc7dc Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 14 Mar 2024 11:27:01 +0000 Subject: [PATCH 03/11] fixup --- .../Transforms/Vectorize/LoopVectorize.cpp | 5 ++-- llvm/lib/Transforms/Vectorize/VPlan.cpp | 14 +++++------ llvm/lib/Transforms/Vectorize/VPlan.h | 4 +--- .../Transforms/Vectorize/VPlanHCFGBuilder.cpp | 2 +- .../Transforms/Vectorize/VPlanTransforms.cpp | 7 +++--- .../vplan-printing-before-execute.ll | 24 +++++++++---------- 6 files changed, 26 insertions(+), 30 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index a16615c04730c..6141d5454f36b 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8754,11 +8754,10 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { if (!Member->getType()->isVoidTy()) { VPValue *OriginalV = MemberR->getVPSingleValue(); OriginalV->replaceAllUsesWith(VPIG->getVPValue(J)); + VPIG->getVPValue(J)->takeName(MemberR->getVPSingleValue()); J++; } MemberR->eraseFromParent(); - if (!Member->getType()->isVoidTy()) - VPIG->getVPValue(J - 1)->setName(Member->getName()); } } @@ -9016,7 +9015,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( // ensure that it comes after all of it's inputs, including CondOp. // Note that this transformation may leave over dead recipes (including // CurrentLink), which will be cleaned by a later VPlan transform. - LinkVPBB->appendRecipe(RedRecipe, CurrentLinkI->getName()); + LinkVPBB->appendRecipe(RedRecipe); CurrentLink->replaceAllUsesWith(RedRecipe); PreviousLink = RedRecipe; } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 459b3b9e3666f..ba83f39e8c340 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -612,13 +612,13 @@ void VPBasicBlock::insert(VPRecipeBase *Recipe, iterator InsertPt) { Recipe->Parent = this; Recipes.insert(InsertPt, Recipe); - if (Recipe->getNumDefinedValues() == 1) { - VPValue *VPV = Recipe->getVPSingleValue(); - Value *UV = VPV->getUnderlyingValue(); - VPlan *ParentPlan = getPlan(); - if (UV && ParentPlan && !ParentPlan->hasName(VPV)) - VPV->setName(UV->getName()); - } + if (Recipe->getNumDefinedValues() != 1) + return; + VPValue *VPV = Recipe->getVPSingleValue(); + Value *UV = VPV->getUnderlyingValue(); + VPlan *ParentPlan = getPlan(); + if (UV && ParentPlan && !ParentPlan->hasName(VPV)) + VPV->setName(UV->getName()); } #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index b99827321eb03..0ca25ee0cf1c5 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2651,9 +2651,7 @@ class VPBasicBlock : public VPBlockBase { /// Augment the existing recipes of a VPBasicBlock with an additional /// \p Recipe as the last recipe. - void appendRecipe(VPRecipeBase *Recipe, const Twine &Name = "") { - insert(Recipe, end()); - } + void appendRecipe(VPRecipeBase *Recipe) { insert(Recipe, end()); } /// The method which generates the output IR instructions that correspond to /// this VPBasicBlock, thereby "executing" the VPlan. diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp index eb1c401ed5c8a..963acaa08ba64 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp @@ -309,7 +309,7 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB, // an empty VPInstruction that we will fix once the whole plain CFG has // been built. NewVPV = new VPWidenPHIRecipe(Phi); - VPBB->appendRecipe(cast(NewVPV), Inst->getName()); + VPBB->appendRecipe(cast(NewVPV)); PhisToFix.push_back(Phi); } else { // Translate LLVM-IR operands into VPValue operands and set them in the diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 3c9fc13a801b0..84c0f4063d75a 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -160,9 +160,8 @@ static bool sinkScalarOperands(VPlan &Plan) { Instruction *I = cast( cast(SinkCandidate)->getUnderlyingValue()); auto *Clone = new VPReplicateRecipe(I, SinkCandidate->operands(), true); - // TODO: add ".cloned" suffix to name of Clone's VPValue. - Clone->insertBefore(SinkCandidate); + Clone->setName(SinkCandidate->getName() + ".cloned"); SinkCandidate->replaceUsesWithIf(Clone, [SinkTo](VPUser &U, unsigned) { return cast(&U)->getParent() != SinkTo; }); @@ -317,9 +316,9 @@ static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe, PredRecipe->replaceAllUsesWith(PHIRecipe); PHIRecipe->setOperand(0, RecipeWithoutMask); } + Plan.takeName(PredRecipe, RecipeWithoutMask); + PredRecipe->eraseFromParent(); - Plan.setName(RecipeWithoutMask, - RecipeWithoutMask->getUnderlyingValue()->getName()); auto *Exiting = new VPBasicBlock(Twine(RegionName) + ".continue", PHIRecipe); VPRegionBlock *Region = new VPRegionBlock(Entry, Exiting, RegionName, true); diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll index 1dddbfe20a2ed..dde5ffc1a6fd3 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll @@ -23,12 +23,12 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) { ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> -; CHECK-NEXT: EMIT ir<%p.src> = WIDEN-POINTER-INDUCTION ir<%A>, 1 -; CHECK-NEXT: vp<[[VPTR:%.]]> = vector-pointer ir<%p.src> -; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VPTR]]> -; CHECK-NEXT: WIDEN ir<%add> = add nsw ir<%l>, ir<10> -; CHECK-NEXT: vp<[[VPTR2:%.+]]> = vector-pointer ir<%p.src> -; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, ir<%add> +; CHECK-NEXT: EMIT vp<%p.src> = WIDEN-POINTER-INDUCTION ir<%A>, 1 +; CHECK-NEXT: vp<[[VPTR:%.]]> = vector-pointer vp<%p.src> +; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VPTR]]> +; CHECK-NEXT: WIDEN vp<%add> = add nsw vp<%l>, ir<10> +; CHECK-NEXT: vp<[[VPTR2:%.+]]> = vector-pointer vp<%p.src> +; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, vp<%add> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV:%.+]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors @@ -54,12 +54,12 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) { ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> -; CHECK-NEXT: EMIT ir<%p.src> = WIDEN-POINTER-INDUCTION ir<%A>, 1 -; CHECK-NEXT: vp<[[VPTR:%.]]> = vector-pointer ir<%p.src> -; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VPTR]]> -; CHECK-NEXT: WIDEN ir<%add> = add nsw ir<%l>, ir<10> -; CHECK-NEXT: vp<[[VPTR2:%.+]]> = vector-pointer ir<%p.src> -; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, ir<%add> +; CHECK-NEXT: EMIT vp<%p.src> = WIDEN-POINTER-INDUCTION ir<%A>, 1 +; CHECK-NEXT: vp<[[VPTR:%.]]> = vector-pointer vp<%p.src> +; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VPTR]]> +; CHECK-NEXT: WIDEN vp<%add> = add nsw vp<%l>, ir<10> +; CHECK-NEXT: vp<[[VPTR2:%.+]]> = vector-pointer vp<%p.src> +; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, vp<%add> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV:%.+]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-cond ir ; CHECK-NEXT: No successors From 84117c241b7b9998c2487f786d29d68c9b1d932a Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 27 Mar 2024 14:54:52 +0000 Subject: [PATCH 04/11] !fixup merge --- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 4 +++- .../Transforms/Vectorize/VPlanTransforms.cpp | 3 ++- .../LoopVectorize/AArch64/sve-widen-gep.ll | 16 ++++++------- .../LoopVectorize/X86/small-size.ll | 24 +++++++++---------- .../vplan-printing-before-execute.ll | 12 +++++----- .../vplan-sink-scalars-and-merge.ll | 14 +++++------ 6 files changed, 38 insertions(+), 35 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index d1e9748fa397b..99f034cfd17d3 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -296,15 +296,17 @@ bool VPInstruction::canGenerateScalarForFirstLane() const { Value *VPInstruction::generatePerLane(VPTransformState &State, const VPIteration &Lane) { IRBuilderBase &Builder = State.Builder; - assert(getOpcode() == VPInstruction::PtrAdd && "only PtrAdd opcodes are supported for now"); + +std::string Name = getName(); return Builder.CreatePtrAdd(State.get(getOperand(0), Lane), State.get(getOperand(1), Lane), Name); } Value *VPInstruction::generatePerPart(VPTransformState &State, unsigned Part) { IRBuilderBase &Builder = State.Builder; +std::string Name = getName(); if (Instruction::isBinaryOp(getOpcode())) { bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index b250fd2383143..b6bda9f791feb 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -582,10 +582,11 @@ static void legalizeAndOptimizeInductions(VPlan &Plan, ScalarEvolution &SE) { auto *Recipe = new VPInstruction(VPInstruction::PtrAdd, {PtrIV->getStartValue(), Steps->getVPSingleValue()}, - PtrIV->getDebugLoc(), "next.gep"); + PtrIV->getDebugLoc()); Recipe->insertAfter(Steps); PtrIV->replaceAllUsesWith(Recipe); + Recipe->setName("next.gep"); continue; } diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll index 3bab341e1c248..ef0027ce4ed9a 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll @@ -19,18 +19,18 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: EMIT ir<%ptr.iv.2> = WIDEN-POINTER-INDUCTION ir<%start.2>, 1 +; CHECK-NEXT: EMIT vp<%ptr.iv.2> = WIDEN-POINTER-INDUCTION ir<%start.2>, 1 ; CHECK-NEXT: vp<[[PTR_IDX:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * ir<8> ; CHECK-NEXT: vp<[[PTR_IDX_STEPS:%.+]]> = SCALAR-STEPS vp<[[PTR_IDX]]>, ir<8> ; CHECK-NEXT: EMIT vp<[[PTR_IV_1:%.+]]> = ptradd ir<%start.1>, vp<[[PTR_IDX_STEPS]]> -; CHECK-NEXT: WIDEN-GEP Var[Inv] ir<%ptr.iv.2.next> = getelementptr inbounds ir<%ptr.iv.2>, ir<1> +; CHECK-NEXT: WIDEN-GEP Var[Inv] vp<%ptr.iv.2.next> = getelementptr inbounds vp<%ptr.iv.2>, ir<1> ; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<[[PTR_IV_1]]> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%ptr.iv.2.next> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%ptr.iv.2> -; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR2]]> -; CHECK-NEXT: WIDEN ir<%add> = add ir<%lv>, ir<1> -; CHECK-NEXT: vp<[[VEC_PTR3:%.+]]> = vector-pointer ir<%ptr.iv.2> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR3]]>, ir<%add> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%ptr.iv.2.next> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%ptr.iv.2> +; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR2]]> +; CHECK-NEXT: WIDEN vp<%add> = add vp<%lv>, ir<1> +; CHECK-NEXT: vp<[[VEC_PTR3:%.+]]> = vector-pointer vp<%ptr.iv.2> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR3]]>, vp<%add> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors diff --git a/llvm/test/Transforms/LoopVectorize/X86/small-size.ll b/llvm/test/Transforms/LoopVectorize/X86/small-size.ll index dc474fbf67ce8..5fde9a4947712 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/small-size.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/small-size.ll @@ -291,7 +291,7 @@ define void @example3(i32 %n, ptr noalias nocapture %p, ptr noalias nocapture %q ; CHECK: pred.store.continue: ; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP3]], i64 1 ; CHECK-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF15:%.*]], label [[PRED_STORE_CONTINUE16:%.*]] -; CHECK: pred.store.if15: +; CHECK: pred.store.if14: ; CHECK-NEXT: [[TMP7:%.*]] = or disjoint i64 [[OFFSET_IDX]], 4 ; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP7]] ; CHECK-NEXT: [[TMP8:%.*]] = or disjoint i64 [[OFFSET_IDX8]], 4 @@ -299,10 +299,10 @@ define void @example3(i32 %n, ptr noalias nocapture %p, ptr noalias nocapture %q ; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[NEXT_GEP10]], align 16 ; CHECK-NEXT: store i32 [[TMP9]], ptr [[NEXT_GEP5]], align 16 ; CHECK-NEXT: br label [[PRED_STORE_CONTINUE16]] -; CHECK: pred.store.continue16: +; CHECK: pred.store.continue15: ; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2 ; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_STORE_IF17:%.*]], label [[PRED_STORE_CONTINUE18:%.*]] -; CHECK: pred.store.if17: +; CHECK: pred.store.if16: ; CHECK-NEXT: [[TMP11:%.*]] = or disjoint i64 [[OFFSET_IDX]], 8 ; CHECK-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP11]] ; CHECK-NEXT: [[TMP12:%.*]] = or disjoint i64 [[OFFSET_IDX8]], 8 @@ -310,10 +310,10 @@ define void @example3(i32 %n, ptr noalias nocapture %p, ptr noalias nocapture %q ; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[NEXT_GEP11]], align 16 ; CHECK-NEXT: store i32 [[TMP13]], ptr [[NEXT_GEP6]], align 16 ; CHECK-NEXT: br label [[PRED_STORE_CONTINUE18]] -; CHECK: pred.store.continue18: +; CHECK: pred.store.continue17: ; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3 ; CHECK-NEXT: br i1 [[TMP14]], label [[PRED_STORE_IF19:%.*]], label [[PRED_STORE_CONTINUE20]] -; CHECK: pred.store.if19: +; CHECK: pred.store.if18: ; CHECK-NEXT: [[TMP15:%.*]] = or disjoint i64 [[OFFSET_IDX]], 12 ; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP15]] ; CHECK-NEXT: [[TMP16:%.*]] = or disjoint i64 [[OFFSET_IDX8]], 12 @@ -321,7 +321,7 @@ define void @example3(i32 %n, ptr noalias nocapture %p, ptr noalias nocapture %q ; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[NEXT_GEP12]], align 16 ; CHECK-NEXT: store i32 [[TMP17]], ptr [[NEXT_GEP7]], align 16 ; CHECK-NEXT: br label [[PRED_STORE_CONTINUE20]] -; CHECK: pred.store.continue20: +; CHECK: pred.store.continue19: ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 ; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -471,7 +471,7 @@ define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst) ; CHECK: pred.store.continue: ; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1 ; CHECK-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13:%.*]] -; CHECK: pred.store.if12: +; CHECK: pred.store.if11: ; CHECK-NEXT: [[TMP7:%.*]] = or disjoint i64 [[OFFSET_IDX7]], 4 ; CHECK-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]] ; CHECK-NEXT: [[TMP8:%.*]] = or disjoint i64 [[OFFSET_IDX]], 2 @@ -481,10 +481,10 @@ define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst) ; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i32 [[TMP10]], 7 ; CHECK-NEXT: store i32 [[TMP11]], ptr [[NEXT_GEP9]], align 4 ; CHECK-NEXT: br label [[PRED_STORE_CONTINUE13]] -; CHECK: pred.store.continue13: +; CHECK: pred.store.continue12: ; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2 ; CHECK-NEXT: br i1 [[TMP12]], label [[PRED_STORE_IF14:%.*]], label [[PRED_STORE_CONTINUE15:%.*]] -; CHECK: pred.store.if14: +; CHECK: pred.store.if13: ; CHECK-NEXT: [[TMP13:%.*]] = or disjoint i64 [[OFFSET_IDX7]], 8 ; CHECK-NEXT: [[NEXT_GEP10:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP13]] ; CHECK-NEXT: [[TMP14:%.*]] = or disjoint i64 [[OFFSET_IDX]], 4 @@ -494,10 +494,10 @@ define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst) ; CHECK-NEXT: [[TMP17:%.*]] = shl nuw nsw i32 [[TMP16]], 7 ; CHECK-NEXT: store i32 [[TMP17]], ptr [[NEXT_GEP10]], align 4 ; CHECK-NEXT: br label [[PRED_STORE_CONTINUE15]] -; CHECK: pred.store.continue15: +; CHECK: pred.store.continue14: ; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3 ; CHECK-NEXT: br i1 [[TMP18]], label [[PRED_STORE_IF16:%.*]], label [[PRED_STORE_CONTINUE17]] -; CHECK: pred.store.if16: +; CHECK: pred.store.if15: ; CHECK-NEXT: [[TMP19:%.*]] = or disjoint i64 [[OFFSET_IDX7]], 12 ; CHECK-NEXT: [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP19]] ; CHECK-NEXT: [[TMP20:%.*]] = or disjoint i64 [[OFFSET_IDX]], 6 @@ -507,7 +507,7 @@ define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst) ; CHECK-NEXT: [[TMP23:%.*]] = shl nuw nsw i32 [[TMP22]], 7 ; CHECK-NEXT: store i32 [[TMP23]], ptr [[NEXT_GEP11]], align 4 ; CHECK-NEXT: br label [[PRED_STORE_CONTINUE17]] -; CHECK: pred.store.continue17: +; CHECK: pred.store.continue16: ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 ; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260 ; CHECK-NEXT: br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll index ca9dfdc6f6d29..036386e7dde76 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll @@ -26,10 +26,10 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) { ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[PADD:%.+]]> = ptradd ir<%A>, vp<[[STEPS]]> ; CHECK-NEXT: vp<[[VPTR:%.]]> = vector-pointer vp<[[PADD]]> -; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VPTR]]> -; CHECK-NEXT: WIDEN ir<%add> = add nsw ir<%l>, ir<10> +; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VPTR]]> +; CHECK-NEXT: WIDEN vp<%add> = add nsw vp<%l>, ir<10> ; CHECK-NEXT: vp<[[VPTR2:%.+]]> = vector-pointer vp<[[PADD]]> -; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, ir<%add> +; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, vp<%add> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV:%.+]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors @@ -58,10 +58,10 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) { ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[PADD:%.+]]> = ptradd ir<%A>, vp<[[STEPS]]> ; CHECK-NEXT: vp<[[VPTR:%.]]> = vector-pointer vp<[[PADD]]> -; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VPTR]]> -; CHECK-NEXT: WIDEN ir<%add> = add nsw ir<%l>, ir<10> +; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VPTR]]> +; CHECK-NEXT: WIDEN vp<%add> = add nsw vp<%l>, ir<10> ; CHECK-NEXT: vp<[[VPTR2:%.+]]> = vector-pointer vp<[[PADD]]> -; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, ir<%add> +; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, vp<%add> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV:%.+]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-cond ir ; CHECK-NEXT: No successors diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll index 961d83a670c25..723e7d7baf66a 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll @@ -1116,11 +1116,11 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) { ; CHECK-NEXT: vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<0> + vp<%3> * ir<-1> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1> ; CHECK-NEXT: EMIT vp<[[PTR_IV:%.+]]> = ptradd ir<%start>, vp<[[STEPS]]> -; CHECK-NEXT: CLONE ir<%ptr.iv.next> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%ptr.iv.next> -; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN ir<%c.1> = icmp eq ir<%l>, ir<0> -; CHECK-NEXT: EMIT vp<[[NEG:%.+]]> = not ir<%c.1> +; CHECK-NEXT: CLONE vp<%ptr.iv.next.cloned> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) vp<%ptr.iv.next.cloned> +; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN vp<%c.1> = icmp eq vp<%l>, ir<0> +; CHECK-NEXT: EMIT vp<[[NEG:%.+]]> = not vp<%c.1> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -1129,8 +1129,8 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%ptr.iv.next> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1> -; CHECK-NEXT: REPLICATE store ir<95>, ir<%ptr.iv.next> +; CHECK-NEXT: REPLICATE vp<%ptr.iv.next> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1> +; CHECK-NEXT: REPLICATE store ir<95>, vp<%ptr.iv.next> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: From 529d103194bfe0dcb105076d69d3909d79c718e3 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 3 Apr 2024 15:53:12 +0100 Subject: [PATCH 05/11] !fixup fix formatting --- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 40cb72ac219c3..eb85a4fb35106 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -299,14 +299,14 @@ Value *VPInstruction::generatePerLane(VPTransformState &State, assert(getOpcode() == VPInstruction::PtrAdd && "only PtrAdd opcodes are supported for now"); -std::string Name = getName(); + std::string Name = getName(); return Builder.CreatePtrAdd(State.get(getOperand(0), Lane), State.get(getOperand(1), Lane), Name); } Value *VPInstruction::generatePerPart(VPTransformState &State, unsigned Part) { IRBuilderBase &Builder = State.Builder; -std::string Name = getName(); + std::string Name = getName(); if (Instruction::isBinaryOp(getOpcode())) { bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this); From 857d9258d478df8c03d8ea0efaf5ca125225a66a Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 3 Apr 2024 16:12:22 +0100 Subject: [PATCH 06/11] !fixup fix formatting --- llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h | 7 ++++--- llvm/lib/Transforms/Vectorize/VPlan.h | 3 +-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h index 1c8a2789712a0..8ae9a83666326 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -179,9 +179,10 @@ class VPBuilder { VPValue *createOr(VPValue *LHS, VPValue *RHS, DebugLoc DL = {}, const Twine &Name = "") { - return tryInsertInstruction(new VPInstruction( - Instruction::BinaryOps::Or, {LHS, RHS}, - VPRecipeWithIRFlags::DisjointFlagsTy(false), DL), Name); + return tryInsertInstruction( + new VPInstruction(Instruction::BinaryOps::Or, {LHS, RHS}, + VPRecipeWithIRFlags::DisjointFlagsTy(false), DL), + Name); } VPValue *createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal, diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 845792b9ad30a..4ab252cfa42da 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1223,8 +1223,7 @@ class VPInstruction : public VPRecipeWithIRFlags { Opcode(Opcode) {} VPInstruction(unsigned Opcode, std::initializer_list Operands, - DisjointFlagsTy DisjointFlag, DebugLoc DL = {} - ) + DisjointFlagsTy DisjointFlag, DebugLoc DL = {}) : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, DisjointFlag, DL), Opcode(Opcode) { assert(Opcode == Instruction::Or && "only OR opcodes can be disjoint"); From 8b303322e59b3d13ecf229db1515fdfcfea04777 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 9 Apr 2024 16:55:50 +0100 Subject: [PATCH 07/11] !fixup move deduplication to VPSlotTracker. --- .../Vectorize/LoopVectorizationPlanner.h | 30 +-- .../Transforms/Vectorize/LoopVectorize.cpp | 3 +- llvm/lib/Transforms/Vectorize/VPlan.cpp | 133 ++++------ llvm/lib/Transforms/Vectorize/VPlan.h | 84 ++----- .../Transforms/Vectorize/VPlanHCFGBuilder.cpp | 4 +- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 15 +- .../Transforms/Vectorize/VPlanTransforms.cpp | 13 +- llvm/lib/Transforms/Vectorize/VPlanValue.h | 46 ++-- .../AArch64/sve-tail-folding-forced.ll | 14 +- .../LoopVectorize/AArch64/sve-widen-gep.ll | 16 +- .../AArch64/synthesize-mask-for-call.ll | 84 +++---- .../widen-call-with-intrinsic-or-libfunc.ll | 28 +-- .../PowerPC/vplan-force-tail-with-evl.ll | 36 +-- .../RISCV/riscv-vector-reverse.ll | 36 +-- .../LoopVectorize/X86/small-size.ll | 24 +- .../first-order-recurrence-chains-vplan.ll | 46 ++-- ...-order-recurrence-sink-replicate-region.ll | 132 +++++----- .../Transforms/LoopVectorize/icmp-uniforms.ll | 10 +- .../interleave-and-scalarize-only.ll | 20 +- .../LoopVectorize/vplan-dot-printing.ll | 14 +- .../LoopVectorize/vplan-iv-transforms.ll | 6 +- .../vplan-printing-before-execute.ll | 12 +- .../vplan-printing-outer-loop.ll | 26 +- .../LoopVectorize/vplan-printing.ll | 224 ++++++++--------- .../vplan-sink-scalars-and-merge-vf1.ll | 8 +- .../vplan-sink-scalars-and-merge.ll | 238 +++++++++--------- .../vplan-unused-interleave-group.ll | 6 +- 27 files changed, 610 insertions(+), 698 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h index 2c33491898e42..ece2a34f180cb 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -47,19 +47,16 @@ class VPBuilder { VPBasicBlock::iterator InsertPt = VPBasicBlock::iterator(); /// Insert \p VPI in BB at InsertPt if BB is set. - VPInstruction *tryInsertInstruction(VPInstruction *VPI, - const Twine &Name = "") { - if (BB) { + VPInstruction *tryInsertInstruction(VPInstruction *VPI) { + if (BB) BB->insert(VPI, InsertPt); - VPI->setName(Name); - } return VPI; } VPInstruction *createInstruction(unsigned Opcode, ArrayRef Operands, DebugLoc DL, const Twine &Name = "") { - return tryInsertInstruction(new VPInstruction(Opcode, Operands, DL), Name); + return tryInsertInstruction(new VPInstruction(Opcode, Operands, DL, Name)); } VPInstruction *createInstruction(unsigned Opcode, @@ -162,7 +159,7 @@ class VPBuilder { VPRecipeWithIRFlags::WrapFlagsTy WrapFlags, DebugLoc DL = {}, const Twine &Name = "") { return tryInsertInstruction( - new VPInstruction(Opcode, Operands, WrapFlags, DL), Name); + new VPInstruction(Opcode, Operands, WrapFlags, DL, Name)); } VPValue *createNot(VPValue *Operand, DebugLoc DL = {}, const Twine &Name = "") { @@ -177,21 +174,20 @@ class VPBuilder { VPValue *createOr(VPValue *LHS, VPValue *RHS, DebugLoc DL = {}, const Twine &Name = "") { - return tryInsertInstruction( - new VPInstruction(Instruction::BinaryOps::Or, {LHS, RHS}, - VPRecipeWithIRFlags::DisjointFlagsTy(false), DL), - Name); + return tryInsertInstruction(new VPInstruction( + Instruction::BinaryOps::Or, {LHS, RHS}, + VPRecipeWithIRFlags::DisjointFlagsTy(false), DL, Name)); } VPValue *createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal, DebugLoc DL = {}, const Twine &Name = "", std::optional FMFs = std::nullopt) { - auto *Select = FMFs - ? new VPInstruction(Instruction::Select, - {Cond, TrueVal, FalseVal}, *FMFs, DL) - : new VPInstruction(Instruction::Select, - {Cond, TrueVal, FalseVal}, DL); - return tryInsertInstruction(Select, Name); + auto *Select = + FMFs ? new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal}, + *FMFs, DL, Name) + : new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal}, + DL, Name); + return tryInsertInstruction(Select); } /// Create a new ICmp VPInstruction with predicate \p Pred and operands \p A diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 14a7a558118ee..5535cc55e9321 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7226,7 +7226,7 @@ VPValue *VPBuilder::createICmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B, assert(Pred >= CmpInst::FIRST_ICMP_PREDICATE && Pred <= CmpInst::LAST_ICMP_PREDICATE && "invalid predicate"); return tryInsertInstruction( - new VPInstruction(Instruction::ICmp, Pred, A, B, DL), Name); + new VPInstruction(Instruction::ICmp, Pred, A, B, DL, Name)); } // This function will select a scalable VF if the target supports scalable @@ -8797,7 +8797,6 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { if (!Member->getType()->isVoidTy()) { VPValue *OriginalV = MemberR->getVPSingleValue(); OriginalV->replaceAllUsesWith(VPIG->getVPValue(J)); - VPIG->getVPValue(J)->takeName(MemberR->getVPSingleValue()); J++; } MemberR->eraseFromParent(); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 7a978909e711a..c4e7b4d12d325 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -597,21 +597,6 @@ bool VPBasicBlock::isExiting() const { return getParent() && getParent()->getExitingBasicBlock() == this; } -void VPBasicBlock::insert(VPRecipeBase *Recipe, iterator InsertPt) { - assert(Recipe && "No recipe to append."); - assert(!Recipe->Parent && "Recipe already in VPlan"); - Recipe->Parent = this; - Recipes.insert(InsertPt, Recipe); - - if (Recipe->getNumDefinedValues() != 1) - return; - VPValue *VPV = Recipe->getVPSingleValue(); - Value *UV = VPV->getUnderlyingValue(); - VPlan *ParentPlan = getPlan(); - if (UV && ParentPlan && !ParentPlan->hasName(VPV)) - VPV->setName(UV->getName()); -} - #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void VPBlockBase::printSuccessors(raw_ostream &O, const Twine &Indent) const { if (getSuccessors().empty()) { @@ -1070,10 +1055,8 @@ static void remapOperands(VPBlockBase *Entry, VPBlockBase *NewEntry, assert(OldR.getNumDefinedValues() == NewR.getNumDefinedValues() && "recipes must define the same number of operands"); for (const auto &[OldV, NewV] : - zip(OldR.definedValues(), NewR.definedValues())) { + zip(OldR.definedValues(), NewR.definedValues())) Old2NewVPValues[OldV] = NewV; - NewV->setName(OldV->getName()); - } } } @@ -1131,25 +1114,6 @@ VPlan *VPlan::duplicate() { return NewPlan; } -void VPlan::setName(const VPValue *V, const Twine &Name) { -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - std::string N = Name.str(); - if (N.empty()) - return; - std::string CurrName = N; - - if (UsedNames.contains(N)) - CurrName = N + ".1"; - unsigned Cnt = 2; - while (UsedNames.contains(CurrName)) { - CurrName = N + "." + std::to_string(Cnt); - Cnt += 1; - } - VPValue2Name[V] = CurrName; - UsedNames.insert(CurrName); -#endif -} - #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) Twine VPlanPrinter::getUID(const VPBlockBase *Block) { @@ -1336,12 +1300,7 @@ void VPValue::replaceUsesWithIf( #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void VPValue::printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const { - if (!getDefiningRecipe() && getUnderlyingValue()) { - OS << "ir<"; - getUnderlyingValue()->printAsOperand(OS, false); - OS << ">"; - } else - OS << Tracker.getName(this); + OS << Tracker.getName(this); } void VPUser::printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const { @@ -1351,31 +1310,6 @@ void VPUser::printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const { } #endif -std::string VPValue::getName() { -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - if (auto *Def = getDefiningRecipe()) - return Def->getParent()->getPlan()->getName(this); - assert(isLiveIn() && "called getName() on unsupported VPValue"); - return getLiveInIRValue()->getName().str(); -#else - return ""; -#endif -} - -void VPValue::setName(const Twine &Name) { -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - auto *Def = getDefiningRecipe(); - Def->getParent()->getPlan()->setName(this, Name); -#endif -} - -void VPValue::takeName(const VPValue *Src) { -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - auto *Def = getDefiningRecipe(); - Def->getParent()->getPlan()->takeName(Src, this); -#endif -} - void VPInterleavedAccessInfo::visitRegion(VPRegionBlock *Region, Old2NewTy &Old2New, InterleavedAccessInfo &IAI) { @@ -1428,40 +1362,73 @@ VPInterleavedAccessInfo::VPInterleavedAccessInfo(VPlan &Plan, visitRegion(Plan.getVectorLoopRegion(), Old2New, IAI); } -void VPSlotTracker::assignSlot(const VPValue *V) { +void VPSlotTracker::assignSlotOrName(const VPValue *V) { + if (auto *UV = V->getUnderlyingValue()) { + std::string Name; + raw_string_ostream S(Name); + UV->printAsOperand(S, false); + deduplicateName(V, Name); + return; + } assert(!Slots.contains(V) && "VPValue already has a slot!"); Slots[V] = NextSlot++; } -void VPSlotTracker::assignSlots(const VPlan &Plan) { +void VPSlotTracker::assignSlotsOrNames(const VPlan &Plan) { if (Plan.VFxUF.getNumUsers() > 0) - assignSlot(&Plan.VFxUF); - assignSlot(&Plan.VectorTripCount); + assignSlotOrName(&Plan.VFxUF); + assignSlotOrName(&Plan.VectorTripCount); if (Plan.BackedgeTakenCount) - assignSlot(Plan.BackedgeTakenCount); - assignSlots(Plan.getPreheader()); + assignSlotOrName(Plan.BackedgeTakenCount); + for (VPValue *LI : Plan.VPLiveInsToFree) + assignSlotOrName(LI); + assignSlotsOrNames(Plan.getPreheader()); ReversePostOrderTraversal> RPOT(VPBlockDeepTraversalWrapper(Plan.getEntry())); for (const VPBasicBlock *VPBB : VPBlockUtils::blocksOnly(RPOT)) - assignSlots(VPBB); + assignSlotsOrNames(VPBB); } -void VPSlotTracker::assignSlots(const VPBasicBlock *VPBB) { +void VPSlotTracker::assignSlotsOrNames(const VPBasicBlock *VPBB) { for (const VPRecipeBase &Recipe : *VPBB) for (VPValue *Def : Recipe.definedValues()) - assignSlot(Def); + assignSlotOrName(Def); +} + +void VPSlotTracker::deduplicateName(const VPValue *V, StringRef Name) { + assert(!Name.empty() && "Name cannot be be empty."); + std::string NewName = Name.str(); + const auto &[A, AssignedInserted] = AssignedNames.insert({V, NewName}); + if (!AssignedInserted || V->isLiveIn()) + return; + + const auto &[C, UseInserted] = NameUseCount.insert({NewName, 0}); + if (!UseInserted) { + C->second++; + NewName = NewName + "." + std::to_string(C->second); + A->second = NewName; + } } std::string VPSlotTracker::getName(const VPValue *V) const { - if (!Plan) - return ""; - std::string N = Plan->getName(V); - if (!N.empty()) - return (Twine("vp<%") + N + ">").str(); + std::string Name = AssignedNames.lookup(V); + if (!Name.empty()) { + assert( + V->getUnderlyingValue() && + "Can only have assigned names for VPValues with an underlying value"); + return (Twine("ir<") + Name + ">").str(); + } - return (Twine("vp<%") + std::to_string(getSlot(V)) + ">").str(); + assert( + !V->getUnderlyingValue() && + "Must not have assigned names for VPValues without an underlying value"); + unsigned Slot = getSlot(V); + if (Slot == unsigned(-1)) + return ""; + else + return (Twine("vp<%") + std::to_string(Slot) + ">").str(); } bool vputils::onlyFirstLaneUsed(const VPValue *Def) { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index bdff80c2b1285..5dc905a3c407b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -32,7 +32,6 @@ #include "llvm/ADT/SmallBitVector.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringSet.h" #include "llvm/ADT/Twine.h" #include "llvm/ADT/ilist.h" #include "llvm/ADT/ilist_node.h" @@ -1186,6 +1185,9 @@ class VPInstruction : public VPRecipeWithIRFlags { typedef unsigned char OpcodeTy; OpcodeTy Opcode; + /// An optional name that can be used for the generated IR instruction. + const std::string Name; + /// Returns true if this VPInstruction generates scalar values for all lanes. /// Most VPInstructions generate a single value per part, either vector or /// scalar. VPReplicateRecipe takes care of generating multiple (scalar) @@ -1216,37 +1218,39 @@ class VPInstruction : public VPRecipeWithIRFlags { #endif public: - VPInstruction(unsigned Opcode, ArrayRef Operands, DebugLoc DL) + VPInstruction(unsigned Opcode, ArrayRef Operands, DebugLoc DL, + const Twine &Name = "") : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, DL), - Opcode(Opcode) {} + Opcode(Opcode), Name(Name.str()) {} VPInstruction(unsigned Opcode, std::initializer_list Operands, - DebugLoc DL = {}) - : VPInstruction(Opcode, ArrayRef(Operands), DL) {} + DebugLoc DL = {}, const Twine &Name = "") + : VPInstruction(Opcode, ArrayRef(Operands), DL, Name) {} VPInstruction(unsigned Opcode, CmpInst::Predicate Pred, VPValue *A, - VPValue *B, DebugLoc DL = {}); + VPValue *B, DebugLoc DL = {}, const Twine &Name = ""); VPInstruction(unsigned Opcode, std::initializer_list Operands, - WrapFlagsTy WrapFlags, DebugLoc DL = {}) + WrapFlagsTy WrapFlags, DebugLoc DL = {}, const Twine &Name = "") : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, WrapFlags, DL), - Opcode(Opcode) {} + Opcode(Opcode), Name(Name.str()) {} VPInstruction(unsigned Opcode, std::initializer_list Operands, - DisjointFlagsTy DisjointFlag, DebugLoc DL = {}) + DisjointFlagsTy DisjointFlag, DebugLoc DL = {}, + const Twine &Name = "") : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, DisjointFlag, DL), - Opcode(Opcode) { + Opcode(Opcode), Name(Name.str()) { assert(Opcode == Instruction::Or && "only OR opcodes can be disjoint"); } VPInstruction(unsigned Opcode, std::initializer_list Operands, - FastMathFlags FMFs, DebugLoc DL = {}); + FastMathFlags FMFs, DebugLoc DL = {}, const Twine &Name = ""); VP_CLASSOF_IMPL(VPDef::VPInstructionSC) VPRecipeBase *clone() override { SmallVector Operands(operands()); - auto *New = new VPInstruction(Opcode, Operands, getDebugLoc()); + auto *New = new VPInstruction(Opcode, Operands, getDebugLoc(), Name); New->transferFlags(*this); return New; } @@ -2741,7 +2745,12 @@ class VPBasicBlock : public VPBlockBase { return V->getVPBlockID() == VPBlockBase::VPBasicBlockSC; } - void insert(VPRecipeBase *Recipe, iterator InsertPt); + void insert(VPRecipeBase *Recipe, iterator InsertPt) { + assert(Recipe && "No recipe to append."); + assert(!Recipe->Parent && "Recipe already in VPlan"); + Recipe->Parent = this; + Recipes.insert(InsertPt, Recipe); + } /// Augment the existing recipes of a VPBasicBlock with an additional /// \p Recipe as the last recipe. @@ -2964,11 +2973,6 @@ class VPlan { /// been modeled in VPlan directly. DenseMap SCEVToExpansion; -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - DenseMap VPValue2Name; - StringSet<> UsedNames; -#endif - public: /// Construct a VPlan with original preheader \p Preheader, trip count \p TC /// and \p Entry to the plan. At the moment, \p Preheader and \p Entry need to @@ -3099,50 +3103,6 @@ class VPlan { LLVM_DUMP_METHOD void dump() const; #endif - /// Set the name for \p V to \p Name if it is not empty. \p V must not have a - /// name assigned already. - void setName(const VPValue *V, const Twine &Name); - - /// Remove the assigned name for \p V, if there is one. - void removeName(const VPValue *V) { -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - if (VPValue2Name.contains(V)) { - UsedNames.erase(VPValue2Name[V]); - VPValue2Name.erase(V); - } -#endif - } - - /// Take the name for \p Src and move it to \p Dst. The name of \p Src will be - /// empty afterwards. - void takeName(const VPValue *Src, const VPValue *Dst) { -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - std::string N = VPValue2Name.lookup(Src); - if (N.empty()) - return; - VPValue2Name[Dst] = N; - VPValue2Name.erase(Src); -#endif - } - - /// Return the name assigned to \p V or an empty string if no name has been - /// assigned. - std::string getName(const VPValue *V) const { -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - return VPValue2Name.lookup(V); -#else - return ""; -#endif - } - - bool hasName(const VPValue *V) const { -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - return VPValue2Name.contains(V); -#else - return false; -#endif - } - /// Returns the VPRegionBlock of the vector loop. VPRegionBlock *getVectorLoopRegion() { return cast(getEntry()->getSingleSuccessor()); diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp index a53f0c9e69753..b57ff2840a729 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp @@ -320,8 +320,8 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB, // Build VPInstruction for any arbitrary Instruction without specific // representation in VPlan. - NewVPV = cast(VPIRBuilder.createNaryOp( - Inst->getOpcode(), VPOperands, Inst, Inst->getName())); + NewVPV = cast( + VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst)); } IRDef2VPValue[Inst] = NewVPV; diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index e062e27f2f802..625319954e9b7 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -224,8 +224,6 @@ void VPRecipeBase::removeFromParent() { iplist::iterator VPRecipeBase::eraseFromParent() { assert(getParent() && "Recipe not in any VPBasicBlock"); - for (auto *V : definedValues()) - getParent()->getPlan()->removeName(V); return getParent()->getRecipeList().erase(getIterator()); } @@ -255,19 +253,20 @@ FastMathFlags VPRecipeWithIRFlags::getFastMathFlags() const { } VPInstruction::VPInstruction(unsigned Opcode, CmpInst::Predicate Pred, - VPValue *A, VPValue *B, DebugLoc DL) + VPValue *A, VPValue *B, DebugLoc DL, + const Twine &Name) : VPRecipeWithIRFlags(VPDef::VPInstructionSC, ArrayRef({A, B}), Pred, DL), - Opcode(Opcode) { + Opcode(Opcode), Name(Name.str()) { assert(Opcode == Instruction::ICmp && "only ICmp predicates supported at the moment"); } VPInstruction::VPInstruction(unsigned Opcode, std::initializer_list Operands, - FastMathFlags FMFs, DebugLoc DL) + FastMathFlags FMFs, DebugLoc DL, const Twine &Name) : VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, FMFs, DL), - Opcode(Opcode) { + Opcode(Opcode), Name(Name.str()) { // Make sure the VPInstruction is a floating-point operation. assert(isFPMathOp() && "this op can't take fast-math flags"); } @@ -297,17 +296,15 @@ bool VPInstruction::canGenerateScalarForFirstLane() const { Value *VPInstruction::generatePerLane(VPTransformState &State, const VPIteration &Lane) { IRBuilderBase &Builder = State.Builder; + assert(getOpcode() == VPInstruction::PtrAdd && "only PtrAdd opcodes are supported for now"); - - std::string Name = getName(); return Builder.CreatePtrAdd(State.get(getOperand(0), Lane), State.get(getOperand(1), Lane), Name); } Value *VPInstruction::generatePerPart(VPTransformState &State, unsigned Part) { IRBuilderBase &Builder = State.Builder; - std::string Name = getName(); if (Instruction::isBinaryOp(getOpcode())) { bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index e539c491ed9f7..1256e4d8fda50 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -88,10 +88,9 @@ void VPlanTransforms::VPInstructionsToVPRecipes( } NewRecipe->insertBefore(&Ingredient); - if (NewRecipe->getNumDefinedValues() == 1) { + if (NewRecipe->getNumDefinedValues() == 1) VPV->replaceAllUsesWith(NewRecipe->getVPSingleValue()); - NewRecipe->getVPSingleValue()->takeName(VPV); - } else + else assert(NewRecipe->getNumDefinedValues() == 0 && "Only recpies with zero or one defined values expected"); Ingredient.eraseFromParent(); @@ -160,8 +159,9 @@ static bool sinkScalarOperands(VPlan &Plan) { continue; Instruction *I = SinkCandidate->getUnderlyingInstr(); auto *Clone = new VPReplicateRecipe(I, SinkCandidate->operands(), true); + // TODO: add ".cloned" suffix to name of Clone's VPValue. + Clone->insertBefore(SinkCandidate); - Clone->setName(SinkCandidate->getName() + ".cloned"); SinkCandidate->replaceUsesWithIf(Clone, [SinkTo](VPUser &U, unsigned) { return cast(&U)->getParent() != SinkTo; }); @@ -316,8 +316,6 @@ static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe, PredRecipe->replaceAllUsesWith(PHIRecipe); PHIRecipe->setOperand(0, RecipeWithoutMask); } - Plan.takeName(PredRecipe, RecipeWithoutMask); - PredRecipe->eraseFromParent(); auto *Exiting = new VPBasicBlock(Twine(RegionName) + ".continue", PHIRecipe); VPRegionBlock *Region = new VPRegionBlock(Entry, Exiting, RegionName, true); @@ -590,11 +588,10 @@ static void legalizeAndOptimizeInductions(VPlan &Plan, ScalarEvolution &SE) { auto *Recipe = new VPInstruction(VPInstruction::PtrAdd, {PtrIV->getStartValue(), Steps->getVPSingleValue()}, - PtrIV->getDebugLoc()); + PtrIV->getDebugLoc(), "next.gep"); Recipe->insertAfter(Steps); PtrIV->replaceAllUsesWith(Recipe); - Recipe->setName("next.gep"); continue; } diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h index a55be92e059a4..15094c6a60004 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -23,6 +23,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" #include "llvm/ADT/TinyPtrVector.h" #include "llvm/ADT/iterator_range.h" @@ -187,21 +188,6 @@ class VPValue { /// TODO: Also handle recipes defined in pre-header blocks. bool isDefinedOutsideVectorRegions() const { return !hasDefiningRecipe(); } - /// Returns the name associated with the VPValue. In case the VPValue has a - /// defining recipe, look up the name in the containing VPlan. In case it is a - /// live-in with an underlying IR value, return the name of the underlying - /// value. Calling getName() in other cases is invalid. - std::string getName(); - - /// Sets the name for this VPValue to \p Name. The VPValue must have a - /// defining recipe inserted in a VPlan. - void setName(const Twine &Name); - - /// Take the name for \p Src and move it to this VPValue. The name of \p Src - /// will be empty afterwards. The VPValue must have a defining recipe inserted - /// in a VPlan. - void takeName(const VPValue *Src); - // Set \p Val as the underlying Value of this VPValue. void setUnderlyingValue(Value *Val) { assert(!UnderlyingVal && "Underlying Value is already set."); @@ -458,22 +444,32 @@ class VPDef { class VPlan; class VPBasicBlock; -/// This class can be used to assign consecutive numbers to all VPValues in a -/// VPlan and allows querying the numbering for printing, similar to the -/// ModuleSlotTracker for IR values. +/// This class can be used to assign consecutive numbers to VPValues in a +/// VPlan without underlying values and deduplicated names to VPValues with +/// underlying values. Allows querying the numbering and deduplicated names for +/// printing, similar to the ModuleSlotTracker for IR values. class VPSlotTracker { - const VPlan *Plan; + /// Keep track of de-duplicated names assigned to VPValues with underlying IR + /// values + DenseMap AssignedNames; + /// Keep track of the next number to use to deduplicate the base name. + StringMap NameUseCount; + DenseMap Slots; unsigned NextSlot = 0; - void assignSlot(const VPValue *V); - void assignSlots(const VPlan &Plan); - void assignSlots(const VPBasicBlock *VPBB); + void assignSlotOrName(const VPValue *V); + void assignSlotsOrNames(const VPlan &Plan); + void assignSlotsOrNames(const VPBasicBlock *VPBB); + + /// Create a deduplicated version of \p Name for \p V by appending ".Number" + /// to \p Name if there are multiple uses of that name. + void deduplicateName(const VPValue *V, StringRef Name); public: - VPSlotTracker(const VPlan *Plan = nullptr) : Plan(Plan) { + VPSlotTracker(const VPlan *Plan = nullptr) { if (Plan) - assignSlots(*Plan); + assignSlotsOrNames(*Plan); } unsigned getSlot(const VPValue *V) const { @@ -483,7 +479,7 @@ class VPSlotTracker { return I->second; } - std::string getName(const VPValue *V) const; + std::string getName(const VPValue *) const; }; } // namespace llvm diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll index 33708bad471da..cc72dfa4ce639 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll @@ -19,20 +19,20 @@ target triple = "aarch64-unknown-linux-gnu" ; VPLANS-EMPTY: ; VPLANS-NEXT: vector.ph: ; VPLANS-NEXT: EMIT vp<[[NEWTC:%[0-9]+]]> = TC > VF ? TC - VF : 0 vp<[[TC]]> -; VPLANS-NEXT: EMIT vp<%index.part.next> = VF * Part + ir<0> -; VPLANS-NEXT: EMIT vp<%active.lane.mask.entry> = active lane mask vp<%index.part.next>, vp<[[TC]]> +; VPLANS-NEXT: EMIT vp<[[VF:%[0-9]+]]> = VF * Part + ir<0> +; VPLANS-NEXT: EMIT vp<[[LANEMASK_ENTRY:%[0-9]+]]> = active lane mask vp<[[VF]]>, vp<[[TC]]> ; VPLANS-NEXT: Successor(s): vector loop ; VPLANS-EMPTY: ; VPLANS-NEXT: vector loop: { ; VPLANS-NEXT: vector.body: ; VPLANS-NEXT: EMIT vp<[[INDV:%[0-9]+]]> = CANONICAL-INDUCTION -; VPLANS-NEXT: ACTIVE-LANE-MASK-PHI vp<[[LANEMASK_PHI:%[0-9]+]]> = phi vp<%active.lane.mask.entry>, vp<[[LANEMASK_LOOP:%.+]]> +; VPLANS-NEXT: ACTIVE-LANE-MASK-PHI vp<[[LANEMASK_PHI:%[0-9]+]]> = phi vp<[[LANEMASK_ENTRY]]>, vp<[[LANEMASK_LOOP:%[0-9]+]]> ; VPLANS-NEXT: vp<[[STEP:%[0-9]+]]> = SCALAR-STEPS vp<[[INDV]]>, ir<1> -; VPLANS-NEXT: CLONE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEP]]> -; VPLANS-NEXT: vp<[[VEC_PTR:%[0-9]+]]> = vector-pointer vp<%gep> +; VPLANS-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEP]]> +; VPLANS-NEXT: vp<[[VEC_PTR:%[0-9]+]]> = vector-pointer ir<%gep> ; VPLANS-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%val>, vp<[[LANEMASK_PHI]]> -; VPLANS-NEXT: EMIT vp<[[INDV_UPDATE:%.+]]> = add vp<[[INDV]]>, vp<[[VFxUF]]> -; VPLANS-NEXT: EMIT vp<[[INC:%.+]]> = VF * Part + vp<[[INDV]]> +; VPLANS-NEXT: EMIT vp<[[INDV_UPDATE:%[0-9]+]]> = add vp<[[INDV]]>, vp<[[VFxUF]]> +; VPLANS-NEXT: EMIT vp<[[INC:%[0-9]+]]> = VF * Part + vp<[[INDV]]> ; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP]]> = active lane mask vp<[[INC]]>, vp<[[NEWTC]]> ; VPLANS-NEXT: EMIT vp<[[NOT:%[0-9]+]]> = not vp<[[LANEMASK_LOOP]]> ; VPLANS-NEXT: EMIT branch-on-cond vp<[[NOT]]> diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll index ef0027ce4ed9a..3bab341e1c248 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll @@ -19,18 +19,18 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: EMIT vp<%ptr.iv.2> = WIDEN-POINTER-INDUCTION ir<%start.2>, 1 +; CHECK-NEXT: EMIT ir<%ptr.iv.2> = WIDEN-POINTER-INDUCTION ir<%start.2>, 1 ; CHECK-NEXT: vp<[[PTR_IDX:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * ir<8> ; CHECK-NEXT: vp<[[PTR_IDX_STEPS:%.+]]> = SCALAR-STEPS vp<[[PTR_IDX]]>, ir<8> ; CHECK-NEXT: EMIT vp<[[PTR_IV_1:%.+]]> = ptradd ir<%start.1>, vp<[[PTR_IDX_STEPS]]> -; CHECK-NEXT: WIDEN-GEP Var[Inv] vp<%ptr.iv.2.next> = getelementptr inbounds vp<%ptr.iv.2>, ir<1> +; CHECK-NEXT: WIDEN-GEP Var[Inv] ir<%ptr.iv.2.next> = getelementptr inbounds ir<%ptr.iv.2>, ir<1> ; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<[[PTR_IV_1]]> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%ptr.iv.2.next> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%ptr.iv.2> -; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR2]]> -; CHECK-NEXT: WIDEN vp<%add> = add vp<%lv>, ir<1> -; CHECK-NEXT: vp<[[VEC_PTR3:%.+]]> = vector-pointer vp<%ptr.iv.2> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR3]]>, vp<%add> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%ptr.iv.2.next> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%ptr.iv.2> +; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR2]]> +; CHECK-NEXT: WIDEN ir<%add> = add ir<%lv>, ir<1> +; CHECK-NEXT: vp<[[VEC_PTR3:%.+]]> = vector-pointer ir<%ptr.iv.2> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR3]]>, ir<%add> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll index 02e348a8e6f57..ad0d40ee7fbde 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll @@ -21,13 +21,13 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep> -; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: REPLICATE vp<%call> = call @foo(vp<%load>) -; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call> +; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep> +; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: REPLICATE ir<%call> = call @foo(ir<%load>) +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors @@ -50,13 +50,13 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[STEPS]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep> -; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>) (using library function: foo_vector_fixed4_nomask) -; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call> +; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep> +; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed4_nomask) +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors @@ -84,13 +84,13 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep> -; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>) (using library function: foo_vector_fixed2_nomask) -; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%call> +; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep> +; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed2_nomask) +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%call> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXST:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors @@ -113,13 +113,13 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep> -; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>, ir) (using library function: foo_vector_fixed4_mask) -; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call> +; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep> +; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>, ir) (using library function: foo_vector_fixed4_mask) +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors @@ -146,13 +146,13 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep> -; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>) (using library function: foo_vector_fixed2_nomask) -; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call> +; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep> +; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed2_nomask) +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors @@ -175,13 +175,13 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep> -; CHECK-NEXT: WIDEN vp<%load> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN-CALL vp<%call> = call @foo(vp<%load>) (using library function: foo_vector_fixed4_nomask) -; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call> +; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%b>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep> +; CHECK-NEXT: WIDEN ir<%load> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed4_nomask) +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll index 9b8cbb37ea2de..f67b932b04a0d 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll @@ -19,13 +19,13 @@ target triple = "arm64-apple-ios" ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.src> -; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN-CAST vp<%conv> = fpext vp<%l> to double -; CHECK-NEXT: WIDEN-CALL vp<%s> = call @llvm.sin.f64(vp<%conv>) (using library function: __simd_sin_v2f64) -; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store vp<%s>, vp<%gep.dst> +; CHECK-NEXT: CLONE ir<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.src> +; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN-CAST ir<%conv> = fpext ir<%l> to double +; CHECK-NEXT: WIDEN-CALL ir<%s> = call @llvm.sin.f64(ir<%conv>) (using library function: __simd_sin_v2f64) +; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep.dst> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors @@ -48,13 +48,13 @@ target triple = "arm64-apple-ios" ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.src> -; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN-CAST vp<%conv> = fpext vp<%l> to double -; CHECK-NEXT: WIDEN-CALL vp<%s> = call @llvm.sin.f64(vp<%conv>) (using vector intrinsic) -; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store vp<%s>, vp<%gep.dst> +; CHECK-NEXT: CLONE ir<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.src> +; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN-CAST ir<%conv> = fpext ir<%l> to double +; CHECK-NEXT: WIDEN-CALL ir<%s> = call @llvm.sin.f64(ir<%conv>) (using vector intrinsic) +; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep.dst> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/vplan-force-tail-with-evl.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/vplan-force-tail-with-evl.ll index 5d1a471c5d166..bd52c2a8f0645 100644 --- a/llvm/test/Transforms/LoopVectorize/PowerPC/vplan-force-tail-with-evl.ll +++ b/llvm/test/Transforms/LoopVectorize/PowerPC/vplan-force-tail-with-evl.ll @@ -18,37 +18,37 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) { ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<%3> = CANONICAL-INDUCTION ir<0>, vp<%16> +; CHECK-NEXT: EMIT vp<%3> = CANONICAL-INDUCTION ir<0>, vp<%8> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<%5> = icmp ule ir<%iv>, vp<%2> +; CHECK-NEXT: EMIT vp<%4> = icmp ule ir<%iv>, vp<%2> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { ; CHECK-NEXT: pred.store.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<%5> +; CHECK-NEXT: BRANCH-ON-MASK vp<%4> ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: vp<%6> = SCALAR-STEPS vp<%3>, ir<1> -; CHECK-NEXT: REPLICATE ir<%arrayidx> = getelementptr inbounds ir<%b>, vp<%6> +; CHECK-NEXT: vp<%5> = SCALAR-STEPS vp<%3>, ir<1> +; CHECK-NEXT: REPLICATE ir<%arrayidx> = getelementptr inbounds ir<%b>, vp<%5> ; CHECK-NEXT: REPLICATE ir<%0> = load ir<%arrayidx> -; CHECK-NEXT: REPLICATE ir<%arrayidx2> = getelementptr inbounds ir<%c>, vp<%6> +; CHECK-NEXT: REPLICATE ir<%arrayidx2> = getelementptr inbounds ir<%c>, vp<%5> ; CHECK-NEXT: REPLICATE ir<%1> = load ir<%arrayidx2> -; CHECK-NEXT: REPLICATE ir<%arrayidx4> = getelementptr inbounds ir<%a>, vp<%6> +; CHECK-NEXT: REPLICATE ir<%arrayidx4> = getelementptr inbounds ir<%a>, vp<%5> ; CHECK-NEXT: REPLICATE ir<%add> = add nsw ir<%1>, ir<%0> ; CHECK-NEXT: REPLICATE store ir<%add>, ir<%arrayidx4> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%14> = ir<%0> -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%15> = ir<%1> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%6> = ir<%0> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<%7> = ir<%1> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): for.body.2 ; CHECK-EMPTY: ; CHECK-NEXT: for.body.2: -; CHECK-NEXT: EMIT vp<%16> = add vp<%3>, vp<%0> -; CHECK-NEXT: EMIT branch-on-count vp<%16>, vp<%1> +; CHECK-NEXT: EMIT vp<%8> = add vp<%3>, vp<%0> +; CHECK-NEXT: EMIT branch-on-count vp<%8>, vp<%1> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -83,17 +83,17 @@ define void @safe_dep(ptr %p) { ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%10> +; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%6> ; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1> ; CHECK-NEXT: CLONE ir<%a1> = getelementptr ir<%p>, vp<%3> -; CHECK-NEXT: vp<%5> = vector-pointer ir<%a1> -; CHECK-NEXT: WIDEN ir<%v> = load vp<%5> +; CHECK-NEXT: vp<%4> = vector-pointer ir<%a1> +; CHECK-NEXT: WIDEN ir<%v> = load vp<%4> ; CHECK-NEXT: CLONE ir<%offset> = add vp<%3>, ir<100> ; CHECK-NEXT: CLONE ir<%a2> = getelementptr ir<%p>, ir<%offset> -; CHECK-NEXT: vp<%9> = vector-pointer ir<%a2> -; CHECK-NEXT: WIDEN store vp<%9>, ir<%v> -; CHECK-NEXT: EMIT vp<%10> = add nuw vp<%2>, vp<%0> -; CHECK-NEXT: EMIT branch-on-count vp<%10>, vp<%1> +; CHECK-NEXT: vp<%5> = vector-pointer ir<%a2> +; CHECK-NEXT: WIDEN store vp<%5>, ir<%v> +; CHECK-NEXT: EMIT vp<%6> = add nuw vp<%2>, vp<%0> +; CHECK-NEXT: EMIT branch-on-count vp<%6>, vp<%1> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll index 5440c5324a42b..72d9691b2bb87 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll @@ -64,15 +64,15 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1> -; CHECK-NEXT: CLONE vp<%i.0> = add nsw vp<[[STEPS]]>, ir<-1> -; CHECK-NEXT: CLONE vp<%idxprom> = zext vp<%i.0> -; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%B>, vp<%idxprom> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) vp<%arrayidx> -; CHECK-NEXT: WIDEN vp<[[L:%.+]]> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN vp<%add9> = add vp<[[L]]>, ir<1> -; CHECK-NEXT: CLONE vp<%arrayidx3> = getelementptr inbounds ir<%A>, vp<%idxprom> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) vp<%arrayidx3> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add9> +; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1> +; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0> +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%arrayidx> +; CHECK-NEXT: WIDEN ir<%1> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN ir<%add9> = add ir<%1>, ir<1> +; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) ir<%arrayidx3> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add9> ; CHECK-NEXT: EMIT vp<[[IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -205,15 +205,15 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1> ; CHECK-NEXT: vp<[[STEPS]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1> -; CHECK-NEXT: CLONE vp<%i.0> = add nsw vp<[[STEPS]]>, ir<-1> -; CHECK-NEXT: CLONE vp<%idxprom> = zext vp<%i.0> -; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%B>, vp<%idxprom> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) vp<%arrayidx> -; CHECK-NEXT: WIDEN vp<[[L:%.+]]> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN vp<%conv1> = fadd vp<[[L]]>, ir<1.000000e+00> -; CHECK-NEXT: CLONE vp<%arrayidx3> = getelementptr inbounds ir<%A>, vp<%idxprom> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) vp<%arrayidx3> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%conv1> +; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1> +; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0> +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%arrayidx> +; CHECK-NEXT: WIDEN ir<%1> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN ir<%conv1> = fadd ir<%1>, ir<1.000000e+00> +; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer (reverse) ir<%arrayidx3> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%conv1> ; CHECK-NEXT: EMIT vp<[[IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors diff --git a/llvm/test/Transforms/LoopVectorize/X86/small-size.ll b/llvm/test/Transforms/LoopVectorize/X86/small-size.ll index 5fde9a4947712..dc474fbf67ce8 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/small-size.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/small-size.ll @@ -291,7 +291,7 @@ define void @example3(i32 %n, ptr noalias nocapture %p, ptr noalias nocapture %q ; CHECK: pred.store.continue: ; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP3]], i64 1 ; CHECK-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF15:%.*]], label [[PRED_STORE_CONTINUE16:%.*]] -; CHECK: pred.store.if14: +; CHECK: pred.store.if15: ; CHECK-NEXT: [[TMP7:%.*]] = or disjoint i64 [[OFFSET_IDX]], 4 ; CHECK-NEXT: [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP7]] ; CHECK-NEXT: [[TMP8:%.*]] = or disjoint i64 [[OFFSET_IDX8]], 4 @@ -299,10 +299,10 @@ define void @example3(i32 %n, ptr noalias nocapture %p, ptr noalias nocapture %q ; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[NEXT_GEP10]], align 16 ; CHECK-NEXT: store i32 [[TMP9]], ptr [[NEXT_GEP5]], align 16 ; CHECK-NEXT: br label [[PRED_STORE_CONTINUE16]] -; CHECK: pred.store.continue15: +; CHECK: pred.store.continue16: ; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2 ; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_STORE_IF17:%.*]], label [[PRED_STORE_CONTINUE18:%.*]] -; CHECK: pred.store.if16: +; CHECK: pred.store.if17: ; CHECK-NEXT: [[TMP11:%.*]] = or disjoint i64 [[OFFSET_IDX]], 8 ; CHECK-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP11]] ; CHECK-NEXT: [[TMP12:%.*]] = or disjoint i64 [[OFFSET_IDX8]], 8 @@ -310,10 +310,10 @@ define void @example3(i32 %n, ptr noalias nocapture %p, ptr noalias nocapture %q ; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[NEXT_GEP11]], align 16 ; CHECK-NEXT: store i32 [[TMP13]], ptr [[NEXT_GEP6]], align 16 ; CHECK-NEXT: br label [[PRED_STORE_CONTINUE18]] -; CHECK: pred.store.continue17: +; CHECK: pred.store.continue18: ; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3 ; CHECK-NEXT: br i1 [[TMP14]], label [[PRED_STORE_IF19:%.*]], label [[PRED_STORE_CONTINUE20]] -; CHECK: pred.store.if18: +; CHECK: pred.store.if19: ; CHECK-NEXT: [[TMP15:%.*]] = or disjoint i64 [[OFFSET_IDX]], 12 ; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP15]] ; CHECK-NEXT: [[TMP16:%.*]] = or disjoint i64 [[OFFSET_IDX8]], 12 @@ -321,7 +321,7 @@ define void @example3(i32 %n, ptr noalias nocapture %p, ptr noalias nocapture %q ; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[NEXT_GEP12]], align 16 ; CHECK-NEXT: store i32 [[TMP17]], ptr [[NEXT_GEP7]], align 16 ; CHECK-NEXT: br label [[PRED_STORE_CONTINUE20]] -; CHECK: pred.store.continue19: +; CHECK: pred.store.continue20: ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 ; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -471,7 +471,7 @@ define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst) ; CHECK: pred.store.continue: ; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1 ; CHECK-NEXT: br i1 [[TMP6]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13:%.*]] -; CHECK: pred.store.if11: +; CHECK: pred.store.if12: ; CHECK-NEXT: [[TMP7:%.*]] = or disjoint i64 [[OFFSET_IDX7]], 4 ; CHECK-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]] ; CHECK-NEXT: [[TMP8:%.*]] = or disjoint i64 [[OFFSET_IDX]], 2 @@ -481,10 +481,10 @@ define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst) ; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i32 [[TMP10]], 7 ; CHECK-NEXT: store i32 [[TMP11]], ptr [[NEXT_GEP9]], align 4 ; CHECK-NEXT: br label [[PRED_STORE_CONTINUE13]] -; CHECK: pred.store.continue12: +; CHECK: pred.store.continue13: ; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2 ; CHECK-NEXT: br i1 [[TMP12]], label [[PRED_STORE_IF14:%.*]], label [[PRED_STORE_CONTINUE15:%.*]] -; CHECK: pred.store.if13: +; CHECK: pred.store.if14: ; CHECK-NEXT: [[TMP13:%.*]] = or disjoint i64 [[OFFSET_IDX7]], 8 ; CHECK-NEXT: [[NEXT_GEP10:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP13]] ; CHECK-NEXT: [[TMP14:%.*]] = or disjoint i64 [[OFFSET_IDX]], 4 @@ -494,10 +494,10 @@ define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst) ; CHECK-NEXT: [[TMP17:%.*]] = shl nuw nsw i32 [[TMP16]], 7 ; CHECK-NEXT: store i32 [[TMP17]], ptr [[NEXT_GEP10]], align 4 ; CHECK-NEXT: br label [[PRED_STORE_CONTINUE15]] -; CHECK: pred.store.continue14: +; CHECK: pred.store.continue15: ; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3 ; CHECK-NEXT: br i1 [[TMP18]], label [[PRED_STORE_IF16:%.*]], label [[PRED_STORE_CONTINUE17]] -; CHECK: pred.store.if15: +; CHECK: pred.store.if16: ; CHECK-NEXT: [[TMP19:%.*]] = or disjoint i64 [[OFFSET_IDX7]], 12 ; CHECK-NEXT: [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP19]] ; CHECK-NEXT: [[TMP20:%.*]] = or disjoint i64 [[OFFSET_IDX]], 6 @@ -507,7 +507,7 @@ define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst) ; CHECK-NEXT: [[TMP23:%.*]] = shl nuw nsw i32 [[TMP22]], 7 ; CHECK-NEXT: store i32 [[TMP23]], ptr [[NEXT_GEP11]], align 4 ; CHECK-NEXT: br label [[PRED_STORE_CONTINUE17]] -; CHECK: pred.store.continue16: +; CHECK: pred.store.continue17: ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4 ; CHECK-NEXT: [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], 260 ; CHECK-NEXT: br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll index 7db4d65df2429..c04178a1c13e2 100644 --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll @@ -15,17 +15,17 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) { ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.1> = phi ir<22>, vp<%for.1.next> -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.2> = phi ir<33>, vp<[[FOR1_SPLICE:%.+]]> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.1> = phi ir<22>, ir<%for.1.next> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.2> = phi ir<33>, vp<[[FOR1_SPLICE:%.+]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%gep.ptr> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.ptr> -; CHECK-NEXT: WIDEN vp<%for.1.next> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: EMIT vp<[[FOR1_SPLICE]]> = first-order splice vp<%for.1>, vp<%for.1.next> -; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE:%.+]]> = first-order splice vp<%for.2>, vp<[[FOR1_SPLICE]]> -; CHECK-NEXT: WIDEN vp<%add> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%gep.ptr> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add> +; CHECK-NEXT: CLONE ir<%gep.ptr> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.ptr> +; CHECK-NEXT: WIDEN ir<%for.1.next> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: EMIT vp<[[FOR1_SPLICE]]> = first-order splice ir<%for.1>, ir<%for.1.next> +; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE:%.+]]> = first-order splice ir<%for.2>, vp<[[FOR1_SPLICE]]> +; CHECK-NEXT: WIDEN ir<%add> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%gep.ptr> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors @@ -68,20 +68,20 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) { ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.1> = phi ir<22>, vp<%for.1.next> -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.2> = phi ir<33>, vp<[[FOR1_SPLICE:%.+]]> -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for.3> = phi ir<33>, vp<[[FOR2_SPLICE:%.+]]> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.1> = phi ir<22>, ir<%for.1.next> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.2> = phi ir<33>, vp<[[FOR1_SPLICE:%.+]]> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for.3> = phi ir<33>, vp<[[FOR2_SPLICE:%.+]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%gep.ptr> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.ptr> -; CHECK-NEXT: WIDEN vp<%for.1.next> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: EMIT vp<[[FOR1_SPLICE]]> = first-order splice vp<%for.1>, vp<%for.1.next> -; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE]]> = first-order splice vp<%for.2>, vp<[[FOR1_SPLICE]]> -; CHECK-NEXT: EMIT vp<[[FOR3_SPLICE:%.+]]> = first-order splice vp<%for.3>, vp<[[FOR2_SPLICE]]> -; CHECK-NEXT: WIDEN vp<%add.1> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]> -; CHECK-NEXT: WIDEN vp<%add.2> = add vp<%add.1>, vp<[[FOR3_SPLICE]]> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%gep.ptr> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add.2> +; CHECK-NEXT: CLONE ir<%gep.ptr> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.ptr> +; CHECK-NEXT: WIDEN ir<%for.1.next> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: EMIT vp<[[FOR1_SPLICE]]> = first-order splice ir<%for.1>, ir<%for.1.next> +; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE]]> = first-order splice ir<%for.2>, vp<[[FOR1_SPLICE]]> +; CHECK-NEXT: EMIT vp<[[FOR3_SPLICE:%.+]]> = first-order splice ir<%for.3>, vp<[[FOR2_SPLICE]]> +; CHECK-NEXT: WIDEN ir<%add.1> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]> +; CHECK-NEXT: WIDEN ir<%add.2> = add ir<%add.1>, vp<[[FOR3_SPLICE]]> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%gep.ptr> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add.2> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll index 06f7a7be352f4..833d55f09294e 100644 --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll @@ -20,10 +20,10 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<[[FOR:%.+]]> = phi ir<0>, vp<%conv> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> ; CHECK-NEXT: vp<[[STEPS:%.]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> +; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> ; CHECK-NEXT: Successor(s): pred.load ; CHECK-EMPTY: ; CHECK-NEXT: pred.load: { @@ -32,19 +32,19 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize ; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.if: -; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE vp<%lv> = load vp<%gep> (S->V) +; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep> (S->V) ; CHECK-NEXT: Successor(s): pred.load.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = vp<%lv> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%lv> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.0 ; CHECK-EMPTY: ; CHECK-NEXT: loop.0: -; CHECK-NEXT: WIDEN-CAST vp<%conv> = sext vp<[[PRED1]]> to i32 -; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<[[FOR]]>, vp<%conv> +; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext vp<[[PRED1]]> to i32 +; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%0>, ir<%conv> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -53,14 +53,14 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x> -; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE vp<%add> = add vp<%conv>, vp<%rem> -; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep.dst> +; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> +; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE ir<%add> = add ir<%conv>, ir<%rem> +; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.dst> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%rem> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%rem> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.2 @@ -111,11 +111,11 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%recur> = phi ir<0>, vp<%recur.next> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> -; CHECK-NEXT: WIDEN-CAST vp<%recur.next> = sext ir<%y> to i32 -; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%recur>, vp<%recur.next> +; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> +; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32 +; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur>, ir<%recur.next> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -124,15 +124,15 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x> +; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE vp<%add> = add vp<%rem>, vp<%recur.next> -; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep> +; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE ir<%add> = add ir<%rem>, ir<%recur.next> +; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%rem> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%rem> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.1 @@ -181,12 +181,12 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%recur> = phi ir<0>, vp<%recur.next> -; CHECK-NEXT: WIDEN-REDUCTION-PHI vp<%and.red> = phi ir<1234>, vp<%and.red.next> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next> +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%and.red> = phi ir<1234>, ir<%and.red.next> ; CHECK-NEXT: EMIT vp<[[WIDEN_CAN:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<[[WIDEN_CAN]]>, vp<[[BTC]]> -; CHECK-NEXT: WIDEN-CAST vp<%recur.next> = sext ir<%y> to i32 -; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%recur>, vp<%recur.next> +; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32 +; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur>, ir<%recur.next> ; CHECK-NEXT: Successor(s): pred.srem ; CHECK-EMPTY: ; CHECK-NEXT: pred.srem: { @@ -195,19 +195,19 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: Successor(s): pred.srem.if, pred.srem.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.srem.if: -; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x> (S->V) +; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> (S->V) ; CHECK-NEXT: Successor(s): pred.srem.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.srem.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%rem> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%rem> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.0 ; CHECK-EMPTY: ; CHECK-NEXT: loop.0: -; CHECK-NEXT: WIDEN vp<%add> = add vp<[[PRED]]>, vp<%recur.next> -; CHECK-NEXT: WIDEN vp<%and.red.next> = and vp<%and.red>, vp<%add> -; CHECK-NEXT: EMIT vp<[[SEL:%.+]]> = select vp<[[MASK]]>, vp<%and.red.next>, vp<%and.red> +; CHECK-NEXT: WIDEN ir<%add> = add vp<[[PRED]]>, ir<%recur.next> +; CHECK-NEXT: WIDEN ir<%and.red.next> = and ir<%and.red>, ir<%add> +; CHECK-NEXT: EMIT vp<[[SEL:%.+]]> = select vp<[[MASK]]>, ir<%and.red.next>, ir<%and.red> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -215,7 +215,7 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: ; CHECK-NEXT: middle.block: -; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result vp<%and.red>, vp<[[SEL]]> +; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%and.red>, vp<[[SEL]]> ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: Live-out i32 %res = vp<[[RED_RES]]> @@ -257,11 +257,11 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<[[FOR:%.+]]> = phi ir<0>, vp<%conv> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> -; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> +; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> +; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> ; CHECK-NEXT: Successor(s): pred.load ; CHECK-EMPTY: ; CHECK-NEXT: pred.load: { @@ -270,18 +270,18 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr ; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.if: -; CHECK-NEXT: REPLICATE vp<%lv> = load vp<%gep> (S->V) +; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep> (S->V) ; CHECK-NEXT: Successor(s): pred.load.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.0 ; CHECK-EMPTY: ; CHECK-NEXT: loop.0: -; CHECK-NEXT: WIDEN-CAST vp<%conv> = sext vp<[[PRED]]> to i32 -; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<[[FOR]]>, vp<%conv> +; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext vp<[[PRED]]> to i32 +; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%0>, ir<%conv> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK: pred.store: { @@ -290,18 +290,18 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK: pred.store.if: -; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x> -; CHECK-NEXT: REPLICATE vp<%lv.2> = load vp<%gep> -; CHECK-NEXT: REPLICATE vp<%conv.lv.2> = sext vp<%lv.2> -; CHECK-NEXT: REPLICATE vp<%add.1> = add vp<%conv>, vp<%rem> -; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE vp<%add> = add vp<%add.1>, vp<%conv.lv.2> -; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep.dst> +; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> +; CHECK-NEXT: REPLICATE ir<%lv.2> = load ir<%gep> +; CHECK-NEXT: REPLICATE ir<%conv.lv.2> = sext ir<%lv.2> +; CHECK-NEXT: REPLICATE ir<%add.1> = add ir<%conv>, ir<%rem> +; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE ir<%add> = add ir<%add.1>, ir<%conv.lv.2> +; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.dst> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = vp<%rem> -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%lv.2> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%rem> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%lv.2> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.3 @@ -360,11 +360,11 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%recur> = phi ir<0>, vp<%recur.next> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> -; CHECK-NEXT: WIDEN-CAST vp<%recur.next> = sext ir<%y> to i32 -; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%recur>, vp<%recur.next> +; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> +; CHECK-NEXT: WIDEN-CAST ir<%recur.next> = sext ir<%y> to i32 +; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%recur>, ir<%recur.next> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -374,17 +374,17 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE vp<%rem> = srem vp<[[SPLICE]]>, ir<%x> -; CHECK-NEXT: REPLICATE vp<%rem.div> = sdiv ir<20>, vp<%rem> -; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store vp<%rem.div>, vp<%gep> -; CHECK-NEXT: REPLICATE vp<%gep.2> = getelementptr ir<%dst.2>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store vp<%rem.div>, vp<%gep.2> +; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> +; CHECK-NEXT: REPLICATE ir<%rem.div> = sdiv ir<20>, ir<%rem> +; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE store ir<%rem.div>, ir<%gep> +; CHECK-NEXT: REPLICATE ir<%gep.2> = getelementptr ir<%dst.2>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE store ir<%rem.div>, ir<%gep.2> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%rem> -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%rem.div> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%rem> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%rem.div> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.3 @@ -435,12 +435,12 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%.pn> = phi ir<0>, vp<[[L:%.+]]> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%.pn> = phi ir<0>, ir<[[L:%.+]]> ; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<2> + vp<[[CAN_IV]]> * ir<1> ; CHECK-NEXT: EMIT vp<[[WIDE_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]> ; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp ule vp<[[WIDE_IV]]>, vp<[[BTC]]> -; CHECK-NEXT: CLONE vp<[[L]]> = load ir<%src> -; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%.pn>, vp<[[L]]> +; CHECK-NEXT: CLONE ir<[[L]]> = load ir<%src> +; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%.pn>, ir<[[L]]> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -449,14 +449,14 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE vp<%val> = sdiv vp<[[SPLICE]]>, ir<%x> +; CHECK-NEXT: REPLICATE ir<%val> = sdiv vp<[[SPLICE]]>, ir<%x> ; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr ir<%dst>, vp<[[SCALAR_STEPS]]> -; CHECK-NEXT: REPLICATE store vp<%val>, vp<%gep.dst> +; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[SCALAR_STEPS]]> +; CHECK-NEXT: REPLICATE store ir<%val>, ir<%gep.dst> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_VAL:%.+]]> = vp<%val> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_VAL:%.+]]> = ir<%val> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.1 diff --git a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll index f0b8ded142210..b4ab6f7e8ceb7 100644 --- a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll +++ b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll @@ -49,9 +49,9 @@ for.end: ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[COND:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> -; CHECK-NEXT: WIDEN vp<%cond0> = icmp ult vp<%iv>, ir<13> -; CHECK-NEXT: WIDEN-SELECT vp<%s> = select vp<%cond0>, ir<10>, ir<20> +; CHECK-NEXT: EMIT vp<[[COND:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> +; CHECK-NEXT: WIDEN ir<%cond0> = icmp ult ir<%iv>, ir<13> +; CHECK-NEXT: WIDEN-SELECT ir<%s> = select ir<%cond0>, ir<10>, ir<20> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -61,8 +61,8 @@ for.end: ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store vp<%s>, vp<%gep> +; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: diff --git a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll index 54a66dd424fae..81cc2024bb31a 100644 --- a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll +++ b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll @@ -21,9 +21,9 @@ ; DBG-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; DBG-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%start> + vp<[[CAN_IV]]> * ir<1> ; DBG-NEXT: vp<[[IV_STEPS:%.]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1> -; DBG-NEXT: CLONE vp<%min> = call @llvm.smin.i32(vp<[[IV_STEPS]]>, ir<65535>) -; DBG-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%dst>, vp<[[IV_STEPS]]> -; DBG-NEXT: CLONE store vp<%min>, vp<%arrayidx> +; DBG-NEXT: CLONE ir<%min> = call @llvm.smin.i32(vp<[[IV_STEPS]]>, ir<65535>) +; DBG-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%dst>, vp<[[IV_STEPS]]> +; DBG-NEXT: CLONE store ir<%min>, ir<%arrayidx> ; DBG-NEXT: EMIT vp<[[INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; DBG-NEXT: EMIT branch-on-count vp<[[INC]]>, vp<[[VEC_TC]]> ; DBG-NEXT: No successors @@ -89,14 +89,14 @@ declare i32 @llvm.smin.i32(i32, i32) ; DBG-EMPTY: ; DBG-NEXT: pred.store.if: ; DBG-NEXT: vp<[[STEPS2:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; DBG-NEXT: CLONE vp<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS2]]> -; DBG-NEXT: CLONE vp<%l> = load vp<%gep.src> -; DBG-NEXT: CLONE vp<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS2]]> -; DBG-NEXT: CLONE store vp<%l>, vp<%gep.dst> +; DBG-NEXT: CLONE ir<%gep.src> = getelementptr inbounds ir<%src>, vp<[[STEPS2]]> +; DBG-NEXT: CLONE ir<%l> = load ir<%gep.src> +; DBG-NEXT: CLONE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS2]]> +; DBG-NEXT: CLONE store ir<%l>, ir<%gep.dst> ; DBG-NEXT: Successor(s): pred.store.continue ; DBG-EMPTY: ; DBG-NEXT: pred.store.continue: -; DBG-NEXT: PHI-PREDICATED-INSTRUCTION vp<{{.+}}> = vp<%l> +; DBG-NEXT: PHI-PREDICATED-INSTRUCTION vp<{{.+}}> = ir<%l> ; DBG-NEXT: No successors ; DBG-NEXT: } ; DBG-NEXT: Successor(s): cond.false.1 @@ -190,10 +190,10 @@ exit: ; DBG-NEXT: vector loop: { ; DBG-NEXT: vector.body: ; DBG-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION -; DBG-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for> = phi ir<0>, vp<[[SCALAR_STEPS:.+]]> +; DBG-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, vp<[[SCALAR_STEPS:.+]]> ; DBG-NEXT: SCALAR-CAST vp<[[TRUNC_IV:%.+]]> = trunc vp<[[CAN_IV]]> to i32 ; DBG-NEXT: vp<[[SCALAR_STEPS]]> = SCALAR-STEPS vp<[[TRUNC_IV]]>, vp<[[CAST]]> -; DBG-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%for>, vp<[[SCALAR_STEPS]]> +; DBG-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%for>, vp<[[SCALAR_STEPS]]> ; DBG-NEXT: CLONE store vp<[[SPLICE]]>, ir<%dst> ; DBG-NEXT: EMIT vp<[[IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; DBG-NEXT: EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VTC]]> diff --git a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll index 85d035f1ddaee..12b8e657aabf1 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll @@ -28,13 +28,13 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw ; CHECK-NEXT: "vector.body:\l" + ; CHECK-NEXT: " EMIT vp\<[[CAN_IV:%.+]]\> = CANONICAL-INDUCTION ir\<0\>, vp\<[[CAN_IV_NEXT:%.+]]\>\l" + ; CHECK-NEXT: " vp\<[[STEPS:%.+]]\> = SCALAR-STEPS vp\<[[CAN_IV]]\>, ir\<1\>\l" + -; CHECK-NEXT: " CLONE vp\<%arrayidx\> = getelementptr inbounds ir\<%y\>, vp\<[[STEPS]]\>\l" + -; CHECK-NEXT: " vp\<[[VEC_PTR:%.+]]\> = vector-pointer vp\<%arrayidx\>\l" + -; CHECK-NEXT: " WIDEN vp\<%lv\> = load vp\<[[VEC_PTR]]\>\l" + -; CHECK-NEXT: " WIDEN-CALL vp\<%call\> = call @llvm.sqrt.f32(vp\<%lv\>) (using vector intrinsic)\l" + -; CHECK-NEXT: " CLONE vp\<%arrayidx2\> = getelementptr inbounds ir\<%x\>, vp\<[[STEPS]]\>\l" + -; CHECK-NEXT: " vp\<[[VEC_PTR2:%.+]]\> = vector-pointer vp\<%arrayidx2\>\l" + -; CHECK-NEXT: " WIDEN store vp\<[[VEC_PTR2]]\>, vp\<%call\>\l" + +; CHECK-NEXT: " CLONE ir\<%arrayidx\> = getelementptr inbounds ir\<%y\>, vp\<[[STEPS]]\>\l" + +; CHECK-NEXT: " vp\<[[VEC_PTR:%.+]]\> = vector-pointer ir\<%arrayidx\>\l" + +; CHECK-NEXT: " WIDEN ir\<%lv\> = load vp\<[[VEC_PTR]]\>\l" + +; CHECK-NEXT: " WIDEN-CALL ir\<%call\> = call @llvm.sqrt.f32(ir\<%lv\>) (using vector intrinsic)\l" + +; CHECK-NEXT: " CLONE ir\<%arrayidx2\> = getelementptr inbounds ir\<%x\>, vp\<[[STEPS]]\>\l" + +; CHECK-NEXT: " vp\<[[VEC_PTR2:%.+]]\> = vector-pointer ir\<%arrayidx2\>\l" + +; CHECK-NEXT: " WIDEN store vp\<[[VEC_PTR2]]\>, ir\<%call\>\l" + ; CHECK-NEXT: " EMIT vp\<[[CAN_IV_NEXT]]\> = add nuw vp\<[[CAN_IV]]\>, vp\<[[VFxUF]]\>\l" + ; CHECK-NEXT: " EMIT branch-on-count vp\<[[CAN_IV_NEXT]]\>, vp\<[[VEC_TC]]\>\l" + ; CHECK-NEXT: "No successors\l" diff --git a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll index 00996e7eee566..7ab2459ada2ed 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll @@ -16,9 +16,9 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) { ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next.p, ir<1> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%gep> = getelementptr inbounds ir<%dst>, vp<[[STEPS:%.+]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%iv> +; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%dst>, vp<[[STEPS:%.+]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%iv> ; CHECK-NEXT: EMIT vp<[[CAN_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_INC]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll index 036386e7dde76..ca9dfdc6f6d29 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll @@ -26,10 +26,10 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) { ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[PADD:%.+]]> = ptradd ir<%A>, vp<[[STEPS]]> ; CHECK-NEXT: vp<[[VPTR:%.]]> = vector-pointer vp<[[PADD]]> -; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VPTR]]> -; CHECK-NEXT: WIDEN vp<%add> = add nsw vp<%l>, ir<10> +; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VPTR]]> +; CHECK-NEXT: WIDEN ir<%add> = add nsw ir<%l>, ir<10> ; CHECK-NEXT: vp<[[VPTR2:%.+]]> = vector-pointer vp<[[PADD]]> -; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, vp<%add> +; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, ir<%add> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV:%.+]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors @@ -58,10 +58,10 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) { ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: EMIT vp<[[PADD:%.+]]> = ptradd ir<%A>, vp<[[STEPS]]> ; CHECK-NEXT: vp<[[VPTR:%.]]> = vector-pointer vp<[[PADD]]> -; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VPTR]]> -; CHECK-NEXT: WIDEN vp<%add> = add nsw vp<%l>, ir<10> +; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VPTR]]> +; CHECK-NEXT: WIDEN ir<%add> = add nsw ir<%l>, ir<10> ; CHECK-NEXT: vp<[[VPTR2:%.+]]> = vector-pointer vp<[[PADD]]> -; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, vp<%add> +; CHECK-NEXT: WIDEN store vp<[[VPTR2]]>, ir<%add> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV:%.+]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-cond ir ; CHECK-NEXT: No successors diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll index 1795c6a4fe33f..65df5973ac75b 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll @@ -15,28 +15,28 @@ define void @foo(i64 %n) { ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: -; CHECK-NEXT: WIDEN-PHI vp<%outer.iv> = phi ir<0>, vp<%outer.iv.next> -; CHECK-NEXT: EMIT vp<%gep.1> = getelementptr ir<@arr2>, ir<0>, vp<%outer.iv> -; CHECK-NEXT: EMIT store vp<%outer.iv>, vp<%gep.1> -; CHECK-NEXT: EMIT vp<%add> = add vp<%outer.iv>, ir<%n> +; CHECK-NEXT: WIDEN-PHI ir<%outer.iv> = phi ir<0>, ir<%outer.iv.next> +; CHECK-NEXT: EMIT ir<%gep.1> = getelementptr ir<@arr2>, ir<0>, ir<%outer.iv> +; CHECK-NEXT: EMIT store ir<%outer.iv>, ir<%gep.1> +; CHECK-NEXT: EMIT ir<%add> = add ir<%outer.iv>, ir<%n> ; CHECK-NEXT: Successor(s): inner ; CHECK-EMPTY: ; CHECK-NEXT: inner: { ; CHECK-NEXT: inner: -; CHECK-NEXT: WIDEN-PHI vp<%inner.iv> = phi ir<0>, vp<%inner.iv.next> -; CHECK-NEXT: EMIT vp<%gep.2> = getelementptr ir<@arr>, ir<0>, vp<%inner.iv>, vp<%outer.iv> -; CHECK-NEXT: EMIT store vp<%add>, vp<%gep.2> -; CHECK-NEXT: EMIT vp<%inner.iv.next> = add vp<%inner.iv>, ir<1> -; CHECK-NEXT: EMIT vp<%inner.ec> = icmp vp<%inner.iv.next>, ir<8> -; CHECK-NEXT: EMIT branch-on-cond vp<%inner.ec> +; CHECK-NEXT: WIDEN-PHI ir<%inner.iv> = phi ir<0>, ir<%inner.iv.next> +; CHECK-NEXT: EMIT ir<%gep.2> = getelementptr ir<@arr>, ir<0>, ir<%inner.iv>, ir<%outer.iv> +; CHECK-NEXT: EMIT store ir<%add>, ir<%gep.2> +; CHECK-NEXT: EMIT ir<%inner.iv.next> = add ir<%inner.iv>, ir<1> +; CHECK-NEXT: EMIT ir<%inner.ec> = icmp ir<%inner.iv.next>, ir<8> +; CHECK-NEXT: EMIT branch-on-cond ir<%inner.ec> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): outer.latch ; CHECK-EMPTY: ; CHECK-NEXT: outer.latch: -; CHECK-NEXT: EMIT vp<%outer.iv.next> = add vp<%outer.iv>, ir<1> -; CHECK-NEXT: EMIT vp<%outer.ec> = icmp vp<%outer.iv.next>, ir<8> -; CHECK-NEXT: EMIT branch-on-cond vp<%outer.ec> +; CHECK-NEXT: EMIT ir<%outer.iv.next> = add ir<%outer.iv>, ir<1> +; CHECK-NEXT: EMIT ir<%outer.ec> = icmp ir<%outer.iv.next>, ir<8> +; CHECK-NEXT: EMIT branch-on-cond ir<%outer.ec> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): middle.block diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll index 56431af1b854e..7056bbe6ba1b7 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll @@ -20,13 +20,13 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx> -; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN-CALL vp<%call> = call @llvm.sqrt.f32(vp<%lv>) -; CHECK-NEXT: CLONE vp<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx2> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%call> +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx> +; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN-CALL ir<%call> = call @llvm.sqrt.f32(ir<%lv>) +; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx2> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -71,15 +71,15 @@ define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x, ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi %iv.next, 0, ir<1> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: WIDEN-GEP Inv[Var] vp<%arrayidx> = getelementptr inbounds ir<%y>, vp<%iv> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx> -; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN vp<%cmp> = icmp eq vp<%arrayidx>, ir<%z> -; CHECK-NEXT: WIDEN-SELECT vp<%sel> = select vp<%cmp>, ir<1.000000e+01>, ir<2.000000e+01> -; CHECK-NEXT: WIDEN vp<%add> = fadd vp<%lv>, vp<%sel> -; CHECK-NEXT: CLONE vp<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx2> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add> +; CHECK-NEXT: WIDEN-GEP Inv[Var] ir<%arrayidx> = getelementptr inbounds ir<%y>, ir<%iv> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx> +; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN ir<%cmp> = icmp eq ir<%arrayidx>, ir<%z> +; CHECK-NEXT: WIDEN-SELECT ir<%sel> = select ir<%cmp>, ir<1.000000e+01>, ir<2.000000e+01> +; CHECK-NEXT: WIDEN ir<%add> = fadd ir<%lv>, ir<%sel> +; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx2> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -124,12 +124,12 @@ define float @print_reduction(i64 %n, ptr noalias %y) { ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> -; CHECK-NEXT: WIDEN-REDUCTION-PHI vp<%red> = phi ir<0.000000e+00>, vp<%red.next.1> +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx> -; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: REDUCE vp<%red.next.1> = vp<%red> + fast reduce.fadd (vp<%lv>) +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx> +; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>) ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -137,7 +137,7 @@ define float @print_reduction(i64 %n, ptr noalias %y) { ; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: ; CHECK-NEXT: middle.block: -; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result vp<%red>, vp<%red.next.1> +; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%red>, ir<%red.next> ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: Live-out float %red.next.lcssa = vp<[[RED_RES]]> @@ -173,12 +173,12 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> -; CHECK-NEXT: WIDEN-REDUCTION-PHI vp<%red> = phi ir<0.000000e+00>, vp<%red.next.1> +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next> ; CHECK-NEXT: vp<[[IV:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[IV]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx> -; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: REDUCE vp<%red.next.1> = vp<%red> + fast reduce.fadd (vp<%lv>) (with final reduction value stored in invariant address sank outside of loop) +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[IV]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx> +; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>) (with final reduction value stored in invariant address sank outside of loop) ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -186,7 +186,7 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no ; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: ; CHECK-NEXT: middle.block: -; CHECK-NEXT: EMIT vp<[[RED_RES:.+]]> = compute-reduction-result vp<%red>, vp<%red.next.1> +; CHECK-NEXT: EMIT vp<[[RED_RES:.+]]> = compute-reduction-result ir<%red>, ir<%red.next> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -227,20 +227,20 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) { ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: WIDEN-INDUCTION %i = phi 0, %i.next, ir<1> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: WIDEN vp<%cmp> = icmp ult vp<%i>, ir<5> +; CHECK-NEXT: WIDEN ir<%cmp> = icmp ult ir<%i>, ir<5> ; CHECK-NEXT: Successor(s): pred.udiv ; CHECK-EMPTY: ; CHECK-NEXT: pred.udiv: { ; CHECK-NEXT: pred.udiv.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<%cmp> +; CHECK-NEXT: BRANCH-ON-MASK ir<%cmp> ; CHECK-NEXT: Successor(s): pred.udiv.if, pred.udiv.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.udiv.if: -; CHECK-NEXT: REPLICATE vp<%tmp4> = udiv ir<%n>, vp<[[STEPS]]> (S->V) +; CHECK-NEXT: REPLICATE ir<%tmp4> = udiv ir<%n>, vp<[[STEPS]]> (S->V) ; CHECK-NEXT: Successor(s): pred.udiv.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.udiv.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%tmp4> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%tmp4> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): if.then.0 @@ -302,19 +302,19 @@ define void @print_interleave_groups(i32 %C, i32 %D) { ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * ir<4> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<4> -; CHECK-NEXT: CLONE vp<%gep.AB.0> = getelementptr inbounds ir<@AB>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at %AB.0, vp<%gep.AB.0> -; CHECK-NEXT: vp<%AB.0> = load from index 0 -; CHECK-NEXT: vp<%AB.1> = load from index 1 -; CHECK-NEXT: vp<%AB.3> = load from index 3 -; CHECK-NEXT: CLONE vp<%iv.plus.3> = add vp<[[STEPS]]>, ir<3> -; CHECK-NEXT: WIDEN vp<%add> = add nsw vp<%AB.0>, vp<%AB.1> -; CHECK-NEXT: CLONE vp<%gep.CD.3> = getelementptr inbounds ir<@CD>, ir<0>, vp<%iv.plus.3> -; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at , vp<%gep.CD.3> -; CHECK-NEXT: store vp<%add> to index 0 +; CHECK-NEXT: CLONE ir<%gep.AB.0> = getelementptr inbounds ir<@AB>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at %AB.0, ir<%gep.AB.0> +; CHECK-NEXT: ir<%AB.0> = load from index 0 +; CHECK-NEXT: ir<%AB.1> = load from index 1 +; CHECK-NEXT: ir<%AB.3> = load from index 3 +; CHECK-NEXT: CLONE ir<%iv.plus.3> = add vp<[[STEPS]]>, ir<3> +; CHECK-NEXT: WIDEN ir<%add> = add nsw ir<%AB.0>, ir<%AB.1> +; CHECK-NEXT: CLONE ir<%gep.CD.3> = getelementptr inbounds ir<@CD>, ir<0>, ir<%iv.plus.3> +; CHECK-NEXT: INTERLEAVE-GROUP with factor 4 at , ir<%gep.CD.3> +; CHECK-NEXT: store ir<%add> to index 0 ; CHECK-NEXT: store ir<1> to index 1 ; CHECK-NEXT: store ir<2> to index 2 -; CHECK-NEXT: store vp<%AB.3> to index 3 +; CHECK-NEXT: store ir<%AB.3> to index 3 ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -369,16 +369,16 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) { ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> -; CHECK-NEXT: WIDEN-REDUCTION-PHI vp<%sum.07> = phi ir<0.000000e+00>, vp<%muladd.1> +; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.07> = phi ir<0.000000e+00>, ir<%muladd> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%arrayidx> -; CHECK-NEXT: WIDEN vp<%l.a> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: CLONE vp<%arrayidx2> = getelementptr inbounds ir<%b>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%arrayidx2> -; CHECK-NEXT: WIDEN vp<%l.b> = load vp<[[VEC_PTR2]]> -; CHECK-NEXT: EMIT vp<[[FMUL:%.+]]> = fmul nnan ninf nsz vp<%l.a>, vp<%l.b> -; CHECK-NEXT: REDUCE vp<[[MULADD:%.+]]> = vp<%sum.07> + nnan ninf nsz reduce.fadd (vp<[[FMUL]]>) +; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx> +; CHECK-NEXT: WIDEN ir<%l.a> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%b>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx2> +; CHECK-NEXT: WIDEN ir<%l.b> = load vp<[[VEC_PTR2]]> +; CHECK-NEXT: EMIT vp<[[FMUL:%.+]]> = fmul nnan ninf nsz ir<%l.a>, ir<%l.b> +; CHECK-NEXT: REDUCE ir<[[MULADD:%.+]]> = ir<%sum.07> + nnan ninf nsz reduce.fadd (vp<[[FMUL]]>) ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -386,7 +386,7 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) { ; CHECK-NEXT: Successor(s): middle.block ; CHECK-EMPTY: ; CHECK-NEXT: middle.block: -; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result vp<%sum.07>, vp<[[MULADD]]> +; CHECK-NEXT: EMIT vp<[[RED_RES:%.+]]> = compute-reduction-result ir<%sum.07>, ir<[[MULADD]]> ; CHECK-NEXT: No successors ; CHECK-EMPTY: ; CHECK-NEXT: Live-out float %muladd.lcssa = vp<[[RED_RES]]> @@ -425,15 +425,15 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%isd> = getelementptr inbounds ir<%asd>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%isd> -; CHECK-NEXT: WIDEN vp<%lsd> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN vp<%psd> = add nuw nsw vp<%lsd>, ir<23> -; CHECK-NEXT: WIDEN vp<%cmp1> = icmp slt vp<%lsd>, ir<100> -; CHECK-NEXT: EMIT vp<[[NOT1:%.+]]> = not vp<%cmp1>, !dbg /tmp/s.c:5:3 -; CHECK-NEXT: WIDEN vp<%cmp2> = icmp sge vp<%lsd>, ir<200> -; CHECK-NEXT: EMIT vp<[[SEL1:%.+]]> = select vp<[[NOT1]]>, vp<%cmp2>, ir, !dbg /tmp/s.c:5:21 -; CHECK-NEXT: EMIT vp<[[OR1:%.+]]> = or vp<[[SEL1]]>, vp<%cmp1> +; CHECK-NEXT: CLONE ir<%isd> = getelementptr inbounds ir<%asd>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%isd> +; CHECK-NEXT: WIDEN ir<%lsd> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN ir<%psd> = add nuw nsw ir<%lsd>, ir<23> +; CHECK-NEXT: WIDEN ir<%cmp1> = icmp slt ir<%lsd>, ir<100> +; CHECK-NEXT: EMIT vp<[[NOT1:%.+]]> = not ir<%cmp1>, !dbg /tmp/s.c:5:3 +; CHECK-NEXT: WIDEN ir<%cmp2> = icmp sge ir<%lsd>, ir<200> +; CHECK-NEXT: EMIT vp<[[SEL1:%.+]]> = select vp<[[NOT1]]>, ir<%cmp2>, ir, !dbg /tmp/s.c:5:21 +; CHECK-NEXT: EMIT vp<[[OR1:%.+]]> = or vp<[[SEL1]]>, ir<%cmp1> ; CHECK-NEXT: Successor(s): pred.sdiv ; CHECK-EMPTY: ; CHECK-NEXT: pred.sdiv: { @@ -442,17 +442,17 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db ; CHECK-NEXT: Successor(s): pred.sdiv.if, pred.sdiv.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.sdiv.if: -; CHECK-NEXT: REPLICATE vp<%sd1> = sdiv vp<%psd>, vp<%lsd> (S->V) +; CHECK-NEXT: REPLICATE ir<%sd1> = sdiv ir<%psd>, ir<%lsd> (S->V) ; CHECK-NEXT: Successor(s): pred.sdiv.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.sdiv.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI:%.+]]> = vp<%sd1> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI:%.+]]> = ir<%sd1> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): if.then.0 ; CHECK-EMPTY: ; CHECK-NEXT: if.then.0: -; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not vp<%cmp2> +; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not ir<%cmp2> ; CHECK-NEXT: EMIT vp<[[SEL2:%.+]]> = select vp<[[NOT1]]>, vp<[[NOT2]]>, ir ; CHECK-NEXT: BLEND ir<%ysd.0> = vp<[[PHI]]> ir<%psd>/vp<[[SEL2]]> ; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%isd> @@ -520,12 +520,12 @@ define void @print_expand_scev(i64 %y, ptr %ptr) { ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: WIDEN-INDUCTION\l" + ; CHECK-NEXT: " %iv = phi %iv.next, 0\l" + -; CHECK-NEXT: " vp<%v2>, vp<[[EXP_SCEV]]> +; CHECK-NEXT: " ir<%v2>, vp<[[EXP_SCEV]]> ; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * vp<[[EXP_SCEV]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, vp<[[EXP_SCEV]]> -; CHECK-NEXT: WIDEN vp<%v3> = add nuw vp<%v2>, ir<1> -; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store vp<%v3>, vp<%gep> +; CHECK-NEXT: WIDEN ir<%v3> = add nuw ir<%v2>, ir<1> +; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE store ir<%v3>, ir<%gep> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors @@ -571,9 +571,9 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) { ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: WIDEN vp<%add> = add vp<%iv>, ir<%off> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep> +; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> +; CHECK-NEXT: WIDEN ir<%add> = add ir<%iv>, ir<%off> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep> ; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<0> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> @@ -584,7 +584,7 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) { ; CHECK-NEXT: middle.block: ; CHECK-NEXT: No successors ; CHECK-EMPTY: -; CHECK-NEXT: Live-out i32 %lcssa = vp<%add> +; CHECK-NEXT: Live-out i32 %lcssa = ir<%add> ; CHECK-NEXT: } ; entry: @@ -618,15 +618,15 @@ define void @print_fast_math_flags(i64 %n, ptr noalias %y, ptr noalias %x, ptr % ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%gep.y> = getelementptr inbounds ir<%y>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.y> -; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN vp<%add> = fadd nnan vp<%lv>, ir<1.000000e+00> -; CHECK-NEXT: WIDEN vp<%mul> = fmul reassoc nnan ninf nsz arcp contract afn vp<%add>, ir<2.000000e+00> -; CHECK-NEXT: WIDEN vp<%div> = fdiv reassoc nsz contract vp<%mul>, ir<2.000000e+00> -; CHECK-NEXT: CLONE vp<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.x> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%div> +; CHECK-NEXT: CLONE ir<%gep.y> = getelementptr inbounds ir<%y>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.y> +; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN ir<%add> = fadd nnan ir<%lv>, ir<1.000000e+00> +; CHECK-NEXT: WIDEN ir<%mul> = fmul reassoc nnan ninf nsz arcp contract afn ir<%add>, ir<2.000000e+00> +; CHECK-NEXT: WIDEN ir<%div> = fdiv reassoc nsz contract ir<%mul>, ir<2.000000e+00> +; CHECK-NEXT: CLONE ir<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.x> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%div> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -671,14 +671,14 @@ define void @print_exact_flags(i64 %n, ptr noalias %x) { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.x> -; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN vp<%div.1> = udiv exact vp<%lv>, ir<20> -; CHECK-NEXT: WIDEN vp<%div.2> = udiv vp<%lv>, ir<60> -; CHECK-NEXT: WIDEN vp<%add> = add nuw nsw vp<%div.1>, vp<%div.2> -; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer vp<%gep.x> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, vp<%add> +; CHECK-NEXT: CLONE ir<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.x> +; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN ir<%div.1> = udiv exact ir<%lv>, ir<20> +; CHECK-NEXT: WIDEN ir<%div.2> = udiv ir<%lv>, ir<60> +; CHECK-NEXT: WIDEN ir<%add> = add nuw nsw ir<%div.1>, ir<%div.2> +; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%gep.x> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%add> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -722,25 +722,25 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%ld.addr> = getelementptr inbounds ir<%src>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%ld.addr> -; CHECK-NEXT: WIDEN vp<%ld.value> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN vp<%ifcond> = fcmp oeq vp<%ld.value>, ir<5.000000e+00> +; CHECK-NEXT: CLONE ir<%ld.addr> = getelementptr inbounds ir<%src>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%ld.addr> +; CHECK-NEXT: WIDEN ir<%ld.value> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN ir<%ifcond> = fcmp oeq ir<%ld.value>, ir<5.000000e+00> ; CHECK-NEXT: Successor(s): pred.call ; CHECK-EMPTY: ; CHECK-NEXT: pred.call: { ; CHECK-NEXT: pred.call.entry: -; CHECK-NEXT: BRANCH-ON-MASK vp<%ifcond> +; CHECK-NEXT: BRANCH-ON-MASK ir<%ifcond> ; CHECK-NEXT: Successor(s): pred.call.if, pred.call.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.call.if: -; CHECK-NEXT: REPLICATE vp<%foo.ret.1> = call nnan ninf nsz @foo(vp<%ld.value>) (S->V) -; CHECK-NEXT: REPLICATE vp<%foo.ret.2> = call @foo(vp<%ld.value>) (S->V) +; CHECK-NEXT: REPLICATE ir<%foo.ret.1> = call nnan ninf nsz @foo(ir<%ld.value>) (S->V) +; CHECK-NEXT: REPLICATE ir<%foo.ret.2> = call @foo(ir<%ld.value>) (S->V) ; CHECK-NEXT: Successor(s): pred.call.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.call.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI1:%.+]]> = vp<%foo.ret.1> -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI2:%.+]]> = vp<%foo.ret.2> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI1:%.+]]> = ir<%foo.ret.1> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PHI2:%.+]]> = ir<%foo.ret.2> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): if.then.1 @@ -804,14 +804,14 @@ define void @print_disjoint_flags(i64 %n, ptr noalias %x) { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.x> -; CHECK-NEXT: WIDEN vp<%lv> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN vp<%or.1> = or disjoint vp<%lv>, ir<1> -; CHECK-NEXT: WIDEN vp<%or.2> = or vp<%lv>, ir<3> -; CHECK-NEXT: WIDEN vp<%add> = add nuw nsw vp<%or.1>, vp<%or.2> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.x> -; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, vp<%add> +; CHECK-NEXT: CLONE ir<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.x> +; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN ir<%or.1> = or disjoint ir<%lv>, ir<1> +; CHECK-NEXT: WIDEN ir<%or.2> = or ir<%lv>, ir<3> +; CHECK-NEXT: WIDEN ir<%add> = add nuw nsw ir<%or.1>, ir<%or.2> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.x> +; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%add> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors @@ -855,11 +855,11 @@ define void @zext_nneg(ptr noalias %p, ptr noalias %p1) { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%idx> = getelementptr ir<%p>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%idx> -; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN-CAST vp<%zext> = zext nneg vp<%l> -; CHECK-NEXT: REPLICATE store vp<%zext>, ir<%p1> +; CHECK-NEXT: CLONE ir<%idx> = getelementptr ir<%p>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%idx> +; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN-CAST ir<%zext> = zext nneg ir<%l> +; CHECK-NEXT: REPLICATE store ir<%zext>, ir<%p1> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll index 7bf5e6bd76bd3..8c18c107b39fc 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll @@ -19,9 +19,9 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%tmp2> = getelementptr ir<%ptr>, vp<[[STEPS]]> -; CHECK-NEXT: CLONE vp<%tmp3> = load vp<%tmp2> -; CHECK-NEXT: CLONE store ir<0>, vp<%tmp2> +; CHECK-NEXT: CLONE ir<%tmp2> = getelementptr ir<%ptr>, vp<[[STEPS]]> +; CHECK-NEXT: CLONE ir<%tmp3> = load ir<%tmp2> +; CHECK-NEXT: CLONE store ir<0>, ir<%tmp2> ; CHECK-NEXT: Successor(s): pred.store ; CHECK: pred.store: { @@ -30,7 +30,7 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK: pred.store.if: -; CHECK-NEXT: CLONE store vp<%tmp3>, vp<%tmp2> +; CHECK-NEXT: CLONE store ir<%tmp3>, ir<%tmp2> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK: pred.store.continue: diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll index 0e19eb5a23050..06bd2e2147032 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll @@ -27,7 +27,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> +; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> ; CHECK-NEXT: Successor(s): pred.store ; CHECK: pred.store: { @@ -37,16 +37,16 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 ; CHECK: pred.store.if: ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b> -; CHECK-NEXT: REPLICATE vp<%add> = add vp<%lv.b>, ir<10> -; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]] -; CHECK-NEXT: REPLICATE vp<%mul> = mul ir<2>, vp<%add> -; CHECK-NEXT: REPLICATE store vp<%mul>, vp<%gep.a> +; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> +; CHECK-NEXT: REPLICATE ir<%add> = add ir<%lv.b>, ir<10> +; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]] +; CHECK-NEXT: REPLICATE ir<%mul> = mul ir<2>, ir<%add> +; CHECK-NEXT: REPLICATE store ir<%mul>, ir<%gep.a> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -96,7 +96,7 @@ exit: ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> +; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> ; CHECK-NEXT: Successor(s): pred.load ; CHECK: pred.load: { @@ -106,17 +106,17 @@ exit: ; CHECK: pred.load.if: ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b> +; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> ; CHECK-NEXT: Successor(s): pred.load.continue ; CHECK: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK: loop.0: -; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, ir<2> +; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<2> ; CHECK-NEXT: Successor(s): pred.store ; CHECK: pred.store: { @@ -125,9 +125,9 @@ exit: ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK: pred.store.if: -; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul> -; CHECK-NEXT: REPLICATE vp<%add> = add vp<[[PRED]]>, ir<10> -; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep.a> +; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul> +; CHECK-NEXT: REPLICATE ir<%add> = add vp<[[PRED]]>, ir<10> +; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.a> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK: pred.store.continue: @@ -180,7 +180,7 @@ exit: ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> +; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> ; CHECK-NEXT: Successor(s): pred.load ; CHECK: pred.load: { @@ -190,18 +190,18 @@ exit: ; CHECK: pred.load.if: ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b> (S->V) +; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V) ; CHECK-NEXT: Successor(s): pred.load.continue ; CHECK: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK: loop.0: -; CHECK-NEXT: WIDEN vp<%add> = add vp<[[PRED]]>, ir<10> -; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, vp<%add> +; CHECK-NEXT: WIDEN ir<%add> = add vp<[[PRED]]>, ir<10> +; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<%add> ; CHECK-NEXT: Successor(s): pred.store ; CHECK: pred.store: { @@ -210,8 +210,8 @@ exit: ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK: pred.store.if: -; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul> -; CHECK-NEXT: REPLICATE store vp<%add>, vp<%gep.a> +; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul> +; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.a> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK: pred.store.continue: @@ -265,10 +265,10 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) { ; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<21> + vp<[[CAN_IV]]> * ir<1> ; CHECK-NEXT: EMIT vp<[[WIDE_CAN_IV:%.+]]> = WIDEN-CANONICAL-INDUCTION vp<[[CAN_IV]]> ; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<[[WIDE_CAN_IV]]>, vp<[[BTC]]> -; CHECK-NEXT: CLONE vp<%gep.A.uniform> = getelementptr inbounds ir<%A>, ir<0> -; CHECK-NEXT: CLONE vp<%lv> = load vp<%gep.A.uniform> -; CHECK-NEXT: WIDEN vp<%cmp> = icmp ult vp<%iv>, ir<%k> -; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not vp<%cmp> +; CHECK-NEXT: CLONE ir<%gep.A.uniform> = getelementptr inbounds ir<%A>, ir<0> +; CHECK-NEXT: CLONE ir<%lv> = load ir<%gep.A.uniform> +; CHECK-NEXT: WIDEN ir<%cmp> = icmp ult ir<%iv>, ir<%k> +; CHECK-NEXT: EMIT vp<[[NOT2:%.+]]> = not ir<%cmp> ; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK]]>, vp<[[NOT2]]>, ir ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: @@ -279,8 +279,8 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) { ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE vp<%gep.B> = getelementptr inbounds ir<%B>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store vp<%lv>, vp<%gep.B> +; CHECK-NEXT: REPLICATE ir<%gep.B> = getelementptr inbounds ir<%B>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE store ir<%lv>, ir<%gep.B> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: @@ -337,10 +337,10 @@ define void @pred_cfg1(i32 %k, i32 %j) { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> -; CHECK-NEXT: WIDEN vp<%c.1> = icmp ult vp<%iv>, ir<%j> -; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, ir<10> -; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]>, vp<%c.1>, ir +; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> +; CHECK-NEXT: WIDEN ir<%c.1> = icmp ult ir<%iv>, ir<%j> +; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10> +; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]>, ir<%c.1>, ir ; CHECK-NEXT: Successor(s): pred.load ; CHECK-EMPTY: ; CHECK-NEXT: pred.load: { @@ -350,18 +350,18 @@ define void @pred_cfg1(i32 %k, i32 %j) { ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.if: ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b> (S->V) +; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V) ; CHECK-NEXT: Successor(s): pred.load.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): then.0.0 ; CHECK-EMPTY: ; CHECK-NEXT: then.0.0: -; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not vp<%c.1> +; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%c.1> ; CHECK-NEXT: EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]>, vp<[[NOT]]>, ir ; CHECK-NEXT: EMIT vp<[[OR:%.+]]> = or vp<[[MASK2]]>, vp<[[MASK3]]> ; CHECK-NEXT: BLEND ir<%p> = ir<0> vp<[[PRED]]>/vp<[[MASK2]]> @@ -373,8 +373,8 @@ define void @pred_cfg1(i32 %k, i32 %j) { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul> -; CHECK-NEXT: REPLICATE store vp<%p>, vp<%gep.a> +; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul> +; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.a> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: @@ -437,11 +437,11 @@ define void @pred_cfg2(i32 %k, i32 %j) { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> -; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, ir<10> -; CHECK-NEXT: WIDEN vp<%c.0> = icmp ult vp<%iv>, ir<%j> -; CHECK-NEXT: WIDEN vp<%c.1> = icmp ugt vp<%iv>, ir<%j> -; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]>, vp<%c.0>, ir +; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> +; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10> +; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j> +; CHECK-NEXT: WIDEN ir<%c.1> = icmp ugt ir<%iv>, ir<%j> +; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1]]>, ir<%c.0>, ir ; CHECK-NEXT: Successor(s): pred.load ; CHECK-EMPTY: ; CHECK-NEXT: pred.load: { @@ -451,18 +451,18 @@ define void @pred_cfg2(i32 %k, i32 %j) { ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.if: ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b> (S->V) +; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V) ; CHECK-NEXT: Successor(s): pred.load.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): then.0.0 ; CHECK-EMPTY: ; CHECK-NEXT: then.0.0: -; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not vp<%c.0> +; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%c.0> ; CHECK-NEXT: EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]>, vp<[[NOT]]>, ir ; CHECK-NEXT: EMIT vp<[[OR:%.+]]> = or vp<[[MASK2]]>, vp<[[MASK3]]> ; CHECK-NEXT: BLEND ir<%p> = ir<0> vp<[[PRED]]>/vp<[[MASK2]]> @@ -475,8 +475,8 @@ define void @pred_cfg2(i32 %k, i32 %j) { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul> -; CHECK-NEXT: REPLICATE store vp<%p>, vp<%gep.a> +; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul> +; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.a> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: @@ -546,10 +546,10 @@ define void @pred_cfg3(i32 %k, i32 %j) { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> -; CHECK-NEXT: WIDEN vp<%mul> = mul vp<%iv>, ir<10> -; CHECK-NEXT: WIDEN vp<%c.0> = icmp ult vp<%iv>, ir<%j> -; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1:%.+]]>, vp<%c.0>, ir +; CHECK-NEXT: EMIT vp<[[MASK1:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> +; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10> +; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j> +; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK1:%.+]]>, ir<%c.0>, ir ; CHECK-NEXT: Successor(s): pred.load ; CHECK-EMPTY: ; CHECK-NEXT: pred.load: { @@ -559,18 +559,18 @@ define void @pred_cfg3(i32 %k, i32 %j) { ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.if: ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b> +; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> ; CHECK-NEXT: Successor(s): pred.load.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%lv.b> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%lv.b> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): then.0.0 ; CHECK-EMPTY: ; CHECK-NEXT: then.0.0: -; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not vp<%c.0> +; CHECK-NEXT: EMIT vp<[[NOT:%.+]]> = not ir<%c.0> ; CHECK-NEXT: EMIT vp<[[MASK3:%.+]]> = select vp<[[MASK1]]>, vp<[[NOT]]>, ir ; CHECK-NEXT: EMIT vp<[[MASK4:%.+]]> = or vp<[[MASK2]]>, vp<[[MASK3]]> ; CHECK-NEXT: BLEND ir<%p> = ir<0> vp<[[PRED]]>/vp<[[MASK2]]> @@ -583,10 +583,10 @@ define void @pred_cfg3(i32 %k, i32 %j) { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<%mul> -; CHECK-NEXT: REPLICATE store ir<0>, vp<%gep.a> -; CHECK-NEXT: REPLICATE vp<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, vp<%mul> -; CHECK-NEXT: REPLICATE store vp<%p>, vp<%gep.c> +; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, ir<%mul> +; CHECK-NEXT: REPLICATE store ir<0>, ir<%gep.a> +; CHECK-NEXT: REPLICATE ir<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, ir<%mul> +; CHECK-NEXT: REPLICATE store ir<%p>, ir<%gep.c> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: @@ -656,7 +656,7 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) { ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> +; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -665,26 +665,26 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE vp<%lv.a> = load vp<%gep.a> -; CHECK-NEXT: REPLICATE vp<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE vp<%lv.b> = load vp<%gep.b> -; CHECK-NEXT: REPLICATE vp<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store vp<%lv.a>, vp<%gep.c> -; CHECK-NEXT: REPLICATE store vp<%lv.b>, vp<%gep.a> +; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a> +; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> +; CHECK-NEXT: REPLICATE ir<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE store ir<%lv.a>, ir<%gep.c> +; CHECK-NEXT: REPLICATE store ir<%lv.b>, ir<%gep.a> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = vp<%lv.a> -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%lv.b> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%lv.a> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%lv.b> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.3 ; CHECK-EMPTY: ; CHECK-NEXT: loop.3: -; CHECK-NEXT: WIDEN vp<%c.0> = icmp ult vp<%iv>, ir<%j> -; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK]]>, vp<%c.0>, ir -; CHECK-NEXT: WIDEN vp<%mul> = mul vp<[[PRED1]]>, vp<[[PRED2]]> +; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j> +; CHECK-NEXT: EMIT vp<[[MASK2:%.+]]> = select vp<[[MASK]]>, ir<%c.0>, ir +; CHECK-NEXT: WIDEN ir<%mul> = mul vp<[[PRED1]]>, vp<[[PRED2]]> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -693,8 +693,8 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE vp<%gep.c.1> = getelementptr inbounds ir<@c>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store vp<%mul>, vp<%gep.c.1> +; CHECK-NEXT: REPLICATE ir<%gep.c.1> = getelementptr inbounds ir<@c>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE store ir<%mul>, ir<%gep.c.1> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: @@ -760,7 +760,7 @@ define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> +; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -770,15 +770,15 @@ define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) { ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE vp<%lv.a> = load vp<%gep.a> -; CHECK-NEXT: REPLICATE vp<%div> = sdiv vp<%lv.a>, vp<%lv.a> -; CHECK-NEXT: REPLICATE store vp<%div>, vp<%gep.a> +; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a> +; CHECK-NEXT: REPLICATE ir<%div> = sdiv ir<%lv.a>, ir<%lv.a> +; CHECK-NEXT: REPLICATE store ir<%div>, ir<%gep.a> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = vp<%lv.a> -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%div> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED1:%.+]]> = ir<%lv.a> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%div> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.2 @@ -827,10 +827,10 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next, ir<1> -; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI vp<%for> = phi ir<0>, vp<[[PRED:%.+]]> +; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, vp<[[PRED:%.+]]> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule vp<%iv>, vp<[[BTC]]> -; CHECK-NEXT: REPLICATE vp<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]> +; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = icmp ule ir<%iv>, vp<[[BTC]]> +; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[STEPS]]> ; CHECK-NEXT: Successor(s): pred.load ; CHECK-EMPTY: ; CHECK-NEXT: pred.load: { @@ -839,17 +839,17 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) { ; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.if: -; CHECK-NEXT: REPLICATE vp<%lv.a> = load vp<%gep.a> +; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a> ; CHECK-NEXT: Successor(s): pred.load.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.load.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED]]> = vp<%lv.a> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED]]> = ir<%lv.a> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.0 ; CHECK-EMPTY: ; CHECK-NEXT: loop.0: -; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice vp<%for>, vp<[[PRED]]> +; CHECK-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%for>, vp<[[PRED]]> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -858,12 +858,12 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE vp<%div> = sdiv vp<[[SPLICE]]>, vp<[[PRED]]> -; CHECK-NEXT: REPLICATE store vp<%div>, vp<%gep.a> +; CHECK-NEXT: REPLICATE ir<%div> = sdiv vp<[[SPLICE]]>, vp<[[PRED]]> +; CHECK-NEXT: REPLICATE store ir<%div>, ir<%gep.a> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = vp<%div> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED2:%.+]]> = ir<%div> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.2 @@ -915,15 +915,15 @@ define void @update_multiple_users(ptr noalias %src, ptr noalias %dst, i1 %c) { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE vp<%l1> = load ir<%src> -; CHECK-NEXT: REPLICATE vp<%l2> = trunc vp<%l1> -; CHECK-NEXT: REPLICATE vp<%cmp> = icmp eq vp<%l1>, ir<0> -; CHECK-NEXT: REPLICATE vp<%sel> = select vp<%cmp>, ir<5>, vp<%l2> -; CHECK-NEXT: REPLICATE store vp<%sel>, ir<%dst> +; CHECK-NEXT: REPLICATE ir<%l1> = load ir<%src> +; CHECK-NEXT: REPLICATE ir<%l2> = trunc ir<%l1> +; CHECK-NEXT: REPLICATE ir<%cmp> = icmp eq ir<%l1>, ir<0> +; CHECK-NEXT: REPLICATE ir<%sel> = select ir<%cmp>, ir<5>, ir<%l2> +; CHECK-NEXT: REPLICATE store ir<%sel>, ir<%dst> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = vp<%l1> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[PRED:%.+]]> = ir<%l1> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.then.1 @@ -973,11 +973,11 @@ define void @sinking_requires_duplication(ptr %addr) { ; CHECK-NEXT: vector.body: ; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> -; CHECK-NEXT: CLONE vp<%gep.cloned> = getelementptr ir<%addr>, vp<[[STEPS]]> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer vp<%gep.cloned> -; CHECK-NEXT: WIDEN vp<[[L:%.+]]> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN vp<%pred> = fcmp oeq vp<[[L]]>, ir<0.000000e+00> -; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = not vp<%pred> +; CHECK-NEXT: CLONE ir<%gep> = getelementptr ir<%addr>, vp<[[STEPS]]> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep> +; CHECK-NEXT: WIDEN ir<%0> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN ir<%pred> = fcmp oeq ir<%0>, ir<0.000000e+00> +; CHECK-NEXT: EMIT vp<[[MASK:%.+]]> = not ir<%pred> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -986,8 +986,8 @@ define void @sinking_requires_duplication(ptr %addr) { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE vp<%gep> = getelementptr ir<%addr>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store ir<1.000000e+01>, vp<%gep> +; CHECK-NEXT: REPLICATE ir<%gep.1> = getelementptr ir<%addr>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE store ir<1.000000e+01>, ir<%gep.1> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: @@ -1055,14 +1055,14 @@ define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr n ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: ; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1> -; CHECK-NEXT: REPLICATE vp<%gep.src> = getelementptr inbounds ir<%src>, vp<[[SCALAR_STEPS]]> -; CHECK-NEXT: REPLICATE vp<%l> = load vp<%gep.src> -; CHECK-NEXT: REPLICATE vp<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[SCALAR_STEPS]]> -; CHECK-NEXT: REPLICATE store vp<%l>, vp<%gep.dst> +; CHECK-NEXT: REPLICATE ir<%gep.src> = getelementptr inbounds ir<%src>, vp<[[SCALAR_STEPS]]> +; CHECK-NEXT: REPLICATE ir<%l> = load ir<%gep.src> +; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[SCALAR_STEPS]]> +; CHECK-NEXT: REPLICATE store ir<%l>, ir<%gep.dst> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: -; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_LOAD:%.+]]> = vp<%l> +; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[P_LOAD:%.+]]> = ir<%l> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; CHECK-NEXT: Successor(s): loop.1 @@ -1116,11 +1116,11 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) { ; CHECK-NEXT: vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<0> + vp<%3> * ir<-1> ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1> ; CHECK-NEXT: EMIT vp<[[PTR_IV:%.+]]> = ptradd ir<%start>, vp<[[STEPS]]> -; CHECK-NEXT: CLONE vp<%ptr.iv.next.cloned> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1> -; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) vp<%ptr.iv.next.cloned> -; CHECK-NEXT: WIDEN vp<%l> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN vp<%c.1> = icmp eq vp<%l>, ir<0> -; CHECK-NEXT: EMIT vp<[[NEG:%.+]]> = not vp<%c.1> +; CHECK-NEXT: CLONE ir<%ptr.iv.next> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1> +; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer (reverse) ir<%ptr.iv.next> +; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]> +; CHECK-NEXT: WIDEN ir<%c.1> = icmp eq ir<%l>, ir<0> +; CHECK-NEXT: EMIT vp<[[NEG:%.+]]> = not ir<%c.1> ; CHECK-NEXT: Successor(s): pred.store ; CHECK-EMPTY: ; CHECK-NEXT: pred.store: { @@ -1129,8 +1129,8 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE vp<%ptr.iv.next> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1> -; CHECK-NEXT: REPLICATE store ir<95>, vp<%ptr.iv.next> +; CHECK-NEXT: REPLICATE ir<%ptr.iv.next.1> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1> +; CHECK-NEXT: REPLICATE store ir<95>, ir<%ptr.iv.next.1> ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: diff --git a/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll b/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll index 27d81de260d3b..5ea27994b356d 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll @@ -18,9 +18,9 @@ define void @test_unused_interleave(ptr %src, i32 %length) { ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%index.next> -; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%2>, vp<%0> -; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<%1> +; CHECK-NEXT: EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%3> +; CHECK-NEXT: EMIT vp<%3> = add nuw vp<%2>, vp<%0> +; CHECK-NEXT: EMIT branch-on-count vp<%3>, vp<%1> ; CHECK-NEXT: No successors ; CHECK-NEXT: } entry: From 55da2ab24b500f4e79a54514c0c212f5cd8eb938 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 12 Apr 2024 13:34:10 +0200 Subject: [PATCH 08/11] !fixup address latest comments, thanks! --- llvm/lib/Transforms/Vectorize/VPlan.cpp | 69 +++++++++++-------- llvm/lib/Transforms/Vectorize/VPlanValue.h | 34 +++++---- .../vplan-sink-scalars-and-merge.ll | 8 +-- 3 files changed, 60 insertions(+), 51 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index c4e7b4d12d325..d3c9dbb088941 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1363,15 +1363,17 @@ VPInterleavedAccessInfo::VPInterleavedAccessInfo(VPlan &Plan, } void VPSlotTracker::assignSlotOrName(const VPValue *V) { - if (auto *UV = V->getUnderlyingValue()) { - std::string Name; - raw_string_ostream S(Name); - UV->printAsOperand(S, false); - deduplicateName(V, Name); + auto *UV = V->getUnderlyingValue(); + if (!UV) { + assert(!VPValue2Name.contains(V) && "VPValue already has a slot!"); + VPValue2Name[V] = (Twine("vp<%") + Twine(NextSlot) + ">").str(); + NextSlot++; return; } - assert(!Slots.contains(V) && "VPValue already has a slot!"); - Slots[V] = NextSlot++; + std::string Name; + raw_string_ostream S(Name); + UV->printAsOperand(S, false); + versionName(V, Name); } void VPSlotTracker::assignSlotsOrNames(const VPlan &Plan) { @@ -1397,38 +1399,47 @@ void VPSlotTracker::assignSlotsOrNames(const VPBasicBlock *VPBB) { assignSlotOrName(Def); } -void VPSlotTracker::deduplicateName(const VPValue *V, StringRef Name) { - assert(!Name.empty() && "Name cannot be be empty."); - std::string NewName = Name.str(); - const auto &[A, AssignedInserted] = AssignedNames.insert({V, NewName}); - if (!AssignedInserted || V->isLiveIn()) +void VPSlotTracker::versionName(const VPValue *V, StringRef Name) { + assert(!Name.empty() && "Name cannot be empty."); + std::string BaseName = (Twine("ir<") + Name + Twine(">")).str(); + + // First assign the base name for V. + const auto &[A, AssignedInserted] = VPValue2Name.insert({V, BaseName}); + assert(AssignedInserted && "name assigned already?"); + if (V->isLiveIn()) return; - const auto &[C, UseInserted] = NameUseCount.insert({NewName, 0}); + // If it is already used by C > 0 other VPValues, increase the version counter + // C and use it for V. + const auto &[C, UseInserted] = BaseName2Version.insert({BaseName, 0}); if (!UseInserted) { C->second++; - NewName = NewName + "." + std::to_string(C->second); - A->second = NewName; + A->second = (BaseName + Twine(".") + Twine(C->second)).str(); } } std::string VPSlotTracker::getName(const VPValue *V) const { - std::string Name = AssignedNames.lookup(V); - if (!Name.empty()) { - assert( - V->getUnderlyingValue() && - "Can only have assigned names for VPValues with an underlying value"); - return (Twine("ir<") + Name + ">").str(); + std::string Name = VPValue2Name.lookup(V); + if (!Name.empty()) + return Name; + + // If no name was assigned, no VPlan was provided when creating the slot + // tracker or it is not reachable from the provided VPlan. This can happen, + // e.g. when trying to print a recipe that has not been inserted into a VPlan + // in a debugger. + const VPRecipeBase *DefR = V->getDefiningRecipe(); + assert((!DefR || !DefR->getParent() || !DefR->getParent()->getPlan()) && + "VPValue defined by a recipe in a VPlan?"); + + // Use the underlying value's name, if there is one. + if (auto *UV = V->getUnderlyingValue()) { + std::string Name; + raw_string_ostream S(Name); + UV->printAsOperand(S, false); + return (Twine("ir<%") + Name + ">").str(); } - assert( - !V->getUnderlyingValue() && - "Must not have assigned names for VPValues without an underlying value"); - unsigned Slot = getSlot(V); - if (Slot == unsigned(-1)) - return ""; - else - return (Twine("vp<%") + std::to_string(Slot) + ">").str(); + return ""; } bool vputils::onlyFirstLaneUsed(const VPValue *Def) { diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h index 15094c6a60004..cc4bb7bc10fd1 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -444,27 +444,29 @@ class VPDef { class VPlan; class VPBasicBlock; -/// This class can be used to assign consecutive numbers to VPValues in a -/// VPlan without underlying values and deduplicated names to VPValues with -/// underlying values. Allows querying the numbering and deduplicated names for -/// printing, similar to the ModuleSlotTracker for IR values. +/// This class can be used to assign names to VPValues. For VPValues without +/// underlying value, assign consecutive numbers and use those as names (wrapped +/// in vp<>). Otherwise, use the name from the underlying value (wrapped in +/// ir<>), apending a .V version number if there are multiple uses of the same +/// name. Allows querying names for VPValues for printing, similar to the +/// ModuleSlotTracker for IR values. class VPSlotTracker { - /// Keep track of de-duplicated names assigned to VPValues with underlying IR + /// Keep track of versioned names assigned to VPValues with underlying IR /// values - DenseMap AssignedNames; + DenseMap VPValue2Name; /// Keep track of the next number to use to deduplicate the base name. - StringMap NameUseCount; + StringMap BaseName2Version; - DenseMap Slots; + /// Number to assign to the next VPValue without underlying value. unsigned NextSlot = 0; void assignSlotOrName(const VPValue *V); void assignSlotsOrNames(const VPlan &Plan); void assignSlotsOrNames(const VPBasicBlock *VPBB); - /// Create a deduplicated version of \p Name for \p V by appending ".Number" + /// Create a versioned variant of \p Name for \p V by appending ".Number" /// to \p Name if there are multiple uses of that name. - void deduplicateName(const VPValue *V, StringRef Name); + void versionName(const VPValue *V, StringRef Name); public: VPSlotTracker(const VPlan *Plan = nullptr) { @@ -472,14 +474,10 @@ class VPSlotTracker { assignSlotsOrNames(*Plan); } - unsigned getSlot(const VPValue *V) const { - auto I = Slots.find(V); - if (I == Slots.end()) - return -1; - return I->second; - } - - std::string getName(const VPValue *) const; + /// Returns the name assigned to \p V, if there is one, otherwise try to + /// construct one from the underlying value, if there's one; else return + /// . + std::string getName(const VPValue *V) const; }; } // namespace llvm diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll index 06bd2e2147032..108b78a70fa1a 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll @@ -986,8 +986,8 @@ define void @sinking_requires_duplication(ptr %addr) { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%gep.1> = getelementptr ir<%addr>, vp<[[STEPS]]> -; CHECK-NEXT: REPLICATE store ir<1.000000e+01>, ir<%gep.1> +; CHECK-NEXT: REPLICATE ir<%gep>.1 = getelementptr ir<%addr>, vp<[[STEPS]]> +; CHECK-NEXT: REPLICATE store ir<1.000000e+01>, ir<%gep>.1 ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: @@ -1129,8 +1129,8 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) { ; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.if: -; CHECK-NEXT: REPLICATE ir<%ptr.iv.next.1> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1> -; CHECK-NEXT: REPLICATE store ir<95>, ir<%ptr.iv.next.1> +; CHECK-NEXT: REPLICATE ir<%ptr.iv.next>.1 = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1> +; CHECK-NEXT: REPLICATE store ir<95>, ir<%ptr.iv.next>.1 ; CHECK-NEXT: Successor(s): pred.store.continue ; CHECK-EMPTY: ; CHECK-NEXT: pred.store.continue: From a407b20f62cba0392f5798ec66ff084a8f9ea9b7 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sat, 13 Apr 2024 08:55:55 +0100 Subject: [PATCH 09/11] !fixup address comments, thanks! --- llvm/lib/Transforms/Vectorize/VPlan.cpp | 69 +++++++++++----------- llvm/lib/Transforms/Vectorize/VPlanValue.h | 14 ++--- 2 files changed, 41 insertions(+), 42 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index d3c9dbb088941..3f49e0489a9d8 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1300,7 +1300,7 @@ void VPValue::replaceUsesWithIf( #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void VPValue::printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const { - OS << Tracker.getName(this); + OS << Tracker.getOrCreateName(this); } void VPUser::printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const { @@ -1362,63 +1362,63 @@ VPInterleavedAccessInfo::VPInterleavedAccessInfo(VPlan &Plan, visitRegion(Plan.getVectorLoopRegion(), Old2New, IAI); } -void VPSlotTracker::assignSlotOrName(const VPValue *V) { +void VPSlotTracker::assignName(const VPValue *V) { + assert(!VPValue2Name.contains(V) && "VPValue already has a name!"); auto *UV = V->getUnderlyingValue(); if (!UV) { - assert(!VPValue2Name.contains(V) && "VPValue already has a slot!"); VPValue2Name[V] = (Twine("vp<%") + Twine(NextSlot) + ">").str(); NextSlot++; return; } + + // Use the name of the underlying Value, wrapped in "ir<>", and versioned by + // appending ".Number" to the name if there are multiple uses. std::string Name; raw_string_ostream S(Name); UV->printAsOperand(S, false); - versionName(V, Name); + assert(!Name.empty() && "Name cannot be empty."); + std::string BaseName = (Twine("ir<") + Name + Twine(">")).str(); + + // First assign the base name for V. + const auto &[A, _] = VPValue2Name.insert({V, BaseName}); + // Integer or FP constants with different types will result in he same string + // due to stripping types. + if (V->isLiveIn() && isa(UV)) + return; + + // If it is already used by C > 0 other VPValues, increase the version counter + // C and use it for V. + const auto &[C, UseInserted] = BaseName2Version.insert({BaseName, 0}); + if (!UseInserted) { + C->second++; + A->second = (BaseName + Twine(".") + Twine(C->second)).str(); + } } -void VPSlotTracker::assignSlotsOrNames(const VPlan &Plan) { +void VPSlotTracker::assignNames(const VPlan &Plan) { if (Plan.VFxUF.getNumUsers() > 0) - assignSlotOrName(&Plan.VFxUF); - assignSlotOrName(&Plan.VectorTripCount); + assignName(&Plan.VFxUF); + assignName(&Plan.VectorTripCount); if (Plan.BackedgeTakenCount) - assignSlotOrName(Plan.BackedgeTakenCount); + assignName(Plan.BackedgeTakenCount); for (VPValue *LI : Plan.VPLiveInsToFree) - assignSlotOrName(LI); - assignSlotsOrNames(Plan.getPreheader()); + assignName(LI); + assignNames(Plan.getPreheader()); ReversePostOrderTraversal> RPOT(VPBlockDeepTraversalWrapper(Plan.getEntry())); for (const VPBasicBlock *VPBB : VPBlockUtils::blocksOnly(RPOT)) - assignSlotsOrNames(VPBB); + assignNames(VPBB); } -void VPSlotTracker::assignSlotsOrNames(const VPBasicBlock *VPBB) { +void VPSlotTracker::assignNames(const VPBasicBlock *VPBB) { for (const VPRecipeBase &Recipe : *VPBB) for (VPValue *Def : Recipe.definedValues()) - assignSlotOrName(Def); -} - -void VPSlotTracker::versionName(const VPValue *V, StringRef Name) { - assert(!Name.empty() && "Name cannot be empty."); - std::string BaseName = (Twine("ir<") + Name + Twine(">")).str(); - - // First assign the base name for V. - const auto &[A, AssignedInserted] = VPValue2Name.insert({V, BaseName}); - assert(AssignedInserted && "name assigned already?"); - if (V->isLiveIn()) - return; - - // If it is already used by C > 0 other VPValues, increase the version counter - // C and use it for V. - const auto &[C, UseInserted] = BaseName2Version.insert({BaseName, 0}); - if (!UseInserted) { - C->second++; - A->second = (BaseName + Twine(".") + Twine(C->second)).str(); - } + assignName(Def); } -std::string VPSlotTracker::getName(const VPValue *V) const { +std::string VPSlotTracker::getOrCreateName(const VPValue *V) const { std::string Name = VPValue2Name.lookup(V); if (!Name.empty()) return Name; @@ -1427,6 +1427,9 @@ std::string VPSlotTracker::getName(const VPValue *V) const { // tracker or it is not reachable from the provided VPlan. This can happen, // e.g. when trying to print a recipe that has not been inserted into a VPlan // in a debugger. + // TODO: Update VPSlotTracker constructor to assign names to recipes & + // VPValues not associated with a VPlan, instead of constructing names ad-hoc + // here. const VPRecipeBase *DefR = V->getDefiningRecipe(); assert((!DefR || !DefR->getParent() || !DefR->getParent()->getPlan()) && "VPValue defined by a recipe in a VPlan?"); diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h index cc4bb7bc10fd1..51c3d5f2324ed 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -460,24 +460,20 @@ class VPSlotTracker { /// Number to assign to the next VPValue without underlying value. unsigned NextSlot = 0; - void assignSlotOrName(const VPValue *V); - void assignSlotsOrNames(const VPlan &Plan); - void assignSlotsOrNames(const VPBasicBlock *VPBB); - - /// Create a versioned variant of \p Name for \p V by appending ".Number" - /// to \p Name if there are multiple uses of that name. - void versionName(const VPValue *V, StringRef Name); + void assignName(const VPValue *V); + void assignNames(const VPlan &Plan); + void assignNames(const VPBasicBlock *VPBB); public: VPSlotTracker(const VPlan *Plan = nullptr) { if (Plan) - assignSlotsOrNames(*Plan); + assignNames(*Plan); } /// Returns the name assigned to \p V, if there is one, otherwise try to /// construct one from the underlying value, if there's one; else return /// . - std::string getName(const VPValue *V) const; + std::string getOrCreateName(const VPValue *V) const; }; } // namespace llvm From 399527672d60225d75fe22111608dead7e1a3df8 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 15 Apr 2024 10:58:17 +0100 Subject: [PATCH 10/11] !fixup address comments, thanks! --- llvm/lib/Transforms/Vectorize/VPlan.cpp | 1 + llvm/lib/Transforms/Vectorize/VPlanValue.h | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 3f49e0489a9d8..86a296bc05a27 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1431,6 +1431,7 @@ std::string VPSlotTracker::getOrCreateName(const VPValue *V) const { // VPValues not associated with a VPlan, instead of constructing names ad-hoc // here. const VPRecipeBase *DefR = V->getDefiningRecipe(); + (void)DefR; assert((!DefR || !DefR->getParent() || !DefR->getParent()->getPlan()) && "VPValue defined by a recipe in a VPlan?"); diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h index 51c3d5f2324ed..da3a768552fc5 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -447,14 +447,14 @@ class VPBasicBlock; /// This class can be used to assign names to VPValues. For VPValues without /// underlying value, assign consecutive numbers and use those as names (wrapped /// in vp<>). Otherwise, use the name from the underlying value (wrapped in -/// ir<>), apending a .V version number if there are multiple uses of the same -/// name. Allows querying names for VPValues for printing, similar to the +/// ir<>), appending a .V version number if there are multiple uses of the same +/// name. Allows querying names for VPValues for printing, similar to the /// ModuleSlotTracker for IR values. class VPSlotTracker { /// Keep track of versioned names assigned to VPValues with underlying IR - /// values + /// values. DenseMap VPValue2Name; - /// Keep track of the next number to use to deduplicate the base name. + /// Keep track of the next number to use to version the base name. StringMap BaseName2Version; /// Number to assign to the next VPValue without underlying value. From 4496d93597b416203bc2b9e0b4ab58c2a65646d3 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 15 Apr 2024 11:37:32 +0100 Subject: [PATCH 11/11] !fixup remove extra `%` from printing underlying values --- llvm/lib/Transforms/Vectorize/VPlan.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 86a296bc05a27..999236ae84898 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1440,7 +1440,7 @@ std::string VPSlotTracker::getOrCreateName(const VPValue *V) const { std::string Name; raw_string_ostream S(Name); UV->printAsOperand(S, false); - return (Twine("ir<%") + Name + ">").str(); + return (Twine("ir<") + Name + ">").str(); } return "";