From 8ae59c78e251e27df6b6d8fcd479886f3b877b02 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sun, 29 Sep 2024 20:55:05 +0100 Subject: [PATCH 1/5] [VPlan] Introduce VPWidenIntrinsicRecipe to separate from libcall. This patch splits off intrinsic hanlding to a new VPWidenIntrinsicRecipe. VPWidenIntrinsicRecipes only need access to the intrinsic ID to widen and the scalar result type (in case the intrinsic is overloaded on the result type). It does not need access to an underlying IR call instruction or function. This means VPWidenIntrinsicRecipe can be created easily without access to underlying IR. --- .../Transforms/Vectorize/LoopVectorize.cpp | 33 ++-- .../Transforms/Vectorize/VPRecipeBuilder.h | 6 +- llvm/lib/Transforms/Vectorize/VPlan.h | 69 ++++++-- .../Transforms/Vectorize/VPlanAnalysis.cpp | 2 + .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 160 +++++++++++------- .../Transforms/Vectorize/VPlanTransforms.cpp | 7 +- llvm/lib/Transforms/Vectorize/VPlanValue.h | 1 + .../LoopVectorize/AArch64/scalable-call.ll | 8 +- .../widen-call-with-intrinsic-or-libfunc.ll | 2 +- .../LoopVectorize/vplan-dot-printing.ll | 2 +- .../LoopVectorize/vplan-printing.ll | 2 +- .../Transforms/Vectorize/VPlanTest.cpp | 6 +- 12 files changed, 197 insertions(+), 101 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index f5ef50934f59f..99a3595539f1d 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -4367,7 +4367,7 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks( [](const auto *R) { return Instruction::Store; }) .Case( [](const auto *R) { return Instruction::Load; }) - .Case( + .Case( [](const auto *R) { return Instruction::Call; }) .Case( @@ -4392,11 +4392,17 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks( OS << "):"; if (Opcode == Instruction::Call) { auto *WidenCall = dyn_cast(R); - Function *CalledFn = - WidenCall ? WidenCall->getCalledScalarFunction() - : cast(R->getOperand(R->getNumOperands() - 1) - ->getLiveInIRValue()); - OS << " call to " << CalledFn->getName(); + StringRef Name = ""; + if (auto *Int = dyn_cast(R)) { + Name = Int->getIntrinsicName(); + } else { + Function *CalledFn = + WidenCall ? WidenCall->getCalledScalarFunction() + : cast(R->getOperand(R->getNumOperands() - 1) + ->getLiveInIRValue()); + Name = CalledFn->getName(); + } + OS << " call to " << Name; } else OS << " " << Instruction::getOpcodeName(Opcode); reportVectorizationInfo(OutString, "InvalidCost", ORE, OrigLoop, nullptr, @@ -4447,6 +4453,7 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF, case VPDef::VPWidenCanonicalIVSC: case VPDef::VPWidenCastSC: case VPDef::VPWidenGEPSC: + case VPDef::VPWidenIntrinsicSC: case VPDef::VPWidenSC: case VPDef::VPWidenSelectSC: case VPDef::VPBlendSC: @@ -8266,7 +8273,7 @@ VPBlendRecipe *VPRecipeBuilder::tryToBlend(PHINode *Phi, return new VPBlendRecipe(Phi, OperandsWithMask); } -VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI, +VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI, ArrayRef Operands, VFRange &Range) { bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange( @@ -8297,8 +8304,9 @@ VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI, }, Range); if (ShouldUseVectorIntrinsic) - return new VPWidenCallRecipe(CI, make_range(Ops.begin(), Ops.end()), ID, - CI->getDebugLoc()); + return new VPWidenIntrinsicRecipe(*CI, ID, + make_range(Ops.begin(), Ops.end() - 1), + CI->getType(), CI->getDebugLoc()); Function *Variant = nullptr; std::optional MaskPos; @@ -8350,9 +8358,8 @@ VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI, Ops.insert(Ops.begin() + *MaskPos, Mask); } - return new VPWidenCallRecipe(CI, make_range(Ops.begin(), Ops.end()), - Intrinsic::not_intrinsic, CI->getDebugLoc(), - Variant); + return new VPWidenCallRecipe( + CI, Variant, make_range(Ops.begin(), Ops.end()), CI->getDebugLoc()); } return nullptr; @@ -9218,7 +9225,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( RecurrenceDescriptor::isFMulAddIntrinsic(CurrentLinkI) && "Expected instruction to be a call to the llvm.fmuladd intrinsic"); assert(((MinVF.isScalar() && isa(CurrentLink)) || - isa(CurrentLink)) && + isa(CurrentLink)) && CurrentLink->getOperand(2) == PreviousLink && "expected a call where the previous link is the added operand"); diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h index 02ef6f7b6fb98..5d4a3b555981c 100644 --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -93,9 +93,9 @@ class VPRecipeBuilder { VPBlendRecipe *tryToBlend(PHINode *Phi, ArrayRef Operands); /// Handle call instructions. If \p CI can be widened for \p Range.Start, - /// return a new VPWidenCallRecipe. Range.End may be decreased to ensure same - /// decision from \p Range.Start to \p Range.End. - VPWidenCallRecipe *tryToWidenCall(CallInst *CI, ArrayRef Operands, + /// return a new VPWidenCallRecipe or VPWidenIntrinsicRecipe. Range.End may be + /// decreased to ensure same decision from \p Range.Start to \p Range.End. + VPSingleDefRecipe *tryToWidenCall(CallInst *CI, ArrayRef Operands, VFRange &Range); /// Check if \p I has an opcode that can be widened and return a VPWidenRecipe diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 594492344d43c..8d62970d7d824 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -886,6 +886,7 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPValue { case VPRecipeBase::VPWidenCanonicalIVSC: case VPRecipeBase::VPWidenCastSC: case VPRecipeBase::VPWidenGEPSC: + case VPRecipeBase::VPWidenIntrinsicSC: case VPRecipeBase::VPWidenSC: case VPRecipeBase::VPWidenEVLSC: case VPRecipeBase::VPWidenSelectSC: @@ -1608,25 +1609,65 @@ class VPScalarCastRecipe : public VPSingleDefRecipe { } }; -/// A recipe for widening Call instructions. -class VPWidenCallRecipe : public VPRecipeWithIRFlags { - /// ID of the vector intrinsic to call when widening the call. If set the - /// Intrinsic::not_intrinsic, a library call will be used instead. +/// A recipe for widening vector intrinsics. +class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags { + /// ID of the vector intrinsic to widen. Intrinsic::ID VectorIntrinsicID; - /// If this recipe represents a library call, Variant stores a pointer to - /// the chosen function. There is a 1:1 mapping between a given VF and the - /// chosen vectorized variant, so there will be a different vplan for each - /// VF with a valid variant. + + /// Scalar type of the result produced by the intrinsic. + Type *ResultTy; + +public: + template + VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID, + iterator_range CallArguments, Type *Ty, + DebugLoc DL = {}) + : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI), + VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty) {} + + ~VPWidenIntrinsicRecipe() override = default; + + VPWidenIntrinsicRecipe *clone() override { + return new VPWidenIntrinsicRecipe(*cast(getUnderlyingValue()), + VectorIntrinsicID, operands(), ResultTy, + getDebugLoc()); + } + + VP_CLASSOF_IMPL(VPDef::VPWidenIntrinsicSC) + + /// Produce a widened version of the vector intrinsic. + void execute(VPTransformState &State) override; + + /// Return the cost of this vector intrinsic. + InstructionCost computeCost(ElementCount VF, + VPCostContext &Ctx) const override; + + Type *getResultTy() const { return ResultTy; } + + /// Return to name of the intrinsic as string. + StringRef getIntrinsicName() const; + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + /// Print the recipe. + void print(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const override; +#endif +}; + +/// A recipe for widening Call instructions using library calls. +class VPWidenCallRecipe : public VPRecipeWithIRFlags { + /// Variant stores a pointer to the chosen function. There is a 1:1 mapping + /// between a given VF and the chosen vectorized variant, so there will be a + /// different VPlan for each VF with a valid variant. Function *Variant; public: template - VPWidenCallRecipe(Value *UV, iterator_range CallArguments, - Intrinsic::ID VectorIntrinsicID, DebugLoc DL = {}, - Function *Variant = nullptr) + VPWidenCallRecipe(Value *UV, Function *Variant, + iterator_range CallArguments, DebugLoc DL = {}) : VPRecipeWithIRFlags(VPDef::VPWidenCallSC, CallArguments, *cast(UV)), - VectorIntrinsicID(VectorIntrinsicID), Variant(Variant) { + Variant(Variant) { assert( isa(getOperand(getNumOperands() - 1)->getLiveInIRValue()) && "last operand must be the called function"); @@ -1635,8 +1676,8 @@ class VPWidenCallRecipe : public VPRecipeWithIRFlags { ~VPWidenCallRecipe() override = default; VPWidenCallRecipe *clone() override { - return new VPWidenCallRecipe(getUnderlyingValue(), operands(), - VectorIntrinsicID, getDebugLoc(), Variant); + return new VPWidenCallRecipe(getUnderlyingValue(), Variant, operands(), + getDebugLoc()); } VP_CLASSOF_IMPL(VPDef::VPWidenCallSC) diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp index 277df0637372d..24f34e43a6e84 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp @@ -268,6 +268,8 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) { VPReplicateRecipe, VPWidenCallRecipe, VPWidenMemoryRecipe, VPWidenSelectRecipe>( [this](const auto *R) { return inferScalarTypeForRecipe(R); }) + .Case( + [](const VPWidenIntrinsicRecipe *R) { return R->getResultTy(); }) .Case([V](const VPInterleaveRecipe *R) { // TODO: Use info from interleave group. return V->getUnderlyingValue()->getType(); diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 0d092b9c10acc..03f6b7d442fb0 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -79,6 +79,8 @@ bool VPRecipeBase::mayWriteToMemory() const { return !cast(this) ->getCalledScalarFunction() ->onlyReadsMemory(); + case VPWidenIntrinsicSC: + return false; case VPBranchOnMaskSC: case VPScalarIVStepsSC: case VPPredInstPHISC: @@ -120,6 +122,8 @@ bool VPRecipeBase::mayReadFromMemory() const { return !cast(this) ->getCalledScalarFunction() ->onlyWritesMemory(); + case VPWidenIntrinsicSC: + return true; case VPBranchOnMaskSC: case VPPredInstPHISC: case VPScalarIVStepsSC: @@ -161,6 +165,8 @@ bool VPRecipeBase::mayHaveSideEffects() const { Function *Fn = cast(this)->getCalledScalarFunction(); return mayWriteToMemory() || !Fn->doesNotThrow() || !Fn->willReturn(); } + case VPWidenIntrinsicSC: + return false; case VPBlendSC: case VPReductionEVLSC: case VPReductionSC: @@ -879,56 +885,105 @@ void VPIRInstruction::print(raw_ostream &O, const Twine &Indent, void VPWidenCallRecipe::execute(VPTransformState &State) { assert(State.VF.isVector() && "not widening"); - Function *CalledScalarFn = getCalledScalarFunction(); - assert(!isDbgInfoIntrinsic(CalledScalarFn->getIntrinsicID()) && - "DbgInfoIntrinsic should have been dropped during VPlan construction"); State.setDebugLocFrom(getDebugLoc()); - bool UseIntrinsic = VectorIntrinsicID != Intrinsic::not_intrinsic; - FunctionType *VFTy = nullptr; - if (Variant) - VFTy = Variant->getFunctionType(); - SmallVector TysForDecl; + FunctionType *VFTy = Variant->getFunctionType(); // Add return type if intrinsic is overloaded on it. - if (UseIntrinsic && - isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, -1)) - TysForDecl.push_back(VectorType::get( - CalledScalarFn->getReturnType()->getScalarType(), State.VF)); SmallVector Args; for (const auto &I : enumerate(arg_operands())) { - // Some intrinsics have a scalar argument - don't replace it with a - // vector. Value *Arg; - if (UseIntrinsic && - isVectorIntrinsicWithScalarOpAtArg(VectorIntrinsicID, I.index())) - Arg = State.get(I.value(), VPLane(0)); // Some vectorized function variants may also take a scalar argument, // e.g. linear parameters for pointers. This needs to be the scalar value // from the start of the respective part when interleaving. - else if (VFTy && !VFTy->getParamType(I.index())->isVectorTy()) + if (!VFTy->getParamType(I.index())->isVectorTy()) Arg = State.get(I.value(), VPLane(0)); else - Arg = State.get(I.value()); - if (UseIntrinsic && - isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, I.index())) - TysForDecl.push_back(Arg->getType()); + Arg = State.get(I.value(), onlyFirstLaneUsed(I.value())); Args.push_back(Arg); } - Function *VectorF; - if (UseIntrinsic) { - // Use vector version of the intrinsic. - Module *M = State.Builder.GetInsertBlock()->getModule(); - VectorF = Intrinsic::getDeclaration(M, VectorIntrinsicID, TysForDecl); - assert(VectorF && "Can't retrieve vector intrinsic."); - } else { -#ifndef NDEBUG - assert(Variant != nullptr && "Can't create vector function."); + assert(Variant != nullptr && "Can't create vector function."); + + auto *CI = cast_or_null(getUnderlyingValue()); + SmallVector OpBundles; + if (CI) + CI->getOperandBundlesAsDefs(OpBundles); + + CallInst *V = State.Builder.CreateCall(Variant, Args, OpBundles); + + if (isa(V)) + V->copyFastMathFlags(CI); + + if (!V->getType()->isVoidTy()) + State.set(this, V); + State.addMetadata(V, CI); +} + +InstructionCost VPWidenCallRecipe::computeCost(ElementCount VF, + VPCostContext &Ctx) const { + TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; + return Ctx.TTI.getCallInstrCost(nullptr, Variant->getReturnType(), + Variant->getFunctionType()->params(), + CostKind); +} + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +void VPWidenCallRecipe::print(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const { + O << Indent << "WIDEN-CALL "; + + Function *CalledFn = getCalledScalarFunction(); + if (CalledFn->getReturnType()->isVoidTy()) + O << "void "; + else { + printAsOperand(O, SlotTracker); + O << " = "; + } + + O << "call"; + printFlags(O); + O << " @" << CalledFn->getName() << "("; + interleaveComma(arg_operands(), O, [&O, &SlotTracker](VPValue *Op) { + Op->printAsOperand(O, SlotTracker); + }); + O << ")"; + + O << " (using library function"; + if (Variant->hasName()) + O << ": " << Variant->getName(); + O << ")"; +} #endif - VectorF = Variant; + +void VPWidenIntrinsicRecipe::execute(VPTransformState &State) { + assert(State.VF.isVector() && "not widening"); + State.setDebugLocFrom(getDebugLoc()); + + SmallVector TysForDecl; + // Add return type if intrinsic is overloaded on it. + if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, -1)) + TysForDecl.push_back(VectorType::get(getResultTy(), State.VF)); + SmallVector Args; + for (const auto &I : enumerate(operands())) { + // Some intrinsics have a scalar argument - don't replace it with a + // vector. + Value *Arg; + if (isVectorIntrinsicWithScalarOpAtArg(VectorIntrinsicID, I.index())) + Arg = State.get(I.value(), VPLane(0)); + else + Arg = State.get(I.value(), onlyFirstLaneUsed(I.value())); + if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, I.index())) + TysForDecl.push_back(Arg->getType()); + Args.push_back(Arg); } - auto *CI = cast_or_null(getUnderlyingInstr()); + // Use vector version of the intrinsic. + Module *M = State.Builder.GetInsertBlock()->getModule(); + Function *VectorF = + Intrinsic::getDeclaration(M, VectorIntrinsicID, TysForDecl); + assert(VectorF && "Can't retrieve vector intrinsic."); + + auto *CI = cast_or_null(getUnderlyingValue()); SmallVector OpBundles; if (CI) CI->getOperandBundlesAsDefs(OpBundles); @@ -942,14 +997,9 @@ void VPWidenCallRecipe::execute(VPTransformState &State) { State.addMetadata(V, CI); } -InstructionCost VPWidenCallRecipe::computeCost(ElementCount VF, - VPCostContext &Ctx) const { +InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF, + VPCostContext &Ctx) const { TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; - if (Variant) { - return Ctx.TTI.getCallInstrCost(nullptr, Variant->getReturnType(), - Variant->getFunctionType()->params(), - CostKind); - } // Some backends analyze intrinsic arguments to determine cost. Use the // underlying value for the operand if it has one. Otherwise try to use the @@ -985,35 +1035,29 @@ InstructionCost VPWidenCallRecipe::computeCost(ElementCount VF, return Ctx.TTI.getIntrinsicInstrCost(CostAttrs, CostKind); } -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -void VPWidenCallRecipe::print(raw_ostream &O, const Twine &Indent, - VPSlotTracker &SlotTracker) const { - O << Indent << "WIDEN-CALL "; +StringRef VPWidenIntrinsicRecipe::getIntrinsicName() const { + return Intrinsic::getBaseName(VectorIntrinsicID); +} - Function *CalledFn = getCalledScalarFunction(); - if (CalledFn->getReturnType()->isVoidTy()) +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +void VPWidenIntrinsicRecipe::print(raw_ostream &O, const Twine &Indent, + VPSlotTracker &SlotTracker) const { + O << Indent << "WIDEN-INTRINSIC "; + if (ResultTy->isVoidTy()) { O << "void "; - else { + } else { printAsOperand(O, SlotTracker); O << " = "; } O << "call"; printFlags(O); - O << " @" << CalledFn->getName() << "("; - interleaveComma(arg_operands(), O, [&O, &SlotTracker](VPValue *Op) { + O << getIntrinsicName() << "("; + + interleaveComma(operands(), O, [&O, &SlotTracker](VPValue *Op) { Op->printAsOperand(O, SlotTracker); }); O << ")"; - - if (VectorIntrinsicID) - O << " (using vector intrinsic)"; - else { - O << " (using library function"; - if (Variant->hasName()) - O << ": " << Variant->getName(); - O << ")"; - } } #endif diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index a878613c4ba48..7da358e7e73f2 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -82,9 +82,10 @@ void VPlanTransforms::VPInstructionsToVPRecipes( } else if (GetElementPtrInst *GEP = dyn_cast(Inst)) { NewRecipe = new VPWidenGEPRecipe(GEP, Ingredient.operands()); } else if (CallInst *CI = dyn_cast(Inst)) { - NewRecipe = new VPWidenCallRecipe( - CI, Ingredient.operands(), getVectorIntrinsicIDForCall(CI, &TLI), - CI->getDebugLoc()); + NewRecipe = new VPWidenIntrinsicRecipe( + *CI, getVectorIntrinsicIDForCall(CI, &TLI), + make_range(Ingredient.op_begin(), Ingredient.op_end() - 1), + CI->getType(), CI->getDebugLoc()); } else if (SelectInst *SI = dyn_cast(Inst)) { NewRecipe = new VPWidenSelectRecipe(*SI, Ingredient.operands()); } else if (auto *CI = dyn_cast(Inst)) { diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h index 4c383244f96f1..f2978b0a758b6 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -350,6 +350,7 @@ class VPDef { VPWidenCanonicalIVSC, VPWidenCastSC, VPWidenGEPSC, + VPWidenIntrinsicSC, VPWidenLoadEVLSC, VPWidenLoadSC, VPWidenStoreEVLSC, diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll index 7797c0bce0baa..a55e53cc9b0bd 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-call.ll @@ -102,7 +102,7 @@ for.end: ; CHECK-REMARKS: UserVF ignored because of invalid costs. ; CHECK-REMARKS-NEXT: t.c:3:10: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): load -; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin.f32 +; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin ; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): store define void @vec_sin_no_mapping(ptr noalias nocapture %dst, ptr noalias nocapture readonly %src, i64 %n) { ; CHECK: @vec_sin_no_mapping @@ -129,8 +129,8 @@ for.cond.cleanup: ; preds = %for.body ; CHECK-REMARKS: UserVF ignored because of invalid costs. ; CHECK-REMARKS-NEXT: t.c:3:10: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): load ; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): fadd -; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin.f32 -; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin.f32 +; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin +; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin ; CHECK-REMARKS-NEXT: t.c:3:40: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): store define void @vec_sin_no_mapping_ite(ptr noalias nocapture %dst, ptr noalias nocapture readonly %src, i64 %n) { ; CHECK: @vec_sin_no_mapping_ite @@ -166,7 +166,7 @@ for.cond.cleanup: ; preds = %for.body ; CHECK-REMARKS: UserVF ignored because of invalid costs. ; CHECK-REMARKS-NEXT: t.c:3:10: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): load -; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin.f32 +; CHECK-REMARKS-NEXT: t.c:3:20: Recipe with invalid costs prevented vectorization at VF=(vscale x 1, vscale x 2): call to llvm.sin ; CHECK-REMARKS-NEXT: t.c:3:30: Recipe with invalid costs prevented vectorization at VF=(vscale x 1): store define void @vec_sin_fixed_mapping(ptr noalias nocapture %dst, ptr noalias nocapture readonly %src, i64 %n) { ; CHECK: @vec_sin_fixed_mapping diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll index a43e1a6755ed7..e9303ec9d3eb7 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll @@ -60,7 +60,7 @@ target triple = "arm64-apple-ios" ; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%gep.src> ; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VEC_PTR]]> ; CHECK-NEXT: WIDEN-CAST ir<%conv> = fpext ir<%l> to double -; CHECK-NEXT: WIDEN-CALL ir<%s> = call reassoc nnan ninf nsz arcp contract afn @llvm.sin.f64(ir<%conv>) (using vector intrinsic) +; CHECK-NEXT: WIDEN-INTRINSIC ir<%s> = call reassoc nnan ninf nsz arcp contract afn llvm.sin(ir<%conv>) ; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]> ; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep.dst> ; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> diff --git a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll index d872fb187a3bc..68dd47537fdfd 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll @@ -31,7 +31,7 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw ; CHECK-NEXT: " CLONE ir\<%arrayidx\> = getelementptr inbounds ir\<%y\>, vp\<[[STEPS]]\>\l" + ; CHECK-NEXT: " vp\<[[VEC_PTR:%.+]]\> = vector-pointer ir\<%arrayidx\>\l" + ; CHECK-NEXT: " WIDEN ir\<%lv\> = load vp\<[[VEC_PTR]]\>\l" + -; CHECK-NEXT: " WIDEN-CALL ir\<%call\> = call @llvm.sqrt.f32(ir\<%lv\>) (using vector intrinsic)\l" + +; CHECK-NEXT: " WIDEN-INTRINSIC ir\<%call\> = call llvm.sqrt(ir\<%lv\>)\l" + ; CHECK-NEXT: " CLONE ir\<%arrayidx2\> = getelementptr inbounds ir\<%x\>, vp\<[[STEPS]]\>\l" + ; CHECK-NEXT: " vp\<[[VEC_PTR2:%.+]]\> = vector-pointer ir\<%arrayidx2\>\l" + ; CHECK-NEXT: " WIDEN store vp\<[[VEC_PTR2]]\>, ir\<%call\>\l" + diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll index 2a55f6d756826..abfbf8574e180 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll @@ -23,7 +23,7 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]> ; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer ir<%arrayidx> ; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VEC_PTR]]> -; CHECK-NEXT: WIDEN-CALL ir<%call> = call @llvm.sqrt.f32(ir<%lv>) +; CHECK-NEXT: WIDEN-INTRINSIC ir<%call> = call llvm.sqrt(ir<%lv>) ; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> ; CHECK-NEXT: vp<[[VEC_PTR2:%.+]]> = vector-pointer ir<%arrayidx2> ; CHECK-NEXT: WIDEN store vp<[[VEC_PTR2]]>, ir<%call> diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp index a5d44e9a57246..2a828a2fbe395 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -901,7 +901,7 @@ TEST(VPRecipeTest, CastVPWidenCallRecipeToVPUserAndVPDef) { Args.push_back(&Op1); Args.push_back(&Op2); Args.push_back(&CalledFn); - VPWidenCallRecipe Recipe(Call, make_range(Args.begin(), Args.end()), false); + VPWidenCallRecipe Recipe(Call, Fn, make_range(Args.begin(), Args.end())); EXPECT_TRUE(isa(&Recipe)); VPRecipeBase *BaseR = &Recipe; EXPECT_TRUE(isa(BaseR)); @@ -1182,7 +1182,7 @@ TEST(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) { Args.push_back(&Op1); Args.push_back(&Op2); Args.push_back(&CalledFn); - VPWidenCallRecipe Recipe(Call, make_range(Args.begin(), Args.end()), false); + VPWidenCallRecipe Recipe(Call, Fn, make_range(Args.begin(), Args.end())); EXPECT_TRUE(Recipe.mayHaveSideEffects()); EXPECT_TRUE(Recipe.mayReadFromMemory()); EXPECT_TRUE(Recipe.mayWriteToMemory()); @@ -1205,7 +1205,7 @@ TEST(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) { Args.push_back(&Op1); Args.push_back(&Op2); Args.push_back(&CalledFn); - VPWidenCallRecipe Recipe(Call, make_range(Args.begin(), Args.end()), false); + VPWidenCallRecipe Recipe(Call, TheFn, make_range(Args.begin(), Args.end())); EXPECT_FALSE(Recipe.mayHaveSideEffects()); EXPECT_FALSE(Recipe.mayReadFromMemory()); EXPECT_FALSE(Recipe.mayWriteToMemory()); From ea64bc7d0e3e22a59128347667f1621b6847c5a7 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 4 Oct 2024 13:22:33 +0100 Subject: [PATCH 2/5] !fixup store read/write/sideeffects property in recipe. --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 +- llvm/lib/Transforms/Vectorize/VPlan.h | 14 +++++++++++++- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 7 ++++--- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index f7a8dcdfd5e52..74d859d81f8cd 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -4397,11 +4397,11 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks( OS << (Pair.second == Subset.front().second ? "" : ", ") << Pair.second; OS << "):"; if (Opcode == Instruction::Call) { - auto *WidenCall = dyn_cast(R); StringRef Name = ""; if (auto *Int = dyn_cast(R)) { Name = Int->getIntrinsicName(); } else { + auto *WidenCall = dyn_cast(R); Function *CalledFn = WidenCall ? WidenCall->getCalledScalarFunction() : cast(R->getOperand(R->getNumOperands() - 1) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 8d62970d7d824..828b55c2fd5dc 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1617,13 +1617,20 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags { /// Scalar type of the result produced by the intrinsic. Type *ResultTy; + bool MayWriteToMemory; + bool MayReadFromMemory; + bool MayHaveSideEffects; + public: template VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID, iterator_range CallArguments, Type *Ty, DebugLoc DL = {}) : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI), - VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty) {} + VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty), + MayWriteToMemory(CI.mayWriteToMemory()), + MayReadFromMemory(CI.mayReadFromMemory()), + MayHaveSideEffects(CI.mayHaveSideEffects()) {} ~VPWidenIntrinsicRecipe() override = default; @@ -1652,6 +1659,11 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags { void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override; #endif + bool mayWriteToMemory() const { return MayWriteToMemory; } + + bool mayReadFromMemory() const { return MayReadFromMemory; } + + bool mayHaveSideEffects() const { return MayHaveSideEffects; } }; /// A recipe for widening Call instructions using library calls. diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 03f6b7d442fb0..47c86c3f7e454 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -80,7 +80,8 @@ bool VPRecipeBase::mayWriteToMemory() const { ->getCalledScalarFunction() ->onlyReadsMemory(); case VPWidenIntrinsicSC: - return false; + return cast(this)->mayWriteToMemory(); + ; case VPBranchOnMaskSC: case VPScalarIVStepsSC: case VPPredInstPHISC: @@ -123,7 +124,7 @@ bool VPRecipeBase::mayReadFromMemory() const { ->getCalledScalarFunction() ->onlyWritesMemory(); case VPWidenIntrinsicSC: - return true; + return cast(this)->mayReadFromMemory(); case VPBranchOnMaskSC: case VPPredInstPHISC: case VPScalarIVStepsSC: @@ -166,7 +167,7 @@ bool VPRecipeBase::mayHaveSideEffects() const { return mayWriteToMemory() || !Fn->doesNotThrow() || !Fn->willReturn(); } case VPWidenIntrinsicSC: - return false; + return cast(this)->mayHaveSideEffects(); case VPBlendSC: case VPReductionEVLSC: case VPReductionSC: From ba896db87a8d159ecf3cf40adfd0a4b844f3f0da Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 7 Oct 2024 14:54:36 +0100 Subject: [PATCH 3/5] !fixup address latest comments, thanks! --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 7 +++---- llvm/lib/Transforms/Vectorize/VPlan.h | 7 +++---- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 5 +---- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 4 ++-- 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 18e820262f217..a62c522d6d6f1 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8293,7 +8293,6 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI, return nullptr; SmallVector Ops(Operands.take_front(CI->arg_size())); - Ops.push_back(Operands.back()); // Is it beneficial to perform intrinsic call compared to lib call? bool ShouldUseVectorIntrinsic = @@ -8304,9 +8303,8 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI, }, Range); if (ShouldUseVectorIntrinsic) - return new VPWidenIntrinsicRecipe(*CI, ID, - make_range(Ops.begin(), Ops.end() - 1), - CI->getType(), CI->getDebugLoc()); + return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(), + CI->getDebugLoc()); Function *Variant = nullptr; std::optional MaskPos; @@ -8358,6 +8356,7 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI, Ops.insert(Ops.begin() + *MaskPos, Mask); } + Ops.push_back(Operands.back()); return new VPWidenCallRecipe( CI, Variant, make_range(Ops.begin(), Ops.end()), CI->getDebugLoc()); } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index c066d623e9bff..699a9364a5ae0 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1627,9 +1627,8 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags { bool MayHaveSideEffects; public: - template VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID, - iterator_range CallArguments, Type *Ty, + ArrayRef CallArguments, Type *Ty, DebugLoc DL = {}) : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty), @@ -1641,8 +1640,8 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags { VPWidenIntrinsicRecipe *clone() override { return new VPWidenIntrinsicRecipe(*cast(getUnderlyingValue()), - VectorIntrinsicID, operands(), ResultTy, - getDebugLoc()); + VectorIntrinsicID, {op_begin(), op_end()}, + ResultTy, getDebugLoc()); } VP_CLASSOF_IMPL(VPDef::VPWidenIntrinsicSC) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 1cb084aa684b6..259da476574e7 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -81,7 +81,6 @@ bool VPRecipeBase::mayWriteToMemory() const { ->onlyReadsMemory(); case VPWidenIntrinsicSC: return cast(this)->mayWriteToMemory(); - ; case VPBranchOnMaskSC: case VPScalarIVStepsSC: case VPPredInstPHISC: @@ -912,9 +911,7 @@ void VPWidenCallRecipe::execute(VPTransformState &State) { CI->getOperandBundlesAsDefs(OpBundles); CallInst *V = State.Builder.CreateCall(Variant, Args, OpBundles); - - if (isa(V)) - V->copyFastMathFlags(CI); + setFlags(V); if (!V->getType()->isVoidTy()) State.set(this, V); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 8ffafcfaa09ab..379bfc0a4394b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -84,8 +84,8 @@ void VPlanTransforms::VPInstructionsToVPRecipes( } else if (CallInst *CI = dyn_cast(Inst)) { NewRecipe = new VPWidenIntrinsicRecipe( *CI, getVectorIntrinsicIDForCall(CI, &TLI), - make_range(Ingredient.op_begin(), Ingredient.op_end() - 1), - CI->getType(), CI->getDebugLoc()); + {Ingredient.op_begin(), Ingredient.op_end() - 1}, CI->getType(), + CI->getDebugLoc()); } else if (SelectInst *SI = dyn_cast(Inst)) { NewRecipe = new VPWidenSelectRecipe(*SI, Ingredient.operands()); } else if (auto *CI = dyn_cast(Inst)) { From 4243f26f55a72a2063fe8575ba525eeadaaa2872 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 8 Oct 2024 13:01:27 +0100 Subject: [PATCH 4/5] !fixup update VPWidenCallRecipe ctor to use ArrayRef. --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 3 +-- llvm/lib/Transforms/Vectorize/VPlan.h | 7 +++---- llvm/unittests/Transforms/Vectorize/VPlanTest.cpp | 6 +++--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index a62c522d6d6f1..193bca7e5d49a 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8357,8 +8357,7 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI, } Ops.push_back(Operands.back()); - return new VPWidenCallRecipe( - CI, Variant, make_range(Ops.begin(), Ops.end()), CI->getDebugLoc()); + return new VPWidenCallRecipe(CI, Variant, Ops, CI->getDebugLoc()); } return nullptr; diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 699a9364a5ae0..7e5834862e53a 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1678,9 +1678,8 @@ class VPWidenCallRecipe : public VPRecipeWithIRFlags { Function *Variant; public: - template VPWidenCallRecipe(Value *UV, Function *Variant, - iterator_range CallArguments, DebugLoc DL = {}) + ArrayRef CallArguments, DebugLoc DL = {}) : VPRecipeWithIRFlags(VPDef::VPWidenCallSC, CallArguments, *cast(UV)), Variant(Variant) { @@ -1692,8 +1691,8 @@ class VPWidenCallRecipe : public VPRecipeWithIRFlags { ~VPWidenCallRecipe() override = default; VPWidenCallRecipe *clone() override { - return new VPWidenCallRecipe(getUnderlyingValue(), Variant, operands(), - getDebugLoc()); + return new VPWidenCallRecipe(getUnderlyingValue(), Variant, + {op_begin(), op_end()}, getDebugLoc()); } VP_CLASSOF_IMPL(VPDef::VPWidenCallSC) diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp index 2a828a2fbe395..372c5aaea5938 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -901,7 +901,7 @@ TEST(VPRecipeTest, CastVPWidenCallRecipeToVPUserAndVPDef) { Args.push_back(&Op1); Args.push_back(&Op2); Args.push_back(&CalledFn); - VPWidenCallRecipe Recipe(Call, Fn, make_range(Args.begin(), Args.end())); + VPWidenCallRecipe Recipe(Call, Fn, Args); EXPECT_TRUE(isa(&Recipe)); VPRecipeBase *BaseR = &Recipe; EXPECT_TRUE(isa(BaseR)); @@ -1182,7 +1182,7 @@ TEST(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) { Args.push_back(&Op1); Args.push_back(&Op2); Args.push_back(&CalledFn); - VPWidenCallRecipe Recipe(Call, Fn, make_range(Args.begin(), Args.end())); + VPWidenCallRecipe Recipe(Call, Fn, Args); EXPECT_TRUE(Recipe.mayHaveSideEffects()); EXPECT_TRUE(Recipe.mayReadFromMemory()); EXPECT_TRUE(Recipe.mayWriteToMemory()); @@ -1205,7 +1205,7 @@ TEST(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) { Args.push_back(&Op1); Args.push_back(&Op2); Args.push_back(&CalledFn); - VPWidenCallRecipe Recipe(Call, TheFn, make_range(Args.begin(), Args.end())); + VPWidenCallRecipe Recipe(Call, TheFn, Args); EXPECT_FALSE(Recipe.mayHaveSideEffects()); EXPECT_FALSE(Recipe.mayReadFromMemory()); EXPECT_FALSE(Recipe.mayWriteToMemory()); From 03424f93914b494a8485779a5f58f18d21cf1103 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 8 Oct 2024 21:45:52 +0100 Subject: [PATCH 5/5] !fixup refine names, add some more comments --- llvm/lib/Transforms/Vectorize/VPlan.h | 28 +++++++++++++------ .../Transforms/Vectorize/VPlanAnalysis.cpp | 5 ++-- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 2 +- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 7e5834862e53a..3aecfaffaf2b8 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1619,11 +1619,16 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags { /// ID of the vector intrinsic to widen. Intrinsic::ID VectorIntrinsicID; - /// Scalar type of the result produced by the intrinsic. + /// Scalar return type of the intrinsic. Type *ResultTy; - bool MayWriteToMemory; + /// True if the intrinsic may read from memory. bool MayReadFromMemory; + + /// True if the intrinsic may read write to memory. + bool MayWriteToMemory; + + /// True if the intrinsic may have side-effects. bool MayHaveSideEffects; public: @@ -1632,8 +1637,8 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags { DebugLoc DL = {}) : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty), - MayWriteToMemory(CI.mayWriteToMemory()), MayReadFromMemory(CI.mayReadFromMemory()), + MayWriteToMemory(CI.mayWriteToMemory()), MayHaveSideEffects(CI.mayHaveSideEffects()) {} ~VPWidenIntrinsicRecipe() override = default; @@ -1653,21 +1658,26 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags { InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override; - Type *getResultTy() const { return ResultTy; } + /// Return the scalar return type of the intrinsic. + Type *getResultType() const { return ResultTy; } /// Return to name of the intrinsic as string. StringRef getIntrinsicName() const; + /// Returns true if the intrinsic may read from memory. + bool mayReadFromMemory() const { return MayReadFromMemory; } + + /// Returns true if the intrinsic may write to memory. + bool mayWriteToMemory() const { return MayWriteToMemory; } + + /// Returns true if the intrinsic may have side-effects. + bool mayHaveSideEffects() const { return MayHaveSideEffects; } + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) /// Print the recipe. void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override; #endif - bool mayWriteToMemory() const { return MayWriteToMemory; } - - bool mayReadFromMemory() const { return MayReadFromMemory; } - - bool mayHaveSideEffects() const { return MayHaveSideEffects; } }; /// A recipe for widening Call instructions using library calls. diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp index 24f34e43a6e84..5a5b3ac19c46a 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp @@ -268,8 +268,9 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) { VPReplicateRecipe, VPWidenCallRecipe, VPWidenMemoryRecipe, VPWidenSelectRecipe>( [this](const auto *R) { return inferScalarTypeForRecipe(R); }) - .Case( - [](const VPWidenIntrinsicRecipe *R) { return R->getResultTy(); }) + .Case([](const VPWidenIntrinsicRecipe *R) { + return R->getResultType(); + }) .Case([V](const VPInterleaveRecipe *R) { // TODO: Use info from interleave group. return V->getUnderlyingValue()->getType(); diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 170c869743188..4e983214318ab 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -961,7 +961,7 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) { SmallVector TysForDecl; // Add return type if intrinsic is overloaded on it. if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, -1)) - TysForDecl.push_back(VectorType::get(getResultTy(), State.VF)); + TysForDecl.push_back(VectorType::get(getResultType(), State.VF)); SmallVector Args; for (const auto &I : enumerate(operands())) { // Some intrinsics have a scalar argument - don't replace it with a