Skip to content

Commit 4a43c98

Browse files
LiqinWengLiqinWeng
committed
[LV][EVL] Support call instruction with EVL-vectorization
1 parent 6b7705c commit 4a43c98

File tree

7 files changed

+400
-95
lines changed

7 files changed

+400
-95
lines changed

‎llvm/include/llvm/IR/VectorBuilder.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -99,11 +99,11 @@ class VectorBuilder {
9999
const Twine &Name = Twine());
100100

101101
/// Emit a VP reduction intrinsic call for recurrence kind.
102-
/// \param RdxID The intrinsic ID of llvm.vector.reduce.*
102+
/// \param ID The intrinsic ID of call Intrinsic
103103
/// \param ValTy The type of operand which the reduction operation is
104104
/// performed.
105105
/// \param VecOpArray The operand list.
106-
Value *createSimpleReduction(Intrinsic::ID RdxID, Type *ValTy,
106+
Value *createSimpleIntrinsic(Intrinsic::ID ID, Type *ValTy,
107107
ArrayRef<Value *> VecOpArray,
108108
const Twine &Name = Twine());
109109
};

‎llvm/lib/Analysis/VectorUtils.cpp

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,9 +116,13 @@ bool llvm::isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
116116
unsigned ScalarOpdIdx) {
117117
switch (ID) {
118118
case Intrinsic::abs:
119+
case Intrinsic::vp_abs:
119120
case Intrinsic::ctlz:
121+
case Intrinsic::vp_ctlz:
120122
case Intrinsic::cttz:
123+
case Intrinsic::vp_cttz:
121124
case Intrinsic::is_fpclass:
125+
case Intrinsic::vp_is_fpclass:
122126
case Intrinsic::powi:
123127
return (ScalarOpdIdx == 1);
124128
case Intrinsic::smul_fix:
@@ -140,10 +144,13 @@ bool llvm::isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID,
140144
case Intrinsic::fptoui_sat:
141145
case Intrinsic::lrint:
142146
case Intrinsic::llrint:
147+
case Intrinsic::vp_lrint:
148+
case Intrinsic::vp_llrint:
143149
case Intrinsic::ucmp:
144150
case Intrinsic::scmp:
145151
return OpdIdx == -1 || OpdIdx == 0;
146152
case Intrinsic::is_fpclass:
153+
case Intrinsic::vp_is_fpclass:
147154
return OpdIdx == 0;
148155
case Intrinsic::powi:
149156
return OpdIdx == -1 || OpdIdx == 1;

‎llvm/lib/IR/VectorBuilder.cpp

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -60,13 +60,12 @@ Value *VectorBuilder::createVectorInstruction(unsigned Opcode, Type *ReturnTy,
6060
return createVectorInstructionImpl(VPID, ReturnTy, InstOpArray, Name);
6161
}
6262

63-
Value *VectorBuilder::createSimpleReduction(Intrinsic::ID RdxID,
64-
Type *ValTy,
63+
Value *VectorBuilder::createSimpleIntrinsic(Intrinsic::ID ID, Type *ValTy,
6564
ArrayRef<Value *> InstOpArray,
6665
const Twine &Name) {
67-
auto VPID = VPIntrinsic::getForIntrinsic(RdxID);
68-
assert(VPReductionIntrinsic::isVPReduction(VPID) &&
69-
"No VPIntrinsic for this reduction");
66+
auto VPID = VPIntrinsic::getForIntrinsic(ID);
67+
assert(VPIntrinsic::isVPIntrinsic(VPID) &&
68+
"No VPIntrinsic for this Intrinsic");
7069
return createVectorInstructionImpl(VPID, ValTy, InstOpArray, Name);
7170
}
7271

‎llvm/lib/Transforms/Utils/LoopUtils.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1300,7 +1300,7 @@ Value *llvm::createSimpleReduction(VectorBuilder &VBuilder, Value *Src,
13001300
Type *SrcEltTy = SrcTy->getElementType();
13011301
Value *Iden = getRecurrenceIdentity(Kind, SrcEltTy, Desc.getFastMathFlags());
13021302
Value *Ops[] = {Iden, Src};
1303-
return VBuilder.createSimpleReduction(Id, SrcTy, Ops);
1303+
return VBuilder.createSimpleIntrinsic(Id, SrcTy, Ops);
13041304
}
13051305

13061306
Value *llvm::createReduction(IRBuilderBase &B,
@@ -1343,7 +1343,7 @@ Value *llvm::createOrderedReduction(VectorBuilder &VBuilder,
13431343
Intrinsic::ID Id = getReductionIntrinsicID(RecurKind::FAdd);
13441344
auto *SrcTy = cast<VectorType>(Src->getType());
13451345
Value *Ops[] = {Start, Src};
1346-
return VBuilder.createSimpleReduction(Id, SrcTy, Ops);
1346+
return VBuilder.createSimpleIntrinsic(Id, SrcTy, Ops);
13471347
}
13481348

13491349
void llvm::propagateIRFlags(Value *I, ArrayRef<Value *> VL, Value *OpValue,

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

Lines changed: 44 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -994,24 +994,39 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) {
994994
Args.push_back(Arg);
995995
}
996996

997-
// Use vector version of the intrinsic.
998-
Module *M = State.Builder.GetInsertBlock()->getModule();
999-
Function *VectorF =
1000-
Intrinsic::getOrInsertDeclaration(M, VectorIntrinsicID, TysForDecl);
1001-
assert(VectorF && "Can't retrieve vector intrinsic.");
1002-
1003-
auto *CI = cast_or_null<CallInst>(getUnderlyingValue());
1004-
SmallVector<OperandBundleDef, 1> OpBundles;
1005-
if (CI)
1006-
CI->getOperandBundlesAsDefs(OpBundles);
997+
if (VPIntrinsic::isVPIntrinsic(VectorIntrinsicID)) {
998+
// Use vector version of the vector predicate Intrinsic
999+
IRBuilderBase &BuilderIR = State.Builder;
1000+
VectorBuilder VBuilder(BuilderIR);
1001+
Value *Mask = BuilderIR.CreateVectorSplat(State.VF, BuilderIR.getTrue());
1002+
VBuilder.setMask(Mask).setEVL(Args.back());
1003+
// Remove the EVL from Args
1004+
Args.pop_back();
1005+
Value *VPInst = VBuilder.createSimpleIntrinsic(
1006+
VectorIntrinsicID, TysForDecl[0], Args, "vp.call");
1007+
if (!VPInst->getType()->isVoidTy())
1008+
State.set(this, VPInst);
1009+
State.addMetadata(VPInst,
1010+
dyn_cast_or_null<Instruction>(getUnderlyingValue()));
1011+
} else {
1012+
// Use vector version of the intrinsic.
1013+
Module *M = State.Builder.GetInsertBlock()->getModule();
1014+
Function *VectorF =
1015+
Intrinsic::getOrInsertDeclaration(M, VectorIntrinsicID, TysForDecl);
1016+
assert(VectorF && "Can't retrieve vector intrinsic.");
10071017

1008-
CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
1018+
auto *CI = cast_or_null<CallInst>(getUnderlyingValue());
1019+
SmallVector<OperandBundleDef, 1> OpBundles;
1020+
if (CI)
1021+
CI->getOperandBundlesAsDefs(OpBundles);
10091022

1010-
setFlags(V);
1023+
CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
1024+
setFlags(V);
10111025

1012-
if (!V->getType()->isVoidTy())
1013-
State.set(this, V);
1014-
State.addMetadata(V, CI);
1026+
if (!V->getType()->isVoidTy())
1027+
State.set(this, V);
1028+
State.addMetadata(V, CI);
1029+
}
10151030
}
10161031

10171032
InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF,
@@ -1024,6 +1039,18 @@ InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF,
10241039
// clear Arguments.
10251040
// TODO: Rework TTI interface to be independent of concrete IR values.
10261041
SmallVector<const Value *> Arguments;
1042+
1043+
Intrinsic::ID FID = VectorIntrinsicID;
1044+
unsigned NumOperands = getNumOperands();
1045+
if (VPIntrinsic::isVPIntrinsic(VectorIntrinsicID)) {
1046+
std::optional<Intrinsic::ID> ID =
1047+
VPIntrinsic::getFunctionalIntrinsicIDForVP(VectorIntrinsicID);
1048+
if (ID) {
1049+
FID = ID.value();
1050+
NumOperands = getNumOperands() - 1;
1051+
}
1052+
}
1053+
10271054
for (const auto &[Idx, Op] : enumerate(operands())) {
10281055
auto *V = Op->getUnderlyingValue();
10291056
if (!V) {
@@ -1039,14 +1066,14 @@ InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF,
10391066

10401067
Type *RetTy = ToVectorTy(Ctx.Types.inferScalarType(this), VF);
10411068
SmallVector<Type *> ParamTys;
1042-
for (unsigned I = 0; I != getNumOperands(); ++I)
1069+
for (unsigned I = 0; I != NumOperands; ++I)
10431070
ParamTys.push_back(
10441071
ToVectorTy(Ctx.Types.inferScalarType(getOperand(I)), VF));
10451072

10461073
// TODO: Rework TTI interface to avoid reliance on underlying IntrinsicInst.
10471074
FastMathFlags FMF = hasFastMathFlags() ? getFastMathFlags() : FastMathFlags();
10481075
IntrinsicCostAttributes CostAttrs(
1049-
VectorIntrinsicID, RetTy, Arguments, ParamTys, FMF,
1076+
FID, RetTy, Arguments, ParamTys, FMF,
10501077
dyn_cast_or_null<IntrinsicInst>(getUnderlyingValue()));
10511078
return Ctx.TTI.getIntrinsicInstrCost(CostAttrs, CostKind);
10521079
}

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1484,6 +1484,24 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
14841484
VPValue *NewMask = GetNewMask(Red->getCondOp());
14851485
return new VPReductionEVLRecipe(*Red, EVL, NewMask);
14861486
})
1487+
.Case<VPWidenIntrinsicRecipe>(
1488+
[&](VPWidenIntrinsicRecipe *CInst) -> VPRecipeBase * {
1489+
auto *CI = cast<CallInst>(CInst->getUnderlyingInstr());
1490+
SmallVector<VPValue *> Ops(CInst->operands());
1491+
Ops.push_back(&EVL);
1492+
Intrinsic::ID VPID = VPIntrinsic::getForIntrinsic(
1493+
CI->getCalledFunction()->getIntrinsicID());
1494+
if (VPID == Intrinsic::not_intrinsic)
1495+
return nullptr;
1496+
// FIXME: In fact, can we really not pass the
1497+
// underlyingInstr? In this case, how to set the Flag and
1498+
// add metadata in execute?
1499+
return new VPWidenIntrinsicRecipe(
1500+
VPID, Ops, TypeInfo.inferScalarType(CInst), false,
1501+
false, false);
1502+
// return new VPWidenIntrinsicRecipe(
1503+
// *CI, VPID, Ops, CI->getType(), CI->getDebugLoc());
1504+
})
14871505
.Case<VPWidenSelectRecipe>([&](VPWidenSelectRecipe *Sel) {
14881506
SmallVector<VPValue *> Ops(Sel->operands());
14891507
Ops.push_back(&EVL);

0 commit comments

Comments
 (0)