diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 931d4d42f56e4..e962a36587d48 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -2152,13 +2152,6 @@ static VPRecipeBase *createEVLRecipe(VPValue *HeaderMask, VPValue *NewMask = GetNewMask(Red->getCondOp()); return new VPReductionEVLRecipe(*Red, EVL, NewMask); }) - .Case([&](VPWidenSelectRecipe *Sel) { - SmallVector Ops(Sel->operands()); - Ops.push_back(&EVL); - return new VPWidenIntrinsicRecipe(Intrinsic::vp_select, Ops, - TypeInfo.inferScalarType(Sel), - Sel->getDebugLoc()); - }) .Case([&](VPInstruction *VPI) -> VPRecipeBase * { if (VPI->getOpcode() == VPInstruction::FirstOrderRecurrenceSplice) { assert(PrevEVL && "Fixed-order recurrences require previous EVL"); diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll index c063e6ff3f482..d485a7432423a 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll @@ -367,7 +367,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) { ; IF-EVL-OUTLOOP-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0 ; IF-EVL-OUTLOOP-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], splat (i1 true), i32 [[TMP9]]) ; IF-EVL-OUTLOOP-NEXT: [[TMP13:%.*]] = icmp slt [[VP_OP_LOAD]], [[VEC_PHI]] -; IF-EVL-OUTLOOP-NEXT: [[TMP14:%.*]] = call @llvm.vp.select.nxv4i32( [[TMP13]], [[VP_OP_LOAD]], [[VEC_PHI]], i32 [[TMP9]]) +; IF-EVL-OUTLOOP-NEXT: [[TMP14:%.*]] = select [[TMP13]], [[VP_OP_LOAD]], [[VEC_PHI]] ; IF-EVL-OUTLOOP-NEXT: [[TMP15]] = call @llvm.vp.merge.nxv4i32( splat (i1 true), [[TMP14]], [[VEC_PHI]], i32 [[TMP9]]) ; IF-EVL-OUTLOOP-NEXT: [[TMP16:%.*]] = zext i32 [[TMP9]] to i64 ; IF-EVL-OUTLOOP-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll index 6cfb4bcdd991a..45357dd6bf0d6 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll @@ -138,20 +138,19 @@ define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noali ; PREDICATED_EVL-NEXT: [[TMP11:%.*]] = zext nneg [[TMP10]] to ; PREDICATED_EVL-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP11]] ; PREDICATED_EVL-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call @llvm.vp.gather.nxv16i8.nxv16p0( align 1 [[TMP12]], [[TMP6]], i32 [[TMP5]]) -; PREDICATED_EVL-NEXT: [[TMP13:%.*]] = icmp slt [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER3]] -; PREDICATED_EVL-NEXT: [[TMP14:%.*]] = call @llvm.vp.select.nxv16i8( [[TMP13]], [[WIDE_MASKED_GATHER3]], [[WIDE_MASKED_GATHER]], i32 [[TMP5]]) -; PREDICATED_EVL-NEXT: [[TMP15:%.*]] = zext nneg [[TMP7]] to -; PREDICATED_EVL-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP15]] -; PREDICATED_EVL-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0( [[TMP14]], align 1 [[TMP16]], [[TMP6]], i32 [[TMP5]]) -; PREDICATED_EVL-NEXT: [[TMP17:%.*]] = sub zeroinitializer, [[TMP14]] -; PREDICATED_EVL-NEXT: [[TMP18:%.*]] = zext nneg [[TMP10]] to -; PREDICATED_EVL-NEXT: [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP18]] -; PREDICATED_EVL-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0( [[TMP17]], align 1 [[TMP19]], [[TMP6]], i32 [[TMP5]]) +; PREDICATED_EVL-NEXT: [[TMP13:%.*]] = call @llvm.smax.nxv16i8( [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER3]]) +; PREDICATED_EVL-NEXT: [[TMP14:%.*]] = zext nneg [[TMP7]] to +; PREDICATED_EVL-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP14]] +; PREDICATED_EVL-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0( [[TMP13]], align 1 [[TMP15]], [[TMP6]], i32 [[TMP5]]) +; PREDICATED_EVL-NEXT: [[TMP16:%.*]] = sub zeroinitializer, [[TMP13]] +; PREDICATED_EVL-NEXT: [[TMP17:%.*]] = zext nneg [[TMP10]] to +; PREDICATED_EVL-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP17]] +; PREDICATED_EVL-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0( [[TMP16]], align 1 [[TMP18]], [[TMP6]], i32 [[TMP5]]) ; PREDICATED_EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP5]], [[EVL_BASED_IV]] ; PREDICATED_EVL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]] ; PREDICATED_EVL-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[BROADCAST_SPLAT2]] -; PREDICATED_EVL-NEXT: [[TMP20:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; PREDICATED_EVL-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; PREDICATED_EVL-NEXT: [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] +; PREDICATED_EVL-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; PREDICATED_EVL: middle.block: ; PREDICATED_EVL-NEXT: br label [[FOR_END:%.*]] ; PREDICATED_EVL: scalar.ph: @@ -364,29 +363,27 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali ; PREDICATED_EVL-NEXT: [[TMP17:%.*]] = zext nneg [[TMP10]] to ; PREDICATED_EVL-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP17]] ; PREDICATED_EVL-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call @llvm.vp.gather.nxv16i8.nxv16p0( align 1 [[TMP18]], [[TMP6]], i32 [[TMP5]]) -; PREDICATED_EVL-NEXT: [[TMP19:%.*]] = icmp slt [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER3]] -; PREDICATED_EVL-NEXT: [[TMP20:%.*]] = call @llvm.vp.select.nxv16i8( [[TMP19]], [[WIDE_MASKED_GATHER3]], [[WIDE_MASKED_GATHER]], i32 [[TMP5]]) -; PREDICATED_EVL-NEXT: [[TMP21:%.*]] = sub zeroinitializer, [[TMP20]] -; PREDICATED_EVL-NEXT: [[TMP22:%.*]] = icmp slt [[WIDE_MASKED_GATHER4]], [[WIDE_MASKED_GATHER5]] -; PREDICATED_EVL-NEXT: [[TMP23:%.*]] = call @llvm.vp.select.nxv16i8( [[TMP22]], [[WIDE_MASKED_GATHER5]], [[WIDE_MASKED_GATHER4]], i32 [[TMP5]]) -; PREDICATED_EVL-NEXT: [[TMP24:%.*]] = sub zeroinitializer, [[TMP23]] -; PREDICATED_EVL-NEXT: [[TMP25:%.*]] = zext nneg [[TMP7]] to +; PREDICATED_EVL-NEXT: [[TMP19:%.*]] = call @llvm.smax.nxv16i8( [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER3]]) +; PREDICATED_EVL-NEXT: [[TMP20:%.*]] = sub zeroinitializer, [[TMP19]] +; PREDICATED_EVL-NEXT: [[TMP21:%.*]] = call @llvm.smax.nxv16i8( [[WIDE_MASKED_GATHER4]], [[WIDE_MASKED_GATHER5]]) +; PREDICATED_EVL-NEXT: [[TMP22:%.*]] = sub zeroinitializer, [[TMP21]] +; PREDICATED_EVL-NEXT: [[TMP23:%.*]] = zext nneg [[TMP7]] to +; PREDICATED_EVL-NEXT: [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP23]] +; PREDICATED_EVL-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0( [[TMP19]], align 1 [[TMP24]], [[TMP6]], i32 [[TMP5]]) +; PREDICATED_EVL-NEXT: [[TMP25:%.*]] = zext nneg [[TMP8]] to ; PREDICATED_EVL-NEXT: [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP25]] ; PREDICATED_EVL-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0( [[TMP20]], align 1 [[TMP26]], [[TMP6]], i32 [[TMP5]]) -; PREDICATED_EVL-NEXT: [[TMP27:%.*]] = zext nneg [[TMP8]] to +; PREDICATED_EVL-NEXT: [[TMP27:%.*]] = zext nneg [[TMP9]] to ; PREDICATED_EVL-NEXT: [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP27]] ; PREDICATED_EVL-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0( [[TMP21]], align 1 [[TMP28]], [[TMP6]], i32 [[TMP5]]) -; PREDICATED_EVL-NEXT: [[TMP29:%.*]] = zext nneg [[TMP9]] to +; PREDICATED_EVL-NEXT: [[TMP29:%.*]] = zext nneg [[TMP10]] to ; PREDICATED_EVL-NEXT: [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP29]] -; PREDICATED_EVL-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0( [[TMP23]], align 1 [[TMP30]], [[TMP6]], i32 [[TMP5]]) -; PREDICATED_EVL-NEXT: [[TMP31:%.*]] = zext nneg [[TMP10]] to -; PREDICATED_EVL-NEXT: [[TMP32:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP31]] -; PREDICATED_EVL-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0( [[TMP24]], align 1 [[TMP32]], [[TMP6]], i32 [[TMP5]]) +; PREDICATED_EVL-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0( [[TMP22]], align 1 [[TMP30]], [[TMP6]], i32 [[TMP5]]) ; PREDICATED_EVL-NEXT: [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP5]], [[EVL_BASED_IV]] ; PREDICATED_EVL-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]] ; PREDICATED_EVL-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[BROADCAST_SPLAT2]] -; PREDICATED_EVL-NEXT: [[TMP33:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; PREDICATED_EVL-NEXT: br i1 [[TMP33]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] +; PREDICATED_EVL-NEXT: [[TMP31:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] +; PREDICATED_EVL-NEXT: br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; PREDICATED_EVL: middle.block: ; PREDICATED_EVL-NEXT: br label [[FOR_END:%.*]] ; PREDICATED_EVL: scalar.ph: diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/preserve-dbg-loc.ll b/llvm/test/Transforms/LoopVectorize/RISCV/preserve-dbg-loc.ll index 93bd44f5c6220..e4892db592d6b 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/preserve-dbg-loc.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/preserve-dbg-loc.ll @@ -9,7 +9,7 @@ define void @vp_select(ptr %a, ptr %b, ptr %c, i64 %N) { ; DEBUGLOC-LABEL: define void @vp_select( ; DEBUGLOC: vector.body: -; DEBUGLOC: = call @llvm.vp.select.nxv4i32( %{{.+}}, %{{.+}}, %{{.+}}, i32 %{{.+}}), !dbg ![[SELLOC:[0-9]+]] +; DEBUGLOC: = select %{{.+}}, %{{.+}}, %{{.+}}, !dbg ![[SELLOC:[0-9]+]] ; DEBUGLOC: loop: ; DEBUGLOC: = select i1 %{{.+}}, i32 %{{.+}}, i32 %{{.+}}, !dbg ![[SELLOC]] ; diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll b/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll index f059730a245be..1e1ed49f9f2ed 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll @@ -44,7 +44,7 @@ define void @type_info_cache_clobber(ptr %dstv, ptr %src, i64 %wide.trip.count) ; CHECK-NEXT: [[TMP23:%.*]] = ashr [[TMP15]], zeroinitializer ; CHECK-NEXT: [[VP_OP3:%.*]] = or [[TMP23]], zeroinitializer ; CHECK-NEXT: [[TMP16:%.*]] = icmp ult [[TMP15]], zeroinitializer -; CHECK-NEXT: [[TMP17:%.*]] = call @llvm.vp.select.nxv8i32( [[TMP16]], [[VP_OP3]], zeroinitializer, i32 [[TMP11]]) +; CHECK-NEXT: [[TMP17:%.*]] = select [[TMP16]], [[VP_OP3]], zeroinitializer ; CHECK-NEXT: [[TMP24:%.*]] = trunc [[TMP17]] to ; CHECK-NEXT: call void @llvm.vp.scatter.nxv8i8.nxv8p0( [[TMP24]], align 1 [[BROADCAST_SPLAT]], splat (i1 true), i32 [[TMP11]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]] ; CHECK-NEXT: [[TMP19:%.*]] = trunc [[VP_OP]] to diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll index 8c44da63e08a6..2926011857ae9 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll @@ -47,7 +47,7 @@ define i32 @cond_add(ptr %a, i64 %n, i32 %start) { ; IF-EVL-OUTLOOP-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 0 ; IF-EVL-OUTLOOP-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP17]], splat (i1 true), i32 [[TMP11]]) ; IF-EVL-OUTLOOP-NEXT: [[TMP18:%.*]] = icmp sgt [[VP_OP_LOAD]], splat (i32 3) -; IF-EVL-OUTLOOP-NEXT: [[TMP19:%.*]] = call @llvm.vp.select.nxv4i32( [[TMP18]], [[VP_OP_LOAD]], zeroinitializer, i32 [[TMP11]]) +; IF-EVL-OUTLOOP-NEXT: [[TMP19:%.*]] = select [[TMP18]], [[VP_OP_LOAD]], zeroinitializer ; IF-EVL-OUTLOOP-NEXT: [[VP_OP:%.*]] = add [[TMP19]], [[VEC_PHI]] ; IF-EVL-OUTLOOP-NEXT: [[TMP20]] = call @llvm.vp.merge.nxv4i32( splat (i1 true), [[VP_OP]], [[VEC_PHI]], i32 [[TMP11]]) ; IF-EVL-OUTLOOP-NEXT: [[TMP22:%.*]] = zext i32 [[TMP11]] to i64 @@ -101,7 +101,7 @@ define i32 @cond_add(ptr %a, i64 %n, i32 %start) { ; IF-EVL-INLOOP-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[TMP17]], i32 0 ; IF-EVL-INLOOP-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP18]], splat (i1 true), i32 [[TMP12]]) ; IF-EVL-INLOOP-NEXT: [[TMP19:%.*]] = icmp sgt [[VP_OP_LOAD]], splat (i32 3) -; IF-EVL-INLOOP-NEXT: [[TMP20:%.*]] = call @llvm.vp.select.nxv4i32( [[TMP19]], [[VP_OP_LOAD]], zeroinitializer, i32 [[TMP12]]) +; IF-EVL-INLOOP-NEXT: [[TMP20:%.*]] = select [[TMP19]], [[VP_OP_LOAD]], zeroinitializer ; IF-EVL-INLOOP-NEXT: [[TMP21:%.*]] = call i32 @llvm.vp.reduce.add.nxv4i32(i32 0, [[TMP20]], splat (i1 true), i32 [[TMP12]]) ; IF-EVL-INLOOP-NEXT: [[TMP22]] = add i32 [[TMP21]], [[VEC_PHI]] ; IF-EVL-INLOOP-NEXT: [[TMP23:%.*]] = zext i32 [[TMP12]] to i64 @@ -543,7 +543,7 @@ define i32 @step_cond_add(ptr %a, i64 %n, i32 %start) { ; IF-EVL-OUTLOOP-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i32 0 ; IF-EVL-OUTLOOP-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP15]], splat (i1 true), i32 [[TMP12]]) ; IF-EVL-OUTLOOP-NEXT: [[TMP16:%.*]] = icmp sgt [[VP_OP_LOAD]], [[VEC_IND]] -; IF-EVL-OUTLOOP-NEXT: [[TMP17:%.*]] = call @llvm.vp.select.nxv4i32( [[TMP16]], [[VP_OP_LOAD]], zeroinitializer, i32 [[TMP12]]) +; IF-EVL-OUTLOOP-NEXT: [[TMP17:%.*]] = select [[TMP16]], [[VP_OP_LOAD]], zeroinitializer ; IF-EVL-OUTLOOP-NEXT: [[TMP18:%.*]] = add [[TMP17]], [[VEC_PHI]] ; IF-EVL-OUTLOOP-NEXT: [[TMP19]] = call @llvm.vp.merge.nxv4i32( splat (i1 true), [[TMP18]], [[VEC_PHI]], i32 [[TMP12]]) ; IF-EVL-OUTLOOP-NEXT: [[TMP20:%.*]] = zext i32 [[TMP12]] to i64 @@ -606,7 +606,7 @@ define i32 @step_cond_add(ptr %a, i64 %n, i32 %start) { ; IF-EVL-INLOOP-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP13]], i32 0 ; IF-EVL-INLOOP-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], splat (i1 true), i32 [[TMP11]]) ; IF-EVL-INLOOP-NEXT: [[TMP15:%.*]] = icmp sgt [[VP_OP_LOAD]], [[VEC_IND]] -; IF-EVL-INLOOP-NEXT: [[TMP16:%.*]] = call @llvm.vp.select.nxv4i32( [[TMP15]], [[VP_OP_LOAD]], zeroinitializer, i32 [[TMP11]]) +; IF-EVL-INLOOP-NEXT: [[TMP16:%.*]] = select [[TMP15]], [[VP_OP_LOAD]], zeroinitializer ; IF-EVL-INLOOP-NEXT: [[TMP17:%.*]] = call i32 @llvm.vp.reduce.add.nxv4i32(i32 0, [[TMP16]], splat (i1 true), i32 [[TMP11]]) ; IF-EVL-INLOOP-NEXT: [[ADD]] = add i32 [[TMP17]], [[RDX]] ; IF-EVL-INLOOP-NEXT: [[TMP19:%.*]] = zext i32 [[TMP11]] to i64 diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll index 83deebe42d38f..8b1441450dd94 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll @@ -602,7 +602,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) { ; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0 ; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], splat (i1 true), i32 [[TMP9]]) ; IF-EVL-NEXT: [[TMP13:%.*]] = icmp slt [[VP_OP_LOAD]], [[VEC_PHI]] -; IF-EVL-NEXT: [[TMP14:%.*]] = call @llvm.vp.select.nxv4i32( [[TMP13]], [[VP_OP_LOAD]], [[VEC_PHI]], i32 [[TMP9]]) +; IF-EVL-NEXT: [[TMP14:%.*]] = select [[TMP13]], [[VP_OP_LOAD]], [[VEC_PHI]] ; IF-EVL-NEXT: [[TMP15]] = call @llvm.vp.merge.nxv4i32( splat (i1 true), [[TMP14]], [[VEC_PHI]], i32 [[TMP9]]) ; IF-EVL-NEXT: [[TMP16:%.*]] = zext i32 [[TMP9]] to i64 ; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]] @@ -723,7 +723,7 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) { ; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0 ; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], splat (i1 true), i32 [[TMP9]]) ; IF-EVL-NEXT: [[TMP13:%.*]] = icmp sgt [[VP_OP_LOAD]], [[VEC_PHI]] -; IF-EVL-NEXT: [[TMP14:%.*]] = call @llvm.vp.select.nxv4i32( [[TMP13]], [[VP_OP_LOAD]], [[VEC_PHI]], i32 [[TMP9]]) +; IF-EVL-NEXT: [[TMP14:%.*]] = select [[TMP13]], [[VP_OP_LOAD]], [[VEC_PHI]] ; IF-EVL-NEXT: [[TMP15]] = call @llvm.vp.merge.nxv4i32( splat (i1 true), [[TMP14]], [[VEC_PHI]], i32 [[TMP9]]) ; IF-EVL-NEXT: [[TMP16:%.*]] = zext i32 [[TMP9]] to i64 ; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]] @@ -844,7 +844,7 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) { ; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0 ; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], splat (i1 true), i32 [[TMP9]]) ; IF-EVL-NEXT: [[TMP13:%.*]] = icmp ult [[VP_OP_LOAD]], [[VEC_PHI]] -; IF-EVL-NEXT: [[TMP14:%.*]] = call @llvm.vp.select.nxv4i32( [[TMP13]], [[VP_OP_LOAD]], [[VEC_PHI]], i32 [[TMP9]]) +; IF-EVL-NEXT: [[TMP14:%.*]] = select [[TMP13]], [[VP_OP_LOAD]], [[VEC_PHI]] ; IF-EVL-NEXT: [[TMP15]] = call @llvm.vp.merge.nxv4i32( splat (i1 true), [[TMP14]], [[VEC_PHI]], i32 [[TMP9]]) ; IF-EVL-NEXT: [[TMP16:%.*]] = zext i32 [[TMP9]] to i64 ; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]] @@ -965,7 +965,7 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) { ; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0 ; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], splat (i1 true), i32 [[TMP9]]) ; IF-EVL-NEXT: [[TMP13:%.*]] = icmp ugt [[VP_OP_LOAD]], [[VEC_PHI]] -; IF-EVL-NEXT: [[TMP14:%.*]] = call @llvm.vp.select.nxv4i32( [[TMP13]], [[VP_OP_LOAD]], [[VEC_PHI]], i32 [[TMP9]]) +; IF-EVL-NEXT: [[TMP14:%.*]] = select [[TMP13]], [[VP_OP_LOAD]], [[VEC_PHI]] ; IF-EVL-NEXT: [[TMP15]] = call @llvm.vp.merge.nxv4i32( splat (i1 true), [[TMP14]], [[VEC_PHI]], i32 [[TMP9]]) ; IF-EVL-NEXT: [[TMP16:%.*]] = zext i32 [[TMP9]] to i64 ; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]] @@ -1311,7 +1311,7 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 { ; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP11]], i32 0 ; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP12]], splat (i1 true), i32 [[TMP9]]) ; IF-EVL-NEXT: [[TMP13:%.*]] = fcmp fast olt [[VP_OP_LOAD]], [[VEC_PHI]] -; IF-EVL-NEXT: [[TMP14:%.*]] = call @llvm.vp.select.nxv4f32( [[TMP13]], [[VP_OP_LOAD]], [[VEC_PHI]], i32 [[TMP9]]) +; IF-EVL-NEXT: [[TMP14:%.*]] = select [[TMP13]], [[VP_OP_LOAD]], [[VEC_PHI]] ; IF-EVL-NEXT: [[TMP15]] = call @llvm.vp.merge.nxv4f32( splat (i1 true), [[TMP14]], [[VEC_PHI]], i32 [[TMP9]]) ; IF-EVL-NEXT: [[TMP16:%.*]] = zext i32 [[TMP9]] to i64 ; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]] @@ -1432,7 +1432,7 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 { ; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP11]], i32 0 ; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP12]], splat (i1 true), i32 [[TMP9]]) ; IF-EVL-NEXT: [[TMP13:%.*]] = fcmp fast ogt [[VP_OP_LOAD]], [[VEC_PHI]] -; IF-EVL-NEXT: [[TMP14:%.*]] = call @llvm.vp.select.nxv4f32( [[TMP13]], [[VP_OP_LOAD]], [[VEC_PHI]], i32 [[TMP9]]) +; IF-EVL-NEXT: [[TMP14:%.*]] = select [[TMP13]], [[VP_OP_LOAD]], [[VEC_PHI]] ; IF-EVL-NEXT: [[TMP15]] = call @llvm.vp.merge.nxv4f32( splat (i1 true), [[TMP14]], [[VEC_PHI]], i32 [[TMP9]]) ; IF-EVL-NEXT: [[TMP16:%.*]] = zext i32 [[TMP9]] to i64 ; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll deleted file mode 100644 index 8c0e4da51f720..0000000000000 --- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll +++ /dev/null @@ -1,77 +0,0 @@ -; REQUIRES: asserts - - ; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize \ - ; RUN: -force-tail-folding-style=data-with-evl \ - ; RUN: -prefer-predicate-over-epilogue=predicate-dont-vectorize \ - ; RUN: -mtriple=riscv64 -mattr=+v -riscv-v-vector-bits-max=128 -disable-output < %s 2>&1 | FileCheck --check-prefix=IF-EVL %s - - define void @vp_select(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) { - ; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1' - ; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI - ; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' - ; IF-EVL: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI - ; - ; IF-EVL: VPlan 'Final VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' { - ; IF-EVL-NEXT: Live-in ir<[[VFUF:%.+]]> = VF * UF - ; IF-EVL-NEXT: Live-in ir<[[VTC:%.+]]> = vector-trip-count - ; IF-EVL-NEXT: Live-in ir<%N> = original trip-count - - ; IF-EVL: ir-bb: - ; IF-EVL-NEXT: Successor(s): ir-bb, ir-bb - - ; IF-EVL: ir-bb: - ; IF-EVL-NEXT: IR %0 = call i64 @llvm.vscale.i64() - ; IF-EVL-NEXT: IR %1 = mul nuw i64 %0, 4 - ; IF-EVL-NEXT: IR %2 = sub i64 %1, 1 - ; IF-EVL-NEXT: IR %n.rnd.up = add i64 %N, %2 - ; IF-EVL-NEXT: IR %n.mod.vf = urem i64 %n.rnd.up, %1 - ; IF-EVL-NEXT: IR %n.vec = sub i64 %n.rnd.up, %n.mod.vf - ; IF-EVL-NEXT: IR %3 = call i64 @llvm.vscale.i64() - ; IF-EVL-NEXT: IR %4 = mul nuw i64 %3, 4 - ; IF-EVL-NEXT: Successor(s): vector.body - ; IF-EVL-EMPTY: - ; IF-EVL-NEXT: vector.body: - ; IF-EVL-NEXT: EMIT-SCALAR vp<[[IV:%.+]]> = phi [ ir<0>, ir-bb ], [ vp<[[IV_NEXT_EXIT:%.+]]>, vector.body ] - ; IF-EVL-NEXT: EMIT-SCALAR vp<[[EVL_PHI:%.+]]> = phi [ ir<0>, ir-bb ], [ vp<[[IV_NEX:%.+]]>, vector.body ] - ; IF-EVL-NEXT: EMIT vp<[[AVL:%.+]]> = sub ir<%N>, vp<[[EVL_PHI]]> - ; IF-EVL-NEXT: EMIT vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]> - ; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[EVL_PHI]]> - ; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]> - ; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]> - ; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%c>, vp<[[EVL_PHI]]> - ; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]> - ; IF-EVL-NEXT: WIDEN ir<[[LD2:%.+]]> = vp.load vp<[[PTR2]]>, vp<[[EVL]]> - ; IF-EVL-NEXT: WIDEN ir<[[CMP:%.+]]> = icmp sgt ir<[[LD1]]>, ir<[[LD2]]> - ; IF-EVL-NEXT: WIDEN ir<[[SUB:%.+]]> = sub ir<0>, ir<[[LD2]]> - ; IF-EVL-NEXT: WIDEN-INTRINSIC vp<[[SELECT:%.+]]> = call llvm.vp.select(ir<[[CMP]]>, ir<[[LD2]]>, ir<[[SUB]]>, vp<[[EVL]]>) - ; IF-EVL-NEXT: WIDEN ir<[[ADD:%.+]]> = add vp<[[SELECT]]>, ir<[[LD1]]> - ; IF-EVL-NEXT: CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[EVL_PHI]]> - ; IF-EVL-NEXT: vp<[[PTR3:%.+]]> = vector-pointer ir<[[GEP3]]> - ; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR3]]>, ir<[[ADD]]>, vp<[[EVL]]> - ; IF-EVL-NEXT: EMIT-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64 - ; IF-EVL-NEXT: EMIT vp<[[IV_NEX]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]> - ; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT]]> = add vp<[[IV]]>, ir<[[VFUF]]> - ; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, ir<[[VTC]]> - - entry: - br label %for.body - - for.body: - %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ] - %arrayidx = getelementptr inbounds i32, ptr %b, i64 %indvars.iv - %0 = load i32, ptr %arrayidx, align 4 - %arrayidx3 = getelementptr inbounds i32, ptr %c, i64 %indvars.iv - %1 = load i32, ptr %arrayidx3, align 4 - %cmp4 = icmp sgt i32 %0, %1 - %2 = sub i32 0, %1 - %cond.p = select i1 %cmp4, i32 %1, i32 %2 - %cond = add i32 %cond.p, %0 - %arrayidx15 = getelementptr inbounds i32, ptr %a, i64 %indvars.iv - store i32 %cond, ptr %arrayidx15, align 4 - %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 - %exitcond.not = icmp eq i64 %indvars.iv.next, %N - br i1 %exitcond.not, label %exit, label %for.body - - exit: - ret void - }