From 3f0acb27e71a0414b1d52f6a6e42fe687a2118e8 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Fri, 31 May 2024 10:24:34 +0100 Subject: [PATCH 1/4] [RISCV] Match strided vector bases in RISCVGatherScatterLowering Currently we only match GEPs with a scalar base pointer, but a common pattern that's emitted from the loop vectorizer is a strided vector base plus some sort of scalar offset: %base = getelementptr i64, ptr %p, %step %gep = getelementptr i64, %base, i64 %offset This is common for accesses into a struct e.g. f[i].b below: struct F { int a; char b; }; void foo(struct F *f) { for (int i = 0; i < 1024; i += 2) { f[i].a++; f[i].b++; } } This patch handles this case in RISCVGatherScatterLowering by recursing on the base pointer if it's a vector. With this we can convert roughly 80% of the indexed loads and stores emitted to strided loads and stores on SPEC CPU 2017, -O3 -march=rva22u64_v --- .../Target/RISCV/RISCVGatherScatterLowering.cpp | 16 ++++++++++++++++ .../test/CodeGen/RISCV/rvv/strided-load-store.ll | 12 ++++-------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp index f0bd25f167d80..f7cca854d2767 100644 --- a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp @@ -349,6 +349,22 @@ RISCVGatherScatterLowering::determineBaseAndStride(Instruction *Ptr, SmallVector Ops(GEP->operands()); + // If the base pointer is a vector, check if it's strided. + if (GEP->getPointerOperand()->getType()->isVectorTy()) { + auto [BaseBase, Stride] = determineBaseAndStride( + cast(GEP->getPointerOperand()), Builder); + // If GEP's offset is scalar then we can add it to the base pointer's base. + auto IsScalar = [](Value *Idx) { return !Idx->getType()->isVectorTy(); }; + if (BaseBase && all_of(GEP->indices(), IsScalar)) { + Builder.SetInsertPoint(GEP); + SmallVector Indices(GEP->indices()); + Value *OffsetBase = + Builder.CreateGEP(GEP->getSourceElementType(), BaseBase, Indices, "", + GEP->isInBounds()); + return {OffsetBase, Stride}; + } + } + // Base pointer needs to be a scalar. Value *ScalarBase = Ops[0]; if (ScalarBase->getType()->isVectorTy()) { diff --git a/llvm/test/CodeGen/RISCV/rvv/strided-load-store.ll b/llvm/test/CodeGen/RISCV/rvv/strided-load-store.ll index 8733c5dc83d65..70412de1d0e91 100644 --- a/llvm/test/CodeGen/RISCV/rvv/strided-load-store.ll +++ b/llvm/test/CodeGen/RISCV/rvv/strided-load-store.ll @@ -301,10 +301,8 @@ define void @constant_stride( %x, ptr %p, i64 %stride) { define @vector_base_scalar_offset(ptr %p, i64 %offset) { ; CHECK-LABEL: @vector_base_scalar_offset( -; CHECK-NEXT: [[STEP:%.*]] = call @llvm.experimental.stepvector.nxv1i64() -; CHECK-NEXT: [[PTRS1:%.*]] = getelementptr i64, ptr [[P:%.*]], [[STEP]] -; CHECK-NEXT: [[PTRS2:%.*]] = getelementptr i64, [[PTRS1]], i64 [[OFFSET:%.*]] -; CHECK-NEXT: [[X:%.*]] = call @llvm.masked.gather.nxv1i64.nxv1p0( [[PTRS2]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) +; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET:%.*]] +; CHECK-NEXT: [[X:%.*]] = call @llvm.riscv.masked.strided.load.nxv1i64.p0.i64( poison, ptr [[TMP1]], i64 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) ; CHECK-NEXT: ret [[X]] ; %step = call @llvm.experimental.stepvector.nxv1i64() @@ -321,10 +319,8 @@ define @vector_base_scalar_offset(ptr %p, i64 %offset) { define @splat_base_scalar_offset(ptr %p, i64 %offset) { ; CHECK-LABEL: @splat_base_scalar_offset( -; CHECK-NEXT: [[HEAD:%.*]] = insertelement poison, ptr [[P:%.*]], i32 0 -; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector [[HEAD]], poison, zeroinitializer -; CHECK-NEXT: [[PTRS:%.*]] = getelementptr i64, [[SPLAT]], i64 [[OFFSET:%.*]] -; CHECK-NEXT: [[X:%.*]] = call @llvm.masked.gather.nxv1i64.nxv1p0( [[PTRS]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) +; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET:%.*]] +; CHECK-NEXT: [[X:%.*]] = call @llvm.riscv.masked.strided.load.nxv1i64.p0.i64( poison, ptr [[TMP1]], i64 0, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) ; CHECK-NEXT: ret [[X]] ; %head = insertelement poison, ptr %p, i32 0 From 62eb32d05e39aa4dcc3fe73d53f5addb43e4ba14 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Fri, 31 May 2024 17:34:36 +0100 Subject: [PATCH 2/4] Check that the pointer operand is an Instruction --- llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp index f7cca854d2767..84fc755c9f260 100644 --- a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp @@ -350,9 +350,10 @@ RISCVGatherScatterLowering::determineBaseAndStride(Instruction *Ptr, SmallVector Ops(GEP->operands()); // If the base pointer is a vector, check if it's strided. - if (GEP->getPointerOperand()->getType()->isVectorTy()) { - auto [BaseBase, Stride] = determineBaseAndStride( - cast(GEP->getPointerOperand()), Builder); + Value *Base = GEP->getPointerOperand(); + if (auto *BaseInst = dyn_cast(Base); + BaseInst && BaseInst->getType()->isVectorTy()) { + auto [BaseBase, Stride] = determineBaseAndStride(BaseInst, Builder); // If GEP's offset is scalar then we can add it to the base pointer's base. auto IsScalar = [](Value *Idx) { return !Idx->getType()->isVectorTy(); }; if (BaseBase && all_of(GEP->indices(), IsScalar)) { @@ -366,7 +367,7 @@ RISCVGatherScatterLowering::determineBaseAndStride(Instruction *Ptr, } // Base pointer needs to be a scalar. - Value *ScalarBase = Ops[0]; + Value *ScalarBase = Base; if (ScalarBase->getType()->isVectorTy()) { ScalarBase = getSplatValue(ScalarBase); if (!ScalarBase) From aa6a40d11f2ba95c38dd9e94c824d28f3a7c5a18 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Fri, 31 May 2024 19:07:01 +0100 Subject: [PATCH 3/4] Move down call to determineBaseAndStride to avoid generating code without passing the checks --- .../RISCV/RISCVGatherScatterLowering.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp index 84fc755c9f260..254587512f38a 100644 --- a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp @@ -353,16 +353,19 @@ RISCVGatherScatterLowering::determineBaseAndStride(Instruction *Ptr, Value *Base = GEP->getPointerOperand(); if (auto *BaseInst = dyn_cast(Base); BaseInst && BaseInst->getType()->isVectorTy()) { - auto [BaseBase, Stride] = determineBaseAndStride(BaseInst, Builder); // If GEP's offset is scalar then we can add it to the base pointer's base. auto IsScalar = [](Value *Idx) { return !Idx->getType()->isVectorTy(); }; - if (BaseBase && all_of(GEP->indices(), IsScalar)) { - Builder.SetInsertPoint(GEP); - SmallVector Indices(GEP->indices()); - Value *OffsetBase = - Builder.CreateGEP(GEP->getSourceElementType(), BaseBase, Indices, "", - GEP->isInBounds()); - return {OffsetBase, Stride}; + if (all_of(GEP->indices(), IsScalar)) { + auto [BaseBase, Stride] = determineBaseAndStride(BaseInst, Builder); + if (BaseBase) { + Builder.SetInsertPoint(GEP); + SmallVector Indices(GEP->indices()); + Value *OffsetBase = + Builder.CreateGEP(GEP->getSourceElementType(), BaseBase, Indices, + "", GEP->isInBounds()); + OffsetBase->takeName(GEP); + return {OffsetBase, Stride}; + } } } From 5303532b92c9dd2e21a92e8562f62428c1edba0d Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Mon, 3 Jun 2024 09:28:38 +0100 Subject: [PATCH 4/4] Name new GEP based off old GEP --- llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp index 254587512f38a..cff46e15251bc 100644 --- a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp @@ -362,8 +362,7 @@ RISCVGatherScatterLowering::determineBaseAndStride(Instruction *Ptr, SmallVector Indices(GEP->indices()); Value *OffsetBase = Builder.CreateGEP(GEP->getSourceElementType(), BaseBase, Indices, - "", GEP->isInBounds()); - OffsetBase->takeName(GEP); + GEP->getName() + "offset", GEP->isInBounds()); return {OffsetBase, Stride}; } }