From 0ed4b572934cc480acc81aed80884fc235cd7192 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20Gau=C3=ABr?= Date: Tue, 18 Feb 2025 13:15:57 +0100 Subject: [PATCH 1/3] [SPIR-V] Add pass to remove spv_ptrcast intrinsics OpenCL is allowed to cast pointers, meaning they can resolve some type mismatches this way. In logical SPIR-V, those are restricted. This new pass legalizes such pointer cast when targeting logical SPIR-V. For now, this pass supports 3 cases we witnessed: - loading a vec3 from a vec4*. - loading a scalar from a vec*. - loading the 1st element of an array. --- llvm/lib/Target/SPIRV/CMakeLists.txt | 1 + llvm/lib/Target/SPIRV/SPIRV.h | 1 + llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 94 ++----- llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp | 35 +++ llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h | 3 + .../Target/SPIRV/SPIRVLegalizePointerLoad.cpp | 265 ++++++++++++++++++ llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp | 2 + llvm/lib/Target/SPIRV/SPIRVUtils.cpp | 11 + llvm/lib/Target/SPIRV/SPIRVUtils.h | 10 + .../CodeGen/SPIRV/pointers/array-skips-gep.ll | 38 +++ .../pointers/getelementptr-downcast-vector.ll | 128 +++++++++ 11 files changed, 517 insertions(+), 71 deletions(-) create mode 100644 llvm/lib/Target/SPIRV/SPIRVLegalizePointerLoad.cpp create mode 100644 llvm/test/CodeGen/SPIRV/pointers/array-skips-gep.ll create mode 100644 llvm/test/CodeGen/SPIRV/pointers/getelementptr-downcast-vector.ll diff --git a/llvm/lib/Target/SPIRV/CMakeLists.txt b/llvm/lib/Target/SPIRV/CMakeLists.txt index efdd8c8d24fbd..d8322f77e0800 100644 --- a/llvm/lib/Target/SPIRV/CMakeLists.txt +++ b/llvm/lib/Target/SPIRV/CMakeLists.txt @@ -27,6 +27,7 @@ add_llvm_target(SPIRVCodeGen SPIRVInstrInfo.cpp SPIRVInstructionSelector.cpp SPIRVStripConvergentIntrinsics.cpp + SPIRVLegalizePointerLoad.cpp SPIRVMergeRegionExitTargets.cpp SPIRVISelLowering.cpp SPIRVLegalizerInfo.cpp diff --git a/llvm/lib/Target/SPIRV/SPIRV.h b/llvm/lib/Target/SPIRV/SPIRV.h index 6d00a046ff7ca..8ccb3bfc25a1a 100644 --- a/llvm/lib/Target/SPIRV/SPIRV.h +++ b/llvm/lib/Target/SPIRV/SPIRV.h @@ -23,6 +23,7 @@ ModulePass *createSPIRVPrepareFunctionsPass(const SPIRVTargetMachine &TM); FunctionPass *createSPIRVStructurizerPass(); FunctionPass *createSPIRVMergeRegionExitTargetsPass(); FunctionPass *createSPIRVStripConvergenceIntrinsicsPass(); +FunctionPass *createSPIRVLegalizePointerLoadPass(SPIRVTargetMachine *TM); FunctionPass *createSPIRVRegularizerPass(); FunctionPass *createSPIRVPreLegalizerCombiner(); FunctionPass *createSPIRVPreLegalizerPass(); diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index 5dfba8427258f..b73e85abfad26 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -57,12 +57,6 @@ void initializeSPIRVEmitIntrinsicsPass(PassRegistry &); namespace { -inline MetadataAsValue *buildMD(Value *Arg) { - LLVMContext &Ctx = Arg->getContext(); - return MetadataAsValue::get( - Ctx, MDNode::get(Ctx, ValueAsMetadata::getConstant(Arg))); -} - class SPIRVEmitIntrinsics : public ModulePass, public InstVisitor { @@ -142,23 +136,10 @@ class SPIRVEmitIntrinsics void preprocessCompositeConstants(IRBuilder<> &B); void preprocessUndefs(IRBuilder<> &B); - CallInst *buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef Types, - Value *Arg, Value *Arg2, ArrayRef Imms, - IRBuilder<> &B) { - SmallVector Args; - Args.push_back(Arg2); - Args.push_back(buildMD(Arg)); - for (auto *Imm : Imms) - Args.push_back(Imm); - return B.CreateIntrinsic(IntrID, {Types}, Args); - } - Type *reconstructType(Value *Op, bool UnknownElemTypeI8, bool IsPostprocessing); void buildAssignType(IRBuilder<> &B, Type *ElemTy, Value *Arg); - void buildAssignPtr(IRBuilder<> &B, Type *ElemTy, Value *Arg); - void updateAssignType(CallInst *AssignCI, Value *Arg, Value *OfType); void replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B); void processInstrAfterVisit(Instruction *I, IRBuilder<> &B); @@ -445,37 +426,6 @@ void SPIRVEmitIntrinsics::buildAssignType(IRBuilder<> &B, Type *Ty, GR->addAssignPtrTypeInstr(Arg, AssignCI); } -void SPIRVEmitIntrinsics::buildAssignPtr(IRBuilder<> &B, Type *ElemTy, - Value *Arg) { - ElemTy = normalizeType(ElemTy); - Value *OfType = PoisonValue::get(ElemTy); - CallInst *AssignPtrTyCI = GR->findAssignPtrTypeInstr(Arg); - if (AssignPtrTyCI == nullptr || - AssignPtrTyCI->getParent()->getParent() != CurrF) { - AssignPtrTyCI = buildIntrWithMD( - Intrinsic::spv_assign_ptr_type, {Arg->getType()}, OfType, Arg, - {B.getInt32(getPointerAddressSpace(Arg->getType()))}, B); - GR->addDeducedElementType(AssignPtrTyCI, ElemTy); - GR->addDeducedElementType(Arg, ElemTy); - GR->addAssignPtrTypeInstr(Arg, AssignPtrTyCI); - } else { - updateAssignType(AssignPtrTyCI, Arg, OfType); - } -} - -void SPIRVEmitIntrinsics::updateAssignType(CallInst *AssignCI, Value *Arg, - Value *OfType) { - AssignCI->setArgOperand(1, buildMD(OfType)); - if (cast(AssignCI)->getIntrinsicID() != - Intrinsic::spv_assign_ptr_type) - return; - - // update association with the pointee type - Type *ElemTy = normalizeType(OfType->getType()); - GR->addDeducedElementType(AssignCI, ElemTy); - GR->addDeducedElementType(Arg, ElemTy); -} - CallInst *SPIRVEmitIntrinsics::buildSpvPtrcast(Function *F, Value *Op, Type *ElemTy) { IRBuilder<> B(Op->getContext()); @@ -495,7 +445,7 @@ CallInst *SPIRVEmitIntrinsics::buildSpvPtrcast(Function *F, Value *Op, B.getInt32(getPointerAddressSpace(OpTy))}; CallInst *PtrCasted = B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args); - buildAssignPtr(B, ElemTy, PtrCasted); + GR->buildAssignPtr(B, ElemTy, PtrCasted); return PtrCasted; } @@ -1026,7 +976,8 @@ bool SPIRVEmitIntrinsics::deduceOperandElementTypeFunctionRet( continue; if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(CI)) { if (Type *PrevElemTy = GR->findDeducedElementType(CI)) { - updateAssignType(AssignCI, CI, getNormalizedPoisonValue(OpElemTy)); + GR->updateAssignType(AssignCI, CI, + getNormalizedPoisonValue(OpElemTy)); propagateElemType(CI, PrevElemTy, VisitedSubst); } } @@ -1212,7 +1163,7 @@ void SPIRVEmitIntrinsics::deduceOperandElementType( {B.getInt32(getPointerAddressSpace(OpTy))}, B); GR->addAssignPtrTypeInstr(Op, CI); } else { - updateAssignType(AssignCI, Op, OpTyVal); + GR->updateAssignType(AssignCI, Op, OpTyVal); DenseSet> VisitedSubst{ std::make_pair(I, Op)}; propagateElemTypeRec(Op, KnownElemTy, PrevElemTy, VisitedSubst); @@ -1522,7 +1473,7 @@ void SPIRVEmitIntrinsics::insertAssignPtrTypeTargetExt( // Our previous guess about the type seems to be wrong, let's update // inferred type according to a new, more precise type information. - updateAssignType(AssignCI, V, getNormalizedPoisonValue(AssignedType)); + GR->updateAssignType(AssignCI, V, getNormalizedPoisonValue(AssignedType)); } void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast( @@ -1579,7 +1530,7 @@ void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast( if (FirstPtrCastOrAssignPtrType) { // If this would be the first spv_ptrcast, do not emit spv_ptrcast and // emit spv_assign_ptr_type instead. - buildAssignPtr(B, ExpectedElementType, Pointer); + GR->buildAssignPtr(B, ExpectedElementType, Pointer); return; } else if (isTodoType(Pointer)) { eraseTodoType(Pointer); @@ -1591,10 +1542,10 @@ void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast( assert(PrevElemTy); DenseSet> VisitedSubst{ std::make_pair(I, Pointer)}; - updateAssignType(AssignCI, Pointer, ExpectedElementVal); + GR->updateAssignType(AssignCI, Pointer, ExpectedElementVal); propagateElemType(Pointer, PrevElemTy, VisitedSubst); } else { - buildAssignPtr(B, ExpectedElementType, Pointer); + GR->buildAssignPtr(B, ExpectedElementType, Pointer); } return; } @@ -1607,7 +1558,7 @@ void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast( auto *PtrCastI = B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args); I->setOperand(OperandToReplace, PtrCastI); // We need to set up a pointee type for the newly created spv_ptrcast. - buildAssignPtr(B, ExpectedElementType, PtrCastI); + GR->buildAssignPtr(B, ExpectedElementType, PtrCastI); } void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I, @@ -1923,7 +1874,7 @@ bool SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(Instruction *I, setInsertPointAfterDef(B, I); if (Type *ElemTy = deduceElementType(I, UnknownElemTypeI8)) { - buildAssignPtr(B, ElemTy, I); + GR->buildAssignPtr(B, ElemTy, I); return false; } return true; @@ -2019,10 +1970,11 @@ void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I, Type *OpTy = Op->getType(); Type *OpTyElem = getPointeeType(OpTy); if (OpTyElem) { - buildAssignPtr(B, OpTyElem, Op); + GR->buildAssignPtr(B, OpTyElem, Op); } else if (isPointerTy(OpTy)) { Type *ElemTy = GR->findDeducedElementType(Op); - buildAssignPtr(B, ElemTy ? ElemTy : deduceElementType(Op, true), Op); + GR->buildAssignPtr(B, ElemTy ? ElemTy : deduceElementType(Op, true), + Op); } else { CallInst *AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type, {OpTy}, @@ -2083,14 +2035,14 @@ void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I, if (!IsConstComposite && isPointerTy(OpTy) && (OpElemTy = GR->findDeducedElementType(Op)) != nullptr && OpElemTy != IntegerType::getInt8Ty(I->getContext())) { - buildAssignPtr(B, IntegerType::getInt8Ty(I->getContext()), NewOp); + GR->buildAssignPtr(B, IntegerType::getInt8Ty(I->getContext()), NewOp); SmallVector Types = {OpTy, OpTy}; SmallVector Args = { NewOp, buildMD(getNormalizedPoisonValue(OpElemTy)), B.getInt32(getPointerAddressSpace(OpTy))}; CallInst *PtrCasted = B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args); - buildAssignPtr(B, OpElemTy, PtrCasted); + GR->buildAssignPtr(B, OpElemTy, PtrCasted); NewOp = PtrCasted; } I->setOperand(OpNo, NewOp); @@ -2172,7 +2124,7 @@ void SPIRVEmitIntrinsics::processParamTypesByFunHeader(Function *F, continue; if (hasPointeeTypeAttr(Arg) && (ElemTy = getPointeeTypeByAttr(Arg)) != nullptr) { - buildAssignPtr(B, ElemTy, Arg); + GR->buildAssignPtr(B, ElemTy, Arg); continue; } // search in function's call sites @@ -2188,7 +2140,7 @@ void SPIRVEmitIntrinsics::processParamTypesByFunHeader(Function *F, break; } if (ElemTy) { - buildAssignPtr(B, ElemTy, Arg); + GR->buildAssignPtr(B, ElemTy, Arg); continue; } if (HaveFunPtrs) { @@ -2200,7 +2152,7 @@ void SPIRVEmitIntrinsics::processParamTypesByFunHeader(Function *F, SmallVector> Ops; deduceOperandElementTypeFunctionPointer(CI, Ops, ElemTy, false); if (ElemTy) { - buildAssignPtr(B, ElemTy, Arg); + GR->buildAssignPtr(B, ElemTy, Arg); break; } } @@ -2219,11 +2171,11 @@ void SPIRVEmitIntrinsics::processParamTypes(Function *F, IRBuilder<> &B) { if (!ElemTy && (ElemTy = deduceFunParamElementType(F, OpIdx)) != nullptr) { if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Arg)) { DenseSet> VisitedSubst; - updateAssignType(AssignCI, Arg, getNormalizedPoisonValue(ElemTy)); + GR->updateAssignType(AssignCI, Arg, getNormalizedPoisonValue(ElemTy)); propagateElemType(Arg, IntegerType::getInt8Ty(F->getContext()), VisitedSubst); } else { - buildAssignPtr(B, ElemTy, Arg); + GR->buildAssignPtr(B, ElemTy, Arg); } } } @@ -2273,7 +2225,7 @@ bool SPIRVEmitIntrinsics::processFunctionPointers(Module &M) { continue; if (II->getIntrinsicID() == Intrinsic::spv_assign_ptr_type || II->getIntrinsicID() == Intrinsic::spv_ptrcast) { - updateAssignType(II, &F, getNormalizedPoisonValue(FPElemTy)); + GR->updateAssignType(II, &F, getNormalizedPoisonValue(FPElemTy)); break; } } @@ -2324,7 +2276,7 @@ void SPIRVEmitIntrinsics::applyDemangledPtrArgTypes(IRBuilder<> &B) { if (!hasPointeeTypeAttr(Arg)) { B.SetInsertPointPastAllocas(Arg->getParent()); B.SetCurrentDebugLocation(DebugLoc()); - buildAssignPtr(B, ElemTy, Arg); + GR->buildAssignPtr(B, ElemTy, Arg); } } else if (isa(Param)) { GR->addDeducedElementType(Param, normalizeType(ElemTy)); @@ -2334,7 +2286,7 @@ void SPIRVEmitIntrinsics::applyDemangledPtrArgTypes(IRBuilder<> &B) { ->getParent() ->getEntryBlock() .getFirstNonPHIOrDbgOrAlloca()); - buildAssignPtr(B, ElemTy, Param); + GR->buildAssignPtr(B, ElemTy, Param); } CallInst *Ref = dyn_cast(Param); if (!Ref) diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp index 0ed414ebc8bbe..7cca1e3d4780c 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -22,6 +22,9 @@ #include "SPIRVUtils.h" #include "llvm/ADT/APInt.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicsSPIRV.h" #include "llvm/IR/Type.h" #include "llvm/Support/Casting.h" #include @@ -1739,3 +1742,35 @@ LLT SPIRVGlobalRegistry::getRegType(SPIRVType *SpvType) const { } return LLT::scalar(64); } + +void SPIRVGlobalRegistry::buildAssignPtr(IRBuilder<> &B, Type *ElemTy, + Value *Arg) { + Value *OfType = PoisonValue::get(ElemTy); + CallInst *AssignPtrTyCI = findAssignPtrTypeInstr(Arg); + Function *CurrF = + B.GetInsertBlock() ? B.GetInsertBlock()->getParent() : nullptr; + if (AssignPtrTyCI == nullptr || + AssignPtrTyCI->getParent()->getParent() != CurrF) { + AssignPtrTyCI = buildIntrWithMD( + Intrinsic::spv_assign_ptr_type, {Arg->getType()}, OfType, Arg, + {B.getInt32(getPointerAddressSpace(Arg->getType()))}, B); + addDeducedElementType(AssignPtrTyCI, ElemTy); + addDeducedElementType(Arg, ElemTy); + addAssignPtrTypeInstr(Arg, AssignPtrTyCI); + } else { + updateAssignType(AssignPtrTyCI, Arg, OfType); + } +} + +void SPIRVGlobalRegistry::updateAssignType(CallInst *AssignCI, Value *Arg, + Value *OfType) { + AssignCI->setArgOperand(1, buildMD(OfType)); + if (cast(AssignCI)->getIntrinsicID() != + Intrinsic::spv_assign_ptr_type) + return; + + // update association with the pointee type + Type *ElemTy = OfType->getType(); + addDeducedElementType(AssignCI, ElemTy); + addDeducedElementType(Arg, ElemTy); +} diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h index 2c24ba79ea8e6..fc9dd297b1994 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h @@ -620,6 +620,9 @@ class SPIRVGlobalRegistry { const TargetRegisterClass *getRegClass(SPIRVType *SpvType) const; LLT getRegType(SPIRVType *SpvType) const; + + void buildAssignPtr(IRBuilder<> &B, Type *ElemTy, Value *Arg); + void updateAssignType(CallInst *AssignCI, Value *Arg, Value *OfType); }; } // end namespace llvm #endif // LLLVM_LIB_TARGET_SPIRV_SPIRVTYPEMANAGER_H diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizePointerLoad.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizePointerLoad.cpp new file mode 100644 index 0000000000000..76a8d690372a9 --- /dev/null +++ b/llvm/lib/Target/SPIRV/SPIRVLegalizePointerLoad.cpp @@ -0,0 +1,265 @@ +//===-- SPIRVLegalizePointerLoad.cpp ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// The LLVM IR has multiple legal patterns we cannot lower to Logical SPIR-V. +// This pass modifies such loads to have an IR we can directly lower to valid +// logical SPIR-V. +// OpenCL can avoid this because they rely on ptrcast, which is not supported +// by logical SPIR-V. +// +// This pass relies on the assign_ptr_type intrinsic to deduce the type of the +// pointed values, must replace all occurences of `ptrcast`. This is why +// unhandled cases are reported as unreachable: we MUST cover all cases. +// +// 1. Loading the first element of an array +// +// %array = [10 x i32] +// %value = load i32, ptr %array +// +// LLVM can skip the GEP instruction, and only request loading the first 4 +// bytes. In logical SPIR-V, we need an OpAccessChain to access the first +// element. This pass will add a getelementptr instruction before the load. +// +// +// 2. Implicit downcast from load +// +// %1 = getelementptr <4 x i32>, ptr %vec4, i64 0 +// %2 = load <3 x i32>, ptr %1 +// +// The pointer in the GEP instruction is only used for offset computations, +// but it doesn't NEED to match the pointed type. OpAccessChain however +// requires this. Also, LLVM loads define the bitwidth of the load, not the +// pointer. In this example, we can guess %vec4 is a vec4 thanks to the GEP +// instruction basetype, but we only want to load the first 3 elements, hence +// do a partial load. In logical SPIR-V, this is not legal. What we must do +// is load the full vector (basetype), extract 3 elements, and recombine them +// to form a 3-element vector. +// +//===----------------------------------------------------------------------===// + +#include "SPIRV.h" +#include "SPIRVSubtarget.h" +#include "SPIRVTargetMachine.h" +#include "SPIRVUtils.h" +#include "llvm/CodeGen/IntrinsicLowering.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicsSPIRV.h" +#include "llvm/Transforms/Utils/Cloning.h" +#include "llvm/Transforms/Utils/LowerMemIntrinsics.h" + +using namespace llvm; + +namespace llvm { +void initializeSPIRVLegalizePointerLoadPass(PassRegistry &); +} + +class SPIRVLegalizePointerLoad : public FunctionPass { + + // Replace all uses of a |Old| with |New| updates the global registry type + // mappings. + void replaceAllUsesWith(Value *Old, Value *New) { + Old->replaceAllUsesWith(New); + GR->updateIfExistDeducedElementType(Old, New, /* deleteOld = */ true); + GR->updateIfExistAssignPtrTypeInstr(Old, New, /* deleteOld = */ true); + } + + // Builds the `spv_assign_type` assigning |Ty| to |Value| at the current + // builder position. + void buildAssignType(IRBuilder<> &B, Type *Ty, Value *Arg) { + Value *OfType = PoisonValue::get(Ty); + CallInst *AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type, + {Arg->getType()}, OfType, Arg, {}, B); + GR->addAssignPtrTypeInstr(Arg, AssignCI); + } + + // Loads a single scalar of type |To| from the vector pointed by |Source| of + // the type |From|. Returns the loaded value. + Value *loadScalarFromVector(IRBuilder<> &B, Value *Source, + FixedVectorType *From) { + + LoadInst *NewLoad = B.CreateLoad(From, Source); + buildAssignType(B, From, NewLoad); + + SmallVector Args = {NewLoad, B.getInt64(0)}; + SmallVector Types = {From->getElementType(), Args[0]->getType(), + Args[1]->getType()}; + Value *Extracted = + B.CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args}); + buildAssignType(B, Extracted->getType(), Extracted); + return Extracted; + } + + // Loads parts of the vector of type |From| from the pointer |Source| and + // create a new vector of type |To|. |To| must be a vector type, and element + // types of |To| and |From| must match. Returns the loaded value. + Value *loadVectorFromVector(IRBuilder<> &B, FixedVectorType *From, + FixedVectorType *To, Value *Source) { + // We expect the codegen to avoid doing implicit bitcast from a load. + assert(To->getElementType() == From->getElementType()); + assert(To->getNumElements() < From->getNumElements()); + + LoadInst *NewLoad = B.CreateLoad(From, Source); + buildAssignType(B, From, NewLoad); + + auto ConstInt = ConstantInt::get(IntegerType::get(B.getContext(), 32), 0); + ElementCount VecElemCount = ElementCount::getFixed(To->getNumElements()); + Value *Output = ConstantVector::getSplat(VecElemCount, ConstInt); + for (unsigned I = 0; I < To->getNumElements(); ++I) { + Value *Extracted = nullptr; + { + SmallVector Args = {NewLoad, B.getInt64(I)}; + SmallVector Types = {To->getElementType(), + Args[0]->getType(), Args[1]->getType()}; + Extracted = + B.CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args}); + buildAssignType(B, Extracted->getType(), Extracted); + } + assert(Extracted != nullptr); + + { + SmallVector Args = {Output, Extracted, B.getInt64(I)}; + SmallVector Types = {Args[0]->getType(), Args[0]->getType(), + Args[1]->getType(), Args[2]->getType()}; + Output = B.CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args}); + buildAssignType(B, Output->getType(), Output); + } + } + return Output; + } + + // Loads the first value in an array pointed by |Source| of type |From|. Load + // flags will be copied from |BadLoad|, which should be the illegal load being + // legalized. Returns the loaded value. + Value *loadFirstValueFromArray(IRBuilder<> &B, ArrayType *From, Value *Source, + LoadInst *BadLoad) { + SmallVector Types = {BadLoad->getPointerOperandType(), + BadLoad->getPointerOperandType()}; + SmallVector Args{/* isInBounds= */ B.getInt1(true), Source, + B.getInt64(0), B.getInt64(0)}; + auto *GEP = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args}); + GR->buildAssignPtr(B, From->getElementType(), GEP); + + const auto *TLI = TM->getSubtargetImpl()->getTargetLowering(); + MachineMemOperand::Flags Flags = TLI->getLoadMemOperandFlags( + *BadLoad, BadLoad->getFunction()->getDataLayout()); + Instruction *LI = B.CreateIntrinsic( + Intrinsic::spv_load, {BadLoad->getOperand(0)->getType()}, + {GEP, B.getInt16(Flags), B.getInt8(BadLoad->getAlign().value())}); + buildAssignType(B, From->getElementType(), LI); + return LI; + } + + // Transforms an illegal partial load into a sequence we can lower to logical + // SPIR-V. + void transformLoad(IRBuilder<> &B, LoadInst *LI, Value *CastedOperand, + Value *OriginalOperand) { + Type *FromTy = GR->findDeducedElementType(OriginalOperand); + Type *ToTy /*-fruity*/ = GR->findDeducedElementType(CastedOperand); + Value *Output = nullptr; + + auto *SAT = dyn_cast(FromTy); + auto *SVT = dyn_cast(FromTy); + auto *DVT = dyn_cast(ToTy); + + B.SetInsertPoint(LI); + + // Destination is the element type of Source, and source is an array -> + // Loading 1st element. + // - float a = array[0]; + if (SAT && SAT->getElementType() == ToTy) + Output = loadFirstValueFromArray(B, SAT, OriginalOperand, LI); + // Destination is the element type of Source, and source is a vector -> + // Vector to scalar. + // - float a = vector.x; + else if (!DVT && SVT && SVT->getElementType() == ToTy) { + Output = loadScalarFromVector(B, OriginalOperand, SVT); + } + // Destination is a smaller vector than source. + // - float3 v3 = vector4; + else if (SVT && DVT) + Output = loadVectorFromVector(B, SVT, DVT, OriginalOperand); + else + llvm_unreachable("Unimplemented implicit down-cast from load."); + + replaceAllUsesWith(LI, Output); + DeadInstructions.push_back(LI); + } + + void legalizePointerCast(IntrinsicInst *II) { + Value *CastedOperand = II; + Value *OriginalOperand = II->getOperand(0); + + IRBuilder<> B(II->getContext()); + std::vector Users; + for (Use &U : II->uses()) + Users.push_back(U.getUser()); + + for (Value *User : Users) { + if (LoadInst *LI = dyn_cast(User)) { + transformLoad(B, LI, CastedOperand, OriginalOperand); + continue; + } + + IntrinsicInst *Intrin = dyn_cast(User); + if (Intrin->getIntrinsicID() == Intrinsic::spv_assign_ptr_type) { + DeadInstructions.push_back(Intrin); + continue; + } + + llvm_unreachable("Unsupported ptrcast user. Please fix."); + } + + DeadInstructions.push_back(II); + } + +public: + SPIRVLegalizePointerLoad(SPIRVTargetMachine *TM) : FunctionPass(ID), TM(TM) { + initializeSPIRVLegalizePointerLoadPass(*PassRegistry::getPassRegistry()); + }; + + virtual bool runOnFunction(Function &F) override { + const SPIRVSubtarget &ST = TM->getSubtarget(F); + GR = ST.getSPIRVGlobalRegistry(); + DeadInstructions.clear(); + + std::vector WorkList; + for (auto &BB : F) { + for (auto &I : BB) { + auto *II = dyn_cast(&I); + if (II && II->getIntrinsicID() == Intrinsic::spv_ptrcast) + WorkList.push_back(II); + } + } + + for (IntrinsicInst *II : WorkList) + legalizePointerCast(II); + + for (Instruction *I : DeadInstructions) + I->eraseFromParent(); + + return DeadInstructions.size() != 0; + } + +private: + SPIRVTargetMachine *TM = nullptr; + SPIRVGlobalRegistry *GR = nullptr; + std::vector DeadInstructions; + +public: + static char ID; +}; + +char SPIRVLegalizePointerLoad::ID = 0; +INITIALIZE_PASS(SPIRVLegalizePointerLoad, "spirv-legalize-bitcast", + "SPIRV legalize bitcast pass", false, false) + +FunctionPass *llvm::createSPIRVLegalizePointerLoadPass(SPIRVTargetMachine *TM) { + return new SPIRVLegalizePointerLoad(TM); +} diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp index 098c7a6fba50e..2a286112df936 100644 --- a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp @@ -209,6 +209,8 @@ void SPIRVPassConfig::addIRPasses() { void SPIRVPassConfig::addISelPrepare() { addPass(createSPIRVEmitIntrinsicsPass(&getTM())); + if (TM.getSubtargetImpl()->isVulkanEnv()) + addPass(createSPIRVLegalizePointerLoadPass(&getTM())); TargetPassConfig::addISelPrepare(); } diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp index c55b735314228..fcfa022a062b2 100644 --- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp @@ -775,6 +775,17 @@ Register createVirtualRegister( MIRBuilder); } +CallInst *buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef Types, + Value *Arg, Value *Arg2, ArrayRef Imms, + IRBuilder<> &B) { + SmallVector Args; + Args.push_back(Arg2); + Args.push_back(buildMD(Arg)); + for (auto *Imm : Imms) + Args.push_back(Imm); + return B.CreateIntrinsic(IntrID, {Types}, Args); +} + // Return true if there is an opaque pointer type nested in the argument. bool isNestedPointer(const Type *Ty) { if (Ty->isPtrOrPtrVectorTy()) diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.h b/llvm/lib/Target/SPIRV/SPIRVUtils.h index 870649879218a..0817a01d0f0e0 100644 --- a/llvm/lib/Target/SPIRV/SPIRVUtils.h +++ b/llvm/lib/Target/SPIRV/SPIRVUtils.h @@ -405,6 +405,16 @@ inline PoisonValue *getNormalizedPoisonValue(Type *Ty) { return PoisonValue::get(normalizeType(Ty)); } +inline MetadataAsValue *buildMD(Value *Arg) { + LLVMContext &Ctx = Arg->getContext(); + return MetadataAsValue::get( + Ctx, MDNode::get(Ctx, ValueAsMetadata::getConstant(Arg))); +} + +CallInst *buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef Types, + Value *Arg, Value *Arg2, ArrayRef Imms, + IRBuilder<> &B); + MachineInstr *getVRegDef(MachineRegisterInfo &MRI, Register Reg); #define SPIRV_BACKEND_SERVICE_FUN_NAME "__spirv_backend_service_fun" diff --git a/llvm/test/CodeGen/SPIRV/pointers/array-skips-gep.ll b/llvm/test/CodeGen/SPIRV/pointers/array-skips-gep.ll new file mode 100644 index 0000000000000..e91e25afd8ec7 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/pointers/array-skips-gep.ll @@ -0,0 +1,38 @@ +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %} + +; CHECK-DAG: %[[#int_ty:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#long_ty:]] = OpTypeInt 64 0 +; CHECK-DAG: %[[#int_10:]] = OpConstant %[[#int_ty]] 10 +; CHECK-DAG: %[[#long_0:]] = OpConstantNull %[[#long_ty]] +; CHECK-DAG: %[[#array_ty:]] = OpTypeArray %[[#int_ty]] %[[#int_10]] +; CHECK-DAG: %[[#array_fp:]] = OpTypePointer Function %[[#array_ty]] +; CHECK-DAG: %[[#array_pp:]] = OpTypePointer Private %[[#array_ty]] +; CHECK-DAG: %[[#int_fp:]] = OpTypePointer Function %[[#int_ty]] +; CHECK-DAG: %[[#int_pp:]] = OpTypePointer Private %[[#int_ty]] + +@gv = external addrspace(10) global [10 x i32] +; CHECK: %[[#gv:]] = OpVariable %[[#array_pp]] Private + +define internal spir_func i32 @foo() { + %array = alloca [10 x i32], align 4 +; CHECK: %[[#array:]] = OpVariable %[[#array_fp:]] Function + + ; Direct load from the pointer index. This requires an OpAccessChain + %1 = load i32, ptr %array, align 4 +; CHECK: %[[#ptr:]] = OpInBoundsAccessChain %[[#int_fp]] %[[#array]] %[[#long_0]] +; CHECK: %[[#val:]] = OpLoad %[[#int_ty]] %[[#ptr]] Aligned 4 + + ret i32 %1 +; CHECK: OpReturnValue %[[#val]] +} + +define internal spir_func i32 @bar() { + ; Direct load from the pointer index. This requires an OpAccessChain + %1 = load i32, ptr addrspace(10) @gv +; CHECK: %[[#ptr:]] = OpInBoundsAccessChain %[[#int_pp]] %[[#gv]] %[[#long_0]] +; CHECK: %[[#val:]] = OpLoad %[[#int_ty]] %[[#ptr]] Aligned 4 + + ret i32 %1 +; CHECK: OpReturnValue %[[#val]] +} diff --git a/llvm/test/CodeGen/SPIRV/pointers/getelementptr-downcast-vector.ll b/llvm/test/CodeGen/SPIRV/pointers/getelementptr-downcast-vector.ll new file mode 100644 index 0000000000000..5bcec1d870651 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/pointers/getelementptr-downcast-vector.ll @@ -0,0 +1,128 @@ +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %} + +; CHECK-DAG: %[[#uint:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#uint_0:]] = OpConstant %[[#uint]] 0 +; CHECK-DAG: %[[#v2:]] = OpTypeVector %[[#uint]] 2 +; CHECK-DAG: %[[#v3:]] = OpTypeVector %[[#uint]] 3 +; CHECK-DAG: %[[#v4:]] = OpTypeVector %[[#uint]] 4 +; CHECK-DAG: %[[#v4_pp:]] = OpTypePointer Private %[[#v4]] +; CHECK-DAG: %[[#v4_fp:]] = OpTypePointer Function %[[#v4]] +; CHECK-DAG: %[[#v3_000:]] = OpConstantComposite %[[#v3]] %[[#uint_0]] %[[#uint_0]] %[[#uint_0]] +; CHECK-DAG: %[[#v2_00:]] = OpConstantComposite %[[#v2]] %[[#uint_0]] %[[#uint_0]] + +define internal spir_func <3 x i32> @foo(ptr addrspace(10) %a) { + + %1 = getelementptr inbounds <4 x i32>, ptr addrspace(10) %a, i64 0 +; CHECK: %[[#tmp:]] = OpInBoundsAccessChain %[[#v4_pp]] %[[#]] + + ; partial loading of a vector: v4 -> v3. + %2 = load <3 x i32>, ptr addrspace(10) %1, align 16 +; CHECK: %[[#load:]] = OpLoad %[[#v4]] %[[#tmp]] Aligned 16 +; CHECK-DAG: %[[#x:]] = OpCompositeExtract %[[#uint]] %[[#load]] 0 +; CHECK-DAG: %[[#y:]] = OpCompositeExtract %[[#uint]] %[[#load]] 1 +; CHECK-DAG: %[[#z:]] = OpCompositeExtract %[[#uint]] %[[#load]] 2 +; CHECK-DAG: %[[#tmp0:]] = OpCompositeInsert %[[#v3]] %[[#x]] %[[#v3_000]] 0 +; CHECK-DAG: %[[#tmp1:]] = OpCompositeInsert %[[#v3]] %[[#y]] %[[#tmp0]] 1 +; CHECK-DAG: %[[#tmp2:]] = OpCompositeInsert %[[#v3]] %[[#z]] %[[#tmp1]] 2 + + ret <3 x i32> %2 +; CHECK: OpReturnValue %[[#tmp2]] +} + +define internal spir_func <3 x i32> @fooDefault(ptr %a) { + + %1 = getelementptr inbounds <4 x i32>, ptr %a, i64 0 +; CHECK: %[[#tmp:]] = OpInBoundsAccessChain %[[#v4_fp]] %[[#]] + + ; partial loading of a vector: v4 -> v3. + %2 = load <3 x i32>, ptr %1, align 16 +; CHECK: %[[#load:]] = OpLoad %[[#v4]] %[[#tmp]] Aligned 16 +; CHECK-DAG: %[[#x:]] = OpCompositeExtract %[[#uint]] %[[#load]] 0 +; CHECK-DAG: %[[#y:]] = OpCompositeExtract %[[#uint]] %[[#load]] 1 +; CHECK-DAG: %[[#z:]] = OpCompositeExtract %[[#uint]] %[[#load]] 2 +; CHECK-DAG: %[[#tmp0:]] = OpCompositeInsert %[[#v3]] %[[#x]] %[[#v3_000]] 0 +; CHECK-DAG: %[[#tmp1:]] = OpCompositeInsert %[[#v3]] %[[#y]] %[[#tmp0]] 1 +; CHECK-DAG: %[[#tmp2:]] = OpCompositeInsert %[[#v3]] %[[#z]] %[[#tmp1]] 2 + + ret <3 x i32> %2 +; CHECK: OpReturnValue %[[#tmp2]] +} + +define internal spir_func <3 x i32> @fooBounds(ptr %a) { + + %1 = getelementptr <4 x i32>, ptr %a, i64 0 +; CHECK: %[[#tmp:]] = OpAccessChain %[[#v4_fp]] %[[#]] + + ; partial loading of a vector: v4 -> v3. + %2 = load <3 x i32>, ptr %1, align 16 +; CHECK: %[[#load:]] = OpLoad %[[#v4]] %[[#tmp]] Aligned 16 +; CHECK-DAG: %[[#x:]] = OpCompositeExtract %[[#uint]] %[[#load]] 0 +; CHECK-DAG: %[[#y:]] = OpCompositeExtract %[[#uint]] %[[#load]] 1 +; CHECK-DAG: %[[#z:]] = OpCompositeExtract %[[#uint]] %[[#load]] 2 +; CHECK-DAG: %[[#tmp0:]] = OpCompositeInsert %[[#v3]] %[[#x]] %[[#v3_000]] 0 +; CHECK-DAG: %[[#tmp1:]] = OpCompositeInsert %[[#v3]] %[[#y]] %[[#tmp0]] 1 +; CHECK-DAG: %[[#tmp2:]] = OpCompositeInsert %[[#v3]] %[[#z]] %[[#tmp1]] 2 + + ret <3 x i32> %2 +; CHECK: OpReturnValue %[[#tmp2]] +} + +define internal spir_func <2 x i32> @bar(ptr addrspace(10) %a) { + + %1 = getelementptr inbounds <4 x i32>, ptr addrspace(10) %a, i64 0 +; CHECK: %[[#tmp:]] = OpInBoundsAccessChain %[[#v4_pp]] %[[#]] + + ; partial loading of a vector: v4 -> v2. + %2 = load <2 x i32>, ptr addrspace(10) %1, align 16 +; CHECK: %[[#load:]] = OpLoad %[[#v4]] %[[#tmp]] Aligned 16 +; CHECK-DAG: %[[#x:]] = OpCompositeExtract %[[#uint]] %[[#load]] 0 +; CHECK-DAG: %[[#y:]] = OpCompositeExtract %[[#uint]] %[[#load]] 1 +; CHECK-DAG: %[[#tmp0:]] = OpCompositeInsert %[[#v2]] %[[#x]] %[[#v2_00]] 0 +; CHECK-DAG: %[[#tmp1:]] = OpCompositeInsert %[[#v2]] %[[#y]] %[[#tmp0]] 1 + + ret <2 x i32> %2 +; CHECK: OpReturnValue %[[#tmp1]] +} + +define internal spir_func i32 @baz(ptr addrspace(10) %a) { + + %1 = getelementptr inbounds <4 x i32>, ptr addrspace(10) %a, i64 0 +; CHECK: %[[#tmp:]] = OpInBoundsAccessChain %[[#v4_pp]] %[[#]] + + ; Loading of the first scalar of a vector: v4 -> int. + %2 = load i32, ptr addrspace(10) %1, align 16 +; CHECK: %[[#load:]] = OpLoad %[[#v4]] %[[#tmp]] Aligned 16 +; CHECK-DAG: %[[#x:]] = OpCompositeExtract %[[#uint]] %[[#load]] 0 + + ret i32 %2 +; CHECK: OpReturnValue %[[#x]] +} + +define internal spir_func i32 @bazDefault(ptr %a) { + + %1 = getelementptr inbounds <4 x i32>, ptr %a, i64 0 +; CHECK: %[[#tmp:]] = OpInBoundsAccessChain %[[#v4_fp]] %[[#]] + + ; Loading of the first scalar of a vector: v4 -> int. + %2 = load i32, ptr %1, align 16 +; CHECK: %[[#load:]] = OpLoad %[[#v4]] %[[#tmp]] Aligned 16 +; CHECK-DAG: %[[#x:]] = OpCompositeExtract %[[#uint]] %[[#load]] 0 + + ret i32 %2 +; CHECK: OpReturnValue %[[#x]] +} + +define internal spir_func i32 @bazBounds(ptr %a) { + + %1 = getelementptr <4 x i32>, ptr %a, i64 0 +; CHECK: %[[#tmp:]] = OpAccessChain %[[#v4_fp]] %[[#]] + + ; Loading of the first scalar of a vector: v4 -> int. + %2 = load i32, ptr %1, align 16 +; CHECK: %[[#load:]] = OpLoad %[[#v4]] %[[#tmp]] Aligned 16 +; CHECK-DAG: %[[#x:]] = OpCompositeExtract %[[#uint]] %[[#load]] 0 + + ret i32 %2 +; CHECK: OpReturnValue %[[#x]] +} From cb95680692e3596fffa0a2fac4dcd2d42f544b5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20Gau=C3=ABr?= Date: Thu, 27 Feb 2025 17:20:24 +0100 Subject: [PATCH 2/3] Update llvm/lib/Target/SPIRV/SPIRVLegalizePointerLoad.cpp Co-authored-by: Steven Perron --- llvm/lib/Target/SPIRV/SPIRVLegalizePointerLoad.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizePointerLoad.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizePointerLoad.cpp index 76a8d690372a9..69951f462d7b4 100644 --- a/llvm/lib/Target/SPIRV/SPIRVLegalizePointerLoad.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVLegalizePointerLoad.cpp @@ -13,8 +13,8 @@ // by logical SPIR-V. // // This pass relies on the assign_ptr_type intrinsic to deduce the type of the -// pointed values, must replace all occurences of `ptrcast`. This is why -// unhandled cases are reported as unreachable: we MUST cover all cases. +// pointee. All occurrences of `ptrcast` must be replaced because the lead to +// invalid SPIR-V. Unhandled cases result in an error. // // 1. Loading the first element of an array // From 9b88fdf6878c419ebd869dc28eb01e4e350b6f04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20Gau=C3=ABr?= Date: Fri, 28 Feb 2025 16:43:04 +0100 Subject: [PATCH 3/3] pr-feedback: move to GR & alternative lowering Moved more EmitIntrinsic function to GR, and changed the lowering to use shuffle and reuse array load logic. --- llvm/lib/Target/SPIRV/CMakeLists.txt | 2 +- llvm/lib/Target/SPIRV/SPIRV.h | 2 +- llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 47 +------ llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp | 43 ++++++ llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h | 5 + ...rLoad.cpp => SPIRVLegalizePointerCast.cpp} | 125 ++++++------------ llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp | 2 +- .../CodeGen/SPIRV/pointers/array-skips-gep.ll | 21 ++- .../pointers/getelementptr-downcast-vector.ll | 70 ++++------ 9 files changed, 134 insertions(+), 183 deletions(-) rename llvm/lib/Target/SPIRV/{SPIRVLegalizePointerLoad.cpp => SPIRVLegalizePointerCast.cpp} (58%) diff --git a/llvm/lib/Target/SPIRV/CMakeLists.txt b/llvm/lib/Target/SPIRV/CMakeLists.txt index d8322f77e0800..4a2b534b948d6 100644 --- a/llvm/lib/Target/SPIRV/CMakeLists.txt +++ b/llvm/lib/Target/SPIRV/CMakeLists.txt @@ -27,7 +27,7 @@ add_llvm_target(SPIRVCodeGen SPIRVInstrInfo.cpp SPIRVInstructionSelector.cpp SPIRVStripConvergentIntrinsics.cpp - SPIRVLegalizePointerLoad.cpp + SPIRVLegalizePointerCast.cpp SPIRVMergeRegionExitTargets.cpp SPIRVISelLowering.cpp SPIRVLegalizerInfo.cpp diff --git a/llvm/lib/Target/SPIRV/SPIRV.h b/llvm/lib/Target/SPIRV/SPIRV.h index 8ccb3bfc25a1a..d765dfe370be2 100644 --- a/llvm/lib/Target/SPIRV/SPIRV.h +++ b/llvm/lib/Target/SPIRV/SPIRV.h @@ -23,7 +23,7 @@ ModulePass *createSPIRVPrepareFunctionsPass(const SPIRVTargetMachine &TM); FunctionPass *createSPIRVStructurizerPass(); FunctionPass *createSPIRVMergeRegionExitTargetsPass(); FunctionPass *createSPIRVStripConvergenceIntrinsicsPass(); -FunctionPass *createSPIRVLegalizePointerLoadPass(SPIRVTargetMachine *TM); +FunctionPass *createSPIRVLegalizePointerCastPass(SPIRVTargetMachine *TM); FunctionPass *createSPIRVRegularizerPass(); FunctionPass *createSPIRVPreLegalizerCombiner(); FunctionPass *createSPIRVPreLegalizerPass(); diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index b73e85abfad26..506bdd2875f48 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -139,8 +139,6 @@ class SPIRVEmitIntrinsics Type *reconstructType(Value *Op, bool UnknownElemTypeI8, bool IsPostprocessing); - void buildAssignType(IRBuilder<> &B, Type *ElemTy, Value *Arg); - void replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B); void processInstrAfterVisit(Instruction *I, IRBuilder<> &B); bool insertAssignPtrTypeIntrs(Instruction *I, IRBuilder<> &B, @@ -254,18 +252,6 @@ bool expectIgnoredInIRTranslation(const Instruction *I) { } } -bool allowEmitFakeUse(const Value *Arg) { - if (isSpvIntrinsic(Arg)) - return false; - if (dyn_cast(Arg) || dyn_cast(Arg) || - dyn_cast(Arg)) - return false; - if (const auto *LI = dyn_cast(Arg)) - if (LI->getType()->isAggregateType()) - return false; - return true; -} - } // namespace char SPIRVEmitIntrinsics::ID = 0; @@ -336,10 +322,7 @@ static void emitAssignName(Instruction *I, IRBuilder<> &B) { void SPIRVEmitIntrinsics::replaceAllUsesWith(Value *Src, Value *Dest, bool DeleteOld) { - Src->replaceAllUsesWith(Dest); - // Update deduced type records - GR->updateIfExistDeducedElementType(Src, Dest, DeleteOld); - GR->updateIfExistAssignPtrTypeInstr(Src, Dest, DeleteOld); + GR->replaceAllUsesWith(Src, Dest, DeleteOld); // Update uncomplete type records if any if (isTodoType(Src)) { if (DeleteOld) @@ -406,26 +389,6 @@ Type *SPIRVEmitIntrinsics::reconstructType(Value *Op, bool UnknownElemTypeI8, return nullptr; } -void SPIRVEmitIntrinsics::buildAssignType(IRBuilder<> &B, Type *Ty, - Value *Arg) { - Value *OfType = getNormalizedPoisonValue(Ty); - CallInst *AssignCI = nullptr; - if (Arg->getType()->isAggregateType() && Ty->isAggregateType() && - allowEmitFakeUse(Arg)) { - LLVMContext &Ctx = Arg->getContext(); - SmallVector ArgMDs{ - MDNode::get(Ctx, ValueAsMetadata::getConstant(OfType)), - MDString::get(Ctx, Arg->getName())}; - B.CreateIntrinsic(Intrinsic::spv_value_md, {}, - {MetadataAsValue::get(Ctx, MDTuple::get(Ctx, ArgMDs))}); - AssignCI = B.CreateIntrinsic(Intrinsic::fake_use, {}, {Arg}); - } else { - AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type, {Arg->getType()}, - OfType, Arg, {}, B); - } - GR->addAssignPtrTypeInstr(Arg, AssignCI); -} - CallInst *SPIRVEmitIntrinsics::buildSpvPtrcast(Function *F, Value *Op, Type *ElemTy) { IRBuilder<> B(Op->getContext()); @@ -1453,7 +1416,7 @@ void SPIRVEmitIntrinsics::insertAssignPtrTypeTargetExt( CallInst *AssignCI = GR->findAssignPtrTypeInstr(V); if (!AssignCI) { - buildAssignType(B, AssignedType, V); + GR->buildAssignType(B, AssignedType, V); return; } @@ -1907,8 +1870,8 @@ void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I, setInsertPointAfterDef(B, I); switch (ResIt->second) { case WellKnownTypes::Event: - buildAssignType(B, TargetExtType::get(I->getContext(), "spirv.Event"), - I); + GR->buildAssignType( + B, TargetExtType::get(I->getContext(), "spirv.Event"), I); break; } } @@ -1953,7 +1916,7 @@ void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I, } } TypeToAssign = restoreMutatedType(GR, I, TypeToAssign); - buildAssignType(B, TypeToAssign, I); + GR->buildAssignType(B, TypeToAssign, I); } for (const auto &Op : I->operands()) { if (isa(Op) || isa(Op) || diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp index 7cca1e3d4780c..e9b517d3eeaaa 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -32,6 +32,20 @@ using namespace llvm; +namespace { + +bool allowEmitFakeUse(const Value *Arg) { + if (isSpvIntrinsic(Arg)) + return false; + if (dyn_cast(Arg) || dyn_cast(Arg) || + dyn_cast(Arg)) + return false; + if (const auto *LI = dyn_cast(Arg)) + if (LI->getType()->isAggregateType()) + return false; + return true; +} + inline unsigned typeToAddressSpace(const Type *Ty) { if (auto PType = dyn_cast(Ty)) return PType->getAddressSpace(); @@ -43,6 +57,8 @@ inline unsigned typeToAddressSpace(const Type *Ty) { report_fatal_error("Unable to convert LLVM type to SPIRVType", true); } +} // anonymous namespace + SPIRVGlobalRegistry::SPIRVGlobalRegistry(unsigned PointerSize) : PointerSize(PointerSize), Bound(0) {} @@ -1743,6 +1759,33 @@ LLT SPIRVGlobalRegistry::getRegType(SPIRVType *SpvType) const { return LLT::scalar(64); } +void SPIRVGlobalRegistry::replaceAllUsesWith(Value *Old, Value *New, + bool DeleteOld) { + Old->replaceAllUsesWith(New); + updateIfExistDeducedElementType(Old, New, DeleteOld); + updateIfExistAssignPtrTypeInstr(Old, New, DeleteOld); +} + +void SPIRVGlobalRegistry::buildAssignType(IRBuilder<> &B, Type *Ty, + Value *Arg) { + Value *OfType = getNormalizedPoisonValue(Ty); + CallInst *AssignCI = nullptr; + if (Arg->getType()->isAggregateType() && Ty->isAggregateType() && + allowEmitFakeUse(Arg)) { + LLVMContext &Ctx = Arg->getContext(); + SmallVector ArgMDs{ + MDNode::get(Ctx, ValueAsMetadata::getConstant(OfType)), + MDString::get(Ctx, Arg->getName())}; + B.CreateIntrinsic(Intrinsic::spv_value_md, {}, + {MetadataAsValue::get(Ctx, MDTuple::get(Ctx, ArgMDs))}); + AssignCI = B.CreateIntrinsic(Intrinsic::fake_use, {}, {Arg}); + } else { + AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type, {Arg->getType()}, + OfType, Arg, {}, B); + } + addAssignPtrTypeInstr(Arg, AssignCI); +} + void SPIRVGlobalRegistry::buildAssignPtr(IRBuilder<> &B, Type *ElemTy, Value *Arg) { Value *OfType = PoisonValue::get(ElemTy); diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h index fc9dd297b1994..778232e7013d4 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h @@ -621,6 +621,11 @@ class SPIRVGlobalRegistry { const TargetRegisterClass *getRegClass(SPIRVType *SpvType) const; LLT getRegType(SPIRVType *SpvType) const; + // Replace all uses of a |Old| with |New| updates the global registry type + // mappings. + void replaceAllUsesWith(Value *Old, Value *New, bool DeleteOld = true); + + void buildAssignType(IRBuilder<> &B, Type *Ty, Value *Arg); void buildAssignPtr(IRBuilder<> &B, Type *ElemTy, Value *Arg); void updateAssignType(CallInst *AssignCI, Value *Arg, Value *OfType); }; diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizePointerLoad.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp similarity index 58% rename from llvm/lib/Target/SPIRV/SPIRVLegalizePointerLoad.cpp rename to llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp index 69951f462d7b4..ec419d25cd317 100644 --- a/llvm/lib/Target/SPIRV/SPIRVLegalizePointerLoad.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp @@ -1,4 +1,4 @@ -//===-- SPIRVLegalizePointerLoad.cpp ----------------------*- C++ -*-===// +//===-- SPIRVLegalizePointerCast.cpp ----------------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -13,8 +13,8 @@ // by logical SPIR-V. // // This pass relies on the assign_ptr_type intrinsic to deduce the type of the -// pointee. All occurrences of `ptrcast` must be replaced because the lead to -// invalid SPIR-V. Unhandled cases result in an error. +// pointed values, must replace all occurences of `ptrcast`. This is why +// unhandled cases are reported as unreachable: we MUST cover all cases. // // 1. Loading the first element of an array // @@ -57,18 +57,10 @@ using namespace llvm; namespace llvm { -void initializeSPIRVLegalizePointerLoadPass(PassRegistry &); +void initializeSPIRVLegalizePointerCastPass(PassRegistry &); } -class SPIRVLegalizePointerLoad : public FunctionPass { - - // Replace all uses of a |Old| with |New| updates the global registry type - // mappings. - void replaceAllUsesWith(Value *Old, Value *New) { - Old->replaceAllUsesWith(New); - GR->updateIfExistDeducedElementType(Old, New, /* deleteOld = */ true); - GR->updateIfExistAssignPtrTypeInstr(Old, New, /* deleteOld = */ true); - } +class SPIRVLegalizePointerCast : public FunctionPass { // Builds the `spv_assign_type` assigning |Ty| to |Value| at the current // builder position. @@ -79,72 +71,37 @@ class SPIRVLegalizePointerLoad : public FunctionPass { GR->addAssignPtrTypeInstr(Arg, AssignCI); } - // Loads a single scalar of type |To| from the vector pointed by |Source| of - // the type |From|. Returns the loaded value. - Value *loadScalarFromVector(IRBuilder<> &B, Value *Source, - FixedVectorType *From) { - - LoadInst *NewLoad = B.CreateLoad(From, Source); - buildAssignType(B, From, NewLoad); - - SmallVector Args = {NewLoad, B.getInt64(0)}; - SmallVector Types = {From->getElementType(), Args[0]->getType(), - Args[1]->getType()}; - Value *Extracted = - B.CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args}); - buildAssignType(B, Extracted->getType(), Extracted); - return Extracted; - } - - // Loads parts of the vector of type |From| from the pointer |Source| and - // create a new vector of type |To|. |To| must be a vector type, and element - // types of |To| and |From| must match. Returns the loaded value. - Value *loadVectorFromVector(IRBuilder<> &B, FixedVectorType *From, - FixedVectorType *To, Value *Source) { + // Loads parts of the vector of type |SourceType| from the pointer |Source| + // and create a new vector of type |TargetType|. |TargetType| must be a vector + // type, and element types of |TargetType| and |SourceType| must match. + // Returns the loaded value. + Value *loadVectorFromVector(IRBuilder<> &B, FixedVectorType *SourceType, + FixedVectorType *TargetType, Value *Source) { // We expect the codegen to avoid doing implicit bitcast from a load. - assert(To->getElementType() == From->getElementType()); - assert(To->getNumElements() < From->getNumElements()); - - LoadInst *NewLoad = B.CreateLoad(From, Source); - buildAssignType(B, From, NewLoad); + assert(TargetType->getElementType() == SourceType->getElementType()); + assert(TargetType->getNumElements() < SourceType->getNumElements()); - auto ConstInt = ConstantInt::get(IntegerType::get(B.getContext(), 32), 0); - ElementCount VecElemCount = ElementCount::getFixed(To->getNumElements()); - Value *Output = ConstantVector::getSplat(VecElemCount, ConstInt); - for (unsigned I = 0; I < To->getNumElements(); ++I) { - Value *Extracted = nullptr; - { - SmallVector Args = {NewLoad, B.getInt64(I)}; - SmallVector Types = {To->getElementType(), - Args[0]->getType(), Args[1]->getType()}; - Extracted = - B.CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args}); - buildAssignType(B, Extracted->getType(), Extracted); - } - assert(Extracted != nullptr); + LoadInst *NewLoad = B.CreateLoad(SourceType, Source); + buildAssignType(B, SourceType, NewLoad); - { - SmallVector Args = {Output, Extracted, B.getInt64(I)}; - SmallVector Types = {Args[0]->getType(), Args[0]->getType(), - Args[1]->getType(), Args[2]->getType()}; - Output = B.CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args}); - buildAssignType(B, Output->getType(), Output); - } - } + SmallVector Mask(/* Size= */ TargetType->getNumElements(), + /* Value= */ 0); + Value *Output = B.CreateShuffleVector(NewLoad, NewLoad, Mask); + buildAssignType(B, TargetType, Output); return Output; } - // Loads the first value in an array pointed by |Source| of type |From|. Load - // flags will be copied from |BadLoad|, which should be the illegal load being - // legalized. Returns the loaded value. - Value *loadFirstValueFromArray(IRBuilder<> &B, ArrayType *From, Value *Source, - LoadInst *BadLoad) { + // Loads the first value in an aggregate pointed by |Source| of containing + // elements of type |ElementType|. Load flags will be copied from |BadLoad|, + // which should be the load being legalized. Returns the loaded value. + Value *loadFirstValueFromAggregate(IRBuilder<> &B, Type *ElementType, + Value *Source, LoadInst *BadLoad) { SmallVector Types = {BadLoad->getPointerOperandType(), BadLoad->getPointerOperandType()}; - SmallVector Args{/* isInBounds= */ B.getInt1(true), Source, - B.getInt64(0), B.getInt64(0)}; + SmallVector Args{/* isInBounds= */ B.getInt1(false), Source, + B.getInt32(0), B.getInt32(0)}; auto *GEP = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args}); - GR->buildAssignPtr(B, From->getElementType(), GEP); + GR->buildAssignPtr(B, ElementType, GEP); const auto *TLI = TM->getSubtargetImpl()->getTargetLowering(); MachineMemOperand::Flags Flags = TLI->getLoadMemOperandFlags( @@ -152,16 +109,16 @@ class SPIRVLegalizePointerLoad : public FunctionPass { Instruction *LI = B.CreateIntrinsic( Intrinsic::spv_load, {BadLoad->getOperand(0)->getType()}, {GEP, B.getInt16(Flags), B.getInt8(BadLoad->getAlign().value())}); - buildAssignType(B, From->getElementType(), LI); + buildAssignType(B, ElementType, LI); return LI; } - // Transforms an illegal partial load into a sequence we can lower to logical - // SPIR-V. + // Replaces the load instruction to get rid of the ptrcast used as source + // operand. void transformLoad(IRBuilder<> &B, LoadInst *LI, Value *CastedOperand, Value *OriginalOperand) { Type *FromTy = GR->findDeducedElementType(OriginalOperand); - Type *ToTy /*-fruity*/ = GR->findDeducedElementType(CastedOperand); + Type *ToTy = GR->findDeducedElementType(CastedOperand); Value *Output = nullptr; auto *SAT = dyn_cast(FromTy); @@ -174,12 +131,14 @@ class SPIRVLegalizePointerLoad : public FunctionPass { // Loading 1st element. // - float a = array[0]; if (SAT && SAT->getElementType() == ToTy) - Output = loadFirstValueFromArray(B, SAT, OriginalOperand, LI); + Output = loadFirstValueFromAggregate(B, SAT->getElementType(), + OriginalOperand, LI); // Destination is the element type of Source, and source is a vector -> // Vector to scalar. // - float a = vector.x; else if (!DVT && SVT && SVT->getElementType() == ToTy) { - Output = loadScalarFromVector(B, OriginalOperand, SVT); + Output = loadFirstValueFromAggregate(B, SVT->getElementType(), + OriginalOperand, LI); } // Destination is a smaller vector than source. // - float3 v3 = vector4; @@ -188,7 +147,7 @@ class SPIRVLegalizePointerLoad : public FunctionPass { else llvm_unreachable("Unimplemented implicit down-cast from load."); - replaceAllUsesWith(LI, Output); + GR->replaceAllUsesWith(LI, Output, /* DeleteOld= */ true); DeadInstructions.push_back(LI); } @@ -220,8 +179,8 @@ class SPIRVLegalizePointerLoad : public FunctionPass { } public: - SPIRVLegalizePointerLoad(SPIRVTargetMachine *TM) : FunctionPass(ID), TM(TM) { - initializeSPIRVLegalizePointerLoadPass(*PassRegistry::getPassRegistry()); + SPIRVLegalizePointerCast(SPIRVTargetMachine *TM) : FunctionPass(ID), TM(TM) { + initializeSPIRVLegalizePointerCastPass(*PassRegistry::getPassRegistry()); }; virtual bool runOnFunction(Function &F) override { @@ -256,10 +215,10 @@ class SPIRVLegalizePointerLoad : public FunctionPass { static char ID; }; -char SPIRVLegalizePointerLoad::ID = 0; -INITIALIZE_PASS(SPIRVLegalizePointerLoad, "spirv-legalize-bitcast", +char SPIRVLegalizePointerCast::ID = 0; +INITIALIZE_PASS(SPIRVLegalizePointerCast, "spirv-legalize-bitcast", "SPIRV legalize bitcast pass", false, false) -FunctionPass *llvm::createSPIRVLegalizePointerLoadPass(SPIRVTargetMachine *TM) { - return new SPIRVLegalizePointerLoad(TM); +FunctionPass *llvm::createSPIRVLegalizePointerCastPass(SPIRVTargetMachine *TM) { + return new SPIRVLegalizePointerCast(TM); } diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp index 2a286112df936..0aa214dd354ee 100644 --- a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp @@ -210,7 +210,7 @@ void SPIRVPassConfig::addIRPasses() { void SPIRVPassConfig::addISelPrepare() { addPass(createSPIRVEmitIntrinsicsPass(&getTM())); if (TM.getSubtargetImpl()->isVulkanEnv()) - addPass(createSPIRVLegalizePointerLoadPass(&getTM())); + addPass(createSPIRVLegalizePointerCastPass(&getTM())); TargetPassConfig::addISelPrepare(); } diff --git a/llvm/test/CodeGen/SPIRV/pointers/array-skips-gep.ll b/llvm/test/CodeGen/SPIRV/pointers/array-skips-gep.ll index e91e25afd8ec7..85663d6c3eeda 100644 --- a/llvm/test/CodeGen/SPIRV/pointers/array-skips-gep.ll +++ b/llvm/test/CodeGen/SPIRV/pointers/array-skips-gep.ll @@ -1,15 +1,14 @@ ; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - | FileCheck %s ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %} -; CHECK-DAG: %[[#int_ty:]] = OpTypeInt 32 0 -; CHECK-DAG: %[[#long_ty:]] = OpTypeInt 64 0 -; CHECK-DAG: %[[#int_10:]] = OpConstant %[[#int_ty]] 10 -; CHECK-DAG: %[[#long_0:]] = OpConstantNull %[[#long_ty]] -; CHECK-DAG: %[[#array_ty:]] = OpTypeArray %[[#int_ty]] %[[#int_10]] +; CHECK-DAG: %[[#uint_ty:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#uint_0:]] = OpConstant %[[#uint_ty]] 0 +; CHECK-DAG: %[[#int_10:]] = OpConstant %[[#uint_ty]] 10 +; CHECK-DAG: %[[#array_ty:]] = OpTypeArray %[[#uint_ty]] %[[#int_10]] ; CHECK-DAG: %[[#array_fp:]] = OpTypePointer Function %[[#array_ty]] ; CHECK-DAG: %[[#array_pp:]] = OpTypePointer Private %[[#array_ty]] -; CHECK-DAG: %[[#int_fp:]] = OpTypePointer Function %[[#int_ty]] -; CHECK-DAG: %[[#int_pp:]] = OpTypePointer Private %[[#int_ty]] +; CHECK-DAG: %[[#int_fp:]] = OpTypePointer Function %[[#uint_ty]] +; CHECK-DAG: %[[#int_pp:]] = OpTypePointer Private %[[#uint_ty]] @gv = external addrspace(10) global [10 x i32] ; CHECK: %[[#gv:]] = OpVariable %[[#array_pp]] Private @@ -20,8 +19,8 @@ define internal spir_func i32 @foo() { ; Direct load from the pointer index. This requires an OpAccessChain %1 = load i32, ptr %array, align 4 -; CHECK: %[[#ptr:]] = OpInBoundsAccessChain %[[#int_fp]] %[[#array]] %[[#long_0]] -; CHECK: %[[#val:]] = OpLoad %[[#int_ty]] %[[#ptr]] Aligned 4 +; CHECK: %[[#ptr:]] = OpAccessChain %[[#int_fp]] %[[#array]] %[[#uint_0]] +; CHECK: %[[#val:]] = OpLoad %[[#uint_ty]] %[[#ptr]] Aligned 4 ret i32 %1 ; CHECK: OpReturnValue %[[#val]] @@ -30,8 +29,8 @@ define internal spir_func i32 @foo() { define internal spir_func i32 @bar() { ; Direct load from the pointer index. This requires an OpAccessChain %1 = load i32, ptr addrspace(10) @gv -; CHECK: %[[#ptr:]] = OpInBoundsAccessChain %[[#int_pp]] %[[#gv]] %[[#long_0]] -; CHECK: %[[#val:]] = OpLoad %[[#int_ty]] %[[#ptr]] Aligned 4 +; CHECK: %[[#ptr:]] = OpAccessChain %[[#int_pp]] %[[#gv]] %[[#uint_0]] +; CHECK: %[[#val:]] = OpLoad %[[#uint_ty]] %[[#ptr]] Aligned 4 ret i32 %1 ; CHECK: OpReturnValue %[[#val]] diff --git a/llvm/test/CodeGen/SPIRV/pointers/getelementptr-downcast-vector.ll b/llvm/test/CodeGen/SPIRV/pointers/getelementptr-downcast-vector.ll index 5bcec1d870651..d4131fa8a2658 100644 --- a/llvm/test/CodeGen/SPIRV/pointers/getelementptr-downcast-vector.ll +++ b/llvm/test/CodeGen/SPIRV/pointers/getelementptr-downcast-vector.ll @@ -1,15 +1,15 @@ ; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - | FileCheck %s ; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %} -; CHECK-DAG: %[[#uint:]] = OpTypeInt 32 0 -; CHECK-DAG: %[[#uint_0:]] = OpConstant %[[#uint]] 0 -; CHECK-DAG: %[[#v2:]] = OpTypeVector %[[#uint]] 2 -; CHECK-DAG: %[[#v3:]] = OpTypeVector %[[#uint]] 3 -; CHECK-DAG: %[[#v4:]] = OpTypeVector %[[#uint]] 4 -; CHECK-DAG: %[[#v4_pp:]] = OpTypePointer Private %[[#v4]] -; CHECK-DAG: %[[#v4_fp:]] = OpTypePointer Function %[[#v4]] -; CHECK-DAG: %[[#v3_000:]] = OpConstantComposite %[[#v3]] %[[#uint_0]] %[[#uint_0]] %[[#uint_0]] -; CHECK-DAG: %[[#v2_00:]] = OpConstantComposite %[[#v2]] %[[#uint_0]] %[[#uint_0]] +; CHECK-DAG: %[[#uint:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#uint_pp:]] = OpTypePointer Private %[[#uint]] +; CHECK-DAG: %[[#uint_fp:]] = OpTypePointer Function %[[#uint]] +; CHECK-DAG: %[[#uint_0:]] = OpConstant %[[#uint]] 0 +; CHECK-DAG: %[[#v2:]] = OpTypeVector %[[#uint]] 2 +; CHECK-DAG: %[[#v3:]] = OpTypeVector %[[#uint]] 3 +; CHECK-DAG: %[[#v4:]] = OpTypeVector %[[#uint]] 4 +; CHECK-DAG: %[[#v4_pp:]] = OpTypePointer Private %[[#v4]] +; CHECK-DAG: %[[#v4_fp:]] = OpTypePointer Function %[[#v4]] define internal spir_func <3 x i32> @foo(ptr addrspace(10) %a) { @@ -19,15 +19,10 @@ define internal spir_func <3 x i32> @foo(ptr addrspace(10) %a) { ; partial loading of a vector: v4 -> v3. %2 = load <3 x i32>, ptr addrspace(10) %1, align 16 ; CHECK: %[[#load:]] = OpLoad %[[#v4]] %[[#tmp]] Aligned 16 -; CHECK-DAG: %[[#x:]] = OpCompositeExtract %[[#uint]] %[[#load]] 0 -; CHECK-DAG: %[[#y:]] = OpCompositeExtract %[[#uint]] %[[#load]] 1 -; CHECK-DAG: %[[#z:]] = OpCompositeExtract %[[#uint]] %[[#load]] 2 -; CHECK-DAG: %[[#tmp0:]] = OpCompositeInsert %[[#v3]] %[[#x]] %[[#v3_000]] 0 -; CHECK-DAG: %[[#tmp1:]] = OpCompositeInsert %[[#v3]] %[[#y]] %[[#tmp0]] 1 -; CHECK-DAG: %[[#tmp2:]] = OpCompositeInsert %[[#v3]] %[[#z]] %[[#tmp1]] 2 +; CHECK: %[[#val:]] = OpVectorShuffle %[[#v3]] %[[#load]] %[[#load]] 0 0 0 ret <3 x i32> %2 -; CHECK: OpReturnValue %[[#tmp2]] +; CHECK: OpReturnValue %[[#val]] } define internal spir_func <3 x i32> @fooDefault(ptr %a) { @@ -38,15 +33,10 @@ define internal spir_func <3 x i32> @fooDefault(ptr %a) { ; partial loading of a vector: v4 -> v3. %2 = load <3 x i32>, ptr %1, align 16 ; CHECK: %[[#load:]] = OpLoad %[[#v4]] %[[#tmp]] Aligned 16 -; CHECK-DAG: %[[#x:]] = OpCompositeExtract %[[#uint]] %[[#load]] 0 -; CHECK-DAG: %[[#y:]] = OpCompositeExtract %[[#uint]] %[[#load]] 1 -; CHECK-DAG: %[[#z:]] = OpCompositeExtract %[[#uint]] %[[#load]] 2 -; CHECK-DAG: %[[#tmp0:]] = OpCompositeInsert %[[#v3]] %[[#x]] %[[#v3_000]] 0 -; CHECK-DAG: %[[#tmp1:]] = OpCompositeInsert %[[#v3]] %[[#y]] %[[#tmp0]] 1 -; CHECK-DAG: %[[#tmp2:]] = OpCompositeInsert %[[#v3]] %[[#z]] %[[#tmp1]] 2 +; CHECK: %[[#val:]] = OpVectorShuffle %[[#v3]] %[[#load]] %[[#load]] 0 0 0 ret <3 x i32> %2 -; CHECK: OpReturnValue %[[#tmp2]] +; CHECK: OpReturnValue %[[#val]] } define internal spir_func <3 x i32> @fooBounds(ptr %a) { @@ -57,15 +47,10 @@ define internal spir_func <3 x i32> @fooBounds(ptr %a) { ; partial loading of a vector: v4 -> v3. %2 = load <3 x i32>, ptr %1, align 16 ; CHECK: %[[#load:]] = OpLoad %[[#v4]] %[[#tmp]] Aligned 16 -; CHECK-DAG: %[[#x:]] = OpCompositeExtract %[[#uint]] %[[#load]] 0 -; CHECK-DAG: %[[#y:]] = OpCompositeExtract %[[#uint]] %[[#load]] 1 -; CHECK-DAG: %[[#z:]] = OpCompositeExtract %[[#uint]] %[[#load]] 2 -; CHECK-DAG: %[[#tmp0:]] = OpCompositeInsert %[[#v3]] %[[#x]] %[[#v3_000]] 0 -; CHECK-DAG: %[[#tmp1:]] = OpCompositeInsert %[[#v3]] %[[#y]] %[[#tmp0]] 1 -; CHECK-DAG: %[[#tmp2:]] = OpCompositeInsert %[[#v3]] %[[#z]] %[[#tmp1]] 2 +; CHECK: %[[#val:]] = OpVectorShuffle %[[#v3]] %[[#load]] %[[#load]] 0 0 0 ret <3 x i32> %2 -; CHECK: OpReturnValue %[[#tmp2]] +; CHECK: OpReturnValue %[[#val]] } define internal spir_func <2 x i32> @bar(ptr addrspace(10) %a) { @@ -76,13 +61,10 @@ define internal spir_func <2 x i32> @bar(ptr addrspace(10) %a) { ; partial loading of a vector: v4 -> v2. %2 = load <2 x i32>, ptr addrspace(10) %1, align 16 ; CHECK: %[[#load:]] = OpLoad %[[#v4]] %[[#tmp]] Aligned 16 -; CHECK-DAG: %[[#x:]] = OpCompositeExtract %[[#uint]] %[[#load]] 0 -; CHECK-DAG: %[[#y:]] = OpCompositeExtract %[[#uint]] %[[#load]] 1 -; CHECK-DAG: %[[#tmp0:]] = OpCompositeInsert %[[#v2]] %[[#x]] %[[#v2_00]] 0 -; CHECK-DAG: %[[#tmp1:]] = OpCompositeInsert %[[#v2]] %[[#y]] %[[#tmp0]] 1 +; CHECK: %[[#val:]] = OpVectorShuffle %[[#v2]] %[[#load]] %[[#load]] 0 0 ret <2 x i32> %2 -; CHECK: OpReturnValue %[[#tmp1]] +; CHECK: OpReturnValue %[[#val]] } define internal spir_func i32 @baz(ptr addrspace(10) %a) { @@ -92,11 +74,11 @@ define internal spir_func i32 @baz(ptr addrspace(10) %a) { ; Loading of the first scalar of a vector: v4 -> int. %2 = load i32, ptr addrspace(10) %1, align 16 -; CHECK: %[[#load:]] = OpLoad %[[#v4]] %[[#tmp]] Aligned 16 -; CHECK-DAG: %[[#x:]] = OpCompositeExtract %[[#uint]] %[[#load]] 0 +; CHECK: %[[#ptr:]] = OpAccessChain %[[#uint_pp]] %[[#tmp]] %[[#uint_0]] +; CHECK: %[[#val:]] = OpLoad %[[#uint]] %[[#ptr]] Aligned 16 ret i32 %2 -; CHECK: OpReturnValue %[[#x]] +; CHECK: OpReturnValue %[[#val]] } define internal spir_func i32 @bazDefault(ptr %a) { @@ -106,11 +88,11 @@ define internal spir_func i32 @bazDefault(ptr %a) { ; Loading of the first scalar of a vector: v4 -> int. %2 = load i32, ptr %1, align 16 -; CHECK: %[[#load:]] = OpLoad %[[#v4]] %[[#tmp]] Aligned 16 -; CHECK-DAG: %[[#x:]] = OpCompositeExtract %[[#uint]] %[[#load]] 0 +; CHECK: %[[#ptr:]] = OpAccessChain %[[#uint_fp]] %[[#tmp]] %[[#uint_0]] +; CHECK: %[[#val:]] = OpLoad %[[#uint]] %[[#ptr]] Aligned 16 ret i32 %2 -; CHECK: OpReturnValue %[[#x]] +; CHECK: OpReturnValue %[[#val]] } define internal spir_func i32 @bazBounds(ptr %a) { @@ -120,9 +102,9 @@ define internal spir_func i32 @bazBounds(ptr %a) { ; Loading of the first scalar of a vector: v4 -> int. %2 = load i32, ptr %1, align 16 -; CHECK: %[[#load:]] = OpLoad %[[#v4]] %[[#tmp]] Aligned 16 -; CHECK-DAG: %[[#x:]] = OpCompositeExtract %[[#uint]] %[[#load]] 0 +; CHECK: %[[#ptr:]] = OpAccessChain %[[#uint_fp]] %[[#tmp]] %[[#uint_0]] +; CHECK: %[[#val:]] = OpLoad %[[#uint]] %[[#ptr]] Aligned 16 ret i32 %2 -; CHECK: OpReturnValue %[[#x]] +; CHECK: OpReturnValue %[[#val]] }