diff --git a/llvm/lib/Target/SPIRV/CMakeLists.txt b/llvm/lib/Target/SPIRV/CMakeLists.txt index efdd8c8d24fbd..4a2b534b948d6 100644 --- a/llvm/lib/Target/SPIRV/CMakeLists.txt +++ b/llvm/lib/Target/SPIRV/CMakeLists.txt @@ -27,6 +27,7 @@ add_llvm_target(SPIRVCodeGen SPIRVInstrInfo.cpp SPIRVInstructionSelector.cpp SPIRVStripConvergentIntrinsics.cpp + SPIRVLegalizePointerCast.cpp SPIRVMergeRegionExitTargets.cpp SPIRVISelLowering.cpp SPIRVLegalizerInfo.cpp diff --git a/llvm/lib/Target/SPIRV/SPIRV.h b/llvm/lib/Target/SPIRV/SPIRV.h index 6d00a046ff7ca..d765dfe370be2 100644 --- a/llvm/lib/Target/SPIRV/SPIRV.h +++ b/llvm/lib/Target/SPIRV/SPIRV.h @@ -23,6 +23,7 @@ ModulePass *createSPIRVPrepareFunctionsPass(const SPIRVTargetMachine &TM); FunctionPass *createSPIRVStructurizerPass(); FunctionPass *createSPIRVMergeRegionExitTargetsPass(); FunctionPass *createSPIRVStripConvergenceIntrinsicsPass(); +FunctionPass *createSPIRVLegalizePointerCastPass(SPIRVTargetMachine *TM); FunctionPass *createSPIRVRegularizerPass(); FunctionPass *createSPIRVPreLegalizerCombiner(); FunctionPass *createSPIRVPreLegalizerPass(); diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index 5dfba8427258f..506bdd2875f48 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -57,12 +57,6 @@ void initializeSPIRVEmitIntrinsicsPass(PassRegistry &); namespace { -inline MetadataAsValue *buildMD(Value *Arg) { - LLVMContext &Ctx = Arg->getContext(); - return MetadataAsValue::get( - Ctx, MDNode::get(Ctx, ValueAsMetadata::getConstant(Arg))); -} - class SPIRVEmitIntrinsics : public ModulePass, public InstVisitor { @@ -142,24 +136,9 @@ class SPIRVEmitIntrinsics void preprocessCompositeConstants(IRBuilder<> &B); void preprocessUndefs(IRBuilder<> &B); - CallInst *buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef Types, - Value *Arg, Value *Arg2, ArrayRef Imms, - IRBuilder<> &B) { - SmallVector Args; - Args.push_back(Arg2); - Args.push_back(buildMD(Arg)); - for (auto *Imm : Imms) - Args.push_back(Imm); - return B.CreateIntrinsic(IntrID, {Types}, Args); - } - Type *reconstructType(Value *Op, bool UnknownElemTypeI8, bool IsPostprocessing); - void buildAssignType(IRBuilder<> &B, Type *ElemTy, Value *Arg); - void buildAssignPtr(IRBuilder<> &B, Type *ElemTy, Value *Arg); - void updateAssignType(CallInst *AssignCI, Value *Arg, Value *OfType); - void replaceMemInstrUses(Instruction *Old, Instruction *New, IRBuilder<> &B); void processInstrAfterVisit(Instruction *I, IRBuilder<> &B); bool insertAssignPtrTypeIntrs(Instruction *I, IRBuilder<> &B, @@ -273,18 +252,6 @@ bool expectIgnoredInIRTranslation(const Instruction *I) { } } -bool allowEmitFakeUse(const Value *Arg) { - if (isSpvIntrinsic(Arg)) - return false; - if (dyn_cast(Arg) || dyn_cast(Arg) || - dyn_cast(Arg)) - return false; - if (const auto *LI = dyn_cast(Arg)) - if (LI->getType()->isAggregateType()) - return false; - return true; -} - } // namespace char SPIRVEmitIntrinsics::ID = 0; @@ -355,10 +322,7 @@ static void emitAssignName(Instruction *I, IRBuilder<> &B) { void SPIRVEmitIntrinsics::replaceAllUsesWith(Value *Src, Value *Dest, bool DeleteOld) { - Src->replaceAllUsesWith(Dest); - // Update deduced type records - GR->updateIfExistDeducedElementType(Src, Dest, DeleteOld); - GR->updateIfExistAssignPtrTypeInstr(Src, Dest, DeleteOld); + GR->replaceAllUsesWith(Src, Dest, DeleteOld); // Update uncomplete type records if any if (isTodoType(Src)) { if (DeleteOld) @@ -425,57 +389,6 @@ Type *SPIRVEmitIntrinsics::reconstructType(Value *Op, bool UnknownElemTypeI8, return nullptr; } -void SPIRVEmitIntrinsics::buildAssignType(IRBuilder<> &B, Type *Ty, - Value *Arg) { - Value *OfType = getNormalizedPoisonValue(Ty); - CallInst *AssignCI = nullptr; - if (Arg->getType()->isAggregateType() && Ty->isAggregateType() && - allowEmitFakeUse(Arg)) { - LLVMContext &Ctx = Arg->getContext(); - SmallVector ArgMDs{ - MDNode::get(Ctx, ValueAsMetadata::getConstant(OfType)), - MDString::get(Ctx, Arg->getName())}; - B.CreateIntrinsic(Intrinsic::spv_value_md, {}, - {MetadataAsValue::get(Ctx, MDTuple::get(Ctx, ArgMDs))}); - AssignCI = B.CreateIntrinsic(Intrinsic::fake_use, {}, {Arg}); - } else { - AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type, {Arg->getType()}, - OfType, Arg, {}, B); - } - GR->addAssignPtrTypeInstr(Arg, AssignCI); -} - -void SPIRVEmitIntrinsics::buildAssignPtr(IRBuilder<> &B, Type *ElemTy, - Value *Arg) { - ElemTy = normalizeType(ElemTy); - Value *OfType = PoisonValue::get(ElemTy); - CallInst *AssignPtrTyCI = GR->findAssignPtrTypeInstr(Arg); - if (AssignPtrTyCI == nullptr || - AssignPtrTyCI->getParent()->getParent() != CurrF) { - AssignPtrTyCI = buildIntrWithMD( - Intrinsic::spv_assign_ptr_type, {Arg->getType()}, OfType, Arg, - {B.getInt32(getPointerAddressSpace(Arg->getType()))}, B); - GR->addDeducedElementType(AssignPtrTyCI, ElemTy); - GR->addDeducedElementType(Arg, ElemTy); - GR->addAssignPtrTypeInstr(Arg, AssignPtrTyCI); - } else { - updateAssignType(AssignPtrTyCI, Arg, OfType); - } -} - -void SPIRVEmitIntrinsics::updateAssignType(CallInst *AssignCI, Value *Arg, - Value *OfType) { - AssignCI->setArgOperand(1, buildMD(OfType)); - if (cast(AssignCI)->getIntrinsicID() != - Intrinsic::spv_assign_ptr_type) - return; - - // update association with the pointee type - Type *ElemTy = normalizeType(OfType->getType()); - GR->addDeducedElementType(AssignCI, ElemTy); - GR->addDeducedElementType(Arg, ElemTy); -} - CallInst *SPIRVEmitIntrinsics::buildSpvPtrcast(Function *F, Value *Op, Type *ElemTy) { IRBuilder<> B(Op->getContext()); @@ -495,7 +408,7 @@ CallInst *SPIRVEmitIntrinsics::buildSpvPtrcast(Function *F, Value *Op, B.getInt32(getPointerAddressSpace(OpTy))}; CallInst *PtrCasted = B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args); - buildAssignPtr(B, ElemTy, PtrCasted); + GR->buildAssignPtr(B, ElemTy, PtrCasted); return PtrCasted; } @@ -1026,7 +939,8 @@ bool SPIRVEmitIntrinsics::deduceOperandElementTypeFunctionRet( continue; if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(CI)) { if (Type *PrevElemTy = GR->findDeducedElementType(CI)) { - updateAssignType(AssignCI, CI, getNormalizedPoisonValue(OpElemTy)); + GR->updateAssignType(AssignCI, CI, + getNormalizedPoisonValue(OpElemTy)); propagateElemType(CI, PrevElemTy, VisitedSubst); } } @@ -1212,7 +1126,7 @@ void SPIRVEmitIntrinsics::deduceOperandElementType( {B.getInt32(getPointerAddressSpace(OpTy))}, B); GR->addAssignPtrTypeInstr(Op, CI); } else { - updateAssignType(AssignCI, Op, OpTyVal); + GR->updateAssignType(AssignCI, Op, OpTyVal); DenseSet> VisitedSubst{ std::make_pair(I, Op)}; propagateElemTypeRec(Op, KnownElemTy, PrevElemTy, VisitedSubst); @@ -1502,7 +1416,7 @@ void SPIRVEmitIntrinsics::insertAssignPtrTypeTargetExt( CallInst *AssignCI = GR->findAssignPtrTypeInstr(V); if (!AssignCI) { - buildAssignType(B, AssignedType, V); + GR->buildAssignType(B, AssignedType, V); return; } @@ -1522,7 +1436,7 @@ void SPIRVEmitIntrinsics::insertAssignPtrTypeTargetExt( // Our previous guess about the type seems to be wrong, let's update // inferred type according to a new, more precise type information. - updateAssignType(AssignCI, V, getNormalizedPoisonValue(AssignedType)); + GR->updateAssignType(AssignCI, V, getNormalizedPoisonValue(AssignedType)); } void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast( @@ -1579,7 +1493,7 @@ void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast( if (FirstPtrCastOrAssignPtrType) { // If this would be the first spv_ptrcast, do not emit spv_ptrcast and // emit spv_assign_ptr_type instead. - buildAssignPtr(B, ExpectedElementType, Pointer); + GR->buildAssignPtr(B, ExpectedElementType, Pointer); return; } else if (isTodoType(Pointer)) { eraseTodoType(Pointer); @@ -1591,10 +1505,10 @@ void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast( assert(PrevElemTy); DenseSet> VisitedSubst{ std::make_pair(I, Pointer)}; - updateAssignType(AssignCI, Pointer, ExpectedElementVal); + GR->updateAssignType(AssignCI, Pointer, ExpectedElementVal); propagateElemType(Pointer, PrevElemTy, VisitedSubst); } else { - buildAssignPtr(B, ExpectedElementType, Pointer); + GR->buildAssignPtr(B, ExpectedElementType, Pointer); } return; } @@ -1607,7 +1521,7 @@ void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast( auto *PtrCastI = B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args); I->setOperand(OperandToReplace, PtrCastI); // We need to set up a pointee type for the newly created spv_ptrcast. - buildAssignPtr(B, ExpectedElementType, PtrCastI); + GR->buildAssignPtr(B, ExpectedElementType, PtrCastI); } void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I, @@ -1923,7 +1837,7 @@ bool SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(Instruction *I, setInsertPointAfterDef(B, I); if (Type *ElemTy = deduceElementType(I, UnknownElemTypeI8)) { - buildAssignPtr(B, ElemTy, I); + GR->buildAssignPtr(B, ElemTy, I); return false; } return true; @@ -1956,8 +1870,8 @@ void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I, setInsertPointAfterDef(B, I); switch (ResIt->second) { case WellKnownTypes::Event: - buildAssignType(B, TargetExtType::get(I->getContext(), "spirv.Event"), - I); + GR->buildAssignType( + B, TargetExtType::get(I->getContext(), "spirv.Event"), I); break; } } @@ -2002,7 +1916,7 @@ void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I, } } TypeToAssign = restoreMutatedType(GR, I, TypeToAssign); - buildAssignType(B, TypeToAssign, I); + GR->buildAssignType(B, TypeToAssign, I); } for (const auto &Op : I->operands()) { if (isa(Op) || isa(Op) || @@ -2019,10 +1933,11 @@ void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I, Type *OpTy = Op->getType(); Type *OpTyElem = getPointeeType(OpTy); if (OpTyElem) { - buildAssignPtr(B, OpTyElem, Op); + GR->buildAssignPtr(B, OpTyElem, Op); } else if (isPointerTy(OpTy)) { Type *ElemTy = GR->findDeducedElementType(Op); - buildAssignPtr(B, ElemTy ? ElemTy : deduceElementType(Op, true), Op); + GR->buildAssignPtr(B, ElemTy ? ElemTy : deduceElementType(Op, true), + Op); } else { CallInst *AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type, {OpTy}, @@ -2083,14 +1998,14 @@ void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I, if (!IsConstComposite && isPointerTy(OpTy) && (OpElemTy = GR->findDeducedElementType(Op)) != nullptr && OpElemTy != IntegerType::getInt8Ty(I->getContext())) { - buildAssignPtr(B, IntegerType::getInt8Ty(I->getContext()), NewOp); + GR->buildAssignPtr(B, IntegerType::getInt8Ty(I->getContext()), NewOp); SmallVector Types = {OpTy, OpTy}; SmallVector Args = { NewOp, buildMD(getNormalizedPoisonValue(OpElemTy)), B.getInt32(getPointerAddressSpace(OpTy))}; CallInst *PtrCasted = B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args); - buildAssignPtr(B, OpElemTy, PtrCasted); + GR->buildAssignPtr(B, OpElemTy, PtrCasted); NewOp = PtrCasted; } I->setOperand(OpNo, NewOp); @@ -2172,7 +2087,7 @@ void SPIRVEmitIntrinsics::processParamTypesByFunHeader(Function *F, continue; if (hasPointeeTypeAttr(Arg) && (ElemTy = getPointeeTypeByAttr(Arg)) != nullptr) { - buildAssignPtr(B, ElemTy, Arg); + GR->buildAssignPtr(B, ElemTy, Arg); continue; } // search in function's call sites @@ -2188,7 +2103,7 @@ void SPIRVEmitIntrinsics::processParamTypesByFunHeader(Function *F, break; } if (ElemTy) { - buildAssignPtr(B, ElemTy, Arg); + GR->buildAssignPtr(B, ElemTy, Arg); continue; } if (HaveFunPtrs) { @@ -2200,7 +2115,7 @@ void SPIRVEmitIntrinsics::processParamTypesByFunHeader(Function *F, SmallVector> Ops; deduceOperandElementTypeFunctionPointer(CI, Ops, ElemTy, false); if (ElemTy) { - buildAssignPtr(B, ElemTy, Arg); + GR->buildAssignPtr(B, ElemTy, Arg); break; } } @@ -2219,11 +2134,11 @@ void SPIRVEmitIntrinsics::processParamTypes(Function *F, IRBuilder<> &B) { if (!ElemTy && (ElemTy = deduceFunParamElementType(F, OpIdx)) != nullptr) { if (CallInst *AssignCI = GR->findAssignPtrTypeInstr(Arg)) { DenseSet> VisitedSubst; - updateAssignType(AssignCI, Arg, getNormalizedPoisonValue(ElemTy)); + GR->updateAssignType(AssignCI, Arg, getNormalizedPoisonValue(ElemTy)); propagateElemType(Arg, IntegerType::getInt8Ty(F->getContext()), VisitedSubst); } else { - buildAssignPtr(B, ElemTy, Arg); + GR->buildAssignPtr(B, ElemTy, Arg); } } } @@ -2273,7 +2188,7 @@ bool SPIRVEmitIntrinsics::processFunctionPointers(Module &M) { continue; if (II->getIntrinsicID() == Intrinsic::spv_assign_ptr_type || II->getIntrinsicID() == Intrinsic::spv_ptrcast) { - updateAssignType(II, &F, getNormalizedPoisonValue(FPElemTy)); + GR->updateAssignType(II, &F, getNormalizedPoisonValue(FPElemTy)); break; } } @@ -2324,7 +2239,7 @@ void SPIRVEmitIntrinsics::applyDemangledPtrArgTypes(IRBuilder<> &B) { if (!hasPointeeTypeAttr(Arg)) { B.SetInsertPointPastAllocas(Arg->getParent()); B.SetCurrentDebugLocation(DebugLoc()); - buildAssignPtr(B, ElemTy, Arg); + GR->buildAssignPtr(B, ElemTy, Arg); } } else if (isa(Param)) { GR->addDeducedElementType(Param, normalizeType(ElemTy)); @@ -2334,7 +2249,7 @@ void SPIRVEmitIntrinsics::applyDemangledPtrArgTypes(IRBuilder<> &B) { ->getParent() ->getEntryBlock() .getFirstNonPHIOrDbgOrAlloca()); - buildAssignPtr(B, ElemTy, Param); + GR->buildAssignPtr(B, ElemTy, Param); } CallInst *Ref = dyn_cast(Param); if (!Ref) diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp index 0ed414ebc8bbe..e9b517d3eeaaa 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -22,6 +22,9 @@ #include "SPIRVUtils.h" #include "llvm/ADT/APInt.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicsSPIRV.h" #include "llvm/IR/Type.h" #include "llvm/Support/Casting.h" #include @@ -29,6 +32,20 @@ using namespace llvm; +namespace { + +bool allowEmitFakeUse(const Value *Arg) { + if (isSpvIntrinsic(Arg)) + return false; + if (dyn_cast(Arg) || dyn_cast(Arg) || + dyn_cast(Arg)) + return false; + if (const auto *LI = dyn_cast(Arg)) + if (LI->getType()->isAggregateType()) + return false; + return true; +} + inline unsigned typeToAddressSpace(const Type *Ty) { if (auto PType = dyn_cast(Ty)) return PType->getAddressSpace(); @@ -40,6 +57,8 @@ inline unsigned typeToAddressSpace(const Type *Ty) { report_fatal_error("Unable to convert LLVM type to SPIRVType", true); } +} // anonymous namespace + SPIRVGlobalRegistry::SPIRVGlobalRegistry(unsigned PointerSize) : PointerSize(PointerSize), Bound(0) {} @@ -1739,3 +1758,62 @@ LLT SPIRVGlobalRegistry::getRegType(SPIRVType *SpvType) const { } return LLT::scalar(64); } + +void SPIRVGlobalRegistry::replaceAllUsesWith(Value *Old, Value *New, + bool DeleteOld) { + Old->replaceAllUsesWith(New); + updateIfExistDeducedElementType(Old, New, DeleteOld); + updateIfExistAssignPtrTypeInstr(Old, New, DeleteOld); +} + +void SPIRVGlobalRegistry::buildAssignType(IRBuilder<> &B, Type *Ty, + Value *Arg) { + Value *OfType = getNormalizedPoisonValue(Ty); + CallInst *AssignCI = nullptr; + if (Arg->getType()->isAggregateType() && Ty->isAggregateType() && + allowEmitFakeUse(Arg)) { + LLVMContext &Ctx = Arg->getContext(); + SmallVector ArgMDs{ + MDNode::get(Ctx, ValueAsMetadata::getConstant(OfType)), + MDString::get(Ctx, Arg->getName())}; + B.CreateIntrinsic(Intrinsic::spv_value_md, {}, + {MetadataAsValue::get(Ctx, MDTuple::get(Ctx, ArgMDs))}); + AssignCI = B.CreateIntrinsic(Intrinsic::fake_use, {}, {Arg}); + } else { + AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type, {Arg->getType()}, + OfType, Arg, {}, B); + } + addAssignPtrTypeInstr(Arg, AssignCI); +} + +void SPIRVGlobalRegistry::buildAssignPtr(IRBuilder<> &B, Type *ElemTy, + Value *Arg) { + Value *OfType = PoisonValue::get(ElemTy); + CallInst *AssignPtrTyCI = findAssignPtrTypeInstr(Arg); + Function *CurrF = + B.GetInsertBlock() ? B.GetInsertBlock()->getParent() : nullptr; + if (AssignPtrTyCI == nullptr || + AssignPtrTyCI->getParent()->getParent() != CurrF) { + AssignPtrTyCI = buildIntrWithMD( + Intrinsic::spv_assign_ptr_type, {Arg->getType()}, OfType, Arg, + {B.getInt32(getPointerAddressSpace(Arg->getType()))}, B); + addDeducedElementType(AssignPtrTyCI, ElemTy); + addDeducedElementType(Arg, ElemTy); + addAssignPtrTypeInstr(Arg, AssignPtrTyCI); + } else { + updateAssignType(AssignPtrTyCI, Arg, OfType); + } +} + +void SPIRVGlobalRegistry::updateAssignType(CallInst *AssignCI, Value *Arg, + Value *OfType) { + AssignCI->setArgOperand(1, buildMD(OfType)); + if (cast(AssignCI)->getIntrinsicID() != + Intrinsic::spv_assign_ptr_type) + return; + + // update association with the pointee type + Type *ElemTy = OfType->getType(); + addDeducedElementType(AssignCI, ElemTy); + addDeducedElementType(Arg, ElemTy); +} diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h index 2c24ba79ea8e6..778232e7013d4 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h @@ -620,6 +620,14 @@ class SPIRVGlobalRegistry { const TargetRegisterClass *getRegClass(SPIRVType *SpvType) const; LLT getRegType(SPIRVType *SpvType) const; + + // Replace all uses of a |Old| with |New| updates the global registry type + // mappings. + void replaceAllUsesWith(Value *Old, Value *New, bool DeleteOld = true); + + void buildAssignType(IRBuilder<> &B, Type *Ty, Value *Arg); + void buildAssignPtr(IRBuilder<> &B, Type *ElemTy, Value *Arg); + void updateAssignType(CallInst *AssignCI, Value *Arg, Value *OfType); }; } // end namespace llvm #endif // LLLVM_LIB_TARGET_SPIRV_SPIRVTYPEMANAGER_H diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp new file mode 100644 index 0000000000000..ec419d25cd317 --- /dev/null +++ b/llvm/lib/Target/SPIRV/SPIRVLegalizePointerCast.cpp @@ -0,0 +1,224 @@ +//===-- SPIRVLegalizePointerCast.cpp ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// The LLVM IR has multiple legal patterns we cannot lower to Logical SPIR-V. +// This pass modifies such loads to have an IR we can directly lower to valid +// logical SPIR-V. +// OpenCL can avoid this because they rely on ptrcast, which is not supported +// by logical SPIR-V. +// +// This pass relies on the assign_ptr_type intrinsic to deduce the type of the +// pointed values, must replace all occurences of `ptrcast`. This is why +// unhandled cases are reported as unreachable: we MUST cover all cases. +// +// 1. Loading the first element of an array +// +// %array = [10 x i32] +// %value = load i32, ptr %array +// +// LLVM can skip the GEP instruction, and only request loading the first 4 +// bytes. In logical SPIR-V, we need an OpAccessChain to access the first +// element. This pass will add a getelementptr instruction before the load. +// +// +// 2. Implicit downcast from load +// +// %1 = getelementptr <4 x i32>, ptr %vec4, i64 0 +// %2 = load <3 x i32>, ptr %1 +// +// The pointer in the GEP instruction is only used for offset computations, +// but it doesn't NEED to match the pointed type. OpAccessChain however +// requires this. Also, LLVM loads define the bitwidth of the load, not the +// pointer. In this example, we can guess %vec4 is a vec4 thanks to the GEP +// instruction basetype, but we only want to load the first 3 elements, hence +// do a partial load. In logical SPIR-V, this is not legal. What we must do +// is load the full vector (basetype), extract 3 elements, and recombine them +// to form a 3-element vector. +// +//===----------------------------------------------------------------------===// + +#include "SPIRV.h" +#include "SPIRVSubtarget.h" +#include "SPIRVTargetMachine.h" +#include "SPIRVUtils.h" +#include "llvm/CodeGen/IntrinsicLowering.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicsSPIRV.h" +#include "llvm/Transforms/Utils/Cloning.h" +#include "llvm/Transforms/Utils/LowerMemIntrinsics.h" + +using namespace llvm; + +namespace llvm { +void initializeSPIRVLegalizePointerCastPass(PassRegistry &); +} + +class SPIRVLegalizePointerCast : public FunctionPass { + + // Builds the `spv_assign_type` assigning |Ty| to |Value| at the current + // builder position. + void buildAssignType(IRBuilder<> &B, Type *Ty, Value *Arg) { + Value *OfType = PoisonValue::get(Ty); + CallInst *AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type, + {Arg->getType()}, OfType, Arg, {}, B); + GR->addAssignPtrTypeInstr(Arg, AssignCI); + } + + // Loads parts of the vector of type |SourceType| from the pointer |Source| + // and create a new vector of type |TargetType|. |TargetType| must be a vector + // type, and element types of |TargetType| and |SourceType| must match. + // Returns the loaded value. + Value *loadVectorFromVector(IRBuilder<> &B, FixedVectorType *SourceType, + FixedVectorType *TargetType, Value *Source) { + // We expect the codegen to avoid doing implicit bitcast from a load. + assert(TargetType->getElementType() == SourceType->getElementType()); + assert(TargetType->getNumElements() < SourceType->getNumElements()); + + LoadInst *NewLoad = B.CreateLoad(SourceType, Source); + buildAssignType(B, SourceType, NewLoad); + + SmallVector Mask(/* Size= */ TargetType->getNumElements(), + /* Value= */ 0); + Value *Output = B.CreateShuffleVector(NewLoad, NewLoad, Mask); + buildAssignType(B, TargetType, Output); + return Output; + } + + // Loads the first value in an aggregate pointed by |Source| of containing + // elements of type |ElementType|. Load flags will be copied from |BadLoad|, + // which should be the load being legalized. Returns the loaded value. + Value *loadFirstValueFromAggregate(IRBuilder<> &B, Type *ElementType, + Value *Source, LoadInst *BadLoad) { + SmallVector Types = {BadLoad->getPointerOperandType(), + BadLoad->getPointerOperandType()}; + SmallVector Args{/* isInBounds= */ B.getInt1(false), Source, + B.getInt32(0), B.getInt32(0)}; + auto *GEP = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args}); + GR->buildAssignPtr(B, ElementType, GEP); + + const auto *TLI = TM->getSubtargetImpl()->getTargetLowering(); + MachineMemOperand::Flags Flags = TLI->getLoadMemOperandFlags( + *BadLoad, BadLoad->getFunction()->getDataLayout()); + Instruction *LI = B.CreateIntrinsic( + Intrinsic::spv_load, {BadLoad->getOperand(0)->getType()}, + {GEP, B.getInt16(Flags), B.getInt8(BadLoad->getAlign().value())}); + buildAssignType(B, ElementType, LI); + return LI; + } + + // Replaces the load instruction to get rid of the ptrcast used as source + // operand. + void transformLoad(IRBuilder<> &B, LoadInst *LI, Value *CastedOperand, + Value *OriginalOperand) { + Type *FromTy = GR->findDeducedElementType(OriginalOperand); + Type *ToTy = GR->findDeducedElementType(CastedOperand); + Value *Output = nullptr; + + auto *SAT = dyn_cast(FromTy); + auto *SVT = dyn_cast(FromTy); + auto *DVT = dyn_cast(ToTy); + + B.SetInsertPoint(LI); + + // Destination is the element type of Source, and source is an array -> + // Loading 1st element. + // - float a = array[0]; + if (SAT && SAT->getElementType() == ToTy) + Output = loadFirstValueFromAggregate(B, SAT->getElementType(), + OriginalOperand, LI); + // Destination is the element type of Source, and source is a vector -> + // Vector to scalar. + // - float a = vector.x; + else if (!DVT && SVT && SVT->getElementType() == ToTy) { + Output = loadFirstValueFromAggregate(B, SVT->getElementType(), + OriginalOperand, LI); + } + // Destination is a smaller vector than source. + // - float3 v3 = vector4; + else if (SVT && DVT) + Output = loadVectorFromVector(B, SVT, DVT, OriginalOperand); + else + llvm_unreachable("Unimplemented implicit down-cast from load."); + + GR->replaceAllUsesWith(LI, Output, /* DeleteOld= */ true); + DeadInstructions.push_back(LI); + } + + void legalizePointerCast(IntrinsicInst *II) { + Value *CastedOperand = II; + Value *OriginalOperand = II->getOperand(0); + + IRBuilder<> B(II->getContext()); + std::vector Users; + for (Use &U : II->uses()) + Users.push_back(U.getUser()); + + for (Value *User : Users) { + if (LoadInst *LI = dyn_cast(User)) { + transformLoad(B, LI, CastedOperand, OriginalOperand); + continue; + } + + IntrinsicInst *Intrin = dyn_cast(User); + if (Intrin->getIntrinsicID() == Intrinsic::spv_assign_ptr_type) { + DeadInstructions.push_back(Intrin); + continue; + } + + llvm_unreachable("Unsupported ptrcast user. Please fix."); + } + + DeadInstructions.push_back(II); + } + +public: + SPIRVLegalizePointerCast(SPIRVTargetMachine *TM) : FunctionPass(ID), TM(TM) { + initializeSPIRVLegalizePointerCastPass(*PassRegistry::getPassRegistry()); + }; + + virtual bool runOnFunction(Function &F) override { + const SPIRVSubtarget &ST = TM->getSubtarget(F); + GR = ST.getSPIRVGlobalRegistry(); + DeadInstructions.clear(); + + std::vector WorkList; + for (auto &BB : F) { + for (auto &I : BB) { + auto *II = dyn_cast(&I); + if (II && II->getIntrinsicID() == Intrinsic::spv_ptrcast) + WorkList.push_back(II); + } + } + + for (IntrinsicInst *II : WorkList) + legalizePointerCast(II); + + for (Instruction *I : DeadInstructions) + I->eraseFromParent(); + + return DeadInstructions.size() != 0; + } + +private: + SPIRVTargetMachine *TM = nullptr; + SPIRVGlobalRegistry *GR = nullptr; + std::vector DeadInstructions; + +public: + static char ID; +}; + +char SPIRVLegalizePointerCast::ID = 0; +INITIALIZE_PASS(SPIRVLegalizePointerCast, "spirv-legalize-bitcast", + "SPIRV legalize bitcast pass", false, false) + +FunctionPass *llvm::createSPIRVLegalizePointerCastPass(SPIRVTargetMachine *TM) { + return new SPIRVLegalizePointerCast(TM); +} diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp index 098c7a6fba50e..0aa214dd354ee 100644 --- a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp @@ -209,6 +209,8 @@ void SPIRVPassConfig::addIRPasses() { void SPIRVPassConfig::addISelPrepare() { addPass(createSPIRVEmitIntrinsicsPass(&getTM())); + if (TM.getSubtargetImpl()->isVulkanEnv()) + addPass(createSPIRVLegalizePointerCastPass(&getTM())); TargetPassConfig::addISelPrepare(); } diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp index c55b735314228..fcfa022a062b2 100644 --- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp @@ -775,6 +775,17 @@ Register createVirtualRegister( MIRBuilder); } +CallInst *buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef Types, + Value *Arg, Value *Arg2, ArrayRef Imms, + IRBuilder<> &B) { + SmallVector Args; + Args.push_back(Arg2); + Args.push_back(buildMD(Arg)); + for (auto *Imm : Imms) + Args.push_back(Imm); + return B.CreateIntrinsic(IntrID, {Types}, Args); +} + // Return true if there is an opaque pointer type nested in the argument. bool isNestedPointer(const Type *Ty) { if (Ty->isPtrOrPtrVectorTy()) diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.h b/llvm/lib/Target/SPIRV/SPIRVUtils.h index 870649879218a..0817a01d0f0e0 100644 --- a/llvm/lib/Target/SPIRV/SPIRVUtils.h +++ b/llvm/lib/Target/SPIRV/SPIRVUtils.h @@ -405,6 +405,16 @@ inline PoisonValue *getNormalizedPoisonValue(Type *Ty) { return PoisonValue::get(normalizeType(Ty)); } +inline MetadataAsValue *buildMD(Value *Arg) { + LLVMContext &Ctx = Arg->getContext(); + return MetadataAsValue::get( + Ctx, MDNode::get(Ctx, ValueAsMetadata::getConstant(Arg))); +} + +CallInst *buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef Types, + Value *Arg, Value *Arg2, ArrayRef Imms, + IRBuilder<> &B); + MachineInstr *getVRegDef(MachineRegisterInfo &MRI, Register Reg); #define SPIRV_BACKEND_SERVICE_FUN_NAME "__spirv_backend_service_fun" diff --git a/llvm/test/CodeGen/SPIRV/pointers/array-skips-gep.ll b/llvm/test/CodeGen/SPIRV/pointers/array-skips-gep.ll new file mode 100644 index 0000000000000..85663d6c3eeda --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/pointers/array-skips-gep.ll @@ -0,0 +1,37 @@ +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %} + +; CHECK-DAG: %[[#uint_ty:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#uint_0:]] = OpConstant %[[#uint_ty]] 0 +; CHECK-DAG: %[[#int_10:]] = OpConstant %[[#uint_ty]] 10 +; CHECK-DAG: %[[#array_ty:]] = OpTypeArray %[[#uint_ty]] %[[#int_10]] +; CHECK-DAG: %[[#array_fp:]] = OpTypePointer Function %[[#array_ty]] +; CHECK-DAG: %[[#array_pp:]] = OpTypePointer Private %[[#array_ty]] +; CHECK-DAG: %[[#int_fp:]] = OpTypePointer Function %[[#uint_ty]] +; CHECK-DAG: %[[#int_pp:]] = OpTypePointer Private %[[#uint_ty]] + +@gv = external addrspace(10) global [10 x i32] +; CHECK: %[[#gv:]] = OpVariable %[[#array_pp]] Private + +define internal spir_func i32 @foo() { + %array = alloca [10 x i32], align 4 +; CHECK: %[[#array:]] = OpVariable %[[#array_fp:]] Function + + ; Direct load from the pointer index. This requires an OpAccessChain + %1 = load i32, ptr %array, align 4 +; CHECK: %[[#ptr:]] = OpAccessChain %[[#int_fp]] %[[#array]] %[[#uint_0]] +; CHECK: %[[#val:]] = OpLoad %[[#uint_ty]] %[[#ptr]] Aligned 4 + + ret i32 %1 +; CHECK: OpReturnValue %[[#val]] +} + +define internal spir_func i32 @bar() { + ; Direct load from the pointer index. This requires an OpAccessChain + %1 = load i32, ptr addrspace(10) @gv +; CHECK: %[[#ptr:]] = OpAccessChain %[[#int_pp]] %[[#gv]] %[[#uint_0]] +; CHECK: %[[#val:]] = OpLoad %[[#uint_ty]] %[[#ptr]] Aligned 4 + + ret i32 %1 +; CHECK: OpReturnValue %[[#val]] +} diff --git a/llvm/test/CodeGen/SPIRV/pointers/getelementptr-downcast-vector.ll b/llvm/test/CodeGen/SPIRV/pointers/getelementptr-downcast-vector.ll new file mode 100644 index 0000000000000..d4131fa8a2658 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/pointers/getelementptr-downcast-vector.ll @@ -0,0 +1,110 @@ +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan-compute %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan %s -o - -filetype=obj | spirv-val %} + +; CHECK-DAG: %[[#uint:]] = OpTypeInt 32 0 +; CHECK-DAG: %[[#uint_pp:]] = OpTypePointer Private %[[#uint]] +; CHECK-DAG: %[[#uint_fp:]] = OpTypePointer Function %[[#uint]] +; CHECK-DAG: %[[#uint_0:]] = OpConstant %[[#uint]] 0 +; CHECK-DAG: %[[#v2:]] = OpTypeVector %[[#uint]] 2 +; CHECK-DAG: %[[#v3:]] = OpTypeVector %[[#uint]] 3 +; CHECK-DAG: %[[#v4:]] = OpTypeVector %[[#uint]] 4 +; CHECK-DAG: %[[#v4_pp:]] = OpTypePointer Private %[[#v4]] +; CHECK-DAG: %[[#v4_fp:]] = OpTypePointer Function %[[#v4]] + +define internal spir_func <3 x i32> @foo(ptr addrspace(10) %a) { + + %1 = getelementptr inbounds <4 x i32>, ptr addrspace(10) %a, i64 0 +; CHECK: %[[#tmp:]] = OpInBoundsAccessChain %[[#v4_pp]] %[[#]] + + ; partial loading of a vector: v4 -> v3. + %2 = load <3 x i32>, ptr addrspace(10) %1, align 16 +; CHECK: %[[#load:]] = OpLoad %[[#v4]] %[[#tmp]] Aligned 16 +; CHECK: %[[#val:]] = OpVectorShuffle %[[#v3]] %[[#load]] %[[#load]] 0 0 0 + + ret <3 x i32> %2 +; CHECK: OpReturnValue %[[#val]] +} + +define internal spir_func <3 x i32> @fooDefault(ptr %a) { + + %1 = getelementptr inbounds <4 x i32>, ptr %a, i64 0 +; CHECK: %[[#tmp:]] = OpInBoundsAccessChain %[[#v4_fp]] %[[#]] + + ; partial loading of a vector: v4 -> v3. + %2 = load <3 x i32>, ptr %1, align 16 +; CHECK: %[[#load:]] = OpLoad %[[#v4]] %[[#tmp]] Aligned 16 +; CHECK: %[[#val:]] = OpVectorShuffle %[[#v3]] %[[#load]] %[[#load]] 0 0 0 + + ret <3 x i32> %2 +; CHECK: OpReturnValue %[[#val]] +} + +define internal spir_func <3 x i32> @fooBounds(ptr %a) { + + %1 = getelementptr <4 x i32>, ptr %a, i64 0 +; CHECK: %[[#tmp:]] = OpAccessChain %[[#v4_fp]] %[[#]] + + ; partial loading of a vector: v4 -> v3. + %2 = load <3 x i32>, ptr %1, align 16 +; CHECK: %[[#load:]] = OpLoad %[[#v4]] %[[#tmp]] Aligned 16 +; CHECK: %[[#val:]] = OpVectorShuffle %[[#v3]] %[[#load]] %[[#load]] 0 0 0 + + ret <3 x i32> %2 +; CHECK: OpReturnValue %[[#val]] +} + +define internal spir_func <2 x i32> @bar(ptr addrspace(10) %a) { + + %1 = getelementptr inbounds <4 x i32>, ptr addrspace(10) %a, i64 0 +; CHECK: %[[#tmp:]] = OpInBoundsAccessChain %[[#v4_pp]] %[[#]] + + ; partial loading of a vector: v4 -> v2. + %2 = load <2 x i32>, ptr addrspace(10) %1, align 16 +; CHECK: %[[#load:]] = OpLoad %[[#v4]] %[[#tmp]] Aligned 16 +; CHECK: %[[#val:]] = OpVectorShuffle %[[#v2]] %[[#load]] %[[#load]] 0 0 + + ret <2 x i32> %2 +; CHECK: OpReturnValue %[[#val]] +} + +define internal spir_func i32 @baz(ptr addrspace(10) %a) { + + %1 = getelementptr inbounds <4 x i32>, ptr addrspace(10) %a, i64 0 +; CHECK: %[[#tmp:]] = OpInBoundsAccessChain %[[#v4_pp]] %[[#]] + + ; Loading of the first scalar of a vector: v4 -> int. + %2 = load i32, ptr addrspace(10) %1, align 16 +; CHECK: %[[#ptr:]] = OpAccessChain %[[#uint_pp]] %[[#tmp]] %[[#uint_0]] +; CHECK: %[[#val:]] = OpLoad %[[#uint]] %[[#ptr]] Aligned 16 + + ret i32 %2 +; CHECK: OpReturnValue %[[#val]] +} + +define internal spir_func i32 @bazDefault(ptr %a) { + + %1 = getelementptr inbounds <4 x i32>, ptr %a, i64 0 +; CHECK: %[[#tmp:]] = OpInBoundsAccessChain %[[#v4_fp]] %[[#]] + + ; Loading of the first scalar of a vector: v4 -> int. + %2 = load i32, ptr %1, align 16 +; CHECK: %[[#ptr:]] = OpAccessChain %[[#uint_fp]] %[[#tmp]] %[[#uint_0]] +; CHECK: %[[#val:]] = OpLoad %[[#uint]] %[[#ptr]] Aligned 16 + + ret i32 %2 +; CHECK: OpReturnValue %[[#val]] +} + +define internal spir_func i32 @bazBounds(ptr %a) { + + %1 = getelementptr <4 x i32>, ptr %a, i64 0 +; CHECK: %[[#tmp:]] = OpAccessChain %[[#v4_fp]] %[[#]] + + ; Loading of the first scalar of a vector: v4 -> int. + %2 = load i32, ptr %1, align 16 +; CHECK: %[[#ptr:]] = OpAccessChain %[[#uint_fp]] %[[#tmp]] %[[#uint_0]] +; CHECK: %[[#val:]] = OpLoad %[[#uint]] %[[#ptr]] Aligned 16 + + ret i32 %2 +; CHECK: OpReturnValue %[[#val]] +}