Skip to content

Commit 67d4c99

Browse files
sdesmalen-armsdesmalen-arm
committed
Add support for (expressing) vscale.
In LLVM IR, vscale can be represented with an intrinsic. For some targets, this is equivalent to the constexpr: getelementptr <vscale x 1 x i8>, <vscale x 1 x i8>* null, i32 1 This can be used to propagate the value in CodeGenPrepare. In ISel we add a node that can be legalized to one or more instructions to materialize the runtime vector length. This patch also adds SVE CodeGen support for VSCALE, which maps this node to RDVL instructions (for scaled multiples of 16bytes) or CNT[HSD] instructions (scaled multiples of 2, 4, or 8 bytes, respectively). Reviewers: rengolin, cameron.mcinally, hfinkel, sebpop, SjoerdMeijer, efriedma, lattner Reviewed by: efriedma Tags: #llvm Differential Revision: https://reviews.llvm.org/D68203
1 parent e57a9ab commit 67d4c99

18 files changed

+402
-2
lines changed

‎llvm/docs/LangRef.rst

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17889,6 +17889,34 @@ information on the *based on* terminology see
1788917889
mask argument does not match the pointer size of the target, the mask is
1789017890
zero-extended or truncated accordingly.
1789117891

17892+
.. _int_vscale:
17893+
17894+
'``llvm.vscale``' Intrinsic
17895+
^^^^^^^^^^^^^^^^^^^^^^^^^^^
17896+
17897+
Syntax:
17898+
"""""""
17899+
17900+
::
17901+
17902+
declare i32 llvm.vscale.i32()
17903+
declare i64 llvm.vscale.i64()
17904+
17905+
Overview:
17906+
"""""""""
17907+
17908+
The ``llvm.vscale`` intrinsic returns the value for ``vscale`` in scalable
17909+
vectors such as ``<vscale x 16 x i8>``.
17910+
17911+
Semantics:
17912+
""""""""""
17913+
17914+
``vscale`` is a positive value that is constant throughout program
17915+
execution, but is unknown at compile time.
17916+
If the result value does not fit in the result type, then the result is
17917+
a :ref:`poison value <poisonvalues>`.
17918+
17919+
1789217920
Stack Map Intrinsics
1789317921
--------------------
1789417922

‎llvm/include/llvm/CodeGen/ISDOpcodes.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -921,6 +921,11 @@ namespace ISD {
921921
/// known nonzero constant. The only operand here is the chain.
922922
GET_DYNAMIC_AREA_OFFSET,
923923

924+
/// VSCALE(IMM) - Returns the runtime scaling factor used to calculate the
925+
/// number of elements within a scalable vector. IMM is a constant integer
926+
/// multiplier that is applied to the runtime value.
927+
VSCALE,
928+
924929
/// Generic reduction nodes. These nodes represent horizontal vector
925930
/// reduction operations, producing a scalar result.
926931
/// The STRICT variants perform reductions in sequential order. The first

‎llvm/include/llvm/CodeGen/SelectionDAG.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -914,6 +914,13 @@ class SelectionDAG {
914914
return getNode(ISD::UNDEF, SDLoc(), VT);
915915
}
916916

917+
/// Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
918+
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm) {
919+
assert(MulImm.getMinSignedBits() <= VT.getSizeInBits() &&
920+
"Immediate does not fit VT");
921+
return getNode(ISD::VSCALE, DL, VT, getConstant(MulImm, DL, VT));
922+
}
923+
917924
/// Return a GLOBAL_OFFSET_TABLE node. This does not have a useful SDLoc.
918925
SDValue getGLOBAL_OFFSET_TABLE(EVT VT) {
919926
return getNode(ISD::GLOBAL_OFFSET_TABLE, SDLoc(), VT);

‎llvm/include/llvm/IR/Intrinsics.td

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1342,6 +1342,11 @@ def int_preserve_struct_access_index : Intrinsic<[llvm_anyptr_ty],
13421342
[IntrNoMem, ImmArg<1>,
13431343
ImmArg<2>]>;
13441344

1345+
//===---------- Intrinsics to query properties of scalable vectors --------===//
1346+
def int_vscale : Intrinsic<[llvm_anyint_ty], [], [IntrNoMem]>;
1347+
1348+
//===----------------------------------------------------------------------===//
1349+
13451350
//===----------------------------------------------------------------------===//
13461351
// Target-specific intrinsics
13471352
//===----------------------------------------------------------------------===//

‎llvm/include/llvm/IR/PatternMatch.h

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
#include "llvm/ADT/APInt.h"
3333
#include "llvm/IR/Constant.h"
3434
#include "llvm/IR/Constants.h"
35+
#include "llvm/IR/DataLayout.h"
3536
#include "llvm/IR/InstrTypes.h"
3637
#include "llvm/IR/Instruction.h"
3738
#include "llvm/IR/Instructions.h"
@@ -2002,6 +2003,42 @@ inline ExtractValue_match<Ind, Val_t> m_ExtractValue(const Val_t &V) {
20022003
return ExtractValue_match<Ind, Val_t>(V);
20032004
}
20042005

2006+
/// Matches patterns for `vscale`. This can either be a call to `llvm.vscale` or
2007+
/// the constant expression
2008+
/// `ptrtoint(gep <vscale x 1 x i8>, <vscale x 1 x i8>* null, i32 1>`
2009+
/// under the right conditions determined by DataLayout.
2010+
struct VScaleVal_match {
2011+
private:
2012+
template <typename Base, typename Offset>
2013+
inline BinaryOp_match<Base, Offset, Instruction::GetElementPtr>
2014+
m_OffsetGep(const Base &B, const Offset &O) {
2015+
return BinaryOp_match<Base, Offset, Instruction::GetElementPtr>(B, O);
2016+
}
2017+
2018+
public:
2019+
const DataLayout &DL;
2020+
VScaleVal_match(const DataLayout &DL) : DL(DL) {}
2021+
2022+
template <typename ITy> bool match(ITy *V) {
2023+
if (m_Intrinsic<Intrinsic::vscale>().match(V))
2024+
return true;
2025+
2026+
if (m_PtrToInt(m_OffsetGep(m_Zero(), m_SpecificInt(1))).match(V)) {
2027+
Type *PtrTy = cast<Operator>(V)->getOperand(0)->getType();
2028+
Type *DerefTy = PtrTy->getPointerElementType();
2029+
if (DerefTy->isVectorTy() && DerefTy->getVectorIsScalable() &&
2030+
DL.getTypeAllocSizeInBits(DerefTy).getKnownMinSize() == 8)
2031+
return true;
2032+
}
2033+
2034+
return false;
2035+
}
2036+
};
2037+
2038+
inline VScaleVal_match m_VScale(const DataLayout &DL) {
2039+
return VScaleVal_match(DL);
2040+
}
2041+
20052042
} // end namespace PatternMatch
20062043
} // end namespace llvm
20072044

‎llvm/include/llvm/Target/TargetSelectionDAG.td

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -316,6 +316,7 @@ def vt : SDNode<"ISD::VALUETYPE" , SDTOther , [], "VTSDNode">;
316316
def bb : SDNode<"ISD::BasicBlock", SDTOther , [], "BasicBlockSDNode">;
317317
def cond : SDNode<"ISD::CONDCODE" , SDTOther , [], "CondCodeSDNode">;
318318
def undef : SDNode<"ISD::UNDEF" , SDTUNDEF , []>;
319+
def vscale : SDNode<"ISD::VSCALE" , SDTIntUnaryOp, []>;
319320
def globaladdr : SDNode<"ISD::GlobalAddress", SDTPtrLeaf, [],
320321
"GlobalAddressSDNode">;
321322
def tglobaladdr : SDNode<"ISD::TargetGlobalAddress", SDTPtrLeaf, [],

‎llvm/lib/Analysis/ConstantFolding.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -828,7 +828,8 @@ Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP,
828828
Type *SrcElemTy = GEP->getSourceElementType();
829829
Type *ResElemTy = GEP->getResultElementType();
830830
Type *ResTy = GEP->getType();
831-
if (!SrcElemTy->isSized())
831+
if (!SrcElemTy->isSized() ||
832+
(SrcElemTy->isVectorTy() && SrcElemTy->getVectorIsScalable()))
832833
return nullptr;
833834

834835
if (Constant *C = CastGEPIndices(SrcElemTy, Ops, ResTy,

‎llvm/lib/CodeGen/CodeGenPrepare.cpp

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2010,6 +2010,22 @@ bool CodeGenPrepare::optimizeCallInst(CallInst *CI, bool &ModifiedDT) {
20102010
return despeculateCountZeros(II, TLI, DL, ModifiedDT);
20112011
case Intrinsic::dbg_value:
20122012
return fixupDbgValue(II);
2013+
case Intrinsic::vscale: {
2014+
// If datalayout has no special restrictions on vector data layout,
2015+
// replace `llvm.vscale` by an equivalent constant expression
2016+
// to benefit from cheap constant propagation.
2017+
Type *ScalableVectorTy =
2018+
VectorType::get(Type::getInt8Ty(II->getContext()), 1, true);
2019+
if (DL->getTypeAllocSize(ScalableVectorTy).getKnownMinSize() == 8) {
2020+
auto Null = Constant::getNullValue(ScalableVectorTy->getPointerTo());
2021+
auto One = ConstantInt::getSigned(II->getType(), 1);
2022+
auto *CGep =
2023+
ConstantExpr::getGetElementPtr(ScalableVectorTy, Null, One);
2024+
II->replaceAllUsesWith(ConstantExpr::getPtrToInt(CGep, II->getType()));
2025+
II->eraseFromParent();
2026+
return true;
2027+
}
2028+
}
20132029
}
20142030

20152031
if (TLI) {

‎llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
9191
case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
9292
case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
9393
case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
94+
case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break;
9495

9596
case ISD::EXTRACT_SUBVECTOR:
9697
Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
@@ -1179,6 +1180,13 @@ SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
11791180
N->getValueType(0)));
11801181
}
11811182

1183+
SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) {
1184+
EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1185+
1186+
APInt MulImm = cast<ConstantSDNode>(N->getOperand(0))->getAPIntValue();
1187+
return DAG.getVScale(SDLoc(N), VT, MulImm.sextOrSelf(VT.getSizeInBits()));
1188+
}
1189+
11821190
SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
11831191
SDValue Chain = N->getOperand(0); // Get the chain.
11841192
SDValue Ptr = N->getOperand(1); // Get the pointer.

‎llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
326326
SDValue PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo);
327327
SDValue PromoteIntRes_UNDEF(SDNode *N);
328328
SDValue PromoteIntRes_VAARG(SDNode *N);
329+
SDValue PromoteIntRes_VSCALE(SDNode *N);
329330
SDValue PromoteIntRes_XMULO(SDNode *N, unsigned ResNo);
330331
SDValue PromoteIntRes_ADDSUBSAT(SDNode *N);
331332
SDValue PromoteIntRes_MULFIX(SDNode *N);

0 commit comments

Comments
 (0)