50#include "llvm/IR/IntrinsicsNVPTX.h"
76#define DEBUG_TYPE "nvptx-lower"
86 cl::desc(
"NVPTX Specific: FMA contraction (0: don't do it"
87 " 1: do it 2: do it aggressively"),
93 "NVPTX Specific: Override the precision of the lowering for f32 fdiv"),
98 "Use IEEE Compliant F32 div.rnd if available (default)"),
100 "Use IEEE Compliant F32 div.rnd if available, no FTZ")),
105 cl::desc(
"NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."),
111 "nvptx-approx-log2f32",
112 cl::desc(
"NVPTX Specific: whether to use lg2.approx for log2"),
116 "nvptx-force-min-byval-param-align",
cl::Hidden,
117 cl::desc(
"NVPTX Specific: force 4-byte minimal alignment for byval"
118 " params of device functions."),
129 if (Flags.hasApproximateFuncs())
142 if (Flags.hasApproximateFuncs())
198static std::optional<std::pair<unsigned int, MVT>>
205 return {{4, MVT::i64}};
212 if (VectorVT == MVT::i128 || VectorVT == MVT::f128)
213 return {{2, MVT::i64}};
221 unsigned PackRegSize;
234 if (!CanLowerTo256Bit)
241 return std::pair(NumElts, EltVT);
249 if (!CanLowerTo256Bit)
271 if (!CanLowerTo256Bit)
279 return std::pair(NumElts, EltVT);
289 const unsigned NPerReg = PackRegSize / EltVT.
getSizeInBits();
310 for (
const auto [VT, Off] :
zip(TempVTs, TempOffsets)) {
316 if (VT.getScalarType() == MVT::i8) {
317 if (RegisterVT == MVT::i16)
318 RegisterVT = MVT::i8;
319 else if (RegisterVT == MVT::v2i16)
320 RegisterVT = MVT::v2i8;
322 assert(RegisterVT == MVT::v4i8 &&
323 "Expected v4i8, v2i16, or i16 for i8 RegisterVT");
330 for (
unsigned I :
seq(NumRegs)) {
351 if (V.getValueType() == VT) {
352 assert(
I == 0 &&
"Index must be 0 for scalar value");
369 return GetElement(0);
395 "Promotion is not suitable for scalars of size larger than 64-bits");
429 if (ParamAlignment < AccessSize)
432 if (Offsets[Idx] & (AccessSize - 1))
435 EVT EltVT = ValueVTs[Idx];
439 if (EltSize >= AccessSize)
442 unsigned NumElts = AccessSize / EltSize;
444 if (AccessSize != EltSize * NumElts)
448 if (Idx + NumElts > ValueVTs.
size())
452 if (NumElts != 4 && NumElts != 2)
455 for (
unsigned j = Idx + 1; j < Idx + NumElts; ++j) {
457 if (ValueVTs[j] != EltVT)
461 if (Offsets[j] - Offsets[j - 1] != EltSize)
480 bool IsVAArg =
false) {
489 const auto GetNumElts = [&](
unsigned I) ->
unsigned {
490 for (
const unsigned AccessSize : {16, 8, 4, 2}) {
492 I, AccessSize, ValueVTs, Offsets, ParamAlignment);
493 assert((NumElts == 1 || NumElts == 2 || NumElts == 4) &&
494 "Unexpected vectorization size");
502 for (
unsigned I = 0,
E = ValueVTs.
size();
I !=
E;) {
503 const unsigned NumElts = GetNumElts(
I);
504 VectorInfo.push_back(NumElts);
507 assert(std::accumulate(VectorInfo.begin(), VectorInfo.end(), 0u) ==
542 bool IsOpSupported = STI.allowFP16Math();
547 case ISD::FMAXNUM_IEEE:
548 case ISD::FMINNUM_IEEE:
551 case ISD::FMAXIMUMNUM:
552 case ISD::FMINIMUMNUM:
553 IsOpSupported &= STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70;
556 IsOpSupported &= STI.getSmVersion() >= 75 && STI.getPTXVersion() >= 70;
564 bool IsOpSupported = STI.hasNativeBF16Support(
Op);
566 Op, VT, IsOpSupported ? Action : NoBF16Action);
571 bool IsOpSupported =
false;
579 IsOpSupported = STI.getSmVersion() >= 90 && STI.getPTXVersion() >= 80;
598 if (STI.hasF32x2Instructions()) {
610 if (STI.getSmVersion() >= 30 && STI.getPTXVersion() > 31)
642 if (STI.hasF32x2Instructions())
670 for (
MVT VT : {MVT::bf16, MVT::f16, MVT::v2bf16, MVT::v2f16, MVT::f32,
671 MVT::v2f32, MVT::f64, MVT::i1, MVT::i8, MVT::i16, MVT::v2i16,
672 MVT::v4i8, MVT::i32, MVT::v2i32, MVT::i64}) {
700 {MVT::i8, MVT::i16, MVT::v2i16, MVT::i32, MVT::i64},
703 if (STI.hasHWROT32()) {
721 for (
MVT ValVT : FloatVTs) {
722 for (
MVT MemVT : FloatVTs) {
734 for (
MVT ValVT : IntVTs)
735 for (
MVT MemVT : IntVTs)
762 if (!
isTypeLegal(VT) && VT.getStoreSizeInBits() <= 256)
799 {MVT::i16, MVT::i32, MVT::i64},
Legal);
825 {MVT::v2i16, MVT::v2i32},
Expand);
838 if (STI.getPTXVersion() >= 43) {
861 ISD::FMAXIMUM, ISD::FMINIMUM, ISD::FMAXIMUMNUM,
869 if (STI.allowFP16Math() || STI.hasBF16Math())
876 if (EltVT == MVT::f32 || EltVT == MVT::f64) {
878 ISD::VECREDUCE_FMAXIMUM, ISD::VECREDUCE_FMINIMUM},
903 for (
const auto &VT : {MVT::bf16, MVT::v2bf16}) {
904 if (!STI.hasNativeBF16Support(
Op) && STI.hasNativeBF16Support(
ISD::FMA)) {
911 const bool IsFP16FP16x2NegAvailable = STI.getSmVersion() >= 53 &&
912 STI.getPTXVersion() >= 60 &&
914 for (
const auto &VT : {MVT::f16, MVT::v2f16})
918 setBF16OperationAction(ISD::FNEG, MVT::bf16,
Legal,
Expand);
919 setBF16OperationAction(ISD::FNEG, MVT::v2bf16,
Legal,
Expand);
924 for (
const auto &
Op : {ISD::FCEIL, ISD::FFLOOR, ISD::FNEARBYINT, ISD::FRINT,
925 ISD::FROUNDEVEN, ISD::FTRUNC}) {
937 if (STI.getSmVersion() < 80 || STI.getPTXVersion() < 71) {
940 if (STI.getSmVersion() < 90 || STI.getPTXVersion() < 78) {
941 for (
MVT VT : {MVT::bf16, MVT::f32, MVT::f64}) {
954 if (STI.getSmVersion() < 90 || STI.getPTXVersion() < 78) {
955 for (
MVT VT : {MVT::i1, MVT::i16, MVT::i32, MVT::i64}) {
984 for (
const auto &
Op :
1000 if (STI.getPTXVersion() >= 65) {
1001 setFP16OperationAction(ISD::FABS, MVT::f16,
Legal,
Promote);
1002 setFP16OperationAction(ISD::FABS, MVT::v2f16,
Legal,
Expand);
1007 setBF16OperationAction(ISD::FABS, MVT::v2bf16,
Legal,
Expand);
1008 setBF16OperationAction(ISD::FABS, MVT::bf16,
Legal,
Promote);
1012 for (
const auto &
Op :
1013 {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM}) {
1024 bool SupportsF32MinMaxNaN =
1025 STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70;
1026 for (
const auto &
Op : {ISD::FMINIMUM, ISD::FMAXIMUM}) {
1046 setFP16OperationAction(ISD::FEXP2, MVT::f16,
Legal,
Promote);
1047 setFP16OperationAction(ISD::FEXP2, MVT::v2f16,
Legal,
Expand);
1048 setBF16OperationAction(ISD::FEXP2, MVT::bf16,
Legal,
Promote);
1049 setBF16OperationAction(ISD::FEXP2, MVT::v2bf16,
Legal,
Expand);
1081 {MVT::v2i32, MVT::v4i32, MVT::v8i32, MVT::v16i32,
1082 MVT::v32i32, MVT::v64i32, MVT::v128i32},
1087 {MVT::v2i32, MVT::v4i32, MVT::v8i32, MVT::v16i32,
1088 MVT::v32i32, MVT::v64i32, MVT::v128i32, MVT::Other},
1101#define MAKE_CASE(V) \
1196 bool Reciprocal)
const {
1217 if (Reciprocal || ExtraSteps > 0) {
1219 return MakeIntrinsicCall(Ftz ? Intrinsic::nvvm_rsqrt_approx_ftz_f
1220 : Intrinsic::nvvm_rsqrt_approx_f);
1221 else if (VT == MVT::f64)
1222 return MakeIntrinsicCall(Intrinsic::nvvm_rsqrt_approx_d);
1227 return MakeIntrinsicCall(Ftz ? Intrinsic::nvvm_sqrt_approx_ftz_f
1228 : Intrinsic::nvvm_sqrt_approx_f);
1236 DAG.
getConstant(Intrinsic::nvvm_rcp_approx_ftz_d,
DL, MVT::i32),
1237 MakeIntrinsicCall(Intrinsic::nvvm_rsqrt_approx_d));
1245 std::optional<unsigned> FirstVAArg,
const CallBase &CB,
1246 unsigned UniqueCallSite)
const {
1249 std::string Prototype;
1251 O <<
"prototype_" << UniqueCallSite <<
" : .callprototype ";
1258 const Align RetAlign = getArgumentAlignment(&CB, RetTy, 0,
DL);
1259 O <<
".param .align " << RetAlign.
value() <<
" .b8 _["
1260 <<
DL.getTypeAllocSize(RetTy) <<
"]";
1264 size = ITy->getBitWidth();
1267 "Floating point type expected here");
1275 O <<
".param .b" <<
size <<
" _";
1277 O <<
".param .b" << PtrVT.getSizeInBits() <<
" _";
1287 const unsigned NumArgs = FirstVAArg.value_or(Args.size());
1289 for (
const unsigned I :
llvm::seq(NumArgs)) {
1290 const auto ArgOuts =
1291 AllOuts.take_while([
I](
auto O) {
return O.OrigArgIndex ==
I; });
1292 AllOuts = AllOuts.drop_front(ArgOuts.size());
1294 Type *Ty = Args[
I].Ty;
1300 if (ArgOuts[0].Flags.isByVal()) {
1303 Type *ETy = Args[
I].IndirectType;
1304 Align InitialAlign = ArgOuts[0].Flags.getNonZeroByValAlign();
1305 Align ParamByValAlign =
1308 O <<
".param .align " << ParamByValAlign.
value() <<
" .b8 _["
1309 << ArgOuts[0].Flags.getByValSize() <<
"]";
1313 getArgumentAlignment(&CB, Ty,
I + AttributeList::FirstArgIndex,
DL);
1314 O <<
".param .align " << ParamAlign.
value() <<
" .b8 _["
1315 <<
DL.getTypeAllocSize(Ty) <<
"]";
1320 (
getValueType(
DL, Ty) == MVT::i8 && ArgOuts[0].VT == MVT::i16)) &&
1321 "type mismatch between callee prototype and arguments");
1327 sz = PtrVT.getSizeInBits();
1329 sz = Ty->getPrimitiveSizeInBits();
1331 O <<
".param .b" << sz <<
" _";
1336 O << (first ?
"" :
",") <<
" .param .align "
1337 << STI.getMaxRequiredAlignment() <<
" .b8 _[]";
1356 return DL.getABITypeAlign(Ty);
1361 if (!DirectCallee) {
1369 return StackAlign.value();
1380 return DL.getABITypeAlign(Ty);
1405 if (
Ptr->getOpcode() == ISD::ADDRSPACECAST) {
1408 Ptr = ASC->getOperand(0);
1427 const EVT ActualVT = V.getValueType();
1428 assert((ActualVT == ExpectedVT ||
1430 "Non-integer argument type size mismatch");
1431 if (ExpectedVT.
bitsGT(ActualVT))
1433 if (ExpectedVT.
bitsLT(ActualVT))
1442 if (CLI.
IsVarArg && (STI.getPTXVersion() < 60 || STI.getSmVersion() < 30))
1444 "Support for variadic functions (unsized array parameter) introduced "
1445 "in PTX ISA version 6.0 and requires target sm_30.");
1457 const auto GetI32 = [&](
const unsigned I) {
1461 const unsigned UniqueCallSite = GlobalUniqueCallSite++;
1469 const auto MakeDeclareScalarParam = [&](
SDValue Symbol,
unsigned Size) {
1475 {StartChain, Symbol, GetI32(SizeBits), DeclareGlue});
1485 {StartChain, Symbol, GetI32(
Align.
value()), GetI32(
Size), DeclareGlue});
1507 "Non-VarArg function with extra arguments");
1510 unsigned VAOffset = 0;
1512 const SDValue VADeclareParam =
1513 CLI.
Args.size() > FirstVAArg
1514 ? MakeDeclareArrayParam(getCallParamSymbol(DAG, FirstVAArg, MVT::i32),
1515 Align(STI.getMaxRequiredAlignment()), 0)
1529 assert(AllOuts.size() == AllOutVals.size() &&
1530 "Outs and OutVals must be the same size");
1534 const auto ArgI = E.index();
1535 const auto Arg = E.value();
1536 const auto ArgOuts =
1537 AllOuts.take_while([&](
auto O) {
return O.OrigArgIndex == ArgI; });
1538 const auto ArgOutVals = AllOutVals.take_front(ArgOuts.size());
1539 AllOuts = AllOuts.drop_front(ArgOuts.size());
1540 AllOutVals = AllOutVals.drop_front(ArgOuts.size());
1542 const bool IsVAArg = (ArgI >= FirstVAArg);
1543 const bool IsByVal = Arg.IsByVal;
1546 getCallParamSymbol(DAG, IsVAArg ? FirstVAArg : ArgI, MVT::i32);
1548 assert((!IsByVal || Arg.IndirectType) &&
1549 "byval arg must have indirect type");
1550 Type *ETy = (IsByVal ? Arg.IndirectType : Arg.Ty);
1552 const Align ArgAlign = [&]() {
1557 const Align InitialAlign = ArgOuts[0].Flags.getNonZeroByValAlign();
1561 return getArgumentAlignment(CB, Arg.Ty, ArgI + 1,
DL);
1564 const unsigned TySize =
DL.getTypeAllocSize(ETy);
1565 assert((!IsByVal || TySize == ArgOuts[0].Flags.getByValSize()) &&
1566 "type size mismatch");
1568 const SDValue ArgDeclare = [&]() {
1570 return VADeclareParam;
1573 return MakeDeclareArrayParam(ParamSymbol, ArgAlign, TySize);
1575 assert(ArgOuts.size() == 1 &&
"We must pass only one value as non-array");
1576 assert((ArgOuts[0].VT.isInteger() || ArgOuts[0].VT.isFloatingPoint()) &&
1577 "Only int and float types are supported as non-array arguments");
1579 return MakeDeclareScalarParam(ParamSymbol, TySize);
1583 assert(ArgOutVals.size() == 1 &&
"We must pass only one value as byval");
1584 SDValue SrcPtr = ArgOutVals[0];
1585 const auto PointerInfo =
refinePtrAS(SrcPtr, DAG,
DL, *
this);
1586 const Align BaseSrcAlign = ArgOuts[0].Flags.getNonZeroByValAlign();
1589 VAOffset =
alignTo(VAOffset, ArgAlign);
1597 for (
const unsigned NumElts : VI) {
1602 DAG.
getLoad(LoadVT, dl, CallChain, SrcAddr, PointerInfo, SrcAlign);
1604 TypeSize ParamOffset = Offsets[J].getWithIncrement(VAOffset);
1609 DAG.
getStore(ArgDeclare, dl, SrcLoad, ParamAddr,
1622 assert(VTs.
size() == Offsets.size() &&
"Size mismatch");
1623 assert(VTs.
size() == ArgOuts.size() &&
"Size mismatch");
1629 const bool ExtendIntegerParam =
1630 Arg.Ty->isIntegerTy() &&
DL.getTypeAllocSizeInBits(Arg.Ty) < 32;
1632 const auto GetStoredValue = [&](
const unsigned I) {
1636 "OutVal type should always be legal");
1640 ExtendIntegerParam ? MVT::i32 : (VTI == MVT::i1 ? MVT::i8 : VTI);
1647 for (
const unsigned NumElts : VI) {
1655 "Vectorization should be disabled for vaargs.");
1661 const EVT TheStoreType = ExtendIntegerParam ? MVT::i32 : EltVT;
1664 assert(VAOffset == 0 &&
"VAOffset must be 0 for non-VA args");
1671 const MaybeAlign CurrentAlign = ExtendIntegerParam
1677 return GetStoredValue(J + K);
1693 const unsigned ResultSize =
DL.getTypeAllocSize(RetTy);
1695 const Align RetAlign = getArgumentAlignment(CB, RetTy, 0,
DL);
1696 MakeDeclareArrayParam(RetSymbol, RetAlign, ResultSize);
1698 MakeDeclareScalarParam(RetSymbol, ResultSize);
1704 if (VADeclareParam) {
1707 VADeclareParam.
getOperand(2), GetI32(VAOffset),
1710 VADeclareParam->
getVTList(), DeclareParamOps);
1721 const bool IsIndirectCall = (!Func && CB) || ConvertToIndirectCall;
1728 assert(CalleeFunc !=
nullptr &&
"Libcall callee must be set.");
1732 CalleeFunc->
addFnAttr(
"nvptx-libcall-callee",
"true");
1735 if (IsIndirectCall) {
1746 HasVAArgs ? std::optional(FirstVAArg) : std::nullopt, *CB,
1748 const char *ProtoStr =
nvTM->getStrPool().save(Proto).data();
1752 CallPrereqs.
push_back(PrototypeDeclare);
1755 const unsigned Proto = IsIndirectCall ? UniqueCallSite : 0;
1756 const unsigned NumArgs =
1763 {CallToken, GetI32(CLI.
IsConvergent), GetI32(IsIndirectCall),
1764 GetI32(Ins.empty() ? 0 : 1), GetI32(NumArgs), Callee, GetI32(Proto)});
1772 assert(VTs.
size() == Ins.size() &&
"Bad value decomposition");
1774 const Align RetAlign = getArgumentAlignment(CB, RetTy, 0,
DL);
1780 const bool ExtendIntegerRetVal =
1781 RetTy->
isIntegerTy() &&
DL.getTypeAllocSizeInBits(RetTy) < 32;
1785 for (
const unsigned NumElts : VI) {
1787 ExtendIntegerRetVal ?
MaybeAlign(std::nullopt)
1792 ExtendIntegerRetVal ? MVT::i32 : (VTI == MVT::i1 ? MVT::i8 : VTI);
1802 for (
const unsigned J :
llvm::seq(NumElts))
1810 UniqueCallSite + 1,
SDValue(), dl);
1831 if (STI.getPTXVersion() < 73 || STI.getSmVersion() < 52) {
1836 "Support for dynamic alloca introduced in PTX ISA version 7.3 and "
1837 "requires target sm_52.",
1871 if (STI.getPTXVersion() < 73 || STI.getSmVersion() < 52) {
1876 "Support for stackrestore requires PTX ISA version >= 7.3 and target "
1879 return Op.getOperand(0);
1893 if (STI.getPTXVersion() < 73 || STI.getSmVersion() < 52) {
1898 "Support for stacksave requires PTX ISA version >= 7.3 and target >= "
1922 unsigned NumOperands =
Node->getNumOperands();
1923 for (
unsigned i = 0; i < NumOperands; ++i) {
1925 EVT VVT = SubOp.getNode()->getValueType(0);
1928 for (
unsigned j = 0; j < NumSubElem; ++j) {
1939 assert(
A.getValueType() == MVT::i32 &&
B.getValueType() == MVT::i32 &&
1940 Selector.
getValueType() == MVT::i32 &&
"PRMT must have i32 operands");
1958 ArrayRef<std::pair<unsigned /*NodeType*/, unsigned /*NumInputs*/>>
Ops,
1964 while (Level.size() > 1) {
1970 unsigned I = 0,
E = Level.size();
1971 for (;
I + NumInputs <=
E;
I += NumInputs) {
1980 if (ReducedLevel.
empty()) {
1984 assert(
OpIdx <
Ops.size() &&
"no smaller operators for reduction");
1996 Level = ReducedLevel;
1999 return *Level.begin();
2004 switch (ReductionOpcode) {
2005 case ISD::VECREDUCE_FMAX:
2006 return ISD::FMAXNUM;
2007 case ISD::VECREDUCE_FMIN:
2008 return ISD::FMINNUM;
2009 case ISD::VECREDUCE_FMAXIMUM:
2010 return ISD::FMAXIMUM;
2011 case ISD::VECREDUCE_FMINIMUM:
2012 return ISD::FMINIMUM;
2019static std::optional<NVPTXISD::NodeType>
2021 switch (ReductionOpcode) {
2022 case ISD::VECREDUCE_FMAX:
2024 case ISD::VECREDUCE_FMIN:
2026 case ISD::VECREDUCE_FMAXIMUM:
2028 case ISD::VECREDUCE_FMINIMUM:
2031 return std::nullopt;
2041 const SDNodeFlags
Flags =
Op->getFlags();
2044 const unsigned Opcode =
Op->getOpcode();
2045 const EVT EltTy =
Vector.getValueType().getVectorElementType();
2048 const bool CanUseMinMax3 =
2049 EltTy == MVT::f32 && STI.getSmVersion() >= 100 &&
2050 STI.getPTXVersion() >= 88 &&
2051 (Opcode == ISD::VECREDUCE_FMAX || Opcode == ISD::VECREDUCE_FMIN ||
2052 Opcode == ISD::VECREDUCE_FMAXIMUM || Opcode == ISD::VECREDUCE_FMINIMUM);
2056 SmallVector<std::pair<
unsigned ,
unsigned >, 2> ScalarOps;
2059 CanUseMinMax3 && Opcode3Elem)
2060 ScalarOps.push_back({*Opcode3Elem, 3});
2072 EVT FromVT =
Op->getOperand(0)->getValueType(0);
2073 if (FromVT != MVT::v2i8) {
2089 EVT ToVT =
Op->getValueType(0);
2099 EVT VT =
Op->getValueType(0);
2105 return Operand->isUndef() || isa<ConstantSDNode>(Operand) ||
2106 isa<ConstantFPSDNode>(Operand);
2108 if (VT != MVT::v4i8)
2113 uint64_t SelectionValue) ->
SDValue {
2120 return getPRMT(L, R, SelectionValue,
DL, DAG);
2122 auto PRMT__10 = GetPRMT(
Op->getOperand(0),
Op->getOperand(1),
true, 0x3340);
2123 auto PRMT__32 = GetPRMT(
Op->getOperand(2),
Op->getOperand(3),
true, 0x3340);
2124 auto PRMT3210 = GetPRMT(PRMT__10, PRMT__32,
false, 0x5410);
2129 auto GetOperand = [](
SDValue Op,
int N) -> APInt {
2131 EVT VT =
Op->getValueType(0);
2133 return APInt(32, 0);
2135 if (VT == MVT::v2f16 || VT == MVT::v2bf16)
2137 else if (VT == MVT::v2i16 || VT == MVT::v4i8)
2143 if (VT == MVT::v4i8)
2145 return Value.zext(32);
2163 assert(32 % NumElements == 0 &&
"must evenly divide bit length");
2164 const unsigned ShiftAmount = 32 / NumElements;
2165 for (
unsigned ElementNo :
seq(NumElements))
2166 Value |= GetOperand(
Op, ElementNo).shl(ElementNo * ShiftAmount);
2168 return DAG.
getNode(ISD::BITCAST,
DL,
Op->getValueType(0), Const);
2176 EVT VectorVT =
Vector.getValueType();
2178 if (VectorVT == MVT::v4i8) {
2186 Flags.setNoSignedWrap(
Ext.getScalarValueSizeInBits() > 8);
2187 Flags.setNoUnsignedWrap(
Ext.getScalarValueSizeInBits() >= 8);
2188 Ext->setFlags(Flags);
2201 SDLoc dl(
Op.getNode());
2213 EVT VectorVT =
Vector.getValueType();
2215 if (VectorVT != MVT::v4i8)
2219 if (
Value->isUndef())
2231 return DAG.
getNode(ISD::BITCAST,
DL,
Op->getValueType(0), BFI);
2238 if (VectorVT != MVT::v4i8 ||
Op.getValueType() != MVT::v4i8)
2244 uint32_t Selector = 0;
2246 if (
I.value() != -1)
2247 Selector |= (
I.value() << (
I.index() * 4));
2265 EVT VT =
Op.getValueType();
2273 if (VTBits == 32 && STI.getSmVersion() >= 35) {
2326 EVT VT =
Op.getValueType();
2333 if (VTBits == 32 && STI.getSmVersion() >= 35) {
2380 EVT VT =
Op.getValueType();
2394 EVT VT =
Op.getValueType();
2397 return LowerFROUND32(
Op, DAG);
2400 return LowerFROUND64(
Op, DAG);
2416 EVT VT =
Op.getValueType();
2422 const unsigned SignBitMask = 0x80000000;
2425 const unsigned PointFiveInBits = 0x3F000000;
2426 SDValue PointFiveWithSignRaw =
2430 DAG.
getNode(ISD::BITCAST, SL, VT, PointFiveWithSignRaw);
2457 EVT VT =
Op.getValueType();
2476 DAG.
getNode(ISD::FTRUNC, SL, VT,
A);
2486 EVT VT =
N->getValueType(0);
2508 assert(STI.getSmVersion() < 90 || STI.getPTXVersion() < 78);
2510 if (
Op.getValueType() == MVT::bf16) {
2514 DAG.
getNode(
Op.getOpcode(), Loc, MVT::f32,
Op.getOperand(0)),
2524 assert(STI.getSmVersion() < 90 || STI.getPTXVersion() < 78);
2526 if (
Op.getOperand(0).getValueType() == MVT::bf16) {
2529 Op.getOpcode(), Loc,
Op.getValueType(),
2530 DAG.
getNode(ISD::FP_EXTEND, Loc, MVT::f32,
Op.getOperand(0)));
2539 EVT NarrowVT =
Op.getValueType();
2544 if (STI.getSmVersion() < 80 || STI.getPTXVersion() < 70) {
2547 if (STI.getSmVersion() < 90 || STI.getPTXVersion() < 78) {
2549 if (STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70) {
2576 EVT WideVT =
Op.getValueType();
2579 (STI.getSmVersion() < 80 || STI.getPTXVersion() < 71)) {
2581 return DAG.
getNode(ISD::BF16_TO_FP, Loc, WideVT, Narrow);
2584 (STI.getSmVersion() < 90 || STI.getPTXVersion() < 78)) {
2588 if (STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 71) {
2593 return DAG.
getNode(ISD::FP_EXTEND, Loc, WideVT,
Op);
2603 if (
Op.getValueType() != MVT::v2i16)
2605 EVT EltVT =
Op.getValueType().getVectorElementType();
2607 for (
int I = 0,
E =
Op.getValueType().getVectorNumElements();
I <
E;
I++) {
2610 [&](
const SDUse &O) {
2611 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT,
2612 O.get(), DAG.getIntPtrConstant(I, DL));
2627 for (
size_t I = 0;
I <
N->getNumOperands();
I++) {
2644 return Tcgen05StNode;
2649 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg1:
2651 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg2:
2653 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg1:
2655 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg2:
2657 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1:
2659 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2:
2661 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1:
2663 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2:
2665 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1_ashift:
2667 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2_ashift:
2670 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1_ashift:
2673 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2_ashift:
2675 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg1:
2677 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg2:
2679 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg1:
2681 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg2:
2683 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1:
2685 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2:
2687 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1_ashift:
2689 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2_ashift:
2691 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1:
2693 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2:
2696 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1_ashift:
2698 TCGEN05_MMA_SP_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG1_ASHIFT;
2700 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2_ashift:
2702 TCGEN05_MMA_SP_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG2_ASHIFT;
2714 for (
size_t I = 0;
I <
N->getNumOperands();
I++) {
2733 return Tcgen05MMANode;
2737static std::optional<std::pair<SDValue, SDValue>>
2740 EVT ResVT =
N->getValueType(0);
2748 for (
unsigned i = 0; i < NumElts; ++i)
2759 Ops.push_back(
N->getOperand(3));
2760 Ops.push_back(
N->getOperand(4));
2762 Ops.push_back(
N->getOperand(3));
2771 for (
unsigned i = 0; i < NumElts; ++i) {
2778 return {{BuildVector, Chain}};
2790 case Intrinsic::nvvm_tcgen05_st_16x64b_x1:
2791 case Intrinsic::nvvm_tcgen05_st_16x64b_x2:
2792 case Intrinsic::nvvm_tcgen05_st_16x64b_x4:
2793 case Intrinsic::nvvm_tcgen05_st_16x64b_x8:
2794 case Intrinsic::nvvm_tcgen05_st_16x64b_x16:
2795 case Intrinsic::nvvm_tcgen05_st_16x64b_x32:
2796 case Intrinsic::nvvm_tcgen05_st_16x64b_x128:
2797 case Intrinsic::nvvm_tcgen05_st_16x128b_x1:
2798 case Intrinsic::nvvm_tcgen05_st_16x128b_x2:
2799 case Intrinsic::nvvm_tcgen05_st_16x128b_x4:
2800 case Intrinsic::nvvm_tcgen05_st_16x128b_x8:
2801 case Intrinsic::nvvm_tcgen05_st_16x128b_x16:
2802 case Intrinsic::nvvm_tcgen05_st_16x128b_x32:
2803 case Intrinsic::nvvm_tcgen05_st_16x128b_x64:
2804 case Intrinsic::nvvm_tcgen05_st_16x256b_x1:
2805 case Intrinsic::nvvm_tcgen05_st_16x256b_x2:
2806 case Intrinsic::nvvm_tcgen05_st_16x256b_x4:
2807 case Intrinsic::nvvm_tcgen05_st_16x256b_x8:
2808 case Intrinsic::nvvm_tcgen05_st_16x256b_x16:
2809 case Intrinsic::nvvm_tcgen05_st_16x256b_x32:
2810 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x1:
2811 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x2:
2812 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x4:
2813 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x8:
2814 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x16:
2815 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x32:
2816 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x64:
2817 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x128:
2818 case Intrinsic::nvvm_tcgen05_st_32x32b_x1:
2819 case Intrinsic::nvvm_tcgen05_st_32x32b_x2:
2820 case Intrinsic::nvvm_tcgen05_st_32x32b_x4:
2821 case Intrinsic::nvvm_tcgen05_st_32x32b_x8:
2822 case Intrinsic::nvvm_tcgen05_st_32x32b_x16:
2823 case Intrinsic::nvvm_tcgen05_st_32x32b_x32:
2824 case Intrinsic::nvvm_tcgen05_st_16x64b_x64:
2825 case Intrinsic::nvvm_tcgen05_st_32x32b_x64:
2826 case Intrinsic::nvvm_tcgen05_st_32x32b_x128:
2828 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg1:
2829 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg2:
2830 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg1:
2831 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg2:
2832 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg1:
2833 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg2:
2834 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg1:
2835 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg2:
2836 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1:
2837 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2:
2838 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1:
2839 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2:
2840 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1:
2841 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2:
2842 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1:
2843 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2:
2844 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1_ashift:
2845 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2_ashift:
2847 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1_ashift:
2849 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2_ashift:
2850 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1_ashift:
2851 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2_ashift:
2853 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1_ashift:
2855 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2_ashift:
2865 if (
N->getOperand(1).getValueType() != MVT::i128) {
2872 auto Opcode = [&]() {
2874 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_is_canceled:
2876 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_x:
2878 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_y:
2880 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_z:
2888 SDValue TryCancelResponse =
N->getOperand(1);
2889 SDValue Cast = DAG.
getNode(ISD::BITCAST,
DL, MVT::v2i64, TryCancelResponse);
2897 return DAG.
getNode(Opcode,
DL,
N->getVTList(),
2898 {TryCancelResponse0, TryCancelResponse1});
2902 const unsigned Mode = [&]() {
2903 switch (
Op->getConstantOperandVal(0)) {
2904 case Intrinsic::nvvm_prmt:
2906 case Intrinsic::nvvm_prmt_b4e:
2908 case Intrinsic::nvvm_prmt_ecl:
2910 case Intrinsic::nvvm_prmt_ecr:
2912 case Intrinsic::nvvm_prmt_f4e:
2914 case Intrinsic::nvvm_prmt_rc16:
2916 case Intrinsic::nvvm_prmt_rc8:
2924 SDValue B =
Op.getNumOperands() == 4 ?
Op.getOperand(2)
2926 SDValue Selector = (
Op->op_end() - 1)->get();
2931 switch (
Op->getConstantOperandVal(1)) {
2937 case Intrinsic::nvvm_tcgen05_ld_16x64b_x2:
2938 case Intrinsic::nvvm_tcgen05_ld_16x128b_x1:
2939 case Intrinsic::nvvm_tcgen05_ld_32x32b_x2:
2944 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x2:
2952 switch (
Op->getConstantOperandVal(0)) {
2955 case Intrinsic::nvvm_prmt:
2956 case Intrinsic::nvvm_prmt_b4e:
2957 case Intrinsic::nvvm_prmt_ecl:
2958 case Intrinsic::nvvm_prmt_ecr:
2959 case Intrinsic::nvvm_prmt_f4e:
2960 case Intrinsic::nvvm_prmt_rc16:
2961 case Intrinsic::nvvm_prmt_rc8:
2963 case Intrinsic::nvvm_internal_addrspace_wrap:
2964 return Op.getOperand(1);
2965 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_is_canceled:
2966 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_x:
2967 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_y:
2968 case Intrinsic::nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_z:
2978 assert(V.getValueType() == MVT::i64 &&
2979 "Unexpected CTLZ/CTPOP type to legalize");
2988 assert(
A.getValueType() == MVT::i64 &&
B.getValueType() == MVT::i64);
2993 const auto Amt = AmtConst->getZExtValue() & 63;
3020 ? std::make_tuple(AHi, ALo, BHi)
3021 : std::make_tuple(ALo, BHi, BLo);
3048 EVT Ty =
Op.getValueType();
3058 if (Flags.hasNoInfs())
3070 assert(
Op.getValueType() == MVT::i1 &&
"Custom lowering enabled only for i1");
3080 TrueVal = TrueVal.getOperand(0);
3081 FalseVal = FalseVal.getOperand(0);
3083 EVT VT = TrueVal.getSimpleValueType().bitsLE(FalseVal.getSimpleValueType())
3084 ? TrueVal.getValueType()
3085 : FalseVal.getValueType();
3105 switch (
Op.getOpcode()) {
3110 case ISD::ADDRSPACECAST:
3111 return LowerADDRSPACECAST(
Op, DAG);
3119 return LowerBUILD_VECTOR(
Op, DAG);
3121 return LowerBITCAST(
Op, DAG);
3125 return LowerEXTRACT_VECTOR_ELT(
Op, DAG);
3127 return LowerINSERT_VECTOR_ELT(
Op, DAG);
3129 return LowerVECTOR_SHUFFLE(
Op, DAG);
3131 return LowerCONCAT_VECTORS(
Op, DAG);
3132 case ISD::VECREDUCE_FMAX:
3133 case ISD::VECREDUCE_FMIN:
3134 case ISD::VECREDUCE_FMAXIMUM:
3135 case ISD::VECREDUCE_FMINIMUM:
3136 return LowerVECREDUCE(
Op, DAG);
3138 return LowerSTORE(
Op, DAG);
3140 return LowerLOAD(
Op, DAG);
3142 return LowerShiftLeftParts(
Op, DAG);
3145 return LowerShiftRightParts(
Op, DAG);
3149 return LowerFROUND(
Op, DAG);
3151 return LowerFCOPYSIGN(
Op, DAG);
3154 return LowerINT_TO_FP(
Op, DAG);
3157 return LowerFP_TO_INT(
Op, DAG);
3159 return LowerFP_ROUND(
Op, DAG);
3160 case ISD::FP_EXTEND:
3161 return LowerFP_EXTEND(
Op, DAG);
3163 return LowerBR_JT(
Op, DAG);
3165 return LowerVAARG(
Op, DAG);
3167 return LowerVASTART(
Op, DAG);
3186 case ISD::DYNAMIC_STACKALLOC:
3188 case ISD::STACKRESTORE:
3190 case ISD::STACKSAVE:
3193 return LowerCopyToReg_128(
Op, DAG);
3198 return PromoteBinOpIfF32FTZ(
Op, DAG);
3216 unsigned JId = JT->getIndex();
3248 unsigned SrcAS =
N->getSrcAddressSpace();
3249 unsigned DestAS =
N->getDestAddressSpace();
3259 const MVT GenerictVT =
3263 SDValue SharedClusterConversion =
3266 return SharedClusterConversion;
3281 SDNode *
Node =
Op.getNode();
3283 EVT VT =
Node->getValueType(0);
3287 const MaybeAlign MA(
Node->getConstantOperandVal(3));
3290 Tmp1, Tmp2, MachinePointerInfo(V));
3310 MachinePointerInfo(V));
3316 return DAG.
getLoad(VT,
DL, Tmp1, VAList, MachinePointerInfo(SrcV));
3325 SDValue VAReg = getParamSymbol(DAG, -1, PtrVT);
3328 return DAG.
getStore(
Op.getOperand(0),
DL, VAReg,
Op.getOperand(1),
3329 MachinePointerInfo(SV));
3333static std::optional<std::pair<SDValue, SDValue>>
3336 const EVT ResVT = LD->getValueType(0);
3337 const EVT MemVT = LD->getMemoryVT();
3342 return std::nullopt;
3344 const auto NumEltsAndEltVT =
3346 if (!NumEltsAndEltVT)
3347 return std::nullopt;
3348 const auto [NumElts, EltVT] = NumEltsAndEltVT.value();
3350 Align Alignment = LD->getAlign();
3353 if (Alignment < PrefAlign) {
3359 return std::nullopt;
3370 return std::nullopt;
3382 ListVTs.push_back(MVT::Other);
3395 LD->getMemOperand());
3404 for (
const unsigned I :
llvm::seq(NumElts)) {
3409 for (
const unsigned I :
llvm::seq(NumElts)) {
3411 if (LoadEltVT != EltVT)
3419 const MVT BuildVecVT =
3431 Results.append({Res->first, Res->second});
3448 assert(LD->getValueType(0) == MVT::i1 &&
"Custom lowering for i1 load only");
3450 LD->getBasePtr(), LD->getPointerInfo(),
3451 MVT::i8, LD->getAlign(),
3452 LD->getMemOperand()->getFlags());
3463 if (
Op.getValueType() == MVT::i1)
3470 assert(
LD->getValueType(0).isInteger() &&
LD->getMemoryVT().isInteger() &&
3471 "Unexpected fpext-load");
3473 LD->getChain(),
LD->getBasePtr(),
LD->getMemoryVT(),
3474 LD->getMemOperand());
3486 const EVT MemVT =
N->getMemoryVT();
3493 const auto NumEltsAndEltVT =
3495 if (!NumEltsAndEltVT)
3497 const auto [NumElts, EltVT] = NumEltsAndEltVT.value();
3501 Align Alignment =
N->getAlign();
3503 if (Alignment < PrefAlign) {
3530 Ops.push_back(
N->getOperand(0));
3540 for (
const unsigned I :
llvm::seq(NumElts)) {
3543 NumEltsPerSubVector);
3548 for (
const unsigned I :
llvm::seq(NumElts)) {
3558 Ops.push_back(ExtVal);
3563 Ops.append(
N->op_begin() + 2,
N->op_end());
3567 N->getMemoryVT(),
N->getMemOperand());
3575 EVT VT =
Store->getMemoryVT();
3578 return LowerSTOREi1(
Op, DAG);
3590 SDNode *
Node =
Op.getNode();
3599 DAG.
getTruncStore(Tmp1, dl, Tmp3, Tmp2,
ST->getPointerInfo(), MVT::i8,
3600 ST->getAlign(),
ST->getMemOperand()->getFlags());
3609 assert(
Op.getOperand(1).getValueType() == MVT::i128 &&
3610 "Custom lowering for 128-bit CopyToReg only");
3612 SDNode *
Node =
Op.getNode();
3624 NewOps[0] =
Op->getOperand(0);
3625 NewOps[1] =
Op->getOperand(1);
3629 NewOps[4] =
Op->getOperand(3);
3634unsigned NVPTXTargetLowering::getNumRegisters(
3636 std::optional<MVT> RegisterVT = std::nullopt)
const {
3637 if (VT == MVT::i128 && RegisterVT == MVT::i128)
3642bool NVPTXTargetLowering::splitValueIntoRegisterParts(
3644 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID> CC)
const {
3645 if (Val.
getValueType() == MVT::i128 && NumParts == 1) {
3658 StringRef SavedStr =
nvTM->getStrPool().save(
3665 const StringRef SavedStr =
nvTM->getStrPool().save(
"param" + Twine(
I));
3693 for (
const auto &Arg :
F.args()) {
3694 const auto ArgIns = AllIns.take_while(
3695 [&](
auto I) {
return I.OrigArgIndex == Arg.getArgNo(); });
3696 AllIns = AllIns.drop_front(ArgIns.size());
3698 Type *Ty = Arg.getType();
3703 if (Arg.use_empty()) {
3705 for (
const auto &In : ArgIns) {
3706 assert(!In.Used &&
"Arg.use_empty() is true but Arg is used?");
3712 SDValue ArgSymbol = getParamSymbol(DAG, Arg.getArgNo(), PtrVT);
3718 if (Arg.hasByValAttr()) {
3726 assert(ArgIns.size() == 1 &&
"ByVal argument must be a pointer");
3727 const auto &ByvalIn = ArgIns[0];
3729 "Ins type did not match function type");
3730 assert(ByvalIn.VT == PtrVT &&
"ByVal argument must be a pointer");
3735 P.getNode()->setIROrder(Arg.getArgNo() + 1);
3738 P.getNode()->setIROrder(Arg.getArgNo() + 1);
3747 assert(VTs.
size() == ArgIns.size() &&
"Size mismatch");
3748 assert(VTs.
size() == Offsets.size() &&
"Size mismatch");
3751 &
F, Ty, Arg.getArgNo() + AttributeList::FirstArgIndex,
DL);
3755 for (
const unsigned NumElts : VI) {
3757 const EVT LoadVT = VTs[
I] == MVT::i1 ? MVT::i8 : VTs[
I];
3765 DAG.
getLoad(VecVT, dl, Root, VecAddr,
3769 P.getNode()->setIROrder(Arg.getArgNo() + 1);
3770 for (
const unsigned J :
llvm::seq(NumElts)) {
3782 if (!OutChains.
empty())
3795 Type *RetTy =
F.getReturnType();
3798 assert(OutVals.
empty() && Outs.
empty() &&
"Return value expected for void");
3811 const bool ExtendIntegerRetVal =
3812 RetTy->
isIntegerTy() &&
DL.getTypeAllocSizeInBits(RetTy) < 32;
3817 assert(VTs.
size() == OutVals.
size() &&
"Bad return value decomposition");
3819 const auto GetRetVal = [&](
unsigned I) ->
SDValue {
3823 "OutVal type should always be legal");
3827 ExtendIntegerRetVal ? MVT::i32 : (VTI == MVT::i1 ? MVT::i8 : VTI);
3833 for (
const unsigned NumElts : VI) {
3834 const MaybeAlign CurrentAlign = ExtendIntegerRetVal
3839 NumElts, dl, DAG, [&](
unsigned K) {
return GetRetVal(
I + K); });
3856 if (Constraint.
size() > 1)
3872 case Intrinsic::nvvm_match_all_sync_i32p:
3873 case Intrinsic::nvvm_match_all_sync_i64p:
3878 Info.memVT = MVT::i1;
3883 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_col:
3884 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_row:
3885 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_col_stride:
3886 case Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_row_stride:
3887 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_col:
3888 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_row:
3889 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_col_stride:
3890 case Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_row_stride:
3891 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_col:
3892 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_row:
3893 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_col_stride:
3894 case Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_row_stride:
3895 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_col:
3896 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_row:
3897 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_col_stride:
3898 case Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_row_stride:
3899 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_col:
3900 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_row:
3901 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_col_stride:
3902 case Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_row_stride:
3903 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_col:
3904 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_row:
3905 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_col_stride:
3906 case Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_row_stride: {
3908 Info.memVT = MVT::v8f16;
3909 Info.ptrVal =
I.getArgOperand(0);
3912 Info.align =
Align(16);
3915 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_col:
3916 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_col_stride:
3917 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_col_stride:
3918 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_col:
3919 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_row:
3920 case Intrinsic::nvvm_wmma_m16n16k16_load_a_s8_row_stride:
3921 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_row_stride:
3922 case Intrinsic::nvvm_wmma_m16n16k16_load_a_u8_row:
3923 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_col:
3924 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_col_stride:
3925 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_row:
3926 case Intrinsic::nvvm_wmma_m8n32k16_load_a_bf16_row_stride:
3927 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_col:
3928 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_col_stride:
3929 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_col_stride:
3930 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_col:
3931 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_row:
3932 case Intrinsic::nvvm_wmma_m16n16k16_load_b_s8_row_stride:
3933 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_row_stride:
3934 case Intrinsic::nvvm_wmma_m16n16k16_load_b_u8_row:
3935 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_col:
3936 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_col_stride:
3937 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_row:
3938 case Intrinsic::nvvm_wmma_m32n8k16_load_b_bf16_row_stride: {
3940 Info.memVT = MVT::v2i32;
3941 Info.ptrVal =
I.getArgOperand(0);
3944 Info.align =
Align(8);
3948 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_col:
3949 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_col_stride:
3950 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_col_stride:
3951 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_col:
3952 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_row:
3953 case Intrinsic::nvvm_wmma_m32n8k16_load_a_s8_row_stride:
3954 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_row_stride:
3955 case Intrinsic::nvvm_wmma_m32n8k16_load_a_u8_row:
3956 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_col:
3957 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_col_stride:
3958 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_row:
3959 case Intrinsic::nvvm_wmma_m16n16k16_load_a_bf16_row_stride:
3960 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_col:
3961 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_col_stride:
3962 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_row:
3963 case Intrinsic::nvvm_wmma_m16n16k8_load_a_tf32_row_stride:
3965 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_col:
3966 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_col_stride:
3967 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_col_stride:
3968 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_col:
3969 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_row:
3970 case Intrinsic::nvvm_wmma_m8n32k16_load_b_s8_row_stride:
3971 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_row_stride:
3972 case Intrinsic::nvvm_wmma_m8n32k16_load_b_u8_row:
3973 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_col:
3974 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_col_stride:
3975 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_row:
3976 case Intrinsic::nvvm_wmma_m16n16k16_load_b_bf16_row_stride:
3977 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_col:
3978 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_col_stride:
3979 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_row:
3980 case Intrinsic::nvvm_wmma_m16n16k8_load_b_tf32_row_stride:
3981 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x4_b16:
3982 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x4_trans_b16:
3983 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8:
3984 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8x16_b4x16_p64:
3985 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8x16_b6x16_p32:
3986 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x4_b8x16_b4x16_p64:
3987 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x4_b8x16_b6x16_p32: {
3989 Info.memVT = MVT::v4i32;
3990 Info.ptrVal =
I.getArgOperand(0);
3993 Info.align =
Align(16);
3997 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_col:
3998 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_col_stride:
3999 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_col_stride:
4000 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_col:
4001 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_row:
4002 case Intrinsic::nvvm_wmma_m32n8k16_load_b_s8_row_stride:
4003 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_row_stride:
4004 case Intrinsic::nvvm_wmma_m32n8k16_load_b_u8_row:
4006 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_col:
4007 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_col_stride:
4008 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_col_stride:
4009 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_col:
4010 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_row:
4011 case Intrinsic::nvvm_wmma_m8n32k16_load_a_s8_row_stride:
4012 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_row_stride:
4013 case Intrinsic::nvvm_wmma_m8n32k16_load_a_u8_row:
4014 case Intrinsic::nvvm_wmma_m8n8k128_load_a_b1_row:
4015 case Intrinsic::nvvm_wmma_m8n8k128_load_a_b1_row_stride:
4016 case Intrinsic::nvvm_wmma_m8n8k128_load_b_b1_col:
4017 case Intrinsic::nvvm_wmma_m8n8k128_load_b_b1_col_stride:
4018 case Intrinsic::nvvm_wmma_m8n8k32_load_a_s4_row:
4019 case Intrinsic::nvvm_wmma_m8n8k32_load_a_s4_row_stride:
4020 case Intrinsic::nvvm_wmma_m8n8k32_load_a_u4_row_stride:
4021 case Intrinsic::nvvm_wmma_m8n8k32_load_a_u4_row:
4022 case Intrinsic::nvvm_wmma_m8n8k32_load_b_s4_col:
4023 case Intrinsic::nvvm_wmma_m8n8k32_load_b_s4_col_stride:
4024 case Intrinsic::nvvm_wmma_m8n8k32_load_b_u4_col_stride:
4025 case Intrinsic::nvvm_wmma_m8n8k32_load_b_u4_col:
4026 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x1_b16:
4027 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x1_trans_b16:
4028 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x1_b8x16_b4x16_p64:
4029 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x1_b8x16_b6x16_p32: {
4031 Info.memVT = MVT::i32;
4032 Info.ptrVal =
I.getArgOperand(0);
4035 Info.align =
Align(4);
4039 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_col:
4040 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_row:
4041 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_col_stride:
4042 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_row_stride:
4043 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_col:
4044 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_row:
4045 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_col_stride:
4046 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_row_stride:
4047 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_col:
4048 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_row:
4049 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_col_stride:
4050 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_row_stride: {
4052 Info.memVT = MVT::v4f16;
4053 Info.ptrVal =
I.getArgOperand(0);
4056 Info.align =
Align(16);
4060 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_col:
4061 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_row:
4062 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_col_stride:
4063 case Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_row_stride:
4064 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_col:
4065 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_row:
4066 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_col_stride:
4067 case Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_row_stride:
4068 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_col:
4069 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_row:
4070 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_col_stride:
4071 case Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_row_stride:
4072 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_col:
4073 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_row:
4074 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_col_stride:
4075 case Intrinsic::nvvm_wmma_m16n16k8_load_c_f32_row_stride: {
4077 Info.memVT = MVT::v8f32;
4078 Info.ptrVal =
I.getArgOperand(0);
4081 Info.align =
Align(16);
4085 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_col:
4086 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_col_stride:
4087 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_row:
4088 case Intrinsic::nvvm_wmma_m32n8k16_load_a_bf16_row_stride:
4090 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_col:
4091 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_col_stride:
4092 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_row:
4093 case Intrinsic::nvvm_wmma_m8n32k16_load_b_bf16_row_stride:
4095 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_col:
4096 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_col_stride:
4097 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_row:
4098 case Intrinsic::nvvm_wmma_m16n16k16_load_c_s32_row_stride:
4099 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_col:
4100 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_col_stride:
4101 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_row:
4102 case Intrinsic::nvvm_wmma_m32n8k16_load_c_s32_row_stride:
4103 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_col:
4104 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_col_stride:
4105 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_row:
4106 case Intrinsic::nvvm_wmma_m8n32k16_load_c_s32_row_stride: {
4108 Info.memVT = MVT::v8i32;
4109 Info.ptrVal =
I.getArgOperand(0);
4112 Info.align =
Align(16);
4116 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_col:
4117 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_col_stride:
4118 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_row:
4119 case Intrinsic::nvvm_wmma_m8n8k128_load_c_s32_row_stride:
4120 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_col:
4121 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_col_stride:
4122 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_row:
4123 case Intrinsic::nvvm_wmma_m8n8k32_load_c_s32_row_stride:
4124 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x2_b16:
4125 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x2_trans_b16:
4126 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8:
4127 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8x16_b4x16_p64:
4128 case Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8x16_b6x16_p32:
4129 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x2_b8x16_b4x16_p64:
4130 case Intrinsic::nvvm_ldmatrix_sync_aligned_m8n16_x2_b8x16_b6x16_p32: {
4132 Info.memVT = MVT::v2i32;
4133 Info.ptrVal =
I.getArgOperand(0);
4136 Info.align =
Align(8);
4140 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_col:
4141 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_col_stride:
4142 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_row:
4143 case Intrinsic::nvvm_wmma_m8n8k4_load_a_f64_row_stride:
4145 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_col:
4146 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_col_stride:
4147 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_row:
4148 case Intrinsic::nvvm_wmma_m8n8k4_load_b_f64_row_stride: {
4150 Info.memVT = MVT::f64;
4151 Info.ptrVal =
I.getArgOperand(0);
4154 Info.align =
Align(8);
4158 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_col:
4159 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_col_stride:
4160 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_row:
4161 case Intrinsic::nvvm_wmma_m8n8k4_load_c_f64_row_stride: {
4163 Info.memVT = MVT::v2f64;
4164 Info.ptrVal =
I.getArgOperand(0);
4167 Info.align =
Align(16);
4171 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_col:
4172 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_row:
4173 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_col_stride:
4174 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_row_stride:
4175 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_col:
4176 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_row:
4177 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_col_stride:
4178 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_row_stride:
4179 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_col:
4180 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_row:
4181 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_col_stride:
4182 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_row_stride: {
4184 Info.memVT = MVT::v4f16;
4185 Info.ptrVal =
I.getArgOperand(0);
4188 Info.align =
Align(16);
4192 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_col:
4193 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_row:
4194 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_col_stride:
4195 case Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_row_stride:
4196 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_col:
4197 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_row:
4198 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_col_stride:
4199 case Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_row_stride:
4200 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_col:
4201 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_row:
4202 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_col_stride:
4203 case Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_row_stride:
4204 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_col:
4205 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_row:
4206 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_col_stride:
4207 case Intrinsic::nvvm_wmma_m16n16k8_store_d_f32_row_stride: {
4209 Info.memVT = MVT::v8f32;
4210 Info.ptrVal =
I.getArgOperand(0);
4213 Info.align =
Align(16);
4217 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_col:
4218 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_col_stride:
4219 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_row:
4220 case Intrinsic::nvvm_wmma_m16n16k16_store_d_s32_row_stride:
4221 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_col:
4222 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_col_stride:
4223 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_row:
4224 case Intrinsic::nvvm_wmma_m32n8k16_store_d_s32_row_stride:
4225 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_col:
4226 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_col_stride:
4227 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_row:
4228 case Intrinsic::nvvm_wmma_m8n32k16_store_d_s32_row_stride: {
4230 Info.memVT = MVT::v8i32;
4231 Info.ptrVal =
I.getArgOperand(0);
4234 Info.align =
Align(16);
4238 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_col:
4239 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_col_stride:
4240 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_row:
4241 case Intrinsic::nvvm_wmma_m8n8k128_store_d_s32_row_stride:
4242 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_col:
4243 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_col_stride:
4244 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_row:
4245 case Intrinsic::nvvm_wmma_m8n8k32_store_d_s32_row_stride:
4246 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x2_b16:
4247 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x2_trans_b16:
4248 case Intrinsic::nvvm_stmatrix_sync_aligned_m16n8_x2_trans_b8: {
4250 Info.memVT = MVT::v2i32;
4251 Info.ptrVal =
I.getArgOperand(0);
4254 Info.align =
Align(8);
4258 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_col:
4259 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_col_stride:
4260 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_row:
4261 case Intrinsic::nvvm_wmma_m8n8k4_store_d_f64_row_stride: {
4263 Info.memVT = MVT::v2f64;
4264 Info.ptrVal =
I.getArgOperand(0);
4267 Info.align =
Align(16);
4271 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x1_b16:
4272 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x1_trans_b16:
4273 case Intrinsic::nvvm_stmatrix_sync_aligned_m16n8_x1_trans_b8: {
4275 Info.memVT = MVT::i32;
4276 Info.ptrVal =
I.getArgOperand(0);
4279 Info.align =
Align(4);
4283 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x4_b16:
4284 case Intrinsic::nvvm_stmatrix_sync_aligned_m8n8_x4_trans_b16:
4285 case Intrinsic::nvvm_stmatrix_sync_aligned_m16n8_x4_trans_b8: {
4287 Info.memVT = MVT::v4i32;
4288 Info.ptrVal =
I.getArgOperand(0);
4291 Info.align =
Align(16);
4295 case Intrinsic::nvvm_atomic_add_gen_f_cta:
4296 case Intrinsic::nvvm_atomic_add_gen_f_sys:
4297 case Intrinsic::nvvm_atomic_add_gen_i_cta:
4298 case Intrinsic::nvvm_atomic_add_gen_i_sys:
4299 case Intrinsic::nvvm_atomic_and_gen_i_cta:
4300 case Intrinsic::nvvm_atomic_and_gen_i_sys:
4301 case Intrinsic::nvvm_atomic_cas_gen_i_cta:
4302 case Intrinsic::nvvm_atomic_cas_gen_i_sys:
4303 case Intrinsic::nvvm_atomic_dec_gen_i_cta:
4304 case Intrinsic::nvvm_atomic_dec_gen_i_sys:
4305 case Intrinsic::nvvm_atomic_inc_gen_i_cta:
4306 case Intrinsic::nvvm_atomic_inc_gen_i_sys:
4307 case Intrinsic::nvvm_atomic_max_gen_i_cta:
4308 case Intrinsic::nvvm_atomic_max_gen_i_sys:
4309 case Intrinsic::nvvm_atomic_min_gen_i_cta:
4310 case Intrinsic::nvvm_atomic_min_gen_i_sys:
4311 case Intrinsic::nvvm_atomic_or_gen_i_cta:
4312 case Intrinsic::nvvm_atomic_or_gen_i_sys:
4313 case Intrinsic::nvvm_atomic_exch_gen_i_cta:
4314 case Intrinsic::nvvm_atomic_exch_gen_i_sys:
4315 case Intrinsic::nvvm_atomic_xor_gen_i_cta:
4316 case Intrinsic::nvvm_atomic_xor_gen_i_sys: {
4317 auto &
DL =
I.getDataLayout();
4320 Info.ptrVal =
I.getArgOperand(0);
4327 case Intrinsic::nvvm_prefetch_tensormap: {
4328 auto &
DL =
I.getDataLayout();
4331 Info.ptrVal =
I.getArgOperand(0);
4339 case Intrinsic::nvvm_ldu_global_i:
4340 case Intrinsic::nvvm_ldu_global_f:
4341 case Intrinsic::nvvm_ldu_global_p: {
4344 Info.ptrVal =
I.getArgOperand(0);
4351 case Intrinsic::nvvm_tex_1d_v4f32_s32:
4352 case Intrinsic::nvvm_tex_1d_v4f32_f32:
4353 case Intrinsic::nvvm_tex_1d_level_v4f32_f32:
4354 case Intrinsic::nvvm_tex_1d_grad_v4f32_f32:
4355 case Intrinsic::nvvm_tex_1d_array_v4f32_s32:
4356 case Intrinsic::nvvm_tex_1d_array_v4f32_f32:
4357 case Intrinsic::nvvm_tex_1d_array_level_v4f32_f32:
4358 case Intrinsic::nvvm_tex_1d_array_grad_v4f32_f32:
4359 case Intrinsic::nvvm_tex_2d_v4f32_s32:
4360 case Intrinsic::nvvm_tex_2d_v4f32_f32:
4361 case Intrinsic::nvvm_tex_2d_level_v4f32_f32:
4362 case Intrinsic::nvvm_tex_2d_grad_v4f32_f32:
4363 case Intrinsic::nvvm_tex_2d_array_v4f32_s32:
4364 case Intrinsic::nvvm_tex_2d_array_v4f32_f32:
4365 case Intrinsic::nvvm_tex_2d_array_level_v4f32_f32:
4366 case Intrinsic::nvvm_tex_2d_array_grad_v4f32_f32:
4367 case Intrinsic::nvvm_tex_3d_v4f32_s32:
4368 case Intrinsic::nvvm_tex_3d_v4f32_f32:
4369 case Intrinsic::nvvm_tex_3d_level_v4f32_f32:
4370 case Intrinsic::nvvm_tex_3d_grad_v4f32_f32:
4371 case Intrinsic::nvvm_tex_cube_v4f32_f32:
4372 case Intrinsic::nvvm_tex_cube_level_v4f32_f32:
4373 case Intrinsic::nvvm_tex_cube_array_v4f32_f32:
4374 case Intrinsic::nvvm_tex_cube_array_level_v4f32_f32:
4375 case Intrinsic::nvvm_tld4_r_2d_v4f32_f32:
4376 case Intrinsic::nvvm_tld4_g_2d_v4f32_f32:
4377 case Intrinsic::nvvm_tld4_b_2d_v4f32_f32:
4378 case Intrinsic::nvvm_tld4_a_2d_v4f32_f32:
4379 case Intrinsic::nvvm_tex_unified_1d_v4f32_s32:
4380 case Intrinsic::nvvm_tex_unified_1d_v4f32_f32:
4381 case Intrinsic::nvvm_tex_unified_1d_level_v4f32_f32:
4382 case Intrinsic::nvvm_tex_unified_1d_grad_v4f32_f32:
4383 case Intrinsic::nvvm_tex_unified_1d_array_v4f32_s32:
4384 case Intrinsic::nvvm_tex_unified_1d_array_v4f32_f32:
4385 case Intrinsic::nvvm_tex_unified_1d_array_level_v4f32_f32:
4386 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4f32_f32:
4387 case Intrinsic::nvvm_tex_unified_2d_v4f32_s32:
4388 case Intrinsic::nvvm_tex_unified_2d_v4f32_f32:
4389 case Intrinsic::nvvm_tex_unified_2d_level_v4f32_f32:
4390 case Intrinsic::nvvm_tex_unified_2d_grad_v4f32_f32:
4391 case Intrinsic::nvvm_tex_unified_2d_array_v4f32_s32:
4392 case Intrinsic::nvvm_tex_unified_2d_array_v4f32_f32:
4393 case Intrinsic::nvvm_tex_unified_2d_array_level_v4f32_f32:
4394 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4f32_f32:
4395 case Intrinsic::nvvm_tex_unified_3d_v4f32_s32:
4396 case Intrinsic::nvvm_tex_unified_3d_v4f32_f32:
4397 case Intrinsic::nvvm_tex_unified_3d_level_v4f32_f32:
4398 case Intrinsic::nvvm_tex_unified_3d_grad_v4f32_f32:
4399 case Intrinsic::nvvm_tex_unified_cube_v4f32_f32:
4400 case Intrinsic::nvvm_tex_unified_cube_level_v4f32_f32:
4401 case Intrinsic::nvvm_tex_unified_cube_array_v4f32_f32:
4402 case Intrinsic::nvvm_tex_unified_cube_array_level_v4f32_f32:
4403 case Intrinsic::nvvm_tex_unified_cube_grad_v4f32_f32:
4404 case Intrinsic::nvvm_tex_unified_cube_array_grad_v4f32_f32:
4405 case Intrinsic::nvvm_tld4_unified_r_2d_v4f32_f32:
4406 case Intrinsic::nvvm_tld4_unified_g_2d_v4f32_f32:
4407 case Intrinsic::nvvm_tld4_unified_b_2d_v4f32_f32:
4408 case Intrinsic::nvvm_tld4_unified_a_2d_v4f32_f32:
4410 Info.memVT = MVT::v4f32;
4411 Info.ptrVal =
nullptr;
4414 Info.align =
Align(16);
4417 case Intrinsic::nvvm_tex_1d_v4s32_s32:
4418 case Intrinsic::nvvm_tex_1d_v4s32_f32:
4419 case Intrinsic::nvvm_tex_1d_level_v4s32_f32:
4420 case Intrinsic::nvvm_tex_1d_grad_v4s32_f32:
4421 case Intrinsic::nvvm_tex_1d_array_v4s32_s32:
4422 case Intrinsic::nvvm_tex_1d_array_v4s32_f32:
4423 case Intrinsic::nvvm_tex_1d_array_level_v4s32_f32:
4424 case Intrinsic::nvvm_tex_1d_array_grad_v4s32_f32:
4425 case Intrinsic::nvvm_tex_2d_v4s32_s32:
4426 case Intrinsic::nvvm_tex_2d_v4s32_f32:
4427 case Intrinsic::nvvm_tex_2d_level_v4s32_f32:
4428 case Intrinsic::nvvm_tex_2d_grad_v4s32_f32:
4429 case Intrinsic::nvvm_tex_2d_array_v4s32_s32:
4430 case Intrinsic::nvvm_tex_2d_array_v4s32_f32:
4431 case Intrinsic::nvvm_tex_2d_array_level_v4s32_f32:
4432 case Intrinsic::nvvm_tex_2d_array_grad_v4s32_f32:
4433 case Intrinsic::nvvm_tex_3d_v4s32_s32:
4434 case Intrinsic::nvvm_tex_3d_v4s32_f32:
4435 case Intrinsic::nvvm_tex_3d_level_v4s32_f32:
4436 case Intrinsic::nvvm_tex_3d_grad_v4s32_f32:
4437 case Intrinsic::nvvm_tex_cube_v4s32_f32:
4438 case Intrinsic::nvvm_tex_cube_level_v4s32_f32:
4439 case Intrinsic::nvvm_tex_cube_array_v4s32_f32:
4440 case Intrinsic::nvvm_tex_cube_array_level_v4s32_f32:
4441 case Intrinsic::nvvm_tex_cube_v4u32_f32:
4442 case Intrinsic::nvvm_tex_cube_level_v4u32_f32:
4443 case Intrinsic::nvvm_tex_cube_array_v4u32_f32:
4444 case Intrinsic::nvvm_tex_cube_array_level_v4u32_f32:
4445 case Intrinsic::nvvm_tex_1d_v4u32_s32:
4446 case Intrinsic::nvvm_tex_1d_v4u32_f32:
4447 case Intrinsic::nvvm_tex_1d_level_v4u32_f32:
4448 case Intrinsic::nvvm_tex_1d_grad_v4u32_f32:
4449 case Intrinsic::nvvm_tex_1d_array_v4u32_s32:
4450 case Intrinsic::nvvm_tex_1d_array_v4u32_f32:
4451 case Intrinsic::nvvm_tex_1d_array_level_v4u32_f32:
4452 case Intrinsic::nvvm_tex_1d_array_grad_v4u32_f32:
4453 case Intrinsic::nvvm_tex_2d_v4u32_s32:
4454 case Intrinsic::nvvm_tex_2d_v4u32_f32:
4455 case Intrinsic::nvvm_tex_2d_level_v4u32_f32:
4456 case Intrinsic::nvvm_tex_2d_grad_v4u32_f32:
4457 case Intrinsic::nvvm_tex_2d_array_v4u32_s32:
4458 case Intrinsic::nvvm_tex_2d_array_v4u32_f32:
4459 case Intrinsic::nvvm_tex_2d_array_level_v4u32_f32:
4460 case Intrinsic::nvvm_tex_2d_array_grad_v4u32_f32:
4461 case Intrinsic::nvvm_tex_3d_v4u32_s32:
4462 case Intrinsic::nvvm_tex_3d_v4u32_f32:
4463 case Intrinsic::nvvm_tex_3d_level_v4u32_f32:
4464 case Intrinsic::nvvm_tex_3d_grad_v4u32_f32:
4465 case Intrinsic::nvvm_tld4_r_2d_v4s32_f32:
4466 case Intrinsic::nvvm_tld4_g_2d_v4s32_f32:
4467 case Intrinsic::nvvm_tld4_b_2d_v4s32_f32:
4468 case Intrinsic::nvvm_tld4_a_2d_v4s32_f32:
4469 case Intrinsic::nvvm_tld4_r_2d_v4u32_f32:
4470 case Intrinsic::nvvm_tld4_g_2d_v4u32_f32:
4471 case Intrinsic::nvvm_tld4_b_2d_v4u32_f32:
4472 case Intrinsic::nvvm_tld4_a_2d_v4u32_f32:
4473 case Intrinsic::nvvm_tex_unified_1d_v4s32_s32:
4474 case Intrinsic::nvvm_tex_unified_1d_v4s32_f32:
4475 case Intrinsic::nvvm_tex_unified_1d_level_v4s32_f32:
4476 case Intrinsic::nvvm_tex_unified_1d_grad_v4s32_f32:
4477 case Intrinsic::nvvm_tex_unified_1d_array_v4s32_s32:
4478 case Intrinsic::nvvm_tex_unified_1d_array_v4s32_f32:
4479 case Intrinsic::nvvm_tex_unified_1d_array_level_v4s32_f32:
4480 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4s32_f32:
4481 case Intrinsic::nvvm_tex_unified_2d_v4s32_s32:
4482 case Intrinsic::nvvm_tex_unified_2d_v4s32_f32:
4483 case Intrinsic::nvvm_tex_unified_2d_level_v4s32_f32:
4484 case Intrinsic::nvvm_tex_unified_2d_grad_v4s32_f32:
4485 case Intrinsic::nvvm_tex_unified_2d_array_v4s32_s32:
4486 case Intrinsic::nvvm_tex_unified_2d_array_v4s32_f32:
4487 case Intrinsic::nvvm_tex_unified_2d_array_level_v4s32_f32:
4488 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4s32_f32:
4489 case Intrinsic::nvvm_tex_unified_3d_v4s32_s32:
4490 case Intrinsic::nvvm_tex_unified_3d_v4s32_f32:
4491 case Intrinsic::nvvm_tex_unified_3d_level_v4s32_f32:
4492 case Intrinsic::nvvm_tex_unified_3d_grad_v4s32_f32:
4493 case Intrinsic::nvvm_tex_unified_1d_v4u32_s32:
4494 case Intrinsic::nvvm_tex_unified_1d_v4u32_f32:
4495 case Intrinsic::nvvm_tex_unified_1d_level_v4u32_f32:
4496 case Intrinsic::nvvm_tex_unified_1d_grad_v4u32_f32:
4497 case Intrinsic::nvvm_tex_unified_1d_array_v4u32_s32:
4498 case Intrinsic::nvvm_tex_unified_1d_array_v4u32_f32:
4499 case Intrinsic::nvvm_tex_unified_1d_array_level_v4u32_f32:
4500 case Intrinsic::nvvm_tex_unified_1d_array_grad_v4u32_f32:
4501 case Intrinsic::nvvm_tex_unified_2d_v4u32_s32:
4502 case Intrinsic::nvvm_tex_unified_2d_v4u32_f32:
4503 case Intrinsic::nvvm_tex_unified_2d_level_v4u32_f32:
4504 case Intrinsic::nvvm_tex_unified_2d_grad_v4u32_f32:
4505 case Intrinsic::nvvm_tex_unified_2d_array_v4u32_s32:
4506 case Intrinsic::nvvm_tex_unified_2d_array_v4u32_f32:
4507 case Intrinsic::nvvm_tex_unified_2d_array_level_v4u32_f32:
4508 case Intrinsic::nvvm_tex_unified_2d_array_grad_v4u32_f32:
4509 case Intrinsic::nvvm_tex_unified_3d_v4u32_s32:
4510 case Intrinsic::nvvm_tex_unified_3d_v4u32_f32:
4511 case Intrinsic::nvvm_tex_unified_3d_level_v4u32_f32:
4512 case Intrinsic::nvvm_tex_unified_3d_grad_v4u32_f32:
4513 case Intrinsic::nvvm_tex_unified_cube_v4s32_f32:
4514 case Intrinsic::nvvm_tex_unified_cube_level_v4s32_f32:
4515 case Intrinsic::nvvm_tex_unified_cube_array_v4s32_f32:
4516 case Intrinsic::nvvm_tex_unified_cube_array_level_v4s32_f32:
4517 case Intrinsic::nvvm_tex_unified_cube_v4u32_f32:
4518 case Intrinsic::nvvm_tex_unified_cube_level_v4u32_f32:
4519 case Intrinsic::nvvm_tex_unified_cube_array_v4u32_f32:
4520 case Intrinsic::nvvm_tex_unified_cube_array_level_v4u32_f32:
4521 case Intrinsic::nvvm_tex_unified_cube_grad_v4s32_f32:
4522 case Intrinsic::nvvm_tex_unified_cube_grad_v4u32_f32:
4523 case Intrinsic::nvvm_tex_unified_cube_array_grad_v4s32_f32:
4524 case Intrinsic::nvvm_tex_unified_cube_array_grad_v4u32_f32:
4525 case Intrinsic::nvvm_tld4_unified_r_2d_v4s32_f32:
4526 case Intrinsic::nvvm_tld4_unified_g_2d_v4s32_f32:
4527 case Intrinsic::nvvm_tld4_unified_b_2d_v4s32_f32:
4528 case Intrinsic::nvvm_tld4_unified_a_2d_v4s32_f32:
4529 case Intrinsic::nvvm_tld4_unified_r_2d_v4u32_f32:
4530 case Intrinsic::nvvm_tld4_unified_g_2d_v4u32_f32:
4531 case Intrinsic::nvvm_tld4_unified_b_2d_v4u32_f32:
4532 case Intrinsic::nvvm_tld4_unified_a_2d_v4u32_f32:
4534 Info.memVT = MVT::v4i32;
4535 Info.ptrVal =
nullptr;
4538 Info.align =
Align(16);
4541 case Intrinsic::nvvm_suld_1d_i8_clamp:
4542 case Intrinsic::nvvm_suld_1d_v2i8_clamp:
4543 case Intrinsic::nvvm_suld_1d_v4i8_clamp:
4544 case Intrinsic::nvvm_suld_1d_array_i8_clamp:
4545 case Intrinsic::nvvm_suld_1d_array_v2i8_clamp:
4546 case Intrinsic::nvvm_suld_1d_array_v4i8_clamp:
4547 case Intrinsic::nvvm_suld_2d_i8_clamp:
4548 case Intrinsic::nvvm_suld_2d_v2i8_clamp:
4549 case Intrinsic::nvvm_suld_2d_v4i8_clamp:
4550 case Intrinsic::nvvm_suld_2d_array_i8_clamp:
4551 case Intrinsic::nvvm_suld_2d_array_v2i8_clamp:
4552 case Intrinsic::nvvm_suld_2d_array_v4i8_clamp:
4553 case Intrinsic::nvvm_suld_3d_i8_clamp:
4554 case Intrinsic::nvvm_suld_3d_v2i8_clamp:
4555 case Intrinsic::nvvm_suld_3d_v4i8_clamp:
4556 case Intrinsic::nvvm_suld_1d_i8_trap:
4557 case Intrinsic::nvvm_suld_1d_v2i8_trap:
4558 case Intrinsic::nvvm_suld_1d_v4i8_trap:
4559 case Intrinsic::nvvm_suld_1d_array_i8_trap:
4560 case Intrinsic::nvvm_suld_1d_array_v2i8_trap:
4561 case Intrinsic::nvvm_suld_1d_array_v4i8_trap:
4562 case Intrinsic::nvvm_suld_2d_i8_trap:
4563 case Intrinsic::nvvm_suld_2d_v2i8_trap:
4564 case Intrinsic::nvvm_suld_2d_v4i8_trap:
4565 case Intrinsic::nvvm_suld_2d_array_i8_trap:
4566 case Intrinsic::nvvm_suld_2d_array_v2i8_trap:
4567 case Intrinsic::nvvm_suld_2d_array_v4i8_trap:
4568 case Intrinsic::nvvm_suld_3d_i8_trap:
4569 case Intrinsic::nvvm_suld_3d_v2i8_trap:
4570 case Intrinsic::nvvm_suld_3d_v4i8_trap:
4571 case Intrinsic::nvvm_suld_1d_i8_zero:
4572 case Intrinsic::nvvm_suld_1d_v2i8_zero:
4573 case Intrinsic::nvvm_suld_1d_v4i8_zero:
4574 case Intrinsic::nvvm_suld_1d_array_i8_zero:
4575 case Intrinsic::nvvm_suld_1d_array_v2i8_zero:
4576 case Intrinsic::nvvm_suld_1d_array_v4i8_zero:
4577 case Intrinsic::nvvm_suld_2d_i8_zero:
4578 case Intrinsic::nvvm_suld_2d_v2i8_zero:
4579 case Intrinsic::nvvm_suld_2d_v4i8_zero:
4580 case Intrinsic::nvvm_suld_2d_array_i8_zero:
4581 case Intrinsic::nvvm_suld_2d_array_v2i8_zero:
4582 case Intrinsic::nvvm_suld_2d_array_v4i8_zero:
4583 case Intrinsic::nvvm_suld_3d_i8_zero:
4584 case Intrinsic::nvvm_suld_3d_v2i8_zero:
4585 case Intrinsic::nvvm_suld_3d_v4i8_zero:
4587 Info.memVT = MVT::i8;
4588 Info.ptrVal =
nullptr;
4591 Info.align =
Align(16);
4594 case Intrinsic::nvvm_suld_1d_i16_clamp:
4595 case Intrinsic::nvvm_suld_1d_v2i16_clamp:
4596 case Intrinsic::nvvm_suld_1d_v4i16_clamp:
4597 case Intrinsic::nvvm_suld_1d_array_i16_clamp:
4598 case Intrinsic::nvvm_suld_1d_array_v2i16_clamp:
4599 case Intrinsic::nvvm_suld_1d_array_v4i16_clamp:
4600 case Intrinsic::nvvm_suld_2d_i16_clamp:
4601 case Intrinsic::nvvm_suld_2d_v2i16_clamp:
4602 case Intrinsic::nvvm_suld_2d_v4i16_clamp:
4603 case Intrinsic::nvvm_suld_2d_array_i16_clamp:
4604 case Intrinsic::nvvm_suld_2d_array_v2i16_clamp:
4605 case Intrinsic::nvvm_suld_2d_array_v4i16_clamp:
4606 case Intrinsic::nvvm_suld_3d_i16_clamp:
4607 case Intrinsic::nvvm_suld_3d_v2i16_clamp:
4608 case Intrinsic::nvvm_suld_3d_v4i16_clamp:
4609 case Intrinsic::nvvm_suld_1d_i16_trap:
4610 case Intrinsic::nvvm_suld_1d_v2i16_trap:
4611 case Intrinsic::nvvm_suld_1d_v4i16_trap:
4612 case Intrinsic::nvvm_suld_1d_array_i16_trap:
4613 case Intrinsic::nvvm_suld_1d_array_v2i16_trap:
4614 case Intrinsic::nvvm_suld_1d_array_v4i16_trap:
4615 case Intrinsic::nvvm_suld_2d_i16_trap:
4616 case Intrinsic::nvvm_suld_2d_v2i16_trap:
4617 case Intrinsic::nvvm_suld_2d_v4i16_trap:
4618 case Intrinsic::nvvm_suld_2d_array_i16_trap:
4619 case Intrinsic::nvvm_suld_2d_array_v2i16_trap:
4620 case Intrinsic::nvvm_suld_2d_array_v4i16_trap:
4621 case Intrinsic::nvvm_suld_3d_i16_trap:
4622 case Intrinsic::nvvm_suld_3d_v2i16_trap:
4623 case Intrinsic::nvvm_suld_3d_v4i16_trap:
4624 case Intrinsic::nvvm_suld_1d_i16_zero:
4625 case Intrinsic::nvvm_suld_1d_v2i16_zero:
4626 case Intrinsic::nvvm_suld_1d_v4i16_zero:
4627 case Intrinsic::nvvm_suld_1d_array_i16_zero:
4628 case Intrinsic::nvvm_suld_1d_array_v2i16_zero:
4629 case Intrinsic::nvvm_suld_1d_array_v4i16_zero:
4630 case Intrinsic::nvvm_suld_2d_i16_zero:
4631 case Intrinsic::nvvm_suld_2d_v2i16_zero:
4632 case Intrinsic::nvvm_suld_2d_v4i16_zero:
4633 case Intrinsic::nvvm_suld_2d_array_i16_zero:
4634 case Intrinsic::nvvm_suld_2d_array_v2i16_zero:
4635 case Intrinsic::nvvm_suld_2d_array_v4i16_zero:
4636 case Intrinsic::nvvm_suld_3d_i16_zero:
4637 case Intrinsic::nvvm_suld_3d_v2i16_zero:
4638 case Intrinsic::nvvm_suld_3d_v4i16_zero:
4640 Info.memVT = MVT::i16;
4641 Info.ptrVal =
nullptr;
4644 Info.align =
Align(16);
4647 case Intrinsic::nvvm_suld_1d_i32_clamp:
4648 case Intrinsic::nvvm_suld_1d_v2i32_clamp:
4649 case Intrinsic::nvvm_suld_1d_v4i32_clamp:
4650 case Intrinsic::nvvm_suld_1d_array_i32_clamp:
4651 case Intrinsic::nvvm_suld_1d_array_v2i32_clamp:
4652 case Intrinsic::nvvm_suld_1d_array_v4i32_clamp:
4653 case Intrinsic::nvvm_suld_2d_i32_clamp:
4654 case Intrinsic::nvvm_suld_2d_v2i32_clamp:
4655 case Intrinsic::nvvm_suld_2d_v4i32_clamp:
4656 case Intrinsic::nvvm_suld_2d_array_i32_clamp:
4657 case Intrinsic::nvvm_suld_2d_array_v2i32_clamp:
4658 case Intrinsic::nvvm_suld_2d_array_v4i32_clamp:
4659 case Intrinsic::nvvm_suld_3d_i32_clamp:
4660 case Intrinsic::nvvm_suld_3d_v2i32_clamp:
4661 case Intrinsic::nvvm_suld_3d_v4i32_clamp:
4662 case Intrinsic::nvvm_suld_1d_i32_trap:
4663 case Intrinsic::nvvm_suld_1d_v2i32_trap:
4664 case Intrinsic::nvvm_suld_1d_v4i32_trap:
4665 case Intrinsic::nvvm_suld_1d_array_i32_trap:
4666 case Intrinsic::nvvm_suld_1d_array_v2i32_trap:
4667 case Intrinsic::nvvm_suld_1d_array_v4i32_trap:
4668 case Intrinsic::nvvm_suld_2d_i32_trap:
4669 case Intrinsic::nvvm_suld_2d_v2i32_trap:
4670 case Intrinsic::nvvm_suld_2d_v4i32_trap:
4671 case Intrinsic::nvvm_suld_2d_array_i32_trap:
4672 case Intrinsic::nvvm_suld_2d_array_v2i32_trap:
4673 case Intrinsic::nvvm_suld_2d_array_v4i32_trap:
4674 case Intrinsic::nvvm_suld_3d_i32_trap:
4675 case Intrinsic::nvvm_suld_3d_v2i32_trap:
4676 case Intrinsic::nvvm_suld_3d_v4i32_trap:
4677 case Intrinsic::nvvm_suld_1d_i32_zero:
4678 case Intrinsic::nvvm_suld_1d_v2i32_zero:
4679 case Intrinsic::nvvm_suld_1d_v4i32_zero:
4680 case Intrinsic::nvvm_suld_1d_array_i32_zero:
4681 case Intrinsic::nvvm_suld_1d_array_v2i32_zero:
4682 case Intrinsic::nvvm_suld_1d_array_v4i32_zero:
4683 case Intrinsic::nvvm_suld_2d_i32_zero:
4684 case Intrinsic::nvvm_suld_2d_v2i32_zero:
4685 case Intrinsic::nvvm_suld_2d_v4i32_zero:
4686 case Intrinsic::nvvm_suld_2d_array_i32_zero:
4687 case Intrinsic::nvvm_suld_2d_array_v2i32_zero:
4688 case Intrinsic::nvvm_suld_2d_array_v4i32_zero:
4689 case Intrinsic::nvvm_suld_3d_i32_zero:
4690 case Intrinsic::nvvm_suld_3d_v2i32_zero:
4691 case Intrinsic::nvvm_suld_3d_v4i32_zero:
4693 Info.memVT = MVT::i32;
4694 Info.ptrVal =
nullptr;
4697 Info.align =
Align(16);
4700 case Intrinsic::nvvm_suld_1d_i64_clamp:
4701 case Intrinsic::nvvm_suld_1d_v2i64_clamp:
4702 case Intrinsic::nvvm_suld_1d_array_i64_clamp:
4703 case Intrinsic::nvvm_suld_1d_array_v2i64_clamp:
4704 case Intrinsic::nvvm_suld_2d_i64_clamp:
4705 case Intrinsic::nvvm_suld_2d_v2i64_clamp:
4706 case Intrinsic::nvvm_suld_2d_array_i64_clamp:
4707 case Intrinsic::nvvm_suld_2d_array_v2i64_clamp:
4708 case Intrinsic::nvvm_suld_3d_i64_clamp:
4709 case Intrinsic::nvvm_suld_3d_v2i64_clamp:
4710 case Intrinsic::nvvm_suld_1d_i64_trap:
4711 case Intrinsic::nvvm_suld_1d_v2i64_trap:
4712 case Intrinsic::nvvm_suld_1d_array_i64_trap:
4713 case Intrinsic::nvvm_suld_1d_array_v2i64_trap:
4714 case Intrinsic::nvvm_suld_2d_i64_trap:
4715 case Intrinsic::nvvm_suld_2d_v2i64_trap:
4716 case Intrinsic::nvvm_suld_2d_array_i64_trap:
4717 case Intrinsic::nvvm_suld_2d_array_v2i64_trap:
4718 case Intrinsic::nvvm_suld_3d_i64_trap:
4719 case Intrinsic::nvvm_suld_3d_v2i64_trap:
4720 case Intrinsic::nvvm_suld_1d_i64_zero:
4721 case Intrinsic::nvvm_suld_1d_v2i64_zero:
4722 case Intrinsic::nvvm_suld_1d_array_i64_zero:
4723 case Intrinsic::nvvm_suld_1d_array_v2i64_zero:
4724 case Intrinsic::nvvm_suld_2d_i64_zero:
4725 case Intrinsic::nvvm_suld_2d_v2i64_zero:
4726 case Intrinsic::nvvm_suld_2d_array_i64_zero:
4727 case Intrinsic::nvvm_suld_2d_array_v2i64_zero:
4728 case Intrinsic::nvvm_suld_3d_i64_zero:
4729 case Intrinsic::nvvm_suld_3d_v2i64_zero:
4731 Info.memVT = MVT::i64;
4732 Info.ptrVal =
nullptr;
4735 Info.align =
Align(16);
4738 case Intrinsic::nvvm_tcgen05_ld_16x64b_x1:
4739 case Intrinsic::nvvm_tcgen05_ld_32x32b_x1:
4740 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x1: {
4742 Info.memVT = MVT::v1i32;
4743 Info.ptrVal =
I.getArgOperand(0);
4750 case Intrinsic::nvvm_tcgen05_ld_16x64b_x2:
4751 case Intrinsic::nvvm_tcgen05_ld_16x128b_x1:
4752 case Intrinsic::nvvm_tcgen05_ld_32x32b_x2:
4753 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x2: {
4755 Info.memVT = MVT::v2i32;
4756 Info.ptrVal =
I.getArgOperand(0);
4763 case Intrinsic::nvvm_tcgen05_ld_16x64b_x4:
4764 case Intrinsic::nvvm_tcgen05_ld_16x128b_x2:
4765 case Intrinsic::nvvm_tcgen05_ld_32x32b_x4:
4766 case Intrinsic::nvvm_tcgen05_ld_16x256b_x1:
4767 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x4: {
4769 Info.memVT = MVT::v4i32;
4770 Info.ptrVal =
I.getArgOperand(0);
4777 case Intrinsic::nvvm_tcgen05_ld_16x64b_x8:
4778 case Intrinsic::nvvm_tcgen05_ld_16x128b_x4:
4779 case Intrinsic::nvvm_tcgen05_ld_16x256b_x2:
4780 case Intrinsic::nvvm_tcgen05_ld_32x32b_x8:
4781 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x8: {
4783 Info.memVT = MVT::v8i32;
4784 Info.ptrVal =
I.getArgOperand(0);
4791 case Intrinsic::nvvm_tcgen05_ld_16x64b_x16:
4792 case Intrinsic::nvvm_tcgen05_ld_16x128b_x8:
4793 case Intrinsic::nvvm_tcgen05_ld_16x256b_x4:
4794 case Intrinsic::nvvm_tcgen05_ld_32x32b_x16:
4795 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x16: {
4797 Info.memVT = MVT::v16i32;
4798 Info.ptrVal =
I.getArgOperand(0);
4805 case Intrinsic::nvvm_tcgen05_ld_16x64b_x32:
4806 case Intrinsic::nvvm_tcgen05_ld_16x128b_x16:
4807 case Intrinsic::nvvm_tcgen05_ld_16x256b_x8:
4808 case Intrinsic::nvvm_tcgen05_ld_32x32b_x32:
4809 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x32: {
4811 Info.memVT = MVT::v32i32;
4812 Info.ptrVal =
I.getArgOperand(0);
4819 case Intrinsic::nvvm_tcgen05_ld_16x64b_x64:
4820 case Intrinsic::nvvm_tcgen05_ld_16x128b_x32:
4821 case Intrinsic::nvvm_tcgen05_ld_16x256b_x16:
4822 case Intrinsic::nvvm_tcgen05_ld_32x32b_x64:
4823 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x64: {
4825 Info.memVT = MVT::v64i32;
4826 Info.ptrVal =
I.getArgOperand(0);
4833 case Intrinsic::nvvm_tcgen05_ld_16x64b_x128:
4834 case Intrinsic::nvvm_tcgen05_ld_16x128b_x64:
4835 case Intrinsic::nvvm_tcgen05_ld_16x256b_x32:
4836 case Intrinsic::nvvm_tcgen05_ld_32x32b_x128:
4837 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x128: {
4839 Info.memVT = MVT::v128i32;
4840 Info.ptrVal =
I.getArgOperand(0);
4847 case Intrinsic::nvvm_tcgen05_st_16x64b_x1:
4848 case Intrinsic::nvvm_tcgen05_st_32x32b_x1:
4849 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x1: {
4851 Info.memVT = MVT::i32;
4852 Info.ptrVal =
I.getArgOperand(0);
4859 case Intrinsic::nvvm_tcgen05_st_16x64b_x2:
4860 case Intrinsic::nvvm_tcgen05_st_16x128b_x1:
4861 case Intrinsic::nvvm_tcgen05_st_32x32b_x2:
4862 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x2: {
4864 Info.memVT = MVT::v2i32;
4865 Info.ptrVal =
I.getArgOperand(0);
4872 case Intrinsic::nvvm_tcgen05_st_16x64b_x4:
4873 case Intrinsic::nvvm_tcgen05_st_16x128b_x2:
4874 case Intrinsic::nvvm_tcgen05_st_16x256b_x1:
4875 case Intrinsic::nvvm_tcgen05_st_32x32b_x4:
4876 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x4: {
4878 Info.memVT = MVT::v4i32;
4879 Info.ptrVal =
I.getArgOperand(0);
4886 case Intrinsic::nvvm_tcgen05_st_16x64b_x8:
4887 case Intrinsic::nvvm_tcgen05_st_16x128b_x4:
4888 case Intrinsic::nvvm_tcgen05_st_16x256b_x2:
4889 case Intrinsic::nvvm_tcgen05_st_32x32b_x8:
4890 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x8: {
4892 Info.memVT = MVT::v8i32;
4893 Info.ptrVal =
I.getArgOperand(0);
4900 case Intrinsic::nvvm_tcgen05_st_16x64b_x16:
4901 case Intrinsic::nvvm_tcgen05_st_16x128b_x8:
4902 case Intrinsic::nvvm_tcgen05_st_16x256b_x4:
4903 case Intrinsic::nvvm_tcgen05_st_32x32b_x16:
4904 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x16: {
4906 Info.memVT = MVT::v16i32;
4907 Info.ptrVal =
I.getArgOperand(0);
4914 case Intrinsic::nvvm_tcgen05_st_16x64b_x32:
4915 case Intrinsic::nvvm_tcgen05_st_16x128b_x16:
4916 case Intrinsic::nvvm_tcgen05_st_16x256b_x8:
4917 case Intrinsic::nvvm_tcgen05_st_32x32b_x32:
4918 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x32: {
4920 Info.memVT = MVT::v32i32;
4921 Info.ptrVal =
I.getArgOperand(0);
4928 case Intrinsic::nvvm_tcgen05_st_16x64b_x64:
4929 case Intrinsic::nvvm_tcgen05_st_16x128b_x32:
4930 case Intrinsic::nvvm_tcgen05_st_16x256b_x16:
4931 case Intrinsic::nvvm_tcgen05_st_32x32b_x64:
4932 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x64: {
4934 Info.memVT = MVT::v64i32;
4935 Info.ptrVal =
I.getArgOperand(0);
4942 case Intrinsic::nvvm_tcgen05_st_16x64b_x128:
4943 case Intrinsic::nvvm_tcgen05_st_16x128b_x64:
4944 case Intrinsic::nvvm_tcgen05_st_16x256b_x32:
4945 case Intrinsic::nvvm_tcgen05_st_32x32b_x128:
4946 case Intrinsic::nvvm_tcgen05_st_16x32bx2_x128: {
4948 Info.memVT = MVT::v128i32;
4949 Info.ptrVal =
I.getArgOperand(0);
4955 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg1:
4956 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg1:
4957 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg1:
4958 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg1:
4959 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1:
4960 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1:
4961 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg1_ashift:
4963 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg1_ashift:
4964 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1:
4965 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1:
4966 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg1_ashift:
4968 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg1_ashift: {
4971 Info.memVT = MVT::v4i32;
4972 Info.ptrVal =
I.getArgOperand(0);
4975 Info.align =
Align(16);
4979 case Intrinsic::nvvm_tcgen05_mma_shared_disable_output_lane_cg2:
4980 case Intrinsic::nvvm_tcgen05_mma_shared_scale_d_disable_output_lane_cg2:
4981 case Intrinsic::nvvm_tcgen05_mma_sp_shared_disable_output_lane_cg2:
4982 case Intrinsic::nvvm_tcgen05_mma_sp_shared_scale_d_disable_output_lane_cg2:
4983 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2:
4984 case Intrinsic::nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2:
4985 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2:
4986 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2:
4987 case Intrinsic::nvvm_tcgen05_mma_tensor_disable_output_lane_cg2_ashift:
4989 nvvm_tcgen05_mma_tensor_scale_d_disable_output_lane_cg2_ashift:
4990 case Intrinsic::nvvm_tcgen05_mma_sp_tensor_disable_output_lane_cg2_ashift:
4992 nvvm_tcgen05_mma_sp_tensor_scale_d_disable_output_lane_cg2_ashift: {
4995 Info.memVT = MVT::v8i32;
4996 Info.ptrVal =
I.getArgOperand(0);
4999 Info.align =
Align(16);
5017 const Align ABITypeAlign = std::min(
Align(128),
DL.getABITypeAlign(ArgTy));
5022 if (!
F || !
F->hasLocalLinkage() ||
5023 F->hasAddressTaken(
nullptr,
5027 return ABITypeAlign;
5030 return std::max(
Align(16), ABITypeAlign);
5037 Align ArgAlign = InitialAlign;
5052 ArgAlign = std::max(ArgAlign,
Align(4));
5062 std::string ParamName;
5067 ParamStr <<
"_vararg";
5069 ParamStr <<
"_param_" << Idx;
5121 if (Constraint.
size() == 1) {
5122 switch (Constraint[0]) {
5141std::pair<unsigned, const TargetRegisterClass *>
5145 if (Constraint.
size() == 1) {
5146 switch (Constraint[0]) {
5148 return std::make_pair(0U, &NVPTX::B1RegClass);
5151 return std::make_pair(0U, &NVPTX::B16RegClass);
5154 return std::make_pair(0U, &NVPTX::B32RegClass);
5158 return std::make_pair(0U, &NVPTX::B64RegClass);
5160 if (STI.getSmVersion() < 70)
5162 "supported for sm_70 and higher!");
5163 return std::make_pair(0U, &NVPTX::B128RegClass);
5193 return Const && Const->getZExtValue() == 0;
5225 if (M->getOpcode() !=
ISD::MUL || !M.getNode()->hasOneUse())
5233 ((ZeroOpNum == 1) ? N1 : MAD),
5234 ((ZeroOpNum == 1) ? MAD : N1));
5249 (
N->getFlags().hasAllowContract() &&
5262 int nonAddCount = 0;
5271 int orderNo =
N->getIROrder();
5277 if (orderNo - orderNo2 < 500)
5283 bool opIsLive =
false;
5292 int orderNo3 =
User->getIROrder();
5293 if (orderNo3 > orderNo) {
5301 int orderNo3 =
User->getIROrder();
5302 if (orderNo3 > orderNo) {
5337 EVT ElementVT =
N->getValueType(0);
5348 if (U.getValueType() == MVT::Glue || U.getValueType() == MVT::Other)
5350 if (U.getUser()->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
5351 if (N->getOpcode() != ISD::LOAD)
5368 return !U.getUser()->use_empty();
5382 unsigned OldNumOutputs;
5383 switch (
LD->getOpcode()) {
5390 Operands.push_back(DCI.DAG.getIntPtrConstant(
5400 if (ElementVT != MVT::v2f32)
5411 const unsigned NewNumOutputs = OldNumOutputs * 2;
5414 NewVTs.append(
LD->value_begin() + OldNumOutputs,
LD->value_end());
5417 SDValue NewLoad = DCI.DAG.getMemIntrinsicNode(
5418 Opcode,
DL, DCI.DAG.getVTList(NewVTs),
Operands,
LD->getMemoryVT(),
5419 LD->getMemOperand());
5425 for (
unsigned I :
seq(OldNumOutputs))
5426 Results.push_back(DCI.DAG.getBuildVector(
5427 ElementVT,
DL, {NewLoad.getValue(I * 2), NewLoad.getValue(I * 2 + 1)}));
5432 return DCI.DAG.getMergeValues(
Results,
DL);
5447 unsigned Front,
unsigned Back) {
5454 EVT ElementVT =
N->getOperand(Front).getValueType();
5464 switch (
N->getOpcode()) {
5477 if (ElementVT != MVT::v2f32)
5491 for (
SDValue BV :
N->ops().drop_front(Front).drop_back(Back)) {
5497 if (!BV.hasOneUse())
5504 if (
Op.getOpcode() == ISD::BITCAST)
5505 Op =
Op.getOperand(0);
5509 Op->getOperand(0).getValueType() == MVT::i32)
5516 Operands.append({BV.getOperand(0), BV.getOperand(1)});
5518 Operands.append(
N->op_end() - Back,
N->op_end());
5522 ST->getMemoryVT(), ST->getMemOperand());
5533 if (!ST->getValue().getValueType().isSimple())
5546 if (!
N->getValueType(0).isSimple())
5566 if (VT.
isVector() || VT != MVT::i32)
5586 if (VT.
isVector() || !(VT == MVT::f32 || VT == MVT::f64))
5599 switch (MinMax2Opcode) {
5601 case ISD::FMAXIMUMNUM:
5604 case ISD::FMINIMUMNUM:
5619 unsigned PTXVersion,
unsigned SmVersion) {
5622 EVT VT =
N->getValueType(0);
5623 if (VT != MVT::f32 || PTXVersion < 88 || SmVersion < 100)
5628 unsigned MinMaxOp2 =
N->getOpcode();
5658 EVT VT =
N->getValueType(0);
5662 const SDValue &Num =
N->getOperand(0);
5663 const SDValue &Den =
N->getOperand(1);
5666 if (U->getOpcode() == DivOpc && U->getOperand(0) == Num &&
5685 if (!
Op.hasOneUse())
5687 EVT ToVT =
N->getValueType(0);
5688 EVT FromVT =
Op.getValueType();
5689 if (!((ToVT == MVT::i32 && FromVT == MVT::i16) ||
5690 (ToVT == MVT::i64 && FromVT == MVT::i32)))
5697 unsigned ExtOpcode =
N->getOpcode();
5698 unsigned Opcode = 0;
5707 const auto ShiftAmt =
Op.getConstantOperandVal(1);
5730 EVT OrigVT =
Op.getOperand(0).getValueType();
5736 EVT OrigVT =
Op.getOperand(0).getValueType();
5763 IsSigned = (LHSSign ==
Signed);
5767 const APInt &Val = CI->getAPIntValue();
5769 return Val.
isIntN(OptSize);
5778 return LHSSign == RHSSign;
5788 EVT MulType =
N->getValueType(0);
5789 if (MulType != MVT::i32 && MulType != MVT::i64) {
5829 if (MulType == MVT::i32) {
5830 DemotedVT = MVT::i16;
5832 DemotedVT = MVT::i32;
5854 return Const && Const->getZExtValue() == 1;
5862 return Add->getOperand(1);
5865 return Add->getOperand(0);
5906 (ConstOpNo == 1) ?
X : NewMul,
5907 (ConstOpNo == 1) ? NewMul :
X);
5918 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
5968 unsigned int SmVersion) {
5969 EVT CCType =
N->getValueType(0);
5973 EVT AType =
A.getValueType();
5974 if (!(CCType == MVT::v2i1 && (AType == MVT::v2f16 || AType == MVT::v2bf16)))
5977 if (
A.getValueType() == MVT::v2bf16 && SmVersion < 90)
5988 DL, DCI.
DAG.
getVTList(MVT::i1, MVT::i1), {A, B, N->getOperand(2)});
6016 if (!(VectorBits == 16 || VectorBits == 32 || VectorBits == 64))
6021 if (!Index || Index->getZExtValue() == 0)
6036 if (EltVT != EltIVT)
6037 Result = DCI.
DAG.
getNode(ISD::BITCAST,
DL, EltVT, Result);
6039 if (EltVT !=
N->getValueType(0))
6049 if (VectorVT != MVT::v4i8)
6060 for (
int I = 0;
I < 4; ++
I) {
6079 auto VT =
N->getValueType(0);
6086 auto Op0 =
N->getOperand(0);
6087 auto Op1 =
N->getOperand(1);
6094 std::pair<SDValue *, uint64_t *> OpData[2] = {{&Op0, &Op0Bytes},
6100 for (
auto &[
Op, OpBytes] : OpData) {
6102 if (
Op->getOpcode() == ISD::BITCAST)
6103 *
Op =
Op->getOperand(0);
6106 Op->getOperand(0).getValueType() == MVT::i32))
6111 if (!
Op->hasOneUse())
6114 *
Op =
Op->getOperand(0);
6122 assert((*OpBytes == 0x10 || *OpBytes == 0x54) &&
6123 "PRMT selector values out of range");
6125 *
Op =
Op->getOperand(0);
6131 auto &DAG = DCI.
DAG;
6135 (Op1Bytes << 8) | Op0Bytes,
DL, DAG);
6144 assert(ASCN2->getDestAddressSpace() == ASCN1->getSrcAddressSpace());
6147 if (ASCN1->getDestAddressSpace() == ASCN2->getSrcAddressSpace())
6148 return ASCN2->getOperand(0);
6166 const auto GetSelector = [](
unsigned S0,
unsigned S1,
unsigned S2,
6168 return APInt(32, S0 | (
S1 << 4) | (S2 << 8) | (S3 << 12));
6173 return GetSelector(V, V + 1, V + 2, V + 3);
6175 return GetSelector(V, (V - 1) & 7, (V - 2) & 7, (V - 3) & 7);
6177 return GetSelector(V, V, V, V);
6179 return GetSelector(V, std::max(V, 1U), std::max(V, 2U), 3U);
6181 return GetSelector(0, std::min(V, 1U), std::min(V, 2U), V);
6183 unsigned V1 = (V & 1) << 1;
6184 return GetSelector(V1, V1 + 1, V1, V1 + 1);
6192 assert(
A.getBitWidth() == 32 &&
B.getBitWidth() == 32 &&
6193 Selector.
getBitWidth() == 32 &&
"PRMT must have i32 operands");
6197 APInt Result(32, 0);
6202 APInt Byte = BitField.extractBits(8, Idx * 8);
6204 Byte = Byte.ashr(8);
6205 Result.insertBits(Byte,
I * 8);
6220 N->getConstantOperandAPInt(1),
6221 N->getConstantOperandAPInt(2),
6222 N->getConstantOperandVal(3)),
6223 SDLoc(
N),
N->getValueType(0));
6238 switch (R.getOpcode()) {
6243 case ISD::BITCAST: {
6270 for (
auto &
Op : R->ops()) {
6284 R.getValueType(), V, R.getOperand(1));
6300 if (
Reg.getOpcode() != ISD::LOAD) {
6309 DAGCombinerInfo &DCI)
const {
6311 switch (
N->getOpcode()) {
6316 case ISD::ADDRSPACECAST:
6331 case ISD::FMAXIMUMNUM:
6332 case ISD::FMINIMUMNUM:
6334 STI.getSmVersion());
6367 EVT ToVT =
Op->getValueType(0);
6368 if (ToVT != MVT::v2i8) {
6395 case Intrinsic::nvvm_ldu_global_i:
6396 case Intrinsic::nvvm_ldu_global_f:
6397 case Intrinsic::nvvm_ldu_global_p: {
6398 EVT ResVT =
N->getValueType(0);
6410 bool NeedTrunc =
false;
6416 unsigned Opcode = 0;
6424 LdResVTs = DAG.
getVTList(EltVT, EltVT, MVT::Other);
6428 EVT ListVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other };
6441 OtherOps.
append(
N->op_begin() + 2,
N->op_end());
6451 for (
unsigned i = 0; i < NumElts; ++i) {
6469 "Custom handling of non-i8 ldu/ldg?");
6492 case Intrinsic::nvvm_tcgen05_ld_16x64b_x4:
6493 case Intrinsic::nvvm_tcgen05_ld_16x64b_x8:
6494 case Intrinsic::nvvm_tcgen05_ld_16x64b_x16:
6495 case Intrinsic::nvvm_tcgen05_ld_16x64b_x32:
6496 case Intrinsic::nvvm_tcgen05_ld_16x64b_x64:
6497 case Intrinsic::nvvm_tcgen05_ld_16x64b_x128:
6498 case Intrinsic::nvvm_tcgen05_ld_32x32b_x4:
6499 case Intrinsic::nvvm_tcgen05_ld_32x32b_x8:
6500 case Intrinsic::nvvm_tcgen05_ld_32x32b_x16:
6501 case Intrinsic::nvvm_tcgen05_ld_32x32b_x32:
6502 case Intrinsic::nvvm_tcgen05_ld_32x32b_x64:
6503 case Intrinsic::nvvm_tcgen05_ld_32x32b_x128:
6504 case Intrinsic::nvvm_tcgen05_ld_16x128b_x2:
6505 case Intrinsic::nvvm_tcgen05_ld_16x128b_x4:
6506 case Intrinsic::nvvm_tcgen05_ld_16x128b_x8:
6507 case Intrinsic::nvvm_tcgen05_ld_16x128b_x16:
6508 case Intrinsic::nvvm_tcgen05_ld_16x128b_x32:
6509 case Intrinsic::nvvm_tcgen05_ld_16x128b_x64:
6510 case Intrinsic::nvvm_tcgen05_ld_16x256b_x1:
6511 case Intrinsic::nvvm_tcgen05_ld_16x256b_x2:
6512 case Intrinsic::nvvm_tcgen05_ld_16x256b_x4:
6513 case Intrinsic::nvvm_tcgen05_ld_16x256b_x8:
6514 case Intrinsic::nvvm_tcgen05_ld_16x256b_x16:
6515 case Intrinsic::nvvm_tcgen05_ld_16x256b_x32:
6517 Results.push_back(Res->first);
6518 Results.push_back(Res->second);
6522 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x4:
6523 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x8:
6524 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x16:
6525 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x32:
6526 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x64:
6527 case Intrinsic::nvvm_tcgen05_ld_16x32bx2_x128:
6529 Results.push_back(Res->first);
6530 Results.push_back(Res->second);
6545 assert(
Reg.getValueType() == MVT::i128 &&
6546 "Custom lowering for CopyFromReg with 128-bit reg only");
6548 N->getValueType(2)};
6579 assert(
N->getValueType(0) == MVT::i128 &&
6580 "Custom lowering for atomic128 only supports i128");
6588 "Support for b128 atomics introduced in PTX ISA version 8.3 and "
6589 "requires target sm_90.",
6600 for (
const auto &
Op : AN->
ops().drop_front(2)) {
6608 unsigned Opcode =
N->getOpcode() == ISD::ATOMIC_SWAP
6615 {Result.getValue(0), Result.getValue(1)}));
6616 Results.push_back(Result.getValue(2));
6619void NVPTXTargetLowering::ReplaceNodeResults(
6621 switch (
N->getOpcode()) {
6639 case ISD::ATOMIC_CMP_SWAP:
6640 case ISD::ATOMIC_SWAP:
6652 if (Ty->isHalfTy() && STI.getSmVersion() >= 70 &&
6653 STI.getPTXVersion() >= 63)
6655 if (Ty->isBFloatTy() && STI.getSmVersion() >= 90 &&
6656 STI.getPTXVersion() >= 78)
6658 if (Ty->isFloatTy())
6660 if (Ty->isDoubleTy() && STI.hasAtomAddF64())
6666 assert(Ty->isIntegerTy() &&
"Ty should be integer at this point");
6686 if (STI.hasAtomBitwise64())
6707 if (STI.hasAtomMinMax64())
6746 STI.getMinCmpXchgSizeInBits() ||
6753 bool BitwidthSupportedAndIsSeqCst =
6756 STI.getMinCmpXchgSizeInBits();
6793 CASWidth < STI.getMinCmpXchgSizeInBits()))
6816 case ISD::VP_FP_TO_UINT:
6818 return ISD::VP_FP_TO_SINT;
6839 unsigned Mode =
Op.getConstantOperandVal(3);
6849 "PRMT must have i32 operands");
6858 KnownBits Byte = BitField.extractBits(8, Idx * 8);
6869 auto ExtType = LD->getConstantOperandVal(LD->getNumOperands() - 1);
6874 auto DestVT = LD->getValueType(0);
6875 if (DestVT.isVector())
6888 switch (
Op.getOpcode()) {
6915 APInt &Src = Idx < 4 ? DemandedLHS : DemandedRHS;
6916 unsigned ByteStart = (Idx % 4) * 8;
6918 Src.
setBit(ByteStart + 7);
6920 Src.setBits(ByteStart, ByteStart + 8);
6923 return {DemandedLHS, DemandedRHS};
6942 SDValue Op0 = PRMT.getOperand(0);
6943 SDValue Op1 = PRMT.getOperand(1);
6948 unsigned Mode = PRMT.getConstantOperandVal(3);
6953 const unsigned LeadingBytes =
DemandedBits.countLeadingZeros() / 8;
6954 const unsigned SelBits = (4 - LeadingBytes) * 4;
6955 if (Selector.
getLoBits(SelBits) ==
APInt(32, 0x3210).getLoBits(SelBits))
6957 if (Selector.
getLoBits(SelBits) ==
APInt(32, 0x7654).getLoBits(SelBits))
6970 if ((DemandedOp0 && DemandedOp0 != Op0) ||
6971 (DemandedOp1 && DemandedOp1 != Op1)) {
6972 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
6973 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
6985 switch (
Op.getOpcode()) {
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU Register Bank Select
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombineWithOperands - Try DAG combinations for an ADD with operands N0 and N1.
static SDValue PerformADDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
static SDValue PerformVSELECTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformMULCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
static SDValue PerformFADDCombine(SDNode *N, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue PerformBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const ARMSubtarget *Subtarget)
PerformBUILD_VECTORCombine - Target-specific dag combine xforms for ISD::BUILD_VECTOR.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file contains the declarations of entities that describe floating point environment and related ...
Module.h This file contains the declarations for the Module class.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
mir Rename Register Operands
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
Register const TargetRegisterInfo * TRI
NVPTX address space definition.
static bool shouldConvertToIndirectCall(const CallBase *CB, const GlobalAddressSDNode *Func)
static SDValue combineADDRSPACECAST(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static cl::opt< bool > sched4reg("nvptx-sched4reg", cl::desc("NVPTX Specific: schedule for register pressue"), cl::init(false))
static SDValue lowerTcgen05St(SDValue Op, SelectionDAG &DAG)
static SDValue PerformEXTRACTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static cl::opt< NVPTX::DivPrecisionLevel > UsePrecDivF32("nvptx-prec-divf32", cl::Hidden, cl::desc("NVPTX Specific: Override the precision of the lowering for f32 fdiv"), cl::values(clEnumValN(NVPTX::DivPrecisionLevel::Approx, "0", "Use div.approx"), clEnumValN(NVPTX::DivPrecisionLevel::Full, "1", "Use div.full"), clEnumValN(NVPTX::DivPrecisionLevel::IEEE754, "2", "Use IEEE Compliant F32 div.rnd if available (default)"), clEnumValN(NVPTX::DivPrecisionLevel::IEEE754_NoFTZ, "3", "Use IEEE Compliant F32 div.rnd if available, no FTZ")), cl::init(NVPTX::DivPrecisionLevel::IEEE754))
static bool isConstOne(const SDValue &Operand)
static cl::opt< unsigned > FMAContractLevelOpt("nvptx-fma-level", cl::Hidden, cl::desc("NVPTX Specific: FMA contraction (0: don't do it" " 1: do it 2: do it aggressively"), cl::init(2))
static bool IsPTXVectorType(MVT VT)
static SDValue lowerLOADi1(LoadSDNode *LD, SelectionDAG &DAG)
static SDValue lowerIntrinsicVoid(SDValue Op, SelectionDAG &DAG)
static MachinePointerInfo refinePtrAS(SDValue &Ptr, SelectionDAG &DAG, const DataLayout &DL, const TargetLowering &TL)
static SDValue lowerROT(SDValue Op, SelectionDAG &DAG)
static void ComputePTXValueVTs(const TargetLowering &TLI, const DataLayout &DL, LLVMContext &Ctx, CallingConv::ID CallConv, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< uint64_t > &Offsets, uint64_t StartingOffset=0)
ComputePTXValueVTs - For the given Type Ty, returns the set of primitive legal-ish MVTs that compose ...
static void ReplaceBITCAST(SDNode *Node, SelectionDAG &DAG, SmallVectorImpl< SDValue > &Results)
static void replaceAtomicSwap128(SDNode *N, SelectionDAG &DAG, const NVPTXSubtarget &STI, SmallVectorImpl< SDValue > &Results)
static SDValue lowerSTOREVector(SDValue Op, SelectionDAG &DAG, const NVPTXSubtarget &STI)
static SDValue lowerLoadVector(SDNode *N, SelectionDAG &DAG, const NVPTXSubtarget &STI)
static void replaceProxyReg(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI, SmallVectorImpl< SDValue > &Results)
static void ReplaceCopyFromReg_128(SDNode *N, SelectionDAG &DAG, SmallVectorImpl< SDValue > &Results)
static SDValue lowerCTLZCTPOP(SDValue Op, SelectionDAG &DAG)
static SDValue combineMADConstOne(SDValue X, SDValue Add, EVT VT, SDLoc DL, TargetLowering::DAGCombinerInfo &DCI)
static SDValue combinePRMT(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, CodeGenOptLevel OptLevel)
static SDValue combinePackingMovIntoStore(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, unsigned Front, unsigned Back)
Fold packing movs into a store.
static void ReplaceINTRINSIC_W_CHAIN(SDNode *N, SelectionDAG &DAG, SmallVectorImpl< SDValue > &Results)
static SDValue getBuildVectorizedValue(unsigned N, const SDLoc &dl, SelectionDAG &DAG, T GetElement)
static SDValue getExtractVectorizedValue(SDValue V, unsigned I, EVT VT, const SDLoc &dl, SelectionDAG &DAG)
static NVPTXISD::NodeType getMinMax3Opcode(unsigned MinMax2Opcode)
Get 3-input version of a 2-input min/max opcode.
static unsigned canMergeParamLoadStoresStartingAt(unsigned Idx, uint32_t AccessSize, const SmallVectorImpl< EVT > &ValueVTs, const SmallVectorImpl< T > &Offsets, Align ParamAlignment)
static EVT getVectorizedVT(EVT VT, unsigned N, LLVMContext &C)
static SDValue lowerIntrinsicWOChain(SDValue Op, SelectionDAG &DAG)
static SDValue PerformFMinMaxCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, unsigned PTXVersion, unsigned SmVersion)
PerformFMinMaxCombine - Combine (fmaxnum (fmaxnum a, b), c) into (fmaxnum3 a, b, c).
static SDValue combineMulWide(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, CodeGenOptLevel OptLevel)
static SDValue PerformFADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI, CodeGenOptLevel OptLevel)
static SDValue lowerIntrinsicWChain(SDValue Op, SelectionDAG &DAG)
static bool isConstZero(const SDValue &Operand)
static SDValue LowerVectorArith(SDValue Op, SelectionDAG &DAG)
static SDValue LowerTcgen05MMADisableOutputLane(SDValue Op, SelectionDAG &DAG)
static bool IsMulWideOperandDemotable(SDValue Op, unsigned OptSize, OperandSignedness &S)
IsMulWideOperandDemotable - Checks if the provided DAG node is an operand that can be demoted to OptS...
static unsigned getTcgen05MMADisableOutputLane(unsigned IID)
static std::pair< APInt, APInt > getPRMTDemandedBits(const APInt &SelectorVal, const APInt &DemandedBits)
static APInt computePRMT(APInt A, APInt B, APInt Selector, unsigned Mode)
static ISD::NodeType getScalarOpcodeForReduction(unsigned ReductionOpcode)
static SDValue PerformREMCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, CodeGenOptLevel OptLevel)
static SDValue PerformMULCombineWithOperands(SDNode *N, SDValue N0, SDValue N1, TargetLowering::DAGCombinerInfo &DCI)
static void computeKnownBitsForPRMT(const SDValue Op, KnownBits &Known, const SelectionDAG &DAG, unsigned Depth)
static SDValue combineUnpackingMovIntoLoad(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
Fold unpacking movs into a load by increasing the number of return values.
static SDValue LowerClusterLaunchControlQueryCancel(SDValue Op, SelectionDAG &DAG)
static std::optional< std::pair< SDValue, SDValue > > lowerTcgen05Ld(SDNode *N, SelectionDAG &DAG, bool HasOffset=false)
static std::optional< NVPTXISD::NodeType > getScalar3OpcodeForReduction(unsigned ReductionOpcode)
Get 3-input scalar reduction opcode.
static std::optional< std::pair< SDValue, SDValue > > replaceLoadVector(SDNode *N, SelectionDAG &DAG, const NVPTXSubtarget &STI)
replaceLoadVector - Convert vector loads into multi-output scalar loads.
static SDValue expandFSH64(SDValue A, SDValue B, SDValue ShiftAmount, SDLoc DL, unsigned Opcode, SelectionDAG &DAG)
static bool AreMulWideOperandsDemotable(SDValue LHS, SDValue RHS, unsigned OptSize, bool &IsSigned)
AreMulWideOperandsDemotable - Checks if the given LHS and RHS operands can be demoted to OptSize bits...
static SDValue TryMULWIDECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
TryMULWIDECombine - Attempt to replace a multiply of M bits with a multiply of M/2 bits that produces...
static SDValue lowerPrmtIntrinsic(SDValue Op, SelectionDAG &DAG)
static SDValue combineMulSelectConstOne(SDValue X, SDValue Select, EVT VT, SDLoc DL, TargetLowering::DAGCombinerInfo &DCI)
static SDValue buildTreeReduction(const SmallVector< SDValue > &Elements, EVT EltTy, ArrayRef< std::pair< unsigned, unsigned > > Ops, const SDLoc &DL, const SDNodeFlags Flags, SelectionDAG &DAG)
Reduces the elements using the scalar operations provided.
static SDValue combineProxyReg(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SmallVector< unsigned, 16 > VectorizePTXValueVTs(const SmallVectorImpl< EVT > &ValueVTs, const SmallVectorImpl< T > &Offsets, Align ParamAlignment, bool IsVAArg=false)
static SDValue getPRMT(SDValue A, SDValue B, SDValue Selector, SDLoc DL, SelectionDAG &DAG, unsigned Mode=NVPTX::PTXPrmtMode::NONE)
static SDValue matchMADConstOnePattern(SDValue Add)
static SDValue correctParamType(SDValue V, EVT ExpectedVT, ISD::ArgFlagsTy Flags, SelectionDAG &DAG, SDLoc dl)
static ISD::NodeType getExtOpcode(const ISD::ArgFlagsTy &Flags)
static cl::opt< bool > UsePrecSqrtF32("nvptx-prec-sqrtf32", cl::Hidden, cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."), cl::init(true))
static void computeKnownBitsForLoadV(const SDValue Op, KnownBits &Known)
static APInt getPRMTSelector(const APInt &Selector, unsigned Mode)
static EVT promoteScalarIntegerPTX(const EVT VT)
PromoteScalarIntegerPTX Used to make sure the arguments/returns are suitable for passing and promote ...
static SDValue simplifyDemandedBitsForPRMT(SDValue PRMT, const APInt &DemandedBits, SelectionDAG &DAG, const TargetLowering &TLI, unsigned Depth)
static SDValue lowerFREM(SDValue Op, SelectionDAG &DAG)
static SDValue canonicalizePRMTInput(SDValue Op, SelectionDAG &DAG)
static SDValue sinkProxyReg(SDValue R, SDValue Chain, TargetLowering::DAGCombinerInfo &DCI)
static SDValue lowerFSH(SDValue Op, SelectionDAG &DAG)
static SDValue PromoteBinOpToF32(SDNode *N, SelectionDAG &DAG)
static SDValue PerformSETCCCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, unsigned int SmVersion)
static std::optional< std::pair< unsigned int, MVT > > getVectorLoweringShape(EVT VectorEVT, const NVPTXSubtarget &STI, unsigned AddressSpace)
static cl::opt< bool > ForceMinByValParamAlign("nvptx-force-min-byval-param-align", cl::Hidden, cl::desc("NVPTX Specific: force 4-byte minimal alignment for byval" " params of device functions."), cl::init(false))
static cl::opt< bool > UseApproxLog2F32("nvptx-approx-log2f32", cl::desc("NVPTX Specific: whether to use lg2.approx for log2"), cl::init(false))
Whereas CUDA's implementation (see libdevice) uses ex2.approx for exp2(), it does NOT use lg2....
static SDValue lowerSELECT(SDValue Op, SelectionDAG &DAG)
static SDValue combineLOAD(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const NVPTXSubtarget &STI)
static SDValue combineSTORE(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const NVPTXSubtarget &STI)
static SDValue PerformSHLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, CodeGenOptLevel OptLevel)
PerformSHLCombine - Runs PTX-specific DAG combine patterns on SHL nodes.
MachineInstr unsigned OpIdx
const SmallVectorImpl< MachineOperand > & Cond
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
This file defines the SmallVector class.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
This file describes how to lower LLVM code to machine code.
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Class for arbitrary precision integers.
LLVM_ABI APInt getLoBits(unsigned numBits) const
Compute an APInt containing numBits lowbits from this APInt.
uint64_t getZExtValue() const
Get zero extended value.
void setHighBits(unsigned hiBits)
Set the top hiBits bits.
LLVM_ABI APInt getHiBits(unsigned numBits) const
Compute an APInt containing numBits highbits from this APInt.
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool isSignedIntN(unsigned N) const
Check if this APInt has an N-bits signed integer value.
bool slt(const APInt &RHS) const
Signed less than comparison.
LLVM_ABI APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
bool isIntN(unsigned N) const
Check if this APInt has an N-bits unsigned integer value.
bool sge(const APInt &RHS) const
Signed greater or equal comparison.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & back() const
back - Get the last element.
ArrayRef< T > drop_back(size_t N=1) const
Drop the last N elements of the array.
bool empty() const
empty - Check if the array is empty.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
an instruction that atomically reads a memory location, combines it with another value,...
@ Min
*p = old <signed v ? old : v
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
@ UDecWrap
Decrement one until a minimum value or zero.
bool isFloatingPointOperation() const
BinOp getOperation() const
This is an SDNode representing atomic operations.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
FunctionType * getFunctionType() const
This class represents a function call, abstracting a target machine's calling convention.
const APInt & getAPIntValue() const
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
LLVM_ABI Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Diagnostic information for unsupported feature in backend.
void addFnAttr(Attribute::AttrKind Kind)
Add function attributes to this function.
Common base class shared among various IRBuilders.
This is an important class for using LLVM in a threaded context.
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
This class is used to represent ISD::LOAD nodes.
MCSection * getDataSection() const
Instances of this class represent a uniqued identifier for a section in the current translation unit.
StringRef getName() const
getName - Get the symbol name.
static auto integer_fixedlen_vector_valuetypes()
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
static auto integer_valuetypes()
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static auto fixedlen_vector_valuetypes()
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
static MVT getIntegerVT(unsigned BitWidth)
static auto fp_valuetypes()
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
static auto fp_fixedlen_vector_valuetypes()
DenormalMode getDenormalMode(const fltSemantics &FPType) const
Returns the denormal handling type for the default rounding mode of the function.
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
@ EK_Inline
EK_Inline - Jump table entries are emitted inline at their point of use.
const std::vector< MachineJumpTableEntry > & getJumpTables() const
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
This is an abstract virtual class for memory operations.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
EVT getMemoryVT() const
Return the type of the in-memory value.
static unsigned getFromTypeWidthForLoad(const MemSDNode *Mem)
bool hasAtomSwap128() const
bool hasF32x2Instructions() const
bool has256BitVectorLoadStore(unsigned AS) const
AtomicOrdering atomicOperationOrderAfterFenceSplit(const Instruction *I) const override
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
const NVPTXTargetMachine * nvTM
bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0) const override
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
NVPTXTargetLowering(const NVPTXTargetMachine &TM, const NVPTXSubtarget &STI)
std::string getPrototype(const DataLayout &DL, Type *, const ArgListTy &, const SmallVectorImpl< ISD::OutputArg > &, std::optional< unsigned > FirstVAArg, const CallBase &CB, unsigned UniqueCallSite) const
unsigned getPreferredFPToIntOpcode(unsigned Op, EVT FromVT, EVT ToVT) const override
bool useF32FTZ(const MachineFunction &MF) const
SDValue LowerSTACKSAVE(SDValue Op, SelectionDAG &DAG) const
Align getFunctionArgumentAlignment(const Function *F, Type *Ty, unsigned Idx, const DataLayout &DL) const
SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &ExtraSteps, bool &UseOneConst, bool Reciprocal) const override
Hooks for building estimates in place of slower divisions and square roots.
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &dl, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
SDValue LowerSTACKRESTORE(SDValue Op, SelectionDAG &DAG) const
Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
std::string getParamName(const Function *F, int Idx) const
TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const override
Return the preferred vector type legalization action.
NVPTX::DivPrecisionLevel getDivF32Level(const MachineFunction &MF, const SDNode &N) const
bool shouldInsertFencesForAtomic(const Instruction *) const override
Whether AtomicExpandPass should automatically insert fences and reduce ordering for this atomic.
Align getFunctionParamOptimizedAlign(const Function *F, Type *ArgTy, const DataLayout &DL) const
getFunctionParamOptimizedAlign - since function arguments are passed via .param space,...
SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Ctx, EVT VT) const override
Return the ValueType of the result of SETCC operations.
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
isLegalAddressingMode - Return true if the addressing mode represented by AM is legal for this target...
Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
Align getFunctionByValParamAlign(const Function *F, Type *ArgTy, Align InitialAlign, const DataLayout &DL) const
Helper for computing alignment of a device function byval parameter.
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
bool allowFMA(MachineFunction &MF, CodeGenOptLevel OptLevel) const
bool usePrecSqrtF32(const SDNode *N=nullptr) const
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
SDValue LowerCall(CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
MCSection * SelectSectionForGlobal(const GlobalObject *GO, SectionKind Kind, const TargetMachine &TM) const override
~NVPTXTargetObjectFile() override
static LLVM_ABI PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
bool hasOneUse() const
Return true if there is exactly one use of this node.
unsigned getIROrder() const
Return the node ordering.
SDNodeFlags getFlags() const
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
SDVTList getVTList() const
const SDValue & getOperand(unsigned Num) const
bool isUndef() const
Returns true if the node type is UNDEF or POISON.
iterator_range< user_iterator > users()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
unsigned getOpcode() const
SectionKind - This is a simple POD value that classifies the properties of a section.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
LLVM_ABI SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS)
Return an AddrSpaceCastSDNode.
const TargetSubtargetInfo & getSubtarget() const
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
LLVM_ABI SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
LLVM_ABI SDValue getSymbolFunctionGlobalAddress(SDValue Op, Function **TargetFunction=nullptr)
Return a GlobalAddress of the function from the current module with name matching the given ExternalS...
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=LocationSize::precise(0), const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
LLVM_ABI Align getEVTAlign(EVT MemoryVT) const
Compute the default alignment value for the given type.
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
LLVM_ABI SDNode * MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef< SDValue > Ops)
This mutates the specified node to have the specified return type, opcode, and operands.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
const DataLayout & getDataLayout() const
LLVM_ABI SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
LLVM_ABI SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
LLVM_ABI SDValue getBasicBlock(MachineBasicBlock *MBB)
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
LLVM_ABI SDValue getExternalSymbol(const char *Sym, EVT VT)
LLVM_ABI SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
LLVM_ABI KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
LLVM_ABI SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
ArrayRef< int > getMask() const
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
constexpr size_t size() const
size - Get the string size.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
void setMaxDivRemBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum div/rem the backend supports.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
Convenience method to set an operation to Promote and specify the type in a single call.
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
void addBypassSlowDiv(unsigned int SlowBitWidth, unsigned int FastBitWidth)
Tells the code generator which bitwidths to bypass.
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrNegativeOneBooleanContent
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
Align getMinStackArgumentAlignment() const
Return the minimum stack alignment of an argument.
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
std::vector< ArgListEntry > ArgListTy
virtual Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const
virtual Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const
Inserts in the IR a target-specific intrinsic specifying a fence.
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
void setJumpIsExpensive(bool isExpensive=true)
Tells the code generator not to expand logic operations on comparison predicates into separate sequen...
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth=0) const
More limited version of SimplifyDemandedBits that can be used to "lookthrough" ops that don't contrib...
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
TargetLowering(const TargetLowering &)=delete
SDValue expandRoundInexactToOdd(EVT ResultVT, SDValue Op, const SDLoc &DL, SelectionDAG &DAG) const
Truncate Op to ResultVT.
SDValue expandFP_ROUND(SDNode *Node, SelectionDAG &DAG) const
Expand round(fp) to fp conversion.
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
Primary interface to the complete machine description for the target machine.
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
MCSymbol * getSymbol(const GlobalValue *GV) const
FPOpFusion::FPOpFusionMode AllowFPOpFusion
AllowFPOpFusion - This flag is set by the -fp-contract=xxx option.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetFrameLowering * getFrameLowering() const
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isVoidTy() const
Return true if this is 'void'.
Type * getType() const
All values are typed, get the type of this value.
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
LLVM_ABI APInt pow(const APInt &X, int64_t N)
Compute X^N for N>=0.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ BSWAP
Byte Swap and Counting operators.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ ADD
Simple integer binary arithmetic operators.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ SIGN_EXTEND
Conversion operators.
@ SSUBO
Same for subtraction.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
LLVM_ABI bool allOperandsUndef(const SDNode *N)
Return true if the node has at least one operand and all operands of the specified node are ISD::UNDE...
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
@ Bitcast
Perform the operation on a different, but equivalently sized type.
@ ADDRESS_SPACE_SHARED_CLUSTER
@ TCGEN05_MMA_SHARED_SCALE_D_DISABLE_OUTPUT_LANE_CG2
@ TCGEN05_MMA_SP_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG2_ASHIFT
@ TCGEN05_MMA_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG1
@ CLUSTERLAUNCHCONTROL_QUERY_CANCEL_IS_CANCELED
@ TCGEN05_MMA_SP_SHARED_SCALE_D_DISABLE_OUTPUT_LANE_CG2
@ CALL
This node represents a PTX call instruction.
@ TCGEN05_MMA_SP_SHARED_SCALE_D_DISABLE_OUTPUT_LANE_CG1
@ TCGEN05_MMA_SP_TENSOR_DISABLE_OUTPUT_LANE_CG2_ASHIFT
@ TCGEN05_MMA_SP_SHARED_DISABLE_OUTPUT_LANE_CG1
@ TCGEN05_MMA_SP_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG1
@ TCGEN05_MMA_SHARED_SCALE_D_DISABLE_OUTPUT_LANE_CG1
@ TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG2
@ TCGEN05_MMA_SHARED_DISABLE_OUTPUT_LANE_CG2
@ TCGEN05_MMA_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG2
@ TCGEN05_MMA_SP_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG2
@ CLUSTERLAUNCHCONTROL_QUERY_CANCEL_GET_FIRST_CTAID_X
@ TCGEN05_MMA_SP_TENSOR_DISABLE_OUTPUT_LANE_CG1
@ TCGEN05_MMA_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG1_ASHIFT
@ TCGEN05_MMA_SP_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG1_ASHIFT
@ UNPACK_VECTOR
This node is the inverse of NVPTX::BUILD_VECTOR.
@ CLUSTERLAUNCHCONTROL_QUERY_CANCEL_GET_FIRST_CTAID_Y
@ TCGEN05_MMA_SHARED_DISABLE_OUTPUT_LANE_CG1
@ TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG2_ASHIFT
@ TCGEN05_MMA_TENSOR_SCALE_D_DISABLE_OUTPUT_LANE_CG2_ASHIFT
@ DeclareScalarParam
These nodes represent a parameter declaration.
@ CLUSTERLAUNCHCONTROL_QUERY_CANCEL_GET_FIRST_CTAID_Z
@ ATOMIC_CMP_SWAP_B128
These nodes are used to lower atomic instructions with i128 type.
@ BUILD_VECTOR
This node is similar to ISD::BUILD_VECTOR except that the output may be implicitly bitcast to a scala...
@ TCGEN05_MMA_SP_SHARED_DISABLE_OUTPUT_LANE_CG2
@ TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG1_ASHIFT
@ TCGEN05_MMA_SP_TENSOR_DISABLE_OUTPUT_LANE_CG1_ASHIFT
@ TCGEN05_MMA_TENSOR_DISABLE_OUTPUT_LANE_CG1
@ TCGEN05_MMA_SP_TENSOR_DISABLE_OUTPUT_LANE_CG2
bool isPackedVectorTy(EVT VT)
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
NodeAddr< NodeBase * > Node
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
FunctionAddr VTableAddr Value
bool shouldEmitPTXNoReturn(const Value *V, const TargetMachine &TM)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
MaybeAlign getAlign(const CallInst &I, unsigned Index)
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
bool isReleaseOrStronger(AtomicOrdering AO)
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
unsigned promoteScalarArgumentSize(unsigned size)
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
bool shouldPassAsArray(Type *Ty)
CodeGenOptLevel
Code generation optimization level.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Sub
Subtraction of integers.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
DWARFExpression::Operation Op
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
ArrayRef(const T &OneElt) -> ArrayRef< T >
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
bool isKernelFunction(const Function &F)
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Function * getMaybeBitcastedCallee(const CallBase *CB)
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static LLVM_ABI const fltSemantics & IEEEsingle() LLVM_READNONE
This struct is a compact representation of a valid (non-zero power of two) alignment.
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
@ PreserveSign
The sign of a flushed-to-zero number is preserved in the sign of 0.
DenormalModeKind Output
Denormal flushing mode for floating point instruction results in the default floating point environme...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
bool is32BitVector() const
Return true if this is a 32-bit vector type.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool isInteger() const
Return true if this is an integer or a vector integer type.
static LLVM_ABI KnownBits ashr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for ashr(LHS, RHS).
KnownBits concat(const KnownBits &Lo) const
Concatenate the bits from Lo onto the bottom of *this.
unsigned getBitWidth() const
Get the bit width of this value.
void resetAll()
Resets the known state of all bits.
void insertBits(const KnownBits &SubBits, unsigned BitPosition)
Insert the bits from a smaller known bits starting at bitPosition.
This class contains a discriminated union of information about pointers in memory operands,...
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasAllowContract() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
Type * RetTy
Same as OrigRetTy, or partially legalized for soft float libcalls.
bool isAfterLegalizeDAG() const
bool isBeforeLegalize() const
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)