26#include "llvm/IR/IntrinsicsAMDGPU.h"
33#define GET_CALLING_CONV_IMPL
34#include "AMDGPUGenCallingConv.inc"
42 if (StoreSize % 32 == 0)
60 bool IsSigned,
bool SelfMultiply) {
187 {MVT::v2i8, MVT::v4i8, MVT::v2i16, MVT::v3i16, MVT::v4i16})
399 {MVT::f16, MVT::f32},
Legal);
405 {MVT::f16, MVT::f32, MVT::f64},
Expand);
432 {MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32,
433 MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v16f32,
434 MVT::v2f64, MVT::v3f64, MVT::v4f64, MVT::v8f64,
442 {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32,
443 MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32,
444 MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32,
445 MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32,
446 MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32},
451 {MVT::v2f32, MVT::v2i32, MVT::v3f32, MVT::v3i32, MVT::v4f32,
452 MVT::v4i32, MVT::v5f32, MVT::v5i32, MVT::v6f32, MVT::v6i32,
453 MVT::v7f32, MVT::v7i32, MVT::v8f32, MVT::v8i32, MVT::v9f32,
454 MVT::v9i32, MVT::v10i32, MVT::v10f32, MVT::v11i32, MVT::v11f32,
455 MVT::v12i32, MVT::v12f32, MVT::v16i32, MVT::v32f32, MVT::v32i32,
456 MVT::v2f64, MVT::v2i64, MVT::v3f64, MVT::v3i64, MVT::v4f64,
457 MVT::v4i64, MVT::v8f64, MVT::v8i64, MVT::v16f64, MVT::v16i64},
464 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 };
465 for (
MVT VT : ScalarIntVTs) {
500 for (
auto VT : {MVT::i8, MVT::i16})
504 MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32, MVT::v6i32, MVT::v7i32,
505 MVT::v9i32, MVT::v10i32, MVT::v11i32, MVT::v12i32};
507 for (
MVT VT : VectorIntTypes) {
534 MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32, MVT::v6f32, MVT::v7f32,
535 MVT::v9f32, MVT::v10f32, MVT::v11f32, MVT::v12f32};
537 for (
MVT VT : FloatVectorTypes) {
658 case AMDGPUISD::RCP_LEGACY:
659 case AMDGPUISD::RCP_IFLAG:
660 case AMDGPUISD::SIN_HW:
661 case AMDGPUISD::FMUL_LEGACY:
662 case AMDGPUISD::FMIN_LEGACY:
663 case AMDGPUISD::FMAX_LEGACY:
664 case AMDGPUISD::FMED3:
675 unsigned Opc =
N->getOpcode();
696 return (
N->getNumOperands() > 2 &&
N->getOpcode() !=
ISD::SELECT) ||
705 return N->getValueType(0) == MVT::f32;
715 switch (
N->getOpcode()) {
721 case AMDGPUISD::DIV_SCALE:
730 switch (
N->getConstantOperandVal(0)) {
731 case Intrinsic::amdgcn_interp_p1:
732 case Intrinsic::amdgcn_interp_p2:
733 case Intrinsic::amdgcn_interp_mov:
734 case Intrinsic::amdgcn_interp_p1_f16:
735 case Intrinsic::amdgcn_interp_p2_f16:
749 unsigned CostThreshold) {
755 unsigned NumMayIncreaseSize = 0;
756 MVT VT =
N->getValueType(0).getScalarType().getSimpleVT();
761 for (
const SDNode *U :
N->users()) {
766 if (++NumMayIncreaseSize > CostThreshold)
797 bool ForCodeSize)
const {
805 return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64);
810 std::optional<unsigned> ByteOffset)
const {
822 EVT OldVT =
N->getValueType(0);
837 if (OldSize >= 32 && NewSize < 32 && MN->
getAlign() >=
Align(4) &&
852 return (OldSize < 32);
867 if ((LScalarSize >= CastScalarSize) && (CastScalarSize < 32))
872 CastTy, MMO, &
Fast) &&
888 switch (
N->getOpcode()) {
893 unsigned IntrID =
N->getConstantOperandVal(0);
897 unsigned IntrID =
N->getConstantOperandVal(1);
905 case AMDGPUISD::SETCC:
915 switch (
Op.getOpcode()) {
923 case AMDGPUISD::RCP: {
925 EVT VT =
Op.getValueType();
931 return DAG.
getNode(AMDGPUISD::RCP, SL, VT, NegSrc,
Op->getFlags());
951 return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16 || VT == MVT::bf16;
958 return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16 || VT == MVT::bf16;
982 unsigned SrcSize = Source.getSizeInBits();
985 return DestSize < SrcSize && DestSize % 32 == 0 ;
991 unsigned SrcSize = Source->getScalarSizeInBits();
994 if (DestSize== 16 && Subtarget->has16BitInsts())
995 return SrcSize >= 32;
997 return DestSize < SrcSize && DestSize % 32 == 0;
1001 unsigned SrcSize = Src->getScalarSizeInBits();
1004 if (SrcSize == 16 && Subtarget->has16BitInsts())
1005 return DestSize >= 32;
1007 return SrcSize == 32 && DestSize == 64;
1016 if (Src == MVT::i16)
1017 return Dest == MVT::i32 ||Dest == MVT::i64 ;
1019 return Src == MVT::i32 && Dest == MVT::i64;
1024 switch (
N->getOpcode()) {
1047 if (!
N->isDivergent() && DestVT.
isInteger() &&
1075 "Expected shift op");
1077 SDValue ShiftLHS =
N->getOperand(0);
1092 if (
N->getValueType(0) == MVT::i32 &&
N->hasOneUse() &&
1093 (
N->user_begin()->getOpcode() ==
ISD::SRA ||
1094 N->user_begin()->getOpcode() ==
ISD::SRL))
1104 return LHS0 && LHS1 && RHSLd && LHS0->getExtensionType() ==
ISD::ZEXTLOAD &&
1105 LHS1->getAPIntValue() == LHS0->getMemoryVT().getScalarSizeInBits() &&
1108 SDValue LHS =
N->getOperand(0).getOperand(0);
1109 SDValue RHS =
N->getOperand(0).getOperand(1);
1110 return !(IsShiftAndLoad(LHS, RHS) || IsShiftAndLoad(RHS, LHS));
1130 return CC_AMDGPU_CS_CHAIN;
1134 return CC_AMDGPU_Func;
1160 return RetCC_SI_Shader;
1163 return RetCC_SI_Gfx;
1167 return RetCC_AMDGPU_Func;
1205 const unsigned ExplicitOffset = Subtarget->getExplicitKernelArgOffset();
1209 uint64_t ExplicitArgOffset = 0;
1212 unsigned InIndex = 0;
1215 const bool IsByRef = Arg.hasByRefAttr();
1216 Type *BaseArgTy = Arg.getType();
1217 Type *MemArgTy = IsByRef ? Arg.getParamByRefType() : BaseArgTy;
1218 Align Alignment =
DL.getValueOrABITypeAlignment(
1219 IsByRef ? Arg.getParamAlign() : std::nullopt, MemArgTy);
1220 MaxAlign = std::max(Alignment, MaxAlign);
1221 uint64_t AllocSize =
DL.getTypeAllocSize(MemArgTy);
1223 uint64_t ArgOffset =
alignTo(ExplicitArgOffset, Alignment) + ExplicitOffset;
1224 ExplicitArgOffset =
alignTo(ExplicitArgOffset, Alignment) + AllocSize;
1236 &Offsets, ArgOffset);
1238 for (
unsigned Value = 0, NumValues = ValueVTs.
size();
1240 uint64_t BasePartOffset = Offsets[
Value];
1276 }
else if (RegisterVT.
isVector()) {
1279 assert(MemoryBits % NumElements == 0);
1283 MemoryBits / NumElements);
1301 unsigned PartOffset = 0;
1302 for (
unsigned i = 0; i != NumRegs; ++i) {
1304 BasePartOffset + PartOffset,
1322 return DAG.
getNode(AMDGPUISD::ENDPGM,
DL, MVT::Other, Chain);
1343 int ClobberedFI)
const {
1346 int64_t LastByte = FirstByte + MFI.
getObjectSize(ClobberedFI) - 1;
1357 if (FI->getIndex() < 0) {
1359 int64_t InLastByte = InFirstByte;
1362 if ((InFirstByte <= FirstByte && FirstByte <= InLastByte) ||
1363 (FirstByte <= InFirstByte && InFirstByte <= LastByte))
1385 FuncName =
G->getSymbol();
1387 FuncName =
G->getGlobal()->getName();
1422 switch (
Op.getOpcode()) {
1426 "instruction is not implemented yet!");
1478 switch (
N->getOpcode()) {
1519 EVT VT =
Op.getValueType();
1538 Fn,
"unsupported use of BARRIER address space",
1548 if (std::optional<uint32_t>
Address =
1558 GV->
getName() !=
"llvm.amdgcn.module.lds") {
1562 Fn,
"local memory global used by non-kernel function",
1594 EVT VT =
Op.getValueType();
1596 unsigned OpBitSize =
Op.getOperand(0).getValueType().getSizeInBits();
1597 if (OpBitSize >= 32 && OpBitSize % 32 == 0) {
1598 unsigned NewNumElt = OpBitSize / 32;
1599 EVT NewEltVT = (NewNumElt == 1) ? MVT::i32
1601 MVT::i32, NewNumElt);
1602 for (
const SDUse &U :
Op->ops()) {
1608 Args.push_back(NewIn);
1612 NewNumElt *
Op.getNumOperands());
1618 for (
const SDUse &U :
Op->ops())
1628 unsigned Start =
Op.getConstantOperandVal(1);
1629 EVT VT =
Op.getValueType();
1630 EVT SrcVT =
Op.getOperand(0).getValueType();
1635 assert(NumElt % 2 == 0 && NumSrcElt % 2 == 0 &&
"expect legal types");
1639 EVT NewVT = NumElt == 2
1711 Opc = AMDGPUISD::FMIN_LEGACY;
1717 Opc = AMDGPUISD::FMIN_LEGACY;
1723 Opc = AMDGPUISD::FMAX_LEGACY;
1729 Opc = AMDGPUISD::FMAX_LEGACY;
1772 if (LHS == NegTrue && CFalse && CRHS) {
1786std::pair<SDValue, SDValue>
1798 return std::pair(
Lo,
Hi);
1819 HiVT = NumElts - LoNumElts == 1
1822 return std::pair(LoVT, HiVT);
1827std::pair<SDValue, SDValue>
1829 const EVT &LoVT,
const EVT &HiVT,
1831 EVT VT =
N.getValueType();
1835 "More vector elements requested than available!");
1866 EVT VT =
Op.getValueType();
1879 EVT MemVT =
Load->getMemoryVT();
1884 EVT LoMemVT, HiMemVT;
1896 Load->getExtensionType(), SL, LoVT,
Load->getChain(), BasePtr, SrcValue,
1897 LoMemVT, BaseAlign,
Load->getMemOperand()->getFlags(),
Load->getAAInfo());
1900 Load->getExtensionType(), SL, HiVT,
Load->getChain(), HiPtr,
1902 Load->getMemOperand()->getFlags(),
Load->getAAInfo());
1926 EVT VT =
Op.getValueType();
1928 EVT MemVT =
Load->getMemoryVT();
1936 if (NumElements != 3 ||
1937 (BaseAlign <
Align(8) &&
1941 assert(NumElements == 3);
1948 Load->getExtensionType(), SL, WideVT,
Load->getChain(), BasePtr, SrcValue,
1949 WideMemVT, BaseAlign,
Load->getMemOperand()->getFlags());
1974 EVT LoMemVT, HiMemVT;
1989 DAG.
getTruncStore(Chain, SL,
Lo, BasePtr, SrcValue, LoMemVT, BaseAlign,
1990 Store->getMemOperand()->getFlags(),
Store->getAAInfo());
1993 Store->getMemOperand()->getFlags(),
Store->getAAInfo());
2003 EVT VT =
Op.getValueType();
2004 assert(VT == MVT::i32 &&
"LowerDIVREMToFloat expects an i32");
2008 MVT IntVT = MVT::i32;
2009 MVT FltVT = MVT::f32;
2011 unsigned LHSSignBits;
2012 unsigned RHSSignBits;
2016 if (LHSSignBits < 9 || RHSSignBits < 9)
2027 unsigned SignBits = std::min(LHSSignBits, RHSSignBits);
2028 unsigned DivBits = BitSize - SignBits;
2037 if (DivBits > (
Sign ? 23 : 22))
2070 fa, DAG.
getNode(AMDGPUISD::RCP,
DL, FltVT, fb));
2089 EVT VT =
Op.getValueType();
2091 assert(VT == MVT::i64 &&
"LowerUDIVREM64 expects an i64");
2101 std::tie(LHS_Lo, LHS_Hi) = DAG.
SplitScalar(LHS,
DL, HalfVT, HalfVT);
2105 std::tie(RHS_Lo, RHS_Hi) = DAG.
SplitScalar(RHS,
DL, HalfVT, HalfVT);
2164 std::tie(Mulhi1_Lo, Mulhi1_Hi) =
2177 std::tie(Mulhi2_Lo, Mulhi2_Hi) =
2191 std::tie(Mul3_Lo, Mul3_Hi) = DAG.
SplitScalar(Mul3,
DL, HalfVT, HalfVT);
2270 for (
unsigned i = 0; i < halfBitWidth; ++i) {
2271 const unsigned bitPos = halfBitWidth - i - 1;
2302 EVT VT =
Op.getValueType();
2304 if (VT == MVT::i64) {
2310 if (VT == MVT::i32) {
2357 EVT VT =
Op.getValueType();
2365 if (VT == MVT::i32) {
2444 const unsigned FractBits = 52;
2445 const unsigned ExpBits = 11;
2461 assert(
Op.getValueType() == MVT::f64);
2471 const unsigned FractBits = 52;
2483 = DAG.
getConstant((UINT64_C(1) << FractBits) - 1, SL, MVT::i64);
2508 assert(
Op.getValueType() == MVT::f64);
2541 auto VT =
Op.getValueType();
2542 auto Arg =
Op.getOperand(0u);
2554 EVT VT =
Op.getValueType();
2605 switch (Src.getOpcode()) {
2607 return Src.getOperand(0).getValueType() == MVT::f16;
2611 case AMDGPUISD::LOG:
2612 case AMDGPUISD::EXP:
2615 unsigned IntrinsicID = Src.getConstantOperandVal(0);
2616 switch (IntrinsicID) {
2617 case Intrinsic::amdgcn_frexp_mant:
2618 case Intrinsic::amdgcn_log:
2619 case Intrinsic::amdgcn_log_clamp:
2620 case Intrinsic::amdgcn_exp2:
2621 case Intrinsic::amdgcn_sqrt:
2636 return Flags.hasApproximateFuncs();
2651 EVT VT = Src.getValueType();
2664std::pair<SDValue, SDValue>
2685 return {ScaledInput, IsLtSmallestNormal};
2696 EVT VT =
Op.getValueType();
2700 if (VT == MVT::f16) {
2704 SDValue Log = DAG.
getNode(AMDGPUISD::LOG, SL, MVT::f32, Ext, Flags);
2709 auto [ScaledInput, IsLtSmallestNormal] =
2712 return DAG.
getNode(AMDGPUISD::LOG, SL, VT, Src, Flags);
2732 EVT VT =
Op.getValueType();
2738 if (VT == MVT::f16 || Flags.hasApproximateFuncs()) {
2742 bool PromoteToF32 = VT == MVT::f16 && (!Flags.hasApproximateFuncs() ||
2759 SDValue ScaledInput, IsScaled;
2771 if (Subtarget->hasFastFMAF32()) {
2773 const float c_log10 = 0x1.344134p-2f;
2774 const float cc_log10 = 0x1.09f79ep-26f;
2777 const float c_log = 0x1.62e42ep-1f;
2778 const float cc_log = 0x1.efa39ep-25f;
2784 Flags.setAllowContract(
false);
2792 const float ch_log10 = 0x1.344000p-2f;
2793 const float ct_log10 = 0x1.3509f6p-18f;
2796 const float ch_log = 0x1.62e000p-1f;
2797 const float ct_log = 0x1.0bfbe8p-15f;
2809 Flags.setAllowContract(
false);
2816 const bool IsFiniteOnly = Flags.hasNoNaNs() && Flags.hasNoInfs();
2819 if (!IsFiniteOnly) {
2841 EVT VT = Src.getValueType();
2845 double Log2BaseInverted =
2848 if (VT == MVT::f32) {
2851 SDValue LogSrc = DAG.
getNode(AMDGPUISD::LOG, SL, VT, ScaledInput, Flags);
2858 ScaledResultOffset,
Zero, Flags);
2862 if (Subtarget->hasFastFMAF32())
2873 return DAG.
getNode(
ISD::FMUL, SL, VT, Log2Operand, Log2BaseInvertedOperand,
2970 if (!Flags.hasNoInfs()) {
2973 Z = DAG.
getSelect(
DL, MVT::f64, CondHi, Z, PInf, Flags);
2988 EVT VT =
Op.getValueType();
2996 if (VT == MVT::f16) {
3000 SDValue Log = DAG.
getNode(AMDGPUISD::EXP, SL, MVT::f32, Ext, Flags);
3008 return DAG.
getNode(AMDGPUISD::EXP, SL, MVT::f32, Src, Flags);
3028 SDValue Exp2 = DAG.
getNode(AMDGPUISD::EXP, SL, VT, AddInput, Flags);
3041 bool IsExp10)
const {
3044 EVT VT =
X.getValueType();
3049 return DAG.
getNode(VT == MVT::f32 ? (
unsigned)AMDGPUISD::EXP
3051 SL, VT,
Mul, Flags);
3057 EVT VT =
X.getValueType();
3076 SDValue Exp2 = DAG.
getNode(AMDGPUISD::EXP, SL, VT, ExpInput, Flags);
3091 const EVT VT =
X.getValueType();
3093 const unsigned Exp2Op = VT == MVT::f32 ?
static_cast<unsigned>(AMDGPUISD::EXP)
3143 EVT VT =
Op.getValueType();
3207 if (Subtarget->hasFastFMAF32()) {
3209 const float cc_exp = 0x1.4ae0bep-26f;
3210 const float c_exp10 = 0x1.a934f0p+1f;
3211 const float cc_exp10 = 0x1.2f346ep-24f;
3221 const float ch_exp = 0x1.714000p+0f;
3222 const float cl_exp = 0x1.47652ap-12f;
3224 const float ch_exp10 = 0x1.a92000p+1f;
3225 const float cl_exp10 = 0x1.4f0978p-11f;
3240 PL =
getMad(DAG, SL, VT, XH, CL, Mad0, Flags);
3255 DAG.
getConstantFP(IsExp10 ? -0x1.66d3e8p+5f : -0x1.9d1da0p+6f, SL, VT);
3264 if (!Flags.hasNoInfs()) {
3266 DAG.
getConstantFP(IsExp10 ? 0x1.344136p+5f : 0x1.62e430p+6f, SL, VT);
3281 EVT VT =
Op.getValueType();
3298 DAG.
getNode(AMDGPUISD::FMUL_LEGACY, SL, VT,
Y, Log, CoreFlags);
3299 return DAG.
getNode(AMDGPUISD::EXP, SL, VT,
Mul, CoreFlags);
3327 if (Flags.hasNoNaNs())
3342 DAG.
getNOT(SL, YIsInt, SetCCVT));
3359 auto Opc =
Op.getOpcode();
3360 auto Arg =
Op.getOperand(0u);
3361 auto ResultVT =
Op.getValueType();
3363 if (ResultVT != MVT::i8 && ResultVT != MVT::i16)
3367 assert(ResultVT == Arg.getValueType());
3369 const uint64_t NumBits = ResultVT.getFixedSizeInBits();
3376 NewOp = DAG.
getNode(
Opc, SL, MVT::i32, NewOp);
3379 NewOp = DAG.
getNode(
Opc, SL, MVT::i32, NewOp);
3392 unsigned NewOpc = Ctlz ? AMDGPUISD::FFBH_U32 : AMDGPUISD::FFBL_B32;
3396 bool Is64BitScalar = !Src->isDivergent() && Src.getValueType() == MVT::i64;
3398 if (Src.getValueType() == MVT::i32 || Is64BitScalar) {
3412 Op.getValueType().getScalarSizeInBits(), SL, MVT::i32);
3432 OprLo = DAG.
getNode(AddOpc, SL, MVT::i32, OprLo, Const32);
3434 OprHi = DAG.
getNode(AddOpc, SL, MVT::i32, OprHi, Const32);
3449 assert(Src.getValueType() == MVT::i32 &&
"LowerCTLS only supports i32");
3461 assert(FP16Ty == MVT::f16 || FP16Ty == MVT::bf16);
3504 if (
Signed && Subtarget->isGCN()) {
3619 EVT DestVT =
Op.getValueType();
3621 EVT SrcVT = Src.getValueType();
3625 if (SrcVT == MVT::i16) {
3626 if (DestVT == MVT::f16)
3631 return DAG.
getNode(CvtOpc,
DL, DestVT, Ext);
3634 if (DestVT == MVT::bf16 || DestVT == MVT::f16)
3637 if (SrcVT != MVT::i64)
3640 if (DestVT == MVT::f32)
3643 assert(DestVT == MVT::f64);
3662 EVT SrcVT = Src.getValueType();
3664 assert(SrcVT == MVT::f32 || SrcVT == MVT::f64);
3677 if (
Signed && SrcVT == MVT::f32) {
3690 if (SrcVT == MVT::f64) {
3712 SL, MVT::i32, FloorMul);
3718 if (
Signed && SrcVT == MVT::f32) {
3738 return DAG.
getNode(AMDGPUISD::FP_TO_FP16,
DL,
Op.getValueType(), N0);
3740 if (
Op->getFlags().hasApproximateFuncs()) {
3751 assert(Src.getSimpleValueType() == MVT::f64);
3755 const unsigned ExpMask = 0x7ff;
3756 const unsigned ExpBiasf64 = 1023;
3757 const unsigned ExpBiasf16 = 15;
3840 unsigned OpOpcode =
Op.getOpcode();
3841 EVT SrcVT = Src.getValueType();
3842 EVT DestVT =
Op.getValueType();
3845 if (SrcVT == MVT::f16 && DestVT == MVT::i16)
3848 if (SrcVT == MVT::bf16 || (SrcVT == MVT::f16 && DestVT == MVT::i32)) {
3851 return DAG.
getNode(
Op.getOpcode(),
DL, DestVT, PromotedSrc);
3855 if (DestVT == MVT::i16 && (SrcVT == MVT::f32 || SrcVT == MVT::f64)) {
3862 if (DestVT != MVT::i64)
3865 if (SrcVT == MVT::f16 ||
3872 return DAG.
getNode(Ext,
DL, MVT::i64, FpToInt32);
3875 if (SrcVT == MVT::f32 || SrcVT == MVT::f64)
3884 unsigned OpOpcode =
Op.getOpcode();
3885 EVT SrcVT = Src.getValueType();
3886 EVT DstVT =
Op.getValueType();
3887 SDValue SatVTOp =
Op.getNode()->getOperand(1);
3893 assert(SatWidth <= DstWidth &&
"Saturation width cannot exceed result width");
3897 if (SatWidth == DstWidth) {
3898 if ((DstVT == MVT::i32 && (SrcVT == MVT::f32 || SrcVT == MVT::f64)) ||
3899 (DstVT == MVT::i16 && (SrcVT == MVT::f16 || SrcVT == MVT::f32)) ||
3900 (DstVT == MVT::v2i16 && SrcVT == MVT::v2f32))
3909 if (SatWidth < DstWidth && SatWidth <= 32) {
3914 Subtarget->has16BitInsts() && SrcVT == MVT::f16 && SatWidth < 16
3948 if (DstVT == MVT::i64 &&
3949 (SrcVT == MVT::f16 || SrcVT == MVT::bf16 ||
3952 return DAG.
getNode(OpOpcode,
DL, DstVT, Src, Int32VTOp);
3956 if (DstVT == MVT::i32 && (SrcVT == MVT::f16 || SrcVT == MVT::bf16)) {
3958 return DAG.
getNode(
Op.getOpcode(),
DL, DstVT, PromotedSrc, SatVTOp);
3964 if (DstWidth < 32) {
3968 (DstWidth < 16 && Subtarget->has16BitInsts()) ? MVT::i16 : MVT::i32;
3981 MVT VT =
Op.getSimpleValueType();
3995 for (
unsigned I = 0;
I < NElts; ++
I)
4010 EVT VT =
Op.getValueType();
4024 unsigned NewOpcode = Node24->
getOpcode();
4028 case Intrinsic::amdgcn_mul_i24:
4029 NewOpcode = AMDGPUISD::MUL_I24;
4031 case Intrinsic::amdgcn_mul_u24:
4032 NewOpcode = AMDGPUISD::MUL_U24;
4034 case Intrinsic::amdgcn_mulhi_i24:
4035 NewOpcode = AMDGPUISD::MULHI_I24;
4037 case Intrinsic::amdgcn_mulhi_u24:
4038 NewOpcode = AMDGPUISD::MULHI_U24;
4052 if (
LHS.hasOneUse() &&
TLI.SimplifyDemandedBits(
LHS, Demanded, DCI))
4054 if (
RHS.hasOneUse() &&
TLI.SimplifyDemandedBits(
RHS, Demanded, DCI))
4060 SDValue DemandedLHS =
TLI.SimplifyMultipleUseDemandedBits(
LHS, Demanded, DAG);
4061 SDValue DemandedRHS =
TLI.SimplifyMultipleUseDemandedBits(
RHS, Demanded, DAG);
4062 if (DemandedLHS || DemandedRHS)
4064 DemandedLHS ? DemandedLHS :
LHS,
4065 DemandedRHS ? DemandedRHS :
RHS);
4070template <
typename IntTy>
4073 if (Width +
Offset < 32) {
4075 IntTy Result =
static_cast<IntTy
>(Shl) >> (32 - Width);
4076 if constexpr (std::is_signed_v<IntTy>) {
4089 if (M->isVolatile())
4241 EVT SrcVT = Src.getValueType();
4242 if (SrcVT.
bitsGE(ExtVT)) {
4253 unsigned IID =
N->getConstantOperandVal(0);
4255 case Intrinsic::amdgcn_mul_i24:
4256 case Intrinsic::amdgcn_mul_u24:
4257 case Intrinsic::amdgcn_mulhi_i24:
4258 case Intrinsic::amdgcn_mulhi_u24:
4260 case Intrinsic::amdgcn_fract:
4261 case Intrinsic::amdgcn_rsq:
4262 case Intrinsic::amdgcn_rcp_legacy:
4263 case Intrinsic::amdgcn_rsq_legacy:
4264 case Intrinsic::amdgcn_rsq_clamp:
4265 case Intrinsic::amdgcn_tanh:
4266 case Intrinsic::amdgcn_prng_b32: {
4269 return Src.isUndef() ? Src :
SDValue();
4271 case Intrinsic::amdgcn_frexp_exp: {
4277 if (PeekSign == Src)
4314 EVT VT =
N->getValueType(0);
4327 switch (LHS->getOpcode()) {
4335 if (VT == MVT::i32 && RHSVal == 16 &&
X.getValueType() == MVT::i16 &&
4349 unsigned LZ =
Known.countMinLeadingZeros();
4352 EVT XVT =
X.getValueType();
4384 ShiftAmt = DAG.
getNode(
ISD::AND, SL, TargetType, TruncShiftAmt, ShiftMask);
4395 EVT ConcatType = TargetType.getDoubleNumVectorElementsVT(*DAG.
getContext());
4396 unsigned NElts = TargetType.getVectorNumElements();
4401 for (
unsigned I = 0;
I != NElts; ++
I)
4402 HiAndLoOps[2 *
I + 1] = HiOps[
I];
4415 EVT VT =
N->getValueType(0);
4445 }
else if (
Known.getMinValue().getZExtValue() ==
4446 (ElementType.getSizeInBits() - 1)) {
4447 ShiftAmt = ShiftFullAmt;
4454 ShiftAmt = DAG.
getNode(
ISD::AND, SL, TargetType, TruncShiftAmt, ShiftMask);
4462 unsigned NElts = TargetType.getVectorNumElements();
4463 ConcatType = TargetType.getDoubleNumVectorElementsVT(*DAG.
getContext());
4469 for (
unsigned I = 0;
I != NElts; ++
I) {
4470 HiOps[
I] = HiAndLoOps[2 *
I + 1];
4487 CRHS->
getZExtValue() == (ElementType.getSizeInBits() - 1)) {
4488 NewShift = HiShift =
4499 unsigned NElts = TargetType.getVectorNumElements();
4506 for (
unsigned I = 0;
I != NElts; ++
I) {
4507 HiAndLoOps[2 *
I + 1] = HiOps[
I];
4508 HiAndLoOps[2 *
I] = LoOps[
I];
4521 EVT VT =
N->getValueType(0);
4534 unsigned MaskIdx, MaskLen;
4535 if (Mask->getAPIntValue().isShiftedMask(MaskIdx, MaskLen) &&
4536 MaskIdx == RHSVal) {
4575 ShiftAmt = DAG.
getNode(
ISD::AND, SL, TargetType, TruncShiftAmt, ShiftMask);
4584 unsigned NElts = TargetType.getVectorNumElements();
4585 ConcatType = TargetType.getDoubleNumVectorElementsVT(*DAG.
getContext());
4591 for (
unsigned I = 0;
I != NElts; ++
I)
4592 HiOps[
I] = HiAndLoOps[2 *
I + 1];
4606 unsigned NElts = TargetType.getVectorNumElements();
4611 for (
unsigned I = 0;
I != NElts; ++
I)
4612 HiAndLoOps[2 *
I] = LoOps[
I];
4624 EVT VT =
N->getValueType(0);
4653 unsigned BitIndex =
K->getZExtValue();
4654 unsigned PartIndex = BitIndex / SrcEltSize;
4656 if (PartIndex * SrcEltSize == BitIndex &&
4674 EVT SrcVT = Src.getValueType();
4686 const unsigned MaxCstSize =
4688 if (
Known.getMaxValue().ule(MaxCstSize)) {
4720 unsigned MulOpc =
Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
4721 return DAG.
getNode(MulOpc, SL, MVT::i32, N0, N1);
4724 unsigned MulLoOpc =
Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
4725 unsigned MulHiOpc =
Signed ? AMDGPUISD::MULHI_I24 : AMDGPUISD::MULHI_U24;
4745 EVT VT =
N->getValueType(0);
4751 if (!
N->isDivergent())
4773 if (V.hasOneUse() ||
all_of(V->users(), [](
const SDNode *U) ->
bool {
4774 return U->getOpcode() == ISD::MUL;
4783 if (
SDValue MulOper = IsFoldableAdd(N0)) {
4788 if (
SDValue MulOper = IsFoldableAdd(N1)) {
4809 if (Subtarget->hasMulU24() &&
isU24(N0, DAG) &&
isU24(N1, DAG)) {
4813 }
else if (Subtarget->hasMulI24() &&
isI24(N0, DAG) &&
isI24(N1, DAG)) {
4829 if (
N->getValueType(0) != MVT::i32)
4850 unsigned LoOpcode = 0;
4851 unsigned HiOpcode = 0;
4853 if (Subtarget->hasMulI24() &&
isI24(N0, DAG) &&
isI24(N1, DAG)) {
4856 LoOpcode = AMDGPUISD::MUL_I24;
4857 HiOpcode = AMDGPUISD::MULHI_I24;
4860 if (Subtarget->hasMulU24() &&
isU24(N0, DAG) &&
isU24(N1, DAG)) {
4863 LoOpcode = AMDGPUISD::MUL_U24;
4864 HiOpcode = AMDGPUISD::MULHI_U24;
4878 EVT VT =
N->getValueType(0);
4880 if (!Subtarget->hasMulI24() || VT.
isVector())
4889 if (Subtarget->hasSMulHi() && !
N->isDivergent())
4911 EVT VT =
N->getValueType(0);
4922 if (!
N->isDivergent() && Subtarget->hasSMulHi())
4945 unsigned Opc)
const {
4946 EVT VT =
Op.getValueType();
4983 isCttzOpc(RHS.getOpcode()) ? AMDGPUISD::FFBL_B32 : AMDGPUISD::FFBH_U32;
4984 return getFFBX_U32(DAG, CmpLHS, SL,
Opc);
4993 isCttzOpc(LHS.getOpcode()) ? AMDGPUISD::FFBL_B32 : AMDGPUISD::FFBH_U32;
4995 return getFFBX_U32(DAG, CmpLHS, SL,
Opc);
5013 return DAG.
getNode(
Op, SL, VT, NewSelect);
5031 EVT VT =
N.getValueType();
5058 bool ShouldFoldNeg =
true;
5063 ShouldFoldNeg =
false;
5065 ShouldFoldNeg =
false;
5068 if (ShouldFoldNeg) {
5092 Cond, NewLHS, NewRHS);
5094 return DAG.
getNode(LHS.getOpcode(), SL, VT, NewSelect);
5110 EVT VT =
N->getValueType(0);
5118 if (
Cond.hasOneUse()) {
5134 if (VT == MVT::f32 && Subtarget->hasFminFmaxLegacy()) {
5136 CC,
N->getFlags(), DCI);
5164 if (Subtarget->hasInv2PiInlineImm() &&
isInv2Pi(
C->getValueAPF()))
5194 case AMDGPUISD::FMAX_LEGACY:
5195 return AMDGPUISD::FMIN_LEGACY;
5196 case AMDGPUISD::FMIN_LEGACY:
5197 return AMDGPUISD::FMAX_LEGACY;
5228 EVT VT =
N->getValueType(0);
5235 bool MayIgnoreSignedZeroForAllUses =
5237 (N0.
hasOneUse() &&
N->getFlags().hasNoSignedZeros());
5242 if (!MayIgnoreSignedZeroForAllUses)
5267 case AMDGPUISD::FMUL_LEGACY: {
5290 if (!MayIgnoreSignedZeroForAllUses)
5325 case AMDGPUISD::FMAX_LEGACY:
5326 case AMDGPUISD::FMIN_LEGACY: {
5341 if ((
Opc == AMDGPUISD::FMIN_LEGACY ||
Opc == AMDGPUISD::FMAX_LEGACY) &&
5356 case AMDGPUISD::FMED3: {
5364 for (
unsigned I = 0;
I < 3; ++
I)
5368 if (Res.
getOpcode() != AMDGPUISD::FMED3)
5388 case AMDGPUISD::RCP:
5389 case AMDGPUISD::RCP_LEGACY:
5390 case AMDGPUISD::RCP_IFLAG:
5391 case AMDGPUISD::SIN_HW: {
5430 EVT SrcVT = Src.getValueType();
5466 Ops.back() = CastBack;
5516 EVT SrcVT = Src.getValueType();
5552 if (!Subtarget->isGCN())
5558 const auto *
TII = ST.getInstrInfo();
5560 if (!ST.hasVMovB64Inst() || (!SDConstant && !SDFPConstant))
5563 if (ST.has64BitLiterals())
5580 switch(
N->getOpcode()) {
5584 EVT DestVT =
N->getValueType(0);
5596 EVT SrcVT = Src.getValueType();
5626 uint64_t CVal =
C->getZExtValue();
5634 const APInt &Val =
C->getValueAPF().bitcastToAPInt();
5655 if (!(
N->getValueType(0).isVector() &&
5669 case AMDGPUISD::MUL_U24:
5670 case AMDGPUISD::MUL_I24: {
5675 case AMDGPUISD::MULHI_I24:
5676 case AMDGPUISD::MULHI_U24:
5691 case AMDGPUISD::BFE_I32:
5692 case AMDGPUISD::BFE_U32: {
5693 assert(
N->getValueType(0) == MVT::i32 &&
5694 "BFE_I32/BFE_U32 is a 32-bit operation");
5707 SDValue BitsFrom =
N->getOperand(0);
5710 bool Signed =
N->getOpcode() == AMDGPUISD::BFE_I32;
5712 if (OffsetVal == 0) {
5739 CVal->getSExtValue(),
5746 CVal->getZExtValue(),
5752 if ((OffsetVal + WidthVal) >= 32 &&
5753 !(OffsetVal == 16 && WidthVal == 16 && Subtarget->hasSDWA())) {
5756 BitsFrom, ShiftVal);
5762 OffsetVal + WidthVal);
5768 if (
TLI.ShrinkDemandedConstant(BitsFrom, Demanded, TLO) ||
5769 TLI.SimplifyDemandedBits(BitsFrom, Demanded,
Known, TLO)) {
5780 case AMDGPUISD::RCP:
5781 case AMDGPUISD::RCP_IFLAG:
5788 case AMDGPUISD::FMAD_FTZ: {
5792 EVT VT =
N->getValueType(0);
5799 if (N0CFP && N1CFP && N2CFP) {
5800 const auto FTZ = [](
const APFloat &V) {
5801 if (V.isDenormal()) {
5803 return V.isNegative() ? -
Zero :
Zero;
5825 unsigned Depth)
const {
5826 switch (
Op.getOpcode()) {
5828 switch (
Op.getConstantOperandVal(0)) {
5829 case Intrinsic::amdgcn_readfirstlane:
5830 case Intrinsic::amdgcn_readlane:
5831 case Intrinsic::amdgcn_wwm: {
5833 OriginalDemandedElts,
Known, TLO,
Depth + 1))
5837 case Intrinsic::amdgcn_set_inactive:
5838 case Intrinsic::amdgcn_set_inactive_chain_arg: {
5843 OriginalDemandedElts, KnownValue, TLO,
5847 OriginalDemandedElts, KnownInactive, TLO,
5873 bool RawReg)
const {
5878 if (!
MRI.isLiveIn(Reg)) {
5879 VReg =
MRI.createVirtualRegister(RC);
5880 MRI.addLiveIn(Reg, VReg);
5882 VReg =
MRI.getLiveInVirtReg(Reg);
5933 DAG.
getCopyFromReg(Chain, SL, Info->getStackPtrOffsetReg(), MVT::i32);
5944 assert(Arg &&
"Attempting to load missing argument");
5953 unsigned Mask = Arg.
getMask();
5963 unsigned ExplicitArgOffset = Subtarget->getExplicitKernelArgOffset();
5964 const Align Alignment = Subtarget->getAlignmentForImplicitArgPtr();
5965 uint64_t ArgOffset =
5966 alignTo(ExplicitKernArgSize, Alignment) + ExplicitArgOffset;
5989 int &RefinementSteps,
5990 bool &UseOneConstNR,
5991 bool Reciprocal)
const {
5994 if (VT == MVT::f32) {
5995 RefinementSteps = 0;
5996 return DAG.
getNode(AMDGPUISD::RSQ,
SDLoc(Operand), VT, Operand);
6007 int &RefinementSteps)
const {
6010 if (VT == MVT::f32) {
6016 RefinementSteps = 0;
6017 return DAG.
getNode(AMDGPUISD::RCP,
SDLoc(Operand), VT, Operand);
6028 case Intrinsic::amdgcn_workitem_id_x:
6030 case Intrinsic::amdgcn_workitem_id_y:
6032 case Intrinsic::amdgcn_workitem_id_z:
6045 unsigned Opc =
Op.getOpcode();
6050 case AMDGPUISD::CARRY:
6051 case AMDGPUISD::BORROW: {
6056 case AMDGPUISD::BFE_I32:
6057 case AMDGPUISD::BFE_U32: {
6064 if (
Opc == AMDGPUISD::BFE_U32)
6069 case AMDGPUISD::FP_TO_FP16: {
6076 case AMDGPUISD::MUL_U24:
6077 case AMDGPUISD::MUL_I24: {
6081 bool SelfMultiply =
Op.getOperand(0) ==
Op.getOperand(1);
6086 SelfMultiply ? RHSKnown
6095 Op.getScalarValueSizeInBits(),
6096 Opc == AMDGPUISD::MUL_I24, SelfMultiply);
6099 case AMDGPUISD::PERM: {
6108 for (
unsigned I = 0;
I < 32;
I += 8) {
6109 unsigned SelBits = Sel & 0xff;
6114 }
else if (SelBits < 7) {
6115 SelBits = (SelBits & 3) * 8;
6118 }
else if (SelBits == 0x0c) {
6119 Known.Zero |= 0xFFull <<
I;
6120 }
else if (SelBits > 0x0c) {
6121 Known.One |= 0xFFull <<
I;
6127 case AMDGPUISD::BUFFER_LOAD_UBYTE: {
6128 Known.Zero.setHighBits(24);
6131 case AMDGPUISD::BUFFER_LOAD_USHORT: {
6132 Known.Zero.setHighBits(16);
6135 case AMDGPUISD::LDS: {
6139 Known.Zero.setHighBits(16);
6143 case AMDGPUISD::SMIN3:
6144 case AMDGPUISD::SMAX3:
6145 case AMDGPUISD::SMED3:
6146 case AMDGPUISD::UMIN3:
6147 case AMDGPUISD::UMAX3:
6148 case AMDGPUISD::UMED3: {
6167 unsigned IID =
Op.getConstantOperandVal(0);
6169 case Intrinsic::amdgcn_workitem_id_x:
6170 case Intrinsic::amdgcn_workitem_id_y:
6171 case Intrinsic::amdgcn_workitem_id_z: {
6172 unsigned MaxValue = Subtarget->getMaxWorkitemID(
6177 case Intrinsic::amdgcn_readfirstlane:
6178 case Intrinsic::amdgcn_readlane:
6191 unsigned Depth)
const {
6192 switch (
Op.getOpcode()) {
6193 case AMDGPUISD::BFE_I32: {
6198 unsigned SignBits = 32 - (Width->
getZExtValue() & 0x1f) + 1;
6204 return std::max(SignBits, Op0SignBits);
6207 case AMDGPUISD::BFE_U32: {
6209 return Width ? 32 - (Width->
getZExtValue() & 0x1f) : 1;
6212 case AMDGPUISD::CARRY:
6213 case AMDGPUISD::BORROW:
6215 case AMDGPUISD::BUFFER_LOAD_BYTE:
6217 case AMDGPUISD::BUFFER_LOAD_SHORT:
6219 case AMDGPUISD::BUFFER_LOAD_UBYTE:
6221 case AMDGPUISD::BUFFER_LOAD_USHORT:
6223 case AMDGPUISD::FP_TO_FP16:
6225 case AMDGPUISD::SMIN3:
6226 case AMDGPUISD::SMAX3:
6227 case AMDGPUISD::SMED3:
6228 case AMDGPUISD::UMIN3:
6229 case AMDGPUISD::UMAX3:
6230 case AMDGPUISD::UMED3: {
6243 return std::min({Tmp0, Tmp1, Tmp2});
6258 switch (
MI->getOpcode()) {
6259 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
6261 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
6263 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
6265 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
6267 case AMDGPU::G_AMDGPU_SMED3:
6268 case AMDGPU::G_AMDGPU_UMED3: {
6269 auto [Dst, Src0, Src1, Src2] =
MI->getFirst4Regs();
6270 unsigned Tmp2 =
Analysis.computeNumSignBits(Src2, DemandedElts,
Depth + 1);
6273 unsigned Tmp1 =
Analysis.computeNumSignBits(Src1, DemandedElts,
Depth + 1);
6276 unsigned Tmp0 =
Analysis.computeNumSignBits(Src0, DemandedElts,
Depth + 1);
6279 return std::min({Tmp0, Tmp1, Tmp2});
6289 unsigned Opcode =
Op.getOpcode();
6291 case AMDGPUISD::BFE_I32:
6292 case AMDGPUISD::BFE_U32:
6296 Op, DemandedElts, DAG, Kind, ConsiderFlags,
Depth);
6301 unsigned Depth)
const {
6302 unsigned Opcode =
Op.getOpcode();
6304 case AMDGPUISD::FMIN_LEGACY:
6305 case AMDGPUISD::FMAX_LEGACY:
6308 case AMDGPUISD::FMUL_LEGACY:
6309 case AMDGPUISD::CVT_PKRTZ_F16_F32: {
6315 case AMDGPUISD::FMED3:
6316 case AMDGPUISD::FMIN3:
6317 case AMDGPUISD::FMAX3:
6318 case AMDGPUISD::FMINIMUM3:
6319 case AMDGPUISD::FMAXIMUM3:
6320 case AMDGPUISD::FMAD_FTZ: {
6327 case AMDGPUISD::CVT_F32_UBYTE0:
6328 case AMDGPUISD::CVT_F32_UBYTE1:
6329 case AMDGPUISD::CVT_F32_UBYTE2:
6330 case AMDGPUISD::CVT_F32_UBYTE3:
6333 case AMDGPUISD::RCP:
6334 case AMDGPUISD::RSQ:
6335 case AMDGPUISD::RCP_LEGACY:
6336 case AMDGPUISD::RSQ_CLAMP: {
6344 case AMDGPUISD::FRACT: {
6349 case AMDGPUISD::DIV_SCALE:
6350 case AMDGPUISD::DIV_FMAS:
6351 case AMDGPUISD::DIV_FIXUP:
6354 case AMDGPUISD::SIN_HW:
6355 case AMDGPUISD::COS_HW: {
6360 unsigned IntrinsicID =
Op.getConstantOperandVal(0);
6362 switch (IntrinsicID) {
6363 case Intrinsic::amdgcn_cubeid:
6364 case Intrinsic::amdgcn_cvt_off_f32_i4:
6367 case Intrinsic::amdgcn_frexp_mant: {
6372 case Intrinsic::amdgcn_cvt_pkrtz: {
6378 case Intrinsic::amdgcn_rcp:
6379 case Intrinsic::amdgcn_rsq:
6380 case Intrinsic::amdgcn_rcp_legacy:
6381 case Intrinsic::amdgcn_rsq_legacy:
6382 case Intrinsic::amdgcn_rsq_clamp:
6383 case Intrinsic::amdgcn_tanh: {
6390 case Intrinsic::amdgcn_trig_preop:
6391 case Intrinsic::amdgcn_fdot2:
6394 case Intrinsic::amdgcn_fma_legacy:
6411 return MRI.hasOneNonDBGUse(N0);
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
const AArch64InstrInfo & TII
AMDGPU address space definition.
static LLVM_READONLY bool hasSourceMods(const MachineInstr &MI)
static bool isInv2Pi(const APFloat &APF)
static LLVM_READONLY bool opMustUseVOP3Encoding(const MachineInstr &MI, const MachineRegisterInfo &MRI)
returns true if the operation will definitely need to use a 64-bit encoding, and thus will use a VOP3...
static unsigned inverseMinMax(unsigned Opc)
static SDValue extractF64Exponent(SDValue Hi, const SDLoc &SL, SelectionDAG &DAG)
static unsigned workitemIntrinsicDim(unsigned ID)
static int getOrCreateFixedStackObject(MachineFrameInfo &MFI, unsigned Size, int64_t Offset)
static SDValue constantFoldBFE(SelectionDAG &DAG, IntTy Src0, uint32_t Offset, uint32_t Width, const SDLoc &DL)
static SDValue getMad(SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue X, SDValue Y, SDValue C, SDNodeFlags Flags=SDNodeFlags())
static SDValue getAddOneOp(const SDNode *V)
If V is an add of a constant 1, returns the other operand.
static bool canIgnoreLegacyMinMaxTies(const SelectionDAG &DAG, SDNodeFlags Flags, SDValue LHS, SDValue RHS)
static LLVM_READONLY bool selectSupportsSourceMods(const SDNode *N)
Return true if v_cndmask_b32 will support fabs/fneg source modifiers for the type for ISD::SELECT.
static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL, SDValue N0, SDValue N1, unsigned Size, bool Signed)
static bool fnegFoldsIntoOp(const SDNode *N)
static bool isI24(SDValue Op, SelectionDAG &DAG)
static bool isCttzOpc(unsigned Opc)
static bool isU24(SDValue Op, SelectionDAG &DAG)
static bool valueIsKnownNeverF32Denorm(SDValue Src)
Return true if it's known that Src can never be an f32 denormal value.
static SDValue distributeOpThroughSelect(TargetLowering::DAGCombinerInfo &DCI, unsigned Op, const SDLoc &SL, SDValue Cond, SDValue N1, SDValue N2)
static SDValue peekFNeg(SDValue Val)
static SDValue simplifyMul24(SDNode *Node24, TargetLowering::DAGCombinerInfo &DCI)
static bool isCtlzOpc(unsigned Opc)
static LLVM_READNONE bool fnegFoldsIntoOpcode(unsigned Opc)
static bool hasVolatileUser(SDNode *Val)
Interface definition of the TargetLowering class that is common to all AMD GPUs.
Contains the definition of a TargetInstrInfo class that is common to all AMD GPUs.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
block Block Frequency Analysis
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Provides analysis for querying information about KnownBits during GISel passes.
static MaybeAlign getAlign(Value *Ptr)
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
MachineBasicBlock * Folded
SmallVector< MachineOperand, 4 > Cond
MachineRegisterInfo & MRI
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg)
static CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC, bool IsVarArg)
unsigned allocateBarrierGlobal(const DataLayout &DL, const GlobalVariable &GV)
uint64_t getExplicitKernArgSize() const
bool isModuleEntryFunction() const
static std::optional< uint32_t > get32BitAbsoluteAddress(const GlobalValue &GV, unsigned AS)
unsigned allocateLDSGlobal(const DataLayout &DL, const GlobalVariable &GV)
static unsigned numBitsSigned(SDValue Op, SelectionDAG &DAG)
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
This method can be implemented by targets that want to expose additional information about sign bits ...
SDValue performMulhuCombine(SDNode *N, DAGCombinerInfo &DCI) const
EVT getTypeForExtReturn(LLVMContext &Context, EVT VT, ISD::NodeType ExtendKind) const override
Return the type that should be used to zero or sign extend a zeroext/signext integer return value.
SDValue SplitVectorLoad(SDValue Op, SelectionDAG &DAG) const
Split a vector load into 2 loads of half the vector.
SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const
SDValue performLoadCombine(SDNode *N, DAGCombinerInfo &DCI) const
void analyzeFormalArgumentsCompute(CCState &State, const SmallVectorImpl< ISD::InputArg > &Ins) const
The SelectionDAGBuilder will automatically promote function arguments with illegal types.
SDValue LowerF64ToF16Safe(SDValue Src, const SDLoc &DL, SelectionDAG &DAG) const
SDValue LowerFROUND(SDValue Op, SelectionDAG &DAG) const
SDValue storeStackInputValue(SelectionDAG &DAG, const SDLoc &SL, SDValue Chain, SDValue ArgVal, int64_t Offset) const
bool storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT, unsigned NumElem, unsigned AS) const override
Return true if it is expected to be cheaper to do a store of vector constant with the given size and ...
SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool shouldCombineMemoryType(EVT VT) const
SDValue splitBinaryBitConstantOpImpl(DAGCombinerInfo &DCI, const SDLoc &SL, unsigned Opc, SDValue LHS, uint32_t ValLo, uint32_t ValHi) const
Split the 64-bit value LHS into two 32-bit components, and perform the binary operation Opc to it wit...
SDValue lowerUnhandledCall(CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals, StringRef Reason) const
virtual SDValue LowerGlobalAddress(AMDGPUMachineFunctionInfo *MFI, SDValue Op, SelectionDAG &DAG) const
SDValue performAssertSZExtCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool isTruncateFree(EVT Src, EVT Dest) const override
bool aggressivelyPreferBuildVectorSources(EVT VecVT) const override
SDValue LowerFCEIL(SDValue Op, SelectionDAG &DAG) const
TargetLowering::NegatibleCost getConstantNegateCost(const ConstantFPSDNode *C) const
SDValue LowerFLOGUnsafe(SDValue Op, const SDLoc &SL, SelectionDAG &DAG, bool IsLog10, SDNodeFlags Flags) const
SDValue combineFMinMaxLegacy(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True, SDValue False, SDValue CC, SDNodeFlags Flags, DAGCombinerInfo &DCI) const
Flags must be the select flags, not the compare (SELECT_CC flags come from the fcmp and say nothing a...
SDValue performMulhsCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue lowerFEXPUnsafeImpl(SDValue Op, const SDLoc &SL, SelectionDAG &DAG, SDNodeFlags Flags, bool IsExp10) const
bool isSDNodeAlwaysUniform(const SDNode *N) const override
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
SDValue performShlCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
SDValue LowerSDIVREM(SDValue Op, SelectionDAG &DAG) const
bool isFNegFree(EVT VT) const override
Return true if an fneg operation is free to the point where it is never worthwhile to replace it with...
FPImmAction getFPImmAction(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns how the legalizer should materialize the specified FP immediate.
SDValue LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG, bool Signed) const
unsigned computeNumSignBitsForTargetInstr(GISelValueTracking &Analysis, Register R, const APInt &DemandedElts, const MachineRegisterInfo &MRI, unsigned Depth=0) const override
This method can be implemented by targets that want to expose additional information about sign bits ...
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
SDValue LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const
SDValue addTokenForArgument(SDValue Chain, SelectionDAG &DAG, MachineFrameInfo &MFI, int ClobberedFI) const
bool isReassocProfitable(MachineRegisterInfo &MRI, Register N0, Register N1) const override
bool isKnownNeverNaNForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool SNaN=false, unsigned Depth=0) const override
If SNaN is false,.
static bool needsDenormHandlingF32(const SelectionDAG &DAG, SDValue Src, SDNodeFlags Flags)
SDValue lowerFPOW(SDValue Op, SelectionDAG &DAG) const
uint32_t getImplicitParameterOffset(const MachineFunction &MF, const ImplicitParameter Param) const
Helper function that returns the byte offset of the given type of implicit parameter.
SDValue lowerFEXPF64(SDValue Op, SelectionDAG &DAG) const
SDValue LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const
SDValue performSelectCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue performFNegCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const
bool isConstantCostlierToNegate(SDValue N) const
static KnownBits computeKnownBitsForMul24(const KnownBits &LHSKnown, const KnownBits &RHSKnown, unsigned BitWidth, bool IsSigned, bool SelfMultiply)
SDValue loadInputValue(SelectionDAG &DAG, const TargetRegisterClass *RC, EVT VT, const SDLoc &SL, const ArgDescriptor &Arg) const
SDValue lowerFEXP10Unsafe(SDValue Op, const SDLoc &SL, SelectionDAG &DAG, SDNodeFlags Flags) const
Emit approx-funcs appropriate lowering for exp10.
bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtType, EVT ExtVT, std::optional< unsigned > ByteOffset) const override
Return true if it is profitable to reduce a load to a smaller type.
SDValue LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) const
bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const override
Return true if Op can create undef or poison from non-undef & non-poison operands.
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
SDValue performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond, SDValue LHS, SDValue RHS, DAGCombinerInfo &DCI) const
SDValue performSraCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool isSelectSupported(SelectSupportKind) const override
bool isZExtFree(Type *Src, Type *Dest) const override
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
SDValue lowerFEXP2(SDValue Op, SelectionDAG &DAG) const
SDValue LowerCall(CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
SDValue performSrlCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue lowerFEXP(SDValue Op, SelectionDAG &DAG) const
SDValue getIsFinite(SelectionDAG &DAG, SDValue Op, SDNodeFlags Flags) const
bool isLoadBitCastBeneficial(EVT, EVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const final
Return true if the following transform is beneficial: fold (conv (load x)) -> (load (conv*)x) On arch...
std::pair< SDValue, SDValue > splitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HighVT, SelectionDAG &DAG) const
Split a vector value into two parts of types LoVT and HiVT.
AMDGPUTargetLowering(const TargetMachine &TM, const TargetSubtargetInfo &STI, const AMDGPUSubtarget &AMDGPUSTI)
SDValue LowerFLOGCommon(SDValue Op, SelectionDAG &DAG) const
SDValue foldFreeOpFromSelect(TargetLowering::DAGCombinerInfo &DCI, SDValue N) const
SDValue LowerINT_TO_FP32(SDValue Op, SelectionDAG &DAG, bool Signed) const
bool isFAbsFree(EVT VT) const override
Return true if an fabs operation is free to the point where it is never worthwhile to replace it with...
bool isInt64ImmLegal(SDNode *Val, SelectionDAG &DAG) const
Check whether value Val can be supported by v_mov_b64, for the current target.
SDValue loadStackInputValue(SelectionDAG &DAG, EVT VT, const SDLoc &SL, int64_t Offset) const
Similar to CreateLiveInRegister, except value maybe loaded from a stack slot rather than passed in a ...
SDValue LowerFLOG2(SDValue Op, SelectionDAG &DAG) const
static EVT getEquivalentMemType(LLVMContext &Context, EVT VT)
SDValue LowerCTLS(SDValue Op, SelectionDAG &DAG) const
Split a vector store into multiple scalar stores.
SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps, bool &UseOneConstNR, bool Reciprocal) const override
Hooks for building estimates in place of slower divisions and square roots.
SDValue performTruncateCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) const
static SDValue stripBitcast(SDValue Val)
SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const
SDValue CreateLiveInRegister(SelectionDAG &DAG, const TargetRegisterClass *RC, Register Reg, EVT VT, const SDLoc &SL, bool RawReg=false) const
Helper function that adds Reg to the LiveIn list of the DAG's MachineFunction.
SDValue SplitVectorStore(SDValue Op, SelectionDAG &DAG) const
Split a vector store into 2 stores of half the vector.
SDValue LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) const
SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOperations, bool ForCodeSize, NegatibleCost &Cost, unsigned Depth) const override
Return the newly negated expression if the cost is not expensive and set the cost in Cost to indicate...
std::pair< SDValue, SDValue > split64BitValue(SDValue Op, SelectionDAG &DAG) const
Return 64-bit value Op as two 32-bit integers.
SDValue performMulCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue combineFMinMaxLegacyImpl(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True, SDValue False, SDValue CC, SDNodeFlags Flags, DAGCombinerInfo &DCI) const
SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps) const override
Return a reciprocal estimate value for the input operand.
SDValue LowerFNEARBYINT(SDValue Op, SelectionDAG &DAG) const
SDValue LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const
static CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC, bool IsVarArg)
std::pair< SDValue, SDValue > getScaledLogInput(SelectionDAG &DAG, const SDLoc SL, SDValue Op, SDNodeFlags Flags) const
If denormal handling is required return the scaled input to FLOG2, and the check for denormal range.
static CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg)
Selects the correct CCAssignFn for a given CallingConvention value.
bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &OriginalDemandedBits, const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const override
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
static bool allUsesHaveSourceMods(const SDNode *N, unsigned CostThreshold=4)
SDValue LowerFROUNDEVEN(SDValue Op, SelectionDAG &DAG) const
static unsigned numBitsUnsigned(SDValue Op, SelectionDAG &DAG)
SDValue lowerFEXPUnsafe(SDValue Op, const SDLoc &SL, SelectionDAG &DAG, SDNodeFlags Flags) const
SDValue LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const
SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const
static bool allowApproxFunc(const SelectionDAG &DAG, SDNodeFlags Flags)
bool ShouldShrinkFPConstant(EVT VT) const override
If true, then instruction selection should seek to shrink the FP constant of the specified type to a ...
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
SDValue performStoreCombine(SDNode *N, DAGCombinerInfo &DCI) const
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
SDValue performRcpCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue lowerCTLZResults(SDValue Op, SelectionDAG &DAG) const
SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const
SDValue performFAbsCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue LowerFP_TO_INT64(SDValue Op, SelectionDAG &DAG, bool Signed) const
static bool shouldFoldFNegIntoSrc(SDNode *FNeg, SDValue FNegSrc)
bool isNarrowingProfitable(SDNode *N, EVT SrcVT, EVT DestVT) const override
Return true if it's profitable to narrow operations of type SrcVT to DestVT.
SDValue LowerFRINT(SDValue Op, SelectionDAG &DAG) const
SDValue performIntrinsicWOChainCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue LowerUDIVREM(SDValue Op, SelectionDAG &DAG) const
SDValue performMulLoHiCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
SDValue lowerINT_TO_FPImpl(SDValue Op, SelectionDAG &DAG, bool Signed) const
void LowerUDIVREM64(SDValue Op, SelectionDAG &DAG, SmallVectorImpl< SDValue > &Results) const
SDValue WidenOrSplitVectorLoad(SDValue Op, SelectionDAG &DAG) const
Widen a suitably aligned v3 load.
SDValue LowerDIVREMToFloat(SDValue Op, SelectionDAG &DAG, bool sign) const
std::pair< EVT, EVT > getSplitDestVTs(const EVT &VT, SelectionDAG &DAG) const
Split a vector type into two parts.
SDValue getHiHalf64(SDValue Op, SelectionDAG &DAG) const
SDValue LowerINT_TO_FP16(SDValue Op, SelectionDAG &DAG, EVT FP16Ty) const
unsigned getVectorIdxWidth(const DataLayout &) const override
Returns the type to be used for the index operand vector operations.
static const fltSemantics & IEEEsingle()
static const fltSemantics & IEEEdouble()
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEhalf()
static APFloat getQNaN(const fltSemantics &Sem, bool Negative=false, const APInt *payload=nullptr)
Factory for QNaN values.
bool bitwiseIsEqual(const APFloat &RHS) const
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
APInt bitcastToAPInt() const
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
This class represents an incoming formal argument to a Function.
int64_t getOffset() const
unsigned getTargetFlags() const
const BlockAddress * getBlockAddress() const
CCState - This class holds information needed while lowering arguments and return values.
static CCValAssign getCustomMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP)
const APFloat & getValueAPF() const
bool isNegative() const
Return true if the value is negative.
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
A parsed version of the target data layout string in and methods for querying it.
Diagnostic information for unsupported feature in backend.
const DataLayout & getDataLayout() const
Get the data layout of the module this function belongs to.
iterator_range< arg_iterator > args()
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
bool signBitIsZeroOrNaN() const
Return true if the sign bit must be 0, ignoring the sign of nans.
This is an important class for using LLVM in a threaded context.
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
static auto integer_fixedlen_vector_valuetypes()
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
static auto integer_valuetypes()
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
LLVM_ABI int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
int getObjectIndexBegin() const
Return the minimum frame object index.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
DenormalMode getDenormalMode(const fltSemantics &FPType) const
Returns the denormal handling type for the default rounding mode of the function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Representation of each machine instruction.
A description of a memory reference used in the backend.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOInvariant
The memory access always returns the same value (or traps).
Flags getFlags() const
Return the raw flags of the source value,.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This is an abstract virtual class for memory operations.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return the unique MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
SDNodeFlags getFlags() const
SDVTList getVTList() const
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
iterator_range< user_iterator > users()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node, in exactly one operand.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
unsigned getOpcode() const
unsigned getNumOperands() const
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
SIModeRegisterDefaults getMode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI bool isKnownNeverLogicalZero(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
Test whether the given floating point SDValue (or all elements of it, if it is a vector) is known to ...
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
LLVM_ABI unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
const TargetSubtargetInfo & getSubtarget() const
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
LLVM_ABI void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
LLVM_ABI SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false, SDNodeFlags Flags={})
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
LLVM_ABI SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Helper function to build ISD::STORE nodes.
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
bool isConstantValueOfAnyType(SDValue N) const
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
LLVM_ABI SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI KnownFPClass computeKnownFPClass(SDValue Op, FPClassTest InterestedClasses, unsigned Depth=0) const
Determine floating-point class information about Op.
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, UndefPoisonKind Kind=UndefPoisonKind::UndefOrPoison, unsigned Depth=0) const
Return true if this function can prove that Op is never poison and, Kind can be used to track poison ...
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI bool isKnownNeverNaN(SDValue Op, const APInt &DemandedElts, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN in...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getPOISON(EVT VT)
Return a POISON node. POISON does not have a useful SDLoc.
LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM_ABI KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVM_ABI bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
LLVM_ABI SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
LLVM_ABI std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getValue() const
Represent a constant reference to a string, i.e.
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
void setMaxDivRemBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum div/rem the backend supports.
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
virtual bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT, std::optional< unsigned > ByteOffset=std::nullopt) const
Return true if it is profitable to reduce a load to a smaller type.
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
unsigned MaxGluedStoresPerMemcpy
Specify max number of store instructions to glue in inlined memcpy.
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void addBypassSlowDiv(unsigned int SlowBitWidth, unsigned int FastBitWidth)
Tells the code generator which bitwidths to bypass.
void setMaxLargeFPConvertBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum fp to/from int conversion the backend supports.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
SelectSupportKind
Enum that describes what type of support for selects the target has.
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setSupportsUnalignedAtomics(bool UnalignedSupported)
Sets whether unaligned atomic operations are supported.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
FPImmAction
How the legalizer should materialize a floating-point immediate.
@ ConstantPool
Load the immediate from the constant pool.
@ Legal
The target can instruction select the immediate natively.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
unsigned GatherAllAliasesMaxDepth
Depth that GatherAllAliases should continue looking for chain dependencies when trying to find a more...
NegatibleCost
Enum that specifies when a float negation is beneficial.
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
void setJumpIsExpensive(bool isExpensive=true)
Tells the code generator not to expand logic operations on comparison predicates into separate sequen...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
SDValue expandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG) const
Expands an unaligned store to 2 half-size stores for integer values, and possibly more for vectors.
std::pair< SDValue, SDValue > expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Expands an unaligned load to 2 half-size loads for an integer, and possibly more for vectors.
virtual SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, NegatibleCost &Cost, unsigned Depth=0) const
Return the newly negated expression if the cost is not expensive and set the cost in Cost to indicate...
std::pair< SDValue, SDValue > scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Turn load of vector type into a load of the individual elements.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
TargetLowering(const TargetLowering &)=delete
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
Primary interface to the complete machine description for the target machine.
TargetSubtargetInfo - Generic base class for all target subtargets.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
LLVM Value Representation.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ CONSTANT_ADDRESS_32BIT
Address space for 32-bit constant memory.
@ BARRIER
Address space for modeling barrier IDs as addresses.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ CONSTANT_ADDRESS
Address space for constant memory (VTX2).
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
bool isIntrinsicAlwaysUniform(unsigned IntrID)
TargetExtType * isNamedBarrier(const GlobalVariable &GV)
std::optional< APFloat > evaluateRcp(const APFloat &Val)
Evaluate the constant-folded result of v_rcp for Val, accounting for the hardware's denormal flushing...
bool isUniformMMO(const MachineMemOperand *MMO)
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ AMDGPU_CS
Used for Mesa/AMDPAL compute shaders.
@ AMDGPU_VS
Used for Mesa vertex shaders, or AMDPAL last shader stage before rasterization (vertex shader if tess...
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
@ AMDGPU_Gfx
Used for AMD graphics targets.
@ AMDGPU_CS_ChainPreserve
Used on AMDGPUs to give the middle-end more control over argument placement.
@ AMDGPU_HS
Used for Mesa/AMDPAL hull shaders (= tessellation control shaders).
@ AMDGPU_GS
Used for Mesa/AMDPAL geometry shaders.
@ AMDGPU_CS_Chain
Used on AMDGPUs to give the middle-end more control over argument placement.
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
@ Cold
Attempts to make code in the caller as efficient as possible under the assumption that the call is no...
@ SPIR_KERNEL
Used for SPIR kernel functions.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ AMDGPU_ES
Used for AMDPAL shader stage before geometry shader if geometry is in use.
@ AMDGPU_LS
Used for AMDPAL vertex shader if tessellation is in use.
@ C
The default llvm calling convention, compatible with C.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SIGN_EXTEND
Conversion operators.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BRIND
BRIND - Indirect branch.
@ BR_JT
BR_JT - Jumptable branch.
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ CTLS
Count leading redundant sign bits.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Same as FMINNUM/FMAXNUM, except that a signaling NaN operand deterministi...
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ INLINEASM_BR
INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
@ FMINIMUM
FMINIMUM/FMAXIMUM - Same semantics as the llvm.minimum/maximum intrinsics.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
@ INLINEASM
INLINEASM - Represents an inline asm block.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - Same semantics as the llvm.minimumnum/maximumnum intrinsics.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
LLVM_ABI CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
@ Known
Known to have no common set bits.
LLVM_ABI void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs=nullptr, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
SDValue peekFPSignOps(SDValue Val)
Strip fabs/fneg/fcopysign from a value to get the underlying source.
LLVM_ABI ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
To bit_cast(const From &from) noexcept
@ Mul
Product of integers.
@ Fast
Assign the register banks as fast as possible (default).
DWARFExpression::Operation Op
LLVM_ABI ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
UndefPoisonKind
Enumeration to track whether we are interested in Undef, Poison, or both.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
APFloat neg(APFloat X)
Returns the negated value of the argument.
unsigned Log2(Align A)
Returns the log2 of the alignment.
LLVM_ABI bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
MCRegisterClass TargetRegisterClass
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
MCRegister getRegister() const
unsigned getStackOffset() const
DenormalModeKind Input
Denormal treatment kind for floating point instruction inputs in the default floating-point environme...
@ PreserveSign
The sign of a flushed-to-zero number is preserved in the sign of 0.
constexpr bool inputsAreZero() const
Return true if input denormals must be implicitly treated as 0.
static constexpr DenormalMode getPreserveSign()
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
EVT getPow2VectorType(LLVMContext &Context) const
Widens the length of the given vector EVT up to the nearest power of 2 and returns that type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
EVT getRoundIntegerType(LLVMContext &Context) const
Rounds the bit-width of the given integer EVT up to the nearest power of two (and at least to eight),...
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isExtended() const
Test if the given EVT is extended (as opposed to being simple).
EVT changeElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a type whose attributes match ourselves with the exception of the element type that i...
LLVM_ABI const fltSemantics & getFltSemantics() const
Returns an APFloat semantics tag appropriate for the value type.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool isUnknown() const
Returns true if we don't know any bits.
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
bool isNegative() const
Returns true if this value is known to be negative.
static LLVM_ABI KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
This class contains a discriminated union of information about pointers in memory operands,...
LLVM_ABI bool isDereferenceable(unsigned Size, LLVMContext &C, const DataLayout &DL) const
Return true if memory region [V, V+Offset+Size) is known to be dereferenceable.
static LLVM_ABI MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
MachinePointerInfo getWithOffset(int64_t O) const
These are IR-level optimization flags that may be propagated to SDNodes.
void setAllowContract(bool b)
bool hasNoSignedZeros() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
DenormalMode FP32Denormals
If this is set, neither input or output denormals are flushed for most f32 instructions.
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
bool isBeforeLegalizeOps() const
CombineLevel getDAGCombineLevel()
LLVM_ABI void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
bool isBeforeLegalize() const
LLVM_ABI SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
LLVM_ABI void CommitTargetLoweringOpt(const TargetLoweringOpt &TLO)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...