67 if (
F.getFnAttribute(
"disable-tail-calls").getValueAsBool())
73 AttrBuilder CallerAttrs(
F.getContext(),
F.getAttributes().getRetAttrs());
74 for (
const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
75 Attribute::DereferenceableOrNull, Attribute::NoAlias,
76 Attribute::NonNull, Attribute::NoUndef,
77 Attribute::Range, Attribute::NoFPClass})
78 CallerAttrs.removeAttribute(Attr);
80 if (CallerAttrs.hasAttributes())
84 if (CallerAttrs.contains(Attribute::ZExt) ||
85 CallerAttrs.contains(Attribute::SExt))
96 for (
unsigned I = 0, E = ArgLocs.
size();
I != E; ++
I) {
113 if (
MRI.getLiveInPhysReg(ArgReg) != Reg)
121 return Call.paramHasAttr(ArgIdx, Kind);
126 return Attrs.hasParamAttr(ArgIdx, Kind);
130 return Call.getParamStackAlign(ArgIdx);
135 return Attrs.getParamStackAlignment(ArgIdx);
139 return Call.getParamAlign(ArgIdx);
143 return Attrs.getParamAlignment(ArgIdx);
148template <
typename SourceT>
150 const SourceT &Src,
unsigned ArgIdx) {
151 Entry.IsSExt =
paramHasAttr(Src, ArgIdx, Attribute::SExt);
152 Entry.IsZExt =
paramHasAttr(Src, ArgIdx, Attribute::ZExt);
153 Entry.IsNoExt =
paramHasAttr(Src, ArgIdx, Attribute::NoExt);
154 Entry.IsInReg =
paramHasAttr(Src, ArgIdx, Attribute::InReg);
155 Entry.IsSRet =
paramHasAttr(Src, ArgIdx, Attribute::StructRet);
156 Entry.IsNest =
paramHasAttr(Src, ArgIdx, Attribute::Nest);
157 Entry.IsByVal =
paramHasAttr(Src, ArgIdx, Attribute::ByVal);
158 Entry.IsPreallocated =
paramHasAttr(Src, ArgIdx, Attribute::Preallocated);
159 Entry.IsInAlloca =
paramHasAttr(Src, ArgIdx, Attribute::InAlloca);
160 Entry.IsReturned =
paramHasAttr(Src, ArgIdx, Attribute::Returned);
161 Entry.IsSwiftSelf =
paramHasAttr(Src, ArgIdx, Attribute::SwiftSelf);
162 Entry.IsSwiftAsync =
paramHasAttr(Src, ArgIdx, Attribute::SwiftAsync);
163 Entry.IsSwiftError =
paramHasAttr(Src, ArgIdx, Attribute::SwiftError);
165 Entry.IndirectType =
nullptr;
166 assert(Entry.IsByVal + Entry.IsPreallocated + Entry.IsInAlloca +
169 "multiple ABI attributes?");
171 Entry.IndirectType = Src.getParamByValType(ArgIdx);
172 if (!Entry.Alignment)
175 if (Entry.IsPreallocated)
176 Entry.IndirectType = Src.getParamPreallocatedType(ArgIdx);
177 if (Entry.IsInAlloca)
178 Entry.IndirectType = Src.getParamInAllocaType(ArgIdx);
180 Entry.IndirectType = Src.getParamStructRetType(ArgIdx);
196 if (Attr == Attribute::SExt)
198 else if (Attr == Attribute::ZExt)
200 else if (Attr == Attribute::NoExt)
213 assert(
Ops.size() == FuncTy->getNumParams() &&
214 "argument count does not match the function type");
216 Args.reserve(
Ops.size());
217 for (
unsigned I = 0, E = FuncTy->getNumParams();
I != E; ++
I) {
219 Entry.setAttributes(FuncAttrs,
I);
220 Args.push_back(Entry);
229 CallOptions, dl, Chain);
234std::pair<SDValue, SDValue>
239 if (LibcallImpl == RTLIB::Unsupported)
246 Args.reserve(
Ops.size());
249 for (
unsigned i = 0; i <
Ops.size(); ++i) {
251 Type *Ty = i < OpsTypeOverrides.
size() && OpsTypeOverrides[i]
252 ? OpsTypeOverrides[i]
261 Entry.IsZExt = !Entry.IsSExt;
265 Entry.IsSExt = Entry.IsZExt =
false;
267 Args.push_back(Entry);
274 Type *OrigRetTy = RetTy;
277 bool zeroExtend = !signExtend;
282 signExtend = zeroExtend =
false;
288 RetTy, OrigRetTy, Callee, std::move(Args))
298 LLVMContext &Context, std::vector<EVT> &MemOps,
unsigned Limit,
299 const MemOp &
Op,
unsigned DstAS,
unsigned SrcAS,
303 if (VT == MVT::Other) {
305 VT = MVT::LAST_INTEGER_VALUETYPE;
306 if (
Op.isFixedDstAlign()) {
307 bool LoadsFromSrc =
Op.isMemcpyOrMemmove() && !
Op.isMemcpyStrSrc();
308 while (VT != MVT::i8) {
311 Op.getDstAlign() >= VTSize ||
314 !LoadsFromSrc ||
Op.getSrcAlign() >= VTSize ||
324 MVT LVT = MVT::LAST_INTEGER_VALUETYPE;
335 unsigned NumMemOps = 0;
336 uint64_t
Size =
Op.size();
339 while (VTSize >
Size) {
350 else if (NewVT == MVT::i64 &&
362 if (NewVT == MVT::i8)
371 if (NumMemOps && !
Op.isVolatile() && NewVTSize <
Size &&
373 VT, DstAS,
Op.isFixedDstAlign() ?
Op.getDstAlign() :
Align(1),
383 if (++NumMemOps > Limit)
386 MemOps.push_back(VT);
411static std::pair<RTLIB::Libcall, ISD::CondCode>
413 RTLIB::Libcall TriStateLC, RTLIB::Libcall GenericLC,
418 return {TriStateLC, TriStateCC};
419 return {GenericLC, TriStateCC};
428 bool IsSignaling)
const {
433 assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)
434 &&
"Unsupported setcc type!");
438 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
440 bool ShouldInvertCC =
false;
444#define FP_CMP_LIBCALL(BASE) \
445 RTLIB::getFPLibCall(VT, RTLIB::BASE##_F32, RTLIB::BASE##_F64, \
446 RTLIB::UNKNOWN_LIBCALL, RTLIB::BASE##_F128, \
447 RTLIB::BASE##_PPCF128)
467 ShouldInvertCC =
true;
495 ShouldInvertCC =
true;
504 ShouldInvertCC =
true;
515 ShouldInvertCC =
true;
552 auto ReportNoLibcall = [&]() {
554 Twine(
"no libcall available to soften floating-point ") +
564 if (LC1Impl == RTLIB::Unsupported) {
574 if (ShouldInvertCC) {
576 CCCode = getSetCCInverse(CCCode, RetVT);
579 if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
584 if (LC2Impl == RTLIB::Unsupported) {
590 "unordered call should be simple boolean");
600 auto Call2 =
makeLibCall(DAG, LC2Impl, RetVT,
Ops, CallOptions, dl, Chain);
603 CCCode = getSetCCInverse(CCCode, RetVT);
604 NewLHS = DAG.
getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);
657 if (!TM.shouldAssumeDSOLocal(GV))
677 const APInt &DemandedElts,
680 unsigned Opcode =
Op.getOpcode();
699 if (!Op1C || Op1C->isOpaque())
703 const APInt &
C = Op1C->getAPIntValue();
708 EVT VT =
Op.getValueType();
725 EVT VT =
Op.getValueType();
740 "ShrinkDemandedOp only supports binary operators!");
741 assert(
Op.getNode()->getNumValues() == 1 &&
742 "ShrinkDemandedOp only supports nodes with one result!");
744 EVT VT =
Op.getValueType();
753 Op.getOperand(1).getValueType().getScalarSizeInBits() ==
BitWidth &&
754 "ShrinkDemandedOp only supports operands that have the same size!");
758 if (!
Op.getNode()->hasOneUse())
774 unsigned Opcode =
Op.getOpcode();
784 assert(DemandedSize <= SmallVTBits &&
"Narrowed below demanded bits?");
808 const APInt &DemandedElts,
828 bool AssumeSingleUse)
const {
829 EVT VT =
Op.getValueType();
845 EVT VT =
Op.getValueType();
863 switch (
Op.getOpcode()) {
875 EVT SrcVT = Src.getValueType();
876 EVT DstVT =
Op.getValueType();
882 if (NumSrcEltBits == NumDstEltBits)
887 if (SrcVT.
isVector() && (NumDstEltBits % NumSrcEltBits) == 0) {
888 unsigned Scale = NumDstEltBits / NumSrcEltBits;
891 for (
unsigned i = 0; i != Scale; ++i) {
892 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
893 unsigned BitOffset = EltOffset * NumSrcEltBits;
894 DemandedSrcBits |=
DemandedBits.extractBits(NumSrcEltBits, BitOffset);
902 Src, DemandedSrcBits, DemandedSrcElts, DAG,
Depth + 1))
907 if (IsLE && (NumSrcEltBits % NumDstEltBits) == 0) {
908 unsigned Scale = NumSrcEltBits / NumDstEltBits;
912 for (
unsigned i = 0; i != NumElts; ++i)
913 if (DemandedElts[i]) {
914 unsigned Offset = (i % Scale) * NumDstEltBits;
916 DemandedSrcElts.
setBit(i / Scale);
920 Src, DemandedSrcBits, DemandedSrcElts, DAG,
Depth + 1))
939 return Op.getOperand(0);
941 return Op.getOperand(1);
952 return Op.getOperand(0);
954 return Op.getOperand(1);
964 return Op.getOperand(0);
966 return Op.getOperand(1);
976 DemandedElts, 1,
Depth + 1))
977 return Op.getOperand(0);
980 DemandedElts, 0,
Depth + 1))
981 return Op.getOperand(1);
987 if (std::optional<unsigned> MaxSA =
990 unsigned ShAmt = *MaxSA;
991 unsigned NumSignBits =
994 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
1002 if (std::optional<unsigned> MaxSA =
1005 unsigned ShAmt = *MaxSA;
1009 unsigned NumSignBits =
1048 if (NumSignBits >= (
BitWidth - ExBits + 1))
1061 EVT SrcVT = Src.getValueType();
1062 EVT DstVT =
Op.getValueType();
1063 if (IsLE && DemandedElts == 1 &&
1079 !DemandedElts[CIdx->getZExtValue()])
1089 uint64_t Idx =
Op.getConstantOperandVal(2);
1090 unsigned NumSubElts =
Sub.getValueType().getVectorNumElements();
1093 if (DemandedSubElts == 0)
1103 bool AllUndef =
true, IdentityLHS =
true, IdentityRHS =
true;
1104 for (
unsigned i = 0; i != NumElts; ++i) {
1105 int M = ShuffleMask[i];
1106 if (M < 0 || !DemandedElts[i])
1109 IdentityLHS &= (M == (int)i);
1110 IdentityRHS &= ((M - NumElts) == i);
1116 return Op.getOperand(0);
1118 return Op.getOperand(1);
1138 unsigned Depth)
const {
1139 EVT VT =
Op.getValueType();
1152 unsigned Depth)
const {
1167 return std::nullopt;
1181 "SRL or SRA node is required here!");
1184 if (!N1C || !N1C->
isOne())
1231 unsigned ShiftOpc =
Op.getOpcode();
1232 bool IsSigned =
false;
1236 unsigned NumSigned = std::min(NumSignedA, NumSignedB) - 1;
1241 unsigned NumZero = std::min(NumZeroA, NumZeroB);
1247 if (NumZero >= 2 && NumSigned < NumZero) {
1252 if (NumSigned >= 1) {
1260 if (NumZero >= 1 && NumSigned < NumZero) {
1280 EVT VT =
Op.getValueType();
1281 std::optional<EVT> NarrowVT =
1285 EVT NVT = *NarrowVT;
1286 if (TLO.
LegalTypes() && !
TLI.isOperationLegal(AVGOpc, NVT)) {
1292 Add.getOperand(1)) &&
1302 if (!IsCeil && !
TLI.isOperationLegal(AVGOpc, NVT) &&
1320 "SRL or SRA node is required here!");
1323 if (!N1C || !N1C->
isOne())
1343 unsigned NumSigned =
1347 unsigned NumZero = std::min(
1350 if (NumZero >= 1 && NumSigned < NumZero) {
1353 }
else if (NumSigned >= 1) {
1367 auto IsLegalAfterTypeLegalization = [&](
EVT VT) {
1368 EVT LegalVT =
TLI.getLegalTypeToTransformTo(Ctx, VT);
1370 TLI.isOperationLegal(AVGOpc, LegalVT);
1378 EVT VT =
Op.getValueType();
1381 if (NarrowVT && IsLegalAfterTypeLegalization(*NarrowVT))
1383 else if (!IsLegalAfterTypeLegalization(VT))
1388 if (!
TLI.isOperationLegal(AVGOpc, NVT) &&
1411 unsigned Depth,
bool AssumeSingleUse)
const {
1414 "Mask size mismatches value type size!");
1419 EVT VT =
Op.getValueType();
1421 unsigned NumElts = OriginalDemandedElts.
getBitWidth();
1423 "Unexpected vector size");
1426 APInt DemandedElts = OriginalDemandedElts;
1451 bool HasMultiUse =
false;
1452 if (!AssumeSingleUse && !
Op.getNode()->hasOneUse()) {
1461 }
else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
1473 auto TryFoldToLHS = [&](
const APInt &Mask,
1490 auto TryReduceRHSDemand = [&](
const KnownBits &LHSKnown,
1502 APInt RHSDemandedBits =
1505 : LHSKnown.*KnownMask;
1518 switch (
Op.getOpcode()) {
1522 if (!DemandedElts[0])
1527 unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
1534 if (DemandedElts == 1)
1570 EVT MemVT = LD->getMemoryVT();
1572 Known.Zero.setBitsFrom(MemBits);
1587 APInt DemandedVecElts(DemandedElts);
1589 unsigned Idx = CIdx->getZExtValue();
1593 if (!DemandedElts[Idx])
1610 if (!!DemandedVecElts)
1622 uint64_t Idx =
Op.getConstantOperandVal(2);
1623 unsigned NumSubElts =
Sub.getValueType().getVectorNumElements();
1625 APInt DemandedSrcElts = DemandedElts;
1626 DemandedSrcElts.
clearBits(Idx, Idx + NumSubElts);
1636 Known.setAllConflict();
1637 if (!!DemandedSubElts)
1639 if (!!DemandedSrcElts)
1649 if (NewSub || NewSrc) {
1650 NewSub = NewSub ? NewSub :
Sub;
1651 NewSrc = NewSrc ? NewSrc : Src;
1664 if (Src.getValueType().isScalableVector())
1666 uint64_t Idx =
Op.getConstantOperandVal(1);
1667 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1668 APInt DemandedSrcElts = DemandedElts.
zext(NumSrcElts).
shl(Idx);
1689 Known.setAllConflict();
1690 EVT SubVT =
Op.getOperand(0).getValueType();
1691 unsigned NumSubVecs =
Op.getNumOperands();
1693 for (
unsigned i = 0; i != NumSubVecs; ++i) {
1694 APInt DemandedSubElts =
1695 DemandedElts.
extractBits(NumSubElts, i * NumSubElts);
1697 Known2, TLO,
Depth + 1))
1700 if (!!DemandedSubElts)
1710 APInt DemandedLHS, DemandedRHS;
1715 if (!!DemandedLHS || !!DemandedRHS) {
1719 Known.setAllConflict();
1720 if (!!DemandedLHS) {
1726 if (!!DemandedRHS) {
1738 if (DemandedOp0 || DemandedOp1) {
1739 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1740 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1775 LHSKnown.
One == ~RHSC->getAPIntValue()) {
1798 unsigned NumSubElts =
1819 Known2, TLO,
Depth + 1))
1847 if (DemandedOp0 || DemandedOp1) {
1848 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1849 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1871 Known2, TLO,
Depth + 1)) {
1899 if (DemandedOp0 || DemandedOp1) {
1900 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1901 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1964 if (
C->getAPIntValue() == Known2.
One) {
1973 if (!
C->isAllOnes() &&
DemandedBits.isSubsetOf(
C->getAPIntValue())) {
1985 if (ShiftC->getAPIntValue().ult(
BitWidth)) {
1986 uint64_t ShiftAmt = ShiftC->getZExtValue();
1989 : Ones.
lshr(ShiftAmt);
2006 if (!
C || !
C->isAllOnes())
2016 if (DemandedOp0 || DemandedOp1) {
2017 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
2018 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
2032 Known2, TLO,
Depth + 1))
2047 Known2, TLO,
Depth + 1))
2058 Known2, TLO,
Depth + 1))
2082 DemandedElts, KnownOp0, TLO,
Depth + 1))
2105 Known.Zero.setBitsFrom(1);
2113 if (std::optional<unsigned> KnownSA =
2115 unsigned ShAmt = *KnownSA;
2125 if (std::optional<unsigned> InnerSA =
2127 unsigned C1 = *InnerSA;
2129 int Diff = ShAmt -
C1;
2148 if (ShAmt < InnerBits &&
DemandedBits.getActiveBits() <= InnerBits &&
2166 InnerOp, DemandedElts,
Depth + 2)) {
2167 unsigned InnerShAmt = *SA2;
2168 if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
2170 (InnerBits - InnerShAmt + ShAmt) &&
2193 Known.Zero.setLowBits(ShAmt);
2198 Op0, InDemandedMask, DemandedElts, TLO.
DAG,
Depth + 1);
2209 Op.getNode()->hasOneUse()) {
2220 assert(DemandedSize <= SmallVTBits &&
2221 "Narrowed below demanded bits?");
2247 if (
bool IsNUW = (
Known.countMinLeadingZeros() >= HalfWidth)) {
2248 bool IsNSW =
Known.countMinSignBits() > HalfWidth;
2251 Flags.setNoUnsignedWrap(IsNUW);
2256 NewShiftAmt, Flags);
2282 if (std::optional<unsigned> MaxSA =
2284 unsigned ShAmt = *MaxSA;
2285 unsigned NumSignBits =
2288 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
2298 if (std::optional<unsigned> KnownSA =
2300 unsigned ShAmt = *KnownSA;
2310 if (std::optional<unsigned> InnerSA =
2312 unsigned C1 = *InnerSA;
2314 int Diff = ShAmt -
C1;
2330 if (std::optional<unsigned> InnerSA =
2332 unsigned C1 = *InnerSA;
2334 unsigned Combined = std::min(
C1 + ShAmt,
BitWidth - 1);
2346 if (
Op->getFlags().hasExact())
2377 Known.Zero.setHighBits(ShAmt);
2382 Op0, InDemandedMask, DemandedElts, TLO.
DAG,
Depth + 1);
2396 if (std::optional<unsigned> MaxSA =
2398 unsigned ShAmt = *MaxSA;
2402 unsigned NumSignBits =
2411 DemandedElts,
Depth + 1))
2414 DemandedElts,
Depth + 1))
2438 if (std::optional<unsigned> KnownSA =
2440 unsigned ShAmt = *KnownSA;
2447 if (std::optional<unsigned> InnerSA =
2449 unsigned LowBits =
BitWidth - ShAmt;
2454 if (*InnerSA == ShAmt) {
2464 unsigned NumSignBits =
2466 if (NumSignBits > ShAmt)
2476 if (
Op->getFlags().hasExact())
2508 Known.One.setHighBits(ShAmt);
2513 Op0, InDemandedMask, DemandedElts, TLO.
DAG,
Depth + 1);
2523 DemandedElts,
Depth + 1))
2526 DemandedElts,
Depth + 1))
2539 unsigned Amt = SA->getAPIntValue().urem(
BitWidth);
2561 Known2 <<= (IsFSHL ? Amt : (
BitWidth - Amt));
2569 Op0, Demanded0, DemandedElts, TLO.
DAG,
Depth + 1);
2571 Op1, Demanded1, DemandedElts, TLO.
DAG,
Depth + 1);
2572 if (DemandedOp0 || DemandedOp1) {
2573 DemandedOp0 = DemandedOp0 ? DemandedOp0 : Op0;
2574 DemandedOp1 = DemandedOp1 ? DemandedOp1 : Op1;
2590 unsigned MaxShiftAmt =
2622 unsigned Amt = SA->getAPIntValue().urem(
BitWidth);
2638 DemandedBits.countr_zero() >= (IsROTL ? Amt : RevAmt)) {
2643 DemandedBits.countl_zero() >= (IsROTL ? RevAmt : Amt)) {
2662 unsigned Opc =
Op.getOpcode();
2669 unsigned NumSignBits =
2673 if (NumSignBits >= NumDemandedUpperBits)
2739 unsigned ShiftAmount = NLZ > NTZ ? NLZ - NTZ : NTZ - NLZ;
2786 Known.One.clearAllBits();
2799 unsigned MinSignedBits =
2801 bool AlreadySignExtended = ExVTBits >= MinSignedBits;
2804 if (!AlreadySignExtended) {
2822 InputDemandedBits.
setBit(ExVTBits - 1);
2832 if (
Known.Zero[ExVTBits - 1])
2836 if (
Known.One[ExVTBits - 1]) {
2837 Known.One.setBitsFrom(ExVTBits);
2846 EVT HalfVT =
Op.getOperand(0).getValueType();
2869 EVT SrcVT = Src.getValueType();
2878 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2889 APInt InDemandedElts = DemandedElts.
zext(InElts);
2895 assert(
Known.getBitWidth() == InBits &&
"Src width has changed?");
2900 Src, InDemandedBits, InDemandedElts, TLO.
DAG,
Depth + 1))
2910 EVT SrcVT = Src.getValueType();
2915 APInt InDemandedElts = DemandedElts.
zext(InElts);
2920 InDemandedBits.
setBit(InBits - 1);
2926 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2944 assert(
Known.getBitWidth() == InBits &&
"Src width has changed?");
2950 if (
Known.isNonNegative()) {
2963 Src, InDemandedBits, InDemandedElts, TLO.
DAG,
Depth + 1))
2973 EVT SrcVT = Src.getValueType();
2980 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2985 APInt InDemandedElts = DemandedElts.
zext(InElts);
2989 assert(
Known.getBitWidth() == InBits &&
"Src width has changed?");
2994 Src, InDemandedBits, InDemandedElts, TLO.
DAG,
Depth + 1))
3003 unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
3016 Src, TruncMask, DemandedElts, TLO.
DAG,
Depth + 1))
3021 switch (Src.getOpcode()) {
3032 if (Src.getNode()->hasOneUse()) {
3044 std::optional<unsigned> ShAmtC =
3046 if (!ShAmtC || *ShAmtC >=
BitWidth)
3048 unsigned ShVal = *ShAmtC;
3078 Known.Zero |= ~InMask;
3085 ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
3086 unsigned EltBitWidth = Src.getScalarValueSizeInBits();
3095 if (CIdx->getAPIntValue().ult(NumSrcElts))
3102 DemandedSrcBits = DemandedSrcBits.
trunc(EltBitWidth);
3111 Src, DemandedSrcBits, DemandedSrcElts, TLO.
DAG,
Depth + 1)) {
3113 TLO.
DAG.
getNode(
Op.getOpcode(), dl, VT, DemandedSrc, Idx);
3127 EVT SrcVT = Src.getValueType();
3139 unsigned ShVal =
Op.getValueSizeInBits() - 1;
3149 unsigned Scale =
BitWidth / NumSrcEltBits;
3152 for (
unsigned i = 0; i != Scale; ++i) {
3153 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
3154 unsigned BitOffset = EltOffset * NumSrcEltBits;
3155 DemandedSrcBits |=
DemandedBits.extractBits(NumSrcEltBits, BitOffset);
3162 APInt KnownSrcUndef, KnownSrcZero;
3164 KnownSrcZero, TLO,
Depth + 1))
3169 KnownSrcBits, TLO,
Depth + 1))
3171 }
else if (IsLE && (NumSrcEltBits %
BitWidth) == 0) {
3173 unsigned Scale = NumSrcEltBits /
BitWidth;
3177 for (
unsigned i = 0; i != NumElts; ++i)
3178 if (DemandedElts[i]) {
3181 DemandedSrcElts.
setBit(i / Scale);
3185 APInt KnownSrcUndef, KnownSrcZero;
3187 KnownSrcZero, TLO,
Depth + 1))
3193 KnownSrcBits, TLO,
Depth + 1))
3199 Src, DemandedSrcBits, DemandedSrcElts, TLO.
DAG,
Depth + 1)) {
3221 if (
C &&
C->getAPIntValue().countr_zero() == CTZ) {
3237 if (
Op.getOperand(0).getValueType() !=
Op.getOperand(1).getValueType())
3245 SDValue Op0 =
Op.getOperand(0), Op1 =
Op.getOperand(1);
3250 auto GetDemandedBitsLHSMask = [&](
APInt Demanded,
3259 DemandedElts, KnownOp0, TLO,
Depth + 1) ||
3276 Op0, LoMask, DemandedElts, TLO.
DAG,
Depth + 1);
3278 Op1, LoMask, DemandedElts, TLO.
DAG,
Depth + 1);
3279 if (DemandedOp0 || DemandedOp1) {
3280 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
3281 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
3295 if (
C && !
C->isAllOnes() && !
C->isOne() &&
3296 (
C->getAPIntValue() | HighMask).isAllOnes()) {
3308 auto getShiftLeftAmt = [&HighMask](
SDValue Mul) ->
unsigned {
3335 if (
unsigned ShAmt = getShiftLeftAmt(Op0))
3338 if (
unsigned ShAmt = getShiftLeftAmt(Op1))
3339 return foldMul(
ISD::SUB, Op1.getOperand(0), Op0, ShAmt);
3343 if (
unsigned ShAmt = getShiftLeftAmt(Op1))
3344 return foldMul(
ISD::ADD, Op1.getOperand(0), Op0, ShAmt);
3352 Op.getOpcode() !=
ISD::SUB, Flags.hasNoSignedWrap(),
3353 Flags.hasNoUnsignedWrap(), KnownOp0, KnownOp1);
3368 if (
Known.isNonNegative())
3370 if (
Known.isNegative())
3374 Known.Zero |= SignMask;
3375 Known.One &= ~SignMask;
3406 Known.Zero &= ~SignMask0;
3407 Known.One &= ~SignMask0;
3421 if (!
Known.isSignUnknown()) {
3422 Known.Zero ^= SignMask;
3423 Known.One ^= SignMask;
3434 if (
Op.getValueType().isScalableVector())
3453 auto *C = dyn_cast<ConstantSDNode>(V);
3454 return C && C->isOpaque();
3468 if (HasMultiUse &&
Known.isUnknown() && !OriginalDemandedElts.
isAllOnes())
3475 const APInt &DemandedElts,
3481 APInt KnownUndef, KnownZero;
3495 const APInt &UndefOp0,
3496 const APInt &UndefOp1) {
3499 "Vector binop only");
3504 UndefOp1.
getBitWidth() == NumElts &&
"Bad type for undef analysis");
3506 auto getUndefOrConstantElt = [&](
SDValue V,
unsigned Index,
3507 const APInt &UndefVals) {
3508 if (UndefVals[Index])
3524 for (
unsigned i = 0; i != NumElts; ++i) {
3533 if (
C0 &&
C1 &&
C0.getValueType() == EltVT &&
C1.getValueType() == EltVT)
3543 bool AssumeSingleUse)
const {
3544 EVT VT =
Op.getValueType();
3545 unsigned Opcode =
Op.getOpcode();
3546 APInt DemandedElts = OriginalDemandedElts;
3560 "Mask size mismatches value type element count!");
3569 if (!AssumeSingleUse && !
Op.getNode()->hasOneUse())
3573 if (DemandedElts == 0) {
3591 assert(ShrunkSize % EltSizeInBits == 0 &&
3592 "Shrunk size not a multiple of element size");
3594 "Shrunk size must be < original vector size");
3596 "Shrunk size must be >= demanded size");
3611 auto SimplifyDemandedVectorEltsBinOp = [&](
SDValue Op0,
SDValue Op1) {
3616 if (NewOp0 || NewOp1) {
3619 NewOp1 ? NewOp1 : Op1,
Op->getFlags());
3623 if (TryShrinkBinOp(Op0, Op1))
3631 if (!DemandedElts[0])
3638 EVT SrcVT = Src.getValueType();
3645 for (
unsigned I = 0;
I != NumElts; ++
I) {
3646 if (DemandedElts[
I]) {
3647 unsigned Offset =
I * EltSize;
3660 if (NumSrcElts == NumElts)
3662 KnownZero, TLO,
Depth + 1);
3664 APInt SrcDemandedElts, SrcZero, SrcUndef;
3668 if ((NumElts % NumSrcElts) == 0) {
3669 unsigned Scale = NumElts / NumSrcElts;
3681 for (
unsigned i = 0; i != NumElts; ++i)
3682 if (DemandedElts[i]) {
3683 unsigned Ofs = (i % Scale) * EltSizeInBits;
3684 SrcDemandedBits.
setBits(Ofs, Ofs + EltSizeInBits);
3696 for (
unsigned SubElt = 0; SubElt != Scale; ++SubElt) {
3697 if (!
Known.Zero.extractBits(EltSizeInBits, SubElt * EltSizeInBits)
3700 for (
unsigned SrcElt = 0; SrcElt != NumSrcElts; ++SrcElt) {
3701 unsigned Elt = Scale * SrcElt + SubElt;
3704 if (DemandedElts[Elt] && !SrcUndef[SrcElt])
3712 for (
unsigned i = 0; i != NumSrcElts; ++i) {
3713 if (SrcDemandedElts[i]) {
3715 KnownZero.
setBits(i * Scale, (i + 1) * Scale);
3717 KnownUndef.
setBits(i * Scale, (i + 1) * Scale);
3725 if ((NumSrcElts % NumElts) == 0) {
3726 unsigned Scale = NumSrcElts / NumElts;
3734 for (
unsigned i = 0; i != NumElts; ++i) {
3735 if (DemandedElts[i]) {
3764 if (!IsPromotedLoad)
3775 [&](
SDValue Elt) { return Op.getOperand(0) != Elt; })) {
3777 bool Updated =
false;
3778 for (
unsigned i = 0; i != NumElts; ++i) {
3789 for (
unsigned i = 0; i != NumElts; ++i) {
3791 if (
SrcOp.isUndef()) {
3793 }
else if (EltSizeInBits ==
SrcOp.getScalarValueSizeInBits() &&
3801 EVT SubVT =
Op.getOperand(0).getValueType();
3802 unsigned NumSubVecs =
Op.getNumOperands();
3804 for (
unsigned i = 0; i != NumSubVecs; ++i) {
3807 APInt SubUndef, SubZero;
3811 KnownUndef.
insertBits(SubUndef, i * NumSubElts);
3812 KnownZero.
insertBits(SubZero, i * NumSubElts);
3817 bool FoundNewSub =
false;
3819 for (
unsigned i = 0; i != NumSubVecs; ++i) {
3823 SubOp, SubElts, TLO.
DAG,
Depth + 1);
3824 DemandedSubOps.
push_back(NewSubOp ? NewSubOp : SubOp);
3825 FoundNewSub = NewSubOp ?
true : FoundNewSub;
3840 uint64_t Idx =
Op.getConstantOperandVal(2);
3841 unsigned NumSubElts =
Sub.getValueType().getVectorNumElements();
3843 APInt DemandedSrcElts = DemandedElts;
3844 DemandedSrcElts.
clearBits(Idx, Idx + NumSubElts);
3847 if (!DemandedSubElts)
3850 APInt SubUndef, SubZero;
3856 if (!DemandedSrcElts && !Src.isUndef())
3870 Src, DemandedSrcElts, TLO.
DAG,
Depth + 1);
3873 if (NewSrc || NewSub) {
3874 NewSrc = NewSrc ? NewSrc : Src;
3875 NewSub = NewSub ? NewSub :
Sub;
3877 NewSub,
Op.getOperand(2));
3886 if (Src.getValueType().isScalableVector())
3888 uint64_t Idx =
Op.getConstantOperandVal(1);
3889 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3890 APInt DemandedSrcElts = DemandedElts.
zext(NumSrcElts).
shl(Idx);
3892 APInt SrcUndef, SrcZero;
3916 if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
3917 unsigned Idx = CIdx->getZExtValue();
3918 if (!DemandedElts[Idx])
3921 APInt DemandedVecElts(DemandedElts);
3924 KnownZero, TLO,
Depth + 1))
3933 APInt VecUndef, VecZero;
3947 APInt UndefSel, ZeroSel;
3953 APInt DemandedLHS(DemandedElts);
3954 APInt DemandedRHS(DemandedElts);
3955 APInt UndefLHS, ZeroLHS;
3956 APInt UndefRHS, ZeroRHS;
3964 KnownUndef = UndefLHS & UndefRHS;
3965 KnownZero = ZeroLHS & ZeroRHS;
3969 APInt DemandedSel = DemandedElts & ~KnownZero;
3970 if (DemandedSel != DemandedElts)
3983 APInt DemandedLHS(NumElts, 0);
3984 APInt DemandedRHS(NumElts, 0);
3985 for (
unsigned i = 0; i != NumElts; ++i) {
3986 int M = ShuffleMask[i];
3987 if (M < 0 || !DemandedElts[i])
3989 assert(0 <= M && M < (
int)(2 * NumElts) &&
"Shuffle index out of range");
3990 if (M < (
int)NumElts)
3993 DemandedRHS.
setBit(M - NumElts);
3999 bool FoldLHS = !DemandedLHS && !LHS.isUndef();
4000 bool FoldRHS = !DemandedRHS && !RHS.isUndef();
4001 if (FoldLHS || FoldRHS) {
4002 LHS = FoldLHS ? TLO.
DAG.
getUNDEF(LHS.getValueType()) : LHS;
4003 RHS = FoldRHS ? TLO.
DAG.
getUNDEF(RHS.getValueType()) : RHS;
4010 APInt UndefLHS, ZeroLHS;
4011 APInt UndefRHS, ZeroRHS;
4020 bool Updated =
false;
4021 bool IdentityLHS =
true, IdentityRHS =
true;
4023 for (
unsigned i = 0; i != NumElts; ++i) {
4024 int &M = NewMask[i];
4027 if (!DemandedElts[i] || (M < (
int)NumElts && UndefLHS[M]) ||
4028 (M >= (
int)NumElts && UndefRHS[M - NumElts])) {
4032 IdentityLHS &= (M < 0) || (M == (
int)i);
4033 IdentityRHS &= (M < 0) || ((M - NumElts) == i);
4038 if (Updated && !IdentityLHS && !IdentityRHS && !TLO.
LegalOps) {
4046 for (
unsigned i = 0; i != NumElts; ++i) {
4047 int M = ShuffleMask[i];
4050 }
else if (M < (
int)NumElts) {
4056 if (UndefRHS[M - NumElts])
4058 if (ZeroRHS[M - NumElts])
4067 APInt SrcUndef, SrcZero;
4069 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
4070 APInt DemandedSrcElts = DemandedElts.
zext(NumSrcElts);
4078 Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
4079 DemandedSrcElts == 1) {
4092 if (IsLE && DemandedSrcElts == 1 && Src.getOpcode() ==
ISD::AND &&
4093 Op->isOnlyUserOf(Src.getNode()) &&
4094 Op.getValueSizeInBits() == Src.getValueSizeInBits()) {
4096 EVT SrcVT = Src.getValueType();
4110 ISD::AND,
DL, SrcVT, {Src.getOperand(1), Mask})) {
4124 if (Op0 == Op1 &&
Op->isOnlyUserOf(Op0.
getNode())) {
4125 APInt UndefLHS, ZeroLHS;
4151 APInt UndefRHS, ZeroRHS;
4155 APInt UndefLHS, ZeroLHS;
4162 KnownZero = ZeroLHS & ZeroRHS;
4168 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
4180 APInt UndefRHS, ZeroRHS;
4184 APInt UndefLHS, ZeroLHS;
4189 KnownZero = ZeroLHS & ~UndefRHS;
4190 KnownUndef = UndefLHS & UndefRHS;
4195 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
4206 APInt SrcUndef, SrcZero;
4220 KnownUndef &= DemandedElts;
4221 KnownZero &= DemandedElts;
4226 if (DemandedElts.
isSubsetOf(SrcZero | KnownZero | SrcUndef | KnownUndef))
4233 KnownZero |= SrcZero;
4234 KnownUndef &= SrcUndef;
4235 KnownUndef &= ~KnownZero;
4239 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
4247 KnownZero, TLO,
Depth + 1))
4252 Op.getOperand(0), DemandedElts, TLO.
DAG,
Depth + 1))
4267 KnownZero, TLO,
Depth + 1))
4274 KnownZero, TLO,
Depth))
4280 TLO,
Depth, AssumeSingleUse))
4287 assert((KnownUndef & KnownZero) == 0 &&
"Elements flagged as undef AND zero");
4301 const APInt &DemandedElts,
4303 unsigned Depth)
const {
4308 "Should use MaskedValueIsZero if you don't know whether Op"
4309 " is a target node!");
4316 unsigned Depth)
const {
4323 unsigned Depth)
const {
4336 Align Alignment)
const {
4347 unsigned NumZeroBits =
Known.countMinLeadingZeros();
4357 unsigned Depth)
const {
4366 unsigned Depth)
const {
4371 "Should use ComputeNumSignBits if you don't know whether Op"
4372 " is a target node!");
4389 "Should use SimplifyDemandedVectorElts if you don't know whether Op"
4390 " is a target node!");
4401 "Should use SimplifyDemandedBits if you don't know whether Op"
4402 " is a target node!");
4415 "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
4416 " is a target node!");
4449 "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
4450 " is a target node!");
4457 return DAG.isGuaranteedNotToBeUndefOrPoison(V, Kind, Depth + 1);
4468 "Should use canCreateUndefOrPoison if you don't know whether Op"
4469 " is a target node!");
4476 const APInt &DemandedElts,
4478 unsigned Depth)
const {
4483 "Should use computeKnownFPClass if you don't know whether Op"
4484 " is a target node!");
4488 const APInt &DemandedElts,
4491 unsigned Depth)
const {
4496 "Should use isKnownNeverNaN if you don't know whether Op"
4497 " is a target node!");
4502 const APInt &DemandedElts,
4505 unsigned Depth)
const {
4510 "Should use isSplatValue if you don't know whether Op"
4511 " is a target node!");
4526 CVal = CN->getAPIntValue();
4527 EltWidth =
N.getValueType().getScalarSizeInBits();
4534 CVal = CVal.
trunc(EltWidth);
4540 return CVal.
isOne();
4582 return (
N->isOne() && !SExt) || (SExt && (
N->getValueType(0) != MVT::i1));
4585 return N->isAllOnes() && SExt;
4594 DAGCombinerInfo &DCI)
const {
4623 if (AndC &&
isNullConstant(N1) && AndC->getAPIntValue().isPowerOf2() &&
4626 AndC->getAPIntValue().getActiveBits());
4653 if (isXAndYEqZeroPreferableToXAndYEqY(
Cond, OpVT) &&
4661 if (DCI.isBeforeLegalizeOps() ||
4690 DAGCombinerInfo &DCI)
const {
4694 SelectionDAG &DAG = DCI.DAG;
4731SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
4733 const SDLoc &
DL)
const {
4744 ConstantSDNode *C01;
4753 APInt
I1 =
C1->getAPIntValue();
4773 auto checkConstants = [&
I1, &I01]() ->
bool {
4778 if (checkConstants()) {
4786 if (!checkConstants())
4792 const unsigned KeptBits =
I1.logBase2();
4793 const unsigned KeptBitsMinusOne = I01.
logBase2();
4796 if (KeptBits != (KeptBitsMinusOne + 1))
4801 SelectionDAG &DAG = DCI.DAG;
4810 return DAG.
getSetCC(
DL, SCCVT, SExtInReg,
X, NewCond);
4814SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
4816 DAGCombinerInfo &DCI,
const SDLoc &
DL)
const {
4818 "Should be a comparison with 0.");
4820 "Valid only for [in]equality comparisons.");
4822 unsigned NewShiftOpcode;
4825 SelectionDAG &DAG = DCI.DAG;
4828 auto Match = [&NewShiftOpcode, &
X, &
C, &
Y, &DAG,
this](
SDValue V) {
4832 unsigned OldShiftOpcode =
V.getOpcode();
4833 switch (OldShiftOpcode) {
4845 C =
V.getOperand(0);
4846 ConstantSDNode *
CC =
4850 Y =
V.getOperand(1);
4852 ConstantSDNode *XC =
4855 X, XC,
CC,
Y, OldShiftOpcode, NewShiftOpcode, DAG);
4872 EVT VT =
X.getValueType();
4887 DAGCombinerInfo &DCI)
const {
4890 "Unexpected binop");
4896 SelectionDAG &DAG = DCI.DAG;
4918 if (!DCI.isCalledByLegalizer())
4919 DCI.AddToWorklist(YShl1.
getNode());
4934 if (CTPOP.getOpcode() !=
ISD::CTPOP || !CTPOP.hasOneUse())
4937 EVT CTVT = CTPOP.getValueType();
4938 SDValue CTOp = CTPOP.getOperand(0);
4948 unsigned CostLimit =
TLI.getCustomCtpopCost(CTVT,
Cond);
4958 for (
unsigned i = 0; i <
Passes; i++) {
4969 if (
TLI.isCtpopFast(CTVT))
5004 if (!
C1 || !(
C1->isZero() ||
C1->isAllOnes()))
5007 auto getRotateSource = [](
SDValue X) {
5009 return X.getOperand(0);
5016 if (
SDValue R = getRotateSource(N0))
5049 if (!
C1 || !
C1->isZero())
5074 if (
Or.getOperand(0) ==
Other) {
5075 X =
Or.getOperand(0);
5076 Y =
Or.getOperand(1);
5079 if (
Or.getOperand(1) ==
Other) {
5080 X =
Or.getOperand(1);
5081 Y =
Or.getOperand(0);
5091 if (matchOr(F0, F1)) {
5098 if (matchOr(F1, F0)) {
5114 const SDLoc &dl)
const {
5124 bool N0ConstOrSplat =
5126 bool N1ConstOrSplat =
5134 if (N0ConstOrSplat && !N1ConstOrSplat &&
5137 return DAG.
getSetCC(dl, VT, N1, N0, SwappedCC);
5143 if (!N0ConstOrSplat && !N1ConstOrSplat &&
5148 return DAG.
getSetCC(dl, VT, N1, N0, SwappedCC);
5157 const APInt &
C1 = N1C->getAPIntValue();
5173 !Attr.hasFnAttr(Attribute::MinSize)) {
5177 return DAG.
getNode(LogicOp, dl, VT, IsXZero, IsYZero);
5223 const APInt &
C1 = N1C->getAPIntValue();
5239 if ((
C->getAPIntValue()+1).isPowerOf2()) {
5240 MinBits =
C->getAPIntValue().countr_one();
5251 MinBits = LN0->getMemoryVT().getSizeInBits();
5255 MinBits = LN0->getMemoryVT().getSizeInBits();
5261 unsigned ReqdBits =
Signed ?
C1.getSignificantBits() :
C1.getActiveBits();
5265 MinBits <
C1.getBitWidth() &&
5266 MinBits >= ReqdBits) {
5271 if (MinBits == 1 &&
C1 == 1)
5290 if (TopSetCC.
getValueType() == MVT::i1 && VT == MVT::i1 &&
5323 unsigned bestWidth = 0, bestOffset = 0;
5324 if (Lod->isSimple() && Lod->isUnindexed() &&
5325 (Lod->getMemoryVT().isByteSized() ||
5327 unsigned memWidth = Lod->getMemoryVT().getStoreSizeInBits();
5329 unsigned maskWidth = origWidth;
5333 origWidth = Lod->getMemoryVT().getSizeInBits();
5337 for (
unsigned width = 8; width < origWidth; width *= 2) {
5342 unsigned maxOffset = origWidth - width;
5343 for (
unsigned offset = 0; offset <= maxOffset; offset += 8) {
5344 if (Mask.isSubsetOf(newMask)) {
5345 unsigned ptrOffset =
5347 unsigned IsFast = 0;
5348 assert((ptrOffset % 8) == 0 &&
"Non-Bytealigned pointer offset");
5353 *DAG.
getContext(), Layout, newVT, Lod->getAddressSpace(),
5354 NewAlign, Lod->getMemOperand()->getFlags(), &IsFast) &&
5356 bestOffset = ptrOffset / 8;
5357 bestMask = Mask.lshr(offset);
5370 SDValue Ptr = Lod->getBasePtr();
5371 if (bestOffset != 0)
5374 DAG.
getLoad(newVT, dl, Lod->getChain(), Ptr,
5375 Lod->getPointerInfo().getWithOffset(bestOffset),
5377 Lod->getMMOMetadataForSubAccess());
5392 C1.getBitWidth() - InSize))) {
5452 if (
C1.getSignificantBits() > ExtSrcTyBits)
5456 ExtDstTy != ExtSrcTy &&
"Unexpected types!");
5463 return DAG.
getSetCC(dl, VT, ZextOp,
5465 }
else if ((N1C->isZero() || N1C->isOne()) &&
5512 return DAG.
getSetCC(dl, VT, Val, N1,
5515 }
else if (N1C->isOne()) {
5598 optimizeSetCCOfSignedTruncationCheck(VT, N0, N1,
Cond, DCI, dl))
5605 const APInt &
C1 = N1C->getAPIntValue();
5607 APInt MinVal, MaxVal;
5629 (!N1C->isOpaque() || (
C.getBitWidth() <= 64 &&
5649 (!N1C->isOpaque() || (
C.getBitWidth() <= 64 &&
5697 if (
SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
5698 VT, N0, N1,
Cond, DCI, dl))
5705 bool CmpZero = N1C->isZero();
5706 bool CmpNegOne = N1C->isAllOnes();
5707 if ((CmpZero || CmpNegOne) && N0.
hasOneUse()) {
5710 unsigned EltBits = V.getScalarValueSizeInBits();
5711 if (V.getOpcode() !=
ISD::OR || (EltBits % 2) != 0)
5719 RHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
5722 Hi = RHS.getOperand(0);
5727 LHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
5730 Hi = LHS.getOperand(0);
5738 unsigned HalfBits = EltBits / 2;
5749 if (IsConcat(N0,
Lo,
Hi))
5750 return MergeConcat(
Lo,
Hi);
5788 const APInt &
C1 = N1C->getAPIntValue();
5803 unsigned ShCt = AndRHS->getAPIntValue().logBase2();
5804 if (AndRHS->getAPIntValue().isPowerOf2() &&
5814 unsigned ShCt =
C1.logBase2();
5825 if (
C1.getSignificantBits() <= 64 &&
5831 const APInt &AndRHSC = AndRHS->getAPIntValue();
5837 APInt NewC =
C1.lshr(ShiftBits);
5840 APInt SignedC =
C1.ashr(ShiftBits);
5869 ShiftBits =
C1.countr_one();
5873 ShiftBits =
C1.countr_zero();
5875 APInt RangeWidth = NewC;
5884 const APInt &AddVal = AddC->getAPIntValue();
5886 APInt RangeLower = -AddVal;
5888 (void)RangeLower.
uadd_ov(RangeWidth, Overflow);
5889 if (!RangeWidth.
isZero() && !Overflow) {
5898 return DAG.
getSetCC(dl, VT, ShiftedAdd, CmpRHS, NewCond);
5907 return DAG.
getSetCC(dl, VT, Shift, CmpRHS, NewCond);
5915 assert(!CFP->getValueAPF().isNaN() &&
"Unexpected NaN value");
5936 !
isFPImmLegal(CFP->getValueAPF(), CFP->getValueType(0))) {
5955 if (CFP->getValueAPF().isInfinity()) {
5956 bool IsNegInf = CFP->getValueAPF().isNegative();
5967 return DAG.
getSetCC(dl, VT, N0, N1, NewCond);
5976 "Integer types should be handled by FoldSetCC");
5982 if (UOF ==
unsigned(EqTrue))
5987 if (NewCond !=
Cond &&
5990 return DAG.
getSetCC(dl, VT, N0, N1, NewCond);
5997 if ((isSignedIntSetCC(
Cond) || isUnsignedIntSetCC(
Cond)) &&
6034 bool LegalRHSImm =
false;
6042 DAG.
getConstant(RHSC->getAPIntValue() - LHSR->getAPIntValue(),
6050 DAG.
getConstant(LHSR->getAPIntValue() ^ RHSC->getAPIntValue(),
6060 DAG.
getConstant(SUBC->getAPIntValue() - RHSC->getAPIntValue(),
6065 if (RHSC->getValueType(0).getSizeInBits() <= 64)
6074 if (
SDValue V = foldSetCCWithBinOp(VT, N0, N1,
Cond, dl, DCI))
6080 if (
SDValue V = foldSetCCWithBinOp(VT, N1, N0,
Cond, dl, DCI))
6083 if (
SDValue V = foldSetCCWithAnd(VT, N0, N1,
Cond, dl, DCI))
6086 if (
SDValue V = foldSetCCWithOr(VT, N0, N1,
Cond, dl, DCI))
6095 if (!
isIntDivCheap(VT, Attr) && !Attr.hasFnAttr(Attribute::MinSize)) {
6113 N0 = DAG.
getNOT(dl, Temp, OpVT);
6122 Temp = DAG.
getNOT(dl, N0, OpVT);
6129 Temp = DAG.
getNOT(dl, N1, OpVT);
6136 Temp = DAG.
getNOT(dl, N0, OpVT);
6143 Temp = DAG.
getNOT(dl, N1, OpVT);
6152 N0 = DAG.
getNode(ExtendCode, dl, VT, N0);
6170 if (VT == OldCCVT) {
6198 GA = GASD->getGlobal();
6199 Offset += GASD->getOffset();
6203 if (
N->isAnyAdd()) {
6208 Offset += V->getSExtValue();
6213 Offset += V->getSExtValue();
6234 unsigned S = Constraint.
size();
6237 switch (Constraint[0]) {
6268 if (S > 1 && Constraint[0] ==
'{' && Constraint[S - 1] ==
'}') {
6269 if (S == 8 && Constraint.
substr(1, 6) ==
"memory")
6297 std::vector<SDValue> &
Ops,
6300 if (Constraint.
size() > 1)
6303 char ConstraintLetter = Constraint[0];
6304 switch (ConstraintLetter) {
6324 bool IsBool =
C->getConstantIntValue()->getBitWidth() == 1;
6334 if (ConstraintLetter !=
'n') {
6337 GA->getValueType(0),
6338 Offset + GA->getOffset()));
6343 BA->getBlockAddress(), BA->getValueType(0),
6344 Offset + BA->getOffset(), BA->getTargetFlags()));
6352 const unsigned OpCode =
Op.getOpcode();
6355 Op =
Op.getOperand(1);
6359 Op =
Op.getOperand(0);
6376std::pair<unsigned, const TargetRegisterClass *>
6382 assert(*(Constraint.
end() - 1) ==
'}' &&
"Not a brace enclosed constraint?");
6387 std::pair<unsigned, const TargetRegisterClass *> R =
6399 std::pair<unsigned, const TargetRegisterClass *> S =
6400 std::make_pair(PR, &RC);
6445 unsigned maCount = 0;
6451 unsigned LabelNo = 0;
6454 ConstraintOperands.emplace_back(std::move(CI));
6458 if (OpInfo.multipleAlternatives.size() > maCount)
6459 maCount = OpInfo.multipleAlternatives.size();
6461 OpInfo.ConstraintVT = MVT::Other;
6464 switch (OpInfo.Type) {
6467 if (OpInfo.isIndirect) {
6468 OpInfo.CallOperandVal =
Call.getArgOperand(ArgNo);
6474 assert(!
Call.getType()->isVoidTy() &&
"Bad inline asm!");
6479 assert(ResNo == 0 &&
"Asm only has one result!");
6487 OpInfo.CallOperandVal =
Call.getArgOperand(ArgNo);
6498 if (OpInfo.CallOperandVal) {
6500 if (OpInfo.isIndirect) {
6501 OpTy =
Call.getParamElementType(ArgNo);
6502 assert(
OpTy &&
"Indirect operand must have elementtype attribute");
6507 if (STy->getNumElements() == 1)
6508 OpTy = STy->getElementType(0);
6512 if (!
OpTy->isSingleValueType() &&
OpTy->isSized()) {
6513 unsigned BitSize =
DL.getTypeSizeInBits(
OpTy);
6534 if (!ConstraintOperands.empty()) {
6536 unsigned bestMAIndex = 0;
6537 int bestWeight = -1;
6543 for (maIndex = 0; maIndex < maCount; ++maIndex) {
6545 for (
unsigned cIndex = 0, eIndex = ConstraintOperands.size();
6546 cIndex != eIndex; ++cIndex) {
6555 if (OpInfo.hasMatchingInput()) {
6557 if (OpInfo.ConstraintVT !=
Input.ConstraintVT) {
6558 if ((OpInfo.ConstraintVT.isInteger() !=
6559 Input.ConstraintVT.isInteger()) ||
6560 (OpInfo.ConstraintVT.getSizeInBits() !=
6561 Input.ConstraintVT.getSizeInBits())) {
6572 weightSum += weight;
6575 if (weightSum > bestWeight) {
6576 bestWeight = weightSum;
6577 bestMAIndex = maIndex;
6584 cInfo.selectAlternative(bestMAIndex);
6589 for (
unsigned cIndex = 0, eIndex = ConstraintOperands.size();
6590 cIndex != eIndex; ++cIndex) {
6597 if (OpInfo.hasMatchingInput()) {
6600 if (OpInfo.ConstraintVT !=
Input.ConstraintVT) {
6601 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
6603 OpInfo.ConstraintVT);
6604 std::pair<unsigned, const TargetRegisterClass *> InputRC =
6606 Input.ConstraintVT);
6607 const bool OutOpIsIntOrFP = OpInfo.ConstraintVT.isInteger() ||
6608 OpInfo.ConstraintVT.isFloatingPoint();
6609 const bool InOpIsIntOrFP =
Input.ConstraintVT.isInteger() ||
6610 Input.ConstraintVT.isFloatingPoint();
6611 if ((OutOpIsIntOrFP != InOpIsIntOrFP) ||
6612 (MatchRC.second != InputRC.second)) {
6614 " with a matching output constraint of"
6615 " incompatible type!");
6621 return ConstraintOperands;
6656 if (maIndex >= (
int)
info.multipleAlternatives.size())
6657 rCodes = &
info.Codes;
6659 rCodes = &
info.multipleAlternatives[maIndex].Codes;
6663 for (
const std::string &rCode : *rCodes) {
6666 if (weight > BestWeight)
6667 BestWeight = weight;
6680 Value *CallOperandVal =
info.CallOperandVal;
6683 if (!CallOperandVal)
6686 switch (*constraint) {
6750 Ret.
reserve(OpInfo.Codes.size());
6783 "need immediate or other");
6788 std::vector<SDValue> ResultOps;
6789 TLI.LowerAsmOperandForConstraint(
Op,
P.first, ResultOps, *DAG);
6790 return !ResultOps.empty();
6798 assert(!OpInfo.Codes.empty() &&
"Must have at least one constraint");
6801 if (OpInfo.Codes.size() == 1) {
6802 OpInfo.ConstraintCode = OpInfo.Codes[0];
6809 unsigned BestIdx = 0;
6810 for (
const unsigned E =
G.size();
6817 if (BestIdx + 1 == E) {
6823 OpInfo.ConstraintCode =
G[BestIdx].first;
6824 OpInfo.ConstraintType =
G[BestIdx].second;
6828 if (OpInfo.ConstraintCode ==
"X" && OpInfo.CallOperandVal) {
6832 Value *v = OpInfo.CallOperandVal;
6838 OpInfo.ConstraintCode =
"i";
6845 OpInfo.ConstraintCode = Repl;
6859 EVT VT =
N->getValueType(0);
6863 bool UseSRA =
false;
6870 EVT CT =
C->getValueType(0);
6871 APInt Divisor =
C->getAPIntValue();
6893 "Expected matchUnaryPredicate to return one element for scalable "
6900 Factor = Factors[0];
6918 EVT VT =
N->getValueType(0);
6922 bool UseSRL =
false;
6929 EVT CT =
C->getValueType(0);
6930 APInt Divisor =
C->getAPIntValue();
6955 "Expected matchUnaryPredicate to return one element for scalable "
6962 Factor = Factors[0];
7005 EVT VT =
N->getValueType(0);
7041 bool IsAfterLegalization,
7042 bool IsAfterLegalTypes,
7047 if (
N->getFlags().hasExact())
7050 EVT VT =
N->getValueType(0);
7089 if (
isTypeLegal(VT) && !HasMULHS && !HasSMUL_LOHI && MulVT ==
EVT()) {
7101 if (!HasMULHS && !HasSMUL_LOHI && MulVT ==
EVT())
7107 if (IsAfterLegalTypes && VT.
isVector()) {
7124 APInt Divisor =
C->getAPIntValue().trunc(EltBits);
7126 int NumeratorFactor = 0;
7137 NumeratorFactor = 1;
7140 NumeratorFactor = -1;
7159 SDValue MagicFactor, Factor, Shift, ShiftMask;
7167 Shifts.
size() == 1 && ShiftMasks.
size() == 1 &&
7168 "Expected matchUnaryPredicate to return one element for scalable "
7176 MagicFactor = MagicFactors[0];
7177 Factor = Factors[0];
7179 ShiftMask = ShiftMasks[0];
7200 SDValue Q = GetMULHS(N0, MagicFactor);
7230 bool IsAfterLegalization,
7231 bool IsAfterLegalTypes,
7236 if (
N->getFlags().hasExact())
7239 EVT VT =
N->getValueType(0);
7278 if (
isTypeLegal(VT) && !HasMULHU && !HasUMUL_LOHI && MulVT ==
EVT()) {
7290 if (!HasMULHU && !HasUMUL_LOHI && MulVT ==
EVT())
7303 if (IsAfterLegalTypes && VT.
isVector()) {
7315 const EVT WideSVT = MVT::i64;
7316 const bool HasWideMULHU =
7319 const bool HasWideUMUL_LOHI =
7322 const bool AllowWiden = (HasWideMULHU || HasWideUMUL_LOHI);
7328 const bool AllowEvenToWiden = AllowWiden &&
isZExtFree(VT, WideSVT);
7330 bool UseNPQ =
false, UsePreShift =
false, UsePostShift =
false;
7331 bool UseWiden =
false;
7339 APInt Divisor =
C->getAPIntValue().trunc(EltBits);
7341 SDValue PreShift, MagicFactor, NPQFactor, PostShift;
7345 if (Divisor.
isOne()) {
7346 PreShift = PostShift = DAG.
getUNDEF(ShSVT);
7347 MagicFactor = NPQFactor = DAG.
getUNDEF(SVT);
7351 Divisor, std::min(KnownLeadingZeros, Divisor.
countl_zero()),
7363 "We shouldn't generate an undefined shift!");
7365 "We shouldn't generate an undefined shift!");
7367 "Unexpected pre-shift");
7374 UseNPQ |= magics.
IsAdd;
7375 UsePreShift |= magics.
PreShift != 0;
7391 SDValue PreShift, PostShift, MagicFactor, NPQFactor;
7399 NPQFactors.
size() == 1 && PostShifts.
size() == 1 &&
7400 "Expected matchUnaryPredicate to return one for scalable vectors");
7407 PreShift = PreShifts[0];
7408 MagicFactor = MagicFactors[0];
7409 PostShift = PostShifts[0];
7422 assert(HasWideUMUL_LOHI);
7425 WideN0, MagicFactor);
7457 Q = GetMULHU(Q, MagicFactor);
7470 NPQ = GetMULHU(NPQ, NPQFactor);
7489 return DAG.
getSelect(dl, VT, IsOne, N0, Q);
7503 if (SplatValue !=
Values.end()) {
7508 Replacement = *SplatValue;
7512 if (!AlternativeReplacement)
7515 Replacement = AlternativeReplacement;
7525SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT,
SDValue REMNode,
7528 DAGCombinerInfo &DCI,
7529 const SDLoc &
DL)
const {
7533 for (SDNode *
N : Built)
7534 DCI.AddToWorklist(
N);
7542TargetLowering::prepareUREMEqFold(EVT SETCCVT,
SDValue REMNode,
7544 DAGCombinerInfo &DCI,
const SDLoc &
DL,
7545 SmallVectorImpl<SDNode *> &Created)
const {
7553 "Only applicable for (in)equality comparisons.");
7555 SelectionDAG &DAG = DCI.DAG;
7566 bool ComparingWithAllZeros =
true;
7567 bool AllComparisonsWithNonZerosAreTautological =
true;
7568 bool HadTautologicalLanes =
false;
7569 bool AllLanesAreTautological =
true;
7570 bool HadEvenDivisor =
false;
7571 bool AllDivisorsArePowerOfTwo =
true;
7572 bool HadTautologicalInvertedLanes =
false;
7575 auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {
7581 const APInt &
Cmp = CCmp->getAPIntValue();
7583 ComparingWithAllZeros &=
Cmp.isZero();
7589 bool TautologicalInvertedLane =
D.ule(Cmp);
7590 HadTautologicalInvertedLanes |= TautologicalInvertedLane;
7595 bool TautologicalLane =
D.isOne() || TautologicalInvertedLane;
7596 HadTautologicalLanes |= TautologicalLane;
7597 AllLanesAreTautological &= TautologicalLane;
7603 AllComparisonsWithNonZerosAreTautological &= TautologicalLane;
7606 unsigned K =
D.countr_zero();
7607 assert((!
D.isOne() || (K == 0)) &&
"For divisor '1' we won't rotate.");
7608 APInt D0 =
D.lshr(K);
7611 HadEvenDivisor |= (
K != 0);
7614 AllDivisorsArePowerOfTwo &= D0.
isOne();
7618 unsigned W =
D.getBitWidth();
7620 assert((D0 *
P).isOne() &&
"Multiplicative inverse basic check failed.");
7633 "We are expecting that K is always less than all-ones for ShSVT");
7636 if (TautologicalLane) {
7660 if (AllLanesAreTautological)
7665 if (AllDivisorsArePowerOfTwo)
7670 if (HadTautologicalLanes) {
7685 "Expected matchBinaryPredicate to return one element for "
7696 if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) {
7700 "Expecting that the types on LHS and RHS of comparisons match.");
7710 if (HadEvenDivisor) {
7723 if (!HadTautologicalInvertedLanes)
7729 assert(VT.
isVector() &&
"Can/should only get here for vectors.");
7736 SDValue TautologicalInvertedChannels =
7746 DL, SETCCVT, SETCCVT);
7748 Replacement, NewCC);
7756 TautologicalInvertedChannels);
7766SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT,
SDValue REMNode,
7769 DAGCombinerInfo &DCI,
7770 const SDLoc &
DL)
const {
7774 assert(Built.
size() <= 7 &&
"Max size prediction failed.");
7775 for (SDNode *
N : Built)
7776 DCI.AddToWorklist(
N);
7784TargetLowering::prepareSREMEqFold(EVT SETCCVT,
SDValue REMNode,
7786 DAGCombinerInfo &DCI,
const SDLoc &
DL,
7787 SmallVectorImpl<SDNode *> &Created)
const {
7811 "Only applicable for (in)equality comparisons.");
7813 SelectionDAG &DAG = DCI.DAG;
7827 if (!CompTarget || !CompTarget->
isZero())
7830 bool HadOneDivisor =
false;
7831 bool AllDivisorsAreOnes =
true;
7832 bool HadEvenDivisor =
false;
7833 bool AllDivisorsArePowerOfTwo =
true;
7836 auto BuildSREMPattern = [&](ConstantSDNode *
C) {
7845 APInt
D =
C->getAPIntValue().abs();
7848 HadOneDivisor |=
D.isOne();
7849 AllDivisorsAreOnes &=
D.isOne();
7852 unsigned K =
D.countr_zero();
7853 assert((!
D.isOne() || (K == 0)) &&
"For divisor '1' we won't rotate.");
7854 APInt D0 =
D.
lshr(K);
7857 HadEvenDivisor |= (
K != 0);
7861 AllDivisorsArePowerOfTwo &= D0.
isOne();
7865 unsigned W =
D.getBitWidth();
7867 assert((D0 *
P).isOne() &&
"Multiplicative inverse basic check failed.");
7877 "We are expecting that A is always less than all-ones for SVT");
7879 "We are expecting that K is always less than all-ones for ShSVT");
7916 if (AllDivisorsAreOnes)
7921 if (AllDivisorsArePowerOfTwo)
7924 SDValue PVal, AVal, KVal, QVal;
7926 if (HadOneDivisor) {
7946 QAmts.
size() == 1 &&
7947 "Expected matchUnaryPredicate to return one element for scalable "
7975 if (HadEvenDivisor) {
7993 EVT VT =
Op.getValueType();
8018 bool LegalOps,
bool OptForSize,
8020 unsigned Depth)
const {
8024 return Op.getOperand(0);
8034 EVT VT =
Op.getValueType();
8035 unsigned Opcode =
Op.getOpcode();
8045 auto RemoveDeadNode = [&](
SDValue N) {
8046 if (
N &&
N.getNode()->use_empty())
8055 std::list<HandleSDNode> Handles;
8066 if (LegalOps && !IsOpLegal)
8095 return !N.isUndef() && !isa<ConstantFPSDNode>(N);
8103 return N.isUndef() ||
8104 isFPImmLegal(neg(cast<ConstantFPSDNode>(N)->getValueAPF()), VT,
8108 if (LegalOps && !IsOpLegal)
8125 if (!Flags.hasNoSignedZeros())
8139 Handles.emplace_back(NegX);
8150 if (NegX && (CostX <= CostY)) {
8154 RemoveDeadNode(NegY);
8163 RemoveDeadNode(NegX);
8170 if (!Flags.hasNoSignedZeros())
8195 Handles.emplace_back(NegX);
8206 if (NegX && (CostX <= CostY)) {
8210 RemoveDeadNode(NegY);
8216 if (
C->isExactlyValue(2.0) &&
Op.getOpcode() ==
ISD::FMUL)
8224 RemoveDeadNode(NegX);
8232 if (!Flags.hasNoSignedZeros())
8235 SDValue X =
Op.getOperand(0),
Y =
Op.getOperand(1), Z =
Op.getOperand(2);
8244 Handles.emplace_back(NegZ);
8252 Handles.emplace_back(NegX);
8263 if (NegX && (CostX <= CostY)) {
8264 Cost = std::min(CostX, CostZ);
8267 RemoveDeadNode(NegY);
8273 Cost = std::min(CostY, CostZ);
8276 RemoveDeadNode(NegX);
8286 return DAG.
getNode(Opcode,
DL, VT, NegV);
8302 RemoveDeadNode(NegLHS);
8307 Handles.emplace_back(NegLHS);
8320 RemoveDeadNode(NegLHS);
8321 RemoveDeadNode(NegRHS);
8325 Cost = std::min(CostLHS, CostRHS);
8326 return DAG.
getSelect(
DL, VT,
Op.getOperand(0), NegLHS, NegRHS);
8355 if (!HasMULHU && !HasMULHS && !HasUMUL_LOHI && !HasSMUL_LOHI)
8367 if ((
Signed && HasSMUL_LOHI) || (!
Signed && HasUMUL_LOHI)) {
8370 Hi =
Lo.getValue(1);
8396 if (MakeMUL_LOHI(LL, RL,
Lo,
Hi,
false)) {
8397 Result.push_back(
Lo);
8398 Result.push_back(
Hi);
8401 Result.push_back(
Zero);
8402 Result.push_back(
Zero);
8413 if (MakeMUL_LOHI(LL, RL,
Lo,
Hi,
true)) {
8414 Result.push_back(
Lo);
8415 Result.push_back(
Hi);
8420 unsigned ShiftAmount = OuterBitSize - InnerBitSize;
8435 if (!MakeMUL_LOHI(LL, RL,
Lo,
Hi,
false))
8438 Result.push_back(
Lo);
8445 Result.push_back(
Hi);
8458 if (!MakeMUL_LOHI(LL, RH,
Lo,
Hi,
false))
8465 if (!MakeMUL_LOHI(LH, RL,
Lo,
Hi,
false))
8518 N->getOperand(0),
N->getOperand(1), Result, HiLoVT,
8519 DAG, Kind, LL, LH, RL, RH);
8521 assert(Result.size() == 2);
8556bool TargetLowering::expandUDIVREMByConstantViaUREMDecomposition(
8559 unsigned Opcode =
N->getOpcode();
8560 EVT VT =
N->getValueType(0);
8568 unsigned TrailingZeros = 0;
8577 if (Divisor.
uge(HalfMaxPlus1))
8582 unsigned BestChunkWidth = 0, AltChunkWidth = 0;
8583 for (
unsigned I = HBitWidth,
E = HBitWidth / 2;
I >
E; --
I) {
8585 if (
I == HBitWidth - 1)
8597 if (
I != HBitWidth &&
Mod == Divisor - 1)
8601 bool Alternate =
false;
8602 if (!BestChunkWidth) {
8606 BestChunkWidth = AltChunkWidth;
8611 assert(!LL == !LH &&
"Expected both input halves or no input halves!");
8613 std::tie(LL, LH) = DAG.
SplitScalar(
N->getOperand(0), dl, HiLoVT, HiLoVT);
8618 assert(ShiftAmt > 0 && ShiftAmt < HBitWidth);
8636 if (ShiftAmt < HBitWidth) {
8637 Lo = GetFSHR(
Lo,
Hi, ShiftAmt);
8640 }
else if (ShiftAmt == HBitWidth) {
8653 SDValue PartialRemL, PartialRemH;
8654 if (TrailingZeros && Opcode !=
ISD::UDIV) {
8656 if (TrailingZeros < HBitWidth) {
8660 }
else if (TrailingZeros == HBitWidth) {
8675 if (BestChunkWidth == HBitWidth) {
8678 ShiftRight(LL, LH, TrailingZeros);
8684 SDVTList VTList = DAG.
getVTList(HiLoVT, SetCCType);
8707 for (
unsigned I = 0;
I <
BitWidth - TrailingZeros;
I += BestChunkWidth) {
8709 unsigned Shift =
I + TrailingZeros;
8713 else if (Shift >= HBitWidth)
8718 Chunk = GetFSHR(LL, LH, Shift);
8720 if (
I + BestChunkWidth <
BitWidth - TrailingZeros)
8726 unsigned ChunkNum =
I / BestChunkWidth;
8727 unsigned Opc = (Alternate && (ChunkNum % 2) != 0) ?
ISD::SUB : ISD::
ADD;
8728 Sum = DAG.
getNode(
Opc, dl, HiLoVT, Sum, Chunk);
8760 if (BestChunkWidth != HBitWidth)
8761 ShiftRight(LL, LH, TrailingZeros);
8778 std::tie(QuotL, QuotH) = DAG.
SplitScalar(Quotient, dl, HiLoVT, HiLoVT);
8786 if (TrailingZeros) {
8787 if (TrailingZeros < HBitWidth) {
8799 }
else if (TrailingZeros == HBitWidth) {
8821bool TargetLowering::expandUDIVREMByConstantViaUMulHiMagic(
8822 SDNode *
N,
const APInt &Divisor, SmallVectorImpl<SDValue> &Result,
8829 assert(!Divisor.
isOne() &&
"Magic algorithm does not work for division by 1");
8834 SmallVectorImpl<SDValue> &
Result) {
8838 return expandMUL_LOHI(
Opc, VT,
DL,
LHS,
RHS, Result, HiLoVT, DAG,
8848 DAG.
getVTList(HiLoVT, MVT::i1), LL, RL);
8853 DAG.
getVTList(HiLoVT, MVT::i1), LH, RH, Overflow);
8855 return std::make_pair(OutL, OutH);
8861 if (Shift < HBitWidth) {
8865 return std::make_pair(ResL, ResH);
8868 if (Shift == HBitWidth)
8869 return std::make_pair(LH,
Zero);
8870 assert(Shift - HBitWidth < HBitWidth &&
8871 "We shouldn't generate an undefined shift");
8880 Divisor, std::min(KnownLeadingZeros, Divisor.
countl_zero()));
8882 assert(!LL == !LH &&
"Expected both input halves or no input halves!");
8888 std::tie(QL, QH) = MakeSRLLong(QL, QH, Magics.
PreShift);
8898 auto [NPQL, NPQH] = MakeAddSubLong(
ISD::SUB, LL, LH, QL, QH);
8899 std::tie(NPQL, NPQH) = MakeSRLLong(NPQL, NPQH, 1);
8900 std::tie(QL, QH) = MakeAddSubLong(
ISD::ADD, NPQL, NPQH, QL, QH);
8904 std::tie(QL, QH) = MakeSRLLong(QL, QH, Magics.
PostShift);
8906 unsigned Opcode =
N->getOpcode();
8914 if (!MakeMUL_LOHIByConst(
ISD::MUL, QL, QH, Divisor, MulResult))
8920 MakeAddSubLong(
ISD::SUB, LL, LH, MulResult[0], MulResult[1]);
8933 unsigned Opcode =
N->getOpcode();
8940 "Unexpected opcode");
8946 APInt Divisor = CN->getAPIntValue();
8951 bool CanDecomposeUREMWithoutMulHi =
8954 RTLIB::Unsupported &&
8956 if (!CanDecomposeUREMWithoutMulHi &&
8969 if (expandUDIVREMByConstantViaUREMDecomposition(
N, Divisor, Result, HiLoVT,
8973 if (expandUDIVREMByConstantViaUMulHiMagic(
N, Divisor, Result, HiLoVT, DAG, LL,
8990 EVT VT =
Node->getValueType(0);
9006 EVT ShVT = Z.getValueType();
9075 EVT VT =
Node->getValueType(0);
9093 if (!AllowVectorOps && VT.
isVector() &&
9111 ShVal = DAG.
getNode(ShOpc,
DL, VT, Op0, ShAmt);
9113 HsVal = DAG.
getNode(HsOpc,
DL, VT, Op0, HsAmt);
9119 ShVal = DAG.
getNode(ShOpc,
DL, VT, Op0, ShAmt);
9140 EVT VT,
unsigned HalveDepth = 0,
9141 unsigned TotalDepth = 0) {
9146 if (!
TLI.isTypeLegal(VT))
9156 if (
TLI.isTypeLegal(HalfVT) &&
9164 if (
TLI.isTypeLegal(WideVT) &&
9173 EVT VT =
Node->getValueType(0);
9177 unsigned Opcode =
Node->getOpcode();
9192 unsigned HalfBW = BW / 2;
9277 const APInt &YVal =
C->getAPIntValue();
9281 for (
unsigned I = 1;
I <
N;
I <<= 1) {
9354 while (S < 32 &&
divideCeil(BW, S) > (1u << S))
9359 unsigned HolesCost = S * S + 3 * S + S * (S - 1) + (S - 1);
9363 unsigned NaiveCost = 0;
9364 for (
unsigned I = 0;
I < BW; ++
I) {
9382 if (HolesCost < NaiveCost &&
9392 for (
unsigned I = 0;
I < S; ++
I) {
9405 for (
unsigned I = 0;
I < S; ++
I) {
9407 for (
unsigned J = 0; J < S; ++J) {
9408 unsigned K = (
I + S - J) % S;
9425 for (
unsigned I = 0;
I < BW; ++
I) {
9493 unsigned ShAmt = Opcode ==
ISD::CLMULR ? BW - 1 : BW;
9513 if (!Flags.hasNoInfs()) {
9526 Results.push_back(ResultFrac);
9533 EVT VT =
Node->getValueType(0);
9550 for (
unsigned I = 1;
I < BW;
I *= 2) {
9572 EVT VT =
Node->getValueType(0);
9589 for (
unsigned S = 0; S < LogBW; ++S) {
9590 unsigned ShiftS = 1u << S;
9596 if (S + 1 < LogBW) {
9609 for (
int S = (
int)LogBW - 1; S >= 0; --S) {
9624 assert(
Node->getNumOperands() == 3 &&
"Not a double-shift!");
9625 EVT VT =
Node->getValueType(0);
9683 EVT VT =
Node->getValueType(0);
9686 Flags.setNoFPExcept(
true);
9698 EVT ResVT =
Node->getValueType(0);
9702 const uint64_t SemEnum =
Node->getConstantOperandVal(1);
9704 const auto RoundMode =
9706 const bool Saturate =
Node->getConstantOperandVal(3) != 0;
9719 "destination format (semantics enum " +
9720 Twine(SemEnum) +
")");
9725 switch (RoundMode) {
9734 "CONVERT_TO_ARBITRARY_FP: unsupported rounding mode (enum " +
9735 Twine(
static_cast<int>(RoundMode)) +
")");
9743 const unsigned DstMant = DstPrecision - 1;
9746 const unsigned DstExpBits = DstBits - (DstHasSign ? 1 : 0) - DstMant;
9748 const unsigned DstExpMax = (1U << DstExpBits) - 1;
9749 const uint64_t DstMantMask = (DstMant > 0) ? ((1ULL << DstMant) - 1) : 0;
9754 const unsigned DstExpMaxNormal =
9763 uint64_t DstMaxMantAtMaxExp = DstMantMask;
9766 DstMaxMantAtMaxExp = DstMantMask - 1;
9773 const unsigned SrcMant = SrcPrecision - 1;
9774 const uint64_t SrcMantMask = (1ULL << SrcMant) - 1;
9805 EVT FrexpExpScalarVT =
9825 switch (RoundMode) {
9865 if (SrcMant > DstMant) {
9866 const unsigned Shift = SrcMant - DstMant;
9896 RoundUp = ComputeRoundUp(RoundBit, StickyBits, LSB);
9911 DAG.
getSetCC(dl, SetCCVT, RoundedMant,
9946 int64_t MantDelta =
static_cast<int64_t
>(SrcMant) - DstMant;
9987 DenormRoundUp = ComputeRoundUp(DenormRoundBit, HasSticky, DenormLSB);
9992 DenormRoundUp = DAG.
getSelect(dl, IntVT, ShiftGEOne, DenormRoundUp,
Zero);
10001 DAG.
getSetCC(dl, SetCCVT, DenormRoundedMant,
10004 DAG.
getSelect(dl, IntVT, DenormMantOF,
Zero, DenormRoundedMant);
10028 DAG.
getSetCC(dl, SetCCVT, AdjMant,
10040 uint64_t MaxFinite =
10041 ((uint64_t)DstExpMaxNormal << DstMant) | DstMaxMantAtMaxExp;
10046 uint64_t InfBits = (uint64_t)DstExpMax << DstMant;
10060 DAG.
getNode(
ISD::OR, dl, IntVT, SignShifted, NormExpShifted), AdjMant);
10066 const uint64_t QNaNBit = (DstMant > 0) ? (1ULL << (DstMant - 1)) : 0;
10068 DAG.
getConstant(((uint64_t)DstExpMax << DstMant) | QNaNBit, dl, IntVT);
10072 NaNResult = DAG.
getConstant(((uint64_t)DstExpMax << DstMant) | DstMantMask,
10083 uint64_t InfBits = (uint64_t)DstExpMax << DstMant;
10086 }
else if (Saturate) {
10088 uint64_t MaxFinite =
10089 ((uint64_t)DstExpMaxNormal << DstMant) | DstMaxMantAtMaxExp;
10097 SDValue ZeroResult = SignShifted;
10101 DAG.
getSelect(dl, IntVT, ExpIsNeg, DenormResult, NormResult);
10105 SDValue Result = FiniteResult;
10106 Result = DAG.
getSelect(dl, IntVT, IsZero, ZeroResult, Result);
10107 Result = DAG.
getSelect(dl, IntVT, IsInf, InfResult, Result);
10113 if (!DstHasSign && Saturate) {
10116 Result = DAG.
getSelect(dl, IntVT, IsNegative,
Zero, Result);
10119 Result = DAG.
getSelect(dl, IntVT, IsNaN, NaNResult, Result);
10129 EVT DstVT =
Node->getValueType(0);
10133 const uint64_t SemEnum =
Node->getConstantOperandVal(1);
10147 "source format (semantics enum " +
10148 Twine(SemEnum) +
")");
10155 const unsigned SrcMant = SrcPrecision - 1;
10158 const unsigned SrcExp = SrcBits - (SrcHasSign ? 1 : 0) - SrcMant;
10166 const unsigned DstExpBits = DstBits - DstMant - 1;
10168 const int DstBias = 1 - DstMinExp;
10169 const uint64_t DstExpAllOnes = (1ULL << DstExpBits) - 1;
10173 EVT IntVT = IntScalarVT;
10185 "CONVERT_FROM_ARBITRARY_FP: the requested integer value type for its "
10186 "legalization is not supported");
10201 const uint64_t MantMask = (SrcMant > 0) ? ((1ULL << SrcMant) - 1) : 0;
10202 const uint64_t ExpMask = (1ULL << SrcExp) - 1;
10237 IsNaN = DAG.
getNode(
ISD::AND, dl, SetCCVT, IsExpAllOnes, IsMantNonZero);
10243 IsNaN = DAG.
getNode(
ISD::AND, dl, SetCCVT, IsExpAllOnes, IsMantAllOnes);
10248 IsInf = DAG.
getNode(
ISD::AND, dl, SetCCVT, IsExpAllOnes, IsMantZero);
10257 const int BiasAdjust = DstBias - SrcBias;
10264 if (DstMant > SrcMant) {
10267 NormDstMant = DAG.
getNode(
ISD::SHL, dl, IntVT, MantField, NormDstMantShift);
10269 NormDstMant = MantField;
10282 SDValue DenormResult = NormResult;
10283 if (BiasAdjust != 0) {
10288 const int DenormExpConst =
10289 (int)IntVTBits + DstBias - SrcBias - (
int)SrcMant;
10297 DAG.
getConstant(IntVTBits - 1, dl, IntVT), LeadingZeros);
10302 const unsigned ShiftSub = IntVTBits - 1 - DstMant;
10317 DAG.
getSelect(dl, IntVT, IsDenorm, DenormResult, NormResult);
10319 const uint64_t QNaNBit = (DstMant > 0) ? (1ULL << (DstMant - 1)) : 0;
10321 DAG.
getConstant((DstExpAllOnes << DstMant) | QNaNBit, dl, IntVT);
10325 DAG.
getConstant(DstExpAllOnes << DstMant, dl, IntVT));
10333 DAG.
getSetCC(dl, SetCCVT, NormDstExp,
10336 DAG.
getSelect(dl, IntVT, IsOverflow, InfResult, FiniteResult);
10339 SDValue ZeroResult = SignShifted;
10341 SDValue Result = FiniteResult;
10342 Result = DAG.
getSelect(dl, IntVT, IsZero, ZeroResult, Result);
10343 Result = DAG.
getSelect(dl, IntVT, IsInf, InfResult, Result);
10344 Result = DAG.
getSelect(dl, IntVT, IsNaN, NaNResult, Result);
10346 if (!DstVT.
bitsEq(IntVT)) {
10358 PtrInfo, IntScalarVT, Alignment);
10369 unsigned OpNo =
Node->isStrictFPOpcode() ? 1 : 0;
10371 EVT SrcVT = Src.getValueType();
10372 EVT DstVT =
Node->getValueType(0);
10376 if (SrcVT != MVT::f32 || DstVT != MVT::i64)
10379 if (
Node->isStrictFPOpcode())
10442 unsigned OpNo =
Node->isStrictFPOpcode() ? 1 : 0;
10445 EVT SrcVT = Src.getValueType();
10446 EVT DstVT =
Node->getValueType(0);
10467 if (
Node->isStrictFPOpcode()) {
10469 {
Node->getOperand(0), Src });
10470 Chain = Result.getValue(1);
10484 if (
Node->isStrictFPOpcode()) {
10486 Node->getOperand(0),
true);
10511 if (
Node->isStrictFPOpcode()) {
10513 { Chain, Src, FltOfs });
10545 if (
Node->isStrictFPOpcode())
10549 EVT SrcVT = Src.getValueType();
10550 EVT DstVT =
Node->getValueType(0);
10554 if (
Node->getFlags().hasNonNeg() &&
10602 unsigned Opcode =
Node->getOpcode();
10607 if (
Node->getFlags().hasNoNaNs()) {
10609 EVT VT =
Node->getValueType(0);
10628 EVT VT =
Node->getValueType(0);
10631 "Expanding fminnum/fmaxnum for scalable vectors is undefined.");
10641 if (!
Node->getFlags().hasNoNaNs()) {
10654 return DAG.
getNode(NewOp, dl, VT, Quiet0, Quiet1,
Node->getFlags());
10659 if (
Node->getFlags().hasNoNaNs() ||
10662 unsigned IEEE2018Op =
10665 return DAG.
getNode(IEEE2018Op, dl, VT,
Node->getOperand(0),
10666 Node->getOperand(1),
Node->getFlags());
10684 if (!
TLI.isTypeLegal(IntVT) &&
10699 unsigned Opc =
N->getOpcode();
10700 EVT VT =
N->getValueType(0);
10713 bool MinMaxMustRespectOrderedZero =
false;
10717 MinMaxMustRespectOrderedZero =
true;
10731 if (!
N->getFlags().hasNoNaNs() &&
10740 if (!MinMaxMustRespectOrderedZero && !
N->getFlags().hasNoSignedZeros() &&
10757 unsigned Opc =
Node->getOpcode();
10758 EVT VT =
Node->getValueType(0);
10767 if (!Flags.hasNoNaNs()) {
10778 return DAG.
getNode(NewOp,
DL, VT, LHS, RHS, Flags);
10783 if (Flags.hasNoNaNs() ||
10785 unsigned IEEE2019Op =
10788 return DAG.
getNode(IEEE2019Op,
DL, VT, LHS, RHS, Flags);
10793 if ((Flags.hasNoNaNs() ||
10799 return DAG.
getNode(IEEE2008Op,
DL, VT, LHS, RHS, Flags);
10844 bool IsOrdered = NanTest ==
fcNone;
10845 bool IsUnordered = NanTest ==
fcNan;
10848 if (!IsOrdered && !IsUnordered)
10849 return std::nullopt;
10851 if (OrderedMask ==
fcZero &&
10857 return std::nullopt;
10864 EVT OperandVT =
Op.getValueType();
10876 if (OperandVT == MVT::ppcf128) {
10879 OperandVT = MVT::f64;
10886 bool IsF80 = (ScalarFloatVT == MVT::f80);
10890 if (Flags.hasNoFPExcept() &&
10893 bool IsInvertedFP =
false;
10897 FPTestMask = InvertedFPCheck;
10898 IsInvertedFP =
true;
10910 OrderedFPTestMask = FPTestMask;
10912 const bool IsOrdered = FPTestMask == OrderedFPTestMask;
10914 if (std::optional<bool> IsCmp0 =
10917 *IsCmp0 ? OrderedCmpOpcode : UnorderedCmpOpcode,
10924 *IsCmp0 ? OrderedCmpOpcode : UnorderedCmpOpcode);
10927 if (FPTestMask ==
fcNan &&
10933 bool IsOrderedInf = FPTestMask ==
fcInf;
10936 : UnorderedCmpOpcode,
10947 IsOrderedInf ? OrderedCmpOpcode : UnorderedCmpOpcode);
10952 : UnorderedCmpOpcode,
10963 IsOrdered ? OrderedCmpOpcode : UnorderedCmpOpcode);
10982 return DAG.
getSetCC(
DL, ResultVT, Abs, SmallestNormal,
10983 IsOrdered ? OrderedOp : UnorderedOp);
11006 DAG.
getSetCC(
DL, ResultVT, Abs, SmallestNormal, IsNormalOp);
11008 return DAG.
getNode(LogicOp,
DL, ResultVT, IsFinite, IsNormal);
11015 bool IsInverted =
false;
11018 Test = InvertedCheck;
11032 const unsigned ExplicitIntBitInF80 = 63;
11033 APInt ExpMask = Inf;
11035 ExpMask.
clearBit(ExplicitIntBitInF80);
11037 APInt QNaNBitMask =
11049 const auto appendResult = [&](
SDValue PartialRes) {
11059 const auto getIntBitIsSet = [&]() ->
SDValue {
11060 if (!IntBitIsSetV) {
11061 APInt IntBitMask(BitSize, 0);
11062 IntBitMask.
setBit(ExplicitIntBitInF80);
11067 return IntBitIsSetV;
11088 "finite check requires IEEE-like FP");
11106 appendResult(PartialRes);
11115 appendResult(ExpIsZero);
11122 if (
unsigned PartialCheck =
Test &
fcZero) {
11125 else if (PartialCheck ==
fcZero)
11129 appendResult(PartialRes);
11142 appendResult(PartialRes);
11145 if (
unsigned PartialCheck =
Test &
fcInf) {
11148 else if (PartialCheck ==
fcInf)
11155 appendResult(PartialRes);
11158 if (
unsigned PartialCheck =
Test &
fcNan) {
11159 APInt InfWithQnanBit = Inf | QNaNBitMask;
11161 if (PartialCheck ==
fcNan) {
11174 }
else if (PartialCheck ==
fcQNan) {
11186 appendResult(PartialRes);
11191 APInt ExpLSB = ExpMask & ~(ExpMask.
shl(1));
11194 APInt ExpLimit = ExpMask - ExpLSB;
11207 appendResult(PartialRes);
11221 return TLI.isOperationLegalOrCustom(
ISD::ADD, VT) &&
11224 (Len == 8 ||
TLI.isOperationLegalOrCustom(
ISD::MUL, VT)) &&
11225 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, VT);
11230 EVT VT =
Node->getValueType(0);
11238 unsigned LZ =
Known.countMinLeadingZeros();
11239 unsigned TZ =
Known.countMinTrailingZeros();
11240 unsigned ShiftedActiveBits =
Known.getBitWidth() - (LZ + TZ);
11243 unsigned EffectiveLen = Len;
11244 if (ShiftedActiveBits > 0 && ShiftedActiveBits < Len)
11245 EffectiveLen = std::min(
alignTo(ShiftedActiveBits, 8), Len);
11250 if (!(Len <= 128 && Len % 8 == 0))
11258 if (EffectiveLen < Len && TZ > 0) {
11291 if (EffectiveLen <= 8)
11297 if (EffectiveLen == 16 && !VT.
isVector()) {
11315 for (
unsigned Shift = 8; Shift < EffectiveLen; Shift *= 2) {
11326 EVT VT =
Node->getValueType(0);
11343 return DAG.
getSelect(dl, VT, SrcIsZero,
11365 for (
unsigned i = 0; (1U << i) < NumBitsPerElt; ++i) {
11376 EVT VT =
Node->getValueType(0);
11402 :
APInt(64, 0x0218A392CD3D5DBFULL);
11415 for (
unsigned i = 0; i <
BitWidth; i++) {
11441 EVT VT =
Node->getValueType(0);
11457 return DAG.
getSelect(dl, VT, SrcIsZero,
11518 SDValue Source =
N->getOperand(0);
11521 EVT SrcVT = Source.getValueType();
11522 EVT ResVT =
N->getValueType(0);
11538 return DAG.
getNode(ISD::VP_REDUCE_UMIN,
DL, ResVT, ExtEVL,
Select, Mask, EVL);
11546static std::pair<SDValue, SDValue>
11549 EVT MaskVT = Mask.getValueType();
11558 uint64_t EltWidth =
TLI.getBitWidthForCttzElements(
11575 StepVecVT =
TLI.getTypeToTransformTo(*DAG.
getContext(), StepVecVT);
11600 return {Mask, StepVec};
11607 N->getOperand(0),
true,
DL, DAG);
11612 EVT MaskVT =
N->getOperand(0).getValueType();
11613 EVT ResVT =
N->getValueType(0);
11643 EVT StepVecVT = StepVec.getValueType();
11660 EVT VT =
N->getValueType(0);
11661 EVT OpVT = Start.getValueType();
11691 StepVector.getValueType());
11694 StepVector.getValueType());
11695 if (Mask.getValueType() != VT)
11703 EVT VT =
N->getValueType(0);
11704 SDValue SourceValue =
N->getOperand(0);
11705 SDValue SinkValue =
N->getOperand(1);
11706 SDValue EltSizeInBytes =
N->getOperand(2);
11719 SDValue SourceAheadOfOrEqualToSink =
11727 if (IsReadAfterWrite)
11728 Diff = DAG.
getSelect(
DL, AddrVT, SourceAheadOfOrEqualToSink,
11736 SDValue NoAlias = SourceAheadOfOrEqualToSink;
11737 if (IsReadAfterWrite)
11744 DL, AddrVT, NoAlias,
11753 bool IsNegative)
const {
11755 EVT VT =
N->getValueType(0);
11818 EVT VT =
N->getValueType(0);
11821 bool IsSigned =
N->getOpcode() ==
ISD::ABDS;
11901 EVT VT =
N->getValueType(0);
11905 unsigned Opc =
N->getOpcode();
11914 "Unknown AVG node");
11926 return DAG.
getNode(ShiftOpc, dl, VT, Sum,
11934 LHS = DAG.
getNode(ExtOpc, dl, ExtVT, LHS);
11935 RHS = DAG.
getNode(ExtOpc, dl, ExtVT, RHS);
11963 ISD::SHL, dl, VT, ZeroExtOverflow,
11984 EVT VT =
N->getValueType(0);
11991 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
12058 EVT VT =
N->getValueType(0);
12101 for (
unsigned I = 0, J = Sz-1;
I < Sz; ++
I, --J) {
12117std::pair<SDValue, SDValue>
12121 SDValue Chain = LD->getChain();
12122 SDValue BasePTR = LD->getBasePtr();
12123 EVT SrcVT = LD->getMemoryVT();
12124 EVT DstVT = LD->getValueType(0);
12155 ISD::EXTLOAD, SL, LoadVT, Chain, BasePTR, LD->getPointerInfo(),
12156 SrcIntVT, LD->getBaseAlign(), LD->getMemOperand()->getFlags(),
12157 LD->getMMOMetadataForSubAccess());
12160 for (
unsigned Idx = 0; Idx < NumElem; ++Idx) {
12161 unsigned ShiftIntoIdx =
12179 return std::make_pair(
Value,
Load.getValue(1));
12188 for (
unsigned Idx = 0; Idx < NumElem; ++Idx) {
12190 ExtType, SL, DstEltVT, Chain, BasePTR,
12191 LD->getPointerInfo().getWithOffset(Idx * Stride), SrcEltVT,
12192 LD->getBaseAlign(), LD->getMemOperand()->getFlags(),
12193 LD->getMMOMetadataForSubAccess());
12204 return std::make_pair(
Value, NewChain);
12211 SDValue Chain = ST->getChain();
12212 SDValue BasePtr = ST->getBasePtr();
12214 EVT StVT = ST->getMemoryVT();
12240 for (
unsigned Idx = 0; Idx < NumElem; ++Idx) {
12244 unsigned ShiftIntoIdx =
12253 return DAG.
getStore(Chain, SL, CurrVal, BasePtr, ST->getPointerInfo(),
12254 ST->getBaseAlign(), ST->getMemOperand()->getFlags(),
12255 ST->getMMOMetadataForSubAccess());
12260 assert(Stride &&
"Zero stride!");
12264 for (
unsigned Idx = 0; Idx < NumElem; ++Idx) {
12272 Chain, SL, Elt, Ptr, ST->getPointerInfo().getWithOffset(Idx * Stride),
12273 MemSclVT, ST->getBaseAlign(), ST->getMemOperand()->getFlags(),
12274 ST->getMMOMetadataForSubAccess());
12282std::pair<SDValue, SDValue>
12285 "unaligned indexed loads not implemented!");
12286 SDValue Chain = LD->getChain();
12287 SDValue Ptr = LD->getBasePtr();
12288 EVT VT = LD->getValueType(0);
12289 EVT LoadedVT = LD->getMemoryVT();
12305 LD->getMemOperand());
12307 if (LoadedVT != VT)
12311 return std::make_pair(Result, newLoad.
getValue(1));
12319 unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes;
12325 SDValue StackPtr = StackBase;
12329 EVT StackPtrVT = StackPtr.getValueType();
12335 for (
unsigned i = 1; i < NumRegs; i++) {
12338 RegVT, dl, Chain, Ptr, LD->getPointerInfo().getWithOffset(
Offset),
12339 LD->getBaseAlign(), LD->getMemOperand()->getFlags(),
12340 LD->getMMOMetadataForSubAccess());
12343 Load.getValue(1), dl,
Load, StackPtr,
12354 8 * (LoadedBytes -
Offset));
12357 LD->getPointerInfo().getWithOffset(
Offset), MemVT, LD->getBaseAlign(),
12358 LD->getMemOperand()->getFlags(), LD->getMMOMetadataForSubAccess());
12363 Load.getValue(1), dl,
Load, StackPtr,
12370 Load = DAG.
getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase,
12375 return std::make_pair(
Load, TF);
12379 "Unaligned load of unsupported type.");
12388 Align Alignment = LD->getBaseAlign();
12389 unsigned IncrementSize = NumBits / 8;
12400 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
12401 LD->getMMOMetadataForSubAccess());
12405 LD->getPointerInfo().getWithOffset(IncrementSize),
12406 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
12407 LD->getMMOMetadataForSubAccess());
12409 Hi = DAG.
getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(),
12410 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
12411 LD->getMMOMetadataForSubAccess());
12415 LD->getPointerInfo().getWithOffset(IncrementSize),
12416 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
12417 LD->getMMOMetadataForSubAccess());
12428 return std::make_pair(Result, TF);
12434 "unaligned indexed stores not implemented!");
12435 SDValue Chain = ST->getChain();
12436 SDValue Ptr = ST->getBasePtr();
12437 SDValue Val = ST->getValue();
12439 Align Alignment = ST->getBaseAlign();
12441 EVT StoreMemVT = ST->getMemoryVT();
12457 Result = DAG.
getStore(Chain, dl, Result, Ptr, ST->getPointerInfo(),
12458 Alignment, ST->getMemOperand()->getFlags(),
12459 ST->getMMOMetadataForSubAccess());
12470 unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes;
12478 Chain, dl, Val, StackPtr,
12481 EVT StackPtrVT = StackPtr.getValueType();
12489 for (
unsigned i = 1; i < NumRegs; i++) {
12492 RegVT, dl,
Store, StackPtr,
12497 ST->getPointerInfo().getWithOffset(
Offset), ST->getBaseAlign(),
12498 ST->getMemOperand()->getFlags(), ST->getMMOMetadataForSubAccess()));
12518 ST->getPointerInfo().getWithOffset(
Offset), LoadMemVT,
12519 ST->getBaseAlign(), ST->getMemOperand()->getFlags(),
12520 ST->getMMOMetadataForSubAccess()));
12527 "Unaligned store of unknown type.");
12531 unsigned IncrementSize = NumBits / 8;
12551 ST->getPointerInfo(), NewStoredVT, Alignment,
12552 ST->getMemOperand()->getFlags(), ST->getMMOMetadataForSubAccess());
12557 ST->getPointerInfo().getWithOffset(IncrementSize), NewStoredVT, Alignment,
12558 ST->getMemOperand()->getFlags(), ST->getMMOMetadataForSubAccess());
12569 bool IsCompressedMemory)
const {
12572 EVT MaskVT = Mask.getValueType();
12574 "Incompatible types of Data and Mask");
12575 if (IsCompressedMemory) {
12588 MaskIntVT = MVT::i32;
12607 "Cannot index a scalable vector within a fixed-width vector");
12609 EVT IdxVT = Idx.getValueType();
12631 unsigned SubOpcode =
12644 DAG, VecPtr, VecVT,
12646 Index, PtrArithFlags);
12662 "Converting bits to bytes lost precision");
12664 "Sub-vector must be a vector with matching element type");
12667 "Subvector Index must be constant!");
12675 EVT IdxVT = Index.getValueType();
12707 assert(EmuTlsVar &&
"Cannot find EmuTlsVar ");
12708 Args.emplace_back(DAG.
getGlobalAddress(EmuTlsVar, dl, PtrVT), VoidPtrType);
12715 std::pair<SDValue, SDValue> CallResult =
LowerCallTo(CLI);
12724 "Emulated TLS must have zero offset in GlobalAddressSDNode");
12725 return CallResult.first;
12736 EVT VT =
Op.getOperand(0).getValueType();
12738 if (VT.
bitsLT(MVT::i32)) {
12756 unsigned Opcode =
Node->getOpcode();
12763 return DAG.
getNode(AltOpcode,
DL, VT, Op0, Op1);
12804 {Op0, Op1, DAG.getCondCode(CC)})) {
12811 {Op0, Op1, DAG.getCondCode(CC)})) {
12839 unsigned Opcode =
Node->getOpcode();
12842 EVT VT = LHS.getValueType();
12845 assert(VT == RHS.getValueType() &&
"Expected operands to be the same type");
12875 unsigned OverflowOp;
12890 llvm_unreachable(
"Expected method to receive signed or unsigned saturation "
12891 "addition or subtraction node.");
12899 unsigned BitWidth = LHS.getScalarValueSizeInBits();
12902 SDValue SumDiff = Result.getValue(0);
12903 SDValue Overflow = Result.getValue(1);
12929 "Expected signed saturating add/sub opcode");
12945 bool RHSIsNonNegative =
12947 if (LHSIsNonNegative || RHSIsNonNegative) {
12949 return DAG.
getSelect(dl, VT, Overflow, SatMax, SumDiff);
12953 bool RHSIsNegative =
12955 if (LHSIsNegative || RHSIsNegative) {
12957 return DAG.
getSelect(dl, VT, Overflow, SatMin, SumDiff);
12965 return DAG.
getSelect(dl, VT, Overflow, Result, SumDiff);
12969 unsigned Opcode =
Node->getOpcode();
12972 EVT VT = LHS.getValueType();
12973 EVT ResVT =
Node->getValueType(0);
13005 unsigned Opcode =
Node->getOpcode();
13009 EVT VT = LHS.getValueType();
13014 "Expected a SHLSAT opcode");
13046 EVT VT = LHS.getValueType();
13047 assert(RHS.getValueType() == VT &&
"Mismatching operand types");
13049 assert((HiLHS && HiRHS) || (!HiLHS && !HiRHS));
13051 "Signed flag should only be set when HiLHS and RiRHS are null");
13059 unsigned HalfBits = Bits / 2;
13104 EVT VT = LHS.getValueType();
13105 assert(RHS.getValueType() == VT &&
"Mismatching operand types");
13109 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
13110 if (WideVT == MVT::i16)
13111 LC = RTLIB::MUL_I16;
13112 else if (WideVT == MVT::i32)
13113 LC = RTLIB::MUL_I32;
13114 else if (WideVT == MVT::i64)
13115 LC = RTLIB::MUL_I64;
13116 else if (WideVT == MVT::i128)
13117 LC = RTLIB::MUL_I128;
13120 if (LibcallImpl == RTLIB::Unsupported) {
13163 SDValue Args[] = {LHS, HiLHS, RHS, HiRHS};
13164 Ret =
makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;
13166 SDValue Args[] = {HiLHS, LHS, HiRHS, RHS};
13167 Ret =
makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;
13170 "Ret value is a collection of constituent nodes holding result.");
13187 "Expected a fixed point multiplication opcode");
13192 EVT VT = LHS.getValueType();
13193 unsigned Scale =
Node->getConstantOperandVal(2);
13209 SDValue Product = Result.getValue(0);
13210 SDValue Overflow = Result.getValue(1);
13221 Result = DAG.
getSelect(dl, VT, ProdNeg, SatMin, SatMax);
13222 return DAG.
getSelect(dl, VT, Overflow, Result, Product);
13226 SDValue Product = Result.getValue(0);
13227 SDValue Overflow = Result.getValue(1);
13231 return DAG.
getSelect(dl, VT, Overflow, SatMax, Product);
13236 "Expected scale to be less than the number of bits if signed or at "
13237 "most the number of bits if unsigned.");
13238 assert(LHS.getValueType() == RHS.getValueType() &&
13239 "Expected both operands to be the same type");
13258 Lo = Result.getValue(0);
13259 Hi = Result.getValue(1);
13262 Hi = DAG.
getNode(HiOp, dl, VT, LHS, RHS);
13280 if (Scale == VTSize)
13303 return getSaturatingSelect(
Hi, LowMask, DAG.
getConstant(MaxVal, dl, VT),
13323 return DAG.
getSelect(dl, VT, Overflow, ResultIfOverflow, Result);
13336 Result = getSaturatingSelect(
Hi, LowMask, SatMax, Result,
ISD::SETGT);
13337 Result = getSaturatingSelect(
Hi, HighMask, SatMin, Result,
ISD::SETLT);
13347 "Expected a fixed point division opcode");
13349 EVT VT = LHS.getValueType();
13371 if (LHSLead + RHSTrail < Scale + (
unsigned)(Saturating &&
Signed))
13374 unsigned LHSShift = std::min(LHSLead, Scale);
13375 unsigned RHSShift = Scale - LHSShift;
13439 { LHS, RHS, CarryIn });
13446 LHS.getValueType(), LHS, RHS);
13448 EVT ResultType =
Node->getValueType(1);
13459 DAG.
getSetCC(dl, SetCCType, Result,
13468 SetCC = DAG.
getSetCC(dl, SetCCType, Result, LHS,
CC);
13481 LHS.getValueType(), LHS, RHS);
13483 EVT ResultType =
Node->getValueType(1);
13490 SDValue Sat = DAG.
getNode(OpcSat, dl, LHS.getValueType(), LHS, RHS);
13506 DAG.
getNode(
ISD::XOR, dl, OType, RHSNegative, ResultLowerThanLHS), dl,
13507 ResultType, ResultType);
13514 DAG.
getNode(
ISD::XOR, dl, OType, LHSLessThanRHS, ResultNegative), dl,
13515 ResultType, ResultType);
13522 EVT VT =
Node->getValueType(0);
13530 const APInt &
C = RHSC->getAPIntValue();
13532 if (
C.isPowerOf2()) {
13534 bool UseArithShift =
isSigned && !
C.isMinSignedValue();
13537 Overflow = DAG.
getSetCC(dl, SetCCVT,
13539 dl, VT, Result, ShiftAmt),
13549 static const unsigned Ops[2][3] =
13575 Result = BottomHalf;
13582 Overflow = DAG.
getSetCC(dl, SetCCVT, TopHalf,
13587 EVT RType =
Node->getValueType(1);
13592 "Unexpected result type for S/UMULO legalization");
13598 EVT VT =
Node->getValueType(0);
13636 EVT VT =
Op.getValueType();
13641 bool WidenSrc =
false;
13642 switch (
Node->getOpcode()) {
13695 "Expanding reductions for scalable vectors is undefined.");
13704 for (
unsigned i = 1; i < NumElts; i++)
13705 Res = DAG.
getNode(BaseOpcode, dl, EltVT, Res,
Ops[i], Flags);
13708 if (EltVT !=
Node->getValueType(0))
13724 "Expanding reductions for scalable vectors is undefined.");
13734 for (
unsigned i = 0; i < NumElts; i++)
13735 Res = DAG.
getNode(BaseOpcode, dl, EltVT, Res,
Ops[i], Flags);
13742 EVT VT =
Node->getValueType(0);
13751 Result = DAG.
getNode(DivRemOpc, dl, VTs, Dividend, Divisor).
getValue(1);
13756 SDValue Divide = DAG.
getNode(DivOpc, dl, VT, Dividend, Divisor);
13771 EVT SrcVT = Src.getValueType();
13772 EVT DstVT =
Node->getValueType(0);
13777 assert(SatWidth <= DstWidth &&
13778 "Expected saturation width smaller than result width");
13782 APInt MinInt, MaxInt;
13793 if (SrcVT == MVT::f16 || SrcVT == MVT::bf16) {
13795 SrcVT = Src.getValueType();
13815 auto EmitMinMax = [&](
unsigned MinOpcode,
unsigned MaxOpcode,
13816 bool MayPropagateNaN) {
13826 Clamped = DAG.
getNode(MaxOpcode, dl, SrcVT, Clamped, MinFloatNode);
13828 Clamped = DAG.
getNode(MinOpcode, dl, SrcVT, Clamped, MaxFloatNode);
13831 dl, DstVT, Clamped);
13835 if (!MayPropagateNaN && !IsSigned)
13843 return DAG.
getSelect(dl, DstVT, IsNan, ZeroInt, FpToInt);
13845 if (AreExactFloatBounds) {
13895 EVT OperandVT =
Op.getValueType();
13921 Op.getValueType());
13925 KeepNarrow = DAG.
getNode(
ISD::OR, dl, WideSetCCVT, KeepNarrow, AlreadyOdd);
13936 SDValue Adjust = DAG.
getSelect(dl, ResultIntVT, NarrowIsRd, One, NegativeOne);
13938 Op = DAG.
getSelect(dl, ResultIntVT, KeepNarrow, NarrowBits, Adjusted);
13945 EVT VT =
Node->getValueType(0);
13948 if (
Node->getConstantOperandVal(1) == 1) {
13951 EVT OperandVT =
Op.getValueType();
13959 EVT I32 =
F32.changeTypeToInteger();
13974 if (!
Node->getFlags().hasNoNaNs() &&
14002 EVT VT =
Node->getValueType(0);
14003 unsigned Factor =
Node->getNumOperands();
14006 if (Factor > 2 && Factor % 2 == 0) {
14011 for (
unsigned I = 0;
I < Factor / 2;
I++) {
14014 {
Node->getOperand(
I),
Node->getOperand(
I + Factor / 2)});
14022 for (
unsigned I = 0;
I < Factor / 2;
I++)
14024 for (
unsigned I = 0;
I < Factor / 2;
I++)
14038 for (
unsigned I = 0;
I != Factor; ++
I)
14040 VT,
DL,
Node->getOperand(0),
Node->getOperand(1),
14046 InterleavedElts.
reserve(Factor * NumElts);
14047 for (
unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx)
14048 for (
unsigned OperandIdx = 0; OperandIdx != Factor; ++OperandIdx)
14050 DL, EltVT,
Node->getOperand(OperandIdx), EltIdx));
14052 for (
unsigned ResultIdx = 0; ResultIdx != Factor; ++ResultIdx)
14054 VT,
DL,
ArrayRef(InterleavedElts).slice(ResultIdx * NumElts, NumElts)));
14061 EVT VT =
Node->getValueType(0);
14062 unsigned Factor =
Node->getNumOperands();
14065 if (Factor > 2 && Factor % 2 == 0) {
14077 for (
unsigned I = 0;
I < Factor / 2; ++
I) {
14079 {L.getValue(
I), R.getValue(
I)});
14094 for (
unsigned I = 0;
I != Factor; ++
I)
14102 ConcatElts.
reserve(Factor * NumElts);
14103 for (
unsigned OperandIdx = 0; OperandIdx != Factor; ++OperandIdx)
14104 for (
unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx)
14106 DL, EltVT,
Node->getOperand(OperandIdx), EltIdx));
14109 DeinterleavedElts.
reserve(Factor * NumElts);
14110 for (
unsigned ResultIdx = 0; ResultIdx != Factor; ++ResultIdx) {
14111 for (
unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx)
14112 DeinterleavedElts.
push_back(ConcatElts[ResultIdx + EltIdx * Factor]);
14115 ArrayRef(DeinterleavedElts).slice(ResultIdx * NumElts, NumElts)));
14123 "Unexpected opcode!");
14124 assert((
Node->getValueType(0).isScalableVector() ||
14126 "Fixed length vector types with constant offsets expected to use "
14127 "SHUFFLE_VECTOR!");
14129 EVT VT =
Node->getValueType(0);
14150 EVT PtrVT = StackPtr.getValueType();
14162 DAG.
getStore(StoreV1,
DL, V2, StackPtr2, PtrInfo, Alignment);
14178 return DAG.
getLoad(VT,
DL, StoreV2, StackPtr,
14191 EVT MaskVT = Mask.getValueType();
14208 bool HasPassthru = !Passthru.
isUndef();
14214 Chain = DAG.
getStore(Chain,
DL, Passthru, StackPtr, PtrInfo, Alignment);
14217 APInt PassthruSplatVal;
14220 if (IsSplatPassthru) {
14224 LastWriteVal = DAG.
getConstant(PassthruSplatVal,
DL, ScalarVT);
14225 }
else if (HasPassthru) {
14241 ScalarVT,
DL, Chain, LastElmtPtr,
14247 for (
unsigned I = 0;
I < NumElms;
I++) {
14251 Chain,
DL, ValI, OutPtr,
14263 if (HasPassthru &&
I == NumElms - 1) {
14273 LastWriteVal = DAG.
getSelect(
DL, ScalarVT, AllLanesSelected, ValI,
14276 Chain,
DL, LastWriteVal, OutPtr,
14281 return DAG.
getLoad(VecVT,
DL, Chain, StackPtr, PtrInfo, Alignment);
14290 EVT VecVT =
Node->getValueType(0);
14296 "Cannot expand vector_shuffle_var for scalable vectors.");
14323 EVT VT =
Node->getValueType(0);
14335 EVT ResVT =
Node->getValueType(0);
14349 return DAG.
getSelect(
DL, ResVT, ResLoNotNumElts, ResLo, Sum);
14352 EVT StepVecVT = StepVec.getValueType();
14374 SDValue Source =
N->getOperand(0);
14375 SDValue Needle =
N->getOperand(1);
14377 EVT SourceVT = Source.getValueType();
14379 EVT ResVT =
N->getValueType(0);
14389 if (NeedleVT == SourceVT) {
14392 SourceVT,
DL, Needle, DAG.
getUNDEF(SourceVT),
14406 UseVT =
N->user_begin()->getValueType(0);
14412 if (UseVT != ResVT)
14421 SDValue MulLHS =
N->getOperand(1);
14422 SDValue MulRHS =
N->getOperand(2);
14430 unsigned ExtOpcLHS, ExtOpcRHS;
14431 switch (
N->getOpcode()) {
14451 unsigned Opc =
N->getOpcode();
14454 unsigned CountRatio =
14456 unsigned WidthRatio =
14475 DAG.
getNode(ExtOpcLHS,
DL, ProdVT, MulLHS),
14476 DAG.
getNode(ExtOpcRHS,
DL, ProdVT, MulRHS));
14481 EVT MidVT =
Lo.getValueType()
14482 .widenIntegerVectorElementType(Ctx)
14483 .getHalfNumVectorElementsVT(Ctx);
14493 if (ExtMulOpVT != MulOpVT) {
14494 MulLHS = DAG.
getNode(ExtOpcLHS,
DL, ExtMulOpVT, MulLHS);
14495 MulRHS = DAG.
getNode(ExtOpcRHS,
DL, ExtMulOpVT, MulRHS);
14511 std::deque<SDValue> Subvectors = {Acc};
14512 for (
unsigned I = 0;
I < ScaleFactor;
I++)
14515 unsigned FlatNode =
14519 while (Subvectors.size() > 1) {
14520 Subvectors.push_back(DAG.
getNode(FlatNode,
DL, AccVT,
14521 {Subvectors[0], Subvectors[1]}, Flags));
14522 Subvectors.pop_front();
14523 Subvectors.pop_front();
14526 assert(Subvectors.size() == 1 &&
14527 "There should only be one subvector after tree flattening");
14529 return Subvectors[0];
14536 unsigned MaxSteps) {
14543 if (
Op.getNode() != FPNode)
14546 while (!Worklist.
empty()) {
14552 if (MaxSteps > 0 && Visited.
size() >= MaxSteps)
14580 std::optional<unsigned> CallRetResNo)
const {
14581 if (LC == RTLIB::UNKNOWN_LIBCALL)
14584 RTLIB::LibcallImpl LibcallImpl =
14586 if (LibcallImpl == RTLIB::Unsupported)
14590 EVT VT =
Node->getValueType(0);
14591 unsigned NumResults =
Node->getNumValues();
14602 SDValue StoreValue = ST->getValue();
14603 unsigned ResNo = StoreValue.
getResNo();
14605 if (CallRetResNo == ResNo)
14608 if (!ST->isSimple() || ST->getAddressSpace() != 0)
14611 if (StoresInChain && ST->getChain() != StoresInChain)
14615 if (ST->getAlign() <
14623 ResultStores[ResNo] = ST;
14624 StoresInChain = ST->getChain();
14631 EVT ArgVT =
Op.getValueType();
14633 Args.emplace_back(
Op, ArgTy);
14640 if (ResNo == CallRetResNo)
14642 EVT ResVT =
Node->getValueType(ResNo);
14644 ResultPtrs[ResNo] = ResultPtr;
14645 Args.emplace_back(ResultPtr,
PointerTy);
14657 Type *RetType = CallRetResNo.has_value()
14658 ?
Node->getValueType(*CallRetResNo).getTypeForEVT(Ctx)
14671 if (ResNo == CallRetResNo) {
14677 ResultPtr, PtrInfo);
14683 PtrInfo = ST->getPointerInfo();
14690 Results.push_back(LoadResult);
14700 bool IsSignaling)
const {
14701 MVT OpVT = LHS.getSimpleValueType();
14703 NeedInvert =
false;
14718 bool NeedSwap =
false;
14719 InvCC = getSetCCInverse(CCCode, OpVT);
14735 if (OpVT == MVT::i1) {
14750 DAG.
getNOT(dl, LHS, MVT::i1));
14755 DAG.
getNOT(dl, RHS, MVT::i1));
14760 DAG.
getNOT(dl, LHS, MVT::i1));
14765 DAG.
getNOT(dl, RHS, MVT::i1));
14788 "If SETUE is expanded, SETOEQ or SETUNE must be legal!");
14793 "If SETO is expanded, SETOEQ must be legal!");
14810 NeedInvert = ((
unsigned)CCCode & 0x8U);
14850 SetCC1 = DAG.
getSetCC(dl, VT, LHS, RHS, CC1, Chain, IsSignaling);
14851 SetCC2 = DAG.
getSetCC(dl, VT, LHS, RHS, CC2, Chain, IsSignaling);
14854 SetCC1 = DAG.
getSetCC(dl, VT, LHS, LHS, CC1, Chain, IsSignaling);
14855 SetCC2 = DAG.
getSetCC(dl, VT, RHS, RHS, CC2, Chain, IsSignaling);
14860 LHS = DAG.
getNode(
Opc, dl, VT, SetCC1, SetCC2);
14871 EVT VT =
Node->getValueType(0);
14883 unsigned Opcode =
Node->getOpcode();
14891 if (!V.getValueType().isVector()) {
14902 bool IsStrict =
Node->isStrictFPOpcode();
14939 std::optional<unsigned> ByteOffset;
14943 int Elt = ConstEltNo->getZExtValue();
14957 unsigned IsFast = 0;
14967 DAG, OriginalLoad->
getBasePtr(), InVecVT, EltNo);
14972 if (ResultVT.
bitsGT(VecEltVT)) {
14981 NewPtr, MPI, VecEltVT, Alignment,
14991 if (ResultVT.
bitsLT(VecEltVT))
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU Register Bank Select
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
block Block Frequency Analysis
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
static bool isSigned(unsigned Opcode)
static bool ShrinkDemandedConstant(Instruction *I, unsigned OpNo, const APInt &Demanded)
Check to see if the specified operand of the specified instruction is a constant integer.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static bool isNonZeroModBitWidthOrUndef(const MachineRegisterInfo &MRI, Register Reg, unsigned BW)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
static bool isUndef(const MachineInstr &MI)
Register const TargetRegisterInfo * TRI
Function const char * Passes
if(auto Err=PB.parsePassPipeline(MPM, Passes)) return wrap(std MPM run * Mod
MachineBasicBlock * Folded
SmallVector< MachineOperand, 4 > Cond
MachineRegisterInfo & MRI
Contains matchers for matching SelectionDAG nodes and values.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static std::pair< SDValue, SDValue > getLegalMaskAndStepVector(SDValue Mask, bool ZeroIsPoison, SDLoc DL, SelectionDAG &DAG)
Returns a type-legalized version of Mask as the first item in the pair.
static SDValue foldSetCCWithFunnelShift(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, const SDLoc &dl, SelectionDAG &DAG)
static bool lowerImmediateIfPossible(TargetLowering::ConstraintPair &P, SDValue Op, SelectionDAG *DAG, const TargetLowering &TLI)
If we have an immediate, see if we can lower it.
#define FP_CMP_LIBCALL(BASE)
static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG, const APInt &UndefOp0, const APInt &UndefOp1)
Given a vector binary operation and known undefined elements for each input operand,...
static SDValue BuildExactUDIV(const TargetLowering &TLI, SDNode *N, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created)
Given an exact UDIV by a constant, create a multiplication with the multiplicative inverse of the con...
static void setArgListEntryAttributes(TargetLoweringBase::ArgListEntry &Entry, const SourceT &Src, unsigned ArgIdx)
Set CallLoweringInfo attribute flags based on a call instruction and called function attributes.
static bool canFoldStoreIntoLibCallOutputPointers(StoreSDNode *StoreNode, SDNode *FPNode, unsigned MaxSteps)
Given a store node StoreNode, return true if it is safe to fold that node into FPNode,...
static std::pair< RTLIB::Libcall, ISD::CondCode > selectFPCmpLibcall(const LibcallLoweringInfo &Libcalls, RTLIB::Libcall BoolLC, RTLIB::Libcall TriStateLC, RTLIB::Libcall GenericLC, ISD::CondCode TriStateCC)
Select the libcall and the condition code to test its result against 0 for an ordered floating-point ...
static SDValue isSpecificZeroAfterMaybeRounding(SelectionDAG &DAG, const TargetLowering &TLI, const SDLoc &DL, SDValue Val, FPClassTest FPClass)
static bool canNarrowCLMULToLegal(const TargetLowering &TLI, LLVMContext &Ctx, EVT VT, unsigned HalveDepth=0, unsigned TotalDepth=0)
Check if CLMUL on VT can eventually reach a type with legal CLMUL through a chain of halving decompos...
static SDValue combineShiftToAVGSUB(SDValue Op, TargetLowering::TargetLoweringOpt &TLO, const TargetLowering &TLI, const APInt &DemandedBits, const APInt &DemandedElts, unsigned Depth)
static unsigned getConstraintPiority(TargetLowering::ConstraintType CT)
Return a number indicating our preference for chosing a type of constraint over another,...
static std::optional< EVT > getNarrowAVGType(EVT VT, unsigned KnownBits, LLVMContext &Ctx)
static std::optional< bool > isFCmpEqualZero(FPClassTest Test, const fltSemantics &Semantics, const MachineFunction &MF)
Returns a true value if if this FPClassTest can be performed with an ordered fcmp to 0,...
static SDValue clampDynamicVectorIndex(SelectionDAG &DAG, SDValue Idx, EVT VecVT, const SDLoc &DL, ElementCount SubEC)
static void turnVectorIntoSplatVector(MutableArrayRef< SDValue > Values, std::function< bool(SDValue)> Predicate, SDValue AlternativeReplacement=SDValue())
If all values in Values that don't match the predicate are same 'splat' value, then replace all value...
static MaybeAlign getParamAlign(const CallBase &Call, unsigned ArgIdx)
static MaybeAlign getParamStackAlign(const CallBase &Call, unsigned ArgIdx)
static bool canExpandVectorCTPOP(const TargetLowering &TLI, EVT VT)
static SDValue foldSetCCWithRotate(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, const SDLoc &dl, SelectionDAG &DAG)
static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created)
Given an exact SDIV by a constant, create a multiplication with the multiplicative inverse of the con...
static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT, SDValue N0, const APInt &C1, ISD::CondCode Cond, const SDLoc &dl, SelectionDAG &DAG)
static SDValue combineShiftToAVG(SDValue Op, TargetLowering::TargetLoweringOpt &TLO, const TargetLowering &TLI, const APInt &DemandedBits, const APInt &DemandedElts, unsigned Depth)
static bool paramHasAttr(const CallBase &Call, unsigned ArgIdx, Attribute::AttrKind Kind)
This file describes how to lower LLVM code to machine code.
static int Lookup(ArrayRef< TableEntry > Table, unsigned Opcode)
static SDValue scalarizeVectorStore(StoreSDNode *Store, MVT StoreVT, SelectionDAG &DAG)
Scalarize a vector store, bitcasting to TargetVT to determine the scalar type.
static LLVM_ABI const llvm::fltSemantics & EnumToSemantics(Semantics S)
static constexpr roundingMode rmTowardZero
static LLVM_ABI ExponentType semanticsMinExponent(const fltSemantics &)
static LLVM_ABI bool semanticsHasSignedRepr(const fltSemantics &)
static LLVM_ABI unsigned getSizeInBits(const fltSemantics &Sem)
Returns the size of the floating point number (in bits) in the given semantics.
static constexpr roundingMode rmNearestTiesToEven
static LLVM_ABI ExponentType semanticsMaxExponent(const fltSemantics &)
static LLVM_ABI unsigned int semanticsPrecision(const fltSemantics &)
static LLVM_ABI bool isIEEELikeFP(const fltSemantics &)
opStatus
IEEE-754R 7: Default exception handling.
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
APInt bitcastToAPInt() const
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
static APFloat getNaN(const fltSemantics &Sem, bool Negative=false, uint64_t payload=0)
Factory for NaN values.
Class for arbitrary precision integers.
LLVM_ABI APInt udiv(const APInt &RHS) const
Unsigned division operation.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
static LLVM_ABI void udivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient, APInt &Remainder)
Dual division/remainder interface.
void clearBit(unsigned BitPosition)
Set a given bit to 0.
bool isNegatedPowerOf2() const
Check if this APInt's negated value is a power of two greater than zero.
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
uint64_t getZExtValue() const
Get zero extended value.
LLVM_ABI APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
LLVM_ABI APInt urem(const APInt &RHS) const
Unsigned remainder operation.
void setSignBit()
Set the sign bit to 1.
unsigned getBitWidth() const
Return the number of bits in the APInt.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
static APInt getMinValue(unsigned numBits)
Gets minimum unsigned value of APInt for a specific bit width.
bool isNegative() const
Determine sign of this APInt.
void clearAllBits()
Set every bit to 0.
void ashrInPlace(unsigned ShiftAmt)
Arithmetic right-shift this APInt by ShiftAmt in place.
LLVM_ABI APInt uadd_ov(const APInt &RHS, bool &Overflow) const
void negate()
Negate this APInt in place.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned countl_zero() const
The APInt version of std::countl_zero.
static LLVM_ABI APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
unsigned countLeadingZeros() const
bool isStrictlyPositive() const
Determine if this APInt Value is positive.
void flipAllBits()
Toggle every bit to its opposite value.
LLVM_ABI void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
void clearLowBits(unsigned loBits)
Set bottom loBits bits to 0.
unsigned logBase2() const
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
void setAllBits()
Set every bit to 1.
LLVM_ABI APInt multiplicativeInverse() const
bool isMask(unsigned numBits) const
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
LLVM_ABI APInt sext(unsigned width) const
Sign extend to a new width.
void setBits(unsigned loBit, unsigned hiBit)
Set the bits from loBit (inclusive) to hiBit (exclusive) to 1.
APInt shl(unsigned shiftAmt) const
Left-shift function.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
void clearBits(unsigned LoBit, unsigned HiBit)
Clear the bits from LoBit (inclusive) to HiBit (exclusive) to 0.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
LLVM_ABI APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
bool isOne() const
Determine if this is a value of 1.
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
void clearHighBits(unsigned hiBits)
Set top hiBits bits to 0.
int64_t getSExtValue() const
Get sign extended value.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
unsigned countr_one() const
Count the number of trailing one bits.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
void setBitVal(unsigned BitPosition, bool BitValue)
Set a given bit to a given value.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
AttrKind
This enumeration lists the attributes that can be associated with parameters, function results,...
@ None
No attributes have been set.
A "pseudo-class" with methods for operating on BUILD_VECTORs.
LLVM_ABI ConstantSDNode * getConstantSplatNode(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted constant or null if this is not a constant splat.
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
LLVM_ABI bool isIndirectCall() const
Return true if the callsite is an indirect call.
This class represents a function call, abstracting a target machine's calling convention.
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
ConstantFP - Floating Point Values [float, double].
This class represents a range of values.
const APInt & getAPIntValue() const
This is an important base class in LLVM.
A parsed version of the target data layout string in and methods for querying it.
bool isLittleEndian() const
Layout endianness...
LLVM_ABI Align getABITypeAlign(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
LLVM_ABI Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
static constexpr ElementCount getFixed(ScalarTy MinVal)
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
constexpr bool isScalar() const
Exactly one element.
constexpr bool isPowerOf2() const
Class to represent function types.
AttributeList getAttributes() const
Return the attribute list for this Function.
int64_t getOffset() const
const GlobalValue * getGlobal() const
Module * getParent()
Get the module that this global value is contained inside of...
std::vector< std::string > ConstraintCodeVector
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
This is an important class for using LLVM in a threaded context.
LLVM_ABI void emitError(const Instruction *I, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
Tracks which library functions to use for a particular subtarget or function.
CallingConv::ID getLibcallImplCallingConv(RTLIB::LibcallImpl Call) const
Get the CallingConv that should be used for the specified libcall.
RTLIB::LibcallImpl getLibcallImpl(RTLIB::Libcall Call) const
Return the lowering's selection of implementation call for Call.
RTLIB::LibcallImpl getVectorLibcallImplOrMasked(RTLIB::Libcall Call) const
Return the implementation for Call, or for its masked variant if Call has no implementation.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Context object for machine code objects.
Base class for the full range of assembler expressions which are needed for parsing.
iterator_range< regclass_iterator > regclasses() const
Wrapper class representing physical registers. Should be passed by value.
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx, SMLoc Loc=SMLoc())
bool isInteger() const
Return true if this is an integer or a vector integer type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getIntegerVT(unsigned BitWidth)
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
void setAdjustsStack(bool V)
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
DenormalMode getDenormalMode(const fltSemantics &FPType) const
Returns the denormal handling type for the default rounding mode of the function.
MCSymbol * getJTISymbol(unsigned JTI, MCContext &Ctx, bool isLinkerPrivate=false) const
getJTISymbol - Return the MCSymbol for the specified non-empty jump table.
Function & getFunction()
Return the LLVM function that this machine code represents.
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
@ EK_LabelDifference32
EK_LabelDifference32 - Each entry is the address of the block minus the address of the jump table.
@ EK_BlockAddress
EK_BlockAddress - Each entry is a plain address of block, e.g.: .word LBB123.
Flags getFlags() const
Return the raw flags of the source value,.
static bool clobbersPhysReg(const uint32_t *RegMask, MCRegister PhysReg)
clobbersPhysReg - Returns true if this RegMask clobbers PhysReg.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
unsigned getAddressSpace() const
Return the address space for the associated pointer.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return the unique MachineMemOperand object describing the memory reference performed by operation.
MMOMetadata getMMOMetadataForSubAccess() const
Returns metadata that can be copied unchanged to an access covering all or part of this access's byte...
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
const GlobalVariable * getNamedGlobal(StringRef Name) const
Return the global variable in the module with the specified name, of arbitrary type.
Represent a mutable reference to an array (0 or more elements consecutively in memory),...
Class to represent pointers.
static PointerType * getUnqual(LLVMContext &C)
This constructs an opaque pointer to an object in the default address space (address space zero).
static LLVM_ABI PointerType * get(LLVMContext &C, unsigned AddressSpace)
This constructs an opaque pointer to an object in a numbered address space.
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
SDNodeFlags getFlags() const
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node, in exactly one operand.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
ISD::CondCode getOperandCondCode(unsigned i) const
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC)
bool willNotOverflowAdd(bool IsSigned, SDValue N0, SDValue N1) const
Determine if the result of the addition of 2 nodes can never overflow.
LLVM_ABI Align getReducedAlign(EVT VT, bool UseABI)
In most cases this function returns the ABI alignment for a given type, except for illegal vector typ...
LLVM_ABI bool isKnownNeverLogicalZero(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
Test whether the given floating point SDValue (or all elements of it, if it is a vector) is known to ...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
SDValue getExtractVectorElt(const SDLoc &DL, EVT VT, SDValue Vec, unsigned Idx)
Extract element at Idx from Vec.
LLVM_ABI unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
LLVM_ABI unsigned getHasPredecessorMaxSteps() const
LLVM_ABI SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl, SDNodeFlags Flags={})
Constant fold a setcc to true or false.
bool isKnownNeverSNaN(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
LLVM_ABI void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
LLVM_ABI SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
LLVM_ABI SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
LLVM_ABI SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
LLVM_ABI bool isConstantIntBuildVectorOrConstantInt(SDValue N, bool AllowOpaques=true) const
Test whether the given value is a constant int or similar node.
LLVM_ABI SDValue getJumpTableDebugInfo(int JTI, SDValue Chain, const SDLoc &DL)
LLVM_ABI std::optional< unsigned > getValidMaximumShiftAmount(SDValue V, const APInt &DemandedElts, unsigned Depth=0) const
If a SHL/SRA/SRL node V has shift amounts that are all less than the element bit-width of the shift n...
LLVM_ABI SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
LLVM_ABI SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getExtractSubvector(const SDLoc &DL, EVT VT, SDValue Vec, unsigned Idx)
Return the VT typed sub-vector of Vec at Idx.
SDValue getInsertSubvector(const SDLoc &DL, SDValue Vec, SDValue SubVec, unsigned Idx)
Insert SubVec at the Idx element of Vec.
LLVM_ABI SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false, SDNodeFlags Flags={})
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
bool willNotOverflowSub(bool IsSigned, SDValue N0, SDValue N1) const
Determine if the result of the sub of 2 nodes can never overflow.
LLVM_ABI bool shouldOptForSize() const
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
static constexpr unsigned MaxRecursionDepth
bool isGuaranteedNotToBePoison(SDValue Op, unsigned Depth=0) const
Return true if this function can prove that Op is never poison.
LLVM_ABI std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
LLVM_ABI SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
LLVM_ABI std::optional< unsigned > getValidShiftAmount(SDValue V, const APInt &DemandedElts, unsigned Depth=0) const
If a SHL/SRA/SRL node V has a uniform shift amount that is less than the element bit-width of the shi...
LLVM_ABI SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Helper function to build ISD::STORE nodes.
LLVM_ABI bool doesNodeExist(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops)
Check if a node exists without modifying its flags.
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
LLVM_ABI SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
LLVM_ABI SDValue getTypeSize(const SDLoc &DL, EVT VT, TypeSize TS)
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
LLVM_ABI SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
LLVM_ABI bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
LLVM_ABI void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
LLVM_ABI SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI bool isIdentityElement(unsigned Opc, SDNodeFlags Flags, SDValue V, unsigned OperandNo, unsigned Depth=0) const
Returns true if V is an identity element of Opc with Flags.
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, UndefPoisonKind Kind=UndefPoisonKind::UndefOrPoison, unsigned Depth=0) const
Return true if this function can prove that Op is never poison and, Kind can be used to track poison ...
LLVM_ABI bool isKnownNeverZero(SDValue Op, unsigned Depth=0) const
Test whether the given SDValue is known to contain non-zero value(s).
LLVM_ABI SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SDNodeFlags Flags=SDNodeFlags())
LLVM_ABI SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
LLVM_ABI SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
const LibcallLoweringInfo & getLibcalls() const
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
LLVM_ABI bool isKnownNeverNaN(SDValue Op, const APInt &DemandedElts, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN in...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
LLVM_ABI SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getPOISON(EVT VT)
Return a POISON node. POISON does not have a useful SDLoc.
LLVM_ABI KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVM_ABI SDValue getCondCode(ISD::CondCode Cond)
LLVM_ABI bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
LLVM_ABI bool isKnownToBeAPowerOfTwo(SDValue Val, bool OrZero=false, unsigned Depth=0) const
Test if the given value is known to have exactly one bit set.
LLVM_ABI SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
LLVM_ABI std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
LLVM_ABI SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
static void commuteMask(MutableArrayRef< int > Mask)
Change values in a shuffle permute mask assuming the two vector operands have swapped position.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
Represent a constant reference to a string, i.e.
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr size_t size() const
Get the string size.
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
Class to represent struct types.
LLVM_ABI void setExtAttribute(Attribute::AttrKind Attr)
LLVM_ABI void setAttributes(const CallBase *Call, unsigned ArgIdx)
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
virtual IntegerType * getCmpLibcallReturnType(LLVMContext &Ctx) const
Return the integer type returned by comparison libcalls.
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
virtual bool shouldRemoveRedundantExtend(SDValue Op) const
Return true (the default) if it is profitable to remove a sext_inreg(x) where the sext is redundant,...
virtual bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT, std::optional< unsigned > ByteOffset=std::nullopt) const
Return true if it is profitable to reduce a load to a smaller type.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
virtual bool preferSelectsOverBooleanArithmetic(EVT VT) const
Should we prefer selects to doing arithmetic on boolean types.
virtual bool isLegalICmpImmediate(int64_t) const
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
virtual bool isSafeMemOpType(MVT) const
Returns true if it's safe to use load / store of the specified type to expand memcpy / memset inline.
const TargetMachine & getTargetMachine() const
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
bool isPaddedAtMostSignificantBitsWhenStored(EVT VT) const
Indicates if any padding is guaranteed to go at the most significant bits when storing the type to me...
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize=false) const
Returns true if the specified FP immediate can be materialized without a load from the constant pool.
virtual bool hasBitTest(SDValue X, SDValue Y) const
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
MVT getRegisterType(LLVMContext &Context, EVT VT) const
Return the type of registers that this ValueType will eventually require.
EVT getLegalTypeToTransformTo(LLVMContext &Context, EVT VT) const
Perform getTypeToTransformTo repeatedly until a legal type is obtained.
LegalizeAction getCondCodeAction(ISD::CondCode CC, MVT VT) const
Return how the condition code should be treated: either it is legal, needs to be expanded to some oth...
virtual bool isCommutativeBinOp(unsigned Opcode) const
Returns true if the opcode is a commutative binary operation.
virtual bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const
Should we tranform the IR-optimal check for whether given truncation down into KeptBits would be trun...
bool isLegalRC(const TargetRegisterInfo &TRI, const TargetRegisterClass &RC) const
Return true if the value types that can be represented by the specified register class are all legal.
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
virtual bool shouldExtendTypeInLibCall(EVT Type) const
Returns true if arguments should be extended in lib calls.
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual bool shouldAvoidTransformToShift(EVT VT, unsigned Amount) const
Return true if creating a shift of the type by the given amount is not profitable.
virtual bool isFPExtFree(EVT DestVT, EVT SrcVT) const
Return true if an fpext operation is free (for instance, because single-precision floating-point numb...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal for a comparison of the specified types on this ...
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
TargetLoweringBase(const TargetMachine &TM, const TargetSubtargetInfo &STI)
NOTE: The TargetMachine owns TLOF.
virtual bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
BooleanContent
Enum that describes how the target represents true/false values.
@ ZeroOrOneBooleanContent
@ UndefinedBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual bool isIntDivCheap(EVT VT, AttributeList Attr) const
Return true if integer divide is usually cheaper than a sequence of several shifts,...
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
virtual bool hasAndNotCompare(SDValue Y) const
Return true if the target should transform: (X & Y) == Y ---> (~X & Y) == 0 (X & Y) !...
virtual bool isNarrowingProfitable(SDNode *N, EVT SrcVT, EVT DestVT) const
Return true if it's profitable to narrow operations of type SrcVT to DestVT.
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
virtual bool isCtlzFast() const
Return true if ctlz instruction is fast.
virtual bool shouldUseStrictFP_TO_INT(EVT FpVT, EVT IntVT, bool IsSigned) const
Return true if it is more correct/profitable to use strict FP_TO_INT conversion operations - canonica...
NegatibleCost
Enum that specifies when a float negation is beneficial.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
virtual bool shouldSignExtendTypeInLibCall(Type *Ty, bool IsSigned) const
Returns true if arguments should be sign-extended in lib calls.
std::vector< ArgListEntry > ArgListTy
virtual EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op, const AttributeList &) const
Returns the target specific optimal type for load and store operations as a result of memset,...
virtual EVT getAsmOperandValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
bool isCondCodeLegalOrCustom(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal or custom for a comparison of the specified type...
bool isLoadLegal(EVT ValVT, EVT MemVT, Align Alignment, unsigned AddrSpace, unsigned ExtType, bool Atomic) const
Return true if the specified load with extension is legal on this target.
virtual bool isFAbsFree(EVT VT) const
Return true if an fabs operation is free to the point where it is never worthwhile to replace it with...
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
MulExpansionKind
Enum that specifies when a multiplication should be expanded.
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
SDValue buildSDIVPow2WithCMov(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Build sdiv by power-of-2 with conditional move instructions Ref: "Hacker's Delight" by Henry Warren 1...
virtual ConstraintWeight getMultipleConstraintMatchWeight(AsmOperandInfo &info, int maIndex) const
Examine constraint type and operand type and determine a weight value.
void expandVectorDeinterleave(SDNode *Node, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const
Expand a VECTOR_DEINTERLEAVE without introducing wider vector types.
bool expandMultipleResultFPLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, SDNode *Node, SmallVectorImpl< SDValue > &Results, std::optional< unsigned > CallRetResNo={}) const
Expands a node with multiple results to an FP or vector libcall.
bool expandMULO(SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]MULO.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
SmallVector< ConstraintPair > ConstraintGroup
virtual const MCExpr * getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI, MCContext &Ctx) const
This returns the relocation base for the given PIC jumptable, the same as getPICJumpTableRelocBase,...
virtual Align computeKnownAlignForTargetInstr(GISelValueTracking &Analysis, Register R, const MachineRegisterInfo &MRI, unsigned Depth=0) const
Determine the known alignment for the pointer value R.
bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Vector Op.
virtual bool isUsedByReturnOnly(SDNode *, SDValue &) const
Return true if result of the specified node is used by a return node only.
bool LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, bool &NeedInvert, const SDLoc &dl, SDValue &Chain, bool IsSignaling=false) const
Legalize a SETCC with given LHS and RHS and condition code CC on the current target.
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
virtual unsigned getPreferredShrunkVectorSizeInBits(SDValue Op, const APInt &DemandedElts) const
If only low elements of a vector are demanded, shrink the operation to the returned size in bits by c...
virtual unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const
This method can be implemented by targets that want to expose additional information about sign bits ...
SDValue lowerCmpEqZeroToCtlzSrl(SDValue Op, SelectionDAG &DAG) const
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
void forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl, bool Signed, const SDValue LHS, const SDValue RHS, SDValue &Lo, SDValue &Hi) const
Calculate full product of LHS and RHS either via a libcall or through brute force expansion of the mu...
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
SDValue expandFCANONICALIZE(SDNode *Node, SelectionDAG &DAG) const
Expand FCANONICALIZE to FMUL with 1.
SDValue expandCTLZ(SDNode *N, SelectionDAG &DAG) const
Expand CTLZ/CTLZ_ZERO_POISON nodes.
SDValue expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const
Expand BITREVERSE nodes.
SDValue expandCTTZ(SDNode *N, SelectionDAG &DAG) const
Expand CTTZ/CTTZ_ZERO_POISON nodes.
virtual SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, int JTI, SelectionDAG &DAG) const
Expands target specific indirect branch for the case of JumpTable expansion.
SDValue expandABD(SDNode *N, SelectionDAG &DAG) const
Expand ABDS/ABDU nodes.
virtual bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const
std::vector< AsmOperandInfo > AsmOperandInfoVector
SDValue expandGetActiveLaneMask(SDNode *N, SelectionDAG &DAG) const
Expand GET_ACTIVE_LANE_MASK nodes.
SDValue expandCLMUL(SDNode *N, SelectionDAG &DAG) const
Expand carryless multiply.
SDValue expandShlSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]SHLSAT.
SDValue expandIS_FPCLASS(EVT ResultVT, SDValue Op, FPClassTest Test, SDNodeFlags Flags, const SDLoc &DL, SelectionDAG &DAG) const
Expand check for floating point class.
virtual bool isTargetCanonicalConstantNode(SDValue Op) const
Returns true if the given Opc is considered a canonical constant for the target, which should not be ...
SDValue expandVECTOR_SHUFFLE_VAR(SDNode *Node, SelectionDAG &DAG) const
Expand a VECTOR_SHUFFLE_VAR into variable-index element extracts.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandCttzElts(SDNode *Node, SelectionDAG &DAG) const
Expand a CTTZ_ELTS or CTTZ_ELTS_ZERO_POISON by calculating (VL - i) for each active lane (i),...
SDValue getCheaperNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, unsigned Depth=0) const
This is the helper function to return the newly negated expression only when the cost is cheaper.
virtual unsigned computeNumSignBitsForTargetInstr(GISelValueTracking &Analysis, Register R, const APInt &DemandedElts, const MachineRegisterInfo &MRI, unsigned Depth=0) const
This method can be implemented by targets that want to expose additional information about sign bits ...
SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth=0) const
More limited version of SimplifyDemandedBits that can be used to "lookthrough" ops that don't contrib...
SDValue expandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG) const
Expands an unaligned store to 2 half-size stores for integer values, and possibly more for vectors.
SDValue SimplifyMultipleUseDemandedVectorElts(SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth=0) const
Helper wrapper around SimplifyMultipleUseDemandedBits, demanding all bits from only some vector eleme...
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual bool findOptimalMemOpLowering(LLVMContext &Context, std::vector< EVT > &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS, unsigned SrcAS, const AttributeList &FuncAttributes, EVT *LargestVT=nullptr) const
Determines the optimal series of memory ops to replace the memset / memcpy.
virtual SDValue unwrapAddress(SDValue N) const
void expandSADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::S(ADD|SUB)O.
SDValue expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
bool ShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const
Check to see if the specified operand of the specified instruction is a constant integer.
virtual bool isGuaranteedNotToBeUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, UndefPoisonKind Kind, unsigned Depth) const
Return true if this function can prove that Op is never poison and, Kind can be used to track poison ...
SDValue expandMULH(SDNode *Node, SelectionDAG &DAG) const
SDValue expandVPCTTZElements(SDNode *N, SelectionDAG &DAG) const
Expand VP_CTTZ_ELTS/VP_CTTZ_ELTS_ZERO_POISON nodes.
SDValue BuildSDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, bool IsAfterLegalTypes, SmallVectorImpl< SDNode * > &Created) const
Given an ISD::SDIV node expressing a divide by constant, return a DAG expression to select that will ...
virtual const char * getTargetNodeName(unsigned Opcode) const
This method returns the name of a target specific DAG node.
bool expandFP_TO_UINT(SDNode *N, SDValue &Result, SDValue &Chain, SelectionDAG &DAG) const
Expand float to UINT conversion.
bool parametersInCSRMatch(const MachineRegisterInfo &MRI, const uint32_t *CallerPreservedMask, const SmallVectorImpl< CCValAssign > &ArgLocs, const SmallVectorImpl< SDValue > &OutVals) const
Check whether parameters to a call that are passed in callee saved registers are the same as from the...
virtual bool SimplifyDemandedVectorEltsForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded vector elements, returning true on success...
bool expandREM(SDNode *Node, SDValue &Result, SelectionDAG &DAG) const
Expand an SREM or UREM using SDIV/UDIV or SDIVREM/UDIVREM, if legal.
std::pair< SDValue, SDValue > expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Expands an unaligned load to 2 half-size loads for an integer, and possibly more for vectors.
SDValue expandFMINIMUMNUM_FMAXIMUMNUM(SDNode *N, SelectionDAG &DAG) const
Expand fminimumnum/fmaximumnum into multiple comparison with selects.
void expandVectorInterleave(SDNode *Node, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const
Expand a VECTOR_INTERLEAVE without introducing wider vector types.
void forceExpandMultiply(SelectionDAG &DAG, const SDLoc &dl, bool Signed, SDValue &Lo, SDValue &Hi, SDValue LHS, SDValue RHS, SDValue HiLHS=SDValue(), SDValue HiRHS=SDValue()) const
Calculate the product twice the width of LHS and RHS.
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
virtual bool isTypeDesirableForOp(unsigned, EVT VT) const
Return true if the target has native support for the specified value type and it is 'desirable' to us...
SDValue expandVectorSplice(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::VECTOR_SPLICE.
SDValue getVectorSubVecPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, EVT SubVecVT, SDValue Index, const SDNodeFlags PtrArithFlags=SDNodeFlags()) const
Get a pointer to a sub-vector of type SubVecVT at index Idx located in memory for a vector of type Ve...
SDValue expandLoopDependenceMask(SDNode *N, SelectionDAG &DAG) const
Expand LOOP_DEPENDENCE_MASK nodes.
virtual const char * LowerXConstraint(EVT ConstraintVT) const
Try to replace an X constraint, which matches anything, with another that has more specific requireme...
SDValue expandCTPOP(SDNode *N, SelectionDAG &DAG) const
Expand CTPOP nodes.
virtual void computeKnownBitsForTargetInstr(GISelValueTracking &Analysis, Register R, KnownBits &Known, const APInt &DemandedElts, const MachineRegisterInfo &MRI, unsigned Depth=0) const
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
SDValue BuildUDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, bool IsAfterLegalTypes, SmallVectorImpl< SDNode * > &Created) const
Given an ISD::UDIV node expressing a divide by constant, return a DAG expression to select that will ...
SDValue expandVectorNaryOpBySplitting(SDNode *Node, SelectionDAG &DAG) const
~TargetLowering() override
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
SDValue expandBSWAP(SDNode *N, SelectionDAG &DAG) const
Expand BSWAP nodes.
SDValue expandFMINIMUM_FMAXIMUM(SDNode *N, SelectionDAG &DAG) const
Expand fminimum/fmaximum into multiple comparison with selects.
SDValue CTTZTableLookup(SDNode *N, SelectionDAG &DAG, const SDLoc &DL, EVT VT, SDValue Op, unsigned NumBitsPerElt) const
Expand CTTZ via Table Lookup.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using an n/2-bit algorithm.
virtual void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool isPositionIndependent() const
std::pair< StringRef, TargetLowering::ConstraintType > ConstraintPair
virtual SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, NegatibleCost &Cost, unsigned Depth=0) const
Return the newly negated expression if the cost is not expensive and set the cost in Cost to indicate...
virtual ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const
Examine constraint string and operand type and determine a weight value.
ConstraintGroup getConstraintPreferences(AsmOperandInfo &OpInfo) const
Given an OpInfo with list of constraints codes as strings, return a sorted Vector of pairs of constra...
bool expandFP_TO_SINT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const
Expand float(f32) to SINT(i64) conversion.
virtual SDValue SimplifyMultipleUseDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth) const
More limited version of SimplifyDemandedBits that can be used to "lookthrough" ops that don't contrib...
virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Glue, const SDLoc &DL, const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const
SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, SDValue N1, MutableArrayRef< int > Mask, SelectionDAG &DAG) const
Tries to build a legal vector shuffle using the provided parameters or equivalent variations.
static ArgListTy getArgListForFunctionType(FunctionType *FuncTy, const AttributeList &FuncAttrs, ArrayRef< SDValue > Ops)
Build a call argument list for FuncTy, taking the argument node values from Ops and the parameter typ...
virtual void computeKnownBitsForStackObjectPointer(KnownBits &Known, const MachineFunction &MF, Align Alignment) const
Determine known bits of a pointer to a known valid stack object.
virtual SDValue getPICJumpTableRelocBase(SDValue Table, SelectionDAG &DAG) const
Returns relocation base for the given PIC jumptable.
std::pair< SDValue, SDValue > scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Turn load of vector type into a load of the individual elements.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
virtual bool isDesirableToCommuteXorWithShift(const SDNode *N) const
Return true if it is profitable to combine an XOR of a logical shift to create a logical shift of NOT...
TargetLowering(const TargetLowering &)=delete
virtual bool shouldSimplifyDemandedVectorElts(SDValue Op, const TargetLoweringOpt &TLO) const
Return true if the target supports simplifying demanded vector elements by converting them to undefs.
bool isConstFalseVal(SDValue N) const
Return if the N is a constant or constant vector equal to the false value from getBooleanContents().
SDValue IncrementMemoryAddress(SDValue Addr, SDValue Mask, const SDLoc &DL, EVT DataVT, SelectionDAG &DAG, bool IsCompressedMemory) const
Increments memory address Addr according to the type of the value DataVT that should be stored.
SDValue expandVectorMatch(SDNode *N, SelectionDAG &DAG) const
Expand VECTOR_MATCH nodes.
bool isInTailCallPosition(SelectionDAG &DAG, SDNode *Node, SDValue &Chain) const
Check whether a given call node is in tail position within its function.
SDValue expandCONVERT_TO_ARBITRARY_FP(SDNode *Node, SelectionDAG &DAG) const
Expand CONVERT_TO_ARBITRARY_FP using bit manipulation.
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &UndefElts, const SelectionDAG &DAG, unsigned Depth=0) const
Return true if vector Op has the same value across all DemandedElts, indicating any elements which ma...
SDValue expandRoundInexactToOdd(EVT ResultVT, SDValue Op, const SDLoc &DL, SelectionDAG &DAG) const
Truncate Op to ResultVT.
virtual bool shouldSplitFunctionArgumentsAsLittleEndian(const DataLayout &DL) const
For most targets, an LLVM type must be broken down into multiple smaller types.
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
SDValue expandFunnelShift(SDNode *N, SelectionDAG &DAG) const
Expand funnel shift.
virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const
Return true if folding a constant offset with the given GlobalAddress is legal.
bool isExtendedTrueVal(const ConstantSDNode *N, EVT VT, bool SExt) const
Return if N is a True value when extended to VT.
bool ShrinkDemandedOp(SDValue Op, unsigned BitWidth, const APInt &DemandedBits, TargetLoweringOpt &TLO) const
Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
bool isConstTrueVal(SDValue N) const
Return if the N is a constant or constant vector equal to the true value from getBooleanContents().
SDValue expandFixedPointDiv(unsigned Opcode, const SDLoc &dl, SDValue LHS, SDValue RHS, unsigned Scale, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]DIVFIX[SAT].
SDValue expandPEXT(SDNode *N, SelectionDAG &DAG) const
Expand parallel bit extract (compress).
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
virtual void CollectTargetIntrinsicOperands(const CallInst &I, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG) const
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
bool expandMODF(SDNode *N, SelectionDAG &DAG, SmallVectorImpl< SDValue > &Results) const
Expand MODF node into fractional and integer parts.
SDValue expandVECTOR_COMPRESS(SDNode *Node, SelectionDAG &DAG) const
Expand a vector VECTOR_COMPRESS into a sequence of extract element, store temporarily,...
virtual const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const
This method returns the constant pool value that will be loaded by LD.
SDValue expandFP_ROUND(SDNode *Node, SelectionDAG &DAG) const
Expand round(fp) to fp conversion.
SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const
Try to convert the fminnum/fmaxnum to a compare/select sequence.
SDValue expandCONVERT_FROM_ARBITRARY_FP(SDNode *Node, SelectionDAG &DAG) const
Expand CONVERT_FROM_ARBITRARY_FP using bit manipulation.
SDValue expandROT(SDNode *N, bool AllowVectorOps, SelectionDAG &DAG) const
Expand rotations.
SDValue annotateStackObjectPointer(SDValue Ptr, SelectionDAG &DAG, const SDLoc &DL, Align Alignment) const
Annotate a stack object pointer with known-bits assertions.
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual SDValue getSqrtInputTest(SDValue Operand, SelectionDAG &DAG, const DenormalMode &Mode, SDNodeFlags Flags={}) const
Return a target-dependent comparison result if the input operand is suitable for use with a square ro...
SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Index, const SDNodeFlags PtrArithFlags=SDNodeFlags()) const
Get a pointer to vector element Idx located in memory for a vector of type VecVT starting at a base a...
SDValue expandFMINNUM_FMAXNUM(SDNode *N, SelectionDAG &DAG) const
Expand fminnum/fmaxnum into fminnum_ieee/fmaxnum_ieee with quieted inputs.
virtual bool isGAPlusOffset(SDNode *N, const GlobalValue *&GA, int64_t &Offset) const
Returns true (and the GlobalValue and the offset) if the node is a GlobalAddress + offset.
virtual void computeKnownFPClassForTargetNode(const SDValue Op, KnownFPClass &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const
Determine floating-point class information for a target node.
virtual unsigned getJumpTableEncoding() const
Return the entry encoding for a jump table in the current function.
virtual void computeKnownFPClassForTargetInstr(GISelValueTracking &Analysis, Register R, KnownFPClass &Known, const APInt &DemandedElts, const MachineRegisterInfo &MRI, unsigned Depth=0) const
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::LibcallImpl LibcallImpl, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandCMP(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]CMP.
void expandShiftParts(SDNode *N, SDValue &Lo, SDValue &Hi, SelectionDAG &DAG) const
Expand shift-by-parts.
virtual bool isKnownNeverNaNForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool SNaN=false, unsigned Depth=0) const
If SNaN is false,.
virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const
This method will be invoked for all target nodes and for any target-independent nodes that the target...
SDValue expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[U|S]MULFIX[SAT].
SDValue getInboundsVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Index) const
Get a pointer to vector element Idx located in memory for a vector of type VecVT starting at a base a...
SDValue expandIntMINMAX(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][MIN|MAX].
SDValue expandVectorFindLastActive(SDNode *N, SelectionDAG &DAG) const
Expand VECTOR_FIND_LAST_ACTIVE nodes.
SDValue expandPartialReduceMLA(SDNode *Node, SelectionDAG &DAG) const
Expands PARTIAL_REDUCE_S/UMLA nodes to a series of simpler operations, consisting of zext/sext,...
void expandUADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::U(ADD|SUB)O.
SDValue expandPDEP(SDNode *N, SelectionDAG &DAG) const
Expand parallel bit deposit (expand).
virtual SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Targets may override this function to provide custom SDIV lowering for power-of-2 denominators.
SDValue scalarizeExtractedVectorLoad(EVT ResultVT, const SDLoc &DL, EVT InVecVT, SDValue EltNo, LoadSDNode *OriginalLoad, SelectionDAG &DAG) const
Replace an extraction of a load with a narrowed load.
virtual SDValue BuildSREMPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Targets may override this function to provide custom SREM lowering for power-of-2 denominators.
bool expandUINT_TO_FP(SDNode *N, SDValue &Result, SDValue &Chain, SelectionDAG &DAG) const
Expand UINT(i64) to double(f64) conversion.
bool expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, SDValue LHS, SDValue RHS, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL or [US]MUL_LOHI of n-bit values into two or four nodes, respectively,...
SDValue expandAVG(SDNode *N, SelectionDAG &DAG) const
Expand vector/scalar AVGCEILS/AVGCEILU/AVGFLOORS/AVGFLOORU nodes.
SDValue expandCTLS(SDNode *N, SelectionDAG &DAG) const
Expand CTLS (count leading sign bits) nodes.
void setTypeIdForCallsiteInfo(const CallBase *CB, MachineFunction &MF, MachineFunction::CallSiteInfo &CSInfo) const
Primary interface to the complete machine description for the target machine.
bool isPositionIndependent() const
const Triple & getTargetTriple() const
unsigned EmitCallSiteInfo
The flag enables call site info production.
unsigned EmitCallGraphSection
Emit section containing call graph metadata.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual StringRef getRegAsmName(MCRegister Reg) const
Return the assembly name for Reg.
bool isTypeLegalForClass(const TargetRegisterClass &RC, MVT T) const
Return true if the given TargetRegisterClass has the ValueType T.
TargetSubtargetInfo - Generic base class for all target subtargets.
bool isOSBinFormatCOFF() const
Tests whether the OS uses the COFF binary format.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
bool isIntegerTy() const
True if this is an instance of IntegerType.
LLVM_ABI const fltSemantics & getFltSemantics() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI const Value * stripPointerCastsAndAliases() const
Strip off pointer casts, all-zero GEPs, address space casts, and aliases.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
constexpr bool isKnownMultipleOf(ScalarTy RHS) const
This function tells the caller whether the element count is known at compile time to be a multiple of...
constexpr bool hasKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns true if there exists a value X where RHS*X will result in a value whose quantity matches our ...
constexpr ScalarTy getFixedValue() const
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr ScalarTy getKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns a value X where RHS*X will result in a value whose quantity matches our own.
static constexpr bool isKnownLT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
LLVM_ABI APInt ScaleBitMask(const APInt &A, unsigned NewBitWidth, bool MatchAllBits=false)
Splat/Merge neighboring bits to widen/narrow the bitmask represented by.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
LLVM_ABI bool isConstantSplatVector(SDValue V, APInt &SplatValue)
Node predicates.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ PTRADD
PTRADD represents pointer arithmetic semantics, for targets that opt in using shouldPreservePtrArith(...
@ PARTIAL_REDUCE_SMLA
PARTIAL_REDUCE_[U|S]MLA(Accumulator, Input1, Input2) The partial reduction nodes sign or zero extend ...
@ LOOP_DEPENDENCE_RAW_MASK
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ AVGSUBFLOORS
AVGSUBFLOORS/AVGSUBFLOORU - Averaging subtract - Subtract two integers using an integer of type i[N+1...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ CTTZ_ELTS
Returns the number of number of trailing (least significant) zero elements in a vector.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ VECTOR_FIND_LAST_ACTIVE
Finds the index of the last active mask element Operands: Mask.
@ PSEUDO_FMIN
PSEUDO_FMIN is strictly equivalent to op0 olt op1 ?
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FMULADD
FMULADD - Performs a * b + c, with, or without, intermediate rounding.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ CLMUL
Carry-less multiplication operations.
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ SSUBO
Same for subtraction.
@ BRIND
BRIND - Indirect branch.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor to...
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ GET_ACTIVE_LANE_MASK
GET_ACTIVE_LANE_MASK - this corrosponds to the llvm.get.active.lane.mask intrinsic.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Same as FMINNUM/FMAXNUM, except that a signaling NaN operand deterministi...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ VECTOR_SPLICE_LEFT
VECTOR_SPLICE_LEFT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1, VEC2) left by OFFSET elements an...
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ PEXT
Parallel bit extract (compress) and parallel bit deposit (expand).
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - Same semantics as the llvm.minimum/maximum intrinsics.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ VECTOR_SPLICE_RIGHT
VECTOR_SPLICE_RIGHT(VEC1, VEC2, OFFSET) - Shifts CONCAT_VECTORS(VEC1,VEC2) right by OFFSET elements a...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ CTTZ_ZERO_POISON
Bit counting operators with a poisoned result for zero inputs.
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ CALLSEQ_START
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence,...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor ...
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - Same semantics as the llvm.minimumnum/maximumnum intrinsics.
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ ABS_MIN_POISON
ABS with a poison result for INT_MIN.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
LLVM_ABI NodeType getOppositeSignednessMinMaxOpcode(unsigned MinMaxOpc)
Given a MinMaxOpc of ISD::(U|S)MIN or ISD::(U|S)MAX, returns the corresponding opcode with the opposi...
LLVM_ABI NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
LLVM_ABI CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
unsigned getUnorderedFlavor(CondCode Cond)
This function returns 0 if the condition is always false if an operand is a NaN, 1 if the condition i...
LLVM_ABI bool matchBinaryPredicate(SDValue LHS, SDValue RHS, const APInt &DemandedElts, std::function< bool(ConstantSDNode *, ConstantSDNode *)> Match, bool AllowUndefs=false, bool AllowTypeMismatch=false)
Attempt to match a binary predicate against a pair of scalar/splat constants or every element of a pa...
LLVM_ABI CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
LLVM_ABI StringRef getCondCodeName(CondCode Operation)
Return the name of the given condition code, e.g. "setoeq".
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
LLVM_ABI bool isBuildVectorAllZeros(SDValue V)
Return true if the specified value is a BUILD_VECTOR where all of the elements are 0 or undef.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LLVM_ABI bool isBuildVectorOfConstantSDNodes(SDValue V)
Return true if the specified value is a BUILD_VECTOR node of all ConstantSDNode or undef.
LLVM_ABI NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isUnsignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs an unsigned comparison when used with intege...
bool matchUnaryPredicate(SDValue Op, const APInt &DemandedElts, std::function< bool(ConstantSDNode *)> Match, bool AllowUndefs=false, bool AllowTruncation=false)
Hook for matching ConstantSDNode predicate.
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
match_deferred< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
auto m_Value()
Match an arbitrary value and ignore it.
BinaryOp_match< LHS, RHS, Instruction::Or > m_Or(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
LLVM_ABI Libcall getUREM(EVT VT)
Or< Preds... > m_AnyOf(const Preds &...preds)
TypeSize_match< ElementCount > m_ElementCount(ElementCount &V)
Match any integer constant or constant multiple of VSCALE that can be represented using ElementCount.
bool sd_match(SDValue N, Pattern &&P)
NUses_match< 1, Value_match > m_OneUse()
This is an optimization pass for GlobalISel generic memory operations.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
void stable_sort(R &&Range)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
constexpr uint64_t maxUIntN(uint64_t N)
Gets the maximum value for a N-bit unsigned integer.
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
RelativeUniformCounterPtr Values
LLVM_ABI bool isAllOnesOrAllOnesSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant -1 integer or a splatted vector of a constant -1 integer (with...
@ Known
Known to have no common set bits.
@ Undef
Value of the register doesn't matter.
LLVM_ABI SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI FPClassTest invertFPClassTestIfSimpler(FPClassTest Test, bool UseFCmp)
Evaluates if the specified FP class test is better performed as the inverse (i.e.
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
LLVM_ABI bool isOneOrOneSplatFP(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant floating-point value, or a splatted vector of a constant float...
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
LLVM_ABI bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
LLVM_ABI void reportFatalInternalError(Error Err)
Report a fatal error that indicates a bug in LLVM.
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
LLVM_ABI ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
constexpr bool has_single_bit(T Value) noexcept
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI bool getShuffleDemandedElts(int SrcWidth, ArrayRef< int > Mask, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS, bool AllowUndefElts=false)
Transform a shuffle mask's output demanded element mask into demanded element masks for the 2 operand...
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
LLVM_ABI bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
LLVM_ABI llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
auto find_if_not(R &&Range, UnaryPredicate P)
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
LLVM_ABI ConstantRange getVScaleRange(const Function *F, unsigned BitWidth)
Determine the possible constant range of vscale with the given bit width, based on the vscale_range f...
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ABI bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
@ Mod
The access may modify the value stored in memory.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
To bit_cast(const From &from) noexcept
LLVM_ABI llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ Sub
Subtraction of integers.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
@ Fast
Assign the register banks as fast as possible (default).
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
DWARFExpression::Operation Op
RoundingMode
Rounding mode.
@ TowardZero
roundTowardZero.
@ NearestTiesToEven
roundTiesToEven.
@ TowardPositive
roundTowardPositive.
@ NearestTiesToAway
roundTiesToAway.
@ TowardNegative
roundTowardNegative.
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI bool isZeroOrZeroSplat(SDValue N, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
LLVM_ABI bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
UndefPoisonKind
Enumeration to track whether we are interested in Undef, Poison, or both.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
LLVM_ABI bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
APFloat neg(APFloat X)
Returns the negated value of the argument.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
unsigned Log2(Align A)
Returns the log2 of the alignment.
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
@ Increment
Incrementally increasing token ID.
@ End
Run the machine verifier once, at the end of the pipeline.
@ Keep
No function return thunk.
LLVM_ABI bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
MCRegisterClass TargetRegisterClass
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Represent subnormal handling kind for floating point instruction inputs and outputs.
DenormalModeKind Input
Denormal treatment kind for floating point instruction inputs in the default floating-point environme...
@ PreserveSign
The sign of a flushed-to-zero number is preserved in the sign of 0.
@ PositiveZero
Denormals are flushed to positive zero.
@ IEEE
IEEE-754 denormal numbers preserved.
constexpr bool inputsAreZero() const
Return true if input denormals must be implicitly treated as 0.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
static LLVM_ABI EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
EVT changeVectorElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
std::string getEVTString() const
This function returns value type as a string, e.g. "i32".
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
EVT widenIntegerVectorElementType(LLVMContext &Context) const
Return a VT for an integer vector type with the size of the elements doubled.
EVT changeVectorElementCount(LLVMContext &Context, ElementCount EC) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element coun...
bool isScalableVT() const
Return true if the type is a scalable type.
bool isFixedLengthVector() const
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
EVT widenIntegerElementType(LLVMContext &Context) const
Return a VT for an integer element type with doubled bit width.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
EVT changeElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a type whose attributes match ourselves with the exception of the element type that i...
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
LLVM_ABI const fltSemantics & getFltSemantics() const
Returns an APFloat semantics tag appropriate for the value type.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
bool isInteger() const
Return true if this is an integer or a vector integer type.
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
KnownBits anyextOrTrunc(unsigned BitWidth) const
Return known bits for an "any" extension or truncation of the value we're tracking.
static LLVM_ABI KnownBits smax(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for smax(LHS, RHS).
bool isNonNegative() const
Returns true if this value is known to be non-negative.
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
KnownBits byteSwap() const
static LLVM_ABI std::optional< bool > sge(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SGE result.
unsigned countMaxPopulation() const
Returns the maximum number of bits that could be one.
KnownBits reverseBits() const
KnownBits concat(const KnownBits &Lo) const
Concatenate the bits from Lo onto the bottom of *this.
static LLVM_ABI KnownBits umax(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for umax(LHS, RHS).
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
static LLVM_ABI KnownBits smin(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for smin(LHS, RHS).
static LLVM_ABI std::optional< bool > ugt(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_UGT result.
static LLVM_ABI std::optional< bool > slt(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SLT result.
static LLVM_ABI KnownBits computeForAddSub(bool Add, bool NSW, bool NUW, const KnownBits &LHS, const KnownBits &RHS)
Compute known bits resulting from adding LHS and RHS.
static LLVM_ABI std::optional< bool > ult(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_ULT result.
static LLVM_ABI std::optional< bool > ule(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_ULE result.
bool isNegative() const
Returns true if this value is known to be negative.
static LLVM_ABI KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
static LLVM_ABI std::optional< bool > sle(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SLE result.
static LLVM_ABI std::optional< bool > sgt(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SGT result.
unsigned countMinPopulation() const
Returns the number of bits known to be one.
static LLVM_ABI std::optional< bool > uge(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_UGE result.
static LLVM_ABI KnownBits umin(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for umin(LHS, RHS).
This class contains a discriminated union of information about pointers in memory operands,...
LLVM_ABI unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
static LLVM_ABI MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static LLVM_ABI MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
static bool hasVectorMaskArgument(RTLIB::LibcallImpl Impl)
Returns true if the function has a vector mask argument, which is assumed to be the last argument.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoUnsignedWrap() const
bool hasNoSignedWrap() const
void setNoSignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Magic data for optimising signed division by a constant.
unsigned ShiftAmount
shift amount
static LLVM_ABI SignedDivisionByConstantInfo get(const APInt &D)
Calculate the magic numbers required to implement a signed integer division by a constant as a sequen...
This contains information for each constraint that we are lowering.
std::string ConstraintCode
This contains the actual string for the code, like "m".
LLVM_ABI unsigned getMatchedOperand() const
If this is an input matching constraint, this method returns the output operand it matches.
LLVM_ABI bool isMatchingInputConstraint() const
Return true of this is an input operand that is a matching constraint like "4".
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setIsPostTypeLegalization(bool Value=true)
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setZExtResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setSExtResult(bool Value=true)
CallLoweringInfo & setNoReturn(bool Value=true)
CallLoweringInfo & setChain(SDValue InChain)
bool isBeforeLegalizeOps() const
LLVM_ABI void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
bool isBeforeLegalize() const
LLVM_ABI void CommitTargetLoweringOpt(const TargetLoweringOpt &TLO)
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setIsPostTypeLegalization(bool Value=true)
ArrayRef< EVT > OpsVTBeforeSoften
bool IsPostTypeLegalization
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT)
ArrayRef< Type * > OpsTypeOverrides
MakeLibCallOptions & setIsSigned(bool Value=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)
bool LegalOperations() const
Magic data for optimising unsigned division by a constant.
unsigned PreShift
pre-shift amount
unsigned PostShift
post-shift amount
static LLVM_ABI UnsignedDivisionByConstantInfo get(const APInt &D, unsigned LeadingZeros=0, bool AllowEvenDivisorOptimization=true, bool AllowWidenOptimization=false)
Calculate the magic numbers required to implement an unsigned integer division by a constant as a seq...
bool Widen
use widen optimization
fltNonfiniteBehavior nonFiniteBehavior
fltNanEncoding nanEncoding