61 .removeAttribute(Attribute::NonNull)
78 for (
unsigned I = 0,
E = ArgLocs.
size();
I !=
E; ++
I) {
106 IsSRet = Call->
paramHasAttr(ArgIdx, Attribute::StructRet);
109 IsInAlloca = Call->
paramHasAttr(ArgIdx, Attribute::InAlloca);
110 IsReturned = Call->
paramHasAttr(ArgIdx, Attribute::Returned);
111 IsSwiftSelf = Call->
paramHasAttr(ArgIdx, Attribute::SwiftSelf);
112 IsSwiftError = Call->
paramHasAttr(ArgIdx, Attribute::SwiftError);
121 std::pair<SDValue, SDValue>
125 const SDLoc &dl)
const {
127 Args.reserve(Ops.
size());
130 for (
unsigned i = 0; i < Ops.
size(); ++i) {
142 Args.push_back(Entry);
145 if (LC == RTLIB::UNKNOWN_LIBCALL)
153 bool zeroExtend = !signExtend;
157 signExtend = zeroExtend =
false;
173 unsigned Limit, uint64_t
Size,
174 unsigned DstAlign,
unsigned SrcAlign,
179 unsigned DstAS,
unsigned SrcAS,
188 if (!(SrcAlign == 0 || SrcAlign >= DstAlign))
192 IsMemset, ZeroMemset, MemcpyStrSrc,
217 unsigned NumMemOps = 0;
220 while (VTSize > Size) {
252 if (NumMemOps && AllowOverlap && NewVTSize < Size &&
263 if (++NumMemOps > Limit)
266 MemOps.push_back(VT);
281 &&
"Unsupported setcc type!");
284 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
285 bool ShouldInvertCC =
false;
289 LC1 = (VT ==
MVT::f32) ? RTLIB::OEQ_F32 :
291 (VT ==
MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
295 LC1 = (VT ==
MVT::f32) ? RTLIB::UNE_F32 :
297 (VT ==
MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
301 LC1 = (VT ==
MVT::f32) ? RTLIB::OGE_F32 :
303 (VT ==
MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
307 LC1 = (VT ==
MVT::f32) ? RTLIB::OLT_F32 :
309 (VT ==
MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
313 LC1 = (VT ==
MVT::f32) ? RTLIB::OLE_F32 :
315 (VT ==
MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
319 LC1 = (VT ==
MVT::f32) ? RTLIB::OGT_F32 :
321 (VT ==
MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
324 LC1 = (VT ==
MVT::f32) ? RTLIB::UO_F32 :
326 (VT ==
MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
329 LC1 = (VT ==
MVT::f32) ? RTLIB::O_F32 :
331 (VT ==
MVT::f128) ? RTLIB::O_F128 : RTLIB::O_PPCF128;
335 LC1 = (VT ==
MVT::f32) ? RTLIB::OLT_F32 :
337 (VT ==
MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
338 LC2 = (VT ==
MVT::f32) ? RTLIB::OGT_F32 :
340 (VT ==
MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
343 LC1 = (VT ==
MVT::f32) ? RTLIB::UO_F32 :
345 (VT ==
MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
346 LC2 = (VT ==
MVT::f32) ? RTLIB::OEQ_F32 :
348 (VT ==
MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
352 ShouldInvertCC =
true;
355 LC1 = (VT ==
MVT::f32) ? RTLIB::OGE_F32 :
357 (VT ==
MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
360 LC1 = (VT ==
MVT::f32) ? RTLIB::OGT_F32 :
362 (VT ==
MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
365 LC1 = (VT ==
MVT::f32) ? RTLIB::OLE_F32 :
367 (VT ==
MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
370 LC1 = (VT ==
MVT::f32) ? RTLIB::OLT_F32 :
372 (VT ==
MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
380 SDValue Ops[2] = {NewLHS, NewRHS};
385 NewLHS =
makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl).first;
392 if (LC2 != RTLIB::UNKNOWN_LIBCALL) {
397 NewLHS =
makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl).first;
489 const APInt &
C = Op1C->getAPIntValue();
511 const APInt &Demanded,
514 "ShrinkDemandedOp only supports binary operators!");
516 "ShrinkDemandedOp only supports nodes with one result!");
534 unsigned SmallVTBits = DemandedSize;
537 for (; SmallVTBits < BitWidth; SmallVTBits =
NextPowerOf2(SmallVTBits)) {
546 assert(DemandedSize <= SmallVTBits &&
"Narrowed below demanded bits?");
573 bool AssumeSingleUse)
const {
596 if (DemandedBits == 0 || DemandedElts == 0)
609 if (NumSrcEltBits == NumDstEltBits)
611 Src, DemandedBits, DemandedElts, DAG, Depth + 1))
615 if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0 &&
617 unsigned Scale = NumDstEltBits / NumSrcEltBits;
618 unsigned NumSrcElts = SrcVT.getVectorNumElements();
621 for (
unsigned i = 0; i != Scale; ++i) {
622 unsigned Offset = i * NumSrcEltBits;
625 DemandedSrcBits |= Sub;
626 for (
unsigned j = 0; j != NumElts; ++j)
628 DemandedSrcElts.
setBit((j * Scale) + i);
633 Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
638 if ((NumSrcEltBits % NumDstEltBits) == 0 &&
640 unsigned Scale = NumSrcEltBits / NumDstEltBits;
641 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
644 for (
unsigned i = 0; i != NumElts; ++i)
645 if (DemandedElts[i]) {
646 unsigned Offset = (i % Scale) * NumDstEltBits;
647 DemandedSrcBits.
insertBits(DemandedBits, Offset);
648 DemandedSrcElts.
setBit(i / Scale);
652 Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
708 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&
709 !DemandedElts[CIdx->getZExtValue()])
718 bool AllUndef =
true, IdentityLHS =
true, IdentityRHS =
true;
719 for (
unsigned i = 0; i != NumElts; ++i) {
720 int M = ShuffleMask[i];
721 if (M < 0 || !DemandedElts[i])
724 IdentityLHS &= (M == (int)i);
725 IdentityRHS &= ((M - NumElts) == i);
739 Op, DemandedBits, DemandedElts, DAG, Depth))
756 unsigned Depth,
bool AssumeSingleUse)
const {
757 unsigned BitWidth = OriginalDemandedBits.
getBitWidth();
759 "Mask size mismatches value type size!");
761 unsigned NumElts = OriginalDemandedElts.
getBitWidth();
764 "Unexpected vector size");
767 APInt DemandedElts = OriginalDemandedElts;
780 Known.
One = cast<ConstantSDNode>(
Op)->getAPIntValue();
798 }
else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
811 if (!DemandedElts[0])
844 APInt DemandedVecElts(DemandedElts);
845 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {
846 unsigned Idx = CIdx->getZExtValue();
850 if (!DemandedElts[Idx])
867 if (!!DemandedVecElts) {
868 Known.
One &= KnownVec.
One;
882 APInt BaseElts = DemandedElts;
886 if (Idx.
ule(NumElts - NumSubElts)) {
888 SubElts = DemandedElts.
extractBits(NumSubElts, SubIdx);
904 Known.
One &= KnownSub.
One;
908 Known.
One &= KnownBase.
One;
934 for (
unsigned i = 0; i != NumSubVecs; ++i) {
935 APInt DemandedSubElts =
936 DemandedElts.
extractBits(NumSubElts, i * NumSubElts);
938 Known2, TLO, Depth + 1))
941 if (!!DemandedSubElts) {
942 Known.
One &= Known2.One;
943 Known.
Zero &= Known2.Zero;
952 APInt DemandedLHS(NumElts, 0);
953 APInt DemandedRHS(NumElts, 0);
954 for (
unsigned i = 0; i != NumElts; ++i) {
955 if (!DemandedElts[i])
957 int M = ShuffleMask[i];
965 assert(0 <= M && M < (
int)(2 * NumElts) &&
"Shuffle index out of range");
966 if (M < (
int)NumElts)
969 DemandedRHS.
setBit(M - NumElts);
972 if (!!DemandedLHS || !!DemandedRHS) {
995 Op0, DemandedBits, DemandedLHS, TLO.
DAG, Depth + 1);
997 Op1, DemandedBits, DemandedRHS, TLO.
DAG, Depth + 1);
998 if (DemandedOp0 || DemandedOp1) {
999 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1000 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1019 if ((LHSKnown.
Zero & DemandedBits) ==
1020 (~RHSC->getAPIntValue() & DemandedBits))
1034 LHSKnown.
One == ~RHSC->getAPIntValue()) {
1045 Known2, TLO, Depth + 1))
1052 Op0, DemandedBits, DemandedElts, TLO.
DAG, Depth + 1);
1054 Op1, DemandedBits, DemandedElts, TLO.
DAG, Depth + 1);
1055 if (DemandedOp0 || DemandedOp1) {
1056 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1057 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1094 Known2, TLO, Depth + 1))
1101 Op0, DemandedBits, DemandedElts, TLO.
DAG, Depth + 1);
1103 Op1, DemandedBits, DemandedElts, TLO.
DAG, Depth + 1);
1104 if (DemandedOp0 || DemandedOp1) {
1105 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1106 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1147 Op0, DemandedBits, DemandedElts, TLO.
DAG, Depth + 1);
1149 Op1, DemandedBits, DemandedElts, TLO.
DAG, Depth + 1);
1150 if (DemandedOp0 || DemandedOp1) {
1151 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1152 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1185 if (
C->getAPIntValue() == Known2.
One) {
1194 if (!
C->isAllOnesValue()) {
1195 if (DemandedBits.
isSubsetOf(
C->getAPIntValue())) {
1206 Known = std::move(KnownOut);
1216 assert(!Known.hasConflict() &&
"Bits known to be one AND zero?");
1217 assert(!Known2.hasConflict() &&
"Bits known to be one AND zero?");
1224 Known.One &= Known2.One;
1225 Known.Zero &= Known2.Zero;
1234 assert(!Known.hasConflict() &&
"Bits known to be one AND zero?");
1235 assert(!Known2.hasConflict() &&
"Bits known to be one AND zero?");
1242 Known.One &= Known2.One;
1243 Known.Zero &= Known2.Zero;
1255 BooleanContent::ZeroOrNegativeOneBooleanContent) {
1262 return TLO.CombineTo(Op, Op0);
1270 Known.Zero.setBitsFrom(1);
1279 if (SA->getAPIntValue().uge(BitWidth))
1282 unsigned ShAmt = SA->getZExtValue();
1284 return TLO.CombineTo(Op, Op0);
1294 if (SA2->getAPIntValue().ult(BitWidth)) {
1295 unsigned C1 = SA2->getZExtValue();
1297 int Diff = ShAmt - C1;
1304 return TLO.CombineTo(
1305 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.
getOperand(0), NewSA));
1312 Known, TLO, Depth + 1))
1327 if (ShAmt < InnerBits && DemandedBits.
getActiveBits() <= InnerBits &&
1334 TLO.DAG.getConstant(ShAmt, dl, ShTy));
1335 return TLO.CombineTo(
1347 unsigned InnerShAmt = SA2->getLimitedValue(InnerBits);
1348 if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
1350 (InnerBits - InnerShAmt + ShAmt) &&
1352 SDValue NewSA = TLO.DAG.getConstant(ShAmt - InnerShAmt, dl,
1356 return TLO.CombineTo(
1363 Known.Zero <<= ShAmt;
1364 Known.One <<= ShAmt;
1366 Known.Zero.setLowBits(ShAmt);
1376 if (SA->getAPIntValue().uge(BitWidth))
1379 unsigned ShAmt = SA->getZExtValue();
1381 return TLO.CombineTo(Op, Op0);
1384 APInt InDemandedMask = (DemandedBits << ShAmt);
1400 if (SA2->getAPIntValue().ult(BitWidth)) {
1401 unsigned C1 = SA2->getZExtValue();
1403 int Diff = ShAmt - C1;
1409 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1410 return TLO.CombineTo(
1411 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.
getOperand(0), NewSA));
1421 assert(!Known.hasConflict() &&
"Bits known to be one AND zero?");
1422 Known.Zero.lshrInPlace(ShAmt);
1423 Known.One.lshrInPlace(ShAmt);
1425 Known.Zero.setHighBits(ShAmt);
1438 return TLO.CombineTo(Op, TLO.DAG.getNode(
ISD::SRL, dl, VT, Op0, Op1));
1442 if (SA->getAPIntValue().uge(BitWidth))
1445 unsigned ShAmt = SA->getZExtValue();
1447 return TLO.CombineTo(Op, Op0);
1449 APInt InDemandedMask = (DemandedBits << ShAmt);
1464 assert(!Known.hasConflict() &&
"Bits known to be one AND zero?");
1465 Known.Zero.lshrInPlace(ShAmt);
1466 Known.One.lshrInPlace(ShAmt);
1470 if (Known.Zero[BitWidth - ShAmt - 1] ||
1474 return TLO.CombineTo(
1475 Op, TLO.DAG.getNode(
ISD::SRL, dl, VT, Op0, Op1, Flags));
1482 TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, Op1.
getValueType());
1483 return TLO.CombineTo(Op, TLO.DAG.getNode(
ISD::SRL, dl, VT, Op0, NewSA));
1486 if (Known.One[BitWidth - ShAmt - 1])
1488 Known.One.setHighBits(ShAmt);
1500 unsigned Amt = SA->getAPIntValue().urem(BitWidth);
1506 Known, TLO, Depth + 1))
1513 APInt Demanded0 = DemandedBits.
lshr(IsFSHL ? Amt : (BitWidth - Amt));
1514 APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
1522 Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt));
1523 Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt));
1524 Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1525 Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1526 Known.One |= Known2.One;
1527 Known.Zero |= Known2.Zero;
1537 Known.One = Known2.One.reverseBits();
1538 Known.Zero = Known2.Zero.reverseBits();
1548 unsigned NumSignBits = TLO.DAG.ComputeNumSignBits(Op0);
1549 bool AlreadySignExtended = NumSignBits >= BitWidth - ExVTBits + 1;
1552 if (!AlreadySignExtended) {
1555 EVT ShiftAmtTy = VT;
1556 if (TLO.LegalTypes() && !ShiftAmtTy.
isVector())
1560 TLO.DAG.getConstant(BitWidth - ExVTBits, dl, ShiftAmtTy);
1561 return TLO.CombineTo(Op,
1562 TLO.DAG.getNode(
ISD::SHL, dl, VT, Op0, ShiftAmt));
1568 return TLO.CombineTo(Op, Op0);
1574 InputDemandedBits.
setBit(ExVTBits - 1);
1578 assert(!Known.hasConflict() &&
"Bits known to be one AND zero?");
1584 if (Known.Zero[ExVTBits - 1])
1585 return TLO.CombineTo(
1586 Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT.
getScalarType()));
1589 if (Known.One[ExVTBits - 1]) {
1590 Known.One.setBitsFrom(ExVTBits);
1613 Known.Zero = KnownLo.Zero.zext(BitWidth) |
1614 KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);
1616 Known.One = KnownLo.One.zext(BitWidth) |
1617 KnownHi.One.zext(BitWidth).shl(HalfBitWidth);
1632 if (IsVecInReg && DemandedElts == 1 &&
1634 TLO.DAG.getDataLayout().isLittleEndian())
1635 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1640 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1643 APInt InDemandedBits = DemandedBits.
trunc(InBits);
1648 assert(!Known.hasConflict() &&
"Bits known to be one AND zero?");
1649 assert(Known.getBitWidth() == InBits &&
"Src width has changed?");
1650 Known = Known.zext(BitWidth,
true );
1665 if (IsVecInReg && DemandedElts == 1 &&
1667 TLO.DAG.getDataLayout().isLittleEndian())
1668 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1673 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1676 APInt InDemandedBits = DemandedBits.
trunc(InBits);
1681 InDemandedBits.
setBit(InBits - 1);
1686 assert(!Known.hasConflict() &&
"Bits known to be one AND zero?");
1687 assert(Known.getBitWidth() == InBits &&
"Src width has changed?");
1690 Known = Known.sext(BitWidth);
1693 if (Known.isNonNegative()) {
1697 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1711 if (IsVecInReg && DemandedElts == 1 &&
1713 TLO.DAG.getDataLayout().isLittleEndian())
1714 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1716 APInt InDemandedBits = DemandedBits.
trunc(InBits);
1721 assert(!Known.hasConflict() &&
"Bits known to be one AND zero?");
1722 assert(Known.getBitWidth() == InBits &&
"Src width has changed?");
1723 Known = Known.zext(BitWidth,
false );
1732 APInt TruncMask = DemandedBits.
zext(OperandBitWidth);
1735 Known = Known.trunc(BitWidth);
1739 Src, TruncMask, DemandedElts, TLO.DAG, Depth + 1))
1740 return TLO.CombineTo(Op, TLO.DAG.getNode(
ISD::TRUNCATE, dl, VT, NewSrc));
1757 if (!ShAmt || ShAmt->getAPIntValue().uge(BitWidth))
1761 uint64_t ShVal = ShAmt->getZExtValue();
1763 if (TLO.LegalTypes())
1769 HighBits = HighBits.
trunc(BitWidth);
1771 if (!(HighBits & DemandedBits)) {
1776 return TLO.CombineTo(
1777 Op, TLO.DAG.getNode(
ISD::SRL, dl, VT, NewTrunc, Shift));
1783 assert(!Known.hasConflict() &&
"Bits known to be one AND zero?");
1794 assert(!Known.hasConflict() &&
"Bits known to be one AND zero?");
1796 Known.Zero |= ~InMask;
1807 if (
auto *CIdx = dyn_cast<ConstantSDNode>(Idx))
1808 if (CIdx->getAPIntValue().ult(NumSrcElts))
1813 APInt DemandedSrcBits = DemandedBits;
1814 if (BitWidth > EltBitWidth)
1815 DemandedSrcBits = DemandedSrcBits.
trunc(EltBitWidth);
1822 if (BitWidth > EltBitWidth)
1823 Known = Known.zext(BitWidth,
false );
1846 if (!OpVTLegal && OpVTSizeInBits > 32)
1849 SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);
1850 return TLO.CombineTo(Op,
1851 TLO.DAG.getNode(
ISD::SHL, dl, VT, Sign, ShAmt));
1858 if (SrcVT.
isVector() && (BitWidth % NumSrcEltBits) == 0 &&
1859 TLO.DAG.getDataLayout().isLittleEndian()) {
1860 unsigned Scale = BitWidth / NumSrcEltBits;
1864 for (
unsigned i = 0; i != Scale; ++i) {
1865 unsigned Offset = i * NumSrcEltBits;
1868 DemandedSrcBits |= Sub;
1869 for (
unsigned j = 0; j != NumElts; ++j)
1870 if (DemandedElts[j])
1871 DemandedSrcElts.
setBit((j * Scale) + i);
1875 APInt KnownSrcUndef, KnownSrcZero;
1877 KnownSrcZero, TLO, Depth + 1))
1882 KnownSrcBits, TLO, Depth + 1))
1884 }
else if ((NumSrcEltBits % BitWidth) == 0 &&
1885 TLO.DAG.getDataLayout().isLittleEndian()) {
1886 unsigned Scale = NumSrcEltBits / BitWidth;
1890 for (
unsigned i = 0; i != NumElts; ++i)
1891 if (DemandedElts[i]) {
1892 unsigned Offset = (i % Scale) * BitWidth;
1893 DemandedSrcBits.
insertBits(DemandedBits, Offset);
1894 DemandedSrcElts.
setBit(i / Scale);
1898 APInt KnownSrcUndef, KnownSrcZero;
1900 KnownSrcZero, TLO, Depth + 1))
1906 KnownSrcBits, TLO, Depth + 1))
1913 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1939 TLO.DAG.getNode(Op.
getOpcode(), dl, VT, Op0, Op1, Flags);
1940 return TLO.CombineTo(Op, NewOp);
1948 Op0, LoMask, DemandedElts, TLO.DAG, Depth + 1);
1950 Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);
1951 if (DemandedOp0 || DemandedOp1) {
1954 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1955 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1957 TLO.DAG.getNode(Op.
getOpcode(), dl, VT, Op0, Op1, Flags);
1958 return TLO.CombineTo(Op, NewOp);
1971 SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
1977 return TLO.CombineTo(Op, NewOp);
1991 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2017 const APInt &DemandedElts,
2038 const APInt &UndefOp0,
2039 const APInt &UndefOp1) {
2042 "Vector binop only");
2047 UndefOp1.
getBitWidth() == NumElts &&
"Bad type for undef analysis");
2049 auto getUndefOrConstantElt = [&](
SDValue V,
unsigned Index,
2050 const APInt &UndefVals) {
2051 if (UndefVals[Index])
2054 if (
auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2059 if (isa<ConstantFPSDNode>(Elt) || Elt.
isUndef() || (
C && !
C->isOpaque()))
2067 for (
unsigned i = 0; i != NumElts; ++i) {
2076 if (C0 && C1 && C0.
getValueType() == EltVT && C1.getValueType() == EltVT)
2086 bool AssumeSingleUse)
const {
2088 APInt DemandedElts = OriginalDemandedElts;
2092 "Mask size mismatches value type element count!");
2098 KnownUndef.setAllBits();
2107 if (DemandedElts == 0) {
2108 KnownUndef.setAllBits();
2121 if (!DemandedElts[0]) {
2122 KnownUndef.setAllBits();
2125 KnownUndef.setHighBits(NumElts - 1);
2139 if (NumSrcElts == NumElts)
2141 KnownZero, TLO, Depth + 1);
2143 APInt SrcZero, SrcUndef;
2148 if ((NumElts % NumSrcElts) == 0) {
2149 unsigned Scale = NumElts / NumSrcElts;
2150 for (
unsigned i = 0; i != NumElts; ++i)
2151 if (DemandedElts[i])
2152 SrcDemandedElts.
setBit(i / Scale);
2164 for (
unsigned i = 0; i != NumElts; ++i)
2165 if (DemandedElts[i]) {
2166 unsigned Ofs = (i % Scale) * EltSizeInBits;
2167 SrcDemandedBits.
setBits(Ofs, Ofs + EltSizeInBits);
2177 for (
unsigned i = 0; i != NumSrcElts; ++i) {
2178 if (SrcDemandedElts[i]) {
2180 KnownZero.
setBits(i * Scale, (i + 1) * Scale);
2182 KnownUndef.setBits(i * Scale, (i + 1) * Scale);
2190 if ((NumSrcElts % NumElts) == 0) {
2191 unsigned Scale = NumSrcElts / NumElts;
2192 for (
unsigned i = 0; i != NumElts; ++i)
2193 if (DemandedElts[i])
2194 SrcDemandedElts.
setBits(i * Scale, (i + 1) * Scale);
2202 for (
unsigned i = 0; i != NumElts; ++i) {
2203 if (DemandedElts[i]) {
2207 KnownUndef.setBit(i);
2220 bool Updated =
false;
2221 for (
unsigned i = 0; i != NumElts; ++i) {
2222 if (!DemandedElts[i] && !Ops[i].
isUndef()) {
2224 KnownUndef.setBit(i);
2232 for (
unsigned i = 0; i != NumElts; ++i) {
2235 KnownUndef.setBit(i);
2247 for (
unsigned i = 0; i != NumSubVecs; ++i) {
2250 APInt SubUndef, SubZero;
2254 KnownUndef.
insertBits(SubUndef, i * NumSubElts);
2255 KnownZero.
insertBits(SubZero, i * NumSubElts);
2267 if (Idx.
ugt(NumElts - NumSubElts))
2271 APInt SubUndef, SubZero;
2275 APInt BaseElts = DemandedElts;
2279 if (!BaseElts && !Base.
isUndef())
2289 KnownUndef.insertBits(SubUndef, SubIdx);
2301 APInt SrcUndef, SrcZero;
2317 if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
2319 if (!DemandedElts[Idx])
2322 APInt DemandedVecElts(DemandedElts);
2325 KnownZero, TLO, Depth + 1))
2328 KnownUndef.clearBit(Idx);
2330 KnownUndef.setBit(Idx);
2338 APInt VecUndef, VecZero;
2352 APInt UnusedUndef, UnusedZero;
2354 UnusedZero, TLO, Depth + 1))
2358 APInt DemandedLHS(DemandedElts);
2359 APInt DemandedRHS(DemandedElts);
2360 APInt UndefLHS, ZeroLHS;
2361 APInt UndefRHS, ZeroRHS;
2363 ZeroLHS, TLO, Depth + 1))
2366 ZeroRHS, TLO, Depth + 1))
2369 KnownUndef = UndefLHS & UndefRHS;
2370 KnownZero = ZeroLHS & ZeroRHS;
2377 APInt DemandedLHS(NumElts, 0);
2378 APInt DemandedRHS(NumElts, 0);
2379 for (
unsigned i = 0; i != NumElts; ++i) {
2380 int M = ShuffleMask[i];
2381 if (M < 0 || !DemandedElts[i])
2383 assert(0 <= M && M < (
int)(2 * NumElts) &&
"Shuffle index out of range");
2384 if (M < (
int)NumElts)
2387 DemandedRHS.
setBit(M - NumElts);
2391 APInt UndefLHS, ZeroLHS;
2392 APInt UndefRHS, ZeroRHS;
2394 ZeroLHS, TLO, Depth + 1))
2397 ZeroRHS, TLO, Depth + 1))
2401 bool Updated =
false;
2402 bool IdentityLHS =
true, IdentityRHS =
true;
2404 for (
unsigned i = 0; i != NumElts; ++i) {
2405 int &M = NewMask[i];
2408 if (!DemandedElts[i] || (M < (
int)NumElts && UndefLHS[M]) ||
2409 (M >= (int)NumElts && UndefRHS[M - NumElts])) {
2413 IdentityLHS &= (M < 0) || (M == (
int)i);
2414 IdentityRHS &= (M < 0) || ((M - NumElts) == i);
2419 if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {
2424 return TLO.CombineTo(Op, LegalShuffle);
2428 for (
unsigned i = 0; i != NumElts; ++i) {
2429 int M = ShuffleMask[i];
2431 KnownUndef.setBit(i);
2432 }
else if (M < (
int)NumElts) {
2434 KnownUndef.setBit(i);
2438 if (UndefRHS[M - NumElts])
2439 KnownUndef.setBit(i);
2440 if (ZeroRHS[M - NumElts])
2449 APInt SrcUndef, SrcZero;
2470 KnownUndef.clearAllBits();
2486 APInt UndefRHS, ZeroRHS;
2488 ZeroRHS, TLO, Depth + 1))
2490 APInt UndefLHS, ZeroLHS;
2492 ZeroLHS, TLO, Depth + 1))
2495 KnownZero = ZeroLHS & ZeroRHS;
2504 APInt UndefRHS, ZeroRHS;
2506 ZeroRHS, TLO, Depth + 1))
2508 APInt UndefLHS, ZeroLHS;
2510 ZeroLHS, TLO, Depth + 1))
2513 KnownZero = ZeroLHS;
2514 KnownUndef = UndefLHS & UndefRHS;
2519 APInt SrcUndef, SrcZero;
2521 SrcZero, TLO, Depth + 1))
2524 KnownZero, TLO, Depth + 1))
2531 KnownZero |= SrcZero;
2532 KnownUndef &= SrcUndef;
2533 KnownUndef &= ~KnownZero;
2540 KnownZero, TLO, Depth + 1))
2546 return TLO.CombineTo(Op, TLO.DAG.getConstant(0,
SDLoc(Op), VT));
2547 KnownUndef.clearAllBits();
2553 KnownZero, TLO, Depth))
2559 TLO, Depth, AssumeSingleUse))
2565 assert((KnownUndef & KnownZero) == 0 &&
"Elements flagged as undef AND zero");
2579 const APInt &DemandedElts,
2581 unsigned Depth)
const {
2586 "Should use MaskedValueIsZero if you don't know whether Op" 2587 " is a target node!");
2594 unsigned Depth)
const {
2600 const APInt &DemandedElts,
2602 unsigned Depth)
const {
2603 assert(isa<FrameIndexSDNode>(Op) &&
"expected FrameIndex");
2616 unsigned Depth)
const {
2621 "Should use ComputeNumSignBits if you don't know whether Op" 2622 " is a target node!");
2633 "Should use SimplifyDemandedVectorElts if you don't know whether Op" 2634 " is a target node!");
2645 "Should use SimplifyDemandedBits if you don't know whether Op" 2646 " is a target node!");
2659 "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op" 2660 " is a target node!");
2688 unsigned Depth)
const {
2693 "Should use isKnownNeverNaN if you don't know whether Op" 2694 " is a target node!");
2706 if (
auto *CN = dyn_cast<ConstantSDNode>(N)) {
2707 CVal = CN->getAPIntValue();
2708 }
else if (
auto *BV = dyn_cast<BuildVectorSDNode>(N)) {
2709 auto *CN = BV->getConstantSplatNode();
2715 unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits();
2716 CVal = CN->getAPIntValue();
2718 CVal = CVal.
trunc(BVEltWidth);
2814 return DAG.
getSetCC(DL, VT, N0, Zero, Cond);
2825 if (YConst && YConst->isNullValue())
2831 return DAG.
getSetCC(DL, VT, NewAnd, Zero, Cond);
2847 SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
2849 const SDLoc &DL)
const {
2852 if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
2861 if (!(C01 = dyn_cast<ConstantSDNode>(N0->
getOperand(1))))
2889 auto checkConstants = [&I1, &I01]() ->
bool {
2894 if (checkConstants()) {
2901 if (!checkConstants())
2907 const unsigned KeptBits = I1.
logBase2();
2908 const unsigned KeptBitsMinusOne = I01.
logBase2();
2911 if (KeptBits != (KeptBitsMinusOne + 1))
2917 if (!DAG.getTargetLoweringInfo().shouldTransformSignedTruncationCheck(
2926 SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);
2929 SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);
2935 SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
2940 "Should be a comparison with 0.");
2942 "Valid only for [in]equality comparisons.");
2944 unsigned NewShiftOpcode;
2955 unsigned OldShiftOpcode = V.getOpcode();
2956 switch (OldShiftOpcode) {
2968 C = V.getOperand(0);
2973 Y = V.getOperand(1);
2977 return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
2978 X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
2995 EVT VT = X.getValueType();
3013 "Unexpected binop");
3045 return DAG.
getSetCC(DL, VT, X, YShl1, Cond);
3053 const SDLoc &dl)
const {
3067 return DAG.
getSetCC(dl, VT, N1, N0, SwappedCC);
3078 return DAG.
getSetCC(dl, VT, N1, N0, SwappedCC);
3080 if (
auto *N1C = dyn_cast<ConstantSDNode>(N1.
getNode())) {
3081 const APInt &C1 = N1C->getAPIntValue();
3141 return DAG.
getNode(LogicOpcode, dl, VT, LHS, RHS);
3154 MinBits = N0->getOperand(0).getValueSizeInBits();
3156 }
else if (N0->getOpcode() ==
ISD::AND) {
3158 if (
auto *
C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))
3159 if ((
C->getAPIntValue()+1).isPowerOf2()) {
3160 MinBits =
C->getAPIntValue().countTrailingOnes();
3165 MinBits = N0->getOperand(0).getValueSizeInBits();
3168 }
else if (
auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
3171 MinBits = LN0->getMemoryVT().getSizeInBits();
3175 MinBits = LN0->getMemoryVT().getSizeInBits();
3181 unsigned ReqdBits = Signed ?
3188 MinBits >= ReqdBits) {
3193 if (MinBits == 1 && C1 == 1)
3198 return DAG.
getSetCC(dl, VT, Trunc, C, Cond);
3225 cast<CondCodeSDNode>(TopSetCC.
getOperand(2))->
get(),
3239 N0.getOpcode() ==
ISD::AND && C1 == 0 &&
3240 N0.getNode()->hasOneUse() &&
3241 isa<LoadSDNode>(N0.getOperand(0)) &&
3242 N0.getOperand(0).getNode()->hasOneUse() &&
3243 isa<ConstantSDNode>(N0.getOperand(1))) {
3244 LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));
3246 unsigned bestWidth = 0, bestOffset = 0;
3248 unsigned origWidth = N0.getValueSizeInBits();
3249 unsigned maskWidth = origWidth;
3254 const APInt &
Mask = N0.getConstantOperandAPInt(1);
3255 for (
unsigned width = origWidth / 2; width>=8; width /= 2) {
3257 for (
unsigned offset=0; offset<origWidth/width; offset++) {
3260 bestOffset = (uint64_t)offset * (width/8);
3262 bestOffset = (origWidth/width - offset - 1) * (width/8);
3263 bestMask = Mask.
lshr(offset * (width/8) * 8);
3277 if (bestOffset != 0)
3300 C1.getBitWidth() - InSize))) {
3317 return DAG.
getConstant(C1.isNonNegative(), dl, VT);
3331 EVT newVT = N0.getOperand(0).getValueType();
3350 EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
3352 EVT ExtDstTy = N0.getValueType();
3357 if (C1.getMinSignedBits() > ExtSrcTyBits)
3361 EVT Op0Ty = N0.getOperand(0).getValueType();
3362 if (Op0Ty == ExtSrcTy) {
3372 return DAG.
getSetCC(dl, VT, ZextOp,
3378 }
else if ((N1C->isNullValue() || N1C->isOne()) &&
3383 bool TrueWhenTrue = (Cond ==
ISD::SETEQ) ^ (!N1C->isOne());
3387 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->
get();
3389 N0.getOperand(0).getValueType().isInteger());
3392 return DAG.
getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
3397 N0.getOperand(0).getOpcode() ==
ISD::XOR &&
3398 N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
3399 isa<ConstantSDNode>(N0.getOperand(1)) &&
3400 cast<ConstantSDNode>(N0.getOperand(1))->isOne()) {
3413 N0.getOperand(0).getOpcode() ==
ISD::XOR);
3416 N0.getOperand(0).getOperand(0),
3420 return DAG.
getSetCC(dl, VT, Val, N1,
3423 }
else if (N1C->isOne() &&
3441 cast<ConstantSDNode>(Op0.
getOperand(1))->isOne()) {
3468 if (N0.getOpcode() ==
ISD::UREM && N1C->isNullValue() &&
3473 return DAG.
getSetCC(dl, VT, N0.getOperand(0), N1, Cond);
3477 optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))
3484 const APInt &C1 = N1C->getAPIntValue();
3486 APInt MinVal, MaxVal;
3576 if (
SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
3577 VT, N0, N1, Cond, DCI, dl))
3605 if (
auto *N1C = dyn_cast<ConstantSDNode>(N1.
getNode())) {
3607 const APInt &C1 = N1C->getAPIntValue();
3615 if (
auto *AndRHS = dyn_cast<ConstantSDNode>(N0.
getOperand(1))) {
3619 unsigned ShCt = AndRHS->getAPIntValue().logBase2();
3620 if (AndRHS->getAPIntValue().isPowerOf2() &&
3626 }
else if (Cond ==
ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
3645 if (
auto *AndRHS = dyn_cast<ConstantSDNode>(N0.
getOperand(1))) {
3646 const APInt &AndRHSC = AndRHS->getAPIntValue();
3647 if ((-AndRHSC).isPowerOf2() && (AndRHSC & C1) == C1) {
3657 return DAG.
getSetCC(dl, VT, Shift, CmpRHS, Cond);
3687 return DAG.
getSetCC(dl, VT, Shift, CmpRHS, NewCond);
3693 if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {
3694 auto *CFP = cast<ConstantFPSDNode>(N1);
3695 assert(!CFP->getValueAPF().isNaN() &&
"Unexpected NaN value");
3702 return DAG.
getSetCC(dl, VT, N0, N0, Cond);
3720 if (CFP->getValueAPF().isInfinity()) {
3721 if (CFP->getValueAPF().isNegative()) {
3756 "Integer types should be handled by FoldSetCC");
3762 if (UOF ==
unsigned(EqTrue))
3767 if (NewCond != Cond &&
3770 return DAG.
getSetCC(dl, VT, N0, N1, NewCond);
3796 bool LegalRHSImm =
false;
3798 if (
auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
3799 if (
auto *LHSR = dyn_cast<ConstantSDNode>(N0.
getOperand(1))) {
3804 LHSR->getAPIntValue(),
3816 RHSC->getAPIntValue(),
3827 RHSC->getAPIntValue(),
3834 if (RHSC->getValueType(0).getSizeInBits() <= 64)
3843 if (
SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))
3849 if (
SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))
3852 if (
SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))
3865 if (
SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))
3868 if (
SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))
3881 N0 = DAG.
getNOT(dl, Temp, OpVT);
3890 Temp = DAG.
getNOT(dl, N0, OpVT);
3897 Temp = DAG.
getNOT(dl, N1, OpVT);
3904 Temp = DAG.
getNOT(dl, N0, OpVT);
3911 Temp = DAG.
getNOT(dl, N1, OpVT);
3920 N0 = DAG.
getNode(ExtendCode, dl, VT, N0);
3936 if (
auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {
3937 GA = GASD->getGlobal();
3938 Offset += GASD->getOffset();
3946 if (
auto *V = dyn_cast<ConstantSDNode>(N2)) {
3947 Offset += V->getSExtValue();
3951 if (
auto *V = dyn_cast<ConstantSDNode>(N1)) {
3952 Offset += V->getSExtValue();
3973 unsigned S = Constraint.
size();
3976 switch (Constraint[0]) {
4006 if (S > 1 && Constraint[0] ==
'{' && Constraint[S - 1] ==
'}') {
4007 if (S == 8 && Constraint.
substr(1, 6) ==
"memory")
4034 std::string &Constraint,
4035 std::vector<SDValue> &Ops,
4038 if (Constraint.length() > 1)
return;
4040 char ConstraintLetter = Constraint[0];
4041 switch (ConstraintLetter) {
4065 if ((GA = dyn_cast<GlobalAddressSDNode>(Op)) && ConstraintLetter !=
'n') {
4070 }
else if ((C = dyn_cast<ConstantSDNode>(Op)) &&
4071 ConstraintLetter !=
's') {
4084 }
else if ((BA = dyn_cast<BlockAddressSDNode>(Op)) &&
4085 ConstraintLetter !=
'n') {
4093 if ((C = dyn_cast<ConstantSDNode>(Op.
getOperand(0))))
4097 (C = dyn_cast<ConstantSDNode>(Op.
getOperand(1))))
4112 std::pair<unsigned, const TargetRegisterClass *>
4116 if (Constraint.
empty() || Constraint[0] !=
'{')
4117 return std::make_pair(0u, static_cast<TargetRegisterClass *>(
nullptr));
4118 assert(*(Constraint.
end() - 1) ==
'}' &&
"Not a brace enclosed constraint?");
4123 std::pair<unsigned, const TargetRegisterClass *> R =
4124 std::make_pair(0u, static_cast<const TargetRegisterClass *>(
nullptr));
4136 std::pair<unsigned, const TargetRegisterClass *> S =
4137 std::make_pair(*
I, RC);
4159 assert(!ConstraintCode.empty() &&
"No known constraint!");
4160 return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
4166 assert(!ConstraintCode.empty() &&
"No known constraint!");
4167 return atoi(ConstraintCode.c_str());
4182 unsigned maCount = 0;
4190 ConstraintOperands.emplace_back(std::move(CI));
4200 switch (OpInfo.
Type) {
4216 assert(ResNo == 0 &&
"Asm only has one result!");
4239 if (
StructType *STy = dyn_cast<StructType>(OpTy))
4240 if (STy->getNumElements() == 1)
4241 OpTy = STy->getElementType(0);
4259 }
else if (
PointerType *PT = dyn_cast<PointerType>(OpTy)) {
4269 if (!ConstraintOperands.empty()) {
4271 unsigned bestMAIndex = 0;
4272 int bestWeight = -1;
4278 for (maIndex = 0; maIndex < maCount; ++maIndex) {
4280 for (
unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4281 cIndex != eIndex; ++cIndex) {
4307 weightSum += weight;
4310 if (weightSum > bestWeight) {
4311 bestWeight = weightSum;
4312 bestMAIndex = maIndex;
4317 for (
unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4318 cIndex != eIndex; ++cIndex) {
4328 for (
unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4329 cIndex != eIndex; ++cIndex) {
4340 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
4343 std::pair<unsigned, const TargetRegisterClass *> InputRC =
4348 (MatchRC.second != InputRC.second)) {
4350 " with a matching output constraint of" 4351 " incompatible type!");
4357 return ConstraintOperands;
4385 rCodes = &info.
Codes;
4391 for (
unsigned i = 0,
e = rCodes->size(); i !=
e; ++i) {
4394 if (weight > BestWeight)
4395 BestWeight = weight;
4411 if (!CallOperandVal)
4414 switch (*constraint) {
4417 if (isa<ConstantInt>(CallOperandVal))
4421 if (isa<GlobalValue>(CallOperandVal))
4426 if (isa<ConstantFP>(CallOperandVal))
4473 assert(OpInfo.
Codes.size() > 1 &&
"Doesn't have multiple constraint options");
4474 unsigned BestIdx = 0;
4476 int BestGenerality = -1;
4479 for (
unsigned i = 0,
e = OpInfo.
Codes.size(); i !=
e; ++i) {
4490 "Unhandled multi-letter 'other' constraint");
4491 std::vector<SDValue> ResultOps;
4494 if (!ResultOps.empty()) {
4508 if (Generality > BestGenerality) {
4511 BestGenerality = Generality;
4524 assert(!OpInfo.
Codes.empty() &&
"Must have at least one constraint");
4527 if (OpInfo.
Codes.size() == 1) {
4540 if (isa<BasicBlock>(v) || isa<ConstantInt>(v) || isa<Function>(v)) {
4569 bool UseSRA =
false;
4573 if (
C->isNullValue())
4575 APInt Divisor =
C->getAPIntValue();
4583 APInt Factor = Divisor;
4584 while ((t = Divisor * Factor) != 1)
4601 Factor = Factors[0];
4633 bool IsAfterLegalization,
4654 if (
C->isNullValue())
4657 const APInt &Divisor =
C->getAPIntValue();
4659 int NumeratorFactor = 0;
4670 NumeratorFactor = 1;
4673 NumeratorFactor = -1;
4690 SDValue MagicFactor, Factor, Shift, ShiftMask;
4697 MagicFactor = MagicFactors[0];
4698 Factor = Factors[0];
4700 ShiftMask = ShiftMasks[0];
4729 SDValue SignShift = DAG.
getConstant(EltBits - 1, dl, ShVT);
4742 bool IsAfterLegalization,
4756 bool UseNPQ =
false;
4760 if (
C->isNullValue())
4764 APInt Divisor =
C->getAPIntValue();
4766 unsigned PreShift = 0, PostShift = 0;
4770 if (magics.
a != 0 && !Divisor[0]) {
4773 magics = Divisor.
lshr(PreShift).
magicu(PreShift);
4774 assert(magics.
a == 0 &&
"Should use cheap fixup now");
4782 "We shouldn't generate an undefined shift!");
4783 PostShift = magics.
s;
4786 PostShift = magics.
s - 1;
4808 SDValue PreShift, PostShift, MagicFactor, NPQFactor;
4815 PreShift = PreShifts[0];
4816 MagicFactor = MagicFactors[0];
4817 PostShift = PostShifts[0];
4839 Q = GetMULHU(Q, MagicFactor);
4852 NPQ = GetMULHU(NPQ, NPQFactor);
4867 return DAG.
getSelect(dl, VT, IsOne, N0, Q);
4881 if (SplatValue != Values.
end()) {
4884 return Value == *SplatValue ||
Predicate(Value);
4886 Replacement = *SplatValue;
4890 if (!AlternativeReplacement)
4893 Replacement = AlternativeReplacement;
4906 DAGCombinerInfo &DCI,
4907 const SDLoc &DL)
const {
4909 if (
SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
4912 DCI.AddToWorklist(
N);
4920 TargetLowering::prepareUREMEqFold(
EVT SETCCVT,
SDValue REMNode,
4922 DAGCombinerInfo &DCI,
const SDLoc &DL,
4930 "Only applicable for (in)equality comparisons.");
4948 bool HadOneDivisor =
false;
4949 bool AllDivisorsAreOnes =
true;
4950 bool HadEvenDivisor =
false;
4951 bool AllDivisorsArePowerOfTwo =
true;
4956 if (
C->isNullValue())
4959 const APInt &
D =
C->getAPIntValue();
4966 assert((!D.
isOneValue() || (K == 0)) &&
"For divisor '1' we won't rotate.");
4970 HadEvenDivisor |= (K != 0);
4973 AllDivisorsArePowerOfTwo &= D0.isOneValue();