57 if (
F.getFnAttribute(
"disable-tail-calls").getValueAsBool())
63 AttrBuilder CallerAttrs(
F.getContext(),
F.getAttributes().getRetAttrs());
64 for (
const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
65 Attribute::DereferenceableOrNull, Attribute::NoAlias,
66 Attribute::NonNull, Attribute::NoUndef})
73 if (CallerAttrs.
contains(Attribute::ZExt) ||
74 CallerAttrs.
contains(Attribute::SExt))
85 for (
unsigned I = 0,
E = ArgLocs.
size();
I !=
E; ++
I) {
101 Register ArgReg = cast<RegisterSDNode>(
Value->getOperand(1))->getReg();
102 if (
MRI.getLiveInPhysReg(ArgReg) != Reg)
112 IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);
113 IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);
114 IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);
115 IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);
116 IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);
117 IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);
118 IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated);
119 IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);
120 IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
121 IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
122 IsSwiftAsync = Call->paramHasAttr(ArgIdx, Attribute::SwiftAsync);
123 IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
124 Alignment = Call->getParamStackAlign(ArgIdx);
127 "multiple ABI attributes?");
143std::pair<SDValue, SDValue>
153 Args.reserve(Ops.
size());
156 for (
unsigned i = 0; i < Ops.
size(); ++i) {
159 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.
getContext());
162 Entry.IsZExt = !Entry.IsSExt;
166 Entry.IsSExt = Entry.IsZExt =
false;
168 Args.push_back(Entry);
171 if (LC == RTLIB::UNKNOWN_LIBCALL)
179 bool zeroExtend = !signExtend;
183 signExtend = zeroExtend =
false;
194 return LowerCallTo(CLI);
198 std::vector<EVT> &MemOps,
unsigned Limit,
const MemOp &Op,
unsigned DstAS,
200 if (Limit != ~
unsigned(0) && Op.isMemcpyWithFixedDstAlign() &&
201 Op.getSrcAlign() < Op.getDstAlign())
211 if (Op.isFixedDstAlign())
229 unsigned NumMemOps = 0;
233 while (VTSize >
Size) {
265 if (NumMemOps && Op.allowOverlap() && NewVTSize <
Size &&
267 VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() :
Align(1),
277 if (++NumMemOps > Limit)
280 MemOps.push_back(VT);
295 return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,
305 bool IsSignaling)
const {
311 &&
"Unsupported setcc type!");
314 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
315 bool ShouldInvertCC =
false;
319 LC1 = (VT ==
MVT::f32) ? RTLIB::OEQ_F32 :
321 (VT ==
MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
325 LC1 = (VT ==
MVT::f32) ? RTLIB::UNE_F32 :
327 (VT ==
MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
331 LC1 = (VT ==
MVT::f32) ? RTLIB::OGE_F32 :
333 (VT ==
MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
337 LC1 = (VT ==
MVT::f32) ? RTLIB::OLT_F32 :
339 (VT ==
MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
343 LC1 = (VT ==
MVT::f32) ? RTLIB::OLE_F32 :
345 (VT ==
MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
349 LC1 = (VT ==
MVT::f32) ? RTLIB::OGT_F32 :
351 (VT ==
MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
354 ShouldInvertCC =
true;
357 LC1 = (VT ==
MVT::f32) ? RTLIB::UO_F32 :
359 (VT ==
MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
363 ShouldInvertCC =
true;
366 LC1 = (VT ==
MVT::f32) ? RTLIB::UO_F32 :
368 (VT ==
MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
369 LC2 = (VT ==
MVT::f32) ? RTLIB::OEQ_F32 :
371 (VT ==
MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
375 ShouldInvertCC =
true;
378 LC1 = (VT ==
MVT::f32) ? RTLIB::OGE_F32 :
380 (VT ==
MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
383 LC1 = (VT ==
MVT::f32) ? RTLIB::OGT_F32 :
385 (VT ==
MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
388 LC1 = (VT ==
MVT::f32) ? RTLIB::OLE_F32 :
390 (VT ==
MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
393 LC1 = (VT ==
MVT::f32) ? RTLIB::OLT_F32 :
395 (VT ==
MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
403 SDValue Ops[2] = {NewLHS, NewRHS};
408 auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);
413 if (ShouldInvertCC) {
415 CCCode = getSetCCInverse(CCCode, RetVT);
418 if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
425 auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);
428 CCCode = getSetCCInverse(CCCode, RetVT);
429 NewLHS = DAG.
getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);
443 if (!isPositionIndependent())
457 unsigned JTEncoding = getJumpTableEncoding();
482 if (!
TM.shouldAssumeDSOLocal(*GV->
getParent(), GV))
486 if (isPositionIndependent())
502 const APInt &DemandedElts,
505 unsigned Opcode = Op.getOpcode();
508 if (targetShrinkDemandedConstant(Op,
DemandedBits, DemandedElts, TLO))
518 auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
519 if (!Op1C || Op1C->isOpaque())
523 const APInt &
C = Op1C->getAPIntValue();
528 EVT VT = Op.getValueType();
544 EVT VT = Op.getValueType();
557 assert(Op.getNumOperands() == 2 &&
558 "ShrinkDemandedOp only supports binary operators!");
559 assert(Op.getNode()->getNumValues() == 1 &&
560 "ShrinkDemandedOp only supports nodes with one result!");
562 EVT VT = Op.getValueType();
572 if (!Op.getNode()->hasOneUse())
585 Op.getOpcode(), dl, SmallVT,
588 assert(DemandedSize <= SmallVTBits &&
"Narrowed below demanded bits?");
603 bool Simplified = SimplifyDemandedBits(Op,
DemandedBits, Known, TLO);
612 const APInt &DemandedElts,
620 SimplifyDemandedBits(Op,
DemandedBits, DemandedElts, Known, TLO);
632 bool AssumeSingleUse)
const {
633 EVT VT = Op.getValueType();
649 EVT VT = Op.getValueType();
667 switch (Op.getOpcode()) {
673 EVT SrcVT = Src.getValueType();
674 EVT DstVT = Op.getValueType();
680 if (NumSrcEltBits == NumDstEltBits)
681 if (
SDValue V = SimplifyMultipleUseDemandedBits(
685 if (SrcVT.
isVector() && (NumDstEltBits % NumSrcEltBits) == 0) {
686 unsigned Scale = NumDstEltBits / NumSrcEltBits;
690 for (
unsigned i = 0; i != Scale; ++i) {
691 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
692 unsigned BitOffset = EltOffset * NumSrcEltBits;
695 DemandedSrcBits |= Sub;
696 for (
unsigned j = 0; j != NumElts; ++j)
698 DemandedSrcElts.
setBit((j * Scale) + i);
702 if (
SDValue V = SimplifyMultipleUseDemandedBits(
703 Src, DemandedSrcBits, DemandedSrcElts, DAG,
Depth + 1))
708 if (IsLE && (NumSrcEltBits % NumDstEltBits) == 0) {
709 unsigned Scale = NumSrcEltBits / NumDstEltBits;
713 for (
unsigned i = 0; i != NumElts; ++i)
714 if (DemandedElts[i]) {
715 unsigned Offset = (i % Scale) * NumDstEltBits;
717 DemandedSrcElts.
setBit(i / Scale);
720 if (
SDValue V = SimplifyMultipleUseDemandedBits(
721 Src, DemandedSrcBits, DemandedSrcElts, DAG,
Depth + 1))
735 return Op.getOperand(0);
737 return Op.getOperand(1);
748 return Op.getOperand(0);
750 return Op.getOperand(1);
760 return Op.getOperand(0);
762 return Op.getOperand(1);
768 if (
const APInt *MaxSA =
770 SDValue Op0 = Op.getOperand(0);
771 unsigned ShAmt = MaxSA->getZExtValue();
772 unsigned NumSignBits =
775 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
781 SDValue Op0 = Op.getOperand(0);
782 SDValue Op1 = Op.getOperand(1);
803 SDValue Op0 = Op.getOperand(0);
804 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
811 if (NumSignBits >= (
BitWidth - ExBits + 1))
823 SDValue Src = Op.getOperand(0);
824 EVT SrcVT = Src.getValueType();
825 EVT DstVT = Op.getValueType();
826 if (IsLE && DemandedElts == 1 &&
838 SDValue Vec = Op.getOperand(0);
839 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
842 !DemandedElts[CIdx->getZExtValue()])
850 SDValue Vec = Op.getOperand(0);
851 SDValue Sub = Op.getOperand(1);
856 if (DemandedSubElts == 0)
862 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
866 bool AllUndef =
true, IdentityLHS =
true, IdentityRHS =
true;
867 for (
unsigned i = 0; i != NumElts; ++i) {
868 int M = ShuffleMask[i];
869 if (M < 0 || !DemandedElts[i])
872 IdentityLHS &= (M == (int)i);
873 IdentityRHS &= ((M - NumElts) == i);
877 return DAG.
getUNDEF(Op.getValueType());
879 return Op.getOperand(0);
881 return Op.getOperand(1);
891 if (
SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
901 unsigned Depth)
const {
902 EVT VT = Op.getValueType();
909 return SimplifyMultipleUseDemandedBits(Op,
DemandedBits, DemandedElts, DAG,
915 unsigned Depth)
const {
917 return SimplifyMultipleUseDemandedBits(Op,
DemandedBits, DemandedElts, DAG,
926 const APInt &DemandedElts,
929 "SRL or SRA node is required here!");
932 if (!N1C || !N1C->
isOne())
978 unsigned ShiftOpc = Op.getOpcode();
979 bool IsSigned =
false;
983 unsigned NumSigned = std::min(NumSignedA, NumSignedB) - 1;
988 unsigned NumZero = std::min(NumZeroA, NumZeroB);
994 if (NumZero >= 2 && NumSigned < NumZero) {
999 if (NumSigned >= 1) {
1007 if (NumZero >= 1 && NumSigned < NumZero) {
1027 EVT VT = Op.getValueType();
1036 if (((!IsSigned &&
Add->getFlags().hasNoUnsignedWrap() &&
1038 (IsSigned &&
Add->getFlags().hasNoSignedWrap() &&
1064 unsigned Depth,
bool AssumeSingleUse)
const {
1067 "Mask size mismatches value type size!");
1072 EVT VT = Op.getValueType();
1074 unsigned NumElts = OriginalDemandedElts.
getBitWidth();
1076 "Unexpected vector size");
1079 APInt DemandedElts = OriginalDemandedElts;
1100 cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt());
1105 bool HasMultiUse =
false;
1106 if (!AssumeSingleUse && !Op.getNode()->hasOneUse()) {
1115 }
else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
1124 switch (Op.getOpcode()) {
1128 if (!DemandedElts[0])
1132 SDValue Src = Op.getOperand(0);
1133 unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
1135 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO,
Depth + 1))
1140 if (DemandedElts == 1)
1150 auto *LD = cast<LoadSDNode>(Op);
1151 if (getTargetConstantFromLoad(LD)) {
1157 EVT MemVT = LD->getMemoryVT();
1167 SDValue Vec = Op.getOperand(0);
1168 SDValue Scl = Op.getOperand(1);
1169 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
1174 APInt DemandedVecElts(DemandedElts);
1176 unsigned Idx = CIdx->getZExtValue();
1180 if (!DemandedElts[
Idx])
1187 if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO,
Depth + 1))
1193 if (SimplifyDemandedBits(Vec,
DemandedBits, DemandedVecElts, KnownVec, TLO,
1197 if (!!DemandedVecElts)
1207 SDValue Src = Op.getOperand(0);
1208 SDValue Sub = Op.getOperand(1);
1212 APInt DemandedSrcElts = DemandedElts;
1216 if (SimplifyDemandedBits(Sub,
DemandedBits, DemandedSubElts, KnownSub, TLO,
1219 if (SimplifyDemandedBits(Src,
DemandedBits, DemandedSrcElts, KnownSrc, TLO,
1225 if (!!DemandedSubElts)
1227 if (!!DemandedSrcElts)
1233 SDValue NewSub = SimplifyMultipleUseDemandedBits(
1235 SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1237 if (NewSub || NewSrc) {
1238 NewSub = NewSub ? NewSub : Sub;
1239 NewSrc = NewSrc ? NewSrc : Src;
1251 SDValue Src = Op.getOperand(0);
1252 if (Src.getValueType().isScalableVector())
1255 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1258 if (SimplifyDemandedBits(Src,
DemandedBits, DemandedSrcElts, Known, TLO,
1264 SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1279 EVT SubVT = Op.getOperand(0).getValueType();
1280 unsigned NumSubVecs = Op.getNumOperands();
1282 for (
unsigned i = 0; i != NumSubVecs; ++i) {
1283 APInt DemandedSubElts =
1284 DemandedElts.
extractBits(NumSubElts, i * NumSubElts);
1285 if (SimplifyDemandedBits(Op.getOperand(i),
DemandedBits, DemandedSubElts,
1286 Known2, TLO,
Depth + 1))
1289 if (!!DemandedSubElts)
1296 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
1299 APInt DemandedLHS, DemandedRHS;
1304 if (!!DemandedLHS || !!DemandedRHS) {
1305 SDValue Op0 = Op.getOperand(0);
1306 SDValue Op1 = Op.getOperand(1);
1310 if (!!DemandedLHS) {
1311 if (SimplifyDemandedBits(Op0,
DemandedBits, DemandedLHS, Known2, TLO,
1316 if (!!DemandedRHS) {
1317 if (SimplifyDemandedBits(Op1,
DemandedBits, DemandedRHS, Known2, TLO,
1324 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1326 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1328 if (DemandedOp0 || DemandedOp1) {
1329 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1330 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1338 SDValue Op0 = Op.getOperand(0);
1339 SDValue Op1 = Op.getOperand(1);
1365 LHSKnown.
One == ~RHSC->getAPIntValue()) {
1377 unsigned NumSubElts =
1394 if (SimplifyDemandedBits(Op1,
DemandedBits, DemandedElts, Known, TLO,
1399 Known2, TLO,
Depth + 1))
1422 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1424 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1426 if (DemandedOp0 || DemandedOp1) {
1427 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1428 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1438 SDValue Op0 = Op.getOperand(0);
1439 SDValue Op1 = Op.getOperand(1);
1441 if (SimplifyDemandedBits(Op1,
DemandedBits, DemandedElts, Known, TLO,
1445 if (SimplifyDemandedBits(Op0, ~Known.
One &
DemandedBits, DemandedElts,
1446 Known2, TLO,
Depth + 1))
1465 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1467 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1469 if (DemandedOp0 || DemandedOp1) {
1470 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1471 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1482 for (
int I = 0;
I != 2; ++
I) {
1483 SDValue X = Op.getOperand(
I).getOperand(0);
1484 SDValue C1 = Op.getOperand(
I).getOperand(1);
1485 SDValue Alt = Op.getOperand(1 -
I).getOperand(0);
1486 SDValue C2 = Op.getOperand(1 -
I).getOperand(1);
1488 for (
int J = 0; J != 2; ++J) {
1508 SDValue Op0 = Op.getOperand(0);
1509 SDValue Op1 = Op.getOperand(1);
1511 if (SimplifyDemandedBits(Op1,
DemandedBits, DemandedElts, Known, TLO,
1515 if (SimplifyDemandedBits(Op0,
DemandedBits, DemandedElts, Known2, TLO,
1543 if (
C->getAPIntValue() == Known2.
One) {
1552 if (!
C->isAllOnes() &&
DemandedBits.isSubsetOf(
C->getAPIntValue())) {
1564 if (ShiftC->getAPIntValue().ult(
BitWidth)) {
1565 uint64_t ShiftAmt = ShiftC->getZExtValue();
1568 : Ones.
lshr(ShiftAmt);
1586 if (!
C || !
C->isAllOnes())
1592 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1594 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1596 if (DemandedOp0 || DemandedOp1) {
1597 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1598 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1608 if (SimplifyDemandedBits(Op.getOperand(2),
DemandedBits, Known, TLO,
1611 if (SimplifyDemandedBits(Op.getOperand(1),
DemandedBits, Known2, TLO,
1625 if (SimplifyDemandedBits(Op.getOperand(2),
DemandedBits, DemandedElts,
1626 Known, TLO,
Depth + 1))
1628 if (SimplifyDemandedBits(Op.getOperand(1),
DemandedBits, DemandedElts,
1629 Known2, TLO,
Depth + 1))
1638 if (SimplifyDemandedBits(Op.getOperand(3),
DemandedBits, Known, TLO,
1641 if (SimplifyDemandedBits(Op.getOperand(2),
DemandedBits, Known2, TLO,
1655 SDValue Op0 = Op.getOperand(0);
1656 SDValue Op1 = Op.getOperand(1);
1683 SDValue Op0 = Op.getOperand(0);
1684 SDValue Op1 = Op.getOperand(1);
1687 if (
const APInt *SA =
1689 unsigned ShAmt = SA->getZExtValue();
1699 if (
const APInt *SA2 =
1701 unsigned C1 = SA2->getZExtValue();
1703 int Diff = ShAmt - C1;
1722 if (ShAmt < InnerBits &&
DemandedBits.getActiveBits() <= InnerBits &&
1723 isTypeDesirableForOp(
ISD::SHL, InnerVT)) {
1739 if (
const APInt *SA2 =
1741 unsigned InnerShAmt = SA2->getZExtValue();
1742 if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
1744 (InnerBits - InnerShAmt + ShAmt) &&
1758 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1762 Known.
Zero <<= ShAmt;
1763 Known.
One <<= ShAmt;
1769 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1770 Op0, InDemandedMask, DemandedElts, TLO.
DAG,
Depth + 1);
1788 if (SimplifyDemandedBits(Op0, DemandedFromOp, DemandedElts, Known, TLO,
1791 if (
Flags.hasNoSignedWrap() ||
Flags.hasNoUnsignedWrap()) {
1794 Flags.setNoSignedWrap(
false);
1795 Flags.setNoUnsignedWrap(
false);
1796 Op->setFlags(
Flags);
1806 if (
const APInt *MaxSA =
1808 unsigned ShAmt = MaxSA->getZExtValue();
1809 unsigned NumSignBits =
1812 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
1818 SDValue Op0 = Op.getOperand(0);
1819 SDValue Op1 = Op.getOperand(1);
1824 DemandedElts,
Depth + 1))
1827 if (
const APInt *SA =
1829 unsigned ShAmt = SA->getZExtValue();
1839 if (
const APInt *SA2 =
1841 unsigned C1 = SA2->getZExtValue();
1843 int Diff = ShAmt - C1;
1859 if (Op->getFlags().hasExact())
1863 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1874 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1875 Op0, InDemandedMask, DemandedElts, TLO.
DAG,
Depth + 1);
1885 SDValue Op0 = Op.getOperand(0);
1886 SDValue Op1 = Op.getOperand(1);
1905 DemandedElts,
Depth + 1))
1908 if (
const APInt *SA =
1910 unsigned ShAmt = SA->getZExtValue();
1918 if (Op->getFlags().hasExact())
1926 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1938 Flags.setExact(Op->getFlags().hasExact());
1956 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1957 Op0, InDemandedMask, DemandedElts, TLO.
DAG,
Depth + 1);
1968 SDValue Op0 = Op.getOperand(0);
1969 SDValue Op1 = Op.getOperand(1);
1970 SDValue Op2 = Op.getOperand(2);
1971 bool IsFSHL = (Op.getOpcode() ==
ISD::FSHL);
1974 unsigned Amt = SA->getAPIntValue().urem(
BitWidth);
1979 if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1,
DemandedBits, DemandedElts,
1980 Known, TLO,
Depth + 1))
1989 if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
1992 if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,
2006 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2007 Op0, Demanded0, DemandedElts, TLO.
DAG,
Depth + 1);
2008 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2009 Op1, Demanded1, DemandedElts, TLO.
DAG,
Depth + 1);
2010 if (DemandedOp0 || DemandedOp1) {
2011 DemandedOp0 = DemandedOp0 ? DemandedOp0 : Op0;
2012 DemandedOp1 = DemandedOp1 ? DemandedOp1 : Op1;
2023 if (SimplifyDemandedBits(Op2, DemandedAmtBits, DemandedElts,
2024 Known2, TLO,
Depth + 1))
2031 SDValue Op0 = Op.getOperand(0);
2032 SDValue Op1 = Op.getOperand(1);
2033 bool IsROTL = (Op.getOpcode() ==
ISD::ROTL);
2040 unsigned Amt = SA->getAPIntValue().urem(
BitWidth);
2046 if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
2056 DemandedBits.countr_zero() >= (IsROTL ? Amt : RevAmt)) {
2061 DemandedBits.countl_zero() >= (IsROTL ? RevAmt : Amt)) {
2070 if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,
2078 SDValue Op0 = Op.getOperand(0);
2079 SDValue Op1 = Op.getOperand(1);
2084 return TLO.
CombineTo(Op, *IsULE ? Op0 : Op1);
2086 return TLO.
CombineTo(Op, *IsULT ? Op0 : Op1);
2091 SDValue Op0 = Op.getOperand(0);
2092 SDValue Op1 = Op.getOperand(1);
2097 return TLO.
CombineTo(Op, *IsUGE ? Op0 : Op1);
2099 return TLO.
CombineTo(Op, *IsUGT ? Op0 : Op1);
2103 SDValue Src = Op.getOperand(0);
2105 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
2113 SDValue Src = Op.getOperand(0);
2132 unsigned ShiftAmount = NLZ > NTZ ? NLZ - NTZ : NTZ - NLZ;
2140 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
2159 SDValue Op0 = Op.getOperand(0);
2160 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2165 unsigned MinSignedBits =
2167 bool AlreadySignExtended = ExVTBits >= MinSignedBits;
2170 if (!AlreadySignExtended) {
2188 InputDemandedBits.
setBit(ExVTBits - 1);
2190 if (SimplifyDemandedBits(Op0, InputDemandedBits, DemandedElts, Known, TLO,
2199 if (Known.
Zero[ExVTBits - 1])
2203 if (Known.
One[ExVTBits - 1]) {
2213 EVT HalfVT = Op.getOperand(0).getValueType();
2221 if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO,
Depth + 1))
2224 if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO,
Depth + 1))
2227 Known = KnownHi.
concat(KnownLo);
2235 SDValue Src = Op.getOperand(0);
2236 EVT SrcVT = Src.getValueType();
2245 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2256 APInt InDemandedElts = DemandedElts.
zext(InElts);
2257 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
2265 if (
SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2266 Src, InDemandedBits, InDemandedElts, TLO.
DAG,
Depth + 1))
2275 SDValue Src = Op.getOperand(0);
2276 EVT SrcVT = Src.getValueType();
2285 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2296 APInt InDemandedElts = DemandedElts.
zext(InElts);
2300 InDemandedBits.
setBit(InBits - 1);
2302 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
2320 if (
SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2321 Src, InDemandedBits, InDemandedElts, TLO.
DAG,
Depth + 1))
2330 SDValue Src = Op.getOperand(0);
2331 EVT SrcVT = Src.getValueType();
2338 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2343 APInt InDemandedElts = DemandedElts.
zext(InElts);
2344 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
2352 if (
SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2353 Src, InDemandedBits, InDemandedElts, TLO.
DAG,
Depth + 1))
2358 SDValue Src = Op.getOperand(0);
2362 unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
2364 if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, Known, TLO,
2370 if (
SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2371 Src, TruncMask, DemandedElts, TLO.
DAG,
Depth + 1))
2376 switch (Src.getOpcode()) {
2387 if (Src.getNode()->hasOneUse()) {
2388 const APInt *ShAmtC =
2419 EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2421 if (SimplifyDemandedBits(Op.getOperand(0), ~InMask |
DemandedBits, Known,
2426 Known.
Zero |= ~InMask;
2427 Known.
One &= (~Known.Zero);
2431 SDValue Src = Op.getOperand(0);
2433 ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
2434 unsigned EltBitWidth = Src.getScalarValueSizeInBits();
2442 if (
auto *CIdx = dyn_cast<ConstantSDNode>(
Idx))
2443 if (CIdx->getAPIntValue().ult(NumSrcElts))
2450 DemandedSrcBits = DemandedSrcBits.
trunc(EltBitWidth);
2452 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,
2458 if (
SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2459 Src, DemandedSrcBits, DemandedSrcElts, TLO.
DAG,
Depth + 1)) {
2474 SDValue Src = Op.getOperand(0);
2475 EVT SrcVT = Src.getValueType();
2492 unsigned OpVTSizeInBits = Op.getValueSizeInBits();
2493 if (!OpVTLegal && OpVTSizeInBits > 32)
2495 unsigned ShVal = Op.getValueSizeInBits() - 1;
2505 unsigned Scale =
BitWidth / NumSrcEltBits;
2509 for (
unsigned i = 0; i != Scale; ++i) {
2510 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
2511 unsigned BitOffset = EltOffset * NumSrcEltBits;
2514 DemandedSrcBits |= Sub;
2515 for (
unsigned j = 0; j != NumElts; ++j)
2516 if (DemandedElts[j])
2517 DemandedSrcElts.
setBit((j * Scale) + i);
2521 APInt KnownSrcUndef, KnownSrcZero;
2522 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2523 KnownSrcZero, TLO,
Depth + 1))
2527 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2528 KnownSrcBits, TLO,
Depth + 1))
2530 }
else if (IsLE && (NumSrcEltBits %
BitWidth) == 0) {
2532 unsigned Scale = NumSrcEltBits /
BitWidth;
2536 for (
unsigned i = 0; i != NumElts; ++i)
2537 if (DemandedElts[i]) {
2540 DemandedSrcElts.
setBit(i / Scale);
2544 APInt KnownSrcUndef, KnownSrcZero;
2545 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2546 KnownSrcZero, TLO,
Depth + 1))
2551 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2552 KnownSrcBits, TLO,
Depth + 1))
2571 if (
C &&
C->getAPIntValue().countr_zero() == CTZ) {
2581 if (Op.getOperand(0) == Op.getOperand(1) &&
DemandedBits.ult(4)) {
2591 SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
2595 if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO,
2597 SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO,
2601 if (
Flags.hasNoSignedWrap() ||
Flags.hasNoUnsignedWrap()) {
2604 Flags.setNoSignedWrap(
false);
2605 Flags.setNoUnsignedWrap(
false);
2606 Op->setFlags(
Flags);
2613 isa<ConstantSDNode>(Op0) && cast<ConstantSDNode>(Op0)->isZero())
2618 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2619 Op0, LoMask, DemandedElts, TLO.
DAG,
Depth + 1);
2620 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2621 Op1, LoMask, DemandedElts, TLO.
DAG,
Depth + 1);
2622 if (DemandedOp0 || DemandedOp1) {
2623 Flags.setNoSignedWrap(
false);
2624 Flags.setNoUnsignedWrap(
false);
2625 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
2626 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
2640 if (
C && !
C->isAllOnes() && !
C->isOne() &&
2641 (
C->getAPIntValue() | HighMask).isAllOnes()) {
2645 Flags.setNoSignedWrap(
false);
2646 Flags.setNoUnsignedWrap(
false);
2654 auto getShiftLeftAmt = [&HighMask](
SDValue Mul) ->
unsigned {
2681 if (
unsigned ShAmt = getShiftLeftAmt(Op0))
2684 if (
unsigned ShAmt = getShiftLeftAmt(Op1))
2685 return foldMul(
ISD::SUB, Op1.getOperand(0), Op0, ShAmt);
2689 if (
unsigned ShAmt = getShiftLeftAmt(Op1))
2690 return foldMul(
ISD::ADD, Op1.getOperand(0), Op0, ShAmt);
2702 if (Op.getValueType().isScalableVector())
2704 if (SimplifyDemandedBitsForTargetNode(Op,
DemandedBits, DemandedElts,
2717 if (!isTargetCanonicalConstantNode(Op) &&
2720 const SDNode *
N = Op.getNode();
2746 const APInt &DemandedElts,
2752 APInt KnownUndef, KnownZero;
2754 SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
2766 const APInt &UndefOp0,
2767 const APInt &UndefOp1) {
2770 "Vector binop only");
2775 UndefOp1.
getBitWidth() == NumElts &&
"Bad type for undef analysis");
2777 auto getUndefOrConstantElt = [&](
SDValue V,
unsigned Index,
2778 const APInt &UndefVals) {
2779 if (UndefVals[
Index])
2782 if (
auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2786 auto *
C = dyn_cast<ConstantSDNode>(Elt);
2787 if (isa<ConstantFPSDNode>(Elt) || Elt.
isUndef() || (
C && !
C->isOpaque()))
2795 for (
unsigned i = 0; i != NumElts; ++i) {
2814 bool AssumeSingleUse)
const {
2815 EVT VT = Op.getValueType();
2816 unsigned Opcode = Op.getOpcode();
2817 APInt DemandedElts = OriginalDemandedElts;
2832 "Mask size mismatches value type element count!");
2841 if (!AssumeSingleUse && !Op.getNode()->hasOneUse())
2845 if (DemandedElts == 0) {
2860 auto SimplifyDemandedVectorEltsBinOp = [&](
SDValue Op0,
SDValue Op1) {
2861 SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,
2863 SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,
2865 if (NewOp0 || NewOp1) {
2867 Opcode,
SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, NewOp1 ? NewOp1 : Op1);
2875 if (!DemandedElts[0]) {
2879 SDValue ScalarSrc = Op.getOperand(0);
2883 EVT SrcVT = Src.getValueType();
2895 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2904 SDValue Src = Op.getOperand(0);
2905 EVT SrcVT = Src.getValueType();
2914 if (NumSrcElts == NumElts)
2915 return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
2916 KnownZero, TLO,
Depth + 1);
2918 APInt SrcDemandedElts, SrcZero, SrcUndef;
2922 if ((NumElts % NumSrcElts) == 0) {
2923 unsigned Scale = NumElts / NumSrcElts;
2925 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2935 for (
unsigned i = 0; i != NumElts; ++i)
2936 if (DemandedElts[i]) {
2937 unsigned Ofs = (i % Scale) * EltSizeInBits;
2938 SrcDemandedBits.
setBits(Ofs, Ofs + EltSizeInBits);
2942 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcDemandedElts, Known,
2950 for (
unsigned SubElt = 0; SubElt != Scale; ++SubElt) {
2954 for (
unsigned SrcElt = 0; SrcElt != NumSrcElts; ++SrcElt) {
2955 unsigned Elt = Scale * SrcElt + SubElt;
2956 if (DemandedElts[Elt])
2964 for (
unsigned i = 0; i != NumSrcElts; ++i) {
2965 if (SrcDemandedElts[i]) {
2967 KnownZero.
setBits(i * Scale, (i + 1) * Scale);
2969 KnownUndef.
setBits(i * Scale, (i + 1) * Scale);
2977 if ((NumSrcElts % NumElts) == 0) {
2978 unsigned Scale = NumSrcElts / NumElts;
2980 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2986 for (
unsigned i = 0; i != NumElts; ++i) {
2987 if (DemandedElts[i]) {
3002 [&](
SDValue Elt) { return Op.getOperand(0) != Elt; })) {
3004 bool Updated =
false;
3005 for (
unsigned i = 0; i != NumElts; ++i) {
3006 if (!DemandedElts[i] && !Ops[i].
isUndef()) {
3016 for (
unsigned i = 0; i != NumElts; ++i) {
3018 if (
SrcOp.isUndef()) {
3020 }
else if (EltSizeInBits ==
SrcOp.getScalarValueSizeInBits() &&
3028 EVT SubVT = Op.getOperand(0).getValueType();
3029 unsigned NumSubVecs = Op.getNumOperands();
3031 for (
unsigned i = 0; i != NumSubVecs; ++i) {
3032 SDValue SubOp = Op.getOperand(i);
3034 APInt SubUndef, SubZero;
3035 if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
3038 KnownUndef.
insertBits(SubUndef, i * NumSubElts);
3039 KnownZero.
insertBits(SubZero, i * NumSubElts);
3044 bool FoundNewSub =
false;
3046 for (
unsigned i = 0; i != NumSubVecs; ++i) {
3047 SDValue SubOp = Op.getOperand(i);
3049 SDValue NewSubOp = SimplifyMultipleUseDemandedVectorElts(
3050 SubOp, SubElts, TLO.
DAG,
Depth + 1);
3051 DemandedSubOps.
push_back(NewSubOp ? NewSubOp : SubOp);
3052 FoundNewSub = NewSubOp ?
true : FoundNewSub;
3065 SDValue Src = Op.getOperand(0);
3066 SDValue Sub = Op.getOperand(1);
3070 APInt DemandedSrcElts = DemandedElts;
3073 APInt SubUndef, SubZero;
3074 if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO,
3079 if (!DemandedSrcElts && !Src.isUndef())
3084 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero,
3092 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
3093 Src, DemandedSrcElts, TLO.
DAG,
Depth + 1);
3094 SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
3095 Sub, DemandedSubElts, TLO.
DAG,
Depth + 1);
3096 if (NewSrc || NewSub) {
3097 NewSrc = NewSrc ? NewSrc : Src;
3098 NewSub = NewSub ? NewSub : Sub;
3100 NewSub, Op.getOperand(2));
3108 SDValue Src = Op.getOperand(0);
3109 if (Src.getValueType().isScalableVector())
3112 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3115 APInt SrcUndef, SrcZero;
3116 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
3124 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
3125 Src, DemandedSrcElts, TLO.
DAG,
Depth + 1);
3135 SDValue Vec = Op.getOperand(0);
3136 SDValue Scl = Op.getOperand(1);
3137 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
3141 if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
3142 unsigned Idx = CIdx->getZExtValue();
3143 if (!DemandedElts[
Idx])
3146 APInt DemandedVecElts(DemandedElts);
3148 if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
3149 KnownZero, TLO,
Depth + 1))
3158 APInt VecUndef, VecZero;
3159 if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
3166 SDValue Sel = Op.getOperand(0);
3172 APInt UndefSel, UndefZero;
3173 if (SimplifyDemandedVectorElts(Sel, DemandedElts, UndefSel, UndefZero, TLO,
3178 APInt DemandedLHS(DemandedElts);
3179 APInt DemandedRHS(DemandedElts);
3180 APInt UndefLHS, ZeroLHS;
3181 APInt UndefRHS, ZeroRHS;
3182 if (SimplifyDemandedVectorElts(
LHS, DemandedLHS, UndefLHS, ZeroLHS, TLO,
3185 if (SimplifyDemandedVectorElts(
RHS, DemandedRHS, UndefRHS, ZeroRHS, TLO,
3189 KnownUndef = UndefLHS & UndefRHS;
3190 KnownZero = ZeroLHS & ZeroRHS;
3194 APInt DemandedSel = DemandedElts & ~KnownZero;
3195 if (DemandedSel != DemandedElts)
3196 if (SimplifyDemandedVectorElts(Sel, DemandedSel, UndefSel, UndefZero, TLO,
3205 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
3208 APInt DemandedLHS(NumElts, 0);
3209 APInt DemandedRHS(NumElts, 0);
3210 for (
unsigned i = 0; i != NumElts; ++i) {
3211 int M = ShuffleMask[i];
3212 if (M < 0 || !DemandedElts[i])
3214 assert(0 <= M && M < (
int)(2 * NumElts) &&
"Shuffle index out of range");
3215 if (M < (
int)NumElts)
3218 DemandedRHS.
setBit(M - NumElts);
3222 APInt UndefLHS, ZeroLHS;
3223 APInt UndefRHS, ZeroRHS;
3224 if (SimplifyDemandedVectorElts(
LHS, DemandedLHS, UndefLHS, ZeroLHS, TLO,
3227 if (SimplifyDemandedVectorElts(
RHS, DemandedRHS, UndefRHS, ZeroRHS, TLO,
3232 bool Updated =
false;
3233 bool IdentityLHS =
true, IdentityRHS =
true;
3235 for (
unsigned i = 0; i != NumElts; ++i) {
3236 int &M = NewMask[i];
3239 if (!DemandedElts[i] || (M < (
int)NumElts && UndefLHS[M]) ||
3240 (M >= (
int)NumElts && UndefRHS[M - NumElts])) {
3244 IdentityLHS &= (M < 0) || (M == (
int)i);
3245 IdentityRHS &= (M < 0) || ((M - NumElts) == i);
3250 if (Updated && !IdentityLHS && !IdentityRHS && !TLO.
LegalOps) {
3252 buildLegalVectorShuffle(VT,
DL,
LHS,
RHS, NewMask, TLO.
DAG);
3258 for (
unsigned i = 0; i != NumElts; ++i) {
3259 int M = ShuffleMask[i];
3262 }
else if (M < (
int)NumElts) {
3268 if (UndefRHS[M - NumElts])
3270 if (ZeroRHS[M - NumElts])
3279 APInt SrcUndef, SrcZero;
3280 SDValue Src = Op.getOperand(0);
3281 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3282 APInt DemandedSrcElts = DemandedElts.
zext(NumSrcElts);
3283 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
3290 Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
3291 DemandedSrcElts == 1) {
3304 if (IsLE && DemandedSrcElts == 1 && Src.getOpcode() ==
ISD::AND &&
3305 Op->isOnlyUserOf(Src.getNode()) &&
3306 Op.getValueSizeInBits() == Src.getValueSizeInBits()) {
3308 EVT SrcVT = Src.getValueType();
3315 ISD::AND,
DL, SrcVT, {Src.getOperand(1), Mask})) {
3327 SDValue Op0 = Op.getOperand(0);
3328 SDValue Op1 = Op.getOperand(1);
3329 if (Op0 == Op1 && Op->isOnlyUserOf(Op0.
getNode())) {
3330 APInt UndefLHS, ZeroLHS;
3331 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
3345 SDValue Op0 = Op.getOperand(0);
3346 SDValue Op1 = Op.getOperand(1);
3348 APInt UndefRHS, ZeroRHS;
3349 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
3352 APInt UndefLHS, ZeroLHS;
3353 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
3357 KnownZero = ZeroLHS & ZeroRHS;
3363 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
3372 SDValue Op0 = Op.getOperand(0);
3373 SDValue Op1 = Op.getOperand(1);
3375 APInt UndefRHS, ZeroRHS;
3376 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
3379 APInt UndefLHS, ZeroLHS;
3380 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
3384 KnownZero = ZeroLHS;
3385 KnownUndef = UndefLHS & UndefRHS;
3390 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
3398 SDValue Op0 = Op.getOperand(0);
3399 SDValue Op1 = Op.getOperand(1);
3401 APInt SrcUndef, SrcZero;
3402 if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO,
3407 APInt DemandedElts0 = DemandedElts & ~SrcZero;
3408 if (SimplifyDemandedVectorElts(Op0, DemandedElts0, KnownUndef, KnownZero,
3412 KnownUndef &= DemandedElts0;
3413 KnownZero &= DemandedElts0;
3418 if (DemandedElts.
isSubsetOf(SrcZero | KnownZero | SrcUndef | KnownUndef))
3425 KnownZero |= SrcZero;
3426 KnownUndef &= SrcUndef;
3427 KnownUndef &= ~KnownZero;
3431 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
3438 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
3439 KnownZero, TLO,
Depth + 1))
3451 if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
3452 KnownZero, TLO,
Depth))
3457 if (SimplifyDemandedBits(Op,
DemandedBits, OriginalDemandedElts, Known,
3458 TLO,
Depth, AssumeSingleUse))
3464 assert((KnownUndef & KnownZero) == 0 &&
"Elements flagged as undef AND zero");
3478 const APInt &DemandedElts,
3480 unsigned Depth)
const {
3485 "Should use MaskedValueIsZero if you don't know whether Op"
3486 " is a target node!");
3493 unsigned Depth)
const {
3505 unsigned Depth)
const {
3514 unsigned Depth)
const {
3519 "Should use ComputeNumSignBits if you don't know whether Op"
3520 " is a target node!");
3537 "Should use SimplifyDemandedVectorElts if you don't know whether Op"
3538 " is a target node!");
3549 "Should use SimplifyDemandedBits if you don't know whether Op"
3550 " is a target node!");
3551 computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.
DAG,
Depth);
3563 "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
3564 " is a target node!");
3591 bool PoisonOnly,
unsigned Depth)
const {
3597 "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
3598 " is a target node!");
3604 bool PoisonOnly,
bool ConsiderFlags,
unsigned Depth)
const {
3609 "Should use canCreateUndefOrPoison if you don't know whether Op"
3610 " is a target node!");
3618 unsigned Depth)
const {
3623 "Should use isKnownNeverNaN if you don't know whether Op"
3624 " is a target node!");
3629 const APInt &DemandedElts,
3632 unsigned Depth)
const {
3637 "Should use isSplatValue if you don't know whether Op"
3638 " is a target node!");
3653 CVal = CN->getAPIntValue();
3654 EltWidth =
N.getValueType().getScalarSizeInBits();
3661 CVal = CVal.
trunc(EltWidth);
3667 return CVal.
isOne();
3709 return (
N->isOne() && !SExt) || (SExt && (
N->getValueType(0) !=
MVT::i1));
3712 return N->isAllOnes() && SExt;
3721 DAGCombinerInfo &DCI)
const {
3749 auto *AndC = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
3750 if (AndC &&
isNullConstant(N1) && AndC->getAPIntValue().isPowerOf2() &&
3753 AndC->getAPIntValue().getActiveBits());
3784 if (DCI.isBeforeLegalizeOps() ||
3796 auto *YConst = dyn_cast<ConstantSDNode>(
Y);
3797 if (YConst && YConst->isZero())
3819SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
3824 if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
3833 if (!(C01 = dyn_cast<ConstantSDNode>(N0->
getOperand(1))))
3837 EVT XVT =
X.getValueType();
3861 auto checkConstants = [&
I1, &I01]() ->
bool {
3866 if (checkConstants()) {
3874 if (!checkConstants())
3880 const unsigned KeptBits =
I1.logBase2();
3881 const unsigned KeptBitsMinusOne = I01.
logBase2();
3884 if (KeptBits != (KeptBitsMinusOne + 1))
3908SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
3910 DAGCombinerInfo &DCI,
const SDLoc &
DL)
const {
3912 "Should be a comparison with 0.");
3914 "Valid only for [in]equality comparisons.");
3916 unsigned NewShiftOpcode;
3927 unsigned OldShiftOpcode =
V.getOpcode();
3928 switch (OldShiftOpcode) {
3940 C =
V.getOperand(0);
3945 Y =
V.getOperand(1);
3949 return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
3950 X, XC,
CC,
Y, OldShiftOpcode, NewShiftOpcode, DAG);
3967 EVT VT =
X.getValueType();
3982 DAGCombinerInfo &DCI)
const {
3985 "Unexpected binop");
4012 !DCI.isBeforeLegalize());
4015 if (!DCI.isCalledByLegalizer())
4016 DCI.AddToWorklist(YShl1.
getNode());
4031 if (CTPOP.getOpcode() !=
ISD::CTPOP || !CTPOP.hasOneUse())
4034 EVT CTVT = CTPOP.getValueType();
4035 SDValue CTOp = CTPOP.getOperand(0);
4055 for (
unsigned i = 0; i <
Passes; i++) {
4096 auto getRotateSource = [](
SDValue X) {
4098 return X.getOperand(0);
4105 if (
SDValue R = getRotateSource(N0))
4138 if (!C1 || !C1->
isZero())
4147 if (!ShAmtC || ShAmtC->getAPIntValue().uge(
BitWidth))
4151 unsigned ShAmt = ShAmtC->getZExtValue();
4160 if (
Or.getOperand(0) ==
Other) {
4161 X =
Or.getOperand(0);
4162 Y =
Or.getOperand(1);
4165 if (
Or.getOperand(1) ==
Other) {
4166 X =
Or.getOperand(1);
4167 Y =
Or.getOperand(0);
4177 if (matchOr(F0, F1)) {
4184 if (matchOr(F1, F0)) {
4200 const SDLoc &dl)
const {
4210 bool N0ConstOrSplat =
4212 bool N1ConstOrSplat =
4223 return DAG.
getSetCC(dl, VT, N1, N0, SwappedCC);
4229 if (!N0ConstOrSplat && !N1ConstOrSplat &&
4234 return DAG.
getSetCC(dl, VT, N1, N0, SwappedCC);
4243 const APInt &C1 = N1C->getAPIntValue();
4263 return DAG.
getNode(LogicOp, dl, VT, IsXZero, IsYZero);
4293 if (
auto *N1C = dyn_cast<ConstantSDNode>(N1.
getNode())) {
4294 const APInt &C1 = N1C->getAPIntValue();
4309 if (
auto *
C = dyn_cast<ConstantSDNode>(N0->
getOperand(1)))
4310 if ((
C->getAPIntValue()+1).isPowerOf2()) {
4311 MinBits =
C->getAPIntValue().countr_one();
4319 }
else if (
auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
4322 MinBits = LN0->getMemoryVT().getSizeInBits();
4326 MinBits = LN0->getMemoryVT().getSizeInBits();
4337 MinBits >= ReqdBits) {
4339 if (isTypeDesirableForOp(
ISD::SETCC, MinVT)) {
4342 if (MinBits == 1 && C1 == 1)
4374 cast<CondCodeSDNode>(TopSetCC.
getOperand(2))->get(),
4395 unsigned bestWidth = 0, bestOffset = 0;
4398 unsigned maskWidth = origWidth;
4404 for (
unsigned width = origWidth / 2; width>=8; width /= 2) {
4406 for (
unsigned offset=0; offset<origWidth/width; offset++) {
4407 if (Mask.isSubsetOf(newMask)) {
4409 bestOffset = (
uint64_t)offset * (width/8);
4411 bestOffset = (origWidth/width - offset - 1) * (width/8);
4412 bestMask = Mask.lshr(offset * (width/8) * 8);
4425 if (bestOffset != 0)