80#define DEBUG_TYPE "dagcombine"
82STATISTIC(NodesCombined ,
"Number of dag nodes combined");
83STATISTIC(PreIndexedNodes ,
"Number of pre-indexed nodes created");
84STATISTIC(PostIndexedNodes,
"Number of post-indexed nodes created");
85STATISTIC(OpsNarrowed ,
"Number of load/op/store narrowed");
86STATISTIC(LdStFP2Int ,
"Number of fp load/store pairs transformed to int");
88STATISTIC(NumFPLogicOpsConv,
"Number of logic ops converted to fp ops");
92 cl::desc(
"Enable DAG combiner's use of IR alias analysis"));
96 cl::desc(
"Enable DAG combiner's use of TBAA"));
101 cl::desc(
"Only use DAG-combiner alias analysis in this"
109 cl::desc(
"Bypass the profitability model of load slicing"),
114 cl::desc(
"DAG combiner may split indexing from loads"));
118 cl::desc(
"DAG combiner enable merging multiple stores "
119 "into a wider store"));
123 cl::desc(
"Limit the number of operands to inline for Token Factors"));
127 cl::desc(
"Limit the number of times for the same StoreNode and RootNode "
128 "to bail out in store merging dependence check"));
132 cl::desc(
"DAG combiner enable reducing the width of load/op/store "
137 cl::desc(
"DAG combiner enable load/<replace bytes>/store with "
138 "a narrower store"));
143 "Enable merging extends and rounds into FCOPYSIGN on vector types"));
153 bool LegalDAG =
false;
154 bool LegalOperations =
false;
155 bool LegalTypes =
false;
157 bool DisableGenericCombines;
199 void AddUsersToWorklist(
SDNode *
N) {
205 void AddToWorklistWithUsers(
SDNode *
N) {
206 AddUsersToWorklist(
N);
213 void clearAddedDanglingWorklistEntries() {
215 while (!PruningList.
empty()) {
218 recursivelyDeleteUnusedNodes(
N);
222 SDNode *getNextWorklistEntry() {
224 clearAddedDanglingWorklistEntries();
228 while (!
N && !Worklist.
empty()) {
233 bool GoodWorklistEntry = WorklistMap.
erase(
N);
234 (void)GoodWorklistEntry;
235 assert(GoodWorklistEntry &&
236 "Found a worklist entry without a corresponding map entry!");
246 : DAG(
D), TLI(
D.getTargetLoweringInfo()),
247 STI(
D.getSubtarget().getSelectionDAGInfo()), OptLevel(OL), AA(AA) {
251 MaximumLegalStoreInBits = 0;
257 VT.getSizeInBits().getKnownMinValue() >= MaximumLegalStoreInBits)
258 MaximumLegalStoreInBits = VT.getSizeInBits().getKnownMinValue();
261 void ConsiderForPruning(
SDNode *
N) {
268 void AddToWorklist(
SDNode *
N,
bool IsCandidateForPruning =
true) {
270 "Deleted Node added to Worklist");
277 if (IsCandidateForPruning)
278 ConsiderForPruning(
N);
280 if (WorklistMap.
insert(std::make_pair(
N, Worklist.
size())).second)
285 void removeFromWorklist(
SDNode *
N) {
288 StoreRootCountMap.
erase(
N);
290 auto It = WorklistMap.
find(
N);
291 if (It == WorklistMap.
end())
295 Worklist[It->second] =
nullptr;
296 WorklistMap.
erase(It);
299 void deleteAndRecombine(
SDNode *
N);
300 bool recursivelyDeleteUnusedNodes(
SDNode *
N);
308 return CombineTo(
N, &Res, 1, AddTo);
315 return CombineTo(
N, To, 2, AddTo);
321 unsigned MaximumLegalStoreInBits;
327 unsigned BitWidth =
Op.getScalarValueSizeInBits();
339 AddToWorklist(
Op.getNode());
341 CommitTargetLoweringOpt(TLO);
348 bool SimplifyDemandedVectorElts(
SDValue Op) {
350 if (
Op.getValueType().isScalableVector())
353 unsigned NumElts =
Op.getValueType().getVectorNumElements();
355 return SimplifyDemandedVectorElts(
Op, DemandedElts);
359 const APInt &DemandedElts,
360 bool AssumeSingleUse =
false);
361 bool SimplifyDemandedVectorElts(
SDValue Op,
const APInt &DemandedElts,
362 bool AssumeSingleUse =
false);
364 bool CombineToPreIndexedLoadStore(
SDNode *
N);
365 bool CombineToPostIndexedLoadStore(
SDNode *
N);
390 void ReplaceLoadWithPromotedLoad(
SDNode *Load,
SDNode *ExtLoad);
524 bool refineExtractVectorEltIntoMultipleNarrowExtractVectorElts(
SDNode *
N);
552 template <
class MatchContextClass>
554 template <
class MatchContextClass>
559 bool reassociationCanBreakAddressingModePattern(
unsigned Opc,
568 SDValue reassociateReduction(
unsigned RedOpc,
unsigned Opc,
const SDLoc &
DL,
582 bool NotExtCompare =
false);
583 SDValue convertSelectOfFPConstantsToLoadOffset(
598 const SDLoc &
DL,
bool foldBooleans);
602 SDValue &
CC,
bool MatchStrict =
false)
const;
603 bool isOneUseSetCC(
SDValue N)
const;
625 bool KnownNeverZero =
false,
626 bool InexpensiveOnly =
false,
627 std::optional<EVT> OutVT = std::nullopt);
637 bool DemandHighBits =
true);
641 unsigned PosOpcode,
unsigned NegOpcode,
645 unsigned PosOpcode,
unsigned NegOpcode,
661 SDValue VecIn2,
unsigned LeftIdx,
696 int64_t OffsetFromBase;
699 : MemNode(
N), OffsetFromBase(
Offset) {}
704 StoreSource getStoreSource(
SDValue StoreVal) {
708 return StoreSource::Constant;
712 return StoreSource::Constant;
713 return StoreSource::Unknown;
716 return StoreSource::Extract;
718 return StoreSource::Load;
720 return StoreSource::Unknown;
728 bool isMulAddWithConstProfitable(
SDNode *MulNode,
SDValue AddNode,
735 EVT LoadResultTy,
EVT &ExtVT);
740 EVT &MemVT,
unsigned ShAmt = 0);
748 bool BackwardsPropagateMask(
SDNode *
N);
765 EVT MemVT,
unsigned NumStores,
766 bool IsConstantSrc,
bool UseVector,
780 bool checkMergeStoreCandidatesForDependencies(
788 int64_t ElementSizeBytes)
const;
793 unsigned NumConsecutiveStores,
794 EVT MemVT,
SDNode *Root,
bool AllowVectors);
801 unsigned NumConsecutiveStores,
EVT MemVT,
807 unsigned NumConsecutiveStores,
EVT MemVT,
808 SDNode *Root,
bool AllowVectors,
809 bool IsNonTemporalStore,
bool IsNonTemporalLoad);
828 bool hasOperation(
unsigned Opcode,
EVT VT) {
840 EVT getShiftAmountTy(
EVT LHSTy) {
847 bool isTypeLegal(
const EVT &VT) {
848 if (!LegalTypes)
return true;
853 EVT getSetCCResultType(
EVT VT)
const {
868 explicit WorklistRemover(DAGCombiner &dc)
869 :
SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
872 DC.removeFromWorklist(
N);
880 explicit WorklistInserter(DAGCombiner &dc)
881 :
SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
888class EmptyMatchContext {
894 : DAG(DAG), TLI(TLI) {}
901 template <
typename... ArgT>
SDValue getNode(ArgT &&...Args) {
902 return DAG.
getNode(std::forward<ArgT>(Args)...);
905 bool isOperationLegalOrCustom(
unsigned Op,
EVT VT,
906 bool LegalOnly =
false)
const {
911class VPMatchContext {
919 : DAG(DAG), TLI(TLI), RootMaskOp(), RootVectorLenOp() {
924 if (
auto RootVLenPos =
926 RootVectorLenOp = Root->
getOperand(*RootVLenPos);
944 if (RootMaskOp != MaskOp &&
951 if (RootVectorLenOp != OpVal.
getOperand(*VLenPos))
965 {Operand, RootMaskOp, RootVectorLenOp});
974 {N1, N2, RootMaskOp, RootVectorLenOp});
983 {N1, N2, N3, RootMaskOp, RootVectorLenOp});
991 return DAG.
getNode(VPOpcode,
DL, VT, {Operand, RootMaskOp, RootVectorLenOp},
1000 return DAG.
getNode(VPOpcode,
DL, VT, {N1, N2, RootMaskOp, RootVectorLenOp},
1010 {N1, N2, N3, RootMaskOp, RootVectorLenOp},
Flags);
1013 bool isOperationLegalOrCustom(
unsigned Op,
EVT VT,
1014 bool LegalOnly =
false)
const {
1027 ((DAGCombiner*)
DC)->AddToWorklist(
N);
1032 return ((DAGCombiner*)DC)->CombineTo(
N, &To[0], To.
size(), AddTo);
1037 return ((DAGCombiner*)DC)->CombineTo(
N, Res, AddTo);
1042 return ((DAGCombiner*)DC)->CombineTo(
N, Res0, Res1, AddTo);
1047 return ((DAGCombiner*)DC)->recursivelyDeleteUnusedNodes(
N);
1052 return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
1059void DAGCombiner::deleteAndRecombine(
SDNode *
N) {
1060 removeFromWorklist(
N);
1068 if (
Op->hasOneUse() ||
Op->getNumValues() > 1)
1069 AddToWorklist(
Op.getNode());
1078 unsigned Bits =
Offset + std::max(
LHS.getBitWidth(),
RHS.getBitWidth());
1091 LHS =
N.getOperand(0);
1092 RHS =
N.getOperand(1);
1093 CC =
N.getOperand(2);
1100 LHS =
N.getOperand(1);
1101 RHS =
N.getOperand(2);
1102 CC =
N.getOperand(3);
1114 LHS =
N.getOperand(0);
1115 RHS =
N.getOperand(1);
1116 CC =
N.getOperand(4);
1123bool DAGCombiner::isOneUseSetCC(
SDValue N)
const {
1125 if (isSetCCEquivalent(
N, N0, N1, N2) &&
N->hasOneUse())
1137 MaskForTy = 0xFFULL;
1140 MaskForTy = 0xFFFFULL;
1143 MaskForTy = 0xFFFFFFFFULL;
1162 return !(Const->isOpaque() && NoOpaques);
1165 unsigned BitWidth =
N.getScalarValueSizeInBits();
1170 if (!Const || Const->getAPIntValue().getBitWidth() !=
BitWidth ||
1171 (Const->isOpaque() && NoOpaques))
1190 !cast<ConstantSDNode>(LD->getOperand(2))->isOpaque());
1193bool DAGCombiner::reassociationCanBreakAddressingModePattern(
unsigned Opc,
1210 auto *C2 = dyn_cast<ConstantSDNode>(N1);
1214 const APInt &C2APIntVal = C2->getAPIntValue();
1218 if (
auto *C1 = dyn_cast<ConstantSDNode>(N0.
getOperand(1))) {
1222 const APInt &C1APIntVal = C1->getAPIntValue();
1223 const APInt CombinedValueIntVal = C1APIntVal + C2APIntVal;
1226 const int64_t CombinedValue = CombinedValueIntVal.
getSExtValue();
1229 if (
auto *LoadStore = dyn_cast<MemSDNode>(
Node)) {
1235 AM.
BaseOffs = C2APIntVal.getSExtValue();
1237 unsigned AS =
LoadStore->getAddressSpace();
1249 if (
auto *GA = dyn_cast<GlobalAddressSDNode>(N0.
getOperand(1)))
1262 AM.
BaseOffs = C2APIntVal.getSExtValue();
1264 unsigned AS =
LoadStore->getAddressSpace();
1277SDValue DAGCombiner::reassociateOpsCommutative(
unsigned Opc,
const SDLoc &
DL,
1292 return DAG.
getNode(Opc,
DL, VT, N00, OpNode);
1300 Flags.hasNoUnsignedWrap())
1303 return DAG.
getNode(Opc,
DL, VT, OpNode, N01, NewFlags);
1313 if (N1 == N00 || N1 == N01)
1359 if (CC1 == CC00 && CC1 != CC01) {
1361 return DAG.
getNode(Opc,
DL, VT, OpNode, N01, Flags);
1363 if (CC1 == CC01 && CC1 != CC00) {
1365 return DAG.
getNode(Opc,
DL, VT, OpNode, N00, Flags);
1382 if (!
Flags.hasAllowReassociation() || !
Flags.hasNoSignedZeros())
1385 if (
SDValue Combined = reassociateOpsCommutative(Opc,
DL, N0, N1, Flags))
1387 if (
SDValue Combined = reassociateOpsCommutative(Opc,
DL, N1, N0, Flags))
1395SDValue DAGCombiner::reassociateReduction(
unsigned RedOpc,
unsigned Opc,
1413 assert(
N->getNumValues() == NumTo &&
"Broken CombineTo call!");
1417 dbgs() <<
" and " << NumTo - 1 <<
" other values\n");
1418 for (
unsigned i = 0, e = NumTo; i !=
e; ++i)
1419 assert((!To[i].getNode() ||
1420 N->getValueType(i) == To[i].getValueType()) &&
1421 "Cannot combine value to value of different type!");
1423 WorklistRemover DeadNodes(*
this);
1427 for (
unsigned i = 0, e = NumTo; i !=
e; ++i) {
1428 if (To[i].getNode())
1429 AddToWorklistWithUsers(To[i].getNode());
1437 deleteAndRecombine(
N);
1455 recursivelyDeleteUnusedNodes(TLO.
Old.
getNode());
1461 const APInt &DemandedElts,
1462 bool AssumeSingleUse) {
1470 AddToWorklist(
Op.getNode());
1472 CommitTargetLoweringOpt(TLO);
1479bool DAGCombiner::SimplifyDemandedVectorElts(
SDValue Op,
1480 const APInt &DemandedElts,
1481 bool AssumeSingleUse) {
1483 APInt KnownUndef, KnownZero;
1485 TLO, 0, AssumeSingleUse))
1489 AddToWorklist(
Op.getNode());
1491 CommitTargetLoweringOpt(TLO);
1495void DAGCombiner::ReplaceLoadWithPromotedLoad(
SDNode *Load,
SDNode *ExtLoad) {
1497 EVT VT =
Load->getValueType(0);
1506 AddToWorklist(Trunc.
getNode());
1507 recursivelyDeleteUnusedNodes(Load);
1515 EVT MemVT =
LD->getMemoryVT();
1517 :
LD->getExtensionType();
1520 LD->getChain(),
LD->getBasePtr(),
1521 MemVT,
LD->getMemOperand());
1524 unsigned Opc =
Op.getOpcode();
1528 if (
SDValue Op0 = SExtPromoteOperand(
Op.getOperand(0), PVT))
1532 if (
SDValue Op0 = ZExtPromoteOperand(
Op.getOperand(0), PVT))
1550 EVT OldVT =
Op.getValueType();
1552 bool Replace =
false;
1553 SDValue NewOp = PromoteOperand(
Op, PVT, Replace);
1556 AddToWorklist(NewOp.
getNode());
1559 ReplaceLoadWithPromotedLoad(
Op.getNode(), NewOp.
getNode());
1565 EVT OldVT =
Op.getValueType();
1567 bool Replace =
false;
1568 SDValue NewOp = PromoteOperand(
Op, PVT, Replace);
1571 AddToWorklist(NewOp.
getNode());
1574 ReplaceLoadWithPromotedLoad(
Op.getNode(), NewOp.
getNode());
1582 if (!LegalOperations)
1585 EVT VT =
Op.getValueType();
1591 unsigned Opc =
Op.getOpcode();
1599 assert(PVT != VT &&
"Don't know what type to promote to!");
1603 bool Replace0 =
false;
1605 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
1607 bool Replace1 =
false;
1609 SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
1621 Replace1 &= (N0 != N1) && !N1->
hasOneUse();
1624 CombineTo(
Op.getNode(), RV);
1650 if (!LegalOperations)
1653 EVT VT =
Op.getValueType();
1659 unsigned Opc =
Op.getOpcode();
1667 assert(PVT != VT &&
"Don't know what type to promote to!");
1671 bool Replace =
false;
1674 N0 = SExtPromoteOperand(N0, PVT);
1676 N0 = ZExtPromoteOperand(N0, PVT);
1678 N0 = PromoteOperand(N0, PVT, Replace);
1689 ReplaceLoadWithPromotedLoad(
Op.getOperand(0).getNode(), N0.
getNode());
1699 if (!LegalOperations)
1702 EVT VT =
Op.getValueType();
1708 unsigned Opc =
Op.getOpcode();
1716 assert(PVT != VT &&
"Don't know what type to promote to!");
1726bool DAGCombiner::PromoteLoad(
SDValue Op) {
1727 if (!LegalOperations)
1733 EVT VT =
Op.getValueType();
1739 unsigned Opc =
Op.getOpcode();
1747 assert(PVT != VT &&
"Don't know what type to promote to!");
1752 EVT MemVT =
LD->getMemoryVT();
1754 :
LD->getExtensionType();
1756 LD->getChain(),
LD->getBasePtr(),
1757 MemVT,
LD->getMemOperand());
1766 AddToWorklist(
Result.getNode());
1767 recursivelyDeleteUnusedNodes(
N);
1780bool DAGCombiner::recursivelyDeleteUnusedNodes(
SDNode *
N) {
1781 if (!
N->use_empty())
1791 if (
N->use_empty()) {
1792 for (
const SDValue &ChildN :
N->op_values())
1793 Nodes.
insert(ChildN.getNode());
1795 removeFromWorklist(
N);
1800 }
while (!Nodes.
empty());
1815 WorklistInserter AddNodes(*
this);
1824 AddToWorklist(&
Node,
Node.use_empty());
1832 while (
SDNode *
N = getNextWorklistEntry()) {
1836 if (recursivelyDeleteUnusedNodes(
N))
1839 WorklistRemover DeadNodes(*
this);
1847 for (
SDNode *LN : UpdatedNodes)
1848 AddToWorklistWithUsers(LN);
1859 for (
const SDValue &ChildN :
N->op_values())
1860 if (!CombinedNodes.
count(ChildN.getNode()))
1861 AddToWorklist(ChildN.getNode());
1880 "Node was deleted but visit returned new node!");
1888 N->getNumValues() == 1 &&
"Type mismatch");
1898 AddToWorklistWithUsers(RV.
getNode());
1904 recursivelyDeleteUnusedNodes(
N);
1913 switch (
N->getOpcode()) {
2002 case ISD::FMA:
return visitFMA<EmptyMatchContext>(
N);
2062#define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) case ISD::SDOPC:
2063#include "llvm/IR/VPIntrinsics.def"
2064 return visitVPOp(
N);
2071 if (!DisableGenericCombines)
2077 "Node was deleted but visit returned NULL!");
2084 DagCombineInfo(DAG, Level,
false,
this);
2092 switch (
N->getOpcode()) {
2100 RV = PromoteIntBinOp(
SDValue(
N, 0));
2105 RV = PromoteIntShiftOp(
SDValue(
N, 0));
2126 if (N0 != N1 && (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1))) {
2141 if (
unsigned NumOps =
N->getNumOperands()) {
2142 if (
N->getOperand(0).getValueType() == MVT::Other)
2143 return N->getOperand(0);
2144 if (
N->getOperand(NumOps-1).getValueType() == MVT::Other)
2145 return N->getOperand(NumOps-1);
2146 for (
unsigned i = 1; i < NumOps-1; ++i)
2147 if (
N->getOperand(i).getValueType() == MVT::Other)
2148 return N->getOperand(i);
2156 if (
N->getNumOperands() == 2) {
2158 return N->getOperand(0);
2160 return N->getOperand(1);
2175 AddToWorklist(*(
N->use_begin()));
2180 bool Changed =
false;
2187 for (
unsigned i = 0; i < TFs.
size(); ++i) {
2192 for (
unsigned j = i;
j < TFs.
size();
j++)
2203 switch (
Op.getOpcode()) {
2221 if (SeenOps.
insert(
Op.getNode()).second)
2232 for (
unsigned i = 1, e = TFs.
size(); i < e; i++)
2233 AddToWorklist(TFs[i]);
2245 bool DidPruneOps =
false;
2247 unsigned NumLeftToConsider = 0;
2249 Worklist.
push_back(std::make_pair(
Op.getNode(), NumLeftToConsider++));
2253 auto AddToWorklist = [&](
unsigned CurIdx,
SDNode *
Op,
unsigned OpNumber) {
2259 unsigned OrigOpNumber = 0;
2260 while (OrigOpNumber < Ops.size() && Ops[OrigOpNumber].getNode() !=
Op)
2262 assert((OrigOpNumber != Ops.size()) &&
2263 "expected to find TokenFactor Operand");
2265 for (
unsigned i = CurIdx + 1; i < Worklist.
size(); ++i) {
2266 if (Worklist[i].second == OrigOpNumber) {
2267 Worklist[i].second = OpNumber;
2270 OpWorkCount[OpNumber] += OpWorkCount[OrigOpNumber];
2271 OpWorkCount[OrigOpNumber] = 0;
2272 NumLeftToConsider--;
2275 if (SeenChains.
insert(
Op).second) {
2276 OpWorkCount[OpNumber]++;
2281 for (
unsigned i = 0; i < Worklist.
size() && i < 1024; ++i) {
2283 if (NumLeftToConsider <= 1)
2285 auto CurNode = Worklist[i].first;
2286 auto CurOpNumber = Worklist[i].second;
2287 assert((OpWorkCount[CurOpNumber] > 0) &&
2288 "Node should not appear in worklist");
2289 switch (CurNode->getOpcode()) {
2295 NumLeftToConsider++;
2298 for (
const SDValue &
Op : CurNode->op_values())
2299 AddToWorklist(i,
Op.getNode(), CurOpNumber);
2305 AddToWorklist(i, CurNode->getOperand(0).getNode(), CurOpNumber);
2308 if (
auto *MemNode = dyn_cast<MemSDNode>(CurNode))
2309 AddToWorklist(i, MemNode->getChain().getNode(), CurOpNumber);
2312 OpWorkCount[CurOpNumber]--;
2313 if (OpWorkCount[CurOpNumber] == 0)
2314 NumLeftToConsider--;
2328 if (SeenChains.
count(
Op.getNode()) == 0)
2343 WorklistRemover DeadNodes(*
this);
2349 AddUsersToWorklist(
N);
2353 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i)
2356 }
while (!
N->use_empty());
2357 deleteAndRecombine(
N);
2365 return Const !=
nullptr && !Const->isOpaque() ? Const :
nullptr;
2375 Op =
N->getOperand(0);
2381 N.getValueType().getScalarType() != MVT::i1 ||
2382 cast<CondCodeSDNode>(
N.getOperand(2))->get() !=
ISD::SETNE)
2409 if (LD->isIndexed() || LD->getBasePtr().getNode() !=
N)
2411 VT = LD->getMemoryVT();
2412 AS = LD->getAddressSpace();
2414 if (ST->isIndexed() || ST->getBasePtr().getNode() !=
N)
2416 VT = ST->getMemoryVT();
2417 AS = ST->getAddressSpace();
2419 if (LD->isIndexed() || LD->getBasePtr().getNode() !=
N)
2421 VT = LD->getMemoryVT();
2422 AS = LD->getAddressSpace();
2424 if (ST->isIndexed() || ST->getBasePtr().getNode() !=
N)
2426 VT = ST->getMemoryVT();
2427 AS = ST->getAddressSpace();
2442 }
else if (
N->getOpcode() ==
ISD::SUB) {
2464 bool ShouldCommuteOperands) {
2469 if (ShouldCommuteOperands)
2481 unsigned Opcode =
N->getOpcode();
2482 EVT VT =
N->getValueType(0);
2489 unsigned OpNo = ShouldCommuteOperands ? 0 : 1;
2507 "Unexpected binary operator");
2524 unsigned SelOpNo = 0;
2560 bool CanFoldNonConst =
2566 if (!CanFoldNonConst &&
2574 if (CanFoldNonConst) {
2610 "Expecting add or sub");
2615 bool IsAdd =
N->getOpcode() ==
ISD::ADD;
2616 SDValue C = IsAdd ?
N->getOperand(1) :
N->getOperand(0);
2617 SDValue Z = IsAdd ?
N->getOperand(0) :
N->getOperand(1);
2618 auto *CN = dyn_cast<ConstantSDNode>(
C);
2623 if (Z.getOperand(0).getOpcode() !=
ISD::SETCC ||
2624 Z.getOperand(0).getValueType() != MVT::i1)
2628 SDValue SetCC = Z.getOperand(0);
2639 EVT VT =
C.getValueType();
2651 "Expecting add or sub");
2655 bool IsAdd =
N->getOpcode() ==
ISD::ADD;
2656 SDValue ConstantOp = IsAdd ?
N->getOperand(1) :
N->getOperand(0);
2657 SDValue ShiftOp = IsAdd ?
N->getOperand(0) :
N->getOperand(1);
2680 {ConstantOp, DAG.getConstant(1, DL, VT)})) {
2725 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
2757 if ((!LegalOperations ||
2760 X.getScalarValueSizeInBits() == 1) {
2776 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
2780 if (!reassociationCanBreakAddressingModePattern(
ISD::ADD,
DL,
N, N0, N1)) {
2881 return (!Max && !
Op) ||
2882 (
Max &&
Op &&
Max->getAPIntValue() == (-
Op->getAPIntValue()));
2923 !
N->getFlags().hasNoSignedWrap()))) {
2937 if (
SDValue Combined = visitADDLikeCommutative(N0, N1,
N))
2940 if (
SDValue Combined = visitADDLikeCommutative(N1, N0,
N))
2952 if (
SDValue Combined = visitADDLike(
N))
2988 APInt NewStep = C0 + C1;
2998 APInt NewStep = SV0 + SV1;
3007 unsigned Opcode =
N->getOpcode();
3025 return DAG.
getNode(Opcode,
DL, VT, N1, N0);
3029 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
3049 bool ForceCarryReconstruction =
false) {
3050 bool Masked =
false;
3055 V = V.getOperand(0);
3060 if (ForceCarryReconstruction)
3064 V = V.getOperand(0);
3068 if (ForceCarryReconstruction && V.getValueType() == MVT::i1)
3075 if (V.getResNo() != 1)
3082 EVT VT = V->getValueType(0);
3193 if (TN->
getVT() == MVT::i1) {
3210 DAG.
getVTList(VT, Carry.getValueType()), N0,
3223 if (!
N->hasAnyUseOfValue(1))
3256 if (Force && isa<ConstantSDNode>(V))
3266 EVT VT = V.getValueType();
3268 bool IsFlip =
false;
3271 IsFlip = Const->isOne();
3274 IsFlip = Const->isAllOnes();
3277 IsFlip = (Const->getAPIntValue() & 0x01) == 1;
3282 return V.getOperand(0);
3294 EVT CarryVT =
N->getValueType(1);
3298 if (!
N->hasAnyUseOfValue(1))
3305 return DAG.
getNode(
N->getOpcode(),
DL,
N->getVTList(), N1, N0);
3330 if (
SDValue Combined = visitUADDOLike(N0, N1,
N))
3333 if (
SDValue Combined = visitUADDOLike(N1, N0,
N))
3367 SDValue CarryIn =
N->getOperand(2);
3386 SDValue CarryIn =
N->getOperand(2);
3397 if (!LegalOperations ||
3407 AddToWorklist(CarryExt.
getNode());
3413 if (
SDValue Combined = visitUADDO_CARRYLike(N0, N1, CarryIn,
N))
3416 if (
SDValue Combined = visitUADDO_CARRYLike(N1, N0, CarryIn,
N))
3424 SDValue Ops[] = {N1, N0, CarryIn};
3574 unsigned CarryInOperandNum =
3576 if (Opcode ==
ISD::USUBO && CarryInOperandNum != 1)
3668 SDValue CarryIn =
N->getOperand(2);
3679 if (!LegalOperations ||
3684 if (
SDValue Combined = visitSADDO_CARRYLike(N0, N1, CarryIn,
N))
3687 if (
SDValue Combined = visitSADDO_CARRYLike(N1, N0, CarryIn,
N))
3699 "Illegal truncation");
3725 !(!LegalOperations || hasOperation(
ISD::USUBSAT, DstVT)))
3728 EVT SubVT =
N->getValueType(0);
3786 auto PeekThroughFreeze = [](
SDValue N) {
3788 return N->getOperand(0);
3794 if (PeekThroughFreeze(N0) == PeekThroughFreeze(N1))
3803 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
3811 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
3838 if (
N->getFlags().hasNoUnsignedWrap())
3844 if (
N->getFlags().hasNoSignedWrap())
3987 if (
SDValue V = foldSubToUSubSat(VT,
N))
4049 if ((X0 == S0 && X1 == N1) || (X0 == N1 && X1 == S0))
4061 if (GA->getGlobal() == GB->getGlobal())
4069 if (TN->
getVT() == MVT::i1) {
4123 DAG.
getVTList(VT, Carry.getValueType()), NegX, Zero,
4131 if (!C0->isOpaque()) {
4132 const APInt &C0Val = C0->getAPIntValue();
4133 const APInt &MaybeOnes = ~DAG.computeKnownBits(N1).Zero;
4134 if ((C0Val - MaybeOnes) == (C0Val ^ MaybeOnes))