45 #define DEBUG_TYPE "dagcombine"
47 STATISTIC(NodesCombined ,
"Number of dag nodes combined");
48 STATISTIC(PreIndexedNodes ,
"Number of pre-indexed nodes created");
49 STATISTIC(PostIndexedNodes,
"Number of post-indexed nodes created");
50 STATISTIC(OpsNarrowed ,
"Number of load/op/store narrowed");
51 STATISTIC(LdStFP2Int ,
"Number of fp load/store pairs transformed to int");
52 STATISTIC(SlicedLoads,
"Number of load sliced");
56 CombinerAA(
"combiner-alias-analysis",
cl::Hidden,
57 cl::desc(
"Enable DAG combiner alias-analysis heuristics"));
60 CombinerGlobalAA(
"combiner-global-alias-analysis",
cl::Hidden,
61 cl::desc(
"Enable DAG combiner's use of IR alias analysis"));
65 cl::desc(
"Enable DAG combiner's use of TBAA"));
69 CombinerAAOnlyFunc(
"combiner-aa-only-func",
cl::Hidden,
70 cl::desc(
"Only use DAG-combiner alias analysis in this"
77 StressLoadSlicing(
"combiner-stress-load-slicing",
cl::Hidden,
78 cl::desc(
"Bypass the profitability model of load "
84 cl::desc(
"DAG combiner may split indexing from loads"));
124 void AddUsersToWorklist(
SDNode *
N) {
135 void AddToWorklist(
SDNode *N) {
141 if (WorklistMap.insert(std::make_pair(N, Worklist.size())).second)
142 Worklist.push_back(N);
146 void removeFromWorklist(
SDNode *N) {
147 CombinedNodes.erase(N);
149 auto It = WorklistMap.find(N);
150 if (It == WorklistMap.end())
154 Worklist[It->second] =
nullptr;
155 WorklistMap.erase(It);
158 void deleteAndRecombine(
SDNode *N);
159 bool recursivelyDeleteUnusedNodes(
SDNode *N);
167 return CombineTo(N, &Res, 1, AddTo);
174 return CombineTo(N, To, 2, AddTo);
187 return SimplifyDemandedBits(Op, Demanded);
190 bool SimplifyDemandedBits(
SDValue Op,
const APInt &Demanded);
192 bool CombineToPreIndexedLoadStore(
SDNode *N);
193 bool CombineToPostIndexedLoadStore(
SDNode *N);
195 bool SliceUpLoad(
SDNode *N);
205 SDValue ReplaceExtractVectorEltOfLoadWithNarrowedLoad(
344 bool NotExtCompare =
false);
348 const SDLoc &DL,
bool foldBooleans =
true);
352 bool isOneUseSetCC(
SDValue N)
const;
373 bool DemandHighBits =
true);
377 unsigned PosOpcode,
unsigned NegOpcode,
420 MemNode(N), OffsetFromBase(Offset), SequenceNum(Seq) { }
424 int64_t OffsetFromBase;
427 unsigned SequenceNum;
434 bool isMulAddWithConstProfitable(
SDNode *MulNode,
451 EVT LoadResultTy,
EVT &ExtVT,
EVT &LoadedVT,
459 bool MergeStoresOfConstantsOrVecElts(
461 bool IsConstantSrc,
bool UseVector);
466 void getStoreMergeAndAliasCandidates(
473 bool checkMergeStoreCandidatesForDependencies(
494 OptLevel(OL), LegalOperations(
false), LegalTypes(
false), AA(A) {
495 ForCodeSize = DAG.getMachineFunction().getFunction()->optForSize();
505 EVT getShiftAmountTy(
EVT LHSTy) {
509 auto &
DL = DAG.getDataLayout();
510 return LegalTypes ? TLI.getScalarShiftAmountTy(DL, LHSTy)
511 : TLI.getPointerTy(DL);
516 bool isTypeLegal(
const EVT &VT) {
517 if (!LegalTypes)
return true;
518 return TLI.isTypeLegal(VT);
522 EVT getSetCCResultType(
EVT VT)
const {
523 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
535 explicit WorklistRemover(DAGCombiner &dc)
536 :
SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
539 DC.removeFromWorklist(N);
549 ((DAGCombiner*)
DC)->AddToWorklist(N);
554 return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.
size(), AddTo);
559 return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
565 return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
570 return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
577 void DAGCombiner::deleteAndRecombine(
SDNode *N) {
578 removeFromWorklist(N);
598 unsigned Depth = 0) {
606 if (
Depth > 6)
return 0;
609 default:
return false;
613 return LegalOperations ? 0 : 1;
619 if (LegalOperations &&
640 if (Options->HonorSignDependentRoundingFPMath())
return 0;
660 bool LegalOperations,
unsigned Depth = 0) {
668 assert(
Depth <= 6 &&
"GetNegatedExpression doesn't match isNegatibleForFree");
675 APFloat V = cast<ConstantFPSDNode>(
Op)->getValueAPF();
688 LegalOperations,
Depth+1),
693 LegalOperations,
Depth+1),
714 LegalOperations,
Depth+1),
721 LegalOperations,
Depth+1), Flags);
727 LegalOperations,
Depth+1));
731 LegalOperations,
Depth+1),
777 bool DAGCombiner::isOneUseSetCC(
SDValue N)
const {
787 if (isa<ConstantFPSDNode>(N))
799 return !(Const->isOpaque() && NoOpaques);
819 return Splat->isNullValue();
829 return Splat->isOne() && Splat->getAPIntValue().getBitWidth() == BitWidth;
839 return Splat->isAllOnesValue() &&
840 Splat->getAPIntValue().getBitWidth() == BitWidth;
856 if (
SDNode *R = DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
858 if (
SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT,
L, R))
859 return DAG.getNode(Opc, DL, VT, N0.
getOperand(0), OpNode);
868 AddToWorklist(OpNode.
getNode());
875 if (
SDNode *R = DAG.isConstantIntBuildVectorOrConstantInt(N1.
getOperand(1))) {
876 if (
SDNode *
L = DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
878 if (
SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, R,
L))
879 return DAG.getNode(Opc, DL, VT, N1.
getOperand(0), OpNode);
888 AddToWorklist(OpNode.
getNode());
903 dbgs() <<
"\nWith: ";
905 dbgs() <<
" and " << NumTo-1 <<
" other values\n");
906 for (
unsigned i = 0, e = NumTo;
i != e; ++
i)
909 "Cannot combine value to value of different type!");
911 WorklistRemover DeadNodes(*
this);
912 DAG.ReplaceAllUsesWith(N, To);
915 for (
unsigned i = 0, e = NumTo;
i != e; ++
i) {
916 if (To[
i].getNode()) {
917 AddToWorklist(To[
i].getNode());
918 AddUsersToWorklist(To[
i].getNode());
927 deleteAndRecombine(N);
935 WorklistRemover DeadNodes(*
this);
936 DAG.ReplaceAllUsesOfValueWith(TLO.
Old, TLO.
New);
951 bool DAGCombiner::SimplifyDemandedBits(
SDValue Op,
const APInt &Demanded) {
953 APInt KnownZero, KnownOne;
954 if (!TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne, TLO))
964 dbgs() <<
"\nWith: ";
968 CommitTargetLoweringOpt(TLO);
972 void DAGCombiner::ReplaceLoadWithPromotedLoad(
SDNode *
Load,
SDNode *ExtLoad) {
979 dbgs() <<
"\nWith: ";
982 WorklistRemover DeadNodes(*
this);
983 DAG.ReplaceAllUsesOfValueWith(
SDValue(Load, 0), Trunc);
984 DAG.ReplaceAllUsesOfValueWith(
SDValue(Load, 1),
SDValue(ExtLoad, 1));
985 deleteAndRecombine(Load);
986 AddToWorklist(Trunc.
getNode());
1000 return DAG.getExtLoad(ExtType, DL, PVT,
1019 return DAG.getNode(ExtOpc, DL, PVT, Op);
1033 bool Replace =
false;
1034 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1037 AddToWorklist(NewOp.
getNode());
1048 bool Replace =
false;
1049 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1052 AddToWorklist(NewOp.
getNode());
1056 return DAG.getZeroExtendInReg(NewOp, DL, OldVT);
1063 if (!LegalOperations)
1073 if (TLI.isTypeDesirableForOp(Opc, VT))
1079 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1080 assert(PVT != VT &&
"Don't know what type to promote to!");
1082 bool Replace0 =
false;
1084 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
1088 bool Replace1 =
false;
1094 NN1 = PromoteOperand(N1, PVT, Replace1);
1112 DAG.getNode(Opc, DL, PVT, NN0, NN1));
1121 if (!LegalOperations)
1131 if (TLI.isTypeDesirableForOp(Opc, VT))
1137 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1138 assert(PVT != VT &&
"Don't know what type to promote to!");
1140 bool Replace =
false;
1143 N0 = SExtPromoteOperand(Op.
getOperand(0), PVT);
1145 N0 = ZExtPromoteOperand(Op.
getOperand(0), PVT);
1147 N0 = PromoteOperand(N0, PVT, Replace);
1151 AddToWorklist(N0.getNode());
1159 DAG.getNode(Opc, DL, PVT, N0, Op.
getOperand(1)));
1165 if (!LegalOperations)
1175 if (TLI.isTypeDesirableForOp(Opc, VT))
1181 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1182 assert(PVT != VT &&
"Don't know what type to promote to!");
1193 bool DAGCombiner::PromoteLoad(
SDValue Op) {
1194 if (!LegalOperations)
1207 if (TLI.isTypeDesirableForOp(Opc, VT))
1213 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1214 assert(PVT != VT &&
"Don't know what type to promote to!");
1224 SDValue NewLD = DAG.getExtLoad(ExtType, DL, PVT,
1234 WorklistRemover DeadNodes(*
this);
1235 DAG.ReplaceAllUsesOfValueWith(
SDValue(N, 0), Result);
1237 deleteAndRecombine(N);
1238 AddToWorklist(Result.
getNode());
1250 bool DAGCombiner::recursivelyDeleteUnusedNodes(
SDNode *N) {
1263 Nodes.
insert(ChildN.getNode());
1265 removeFromWorklist(N);
1270 }
while (!Nodes.
empty());
1285 for (
SDNode &Node : DAG.allnodes())
1286 AddToWorklist(&Node);
1294 while (!WorklistMap.empty()) {
1298 N = Worklist.pop_back_val();
1301 bool GoodWorklistEntry = WorklistMap.erase(N);
1302 (void)GoodWorklistEntry;
1303 assert(GoodWorklistEntry &&
1304 "Found a worklist entry without a corresponding map entry!");
1309 if (recursivelyDeleteUnusedNodes(N))
1312 WorklistRemover DeadNodes(*
this);
1318 bool NIsValid = DAG.LegalizeOp(N, UpdatedNodes);
1320 for (
SDNode *LN : UpdatedNodes) {
1322 AddUsersToWorklist(LN);
1333 CombinedNodes.insert(N);
1335 if (!CombinedNodes.count(ChildN.getNode()))
1336 AddToWorklist(ChildN.getNode());
1349 if (RV.getNode() ==
N)
1354 "Node was deleted but visit returned new node!");
1357 RV.getNode()->dump(&DAG));
1360 DAG.ReplaceAllUsesWith(N, RV.getNode());
1365 DAG.ReplaceAllUsesWith(N, &OpV);
1369 AddToWorklist(RV.getNode());
1370 AddUsersToWorklist(RV.getNode());
1376 recursivelyDeleteUnusedNodes(N);
1380 DAG.setRoot(
Dummy.getValue());
1381 DAG.RemoveDeadNodes();
1411 case ISD::OR:
return visitOR(N);
1489 "Node was deleted but visit returned NULL!");
1496 DagCombineInfo(DAG,
Level,
false,
this);
1498 RV = TLI.PerformDAGCombine(N, DagCombineInfo);
1512 RV = PromoteIntBinOp(
SDValue(N, 0));
1517 RV = PromoteIntShiftOp(
SDValue(N, 0));
1522 RV = PromoteExtend(
SDValue(N, 0));
1525 if (PromoteLoad(
SDValue(N, 0)))
1539 if (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1)) {
1559 for (
unsigned i = 1;
i < NumOps-1; ++
i)
1579 bool Changed =
false;
1586 for (
unsigned i = 0;
i < TFs.
size(); ++
i) {
1627 Result = DAG.getEntryNode();
1635 bool UseAA = CombinerAA.getNumOccurrences() > 0 ? CombinerAA
1636 : DAG.getSubtarget().useAA();
1637 return CombineTo(N, Result, UseAA );
1645 WorklistRemover DeadNodes(*
this);
1651 AddUsersToWorklist(N);
1656 deleteAndRecombine(N);
1664 return Const !=
nullptr && !Const->
isOpaque() ? Const :
nullptr;
1675 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
1692 if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
1694 if (!DAG.isConstantIntBuildVectorOrConstantInt(N1))
1738 if (N1.getOpcode() ==
ISD::SUB && N1.getOperand(1).getOpcode() ==
ISD::ADD &&
1739 N0 == N1.getOperand(1).getOperand(0))
1741 N1.getOperand(1).getOperand(1));
1744 if (N1.getOpcode() ==
ISD::SUB && N1.getOperand(1).getOpcode() ==
ISD::ADD &&
1745 N0 == N1.getOperand(1).getOperand(1))
1746 return DAG.getNode(
ISD::SUB, DL, VT, N1.getOperand(0),
1747 N1.getOperand(1).getOperand(0));
1751 N1.getOperand(0).getOpcode() ==
ISD::SUB &&
1752 N0 == N1.getOperand(0).getOperand(1))
1753 return DAG.getNode(N1.getOpcode(), DL, VT, N1.getOperand(0).getOperand(0),
1773 if ((!LegalOperations || TLI.isOperationLegal(
ISD::OR, VT)) &&
1774 VT.isInteger() && DAG.haveNoCommonBitsSet(N0, N1))
1778 if (N1.getOpcode() ==
ISD::SHL && N1.getOperand(0).getOpcode() ==
ISD::SUB &&
1780 return DAG.getNode(
ISD::SUB, DL, VT, N0,
1782 N1.getOperand(0).getOperand(1),
1784 if (N0.getOpcode() ==
ISD::SHL && N0.getOperand(0).getOpcode() ==
ISD::SUB &&
1786 return DAG.getNode(
ISD::SUB, DL, VT, N1,
1788 N0.getOperand(0).getOperand(1),
1793 unsigned NumSignBits = DAG.ComputeNumSignBits(AndOp0);
1794 unsigned DestBits = VT.getScalarSizeInBits();
1798 if (NumSignBits == DestBits &&
1805 N0.getOperand(0).getValueType() ==
MVT::i1 &&
1808 return DAG.getNode(
ISD::SUB, DL, VT, N1, ZExt);
1813 VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
1816 DAG.getConstant(1, DL, VT));
1817 return DAG.getNode(
ISD::SUB, DL, VT, N0, ZExt);
1831 return CombineTo(N, DAG.getNode(
ISD::ADD,
SDLoc(N), VT, N0, N1),
1847 APInt LHSZero, LHSOne;
1848 APInt RHSZero, RHSOne;
1849 DAG.computeKnownBits(N0, LHSZero, LHSOne);
1852 DAG.computeKnownBits(N1, RHSZero, RHSOne);
1856 if ((RHSZero & ~LHSZero) == ~LHSZero || (LHSZero & ~RHSZero) == ~RHSZero)
1857 return CombineTo(N, DAG.getNode(
ISD::OR,
SDLoc(N), VT, N0, N1),
1904 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
1915 return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations, LegalTypes);
1916 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
1917 DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
1927 return DAG.getNode(
ISD::ADD, DL, VT, N0,
1939 if (ShiftAmt && ShiftAmt->
getZExtValue() == BitWidth - 1) {
1941 if (!LegalOperations || TLI.isOperationLegal(NewSh, VT))
1970 if (N0.getOpcode() ==
ISD::ADD && N0.getOperand(0) == N1)
1974 if (N0.getOpcode() ==
ISD::ADD && N0.getOperand(1) == N1)
1989 (N0.getOperand(1).getOpcode() ==
ISD::SUB ||
1990 N0.getOperand(1).getOpcode() ==
ISD::ADD) &&
1991 N0.getOperand(1).getOperand(0) == N1)
1992 return DAG.getNode(N0.getOperand(1).getOpcode(), DL, VT, N0.getOperand(0),
1993 N0.getOperand(1).getOperand(1));
1996 if (N0.getOpcode() ==
ISD::ADD && N0.getOperand(1).getOpcode() ==
ISD::ADD &&
1997 N0.getOperand(1).getOperand(1) == N1)
1998 return DAG.getNode(
ISD::ADD, DL, VT, N0.getOperand(0),
1999 N0.getOperand(1).getOperand(0));
2002 if (N0.getOpcode() ==
ISD::SUB && N0.getOperand(1).getOpcode() ==
ISD::SUB &&
2003 N0.getOperand(1).getOperand(1) == N1)
2004 return DAG.getNode(
ISD::SUB, DL, VT, N0.getOperand(0),
2005 N0.getOperand(1).getOperand(0));
2015 if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) {
2018 return DAG.getGlobalAddress(GA->getGlobal(),
SDLoc(N1C), VT,
2023 if (GA->getGlobal() == GB->getGlobal())
2024 return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(),
2033 DAG.getConstant(1, DL, VT));
2034 return DAG.getNode(
ISD::ADD, DL, VT, N0, ZExt);
2049 return CombineTo(N, DAG.getNode(
ISD::SUB, DL, VT, N0, N1),
2054 return CombineTo(N, DAG.getConstant(0, DL, VT),
2063 return CombineTo(N, DAG.getNode(
ISD::XOR, DL, VT, N1, N0),
2088 return DAG.getConstant(0,
SDLoc(N), VT);
2090 bool N0IsConst =
false;
2091 bool N1IsConst =
false;
2092 bool N1IsOpaqueConst =
false;
2093 bool N0IsOpaqueConst =
false;
2094 APInt ConstValue0, ConstValue1;
2097 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
2103 N0IsConst = isa<ConstantSDNode>(N0);
2105 ConstValue0 = cast<ConstantSDNode>(N0)->getAPIntValue();
2106 N0IsOpaqueConst = cast<ConstantSDNode>(N0)->isOpaque();
2108 N1IsConst = isa<ConstantSDNode>(N1);
2110 ConstValue1 = cast<ConstantSDNode>(N1)->getAPIntValue();
2111 N1IsOpaqueConst = cast<ConstantSDNode>(N1)->isOpaque();
2116 if (N0IsConst && N1IsConst && !N0IsOpaqueConst && !N1IsOpaqueConst)
2121 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
2122 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
2125 if (N1IsConst && ConstValue1 == 0)
2132 if (N1IsConst && ConstValue1 == 1 && IsFullSplat)
2137 return DAG.getNode(
ISD::SUB, DL, VT,
2138 DAG.getConstant(0, DL, VT), N0);
2141 if (N1IsConst && !N1IsOpaqueConst && ConstValue1.
isPowerOf2() &&
2144 return DAG.getNode(
ISD::SHL, DL, VT, N0,
2145 DAG.getConstant(ConstValue1.
logBase2(), DL,
2149 if (N1IsConst && !N1IsOpaqueConst && (-ConstValue1).isPowerOf2() &&
2151 unsigned Log2Val = (-ConstValue1).
logBase2();
2155 return DAG.getNode(
ISD::SUB, DL, VT,
2156 DAG.getConstant(0, DL, VT),
2158 DAG.getConstant(Log2Val, DL,
2174 SDValue Sh(
nullptr, 0),
Y(
nullptr, 0);
2194 if (DAG.isConstantIntBuildVectorOrConstantInt(N1) &&
2196 DAG.isConstantIntBuildVectorOrConstantInt(N0.
getOperand(1)) &&
2197 isMulAddWithConstProfitable(N, N0, N1))
2219 default:
return false;
2244 if (!TLI.isTypeLegal(VT) && !TLI.isOperationCustom(DivRemOpc, VT))
2249 if (!TLI.isOperationLegalOrCustom(DivRemOpc, VT) &&
2254 unsigned OtherOpcode = 0;
2257 if (TLI.isOperationLegalOrCustom(Opcode, VT))
2261 if (TLI.isOperationLegalOrCustom(OtherOpcode, VT))
2277 if ((UserOpc == Opcode || UserOpc == OtherOpcode || UserOpc == DivRemOpc) &&
2281 if (UserOpc == OtherOpcode) {
2282 SDVTList VTs = DAG.getVTList(VT, VT);
2283 combined = DAG.
getNode(DivRemOpc,
SDLoc(Node), VTs, Op0, Op1);
2284 }
else if (UserOpc == DivRemOpc) {
2287 assert(UserOpc == Opcode);
2292 CombineTo(User, combined);
2294 CombineTo(User, combined.
getValue(1));
2307 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
2316 return DAG.FoldConstantArithmetic(
ISD::SDIV, DL, VT, N0C, N1C);
2318 if (N1C && N1C->
isOne())
2323 DAG.getConstant(0, DL, VT), N0);
2327 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
2335 !cast<BinaryWithFlagsSDNode>(
N)->
Flags.hasExact() &&
2357 AddToWorklist(SRL.getNode());
2360 DAG.getConstant(lg2, DL,
2368 AddToWorklist(SRA.getNode());
2369 return DAG.getNode(
ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
SRA);
2375 AttributeSet Attr = DAG.getMachineFunction().getFunction()->getAttributes();
2376 if (N1C && !TLI.isIntDivCheap(N->
getValueType(0), Attr))
2383 if (!N1C || TLI.isIntDivCheap(N->
getValueType(0), Attr))
2389 return DAG.getConstant(0, DL, VT);
2404 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
2419 DAG.isKnownToBeAPowerOfTwo(N1)) {
2420 SDValue LogBase2 = BuildLogBase2(N1, DL);
2421 AddToWorklist(LogBase2.
getNode());
2424 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
2425 AddToWorklist(Trunc.getNode());
2433 DAG.isKnownToBeAPowerOfTwo(N10)) {
2434 SDValue LogBase2 = BuildLogBase2(N10, DL);
2435 AddToWorklist(LogBase2.
getNode());
2438 SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ADDVT);
2439 AddToWorklist(Trunc.
getNode());
2441 AddToWorklist(Add.getNode());
2442 return DAG.getNode(
ISD::SRL, DL, VT, N0, Add);
2447 AttributeSet Attr = DAG.getMachineFunction().getFunction()->getAttributes();
2448 if (N1C && !TLI.isIntDivCheap(N->
getValueType(0), Attr))
2455 if (!N1C || TLI.isIntDivCheap(N->
getValueType(0), Attr))
2461 return DAG.getConstant(0, DL, VT);
2482 if (
SDValue Folded = DAG.FoldConstantArithmetic(Opcode, DL, VT, N0C, N1C))
2488 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
2489 return DAG.getNode(
ISD::UREM, DL, VT, N0, N1);
2492 if (DAG.isKnownToBeAPowerOfTwo(N1)) {
2496 AddToWorklist(Add.getNode());
2497 return DAG.getNode(
ISD::AND, DL, VT, N0, Add);
2501 DAG.isKnownToBeAPowerOfTwo(N1.
getOperand(0))) {
2505 AddToWorklist(Add.getNode());
2506 return DAG.getNode(
ISD::AND, DL, VT, N0, Add);
2510 AttributeSet Attr = DAG.getMachineFunction().getFunction()->getAttributes();
2520 if (N1C && !N1C->
isNullValue() && !TLI.isIntDivCheap(VT, Attr)) {
2525 if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != Div.
getNode()) {
2537 return DivRem.getValue(1);
2541 return DAG.getConstant(0, DL, VT);
2567 return DAG.getConstant(0,
SDLoc(N), VT);
2575 if (TLI.isOperationLegal(
ISD::MUL, NewVT)) {
2580 DAG.getConstant(SimpleSize, DL,
2603 return DAG.getConstant(0, DL, VT);
2611 if (TLI.isOperationLegal(
ISD::MUL, NewVT)) {
2616 DAG.getConstant(SimpleSize, DL,
2628 SDValue DAGCombiner::SimplifyNodeWithTwoResults(
SDNode *N,
unsigned LoOp,
2633 (!LegalOperations ||
2634 TLI.isOperationLegalOrCustom(LoOp, N->
getValueType(0)))) {
2636 return CombineTo(N, Res, Res);
2642 (!LegalOperations ||
2645 return CombineTo(N, Res, Res);
2649 if (LoExists && HiExists)
2657 if (LoOpt.getNode() && LoOpt.getNode() != Lo.
getNode() &&
2658 (!LegalOperations ||
2659 TLI.isOperationLegal(LoOpt.getOpcode(), LoOpt.getValueType())))
2660 return CombineTo(N, LoOpt, LoOpt);
2667 if (HiOpt.getNode() && HiOpt != Hi &&
2668 (!LegalOperations ||
2669 TLI.isOperationLegal(HiOpt.getOpcode(), HiOpt.getValueType())))
2670 return CombineTo(N, HiOpt, HiOpt);
2689 if (TLI.isOperationLegal(
ISD::MUL, NewVT)) {
2695 DAG.getConstant(SimpleSize, DL,
2700 return CombineTo(N, Lo, Hi);
2720 if (TLI.isOperationLegal(
ISD::MUL, NewVT)) {
2726 DAG.getConstant(SimpleSize, DL,
2731 return CombineTo(N, Lo, Hi);
2741 if (C2->getAPIntValue() == 2)
2751 if (C2->getAPIntValue() == 2)
2765 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
2772 return DAG.FoldConstantArithmetic(N->
getOpcode(),
SDLoc(N), VT, N0C, N1C);
2775 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
2776 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
2784 SDValue DAGCombiner::SimplifyBinOpWithSameOpcodeHands(
SDNode *N) {
2807 (!LegalTypes || TLI.isTypeDesirableForOp(N->
getOpcode(), Op0VT))) ||
2809 (!TLI.isZExtFree(VT, Op0VT) ||
2810 !TLI.isTruncateFree(Op0VT, VT)) &&
2811 TLI.isTypeLegal(Op0VT))) &&
2814 (!LegalOperations || TLI.isOperationLegal(N->
getOpcode(), Op0VT))) {
2818 AddToWorklist(ORNode.
getNode());
2832 AddToWorklist(ORNode.
getNode());
2879 "Inputs to shuffles are not the same type");
2885 if (SVN0->
hasOneUse() && SVN1->hasOneUse() &&
2886 SVN0->
getMask().equals(SVN1->getMask())) {
2893 ShOp = DAG.getConstant(0,
SDLoc(N), VT);
2904 AddToWorklist(NewNode.
getNode());
2905 return DAG.getVectorShuffle(VT,
SDLoc(N), NewNode, ShOp,
2914 ShOp = DAG.getConstant(0,
SDLoc(N), VT);
2925 AddToWorklist(NewNode.
getNode());
2926 return DAG.getVectorShuffle(VT,
SDLoc(N), ShOp, NewNode,
2945 return DAG.getConstant(0,
SDLoc(LocReference), VT);
2947 SDValue LL, LR, RL, RR, CC0, CC1;
2948 if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){
2952 if (LR == RR && isa<ConstantSDNode>(LR) && Op0 == Op1 &&
2957 if (VT == CCVT || (!LegalOperations && VT ==
MVT::i1)) {
2960 AddToWorklist(ORNode.
getNode());
2961 return DAG.getSetCC(
SDLoc(LocReference), VT, ORNode, LR, Op1);
2968 if (VT == CCVT || (!LegalOperations && VT ==
MVT::i1)) {
2971 AddToWorklist(ANDNode.
getNode());
2972 return DAG.getSetCC(
SDLoc(LocReference), VT, ANDNode, LR, Op1);
2978 if (VT == CCVT || (!LegalOperations && VT ==
MVT::i1)) {
2981 AddToWorklist(ORNode.
getNode());
2982 return DAG.getSetCC(
SDLoc(LocReference), VT, ORNode, LR, Op1);
2988 if (LL == RL && isa<ConstantSDNode>(LR) && isa<ConstantSDNode>(RR) &&
2993 if (VT == CCVT || (!LegalOperations && VT ==
MVT::i1)) {
2996 LL, DAG.getConstant(1, DL,
2998 AddToWorklist(ADDNode.
getNode());
2999 return DAG.getSetCC(
SDLoc(LocReference), VT, ADDNode,
3005 if (LL == RR && LR == RL) {
3009 if (LL == RL && LR == RR) {
3013 (!LegalOperations ||
3036 SRLI->getZExtValue());
3043 N0.
getOperand(0), DAG.getConstant(ADDC, DL, VT));
3044 CombineTo(N0.
getNode(), NewAdd);
3046 return SDValue(LocReference, 0);
3061 const APInt &AndMask = CAnd->getAPIntValue();
3074 (ShiftBits + MaskBits <= Size / 2) &&
3075 TLI.isNarrowingProfitable(VT, HalfVT) &&
3076 TLI.isTypeDesirableForOp(
ISD::AND, HalfVT) &&
3077 TLI.isTypeDesirableForOp(
ISD::SRL, HalfVT) &&
3078 TLI.isTruncateFree(VT, HalfVT) &&
3079 TLI.isZExtFree(HalfVT, VT)) {
3086 assert(MaskBits <= Size);
3089 EVT ShiftVT = TLI.getShiftAmountTy(HalfVT, DAG.getDataLayout());
3093 SDValue NewMask = DAG.getConstant(AndMask.
trunc(Size / 2), SL, HalfVT);
3094 SDValue ShiftK = DAG.getConstant(ShiftBits, SL, ShiftVT);
3107 EVT LoadResultTy,
EVT &ExtVT,
EVT &LoadedVT,
3117 if (ExtVT == LoadedVT &&
3118 (!LegalOperations ||
3132 if (!LoadedVT.bitsGT(ExtVT) || !ExtVT.
isRound())
3135 if (LegalOperations &&
3139 if (!TLI.shouldReduceLoadWidth(LoadN,
ISD::ZEXTLOAD, ExtVT))
3157 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
3180 if (N0C && N1C && !N1C->
isOpaque())
3181 return DAG.FoldConstantArithmetic(
ISD::AND,
SDLoc(N), VT, N0C, N1C);
3183 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
3184 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
3191 if (N1C && DAG.MaskedValueIsZero(
SDValue(N, 0),
3193 return DAG.getConstant(0,
SDLoc(N), VT);
3207 if (DAG.MaskedValueIsZero(N0Op0, Mask)) {
3217 CombineTo(N0.
getNode(), Zext);
3240 Constant =
C->getAPIntValue();
3242 APInt SplatValue, SplatUndef;
3243 unsigned SplatBitSize;
3245 bool IsSplat = Vector->isConstantSplat(SplatValue, SplatUndef,
3246 SplatBitSize, HasAnyUndefs);
3250 SplatValue |= SplatUndef;
3256 EVT VT = Vector->getValueType(0);
3262 if (BitWidth > SplatBitSize)
3263 for (SplatValue = SplatValue.
zextOrTrunc(BitWidth);
3264 SplatBitSize < BitWidth;
3265 SplatBitSize = SplatBitSize * 2)
3266 SplatValue |= SplatValue.
shl(SplatBitSize);
3270 if (SplatBitSize % BitWidth == 0) {
3272 for (
unsigned i = 0, n = SplatBitSize/BitWidth;
i < n; ++
i)
3281 bool CanZextLoadProfitably = TLI.isLoadExtLegal(
ISD::ZEXTLOAD,
3292 default: B =
false;
break;
3298 if (B && Constant.isAllOnesValue()) {
3313 CombineTo(Load, To, 3,
true);
3315 CombineTo(Load, NewLoad.getValue(0), NewLoad.getValue(1));
3321 CombineTo(N, (N0.
getNode() ==
Load) ? NewLoad : N0);
3336 : cast<LoadSDNode>(N0);
3339 auto NarrowLoad =
false;
3341 EVT ExtVT, LoadedVT;
3342 if (isAndLoadExtLoad(N1C, LN0, LoadResultTy, ExtVT, LoadedVT,
3350 CombineTo(LN0, NewLoad, NewLoad.
getValue(1));
3361 if (DAG.getDataLayout().isBigEndian()) {
3364 unsigned PtrOff = LVTStoreBytes - EVTStoreBytes;
3367 NewPtr, DAG.getConstant(PtrOff, DL, PtrType));
3368 Alignment =
MinAlign(Alignment, PtrOff);
3371 AddToWorklist(NewPtr.
getNode());
3373 SDValue Load = DAG.getExtLoad(
3378 CombineTo(LN0, Load, Load.getValue(1));
3385 if (
SDValue Combined = visitANDLike(N0, N1, N))
3390 if (
SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
3405 SubRHS.getOperand(0).getScalarValueSizeInBits() == 1)
3408 SubRHS.getOperand(0).getScalarValueSizeInBits() == 1)
3426 BitWidth - MemVT.getScalarSizeInBits())) &&
3446 BitWidth - MemVT.getScalarSizeInBits())) &&
3469 bool DemandHighBits) {
3470 if (!LegalOperations)
3480 bool LookPassAnd0 =
false;
3481 bool LookPassAnd1 =
false;
3493 LookPassAnd0 =
true;
3503 LookPassAnd1 =
true;
3517 if (N01C->
getZExtValue() != 8 || N11C->getZExtValue() != 8)
3529 LookPassAnd0 =
true;
3540 LookPassAnd1 =
true;
3549 if (DemandHighBits && OpSizeInBits > 16) {
3558 if (!LookPassAnd1 &&
3559 !DAG.MaskedValueIsZero(
3565 if (OpSizeInBits > 16) {
3568 DAG.getConstant(OpSizeInBits - 16, DL,
3596 case 0xFF: Num = 0;
break;
3597 case 0xFF00: Num = 1;
break;
3598 case 0xFF0000: Num = 2;
break;
3599 case 0xFF000000: Num = 3;
break;
3605 if (Num == 0 || Num == 2) {
3625 if (Num != 0 && Num != 2)
3633 if (Num != 1 && Num != 3)
3654 if (!LegalOperations)
3705 if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
3715 if (TLI.isOperationLegalOrCustom(
ISD::ROTL, VT))
3717 if (TLI.isOperationLegalOrCustom(
ISD::ROTR, VT))
3718 return DAG.getNode(
ISD::ROTR, DL, VT, BSwap, ShAmt);
3719 return DAG.getNode(
ISD::OR, DL, VT,
3720 DAG.getNode(
ISD::SHL, DL, VT, BSwap, ShAmt),
3721 DAG.getNode(
ISD::SRL, DL, VT, BSwap, ShAmt));
3729 if (!LegalOperations &&
3733 SDLoc(LocReference), VT);
3736 SDValue LL, LR, RL, RR, CC0, CC1;
3737 if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){
3746 if (VT == CCVT || (!LegalOperations && VT ==
MVT::i1)) {
3749 AddToWorklist(ORNode.
getNode());
3750 return DAG.getSetCC(
SDLoc(LocReference), VT, ORNode, LR, Op1);
3757 if (VT == CCVT || (!LegalOperations && VT ==
MVT::i1)) {
3760 AddToWorklist(ANDNode.
getNode());
3761 return DAG.getSetCC(
SDLoc(LocReference), VT, ANDNode, LR, Op1);
3766 if (LL == RR && LR == RL) {
3770 if (LL == RL && LR == RR) {
3774 (!LegalOperations ||
3798 const APInt &LHSMask = N0O1C->getAPIntValue();
3799 const APInt &RHSMask = N1O1C->getAPIntValue();
3801 if (DAG.MaskedValueIsZero(N0.
getOperand(0), RHSMask&~LHSMask) &&
3802 DAG.MaskedValueIsZero(N1.
getOperand(0), LHSMask&~RHSMask)) {
3805 SDLoc DL(LocReference);
3806 return DAG.getNode(
ISD::AND, DL, VT, X,
3807 DAG.getConstant(LHSMask | RHSMask, DL, VT));
3838 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
3850 return DAG.getConstant(
3855 return DAG.getConstant(
3861 if (isa<ShuffleVectorSDNode>(N0) &&
3862 isa<ShuffleVectorSDNode>(N1) &&
3864 TLI.isTypeLegal(VT)) {
3870 if ((ZeroN00 || ZeroN01) && (ZeroN10 || ZeroN11)) {
3871 assert((!ZeroN00 || !ZeroN01) &&
"Both inputs zero!");
3872 assert((!ZeroN10 || !ZeroN11) &&
"Both inputs zero!");
3875 bool CanFold =
true;
3879 for (
int i = 0;
i != NumElts; ++
i) {
3881 int M1 = SV1->getMaskElt(
i);
3884 bool M0Zero = M0 < 0 || (ZeroN00 == (M0 < NumElts));
3885 bool M1Zero = M1 < 0 || (ZeroN10 == (M1 < NumElts));
3889 if ((M0Zero && M1 < 0) || (M1Zero && M0 < 0)) {
3895 if (M0Zero == M1Zero) {
3900 assert((M0 >= 0 || M1 >= 0) &&
"Undef index!");
3906 Mask[
i] = M1Zero ? M0 % NumElts : (M1 % NumElts) + NumElts;
3913 bool LegalMask = TLI.isShuffleMaskLegal(Mask, VT);
3917 LegalMask = TLI.isShuffleMaskLegal(Mask, VT);
3921 return DAG.getVectorShuffle(VT,
SDLoc(N), NewLHS, NewRHS, Mask);
3930 if (N0C && N1C && !N1C->
isOpaque())
3931 return DAG.FoldConstantArithmetic(
ISD::OR,
SDLoc(N), VT, N0C, N1C);
3933 if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
3934 !DAG.isConstantIntBuildVectorOrConstantInt(N1))
3943 if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->
getAPIntValue()))
3946 if (
SDValue Combined = visitORLike(N0, N1, N))
3950 if (
SDValue BSwap = MatchBSwapHWord(N, N0, N1))
3952 if (
SDValue BSwap = MatchBSwapHWordLow(N, N0, N1))
3974 if (
SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
3992 if (DAG.isConstantIntBuildVectorOrConstantInt(Op.
getOperand(1))) {
4049 unsigned MaskLoBits = 0;
4052 if (NegC->getAPIntValue() == EltSize - 1) {
4054 MaskLoBits =
Log2_64(EltSize);
4071 if (PosC->getAPIntValue() == EltSize - 1)
4107 return Width.
getLoBits(MaskLoBits) == 0;
4108 return Width == EltSize;
4118 SDValue InnerNeg,
unsigned PosOpcode,
4119 unsigned NegOpcode,
const SDLoc &DL) {
4129 bool HasPos = TLI.isOperationLegalOrCustom(PosOpcode, VT);
4130 return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, Shifted,
4131 HasPos ? Pos : Neg).getNode();
4143 if (!TLI.isTypeLegal(VT))
return nullptr;
4146 bool HasROTL = TLI.isOperationLegalOrCustom(
ISD::ROTL, VT);
4147 bool HasROTR = TLI.isOperationLegalOrCustom(
ISD::ROTR, VT);
4148 if (!HasROTL && !HasROTR)
return nullptr;
4153 if (!MatchRotateHalf(LHS, LHSShift, LHSMask))
4158 if (!MatchRotateHalf(RHS, RHSShift, RHSMask))
4185 if ((LShVal + RShVal) != EltSizeInBits)
4189 LHSShiftArg, HasROTL ? LHSShiftAmt : RHSShiftAmt);
4194 SDValue Mask = DAG.getConstant(AllBits, DL, VT);
4200 DAG.getConstant(RHSBits, DL, VT)));
4206 DAG.getConstant(LHSBits, DL, VT)));
4221 SDValue LExtOp0 = LHSShiftAmt;
4222 SDValue RExtOp0 = RHSShiftAmt;
4235 SDNode *TryL = MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt,
4236 LExtOp0, RExtOp0,
ISD::ROTL, ISD::ROTR, DL);
4240 SDNode *TryR = MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
4241 RExtOp0, LExtOp0, ISD::ROTR,
ISD::ROTL, DL);
4262 struct BaseIndexOffset {
4266 bool IsIndexSignExt;
4268 BaseIndexOffset() :
Offset(0), IsIndexSignExt(
false) {}
4271 bool IsIndexSignExt) :
4272 Base(Base), Index(Index), Offset(Offset), IsIndexSignExt(IsIndexSignExt) {}
4274 bool equalBaseIndex(
const BaseIndexOffset &Other) {
4275 return Other.Base == Base && Other.Index == Index &&
4276 Other.IsIndexSignExt == IsIndexSignExt;
4281 int64_t PartialOffset = 0) {
4282 bool IsIndexSignExt =
false;
4289 GA->getValueType(0),
4292 GA->getTargetFlags()),
4302 return BaseIndexOffset(Ptr,
SDValue(), PartialOffset, IsIndexSignExt);
4306 if (isa<ConstantSDNode>(Ptr->
getOperand(1))) {
4317 return BaseIndexOffset(Ptr,
SDValue(), PartialOffset, IsIndexSignExt);
4326 IsIndexSignExt =
true;
4331 return BaseIndexOffset(Base, IndexOffset, PartialOffset, IsIndexSignExt);
4337 if (!isa<ConstantSDNode>(Offset))
4338 return BaseIndexOffset(Ptr,
SDValue(), PartialOffset, IsIndexSignExt);
4343 IsIndexSignExt =
true;
4344 }
else IsIndexSignExt =
false;
4346 int64_t Off = cast<ConstantSDNode>(
Offset)->getSExtValue();
4347 return BaseIndexOffset(Base, Index, Off + PartialOffset, IsIndexSignExt);
4359 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
4395 if (TLI.isConstTrueVal(N1.
getNode()) && isSetCCEquivalent(N0, LHS, RHS, CC)) {
4400 if (!LegalOperations ||
4417 isSetCCEquivalent(N0.
getOperand(0), LHS, RHS, CC)){
4430 if (isOneUseSetCC(RHS) || isOneUseSetCC(LHS)) {
4434 AddToWorklist(LHS.
getNode()); AddToWorklist(RHS.getNode());
4442 if (isa<ConstantSDNode>(RHS) || isa<ConstantSDNode>(LHS)) {
4446 AddToWorklist(LHS.
getNode()); AddToWorklist(RHS.getNode());
4455 AddToWorklist(NotX.
getNode());
4464 N00C->getAPIntValue(), DL, VT));
4504 if (
SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N))
4525 bool HighBitSet =
false;
4545 if (!BinOpCst)
return SDValue();
4556 if ((!isShift || !isa<ConstantSDNode>(BinOpLHSVal->
getOperand(1))) &&
4571 if (BinOpRHSSignSet != HighBitSet)
4575 if (!TLI.isDesirableToCommuteWithShift(LHS))
4582 assert(isa<ConstantSDNode>(NewRHS) &&
"Folding was not successful!");
4593 SDValue DAGCombiner::distributeTruncateThroughAnd(
SDNode *N) {
4606 AddToWorklist(Trunc00.
getNode());
4607 AddToWorklist(Trunc01.
getNode());
4635 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
4662 if (N0C && N1C && !N1C->
isOpaque())
4694 APInt c1 = N0C1->getAPIntValue();
4698 APInt Sum = c1 + c2;
4699 if (Sum.
uge(OpSizeInBits))
4719 APInt c1 = N0Op0C1->getAPIntValue();
4725 if (c2.
uge(OpSizeInBits - InnerShiftSize)) {
4727 APInt Sum = c1 + c2;
4728 if (Sum.
uge(OpSizeInBits))
4756 AddToWorklist(NewSHL.
getNode());
4766 cast<BinaryWithFlagsSDNode>(N0)->Flags.hasExact()) {
4785 uint64_t c1 = N0C1->getZExtValue();
4786 if (c1 < OpSizeInBits) {
4791 Mask = Mask.
shl(c2 - c1);
4796 Mask = Mask.
lshr(c1 - c2);
4811 unsigned BitSize = VT.getScalarSizeInBits();
4826 AddToWorklist(Shl0.
getNode());
4827 AddToWorklist(Shl1.
getNode());
4841 if (
SDValue NewSHL = visitShiftByConstant(N, N1C))
4859 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
4866 if (N0C && N1C && !N1C->
isOpaque())
4888 if ((!LegalOperations ||
4898 APInt c1 = N0C1->getAPIntValue();
4902 APInt Sum = c1 + c2;
4903 if (Sum.
uge(OpSizeInBits))
4904 Sum =
APInt(OpSizeInBits, OpSizeInBits - 1);
4935 if ((ShiftAmt > 0) &&
4938 TLI.isTruncateFree(VT, TruncVT)) {
4955 N1.getOperand(0).getOpcode() ==
ISD::AND) {
4956 if (
SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
4970 unsigned LargeShiftVal = LargeShift->getZExtValue();
4995 if (
SDValue NewSRA = visitShiftByConstant(N, N1C))
5009 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
5016 if (N0C && N1C && !N1C->
isOpaque())
5036 APInt c1 = N0C1->getAPIntValue();
5040 APInt Sum = c1 + c2;
5041 if (Sum.
uge(OpSizeInBits))
5061 if (c1 + OpSizeInBits == InnerShiftSize) {
5063 if (c1 + c2 >= InnerShiftSize)
5080 AddToWorklist(Mask.getNode());
5092 if (!LegalTypes || TLI.isTypeDesirableForOp(
ISD::SRL, SmallVT)) {
5099 AddToWorklist(SmallShift.
getNode());
5118 APInt KnownZero, KnownOne;
5127 APInt UnknownBits = ~KnownZero;
5131 if ((UnknownBits & (UnknownBits - 1)) == 0) {
5166 if (
SDValue NewSRL = visitShiftByConstant(N, N1C))
5170 if (
SDValue NarrowLoad = ReduceLoadWidth(N))
5283 if (!(LHS == True && RHS == False) && !(LHS == False && RHS == True))
5295 return DAG.
getNode(Opcode, DL, VT, LHS, RHS);
5306 return DAG.
getNode(Opcode, DL, VT, LHS, RHS);
5334 TLI.getBooleanContents(
false,
true) ==
5336 TLI.getBooleanContents(
false,
false) ==
5359 if (
const ConstantSDNode *N0C = dyn_cast<const ConstantSDNode>(N0)) {
5369 if (
SDValue V = foldSelectOfConstants(N))
5375 AddToWorklist(NOTNode.
getNode());
5381 AddToWorklist(NOTNode.
getNode());
5390 if (SimplifySelectOps(N, N1, N2))
5402 bool normalizeToSequence
5403 = TLI.shouldNormalizeToSelectSequence(*DAG.
getContext(), VT);
5411 if (normalizeToSequence || !InnerSelect.
use_empty())
5420 N1.getValueType(), Cond1, N1, N2);
5421 if (normalizeToSequence || !InnerSelect.
use_empty())
5427 if (N1->getOpcode() ==
ISD::SELECT && N1->hasOneUse()) {
5433 if (!normalizeToSequence) {
5440 if (
SDValue Combined = visitANDLike(N0, N1_0, N))
5452 if (!normalizeToSequence) {
5459 if (
SDValue Combined = visitORLike(N0, N2_0, N))
5469 if (
auto *
C = dyn_cast<ConstantSDNode>(N0->
getOperand(1))) {
5501 if ((!LegalOperations &&
5507 return SimplifySelect(
SDLoc(N), N0, N1, N2);
5527 return std::make_pair(Lo, Hi);
5554 for (
int i = 0;
i < NumElems / 2; ++
i) {
5558 if (BottomHalf ==
nullptr)
5559 BottomHalf = cast<ConstantSDNode>(Cond.
getOperand(
i));
5566 for (
int i = NumElems / 2;
i < NumElems; ++
i) {
5570 if (TopHalf ==
nullptr)
5571 TopHalf = cast<ConstantSDNode>(Cond.
getOperand(
i));
5576 assert(TopHalf && BottomHalf &&
5577 "One half of the selector was all UNDEFs and the other was all the "
5578 "same value. This should have been addressed before this function.");
5607 std::tie(MaskLo, MaskHi) =
SplitVSETCC(Mask.getNode(), DAG);
5617 EVT LoMemVT, HiMemVT;
5621 std::tie(DataLo, DataHi) = DAG.
SplitVector(Data, DL);
5632 SDValue OpsLo[] = { Chain, DataLo, MaskLo, BasePtr, IndexLo };
5636 SDValue OpsHi[] = {Chain, DataHi, MaskHi, BasePtr, IndexHi};
5664 if (TLI.getTypeAction(*DAG.
getContext(), VT) !=
5669 std::tie(MaskLo, MaskHi) =
SplitVSETCC(Mask.getNode(), DAG);
5679 unsigned SecondHalfAlignment =
5680 (Alignment == VT.
getSizeInBits() / 8) ? Alignment / 2 : Alignment;
5682 EVT LoMemVT, HiMemVT;
5686 std::tie(DataLo, DataHi) = DAG.
SplitVector(Data, DL);
5693 Lo = DAG.
getMaskedStore(Chain, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
5697 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
5706 Hi = DAG.
getMaskedStore(Chain, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
5710 AddToWorklist(Lo.getNode());
5711 AddToWorklist(Hi.getNode());
5738 if (TLI.getTypeAction(*DAG.
getContext(), VT) !=
5743 std::tie(MaskLo, MaskHi) =
SplitVSETCC(Mask.getNode(), DAG);
5747 std::tie(Src0Lo, Src0Hi) = DAG.
SplitVector(Src0, DL);
5756 EVT LoMemVT, HiMemVT;
5762 std::tie(IndexLo, IndexHi) = DAG.
SplitVector(Index, DL);
5769 SDValue OpsLo[] = { Chain, Src0Lo, MaskLo, BasePtr, IndexLo };
5773 SDValue OpsHi[] = {Chain, Src0Hi, MaskHi, BasePtr, IndexHi};
5791 SDValue RetOps[] = { GatherRes, Chain };
5813 if (TLI.getTypeAction(*DAG.
getContext(), VT) !=
5818 std::tie(MaskLo, MaskHi) =
SplitVSETCC(Mask.getNode(), DAG);
5822 std::tie(Src0Lo, Src0Hi) = DAG.
SplitVector(Src0, DL);
5834 unsigned SecondHalfAlignment =
5836 Alignment/2 : Alignment;
5838 EVT LoMemVT, HiMemVT;
5846 Lo = DAG.
getMaskedLoad(LoVT, DL, Chain, Ptr, MaskLo, Src0Lo, LoMemVT, MMO,
5849 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
5857 Hi = DAG.
getMaskedLoad(HiVT, DL, Chain, Ptr, MaskHi, Src0Hi, HiMemVT, MMO,
5860 AddToWorklist(Lo.getNode());
5861 AddToWorklist(Hi.getNode());
5874 SDValue RetOps[] = { LoadRes, Chain };
5915 AddToWorklist(Shift.
getNode());
5921 if (SimplifySelectOps(N, N1, N2))
5932 if (TLI.getTypeAction(*DAG.
getContext(), VT) !=
5986 CC,
SDLoc(N),
false)) {
5987 AddToWorklist(SCC.getNode());
5989 if (
ConstantSDNode *SCCC = dyn_cast<ConstantSDNode>(SCC.getNode())) {
5990 if (!SCCC->isNullValue())
5994 }
else if (SCC->isUndef()) {
6001 SCC.getOperand(0), SCC.getOperand(1), N2, N3,
6007 if (SimplifySelectOps(N, N2, N3))
6011 return SimplifySelectCC(
SDLoc(N), N0, N1, N2, N3, CC);
6016 cast<CondCodeSDNode>(N->
getOperand(2))->
get(),
6041 bool LegalOperations) {
6049 &&
"Expected EXTEND dag node in input!");
6054 if (isa<ConstantSDNode>(N0))
6062 (!LegalTypes || (!LegalOperations && TLI.
isTypeLegal(SVT))) &&
6073 for (
unsigned i=0;
i != NumElts; ++
i) {
6083 APInt C = cast<ConstantSDNode>(
Op)->getAPIntValue().zextOrTrunc(EVTBits);
6101 bool HasCopyToRegUses =
false;
6109 if (UI.getUse().getResNo() != N0.
getResNo())
6118 for (
unsigned i = 0;
i != 2; ++
i) {
6122 if (!isa<ConstantSDNode>(UseOp))
6136 HasCopyToRegUses =
true;
6139 if (HasCopyToRegUses) {
6140 bool BothLiveOut =
false;
6143 SDUse &Use = UI.getUse();
6152 return ExtendNodes.
size();
6161 for (
unsigned i = 0, e = SetCCs.
size();
i != e; ++
i) {
6165 for (
unsigned j = 0; j != 2; ++j) {
6186 "Unexpected node type (not an extend)!");
6223 EVT SplitSrcVT = SrcVT;
6224 EVT SplitDstVT = DstVT;
6225 while (!TLI.isLoadExtLegalOrCustom(ExtType, SplitDstVT, SplitSrcVT) &&
6231 if (!TLI.isLoadExtLegalOrCustom(ExtType, SplitDstVT, SplitSrcVT))
6235 const unsigned NumSplits =
6242 for (
unsigned Idx = 0; Idx < NumSplits; Idx++) {
6243 const unsigned Offset = Idx * Stride;
6247 ExtType, DL, SplitDstVT, LN0->
getChain(), BasePtr,
6255 Chains.
push_back(SplitLoad.getValue(1));
6261 CombineTo(N, NewValue);
6267 CombineTo(N0.
getNode(), Trunc, NewChain);
6268 ExtendSetCCUses(SetCCs, Trunc, NewValue, DL,
6292 if (NarrowLoad.getNode() != N0.
getNode()) {
6293 CombineTo(N0.
getNode(), NarrowLoad);
6308 if (OpBits == DestBits) {
6311 if (NumSignBits > DestBits-MidBits)
6313 }
else if (OpBits < DestBits) {
6316 if (NumSignBits > OpBits-MidBits)
6321 if (NumSignBits > OpBits-MidBits)
6328 if (OpBits < DestBits)
6330 else if (OpBits > DestBits)
6341 ((!LegalOperations && !VT.
isVector() &&
6344 bool DoXform =
true;
6349 DoXform &= TLI.isVectorLoadExtDesirable(
SDValue(N, 0));
6356 CombineTo(N, ExtLoad);
6360 ExtendSetCCUses(SetCCs, Trunc, ExtLoad,
SDLoc(N),
6368 if (
SDValue ExtLoad = CombineExtLoad(N))
6377 if ((!LegalOperations && !LN0->
isVolatile()) ||
6383 CombineTo(N, ExtLoad);
6399 (!LegalOperations && TLI.isOperationLegal(N0.
getOpcode(), VT))) {
6402 bool DoXform =
true;
6422 ExtendSetCCUses(SetCCs, Trunc, ExtLoad, DL,
6433 if (VT.isVector() && !LegalOperations &&
6434 TLI.getBooleanContents(N0VT) ==
6449 cast<CondCodeSDNode>(N0.
getOperand(2))->
get());
6455 if (SVT == MatchingVectorType) {
6458 cast<CondCodeSDNode>(N0.
getOperand(2))->
get());
6480 : TLI.getConstTrueVal(DAG, VT, DL);
6482 if (
SDValue SCC = SimplifySelectCC(
6485 cast<CondCodeSDNode>(N0.
getOperand(2))->
get(),
true))
6488 if (!VT.isVector()) {
6490 if (!LegalOperations ||
6496 return DAG.
getSelect(DL, VT, SetCC, ExtTrueVal,
6567 APInt TruncatedBits =
6574 if (TruncatedBits == (KnownZero & TruncatedBits)) {
6589 if (NarrowLoad.getNode() != N0.
getNode()) {
6590 CombineTo(N0.
getNode(), NarrowLoad);
6604 if (NarrowLoad.getNode() != N0.
getNode()) {
6605 CombineTo(N0.
getNode(), NarrowLoad);
6618 if (!LegalOperations || (TLI.isOperationLegal(
ISD::AND, SrcVT) &&
6627 if (!LegalOperations || TLI.isOperationLegal(
ISD::AND, VT)) {
6632 }
else if (SrcVT.
bitsGT(VT)) {
6665 ((!LegalOperations && !VT.
isVector() &&
6668 bool DoXform =
true;
6673 DoXform &= TLI.isVectorLoadExtDesirable(
SDValue(N, 0));
6680 CombineTo(N, ExtLoad);
6685 ExtendSetCCUses(SetCCs, Trunc, ExtLoad,
SDLoc(N),
6693 if (
SDValue ExtLoad = CombineExtLoad(N))
6705 (!LegalOperations && TLI.isOperationLegal(N0.
getOpcode(), VT))) {
6708 bool DoXform =
true;
6712 auto *AndC = cast<ConstantSDNode>(N0.
getOperand(1));
6713 auto NarrowLoad =
false;
6715 EVT ExtVT, LoadedVT;
6716 if (isAndLoadExtLoad(AndC, LN0, LoadResultTy, ExtVT, LoadedVT,
6739 ExtendSetCCUses(SetCCs, Trunc, ExtLoad, DL,
6752 if ((!LegalOperations && !LN0->
isVolatile()) ||
6758 CombineTo(N, ExtLoad);
6769 if (!LegalOperations && VT.isVector() &&
6805 if (
SDValue SCC = SimplifySelectCC(
6808 cast<CondCodeSDNode>(N0.
getOperand(2))->
get(),
true))
6818 unsigned ShAmtVal = cast<ConstantSDNode>(ShAmt)->getZExtValue();
6825 if (ShAmtVal > KnownZeroBits)
6832 if (VT.getSizeInBits() >= 256)
6864 if (NarrowLoad.getNode() != N0.
getNode()) {
6865 CombineTo(N0.
getNode(), NarrowLoad);
6910 bool DoXform =
true;
6920 CombineTo(N, ExtLoad);
6924 ExtendSetCCUses(SetCCs, Trunc, ExtLoad,
SDLoc(N),
6939 if (!LegalOperations || TLI.isLoadExtLegal(ExtType, VT, MemVT)) {
6943 CombineTo(N, ExtLoad);
6958 if (VT.
isVector() && !LegalOperations) {
6968 cast<CondCodeSDNode>(N0.
getOperand(2))->
get());
6977 cast<CondCodeSDNode>(N0.
getOperand(2))->
get());
6984 if (
SDValue SCC = SimplifySelectCC(
6987 cast<CondCodeSDNode>(N0.
getOperand(2))->
get(),
true))
7002 assert(CV &&
"Const value should be ConstSDNode.");
7023 unsigned Amt = RHSC->getZExtValue();
7026 if (Amt >= Mask.getBitWidth())
break;
7027 APInt NewMask = Mask << Amt;
7057 ExtVT = cast<VTSDNode>(N->
getOperand(1))->getVT();
7067 if (LegalOperations && !TLI.isLoadExtLegal(ExtType, VT, ExtVT))
7082 if ((ShAmt & (EVTBits-1)) == 0) {
7090 if (!isa<LoadSDNode>(N0))
return SDValue();
7095 if (cast<LoadSDNode>(N0)->getExtensionType() ==
ISD::SEXTLOAD)
7101 if (ShAmt >= cast<LoadSDNode>(N0)->getMemoryVT().getSizeInBits())
7108 unsigned ShLeftAmt = 0;
7110 ExtVT == VT && TLI.isNarrowingProfitable(N0.
getValueType(), VT)) {
7119 if (!isa<LoadSDNode>(N0) || !N0.
hasOneUse())
7146 if (!TLI.shouldReduceLoadWidth(LN0, ExtType, ExtVT))
7160 ShAmt = LVTStoreBits - EVTStoreBits - ShAmt;
7163 uint64_t PtrOff = ShAmt / 8;
7173 AddToWorklist(NewPtr.
getNode());
7187 WorklistRemover DeadNodes(*
this);
7192 if (ShLeftAmt != 0) {
7216 EVT EVT = cast<VTSDNode>(N1)->getVT();
7267 if (
SDValue NarrowLoad = ReduceLoadWidth(N))
7275 if (ShAmt->getZExtValue()+EVTBits <= VTBits) {
7279 if (VTBits-(ShAmt->getZExtValue()+EVTBits) < InSignBits)
7288 EVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
7289 ((!LegalOperations && !cast<LoadSDNode>(N0)->
isVolatile()) ||
7296 CombineTo(N, ExtLoad);
7297 CombineTo(N0.
getNode(), ExtLoad, ExtLoad.getValue(1));
7298 AddToWorklist(ExtLoad.getNode());
7304 EVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
7305 ((!LegalOperations && !cast<LoadSDNode>(N0)->
isVolatile()) ||
7312 CombineTo(N, ExtLoad);
7313 CombineTo(N0.
getNode(), ExtLoad, ExtLoad.getValue(1));
7328 SDValue DAGCombiner::visitSIGN_EXTEND_VECTOR_INREG(
SDNode *N) {
7342 SDValue DAGCombiner::visitZERO_EXTEND_VECTOR_INREG(
SDNode *N) {
7413 if (isa<ConstantSDNode>(EltNo) &&
isTypeLegal(NVT)) {
7414 int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
7416 int Index = isLE ? (Elt*SizeRatio) : (Elt*SizeRatio + (SizeRatio-1));
7428 if ((!LegalOperations || TLI.isOperationLegal(
ISD::SELECT, SrcVT)) &&
7429 TLI.isTruncateFree(SrcVT, VT)) {
7440 (!LegalOperations || TLI.isOperationLegalOrCustom(
ISD::SHL, VT)) &&
7441 TLI.isTypeDesirableForOp(
ISD::SHL, VT)) {
7443 uint64_t Amt = CAmt->getZExtValue();
7471 if (BuildVectEltTy == TruncVecEltTy) {
7475 unsigned TruncEltOffset = BuildVecNumElts / TruncVecNumElts;
7477 assert((BuildVecNumElts % TruncVecNumElts) == 0 &&
7478 "Invalid number of elements");
7481 for (
unsigned i = 0, e = BuildVecNumElts;
i != e;
i += TruncEltOffset)
7501 if (!LegalTypes || TLI.isTypeDesirableForOp(N0.
getOpcode(), VT)) {
7502 if (
SDValue Reduced = ReduceLoadWidth(N))
7526 unsigned NumDefs = 0;
7547 assert(V.
getNode() &&
"The single defined operand is empty!");
7549 for (
unsigned i = 0, e = VTs.
size();
i != e; ++
i) {
7556 Opnds.push_back(NV);
7570 (!LegalOperations ||
7597 SDValue DAGCombiner::CombineConsecutiveLoads(
SDNode *N, EVT VT) {
7603 LD1->getAddressSpace() != LD2->getAddressSpace())
7605 EVT LD1VT = LD1->getValueType(0);
7609 unsigned Align = LD1->getAlignment();
7613 if (NewAlign <= Align &&
7614 (!LegalOperations || TLI.isOperationLegal(
ISD::LOAD, VT)))
7615 return DAG.
getLoad(VT,
SDLoc(N), LD1->getChain(), LD1->getBasePtr(),
7616 LD1->getPointerInfo(), Align);
7682 bool isSimple = cast<BuildVectorSDNode>(N0)->isConstant();
7686 "Element type of vector ValueType must not be vector!");
7688 return ConstantFoldBITCASTofBUILD_VECTOR(N0.
getNode(), DestEltVT);
7692 if (isa<ConstantSDNode>(N0) || isa<ConstantFPSDNode>(N0)) {
7696 if (!LegalOperations ||
7716 (!LegalOperations || TLI.isOperationLegal(
ISD::LOAD, VT)) &&
7754 AddToWorklist(NewConv.
getNode());
7764 AddToWorklist(FlipBit.
getNode());
7771 AddToWorklist(Hi.getNode());
7773 AddToWorklist(FlipBit.getNode());
7777 AddToWorklist(FlipBits.
getNode());
7811 if (OrigXWidth < VTWidth) {
7814 }
else if (OrigXWidth > VTWidth) {
7824 AddToWorklist(X.getNode());
7830 AddToWorklist(Cst.getNode());
7832 AddToWorklist(X.getNode());
7834 AddToWorklist(XorResult.getNode());
7839 AddToWorklist(XorResult64.getNode());
7843 AddToWorklist(FlipBit.getNode());
7846 AddToWorklist(FlipBits.getNode());
7852 AddToWorklist(X.getNode());
7857 AddToWorklist(Cst.getNode());
7865 if (
SDValue CombineLD = CombineConsecutiveLoads(N0.
getNode(), VT))
7880 auto PeekThroughBitcast = [&](
SDValue Op) {
7899 for (
int i = 0; i != MaskScale; ++
i)
7900 NewMask.
push_back(M < 0 ? -1 : M * MaskScale + i);
7902 bool LegalMask = TLI.isShuffleMaskLegal(NewMask, VT);
7906 LegalMask = TLI.isShuffleMaskLegal(NewMask, VT);
7918 return CombineConsecutiveLoads(N, VT);
7924 ConstantFoldBITCASTofBUILD_VECTOR(
SDNode *BV, EVT DstEltVT) {
7928 if (SrcEltVT == DstEltVT)
return SDValue(BV, 0);
7935 if (SrcBitSize == DstBitSize) {
7964 BV = ConstantFoldBITCASTofBUILD_VECTOR(BV, IntVT).getNode();
7972 SDNode *Tmp = ConstantFoldBITCASTofBUILD_VECTOR(BV, TmpVT).getNode();
7975 return ConstantFoldBITCASTofBUILD_VECTOR(Tmp, DstEltVT);
7983 if (SrcBitSize < DstBitSize) {
7984 unsigned NumInputsPerOutput = DstBitSize/SrcBitSize;
7987 for (
unsigned i = 0, e = BV->getNumOperands(); i != e;
7988 i += NumInputsPerOutput) {
7991 bool EltIsUndef =
true;
7992 for (
unsigned j = 0; j != NumInputsPerOutput; ++j) {
7994 NewBits <<= SrcBitSize;
7999 NewBits |= cast<ConstantSDNode>(
Op)->getAPIntValue().
8000 zextOrTrunc(SrcBitSize).
zext(DstBitSize);
8015 unsigned NumOutputsPerInput = SrcBitSize/DstBitSize;
8017 NumOutputsPerInput*BV->getNumOperands());
8026 APInt OpVal = cast<ConstantSDNode>(
Op)->
8029 for (
unsigned j = 0; j != NumOutputsPerInput; ++j) {
8031 Ops.push_back(DAG.
getConstant(ThisVal, DL, DstEltVT));
8032 OpVal = OpVal.
lshr(DstBitSize);
8055 bool HasFMAD = (LegalOperations && TLI.isOperationLegal(
ISD::FMAD, VT));
8059 AllowFusion && TLI.isFMAFasterThanFMulAndFAdd(VT) &&
8060 (!LegalOperations || TLI.isOperationLegalOrCustom(
ISD::FMA, VT));
8063 if (!HasFMAD && !HasFMA)
8073 bool Aggressive = TLI.enableAggressiveFMAFusion(VT);
8074 bool LookThroughFPExt = TLI.isFPExtFree(VT);
8087 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
8095 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
8100 if (AllowFusion && LookThroughFPExt) {
8105 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
8117 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
8131 N0.
getOpcode() == PreferredFusedOpcode &&
8134 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
8136 DAG.
getNode(PreferredFusedOpcode, SL, VT,
8146 N1->
getOpcode() == PreferredFusedOpcode &&
8149 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
8151 DAG.
getNode(PreferredFusedOpcode, SL, VT,
8157 if (AllowFusion && LookThroughFPExt) {
8160 auto FoldFAddFMAFPExtFMul = [&] (
8162 return DAG.
getNode(PreferredFusedOpcode, SL, VT, X, Y,
8163 DAG.
getNode(PreferredFusedOpcode, SL, VT,
8168 if (N0.
getOpcode() == PreferredFusedOpcode) {
8184 auto FoldFAddFPExtFMAFMul = [&] (
8186 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
8189 DAG.
getNode(PreferredFusedOpcode, SL, VT,
8196 if (N00.
getOpcode() == PreferredFusedOpcode) {
8207 if (N1.
getOpcode() == PreferredFusedOpcode) {
8225 if (N10.
getOpcode() == PreferredFusedOpcode) {
8251 bool HasFMAD = (LegalOperations && TLI.isOperationLegal(
ISD::FMAD, VT));
8255 AllowFusion && TLI.isFMAFasterThanFMulAndFAdd(VT) &&
8256 (!LegalOperations || TLI.isOperationLegalOrCustom(
ISD::FMA, VT));
8259 if (!HasFMAD && !HasFMA)
8268 bool Aggressive = TLI.enableAggressiveFMAFusion(VT);
8269 bool LookThroughFPExt = TLI.isFPExtFree(VT);
8274 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
8283 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
8294 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
8300 if (AllowFusion && LookThroughFPExt) {
8306 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
8320 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
8341 DAG.
getNode(PreferredFusedOpcode, SL, VT,
8363 DAG.
getNode(PreferredFusedOpcode, SL, VT,
8382 N0.
getOpcode() == PreferredFusedOpcode &&
8385 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
8387 DAG.
getNode(PreferredFusedOpcode, SL, VT,
8399 N1.
getOpcode() == PreferredFusedOpcode &&
8403 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
8407 DAG.
getNode(PreferredFusedOpcode, SL, VT,
8413 if (AllowFusion && LookThroughFPExt) {
8416 if (N0.
getOpcode() == PreferredFusedOpcode) {
8421 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
8423 DAG.
getNode(PreferredFusedOpcode, SL, VT,
8441 if (N00.
getOpcode() == PreferredFusedOpcode) {
8444 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
8449 DAG.
getNode(PreferredFusedOpcode, SL, VT,
8461 if (N1.
getOpcode() == PreferredFusedOpcode &&
8467 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
8470 DAG.
getNode(PreferredFusedOpcode, SL, VT,
8494 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
8499 DAG.
getNode(PreferredFusedOpcode, SL, VT,
8517 SDValue DAGCombiner::visitFMULForFMADistributiveCombine(
SDNode *N) {
8529 if (!Options.NoInfsFPMath)
8535 TLI.isFMAFasterThanFMulAndFAdd(VT) &&
8536 (!LegalOperations || TLI.isOperationLegalOrCustom(
ISD::FMA, VT));
8540 bool HasFMAD = Options.UnsafeFPMath &&
8541 (LegalOperations && TLI.isOperationLegal(
ISD::FMAD, VT));
8544 if (!HasFMAD && !HasFMA)
8549 bool Aggressive = TLI.enableAggressiveFMAFusion(VT);
8556 if (XC1 && XC1->isExactlyValue(+1.0))
8558 if (XC1 && XC1->isExactlyValue(-1.0))
8577 if (XC0 && XC0->isExactlyValue(+1.0))
8578 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
8581 if (XC0 && XC0->isExactlyValue(-1.0))
8582 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
8587 if (XC1 && XC1->isExactlyValue(+1.0))
8590 if (XC1 && XC1->isExactlyValue(-1.0))
8612 const SDNodeFlags *Flags = &cast<BinaryWithFlagsSDNode>(
N)->Flags;
8616 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
8624 if (N0CFP && !N1CFP)
8628 if ((!LegalOperations || TLI.isOperationLegalOrCustom(
ISD::FSUB, VT)) &&
8634 if ((!LegalOperations || TLI.isOperationLegalOrCustom(
ISD::FSUB, VT)) &&
8672 if (TLI.isOperationLegalOrCustom(
ISD::FMUL, VT) && !N0CFP && !N1CFP) {
8678 if (CFP01 && !CFP00 && N0.
getOperand(0) == N1) {
8699 if (CFP11 && !CFP10 && N1.
getOperand(0) == N0) {
8736 if (AllowNewConst &&
8748 if (
SDValue Fused = visitFADDForFMACombine(N)) {
8749 AddToWorklist(Fused.getNode());
8763 const SDNodeFlags *Flags = &cast<BinaryWithFlagsSDNode>(
N)->Flags;
8767 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
8782 if (N0CFP && N0CFP->
isZero()) {
8785 if (!LegalOperations || TLI.isOperationLegal(
ISD::FNEG, VT))
8793 if (N1CFP && N1CFP->
isZero())
8815 if (
SDValue Fused = visitFSUBForFMACombine(N)) {
8816 AddToWorklist(Fused.getNode());
8831 const SDNodeFlags *Flags = &cast<BinaryWithFlagsSDNode>(
N)->Flags;
8836 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
8855 if (N1CFP && N1CFP->
isZero())
8875 (BV1 && BV01 && BV1->isConstant() && BV01->isConstant())) {
8901 if (!LegalOperations || TLI.isOperationLegal(
ISD::FNEG, VT))
8909 if (LHSNeg == 2 || RHSNeg == 2)
8918 if (
SDValue Fused = visitFMULForFMADistributiveCombine(N)) {
8919 AddToWorklist(Fused.getNode());
8937 if (isa<ConstantFPSDNode>(N0) &&
8938 isa<ConstantFPSDNode>(N1) &&
8939 isa<ConstantFPSDNode>(N2)) {
8944 if (N0CFP && N0CFP->
isZero())
8946 if (N1CFP && N1CFP->isZero())
8952 if (N1CFP && N1CFP->isExactlyValue(1.0))
8990 if (N1CFP->isExactlyValue(1.0))
8994 if (N1CFP->isExactlyValue(-1.0) &&
8995 (!LegalOperations || TLI.isOperationLegal(
ISD::FNEG, VT))) {
8997 AddToWorklist(RHSNeg.
getNode());
9005 if (N1CFP && N0 == N2) {
9046 unsigned MinUses = TLI.combineRepeatedFPDivisors();
9047 if (!MinUses || N1->
use_size() < MinUses)
9053 for (
auto *U : N1->
uses()) {
9054 if (U->getOpcode() ==
ISD::FDIV && U->getOperand(1) == N1) {
9057 if (UnsafeMath || U->getFlags()->hasAllowReciprocal())
9064 if (Users.
size() < MinUses)
9073 for (
auto *U : Users) {
9075 if (Dividend != FPOne) {
9078 CombineTo(U, NewNode);
9079 }
else if (U != Reciprocal.
getNode()) {
9082 CombineTo(U, Reciprocal);
9096 SDNodeFlags *Flags = &cast<BinaryWithFlagsSDNode>(
N)->Flags;
9100 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
9111 const APFloat &N1APF = N1CFP->getValueAPF();
9117 (!LegalOperations ||
9122 TLI.isFPImmLegal(Recip, VT)))
9173 if (
SDValue RV = BuildReciprocalEstimate(N1, Flags)) {
9184 if (LHSNeg == 2 || RHSNeg == 2)
9193 return CombineRepeatedDivisors;
9208 &cast<BinaryWithFlagsSDNode>(N)->Flags);
9218 if (TLI.isFsqrtCheap(N0, DAG))
9225 return buildSqrtEstimate(N0, &Flags);
9240 return (N1VT == N1Op0VT || N1Op0VT !=
MVT::f128);
9256 const APFloat &V = N1CFP->getValueAPF();
9260 if (!LegalOperations || TLI.isOperationLegal(
ISD::FABS, VT))
9263 if (!LegalOperations || TLI.isOperationLegal(
ISD::FNEG, VT))
9300 (!LegalOperations ||
9314 if (TLI.isOperationLegalOrCustom(
ISD::SELECT_CC, VT) || !LegalOperations) {
9318 (!LegalOperations ||
9332 (!LegalOperations ||
9354 (!LegalOperations ||
9368 if (TLI.isOperationLegalOrCustom(
ISD::SELECT_CC, VT) || !LegalOperations) {
9372 (!LegalOperations ||
9410 unsigned ActualSize =
std::min(InputSize, OutputSize);
9506 EVT EVT = cast<VTSDNode>(N->
getOperand(1))->getVT();
9557 CombineTo(N, ExtLoad);
9617 if (!TLI.isFNegFree(VT) &&
9649 (TLI.isFPImmLegal(CVal, VT) ||
9654 &cast<BinaryWithFlagsSDNode>(N0)->Flags);
9668 if (N0CFP && N1CFP) {
9689 if (N0CFP && N1CFP) {
9722 if (!TLI.isFAbsFree(VT) &&
9806 const APInt &AndConst = cast<ConstantSDNode>(AndOp1)->getAPIntValue();
9809 cast<ConstantSDNode>(Op1)->getAPIntValue()==AndConst.
logBase2()) {
9821 CombineTo(N, NewBRCond,
false);
9824 deleteAndRecombine(Trunc);
9826 WorklistRemover DeadNodes(*
this);
9828 deleteAndRecombine(N1.
getNode());
9847 if (
SDValue Tmp = visitXOR(TheXor)) {
9848 if (Tmp.
getNode() != TheXor) {
9851 dbgs() <<
"\nWith: ";
9854 WorklistRemover DeadNodes(*
this);
9856 deleteAndRecombine(TheXor);
9883 WorklistRemover DeadNodes(*
this);
9885 deleteAndRecombine(N1.
getNode());
9908 CondLHS, CondRHS, CC->
get(),
SDLoc(N),
9930 if (
LoadSDNode *LD = dyn_cast<LoadSDNode>(Use)) {
9936 if (
ST->isIndexed() ||
ST->getBasePtr().getNode() !=
N)
9938 VT =
ST->getMemoryVT();
9939 AS =
ST->getAddressSpace();
9972 bool DAGCombiner::CombineToPreIndexedLoadStore(
SDNode *N) {
9979 if (
LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
9988 if (
ST->isIndexed())
9990 VT =
ST->getMemoryVT();
9994 Ptr =
ST->getBasePtr();
10010 if (!TLI.getPreIndexedAddressParts(N, BasePtr, Offset, AM, DAG))
10016 bool Swapped =
false;
10017 if (isa<ConstantSDNode>(BasePtr)) {
10036 if (isa<FrameIndexSDNode>(BasePtr) || isa<RegisterSDNode>(BasePtr))
10041 SDValue Val = cast<StoreSDNode>(
N)->getValue();
10055 if (isa<ConstantSDNode>(Offset))
10059 SDUse &Use = UI.getUse();
10075 if (!isa<ConstantSDNode>(Op1)) {
10093 bool RealUse =
false;
10113 BasePtr, Offset, AM);
10116 BasePtr, Offset, AM);
10121 dbgs() <<
"\nWith: ";
10124 WorklistRemover DeadNodes(*
this);
10133 deleteAndRecombine(N);
10139 for (
unsigned i = 0, e = OtherUses.
size(); i != e; ++
i) {
10140 unsigned OffsetIdx = 1;
10141 if (OtherUses[i]->getOperand(OffsetIdx).getNode() == BasePtr.
getNode())
10143 assert(OtherUses[i]->getOperand(!OffsetIdx).getNode() ==
10144 BasePtr.
getNode() &&
"Expected BasePtr operand");
10158 cast<ConstantSDNode>(OtherUses[
i]->getOperand(OffsetIdx));
10159 int X0, X1, Y0, Y1;
10160 const APInt &Offset0 = CN->getAPIntValue();
10161 APInt Offset1 = cast<ConstantSDNode>(
Offset)->getAPIntValue();
10163 X0 = (OtherUses[
i]->getOpcode() ==
ISD::SUB && OffsetIdx == 1) ? -1 : 1;
10164 Y0 = (OtherUses[
i]->getOpcode() ==
ISD::SUB && OffsetIdx == 0) ? -1 : 1;
10170 APInt CNV = Offset0;
10171 if (X0 < 0) CNV = -CNV;
10172 if (X1 * Y0 * Y1 < 0) CNV = CNV + Offset1;
10173 else CNV = CNV - Offset1;
10175 SDLoc DL(OtherUses[i]);
10185 deleteAndRecombine(OtherUses[i]);
10190 deleteAndRecombine(Ptr.
getNode());
10199 bool DAGCombiner::CombineToPostIndexedLoadStore(
SDNode *N) {
10203 bool isLoad =
true;
10206 if (
LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
10215 if (
ST->isIndexed())
10217 VT =
ST->getMemoryVT();
10221 Ptr =
ST->getBasePtr();
10238 if (TLI.getPostIndexedAddressParts(N, Op, BasePtr, Offset, AM, DAG)) {
10250 if (isa<FrameIndexSDNode>(BasePtr) || isa<RegisterSDNode>(BasePtr))
10254 bool TryNext =
false;
10262 bool RealUse =
false;
10263 for (
SDNode *UseUse : Use->uses()) {
10282 BasePtr, Offset, AM)
10284 BasePtr, Offset, AM);
10285 ++PostIndexedNodes;
10289 dbgs() <<
"\nWith: ";
10292 WorklistRemover DeadNodes(*
this);
10301 deleteAndRecombine(N);
10306 deleteAndRecombine(Op);
10326 !cast<ConstantSDNode>(Inc)->isOpaque()) &&
10327 "Cannot split out indexing using opaque target constants");
10359 dbgs() <<
"\nWith chain: ";
10360 Chain.getNode()->dump(&DAG);
10362 WorklistRemover DeadNodes(*
this);
10366 deleteAndRecombine(N);
10379 cast<ConstantSDNode>(LD->
getOperand(2))->isOpaque();
10386 Index = SplitIndexingFromLoad(LD);
10389 AddUsersToWorklist(N);
10394 dbgs() <<
"\nWith: ";
10396 dbgs() <<
" and 2 other values\n");
10397 WorklistRemover DeadNodes(*
this);
10401 deleteAndRecombine(N);
10417 return CombineTo(N, Chain.
getOperand(1), Chain);
10430 return CombineTo(N, NewLoad,
SDValue(NewLoad.
getNode(), 1),
true);
10435 bool UseAA = CombinerAA.getNumOccurrences() > 0 ? CombinerAA
10438 if (CombinerAAOnlyFunc.getNumOccurrences() &&
10444 SDValue BetterChain = FindBetterChain(N, Chain);
10447 if (Chain != BetterChain) {
10466 AddToWorklist(Token.
getNode());
10475 if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
10480 if (SliceUpLoad(N))
10499 struct LoadedSlice {
10506 unsigned Truncates;
10507 unsigned CrossRegisterBanksCopies;
10511 Cost(
bool ForCodeSize =
false)
10512 : ForCodeSize(ForCodeSize), Loads(0), Truncates(0),
10513 CrossRegisterBanksCopies(0), ZExts(0), Shift(0) {}
10516 Cost(
const LoadedSlice &
LS,
bool ForCodeSize =
false)
10517 : ForCodeSize(ForCodeSize), Loads(1), Truncates(0),
10518 CrossRegisterBanksCopies(0), ZExts(0), Shift(0) {
10519 EVT TruncType = LS.Inst->getValueType(0);
10520 EVT LoadedType = LS.getLoadedType();
10521 if (TruncType != LoadedType &&
10522 !LS.DAG->getTargetLoweringInfo().isZExtFree(LoadedType, TruncType))
10530 void addSliceGain(
const LoadedSlice &LS) {
10534 LS.Inst->getValueType(0)))
10540 if (LS.canMergeExpensiveCrossRegisterBankCopy())
10541 ++CrossRegisterBanksCopies;
10545 Loads += RHS.Loads;
10546 Truncates += RHS.Truncates;
10547 CrossRegisterBanksCopies += RHS.CrossRegisterBanksCopies;
10548 ZExts += RHS.ZExts;
10549 Shift += RHS.Shift;
10554 return Loads == RHS.Loads && Truncates == RHS.Truncates &&
10555 CrossRegisterBanksCopies == RHS.CrossRegisterBanksCopies &&
10556 ZExts == RHS.ZExts && Shift == RHS.Shift;
10559 bool operator!=(
const Cost &RHS)
const {
return !(*
this == RHS); }
10561 bool operator<(
const Cost &RHS)
const {
10564 unsigned ExpensiveOpsLHS = Loads + CrossRegisterBanksCopies;
10565 unsigned ExpensiveOpsRHS = RHS.Loads + RHS.CrossRegisterBanksCopies;
10568 if (!ForCodeSize && ExpensiveOpsLHS != ExpensiveOpsRHS)
10569 return ExpensiveOpsLHS < ExpensiveOpsRHS;
10570 return (Truncates + ZExts + Shift + ExpensiveOpsLHS) <
10571 (RHS.Truncates + RHS.ZExts + RHS.Shift + ExpensiveOpsRHS);
10574 bool operator>(
const Cost &RHS)
const {
return RHS < *
this; }
10576 bool operator<=(
const Cost &RHS)
const {
return !(RHS < *
this); }
10578 bool operator>=(
const Cost &RHS)
const {
return !(*
this < RHS); }
10593 : Inst(Inst), Origin(Origin), Shift(Shift), DAG(DAG) {}
10598 APInt getUsedBits()
const {
10603 assert(Origin &&
"No original load to compare against.");
10604 unsigned BitWidth = Origin->getValueSizeInBits(0);
10605 assert(Inst &&
"This slice is not bound to an instruction");
10606 assert(Inst->getValueSizeInBits(0) <= BitWidth &&
10607 "Extracted slice is bigger than the whole type!");
10608 APInt UsedBits(Inst->getValueSizeInBits(0), 0);
10610 UsedBits = UsedBits.zext(BitWidth);
10611 UsedBits <<= Shift;
10616 unsigned getLoadedSize()
const {
10617 unsigned SliceSize = getUsedBits().countPopulation();
10618 assert(!(SliceSize & 0x7) &&
"Size is not a multiple of a byte.");
10619 return SliceSize / 8;
10624 EVT getLoadedType()
const {
10625 assert(DAG &&
"Missing context");
10632 unsigned Alignment = Origin->getAlignment();
10633 unsigned Offset = getOffsetFromBase();
10635 Alignment =
MinAlign(Alignment, Alignment + Offset);
10640 bool isLegal()
const {
10642 if (!Origin || !Inst || !DAG)
10646 if (!Origin->getOffset().isUndef())
10652 EVT SliceType = getLoadedType();
10662 EVT PtrType = Origin->getBasePtr().getValueType();
10675 EVT TruncateType = Inst->getValueType(0);
10676 if (TruncateType != SliceType &&
10686 uint64_t getOffsetFromBase()
const {
10687 assert(DAG &&
"Missing context.");
10689 assert(!(Shift & 0x7) &&
"Shifts not aligned on Bytes are not supported.");
10690 uint64_t Offset = Shift / 8;
10692 assert(!(Origin->getValueSizeInBits(0) & 0x7) &&
10693 "The size of the original loaded type is not a multiple of a"
10697 assert(TySizeInBytes > Offset &&
10698 "Invalid shift amount for given loaded size");
10700 Offset = TySizeInBytes - Offset - getLoadedSize();
10711 assert(Inst && Origin &&
"Unable to replace a non-existing slice.");
10712 const SDValue &OldBaseAddr = Origin->getBasePtr();
10713 SDValue BaseAddr = OldBaseAddr;
10715 int64_t Offset =
static_cast<int64_t
>(getOffsetFromBase());
10716 assert(Offset >= 0 &&
"Offset too big to fit in int64_t!");
10726 EVT SliceType = getLoadedType();
10730 DAG->
getLoad(SliceType,
SDLoc(Origin), Origin->getChain(), BaseAddr,
10731 Origin->getPointerInfo().getWithOffset(Offset),
10735 EVT FinalType = Inst->getValueType(0);
10736 if (SliceType != FinalType)
10746 bool canMergeExpensiveCrossRegisterBankCopy()
const {
10747 if (!Inst || !Inst->hasOneUse())
10749 SDNode *Use = *Inst->use_begin();
10752 assert(DAG &&
"Missing context");
10782 if (Inst->getValueType(0) != getLoadedType())
10810 const LoadedSlice &Second) {
10811 assert(First.Origin == Second.Origin && First.Origin &&
10812 "Unable to match different memory origins.");
10813 APInt UsedBits = First.getUsedBits();
10814 assert((UsedBits & Second.getUsedBits()) == 0 &&
10815 "Slices are not supposed to overlap.");
10816 UsedBits |= Second.getUsedBits();
10825 LoadedSlice::Cost &GlobalLSCost) {
10826 unsigned NumberOfSlices = LoadedSlices.
size();
10828 if (NumberOfSlices < 2)
10833 std::sort(LoadedSlices.
begin(), LoadedSlices.
end(),
10834 [](
const LoadedSlice &LHS,
const LoadedSlice &RHS) {
10835 assert(LHS.Origin == RHS.Origin &&
"Different bases not implemented.");
10836 return LHS.getOffsetFromBase() < RHS.getOffsetFromBase();
10838 const TargetLowering &TLI = LoadedSlices[0].DAG->getTargetLoweringInfo();
10841 const LoadedSlice *First =
nullptr;
10842 const LoadedSlice *Second =
nullptr;
10843 for (
unsigned CurrSlice = 0; CurrSlice < NumberOfSlices; ++CurrSlice,
10847 Second = &LoadedSlices[CurrSlice];
10854 EVT LoadedType = First->getLoadedType();
10857 if (LoadedType != Second->getLoadedType())
10861 unsigned RequiredAlignment = 0;
10862 if (!TLI.hasPairedLoad(LoadedType, RequiredAlignment)) {
10868 if (RequiredAlignment > First->getAlignment())
10875 assert(GlobalLSCost.Loads > 0 &&
"We save more loads than we created!");
10876 --GlobalLSCost.Loads;
10893 const APInt &UsedBits,
bool ForCodeSize) {
10894 unsigned NumberOfSlices = LoadedSlices.
size();
10895 if (StressLoadSlicing)
10896 return NumberOfSlices > 1;
10899 if (NumberOfSlices != 2)
10907 LoadedSlice::Cost OrigCost(ForCodeSize), GlobalSlicingCost(ForCodeSize);
10909 OrigCost.Loads = 1;
10910 for (
unsigned CurrSlice = 0; CurrSlice < NumberOfSlices; ++CurrSlice) {
10911 const LoadedSlice &LS = LoadedSlices[CurrSlice];
10913 LoadedSlice::Cost SliceCost(LS, ForCodeSize);
10914 GlobalSlicingCost += SliceCost;
10918 OrigCost.addSliceGain(LS);
10923 return OrigCost > GlobalSlicingCost;
10932 bool DAGCombiner::SliceUpLoad(
SDNode *N) {
10951 UI != UIEnd; ++UI) {
10953 if (UI.getUse().getResNo() != 0)
10957 unsigned Shift = 0;
10962 Shift = cast<ConstantSDNode>(User->
getOperand(1))->getZExtValue();
10980 LoadedSlice
LS(User, LD, Shift, &DAG);
10981 APInt CurrentUsedBits = LS.getUsedBits();
10984 if ((CurrentUsedBits & UsedBits) != 0)
10987 UsedBits |= CurrentUsedBits;
10994 LoadedSlices.push_back(LS);
11009 LSIt = LoadedSlices.begin(),
11010 LSItEnd = LoadedSlices.end();
11011 LSIt != LSItEnd; ++LSIt) {
11012 SDValue SliceInst = LSIt->loadSlice();
11013 CombineTo(LSIt->Inst, SliceInst,
true);
11017 "It takes more than a zext to get to the loaded slice!!");
11030 static std::pair<unsigned, unsigned>
11032 std::pair<unsigned, unsigned> Result(0, 0);
11053 if (ChainOp.getNode() ==
LD) {
11057 if (!isOk)
return Result;
11069 uint64_t NotMask = ~cast<ConstantSDNode>(V->
getOperand(1))->getSExtValue();
11071 if (NotMaskLZ & 7)
return Result;
11073 if (NotMaskTZ & 7)
return Result;
11074 if (NotMaskLZ == 64)
return Result;
11085 switch (MaskedBytes) {
11089 default:
return Result;
11094 if (NotMaskTZ && NotMaskTZ/8 % MaskedBytes)
return Result;
11096 Result.first = MaskedBytes;
11097 Result.second = NotMaskTZ/8;
11109 unsigned NumBytes = MaskInfo.first;
11110 unsigned ByteShift = MaskInfo.second;
11116 ByteShift*8, (ByteShift+NumBytes)*8);
11123 if (!DC->isTypeLegal(VT))
11140 StOffset = ByteShift;
11149 NewAlign =
MinAlign(NewAlign, StOffset);
11188 std::pair<unsigned, unsigned> MaskedLoad;
11190 if (MaskedLoad.first)
11197 if (MaskedLoad.first)
11219 APInt Imm = cast<ConstantSDNode>(N1)->getAPIntValue();
11230 while (NewBW < BitWidth &&
11237 if (NewBW >= BitWidth)
11243 ShAmt = (((ShAmt + NewBW - 1) / NewBW) * NewBW) - NewBW;
11245 std::min(BitWidth, ShAmt + NewBW));
11246 if ((Imm & Mask) == Imm) {
11250 uint64_t PtrOff = ShAmt / 8;
11254 PtrOff = (BitWidth + 7 - NewBW) / 8 - PtrOff;
11276 AddToWorklist(NewPtr.
getNode());
11277 AddToWorklist(NewLD.
getNode());
11278 AddToWorklist(NewVal.
getNode());
11279 WorklistRemover DeadNodes(*
this);
11301 if (!VT.isFloatingPoint() ||
11320 if (LDAlign < ABIAlign || STAlign < ABIAlign)
11331 AddToWorklist(NewLD.
getNode());
11332 AddToWorklist(NewST.
getNode());
11333 WorklistRemover DeadNodes(*
this);
11355 bool DAGCombiner::isMulAddWithConstProfitable(
SDNode *MulNode,
11367 if (Use == MulNode)
11390 if (OtherOp == MulVar)
11417 SDValue DAGCombiner::getMergedConstantVectorStore(
11423 StoreSDNode *St = cast<StoreSDNode>(Stores[
I].MemNode);
11431 bool DAGCombiner::MergeStoresOfConstantsOrVecElts(
11433 unsigned NumStores,
bool IsConstantSrc,
bool UseVector) {
11440 unsigned LatestNodeUsed = 0;
11442 for (
unsigned i=0; i < NumStores; ++
i) {
11447 if (StoreNodes[i].SequenceNum < StoreNodes[LatestNodeUsed].SequenceNum)
11448 LatestNodeUsed =
i;
11454 LSBaseSDNode *LatestOp = StoreNodes[LatestNodeUsed].MemNode;
11455 SDLoc DL(StoreNodes[0].MemNode);
11460 unsigned Elts = NumStores;
11469 if (IsConstantSrc) {
11470 StoredVal = getMergedConstantVectorStore(DAG, DL, StoreNodes, Chains, Ty);
11473 for (
unsigned i = 0; i < NumStores; ++
i) {
11474 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[
i].MemNode);
11477 if (Val.getValueType() != MemVT)
11489 assert(IsConstantSrc &&
"Merged vector elements should use vector store");
11491 unsigned SizeInBits = NumStores * ElementSizeBytes * 8;
11492 APInt StoreInt(SizeInBits, 0);
11497 for (
unsigned i = 0; i < NumStores; ++
i) {
11498 unsigned Idx = IsLE ? (NumStores - 1 -
i) : i;
11499 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[Idx].MemNode);
11502 SDValue Val = St->getValue();
11503 StoreInt <<= ElementSizeBytes * 8;
11505 StoreInt |=
C->getAPIntValue().zext(SizeInBits);
11507 StoreInt |=
C->getValueAPF().bitcastToAPInt().zext(SizeInBits);
11515 StoredVal = DAG.
getConstant(StoreInt, DL, StoreTy);
11526 bool UseAA = CombinerAA.getNumOccurrences() > 0 ? CombinerAA
11530 for (
unsigned i = 0; i < NumStores; ++
i)
11531 CombineTo(StoreNodes[i].MemNode, NewStore);
11534 CombineTo(LatestOp, NewStore);
11536 for (
unsigned i = 0; i < NumStores; ++
i) {
11537 if (StoreNodes[i].MemNode == LatestOp)
11539 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[
i].MemNode);
11552 deleteAndRecombine(St);
11556 StoreNodes.
erase(StoreNodes.
begin() + NumStores, StoreNodes.
end());
11560 void DAGCombiner::getStoreMergeAndAliasCandidates(
11568 if (!BasePtr.Base.getNode())
11572 if (BasePtr.Base.isUndef())
11583 bool UseAA = CombinerAA.getNumOccurrences() > 0 ? CombinerAA
11593 if (
StoreSDNode *OtherST = dyn_cast<StoreSDNode>(*
I)) {
11594 if (
I.getOperandNo() != 0)
11597 if (OtherST->isVolatile() || OtherST->isIndexed())
11600 if (OtherST->getMemoryVT() != MemVT)
11605 if (Ptr.equalBaseIndex(BasePtr))
11606 StoreNodes.
push_back(MemOpLink(OtherST, Ptr.Offset, Seq++));
11615 if (Index != St && !
SDValue(Index, 0)->hasOneUse())
11622 if (!Ptr.equalBaseIndex(BasePtr))
11645 StoreNodes.
push_back(MemOpLink(Index, Ptr.Offset, Seq++));
11653 if (
StoreSDNode *STn = dyn_cast<StoreSDNode>(NextInChain)) {
11657 }
else if (
LoadSDNode *Ldn = dyn_cast<LoadSDNode>(NextInChain)) {
11658 if (Ldn->isVolatile()) {
11665 NextInChain = Ldn->getChain().getNode();
11680 bool DAGCombiner::checkMergeStoreCandidatesForDependencies(
11685 for (
unsigned i = 0; i < StoreNodes.
size(); ++
i) {
11686 SDNode *n = StoreNodes[
i].MemNode;
11692 for (
unsigned i = 0; i < StoreNodes.
size(); ++
i) {
11699 bool DAGCombiner::MergeConsecutiveStores(
11707 Attribute::NoImplicitFloat);
11719 bool IsLoadSrc = isa<LoadSDNode>(StoredVal);
11720 bool IsConstantSrc = isa<ConstantSDNode>(StoredVal) ||
11721 isa<ConstantFPSDNode>(StoredVal);
11725 if (!IsConstantSrc && !IsLoadSrc && !IsExtractVecSrc)
11731 if (MemVT.
isVector() && IsLoadSrc)
11744 getStoreMergeAndAliasCandidates(St, StoreNodes, AliasLoadNodes);
11747 if (StoreNodes.
size() < 2)
11751 bool UseAA = CombinerAA.getNumOccurrences() > 0 ? CombinerAA
11753 if (UseAA && !checkMergeStoreCandidatesForDependencies(StoreNodes))
11763 std::sort(StoreNodes.
begin(), StoreNodes.
end(),
11764 [](MemOpLink LHS, MemOpLink RHS) {
11765 return LHS.OffsetFromBase < RHS.OffsetFromBase ||
11766 (LHS.OffsetFromBase == RHS.OffsetFromBase &&
11767 LHS.SequenceNum < RHS.SequenceNum);
11772 unsigned LastConsecutiveStore = 0;
11773 int64_t StartAddress = StoreNodes[0].OffsetFromBase;
11774 for (
unsigned i = 0, e = StoreNodes.
size(); i < e; ++
i) {
11779 int64_t CurrAddress = StoreNodes[
i].OffsetFromBase;
11780 if (CurrAddress - StartAddress != (ElementSizeBytes * i))
11787 return isAlias(Ldn, StoreNodes[i].MemNode);
11792 LastConsecutiveStore =
i;
11798 unsigned FirstStoreAlign = FirstInChain->
getAlignment();
11803 if (IsConstantSrc) {
11804 unsigned LastLegalType = 0;
11805 unsigned LastLegalVectorType = 0;
11806 bool NonZero =
false;
11807 for (
unsigned i=0; i<LastConsecutiveStore+1; ++
i) {
11808 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[
i].MemNode);
11812 NonZero |= !
C->isNullValue();
11814 NonZero |= !
C->getConstantFPValue()->isNullValue();
11821 unsigned SizeInBits = (i+1) * ElementSizeBytes * 8;
11826 FirstStoreAlign, &IsFast) && IsFast) {
11827 LastLegalType = i+1;
11831 EVT LegalizedStoredValueTy =
11835 FirstStoreAS, FirstStoreAlign, &IsFast) &&
11837 LastLegalType = i + 1;
11851 FirstStoreAlign, &IsFast) && IsFast)
11852 LastLegalVectorType = i + 1;
11857 if (LastLegalType == 0 && LastLegalVectorType == 0)
11860 bool UseVector = (LastLegalVectorType > LastLegalType) && !NoVectors;
11861 unsigned NumElem = UseVector ? LastLegalVectorType : LastLegalType;
11863 return MergeStoresOfConstantsOrVecElts(StoreNodes, MemVT, NumElem,
11869 if (IsExtractVecSrc) {
11870 unsigned NumStoresToMerge = 0;
11872 for (
unsigned i = 0; i < LastConsecutiveStore + 1; ++
i) {
11873 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[
i].MemNode);
11885 unsigned Elts = i + 1;
11894 FirstStoreAlign, &IsFast) && IsFast)
11895 NumStoresToMerge = i + 1;
11898 return MergeStoresOfConstantsOrVecElts(StoreNodes, MemVT, NumStoresToMerge,
11911 BaseIndexOffset LdBasePtr;
11912 for (
unsigned i=0; i<LastConsecutiveStore+1; ++
i) {
11913 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[
i].MemNode);
11918 if (!Ld->hasNUsesOfValue(1, 0))
11922 if (Ld->isVolatile() || Ld->isIndexed())
11930 if (Ld->getMemoryVT() != MemVT)
11935 if (LdBasePtr.Base.getNode()) {
11937 if (!LdPtr.equalBaseIndex(LdBasePtr))
11945 LoadNodes.
push_back(MemOpLink(Ld, LdPtr.Offset, 0));
11948 if (LoadNodes.
size() < 2)
11953 unsigned RequiredAlignment;
11958 LoadSDNode *FirstLoad = cast<LoadSDNode>(LoadNodes[0].MemNode);
11965 unsigned LastConsecutiveLoad = 0;
11967 unsigned LastLegalVectorType = 0;
11968 unsigned LastLegalIntegerType = 0;
11969 StartAddress = LoadNodes[0].OffsetFromBase;
11971 for (
unsigned i = 1; i < LoadNodes.
size(); ++
i) {
11973 if (LoadNodes[i].MemNode->getChain() != FirstChain)
11976 int64_t CurrAddress = LoadNodes[
i].OffsetFromBase;
11977 if (CurrAddress - StartAddress != (ElementSizeBytes * i))
11979 LastConsecutiveLoad =
i;
11982 bool IsFastSt, IsFastLd;
11985 FirstStoreAlign, &IsFastSt) && IsFastSt &&
11987 FirstLoadAlign, &IsFastLd) && IsFastLd) {
11988 LastLegalVectorType = i + 1;
11992 unsigned SizeInBits = (i+1) * ElementSizeBytes * 8;
11996 FirstStoreAlign, &IsFastSt) && IsFastSt &&
11998 FirstLoadAlign, &IsFastLd) && IsFastLd)
11999 LastLegalIntegerType = i + 1;
12003 EVT LegalizedStoredValueTy =
12010 FirstStoreAS, FirstStoreAlign, &IsFastSt) &&
12013 FirstLoadAS, FirstLoadAlign, &IsFastLd) &&
12015 LastLegalIntegerType = i+1;
12021 bool UseVectorTy = LastLegalVectorType > LastLegalIntegerType && !NoVectors;
12022 unsigned LastLegalType = std::max(LastLegalVectorType, LastLegalIntegerType);
12026 unsigned NumElem =
std::min(LastConsecutiveStore, LastConsecutiveLoad) + 1;
12027 NumElem =
std::min(LastLegalType, NumElem);
12034 MergeStoreChains.
push_back(StoreNodes[0].MemNode->getChain());
12037 unsigned LatestNodeUsed = 0;
12038 for (
unsigned i=1; i<NumElem; ++
i) {
12043 if (StoreNodes[i].SequenceNum < StoreNodes[LatestNodeUsed].SequenceNum)
12044 LatestNodeUsed =
i;
12046 MergeStoreChains.
push_back(StoreNodes[i].MemNode->getChain());
12049 LSBaseSDNode *LatestOp = StoreNodes[LatestNodeUsed].MemNode;
12057 unsigned SizeInBits = NumElem * ElementSizeBytes * 8;
12061 SDLoc LoadDL(LoadNodes[0].MemNode);
12062 SDLoc StoreDL(StoreNodes[0].MemNode);
12078 for (
unsigned i = 0; i < NumElem; ++
i) {
12079 LoadSDNode *Ld = cast<LoadSDNode>(LoadNodes[
i].MemNode);
12086 for (
unsigned i = 0; i < NumElem; ++
i)
12087 CombineTo(StoreNodes[i].MemNode, NewStore);
12090 CombineTo(LatestOp, NewStore);
12092 for (
unsigned i = 0; i < NumElem; ++
i) {
12094 if (StoreNodes[i].MemNode == LatestOp)
12096 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[
i].MemNode);
12098 deleteAndRecombine(St);
12102 StoreNodes.
erase(StoreNodes.
begin() + NumElem, StoreNodes.
end());
12125 AddToWorklist(Token.
getNode());
12128 return CombineTo(ST, Token,
false);
12162 bitcastToAPInt().getZExtValue(),
SDLoc(CFP),
12175 return DAG.
getStore(Chain, DL, Tmp,
12198 Alignment =
MinAlign(Alignment, 4U);
12201 Alignment, MMOFlags, AAInfo);
12221 if (((!LegalOperations && !ST->
isVolatile()) ||
12249 return CombineTo(ST, NewStore,
true);
12256 if (
SDValue NewST = TransformFPLoadStorePair(N))
12259 bool UseAA = CombinerAA.getNumOccurrences() > 0 ? CombinerAA
12262 if (CombinerAAOnlyFunc.getNumOccurrences() &&
12274 if (findBetterNeighborChains(ST)) {
12283 if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
12292 SDValue Shorter = GetDemandedBits(
12295 AddToWorklist(Value.
getNode());
12311 if (
LoadSDNode *Ld = dyn_cast<LoadSDNode>(Value)) {
12316 Chain.reachesChainWithoutSideEffects(
SDValue(Ld, 1))) {
12324 if (
StoreSDNode *ST1 = dyn_cast<StoreSDNode>(Chain)) {
12325 if (ST1->getBasePtr() == Ptr && ST->
getMemoryVT() == ST1->getMemoryVT() &&
12327 ST1->isUnindexed() && !ST1->isVolatile()) {
12351 bool Changed = MergeConsecutiveStores(ST, StoreNodes);
12352 if (!Changed)
break;
12355 [ST](
const MemOpLink &
Link) {
return Link.MemNode ==
ST; })) {
12367 if (isa<ConstantFPSDNode>(Value)) {
12368 if (
SDValue NewSt = replaceStoreOfFPConstant(ST))
12372 if (
SDValue NewSt = splitMergedValStore(ST))
12375 return ReduceLoadOpStoreWidth(N);
12480 Alignment / 2, MMOFlags, AAInfo);
12501 if (!isa<ConstantSDNode>(EltNo))
12503 unsigned Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
12513 && isa<ConstantSDNode>(InVec.
getOperand(2))) {
12514 unsigned OtherElt =
12515 cast<ConstantSDNode>(InVec.
getOperand(2))->getZExtValue();
12516 if (Elt < OtherElt) {
12520 AddToWorklist(NewOp.
getNode());
12535 }
else if (InVec.
isUndef()) {
12543 if (Elt < Ops.
size()) {
12546 EVT OpVT = Ops[0].getValueType();
12558 SDValue DAGCombiner::ReplaceExtractVectorEltOfLoadWithNarrowedLoad(
12578 if (
auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo)) {
12579 int Elt = ConstEltNo->getZExtValue();
12600 if (ResultVT.bitsGT(VecEltVT)) {
12608 OriginalLoad->
getChain(), NewPtr, MPI, VecEltVT,
12617 if (ResultVT.bitsLT(VecEltVT))
12622 WorklistRemover DeadNodes(*
this);
12628 AddToWorklist(Load.
getNode());
12629 AddUsersToWorklist(Load.
getNode());
12631 AddToWorklist(EVE);
12667 if (NVT == InEltVT)
12710 if (OrigElt < NumElem) {
12714 OrigElt -= NumElem;
12730 if (!LegalOperations) {
12737 bool BCNumEltsChanged =
false;
12755 BCNumEltsChanged =
true;
12761 if (!LegalOperations && !ConstEltNo && InVec.
hasOneUse() &&
12765 if (
LoadSDNode *OrigLoad = dyn_cast<LoadSDNode>(InVec)) {
12766 if (!OrigLoad->isVolatile()) {
12767 return ReplaceExtractVectorEltOfLoadWithNarrowedLoad(N, VT, Index,
12775 if (!LegalOperations)
return SDValue();
12782 int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
12787 LN0 = cast<LoadSDNode>(InVec);
12795 LN0 = cast<LoadSDNode>(InVec.
getOperand(0));
12796 }
else if ((SVN = dyn_cast<ShuffleVectorSDNode>(InVec))) {
12807 if (BCNumEltsChanged)
12812 int Idx = (Elt > (int)NumElems) ? -1 : SVN->
getMaskElt(Elt);
12823 LN0 = cast<LoadSDNode>(InVec);
12824 Elt = (Idx < (int)NumElems) ? Idx : Idx - (int)NumElems;
12838 return ReplaceExtractVectorEltOfLoadWithNarrowedLoad(N, VT, EltNo, LN0);
12845 SDValue DAGCombiner::reduceBuildVecExtToExtBuildVec(
SDNode *N) {
12865 bool AllAnyExt =
true;
12867 for (
unsigned i = 0; i != NumInScalars; ++
i) {
12876 if (!ZeroExt && !AnyExt) {
12888 else if (InTy != SourceType) {
12895 AllAnyExt &= AnyExt;
12902 bool ValidTypes = SourceType !=
MVT::Other &&
12913 assert(ElemRatio > 1 &&
"Invalid element size ratio");
12925 Cast.
isUndef()) &&
"Invalid cast opcode");
12931 unsigned Index = isLE ? (i * ElemRatio) :
12932 (i * ElemRatio + (ElemRatio - 1));
12934 assert(Index < Ops.
size() &&
"Invalid index");
12941 "Invalid vector size");
12954 SDValue DAGCombiner::reduceBuildVecConvertToConvertBuildVec(
SDNode *N) {
12962 unsigned NumDefs = 0;
12964 for (
unsigned i = 0; i != NumInScalars; ++
i) {
12997 &&
"Should only handle conversion from integer to float.");
13011 for (
unsigned i = 0; i != NumInScalars; ++
i) {
13022 return DAG.
getNode(Opcode, DL, VT, BV);
13028 unsigned LeftIdx) {
13038 unsigned ShuffleNumElems = NumElems;
13042 if (InVT1 != VT || InVT2 != VT) {
13047 assert(NumConcats >= 2 &&
"Concat needs at least two inputs!");
13049 ConcatOps[0] = VecIn1;
13050 ConcatOps[1] = VecIn2 ? VecIn2 : DAG.
getUNDEF(InVT1);
13065 Vec2Offset = NumElems;
13071 if (LegalOperations &&
13078 if (InVT1 != InVT2) {
13082 DAG.
getUNDEF(InVT1), VecIn2, ZeroIdx);
13084 ShuffleNumElems = NumElems * 2;
13106 for (
unsigned i = 0; i != NumElems; ++
i) {
13107 if (VectorMask[i] <= 0)
13111 if (VectorMask[i] == (
int)LeftIdx) {
13112 Mask[
i] = ExtIndex;
13113 }
else if (VectorMask[i] == (
int)LeftIdx + 1) {
13114 Mask[
i] = Vec2Offset + ExtIndex;
13127 if (ShuffleNumElems > NumElems)
13148 bool UsesZeroVector =
false;
13161 for (
unsigned i = 0; i != NumElems; ++
i) {
13171 UsesZeroVector =
true;
13192 unsigned Idx = std::distance(
13194 if (Idx == VecIn.
size())
13197 VectorMask[
i] = Idx;
13201 if (VecIn.
size() < 2)
13224 for (
unsigned In = 0, Len = (VecIn.
size() / 2); In < Len; ++
In) {
13225 unsigned LeftIdx = 2 * In + 1;
13226 SDValue VecLeft = VecIn[LeftIdx];
13228 (LeftIdx + 1) < VecIn.
size() ? VecIn[LeftIdx + 1] :
SDValue();
13230 if (
SDValue Shuffle = createBuildVecShuffle(DL, N, VectorMask, VecLeft,
13231 VecRight, LeftIdx))
13239 if (UsesZeroVector)
13244 if (Shuffles.
size() == 1)
13245 return Shuffles[0];
13248 for (
int &Vec : VectorMask)
13250 Vec = Shuffles.
size() - 1;
13252 Vec = (Vec - 1) / 2;
13266 if (Shuffles.
size() % 2)
13269 for (
unsigned CurSize = Shuffles.
size(); CurSize > 1; CurSize /= 2) {
13271 Shuffles[CurSize] = DAG.
getUNDEF(VT);
13274 for (
unsigned In = 0, Len = CurSize / 2; In < Len; ++
In) {
13276 int Right = 2 * In + 1;
13278 for (
unsigned i = 0; i != NumElems; ++
i) {
13279 if (VectorMask[i] == Left) {
13281 VectorMask[
i] =
In;
13282 }
else if (VectorMask[i] == Right) {
13283 Mask[
i] = i + NumElems;
13284 VectorMask[
i] =
In;
13293 return Shuffles[0];
13303 if (
SDValue V = reduceBuildVecExtToExtBuildVec(N))
13306 if (
SDValue V = reduceBuildVecConvertToConvertBuildVec(N))
13309 if (
SDValue V = reduceBuildVecToShuffle(N))
13331 bool AnyInteger =
false;
13332 bool AnyFP =
false;
13396 Mask.
append((
unsigned)NumOpElts, -1);
13416 Mask.
append((
unsigned)NumOpElts, -1);
13422 int ExtIdx = cast<ConstantSDNode>(Op.
getOperand(1))->getZExtValue();
13431 if (0 == (NumExtElts % NumElts))
13432 ExtIdx /= (NumExtElts / NumElts);
13433 else if (0 == (NumElts % NumExtElts))
13434 ExtIdx *= (NumElts / NumExtElts);
13439 if (SV0.
isUndef() || SV0 == ExtVec) {
13441 for (
int i = 0; i != NumOpElts; ++
i)
13443 }
else if (SV1.isUndef() || SV1 == ExtVec) {
13445 for (
int i = 0; i != NumOpElts; ++
i)
13508 auto IsBuildVectorOrUndef = [](
const SDValue &
Op) {
13519 bool FoundMinVT =
false;
13523 MinVT = (!FoundMinVT || OpSVT.
bitsLE(MinVT)) ? OpSVT : MinVT;
13526 assert(FoundMinVT &&
"Concat vector type mismatch");
13541 for (
unsigned i = 0; i != NumElts; ++
i)
13549 "Concat vector type mismatch");
13580 if (SingleSource.
getNode()) {
13593 unsigned IdentityIndex = i * PartNumElem;
13605 return SingleSource;
13625 assert((Idx % NumElems) == 0 &&
13626 "IDX in concat is not a multiple of the result vector length.");
13646 if (InsIdx && ExtIdx &&
13680 bool FoundSimplification =
false;
13685 bool OpUsed =
false;
13686 for (
int j = 0; j < OpSize; ++j)
13687 if (UsedElements[i * OpSize + j]) {
13688 OpUsedElements[j] =
true;
13694 FoundSimplification |= Op == NewOps.
back();
13695 OpUsedElements.
reset();
13697 if (FoundSimplification)
13710 int Idx = IdxN->getZExtValue();
13711 bool SubVectorUsed =
false;
13713 for (
int i = 0; i < SubSize; ++
i)
13714 if (UsedElements[i + Idx]) {
13715 SubVectorUsed =
true;
13716 SubUsedElements[
i] =
true;
13717 UsedElements[i + Idx] =
false;
13726 if (SimplifiedSubV != SubV || SimplifiedBaseV != BaseV)
13728 SimplifiedBaseV, SimplifiedSubV, V->
getOperand(2));
13738 SmallBitVector N0UsedElements(NumElts,
false), N1UsedElements(NumElts,
false);
13740 if (M >= 0 && M < NumElts)
13741 N0UsedElements[M] =
true;
13742 else if (M >= NumElts)
13743 N1UsedElements[M - NumElts] =
true;
13747 if (S0 == N0 && S1 == N1)
13766 unsigned NumConcats = NumElts / NumElemsPerConcat;
13771 if (NumElemsPerConcat * 2 == NumElts && N1.
isUndef() &&
13773 SVN->
getMask().end(), [](
int i) {
return i == -1; })) {
13782 for (
unsigned I = 0;
I != NumConcats; ++
I) {
13784 unsigned Begin =
I * NumElemsPerConcat;
13785 bool AllUndef =
true, NoUndef =
true;
13786 for (
unsigned J = Begin; J != Begin + NumElemsPerConcat; ++J) {
13794 if (SVN->
getMaskElt(Begin) % NumElemsPerConcat != 0)
13797 for (
unsigned J = 1; J != NumElemsPerConcat; ++J)
13801 unsigned FirstElt = SVN->
getMaskElt(Begin) / NumElemsPerConcat;
13807 }
else if (AllUndef) {
13856 for (
int M : SVN->
getMask()) {
13859 int Idx = M < (int)NumElts ? M : M - NumElts;
13860 SDValue &S = (M < (int)NumElts ? N0 : N1);
13875 if (!Op.
isUndef() && !isa<ConstantSDNode>(
Op) && !isa<ConstantFPSDNode>(Op))
13876 if (!DuplicateOps.
insert(Op).second)
13913 for (
unsigned i = 0; i != NumElts; ++
i) {
13915 if (Idx >= (
int)NumElts) Idx -= NumElts;
13927 bool Changed =
false;
13929 for (
unsigned i = 0; i != NumElts; ++
i) {
13931 if (Idx >= (
int)NumElts) {
13958 "BUILD_VECTOR has wrong number of operands");
13960 bool AllSame =
true;
13961 for (
unsigned i = 0; i != NumElts; ++
i) {
13968 if (!Base.getNode())
13970 for (
unsigned i = 0; i != NumElts; ++
i) {
14035 for (
int s = 0; s != Scale; ++s)
14036 NewMask.
push_back(M < 0 ? -1 : Scale * M + s);
14046 EVT ScaleVT = SVT.
bitsLT(InnerSVT) ? VT : InnerVT;
14059 ScaleShuffleMask(InnerSVN->
getMask(), InnerScale);
14061 ScaleShuffleMask(SVN->
getMask(), OuterScale);
14065 for (
int M : OuterMask)
14066 NewMask.
push_back(M < 0 ? -1 : InnerMask[M]);
14098 "Shuffle types don't match");
14102 bool HasSameOp0 = N0 == SV0;
14103 bool IsSV1Undef = SV1.
isUndef();
14104 if (HasSameOp0 || IsSV1Undef || N0 == SV1)
14128 "Shuffle types don't match");
14134 for (
unsigned i = 0; i != NumElts; ++
i) {
14143 if (Idx < (
int)NumElts) {
14153 CurrentVec = (Idx < (int) NumElts) ? OtherSV->
getOperand(0)
14168 Idx = Idx % NumElts;
14169 if (!SV0.getNode() || SV0 == CurrentVec) {
14178 if (SV1.getNode() && SV1 != CurrentVec)
14188 bool isUndefMask =
true;
14189 for (
unsigned i = 0; i != NumElts && isUndefMask; ++
i)
14190 isUndefMask &= Mask[i] < 0;
14195 if (!SV0.getNode())
14197 if (!SV1.getNode())
14272 APInt InsIdx = cast<ConstantSDNode>(N2)->getAPIntValue();
14327 if (LegalOperations)
14345 auto BuildClearMask = [&](
int Split) {
14346 int NumSubElts = NumElts *
Split;
14350 for (
int i = 0; i != NumSubElts; ++
i) {
14351 int EltIdx = i /
Split;
14352 int SubIdx = i %
Split;
14360 if (isa<ConstantSDNode>(Elt))
14361 Bits = cast<ConstantSDNode>(Elt)->getAPIntValue();
14362 else if (isa<ConstantFPSDNode>(Elt))
14363 Bits = cast<ConstantFPSDNode>(Elt)->getValueAPF().bitcastToAPInt();
14369 Bits = Bits.
lshr((Split - SubIdx - 1) * NumSubBits);
14371 Bits = Bits.
lshr(SubIdx * NumSubBits);
14375 Bits = Bits.
trunc(NumSubBits);
14379 else if (Bits == 0)
14402 for (
int Split = 1; Split <= MaxSplit; ++
Split)
14404 if (
SDValue S = BuildClearMask(Split))
14413 "SimplifyVBinOp only works on vectors!");
14425 if (
SDValue Shuffle = XformToShuffleWithZero(N))
14431 if (LegalTypes && isa<ShuffleVectorSDNode>(LHS) &&
14432 isa<ShuffleVectorSDNode>(RHS) && LHS.hasOneUse() && RHS.
hasOneUse() &&
14433 LHS.getOperand(1).isUndef() &&
14438 if (SVN0->
getMask().equals(SVN1->getMask())) {
14444 AddUsersToWorklist(N);
14458 cast<CondCodeSDNode>(N0.
getOperand(2))->
get());
14463 if (SCC.getNode()) {
14469 SCC.getOperand(0), SCC.getOperand(1),
14470 SCC.getOperand(4));
14471 AddToWorklist(SETCC.
getNode());
14473 SCC.getOperand(2), SCC.getOperand(3));
14486 bool DAGCombiner::SimplifySelectOps(
SDNode *TheSelect,
SDValue LHS,
14512 if (Zero && Zero->
isZero() &&
14516 CombineTo(TheSelect, Sqrt);
14544 LLD->
isIndexed() || RLD->isIndexed() ||
14558 RLD->getPointerInfo().getAddrSpace() != 0 ||
14570 (RLD->hasAnyUseOfValue(1) && RLD->isPredecessorOf(CondNode)))
14574 RLD->isPredecessorOf(LLD))
14579 RLD->getBasePtr());
14586 (RLD->hasAnyUseOfValue(1) &&
14587 (RLD->isPredecessorOf(CondLHS) || RLD->isPredecessorOf(CondRHS))))
14604 if (!RLD->isInvariant())
14606 if (!RLD->isDereferenceable())
14623 CombineTo(TheSelect, Load);
14670 if (N2C && ((N2C->getAPIntValue() & (N2C->getAPIntValue() - 1)) == 0)) {
14671 unsigned ShCt = XType.
getSizeInBits() - N2C->getAPIntValue().logBase2() - 1;
14674 AddToWorklist(Shift.
getNode());
14676 if (XType.
bitsGT(AType)) {
14678 AddToWorklist(Shift.
getNode());
14682 Shift = DAG.
getNOT(DL, Shift, AType);
14689 AddToWorklist(Shift.
getNode());
14691 if (XType.
bitsGT(AType)) {
14693 AddToWorklist(Shift.
getNode());
14697 Shift = DAG.
getNOT(DL, Shift, AType);
14706 bool NotExtCompare) {
14708 if (N2 == N3)
return N2;
14716 N0, N1, CC, DL,
false);
14717 if (SCC.getNode()) AddToWorklist(SCC.getNode());
14719 if (
ConstantSDNode *SCCC = dyn_cast_or_null<ConstantSDNode>(SCC.getNode())) {
14722 return !SCCC->isNullValue() ? N2 : N3;
14738 N2.getOperand(0) == N3)
14756 !TLI.
isFPImmLegal(TV->getValueAPF(), TV->getValueType(0)) &&
14757 !TLI.
isFPImmLegal(FV->getValueAPF(), FV->getValueType(0))) &&
14760 (TV->hasOneUse() || FV->hasOneUse())) {
14761 Constant *Elts[] = {
14762 const_cast<ConstantFP*
>(FV->getConstantFPValue()),
14763 const_cast<ConstantFP*>(TV->getConstantFPValue())
14773 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->
getAlignment();
14784 AddToWorklist(Cond.
getNode());
14787 AddToWorklist(CstOffset.getNode());
14790 AddToWorklist(CPIdx.getNode());
14798 if (
SDValue V = foldSelectCCToShiftAnd(DL, N0, N1, N2, N3, CC))
14831 if (N2C &&
isNullConstant(N3) && N2C->getAPIntValue().isPowerOf2() &&
14837 if (NotExtCompare && N2C->isOne())
14842 if (!LegalOperations ||
14851 N2.getValueType());
14854 N2.getValueType(), SCC);
14858 N2.getValueType(), SCC);
14861 AddToWorklist(SCC.
getNode());
14862 AddToWorklist(Temp.
getNode());
14869 ISD::SHL, DL, N2.getValueType(), Temp,
14886 SubC = dyn_cast<ConstantSDNode>(N3.
getOperand(0));
14890 SubC = dyn_cast<ConstantSDNode>(N2.getOperand(0));
14892 EVT XType = N0.getValueType();
14901 AddToWorklist(Shift.
getNode());
14902 AddToWorklist(Add.
getNode());
14919 if (CC == ISD::SETNE)
14922 if (
auto *ValueOnZeroC = dyn_cast<ConstantSDNode>(ValueOnZero)) {
14948 bool foldBooleans) {
14950 DagCombineInfo(DAG,
Level,
false,
this);
14951 return TLI.
SimplifySetCC(VT, N0, N1, Cond, foldBooleans, DagCombineInfo, DL);
14972 std::vector<SDNode*> Built;
14992 std::vector<SDNode *> Built;
15018 std::vector<SDNode*> Built;
15056 if (Enabled == TLI.ReciprocalEstimate::Disabled)
15063 AddToWorklist(Est.getNode());
15071 for (
int i = 0; i < Iterations; ++
i) {
15073 AddToWorklist(NewEst.
getNode());
15076 AddToWorklist(NewEst.getNode());
15079 AddToWorklist(NewEst.getNode());
15082 AddToWorklist(Est.getNode());
15098 unsigned Iterations,
15107 AddToWorklist(HalfArg.
getNode());
15110 AddToWorklist(HalfArg.getNode());
15113 for (
unsigned i = 0; i < Iterations; ++
i) {
15115 AddToWorklist(NewEst.
getNode());
15118 AddToWorklist(NewEst.getNode());
15121 AddToWorklist(NewEst.getNode());
15124 AddToWorklist(Est.getNode());
15130 AddToWorklist(Est.getNode());
15142 unsigned Iterations,
15155 for (
unsigned i = 0; i < Iterations; ++
i) {
15160 AddToWorklist(AEE.getNode());
15163 AddToWorklist(RHS.getNode());
15169 if (Reciprocal || (i + 1) < Iterations) {
15176 AddToWorklist(LHS.getNode());
15179 AddToWorklist(Est.getNode());
15201 if (Enabled == TLI.ReciprocalEstimate::Disabled)
15208 bool UseOneConstNR =
false;
15212 AddToWorklist(Est.getNode());
15215 Est = UseOneConstNR
15216 ? buildSqrtNROneConst(Op, Est, Iterations, Flags, Reciprocal)
15217 : buildSqrtNRTwoConst(Op, Est, Iterations, Flags, Reciprocal);
15228 AddToWorklist(ZeroCmp.
getNode());
15231 ZeroCmp, FPZero, Est);
15232 AddToWorklist(Est.getNode());
15242 return buildSqrtEstimateImpl(Op, Flags,
true);
15246 return buildSqrtEstimateImpl(Op, Flags,
false);
15254 Base =
Ptr; Offset = 0; GV =
nullptr; CV =
nullptr;
15268 GV =
G->getGlobal();
15269 Offset +=
G->getOffset();
15277 CV = C->isMachineConstantPoolEntry() ? (
const void *)C->getMachineCPVal()
15278 : (
const void *)C->getConstVal();
15279 Offset += C->getOffset();
15283 return isa<FrameIndexSDNode>(Base);
15305 int64_t Offset1, Offset2;
15307 const void *CV1, *CV2;
15309 Base1, Offset1, GV1, CV1);
15311 Base2, Offset2, GV2, CV2);
15314 if (Base1 == Base2 || (GV1 && (GV1 == GV2)) || (CV1 && (CV1 == CV2)))
15322 if (isFrameIndex1 && isFrameIndex2) {
15324 Offset1 += MFI.
getObjectOffset(cast<FrameIndexSDNode>(Base1)->getIndex());
15325 Offset2 += MFI.
getObjectOffset(cast<FrameIndexSDNode>(Base2)->getIndex());
15332 if ((isFrameIndex1 || CV1 || GV1) && (isFrameIndex2 || CV2 || GV2))
15354 bool UseAA = CombinerGlobalAA.getNumOccurrences() > 0
15358 if (CombinerAAOnlyFunc.getNumOccurrences() &&
15386 void DAGCombiner::GatherAllAliases(
SDNode *N,
SDValue OriginalChain,
15392 bool IsLoad = isa<LoadSDNode>(
N) && !cast<LSBaseSDNode>(N)->isVolatile();
15396 unsigned Depth = 0;
15401 while (!Chains.
empty()) {
15428 bool IsOpLoad = isa<LoadSDNode>(Chain.
getNode()) &&
15432 if (!(IsLoad && IsOpLoad) &&
15433 isAlias(cast<LSBaseSDNode>(N), cast<LSBaseSDNode>(Chain.
getNode()))) {
15471 GatherAllAliases(N, OldChain, Aliases);
15474 if (Aliases.
size() == 0)
15478 if (Aliases.
size() == 1)
15485 bool DAGCombiner::findBetterNeighborChains(
StoreSDNode *St) {
15491 if (!BasePtr.Base.getNode())
15495 if (BasePtr.Base.isUndef())
15507 if (Index != St && !
SDValue(Index, 0)->hasOneUse())
15517 if (!Ptr.equalBaseIndex(BasePtr))
15526 if (
StoreSDNode *STn = dyn_cast<StoreSDNode>(NextInChain)) {
15528 if (STn->isVolatile() || STn->isIndexed()) {
15535 }
else if (
LoadSDNode *Ldn = dyn_cast<LoadSDNode>(NextInChain)) {
15536 NextInChain = Ldn->getChain().getNode();
15545 bool MadeChangeToSt =
false;
15548 for (
StoreSDNode *ChainedStore : ChainedStores) {
15549 SDValue Chain = ChainedStore->getChain();
15550 SDValue BetterChain = FindBetterChain(ChainedStore, Chain);
15552 if (Chain != BetterChain) {
15553 if (ChainedStore == St)
15554 MadeChangeToSt =
true;
15555 BetterChains.
push_back(std::make_pair(ChainedStore, BetterChain));
15561 for (
auto Replacement : BetterChains)
15562 replaceStoreChain(Replacement.first, Replacement.second);
15564 return MadeChangeToSt;
15571 DAGCombiner(*
this, AA, OptLevel).Run(Level);
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
virtual bool hasBitPreservingFPLogic(EVT VT) const
Return true if it is safe to transform an integer-domain bitwise operation into the equivalent floati...
bool hasNoUnsignedWrap() const
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned char TargetFlags=0)
unsigned getValueSizeInBits(unsigned ResNo) const
Returns MVT::getSizeInBits(getValueType(ResNo)).
bool use_empty() const
Return true if there are no uses of this node.
static MVT getIntegerVT(unsigned BitWidth)
std::string & operator+=(std::string &buffer, StringRef string)
void push_back(const T &Elt)
BUILTIN_OP_END - This must be the last enum value in this list.
A parsed version of the target data layout string in and methods for querying it. ...
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
SDValue getValue(unsigned R) const
const SDValue & getValue() const
static SDValue FoldIntToFPToInt(SDNode *N, SelectionDAG &DAG)
static APInt getSignBit(unsigned BitWidth)
Get the SignBit for a specific bit width.
static void commuteMask(MutableArrayRef< int > Mask)
Change values in a shuffle permute mask assuming the two vector operands have swapped position...
bool isTruncatingStore() const
Return true if the op does a truncation before store.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant, which is required to be operand #1) half of the integer or float value specified as operand #0.
bool hasNoSignedZeros() const
static bool areSlicesNextToEachOther(const LoadedSlice &First, const LoadedSlice &Second)
Check whether or not First and Second are next to each other in memory.
Flags getFlags() const
Return the raw flags of the source value,.
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
LLVMContext * getContext() const
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef...
uint64_t getZExtValue() const
Get zero extended value.
DiagnosticInfoOptimizationBase::Argument NV
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
DELETED_NODE - This is an illegal value that is used to catch errors.
STATISTIC(NumFunctions,"Total number of functions")
LLVM_NODISCARD T pop_back_val()
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
bool isKnownNeverNaN(SDValue Op) const
Test whether the given SDValue is known to never be NaN.
BR_CC - Conditional branch.
bool isNON_TRUNCStore(const SDNode *N)
Returns true if the specified node is a non-truncating store.
bool hasOneUse() const
Return true if there is exactly one use of this node.
virtual bool storeOfVectorConstantIsCheap(EVT MemVT, unsigned NumElem, unsigned AddrSpace) const
Return true if it is expected to be cheaper to do a store of a non-zero vector constant with the give...
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDVTList getVTList() const
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
int getSplatIndex() const
Carry-setting nodes for multiple precision addition and subtraction.
const TargetMachine & getTarget() const
static SDValue combineShuffleOfScalars(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const TargetLowering &TLI)
bool isExtended() const
isExtended - Test if the given EVT is extended (as opposed to being simple).
unsigned getPrefTypeAlignment(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
virtual bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const
Return true if it is cheaper to split the store of a merged int val from a pair of smaller values int...
Clients of various APIs that cause global effects on the DAG can optionally implement this interface...
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
bool operator>(int64_t V1, const APSInt &V2)
static SDValue simplifyShuffleOperandRecursively(SmallBitVector &UsedElements, SDValue V, SelectionDAG &DAG)
const TargetSubtargetInfo & getSubtarget() const
unsigned InferPtrAlignment(SDValue Ptr) const
Infer alignment of a load / store address.
void computeKnownBits(SDValue Op, APInt &KnownZero, APInt &KnownOne, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in the KnownZero/KnownO...
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
CondCode getSetCCAndOperation(CondCode Op1, CondCode Op2, bool isInteger)
Return the result of a logical AND between different comparisons of identical values: ((X op1 Y) & (X...
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, unsigned Bytes, int Dist) const
Return true if loads are next to each other and can be merged.
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
const SDValue & getSrc0() const
Type * getTypeForEVT(LLVMContext &Context) const
getTypeForEVT - This method returns an LLVM type corresponding to the specified EVT.
unsigned getSizeInBits() const
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
bool isMask(unsigned numBits, const APInt &APIVal)
unsigned getNumOperands() const
Return the number of values used by this operation.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly...
void setAllBits()
Set every bit to 1.
unsigned getNumOperands() const
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
The two locations do not alias at all.
const SDValue & getOperand(unsigned Num) const
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT TVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize)
static bool isCommutativeBinOp(unsigned Opcode)
Returns true if the opcode is a commutative binary operation.
virtual bool isStoreBitCastBeneficial(EVT StoreVT, EVT BitcastVT) const
Return true if the following transform is beneficial: (store (y (conv x)), y*)) -> (store x...
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
iv Induction Variable Users
static bool isAllOnesConstantOrAllOnesSplatConstant(SDValue N)
void reserve(size_type N)
static bool FindBaseOffset(SDValue Ptr, SDValue &Base, int64_t &Offset, const GlobalValue *&GV, const void *&CV)
Return true if base is a frame index, which is known not to alias with anything but itself...
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
const SDValue & getBasePtr() const
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1 at the ...
SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align=0, int Offs=0, bool isT=false, unsigned char TargetFlags=0)
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
SDValue BuildUDIV(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, bool IsAfterLegalization, std::vector< SDNode * > *Created) const
Given an ISD::UDIV node expressing a divide by constant, return a DAG expression to select that will ...
std::size_t countLeadingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the most significant bit to the least stopping at the first 1...
static SDNode * ShrinkLoadReplaceStoreWithStore(const std::pair< unsigned, unsigned > &MaskInfo, SDValue IVal, StoreSDNode *St, DAGCombiner *DC)
Check to see if IVal is something that provides a value as specified by MaskInfo. ...
virtual bool isFPImmLegal(const APFloat &, EVT) const
Returns true if the target can instruction select the specified FP immediate natively.
size_type size() const
Determine the number of elements in the SetVector.
EntryToken - This is the marker used to indicate the start of a region.
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
unsigned getResNo() const
get the index which selects a specific result in the SDNode
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
bool bitsLT(EVT VT) const
bitsLT - Return true if this has less bits than VT.
bool isUndef() const
Return true if the type of the node type undefined.
int64_t getSrcValueOffset() const
bool optForMinSize() const
Optimize this function for minimum size (-Oz).
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
bool isAllOnesValue() const
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist)
Returns true if N is a predecessor of any node in Worklist.
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool operator<=(int64_t V1, const APSInt &V2)
static bool CanCombineFCOPYSIGN_EXTEND_ROUND(SDNode *N)
copysign(x, fp_extend(y)) -> copysign(x, y) copysign(x, fp_round(y)) -> copysign(x, y)
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
bool isNegative() const
Determine sign of this APInt.
bool isVector() const
isVector - Return true if this is a vector value type.
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
bool match(Val *V, const Pattern &P)
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
static bool isNullConstantOrNullSplatConstant(SDValue N)
A description of a memory reference used in the backend.
bool isRound() const
isRound - Return true if the size is a power-of-two number of bytes.
struct fuzzer::@269 Flags
bool operator>=(int64_t V1, const APSInt &V2)
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Shift and rotation operations.
virtual bool isLegalAddImmediate(int64_t) const
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
virtual SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps) const
Return a reciprocal estimate value for the input operand.
std::size_t countTrailingOnes(T Value, ZeroBehavior ZB=ZB_Width)
Count the number of ones from the least significant bit to the first zero bit.
static cl::opt< bool > Aggressive("aggressive-ext-opt", cl::Hidden, cl::desc("Aggressive extension optimization"))
static unsigned getPPCf128HiElementSelector(const SelectionDAG &DAG)
Base class for LoadSDNode and StoreSDNode.
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef...
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
A Use represents the edge between a Value definition and its users.
MachineFunction & getMachineFunction() const
SDValue getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
bool HonorSignDependentRoundingFPMath() const
HonorSignDependentRoundingFPMath - Return true if the codegen must assume that the rounding mode of t...
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
static GCRegistry::Add< StatepointGC > D("statepoint-example","an example strategy for statepoint")
CopyToReg - This node has three operands: a chain, a register number to set to this value...
unsigned logBase2(const APInt &APIVal)
Returns the floor log base 2 of the specified APInt value.
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
MachinePointerInfo getWithOffset(int64_t O) const
APInt bitcastToAPInt() const
The memory access is dereferenceable (i.e., doesn't trap).
EVT getScalarType() const
getScalarType - If this is a vector type, return the element type, otherwise return this...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
bool bitsGE(EVT VT) const
bitsGE - Return true if this has no less bits than VT.
int getMaskElt(unsigned Idx) const
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
This class is used to represent EVT's, which are used to parameterize some operations.
SDValue FoldConstantVectorArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, const SDNodeFlags *Flags=nullptr)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
LLVM_NODISCARD bool empty() const
bool getBoolValue() const
Convert APInt to a boolean value.
This class is used to represent an MSTORE node.
APInt lshr(const APInt &LHS, unsigned shiftAmt)
Logical right-shift function.
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getVectorElementType() const
getVectorElementType - Given a vector type, return the type of each element.
static bool isSimple(Instruction *I)
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
APInt shl(unsigned shiftAmt) const
Left-shift function.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
static SDValue combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode CC, const TargetLowering &TLI, SelectionDAG &DAG)
Generate Min/Max node.
static unsigned getAlignment(GlobalVariable *GV)
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
bool insert(const value_type &X)
Insert a new element into the SetVector.
SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
APInt zextOrSelf(unsigned width) const
Zero extend or truncate to width.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
static bool isAnyConstantBuildVector(const SDNode *N)
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Simple integer binary arithmetic operators.
Function Alias Analysis false
bool empty() const
Determine if the SetVector is empty or not.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
unsigned getActiveBits() const
Compute the number of active bits in the value.
bool isLittleEndian() const
Layout endianness...
const SDValue & getBasePtr() const
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTruncating=false, bool IsCompressing=false)
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
const APInt & getAPIntValue() const
virtual bool generateFMAsInMachineCombiner(CodeGenOpt::Level OptLevel) const
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification, or lowering of the constant.
virtual bool isDesirableToTransformToIntegerOp(unsigned, EVT) const
Return true if it is profitable for dag combiner to transform a floating point op of specified opcode...
EVT getMemoryVT() const
Return the type of the in-memory value.
const ConstantInt * getConstantIntValue() const
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
size_t size() const
size - Get the array size.
const DataLayout & getDataLayout() const
bool bitsLE(EVT VT) const
bitsLE - Return true if this has no more bits than VT.
bool isPow2VectorType() const
isPow2VectorType - Returns true if the given vector is a power of 2.
Maximum length of the test input libFuzzer tries to guess a good value based on the corpus and reports it always prefer smaller inputs during the corpus shuffle When libFuzzer itself reports a bug this exit code will be used If indicates the maximal total time in seconds to run the fuzzer minimizes the provided crash input Use with etc Experimental Use value profile to guide fuzzing Number of simultaneous worker processes to run the jobs If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
static bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op, APInt &KnownZero)
UNDEF - An undefined node.
This class is used to represent ISD::STORE nodes.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
virtual bool hasAndNot(SDValue X) const
Return true if the target has a bitwise and-not operation: X = ~A & B This can be used to simplify se...
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
virtual const SelectionDAGTargetInfo * getSelectionDAGInfo() const
SDNode * getNode() const
get the SDNode which holds the desired result
bool isTruncStoreLegal(EVT ValVT, EVT MemVT) const
Return true if the specified store with truncation is legal on this target.
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
MinAlign - A and B are either alignments or offsets.
bool hasPredecessor(const SDNode *N) const
Return true if N is a predecessor of this node.
virtual SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, std::vector< SDNode * > *Created) const
Targets may override this function to provide custom SDIV lowering for power-of-2 denominators...
unsigned getScalarSizeInBits() const
unsigned getStoreSize() const
getStoreSize - Return the number of bytes overwritten by a store of the specified value type...
static bool ExtendUsesToFormExtLoad(SDNode *N, SDValue N0, unsigned ExtOpc, SmallVectorImpl< SDNode * > &ExtendNodes, const TargetLowering &TLI)
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
initializer< Ty > init(const Ty &Val)
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
static SDValue foldBitcastedFPLogic(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI)
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
unsigned getStoreSizeInBits() const
getStoreSizeInBits - Return the number of bits overwritten by a store of the specified value type...
FPOpFusion::FPOpFusionMode AllowFPOpFusion
AllowFPOpFusion - This flag is set by the -fuse-fp-ops=xxx option.
virtual bool isShuffleMaskLegal(const SmallVectorImpl< int > &, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations, those with specific masks.
SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV)
Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to the shuffle node in input but with swa...
const SDValue & getBasePtr() const
bool isZero() const
Return true if the value is positive or negative zero.
std::size_t countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1...
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
APInt trunc(unsigned width) const
Truncate to new width.
size_t use_size() const
Return the number of uses of this node.
constexpr bool isPowerOf2_32(uint32_t Value)
isPowerOf2_32 - This function returns true if the argument is a power of two > 0. ...
MVT - Machine Value Type.
static SDValue combineConcatVectorOfExtracts(SDNode *N, SelectionDAG &DAG)
static SDValue ConvertSelectToConcatVector(SDNode *N, SelectionDAG &DAG)
const SDValue & getOperand(unsigned i) const
The instances of the Type class are immutable: once they are created, they are never changed...
constexpr bool isInt(int64_t x)
isInt - Checks if an integer fits into the given bit width.
This is an important class for using LLVM in a threaded context.
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
Simple binary floating point operators.
bool isNonTemporal() const
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
void Combine(CombineLevel Level, AliasAnalysis &AA, CodeGenOpt::Level OptLevel)
This iterates over the nodes in the SelectionDAG, folding certain types of nodes together, or eliminating superfluous nodes.
This is an important base class in LLVM.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
int64_t getSExtValue() const
Get sign extended value.
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Carry-using nodes for multiple precision addition and subtraction.
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
ConstantFP - Floating Point Values [float, double].
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
unsigned getScalarValueSizeInBits() const
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
void AddToWorklist(SDNode *N)
APInt sext(unsigned width) const
Sign extend to a new width.
AliasResult
The possible results of an alias query.
APInt getLoBits(unsigned numBits) const
Compute an APInt containing numBits lowbits from this APInt.
constexpr bool isPowerOf2_64(uint64_t Value)
isPowerOf2_64 - This function returns true if the argument is a power of two 0 (64 bit edition...
This class provides iterator support for SDUse operands that use a specific SDNode.
unsigned getOriginalAlignment() const
Returns alignment and volatility of the memory access.
bool hasAllowReciprocal() const
bool any_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly...
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang","erlang-compatible garbage collector")
static EVT getFloatingPointVT(unsigned BitWidth)
getFloatingPointVT - Returns the EVT that represents a floating point type with the given number of b...
virtual bool aggressivelyPreferBuildVectorSources(EVT VecVT) const
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
unsigned getOpcode() const
const SDValue & getBasePtr() const
virtual bool isNarrowingProfitable(EVT, EVT) const
Return true if it's profitable to narrow operations of type VT1 to VT2.
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y)...
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
unsigned countPopulation() const
Count the number of bits set.
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType, bool IsExpanding=false)
const SDValue & getValue() const
ConstantSDNode * isConstOrConstSplat(SDValue V)
Returns the SDNode if it is a constant splat BuildVector or constant int.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedMask, APInt &KnownZero, APInt &KnownOne, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
Bit counting operators with an undefined result for zero inputs.
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue BuildSDIV(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, bool IsAfterLegalization, std::vector< SDNode * > *Created) const
Given an ISD::SDIV node expressing a divide by constant, return a DAG expression to select that will ...
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Targets can subclass this to parameterize the SelectionDAG lowering and instruction selection process...
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
EVT - Extended Value Type.
uint64_t NextPowerOf2(uint64_t A)
NextPowerOf2 - Returns the next power of two (in 64-bits) that is strictly greater than A...
iterator erase(const_iterator CI)
const SDValue & getMask() const
const APFloat & getValueAPF() const
const ConstantFP * getConstantFPValue() const
bool bitsEq(EVT VT) const
bitsEq - Return true if this has the same number of bits as VT.
static bool areUsedBitsDense(const APInt &UsedBits)
Check that all bits set in UsedBits form a dense region, i.e., UsedBits looks like 0...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
static void adjustCostForPairing(SmallVectorImpl< LoadedSlice > &LoadedSlices, LoadedSlice::Cost &GlobalLSCost)
Adjust the GlobalLSCost according to the target paring capabilities and the layout of the slices...
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc, or post-dec.
bool isLoadExtLegal(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return true if the specified load with extension is legal on this target.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements)
getVectorVT - Returns the EVT that represents a vector NumElements in length, where each element is o...
This class contains a discriminated union of information about pointers in memory operands...
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
HANDLENODE node - Used as a handle for various purposes.
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE maxNum semantics.
static char isNegatibleForFree(SDValue Op, bool LegalOperations, const TargetLowering &TLI, const TargetOptions *Options, unsigned Depth=0)
Return 1 if we can compute the negated form of the specified expression for the same cost as the expr...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
const MachinePointerInfo & getPointerInfo() const
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
const MDNode * getRanges() const
Returns the Ranges that describes the dereference.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
The memory access writes data.
const SDValue & getOffset() const
bool bitsGT(EVT VT) const
bitsGT - Return true if this has more bits than VT.
ArrayRef< int > getMask() const
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type...
unsigned countTrailingZeros() const
Count the number of trailing zero bits.
Representation for a specific memory location.
TokenFactor - This node takes multiple tokens as input and produces a single token result...
unsigned getABITypeAlignment(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef...
static bool canFoldInAddressingMode(SDNode *N, SDNode *Use, SelectionDAG &DAG, const TargetLowering &TLI)
Return true if 'Use' is a load or a store that uses N as its base pointer and that N may be folded in...
int getRecipEstimateDivEnabled(EVT VT, MachineFunction &MF) const
Return a ReciprocalEstimate enum value for a division of the given type based on the function's attri...
A SetVector that performs no allocations if smaller than a certain size.
void setNoUnsignedWrap(bool b)
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
static void zeroExtendToMatch(APInt &LHS, APInt &RHS, unsigned Offset=0)
void dump() const
Dump this node, for debugging.
auto find(R &&Range, const T &Val) -> decltype(std::begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provides VTs and return the low/high part...
virtual bool isVectorClearMaskLegal(const SmallVectorImpl< int > &, EVT) const
Similar to isShuffleMaskLegal.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
X = FP_ROUND_INREG(Y, VT) - This operator takes an FP register, and rounds it to a floating point val...
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
unsigned logBase2() const
bool isPredecessorOf(const SDNode *N) const
Return true if this node is a predecessor of N.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Type * getType() const
All values are typed, get the type of this value.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
bool allOperandsUndef(const SDNode *N)
Return true if the node has at least one operand and all operands of the specified node are ISD::UNDE...
BRCOND - Conditional branch.
const SDValue & getChain() const
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Byte Swap and Counting operators.
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
LLVM_NODISCARD T pop_back_val()
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
bool hasNoSignedWrap() const
Represents one node in the SelectionDAG.
static SDNode * tryToFoldExtendOfConstant(SDNode *N, const TargetLowering &TLI, SelectionDAG &DAG, bool LegalTypes, bool LegalOperations)
Try to fold a sext/zext/aext dag node into a ConstantSDNode or a build_vector of constants.
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
CondCode getSetCCInverse(CondCode Operation, bool isInteger)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
static unsigned int semanticsPrecision(const fltSemantics &)
int getSqrtRefinementSteps(EVT VT, MachineFunction &MF) const
Return the refinement step count for a square root of the given type based on the function's attribut...
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
static bool isOneConstantOrOneSplatConstant(SDValue N)
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
static std::pair< SDValue, SDValue > SplitVSETCC(const SDNode *N, SelectionDAG &DAG)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
unsigned Log2_32(uint32_t Value)
Log2_32 - This function returns the floor log base 2 of the specified value, -1 if the value is zero...
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
const SDValue & getValue() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
SDNode * isConstantIntBuildVectorOrConstantInt(SDValue N)
Test whether the given value is a constant int or similar node.
Class for arbitrary precision integers.
const Value * getValue() const
Return the base address of the memory access.
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
iterator_range< use_iterator > uses()
static SDNode * getBuildPairElt(SDNode *N, unsigned i)
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, SDNode *Cst1, SDNode *Cst2)
A "pseudo-class" with methods for operating on BUILD_VECTORs.
Select(COND, TRUEVAL, FALSEVAL).
int64_t getSExtValue() const
op_iterator op_begin() const
static SDValue simplifyShuffleOperands(ShuffleVectorSDNode *SVN, SDValue N0, SDValue N1, SelectionDAG &DAG)
virtual const TargetRegisterClass * getRegClassFor(MVT VT) const
Return the register class that should be used for the specified value type.
static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOperations, unsigned Depth=0)
If isNegatibleForFree returns true, return the newly negated expression.
static use_iterator use_end()
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
ZERO_EXTEND - Used for integer types, zeroing the new bits.
AddrMode
ARM Addressing Modes.
ANY_EXTEND - Used for integer types. The high bits are undefined.
static std::pair< unsigned, unsigned > CheckForMaskedLoad(SDValue V, SDValue Ptr, SDValue Chain)
Check to see if V is (and load (ptr), imm), where the load is having specific bytes cleared out...
virtual SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps, bool &UseOneConstNR, bool Reciprocal) const
Hooks for building estimates in place of slower divisions and square roots.
CondCode getSetCCOrOperation(CondCode Op1, CondCode Op2, bool isInteger)
Return the result of a logical OR between different comparisons of identical values: ((X op1 Y) | (X ...
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
static SDValue partitionShuffleOfConcats(SDNode *N, SelectionDAG &DAG)
iterator_range< value_op_iterator > op_values() const
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
Flags
Flags values. These may be or'd together.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
const SDValue & getMask() const
The memory access reads data.
bool operator!=(uint64_t V1, const APInt &V2)
bool isAllOnesValue() const
Determine if all bits are set.
ConstantFPSDNode * isConstOrConstSplatFP(SDValue V)
Returns the SDNode if it is a constant splat BuildVector or constant float.
opStatus
IEEE-754R 7: Default exception handling.
These are IR-level optimization flags that may be propagated to SDNodes.
Represents a use of a SDNode.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
uint64_t getConstantOperandVal(unsigned i) const
Bitwise operators - logical and, logical or, logical xor.
static cl::opt< bool > UseTBAA("use-tbaa-in-sched-mi", cl::Hidden, cl::init(true), cl::desc("Enable use of TBAA during MI DAG construction"))
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AddrSpace) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT, SelectionDAG &DAG, bool LegalOperations, bool LegalTypes)
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
void ReplaceAllUsesWith(SDValue From, SDValue Op)
Modify anything using 'From' to use 'To' instead.
ArrayRef< SDUse > ops() const
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
The memory access always returns the same value (or traps).
const TargetRegisterClass * getCommonSubClass(const TargetRegisterClass *A, const TargetRegisterClass *B, const MVT::SimpleValueType SVT=MVT::SimpleValueType::Any) const
Find the largest common subclass of A and B.
op_iterator op_end() const
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
static SDValue combineConcatVectorOfScalars(SDNode *N, SelectionDAG &DAG)
unsigned countTrailingOnes() const
Count the number of trailing one bits.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
bool isExpandingLoad() const
static SDValue getInputChainForNode(SDNode *N)
Given a node, return its input chain if it has one, otherwise return a null sd operand.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
virtual bool isExtractSubvectorCheap(EVT ResVT, unsigned Index) const
Return true if EXTRACT_SUBVECTOR is cheap for this result type with this index.
This class is used to represent an MSCATTER node.
const SDValue & getIndex() const
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
bool isScalarInteger() const
isScalarInteger - Return true if this is an integer, but not a vector.
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
EVT getValueType() const
Return the ValueType of the referenced return value.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a carry value...
void setUnsafeAlgebra(bool b)
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
int getDivRefinementSteps(EVT VT, MachineFunction &MF) const
Return the refinement step count for a division of the given type based on the function's attributes...
SDValue getIndexedStore(SDValue OrigStoe, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
bool isByteSized() const
isByteSized - Return true if the bit size is a multiple of 8.
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
bool isFloatingPoint() const
isFloatingPoint - Return true if this is a FP, or a vector FP type.
This class is used to form a handle around another node that is persistent and is updated across invo...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, unsigned Alignment=1, bool *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
This class is used to represent an MLOAD node.
bool isSimple() const
isSimple - Test if the given EVT is simple (as opposed to being extended).
bool operator<(int64_t V1, const APSInt &V2)
static ConstantSDNode * getAsNonOpaqueConstant(SDValue N)
If N is a ConstantSDNode with isOpaque() == false return it casted to a ConstantSDNode pointer else n...
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
LLVM Value Representation.
FMA - Perform a * b + c with no intermediate rounding step.
static bool isSlicingProfitable(SmallVectorImpl< LoadedSlice > &LoadedSlices, const APInt &UsedBits, bool ForCodeSize)
Check the profitability of all involved LoadedSlice.
const SDValue & getBasePtr() const
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
A vector that has set insertion semantics.
static bool isShift(MachineInstr *MI, unsigned Opcode, int64_t Imm)
static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned, const TargetLowering &TLI)
Return true if divmod libcall is available.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isTruncatingStore() const
Return true if the op does a truncation before store.
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
This class is used to represent an MGATHER node.
SDValue getValueType(EVT)
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits...
const TargetLowering & getTargetLoweringInfo() const
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.Val alone...
unsigned countLeadingZeros() const
The APInt version of the countLeadingZeros functions in MathExtras.h.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations...
APInt zext(unsigned width) const
Zero extend to a new width.
SetCC operator - This evaluates to a true value iff the condition is true.
static SDNode * isConstantFPBuildVectorOrConstantFP(SDValue N)
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml","ocaml 3.10-compatible collector")
bool operator==(uint64_t V1, const APInt &V2)
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
static bool isVolatile(Instruction *Inst)
static APInt getNullValue(unsigned numBits)
Get the '0' value.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDNode * getUser()
This returns the SDNode that contains this Use.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT SrcTy)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value...
bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
TRUNCATE - Completely drop the high bits.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
static void Split(std::vector< std::string > &V, StringRef S)
Split - Splits a string of comma separated items in to a vector of strings.
bool isUIntN(unsigned N, uint64_t x)
isUIntN - Checks if an unsigned integer fits into the given (dynamic) bit width.
unsigned getAlignment() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
virtual unsigned combineRepeatedFPDivisors() const
Indicate whether this target prefers to combine FDIVs with the same divisor.
FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR - Perform various unary floating point operations.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
getIntegerVT - Returns the EVT that represents an integer with the given number of bits...
unsigned getGatherAllAliasesMaxDepth() const
Fast - This calling convention attempts to make calls as fast as possible (e.g.
const fltSemantics & getSemantics() const
virtual bool hasPairedLoad(EVT, unsigned &) const
Return true if the target supplies and combines to a paired load two loaded values of type LoadedType...
unsigned Log2_64(uint64_t Value)
Log2_64 - This function returns the floor log base 2 of the specified value, -1 if the value is zero...
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
EVT changeVectorElementTypeToInteger() const
changeVectorElementTypeToInteger - Return a vector with the same number of elements as this vector...
unsigned getResNo() const
Convenience function for get().getResNo().
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE minNum semantics.
MVT getSimpleVT() const
getSimpleVT - Return the SimpleValueType held in the specified simple EVT.
const SDNodeFlags * getFlags() const
This could be defined as a virtual function and implemented more simply and directly, but it is not to avoid creating a vtable for this class.
CARRY_FALSE - This node is used when folding other nodes, like ADDC/SUBC, which indicate the carry re...
static bool isBSwapHWordElement(SDValue N, MutableArrayRef< SDNode * > Parts)
Return true if the specified node is an element that makes up a 32-bit packed halfword byteswap...
This file describes how to lower LLVM code to machine code.
void CommitTargetLoweringOpt(const TargetLoweringOpt &TLO)
uint64_t getBaseAlignment() const
Return the minimum known alignment in bytes of the base address, without the offset.
int getRecipEstimateSqrtEnabled(EVT VT, MachineFunction &MF) const
Return a ReciprocalEstimate enum value for a square root of the given type based on the function's at...
virtual bool useAA() const
Enable use of alias analysis during code generation (during MI scheduling, DAGCombine, etc.).
ISD::CondCode get() const
uint64_t getZExtValue() const
static bool isConstantOrConstantVector(SDValue N, bool NoOpaques=false)
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.
This class is used to represent ISD::LOAD nodes.
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.