43 #define DEBUG_TYPE "dagcombine"
45 STATISTIC(NodesCombined ,
"Number of dag nodes combined");
46 STATISTIC(PreIndexedNodes ,
"Number of pre-indexed nodes created");
47 STATISTIC(PostIndexedNodes,
"Number of post-indexed nodes created");
48 STATISTIC(OpsNarrowed ,
"Number of load/op/store narrowed");
49 STATISTIC(LdStFP2Int ,
"Number of fp load/store pairs transformed to int");
50 STATISTIC(SlicedLoads,
"Number of load sliced");
54 CombinerAA(
"combiner-alias-analysis",
cl::Hidden,
55 cl::desc(
"Enable DAG combiner alias-analysis heuristics"));
58 CombinerGlobalAA(
"combiner-global-alias-analysis",
cl::Hidden,
59 cl::desc(
"Enable DAG combiner's use of IR alias analysis"));
63 cl::desc(
"Enable DAG combiner's use of TBAA"));
67 CombinerAAOnlyFunc(
"combiner-aa-only-func",
cl::Hidden,
68 cl::desc(
"Only use DAG-combiner alias analysis in this"
75 StressLoadSlicing(
"combiner-stress-load-slicing",
cl::Hidden,
76 cl::desc(
"Bypass the profitability model of load "
82 cl::desc(
"DAG combiner may split indexing from loads"));
122 void AddUsersToWorklist(
SDNode *
N) {
133 void AddToWorklist(
SDNode *N) {
139 if (WorklistMap.insert(std::make_pair(N, Worklist.size())).second)
140 Worklist.push_back(N);
144 void removeFromWorklist(
SDNode *N) {
145 CombinedNodes.erase(N);
147 auto It = WorklistMap.find(N);
148 if (It == WorklistMap.end())
152 Worklist[It->second] =
nullptr;
153 WorklistMap.erase(It);
156 void deleteAndRecombine(
SDNode *N);
157 bool recursivelyDeleteUnusedNodes(
SDNode *N);
163 return CombineTo(N, &Res, 1, AddTo);
169 return CombineTo(N, To, 2, AddTo);
179 bool SimplifyDemandedBits(
SDValue Op) {
182 return SimplifyDemandedBits(Op, Demanded);
185 bool SimplifyDemandedBits(
SDValue Op,
const APInt &Demanded);
187 bool CombineToPreIndexedLoadStore(
SDNode *N);
188 bool CombineToPostIndexedLoadStore(
SDNode *N);
190 bool SliceUpLoad(
SDNode *N);
200 SDValue ReplaceExtractVectorEltOfLoadWithNarrowedLoad(
329 bool NotExtCompare =
false);
331 SDLoc DL,
bool foldBooleans =
true);
335 bool isOneUseSetCC(
SDValue N)
const;
350 bool DemandHighBits =
true);
354 unsigned PosOpcode,
unsigned NegOpcode,
380 MemOpLink (
LSBaseSDNode *N, int64_t Offset,
unsigned Seq):
381 MemNode(N), OffsetFromBase(Offset), SequenceNum(Seq) { }
385 int64_t OffsetFromBase;
388 unsigned SequenceNum;
403 EVT MemVT,
unsigned NumElem,
404 bool IsConstantSrc,
bool UseVector);
409 void getStoreMergeAndAliasCandidates(
429 OptLevel(OL), LegalOperations(
false), LegalTypes(
false), AA(A) {
430 auto *
F = DAG.getMachineFunction().getFunction();
442 EVT getShiftAmountTy(
EVT LHSTy) {
443 assert(LHSTy.
isInteger() &&
"Shift amount is not an integer type!");
446 auto &
DL = DAG.getDataLayout();
447 return LegalTypes ? TLI.getScalarShiftAmountTy(
DL, LHSTy)
448 : TLI.getPointerTy(
DL);
453 bool isTypeLegal(
const EVT &VT) {
454 if (!LegalTypes)
return true;
455 return TLI.isTypeLegal(VT);
459 EVT getSetCCResultType(
EVT VT)
const {
460 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
472 explicit WorklistRemover(DAGCombiner &dc)
473 :
SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
476 DC.removeFromWorklist(N);
486 ((DAGCombiner*)
DC)->AddToWorklist(N);
490 ((DAGCombiner*)DC)->removeFromWorklist(N);
495 return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.
size(), AddTo);
500 return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
506 return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
511 return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
518 void DAGCombiner::deleteAndRecombine(
SDNode *N) {
519 removeFromWorklist(N);
539 unsigned Depth = 0) {
547 if (Depth > 6)
return 0;
550 default:
return false;
554 return LegalOperations ? 0 : 1;
560 if (LegalOperations &&
573 if (!Options->UnsafeFPMath)
return 0;
580 if (Options->HonorSignDependentRoundingFPMath())
return 0;
600 bool LegalOperations,
unsigned Depth = 0) {
606 assert(Op.
hasOneUse() &&
"Unknown reuse!");
608 assert(Depth <= 6 &&
"GetNegatedExpression doesn't match isNegatibleForFree");
612 APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
625 LegalOperations, Depth+1),
630 LegalOperations, Depth+1),
654 LegalOperations, Depth+1),
661 LegalOperations, Depth+1));
667 LegalOperations, Depth+1));
671 LegalOperations, Depth+1),
708 bool DAGCombiner::isOneUseSetCC(
SDValue N)
const {
723 unsigned SplatBitSize;
734 if (isa<ConstantSDNode>(N))
744 if (isa<ConstantFPSDNode>(N))
764 if (CN && UndefElements.
none() &&
782 if (CN && UndefElements.
none())
796 if (
SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, L, R))
797 return DAG.getNode(Opc, DL, VT, N0.
getOperand(0), OpNode);
806 AddToWorklist(OpNode.
getNode());
816 if (
SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, R, L))
817 return DAG.getNode(Opc, DL, VT, N1.
getOperand(0), OpNode);
826 AddToWorklist(OpNode.
getNode());
837 assert(N->
getNumValues() == NumTo &&
"Broken CombineTo call!");
841 dbgs() <<
"\nWith: ";
843 dbgs() <<
" and " << NumTo-1 <<
" other values\n");
844 for (
unsigned i = 0, e = NumTo; i != e; ++i)
845 assert((!To[i].getNode() ||
847 "Cannot combine value to value of different type!");
849 WorklistRemover DeadNodes(*
this);
850 DAG.ReplaceAllUsesWith(N, To);
853 for (
unsigned i = 0, e = NumTo; i != e; ++i) {
854 if (To[i].getNode()) {
855 AddToWorklist(To[i].getNode());
856 AddUsersToWorklist(To[i].getNode());
865 deleteAndRecombine(N);
873 WorklistRemover DeadNodes(*
this);
874 DAG.ReplaceAllUsesOfValueWith(TLO.
Old, TLO.
New);
889 bool DAGCombiner::SimplifyDemandedBits(
SDValue Op,
const APInt &Demanded) {
891 APInt KnownZero, KnownOne;
892 if (!TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne, TLO))
902 dbgs() <<
"\nWith: ";
906 CommitTargetLoweringOpt(TLO);
910 void DAGCombiner::ReplaceLoadWithPromotedLoad(
SDNode *
Load,
SDNode *ExtLoad) {
917 dbgs() <<
"\nWith: ";
920 WorklistRemover DeadNodes(*
this);
921 DAG.ReplaceAllUsesOfValueWith(
SDValue(Load, 0), Trunc);
922 DAG.ReplaceAllUsesOfValueWith(
SDValue(Load, 1),
SDValue(ExtLoad, 1));
923 deleteAndRecombine(Load);
924 AddToWorklist(Trunc.
getNode());
931 EVT MemVT =
LD->getMemoryVT();
935 :
LD->getExtensionType();
937 return DAG.getExtLoad(ExtType, dl, PVT,
938 LD->getChain(),
LD->getBasePtr(),
939 MemVT,
LD->getMemOperand());
956 return DAG.getNode(ExtOpc, dl, PVT, Op);
970 bool Replace =
false;
971 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
974 AddToWorklist(NewOp.
getNode());
985 bool Replace =
false;
986 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
989 AddToWorklist(NewOp.
getNode());
993 return DAG.getZeroExtendInReg(NewOp, dl, OldVT);
1000 if (!LegalOperations)
1010 if (TLI.isTypeDesirableForOp(Opc, VT))
1016 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1017 assert(PVT != VT &&
"Don't know what type to promote to!");
1019 bool Replace0 =
false;
1021 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
1025 bool Replace1 =
false;
1031 NN1 = PromoteOperand(N1, PVT, Replace1);
1049 DAG.getNode(Opc, dl, PVT, NN0, NN1));
1058 if (!LegalOperations)
1068 if (TLI.isTypeDesirableForOp(Opc, VT))
1074 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1075 assert(PVT != VT &&
"Don't know what type to promote to!");
1077 bool Replace =
false;
1080 N0 = SExtPromoteOperand(Op.
getOperand(0), PVT);
1082 N0 = ZExtPromoteOperand(Op.
getOperand(0), PVT);
1084 N0 = PromoteOperand(N0, PVT, Replace);
1088 AddToWorklist(N0.getNode());
1096 DAG.getNode(Opc, dl, PVT, N0, Op.
getOperand(1)));
1102 if (!LegalOperations)
1112 if (TLI.isTypeDesirableForOp(Opc, VT))
1118 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1119 assert(PVT != VT &&
"Don't know what type to promote to!");
1130 bool DAGCombiner::PromoteLoad(
SDValue Op) {
1131 if (!LegalOperations)
1141 if (TLI.isTypeDesirableForOp(Opc, VT))
1147 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1148 assert(PVT != VT &&
"Don't know what type to promote to!");
1158 SDValue NewLD = DAG.getExtLoad(ExtType, dl, PVT,
1168 WorklistRemover DeadNodes(*
this);
1169 DAG.ReplaceAllUsesOfValueWith(
SDValue(N, 0), Result);
1171 deleteAndRecombine(N);
1172 AddToWorklist(Result.
getNode());
1184 bool DAGCombiner::recursivelyDeleteUnusedNodes(
SDNode *N) {
1197 Nodes.
insert(ChildN.getNode());
1199 removeFromWorklist(N);
1204 }
while (!Nodes.
empty());
1220 E = DAG.allnodes_end();
I != E; ++
I)
1230 while (!WorklistMap.empty()) {
1234 N = Worklist.pop_back_val();
1237 bool GoodWorklistEntry = WorklistMap.erase(N);
1238 (void)GoodWorklistEntry;
1239 assert(GoodWorklistEntry &&
1240 "Found a worklist entry without a corresponding map entry!");
1245 if (recursivelyDeleteUnusedNodes(N))
1248 WorklistRemover DeadNodes(*
this);
1254 bool NIsValid = DAG.LegalizeOp(N, UpdatedNodes);
1256 for (
SDNode *LN : UpdatedNodes) {
1258 AddUsersToWorklist(LN);
1269 CombinedNodes.insert(N);
1271 if (!CombinedNodes.count(ChildN.getNode()))
1272 AddToWorklist(ChildN.getNode());
1285 if (RV.getNode() ==
N)
1290 "Node was deleted but visit returned new node!");
1293 RV.getNode()->dump(&DAG));
1296 DAG.TransferDbgValues(
SDValue(N, 0), RV);
1298 DAG.ReplaceAllUsesWith(N, RV.getNode());
1303 DAG.ReplaceAllUsesWith(N, &OpV);
1307 AddToWorklist(RV.getNode());
1308 AddUsersToWorklist(RV.getNode());
1314 recursivelyDeleteUnusedNodes(N);
1318 DAG.setRoot(
Dummy.getValue());
1319 DAG.RemoveDeadNodes();
1347 case ISD::OR:
return visitOR(N);
1421 "Node was deleted but visit returned NULL!");
1428 DagCombineInfo(DAG,
Level,
false,
this);
1430 RV = TLI.PerformDAGCombine(N, DagCombineInfo);
1444 RV = PromoteIntBinOp(
SDValue(N, 0));
1449 RV = PromoteIntShiftOp(
SDValue(N, 0));
1454 RV = PromoteExtend(
SDValue(N, 0));
1457 if (PromoteLoad(
SDValue(N, 0)))
1471 if (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1)) {
1474 if (
const auto *BinNode = dyn_cast<BinaryWithFlagsSDNode>(N)) {
1496 for (
unsigned i = 1; i < NumOps-1; ++i)
1516 bool Changed =
false;
1523 for (
unsigned i = 0; i < TFs.
size(); ++i) {
1565 Result = DAG.getEntryNode();
1573 bool UseAA = CombinerAA.getNumOccurrences() > 0 ? CombinerAA
1574 : DAG.getSubtarget().useAA();
1575 return CombineTo(N, Result, UseAA );
1583 WorklistRemover DeadNodes(*
this);
1589 AddUsersToWorklist(N);
1594 deleteAndRecombine(N);
1615 return Const !=
nullptr && Const->
isOne();
1622 return Const !=
nullptr && !Const->
isOpaque() ? Const :
nullptr;
1632 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
1651 return DAG.FoldConstantArithmetic(
ISD::ADD,
SDLoc(N), VT, N0C, N1C);
1661 if (!LegalOperations && TLI.isOffsetFoldingLegal(GA) && N1C &&
1663 return DAG.getGlobalAddress(GA->getGlobal(),
SDLoc(N1C), VT,
1670 return DAG.getNode(
ISD::SUB, DL, VT,
1691 if (N1.getOpcode() ==
ISD::SUB && N1.getOperand(1).getOpcode() ==
ISD::ADD &&
1692 N0 == N1.getOperand(1).getOperand(0))
1694 N1.getOperand(1).getOperand(1));
1696 if (N1.getOpcode() ==
ISD::SUB && N1.getOperand(1).getOpcode() ==
ISD::ADD &&
1697 N0 == N1.getOperand(1).getOperand(1))
1698 return DAG.getNode(
ISD::SUB,
SDLoc(N), VT, N1.getOperand(0),
1699 N1.getOperand(1).getOperand(0));
1702 N1.getOperand(0).getOpcode() ==
ISD::SUB &&
1703 N0 == N1.getOperand(0).getOperand(1))
1704 return DAG.getNode(N1.getOpcode(),
SDLoc(N), VT,
1705 N1.getOperand(0).getOperand(0), N1.getOperand(1));
1714 if (isa<ConstantSDNode>(N00) || isa<ConstantSDNode>(N10))
1725 APInt LHSZero, LHSOne;
1726 APInt RHSZero, RHSOne;
1727 DAG.computeKnownBits(N0, LHSZero, LHSOne);
1730 DAG.computeKnownBits(N1, RHSZero, RHSOne);
1734 if ((RHSZero & ~LHSZero) == ~LHSZero || (LHSZero & ~RHSZero) == ~RHSZero){
1735 if (!LegalOperations || TLI.isOperationLegal(
ISD::OR, VT))
1742 if (N1.getOpcode() ==
ISD::SHL && N1.getOperand(0).getOpcode() ==
ISD::SUB &&
1746 N1.getOperand(0).getOperand(1),
1748 if (N0.getOpcode() ==
ISD::SHL && N0.getOperand(0).getOpcode() ==
ISD::SUB &&
1752 N0.getOperand(0).getOperand(1),
1757 unsigned NumSignBits = DAG.ComputeNumSignBits(AndOp0);
1762 if (NumSignBits == DestBits &&
isOneConstant(N1->getOperand(1))) {
1770 N0.getOperand(0).getValueType() ==
MVT::i1 &&
1774 return DAG.getNode(
ISD::SUB, DL, VT, N1, ZExt);
1779 VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
1783 DAG.getConstant(1, DL, VT));
1784 return DAG.getNode(
ISD::SUB, DL, VT, N0, ZExt);
1798 return CombineTo(N, DAG.getNode(
ISD::ADD,
SDLoc(N), VT, N0, N1),
1814 APInt LHSZero, LHSOne;
1815 APInt RHSZero, RHSOne;
1816 DAG.computeKnownBits(N0, LHSZero, LHSOne);
1819 DAG.computeKnownBits(N1, RHSZero, RHSOne);
1823 if ((RHSZero & ~LHSZero) == ~LHSZero || (LHSZero & ~RHSZero) == ~RHSZero)
1824 return CombineTo(N, DAG.getNode(
ISD::OR,
SDLoc(N), VT, N0, N1),
1855 bool LegalOperations,
bool LegalTypes) {
1870 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
1886 return DAG.FoldConstantArithmetic(
ISD::SUB,
SDLoc(N), VT, N0C, N1C);
1890 return DAG.getNode(
ISD::ADD, DL, VT, N0,
1900 if (N0.getOpcode() ==
ISD::ADD && N0.getOperand(0) == N1)
1903 if (N0.getOpcode() ==
ISD::ADD && N0.getOperand(1) == N1)
1917 (N0.getOperand(1).getOpcode() ==
ISD::SUB ||
1918 N0.getOperand(1).getOpcode() ==
ISD::ADD) &&
1919 N0.getOperand(1).getOperand(0) == N1)
1920 return DAG.getNode(N0.getOperand(1).getOpcode(),
SDLoc(N), VT,
1921 N0.getOperand(0), N0.getOperand(1).getOperand(1));
1924 N0.getOperand(1).getOpcode() ==
ISD::ADD &&
1925 N0.getOperand(1).getOperand(1) == N1)
1927 N0.getOperand(0), N0.getOperand(1).getOperand(0));
1930 N0.getOperand(1).getOpcode() ==
ISD::SUB &&
1931 N0.getOperand(1).getOperand(1) == N1)
1933 N0.getOperand(0), N0.getOperand(1).getOperand(0));
1943 if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) {
1946 return DAG.getGlobalAddress(GA->getGlobal(),
SDLoc(N1C), VT,
1951 if (GA->getGlobal() == GB->getGlobal())
1952 return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(),
1962 DAG.getConstant(1, DL, VT));
1963 return DAG.getNode(
ISD::ADD, DL, VT, N0, ZExt);
1977 return CombineTo(N, DAG.getNode(
ISD::SUB,
SDLoc(N), VT, N0, N1),
1984 return CombineTo(N, DAG.getConstant(0, DL, VT),
1996 return CombineTo(N, DAG.getNode(
ISD::XOR,
SDLoc(N), VT, N1, N0),
2022 return DAG.getConstant(0,
SDLoc(N), VT);
2024 bool N0IsConst =
false;
2025 bool N1IsConst =
false;
2026 bool N1IsOpaqueConst =
false;
2027 bool N0IsOpaqueConst =
false;
2028 APInt ConstValue0, ConstValue1;
2031 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
2037 N0IsConst = isa<ConstantSDNode>(N0);
2039 ConstValue0 = cast<ConstantSDNode>(N0)->getAPIntValue();
2040 N0IsOpaqueConst = cast<ConstantSDNode>(N0)->isOpaque();
2042 N1IsConst = isa<ConstantSDNode>(N1);
2044 ConstValue1 = cast<ConstantSDNode>(N1)->getAPIntValue();
2045 N1IsOpaqueConst = cast<ConstantSDNode>(N1)->isOpaque();
2050 if (N0IsConst && N1IsConst && !N0IsOpaqueConst && !N1IsOpaqueConst)
2059 if (N1IsConst && ConstValue1 == 0)
2066 if (N1IsConst && ConstValue1 == 1 && IsFullSplat)
2071 return DAG.getNode(
ISD::SUB, DL, VT,
2072 DAG.getConstant(0, DL, VT), N0);
2075 if (N1IsConst && !N1IsOpaqueConst && ConstValue1.
isPowerOf2() &&
2078 return DAG.getNode(
ISD::SHL, DL, VT, N0,
2083 if (N1IsConst && !N1IsOpaqueConst && (-ConstValue1).isPowerOf2() &&
2085 unsigned Log2Val = (-ConstValue1).
logBase2();
2089 return DAG.getNode(
ISD::SUB, DL, VT,
2090 DAG.getConstant(0, DL, VT),
2092 DAG.getConstant(Log2Val, DL,
2111 SDValue Sh(
nullptr,0),
Y(
nullptr,0);
2126 Sh.getOperand(0),
Y);
2156 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
2163 return DAG.FoldConstantArithmetic(
ISD::SDIV,
SDLoc(N), VT, N0C, N1C);
2165 if (N1C && N1C->
isOne())
2170 return DAG.getNode(
ISD::SUB, DL, VT,
2171 DAG.getConstant(0, DL, VT), N0);
2176 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
2186 !cast<BinaryWithFlagsSDNode>(
N)->
Flags.hasExact() &&
2191 if (TLI.isPow2SDivCheap())
2215 AddToWorklist(SRL.getNode());
2218 DAG.getConstant(lg2, DL,
2226 AddToWorklist(SRA.getNode());
2227 return DAG.getNode(
ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
SRA);
2232 if (N1C && !TLI.isIntDivCheap()) {
2239 return DAG.getConstant(0,
SDLoc(N), VT);
2254 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
2267 return DAG.getNode(
ISD::SRL, DL, VT, N0,
2274 if (SHC->getAPIntValue().isPowerOf2()) {
2279 DAG.getConstant(SHC->getAPIntValue()
2283 return DAG.getNode(
ISD::SRL, DL, VT, N0, Add);
2288 if (N1C && !TLI.isIntDivCheap()) {
2295 return DAG.getConstant(0,
SDLoc(N), VT);
2318 if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
2328 if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != Div.
getNode()) {
2339 return DAG.getConstant(0,
SDLoc(N), VT);
2363 return DAG.getNode(
ISD::AND, DL, VT, N0,
2369 if (SHC->getAPIntValue().isPowerOf2()) {
2376 return DAG.getNode(
ISD::AND, DL, VT, N0, Add);
2387 if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != Div.
getNode()) {
2398 return DAG.getConstant(0,
SDLoc(N), VT);
2425 return DAG.getConstant(0,
SDLoc(N), VT);
2433 if (TLI.isOperationLegal(
ISD::MUL, NewVT)) {
2438 DAG.getConstant(SimpleSize, DL,
2461 return DAG.getConstant(0, DL, VT);
2469 if (TLI.isOperationLegal(
ISD::MUL, NewVT)) {
2474 DAG.getConstant(SimpleSize, DL,
2486 SDValue DAGCombiner::SimplifyNodeWithTwoResults(
SDNode *N,
unsigned LoOp,
2491 (!LegalOperations ||
2492 TLI.isOperationLegalOrCustom(LoOp, N->
getValueType(0)))) {
2494 return CombineTo(N, Res, Res);
2500 (!LegalOperations ||
2503 return CombineTo(N, Res, Res);
2507 if (LoExists && HiExists)
2515 if (LoOpt.getNode() && LoOpt.getNode() != Lo.
getNode() &&
2516 (!LegalOperations ||
2517 TLI.isOperationLegal(LoOpt.getOpcode(), LoOpt.getValueType())))
2518 return CombineTo(N, LoOpt, LoOpt);
2525 if (HiOpt.getNode() && HiOpt != Hi &&
2526 (!LegalOperations ||
2527 TLI.isOperationLegal(HiOpt.getOpcode(), HiOpt.getValueType())))
2528 return CombineTo(N, HiOpt, HiOpt);
2536 if (Res.
getNode())
return Res;
2547 if (TLI.isOperationLegal(
ISD::MUL, NewVT)) {
2553 DAG.getConstant(SimpleSize, DL,
2558 return CombineTo(N, Lo, Hi);
2567 if (Res.
getNode())
return Res;
2578 if (TLI.isOperationLegal(
ISD::MUL, NewVT)) {
2584 DAG.getConstant(SimpleSize, DL,
2589 return CombineTo(N, Lo, Hi);
2599 if (C2->getAPIntValue() == 2)
2609 if (C2->getAPIntValue() == 2)
2618 if (Res.
getNode())
return Res;
2625 if (Res.
getNode())
return Res;
2632 SDValue DAGCombiner::SimplifyBinOpWithSameOpcodeHands(
SDNode *N) {
2655 (!LegalTypes || TLI.isTypeDesirableForOp(N->
getOpcode(), Op0VT))) ||
2657 (!TLI.isZExtFree(VT, Op0VT) ||
2658 !TLI.isTruncateFree(Op0VT, VT)) &&
2659 TLI.isTypeLegal(Op0VT))) &&
2662 (!LegalOperations || TLI.isOperationLegal(N->
getOpcode(), Op0VT))) {
2666 AddToWorklist(ORNode.
getNode());
2680 AddToWorklist(ORNode.
getNode());
2727 "Inputs to shuffles are not the same type");
2733 if (SVN0->
hasOneUse() && SVN1->hasOneUse() &&
2734 SVN0->
getMask().equals(SVN1->getMask())) {
2741 ShOp = DAG.getConstant(0,
SDLoc(N), VT);
2752 AddToWorklist(NewNode.
getNode());
2753 return DAG.getVectorShuffle(VT,
SDLoc(N), NewNode, ShOp,
2762 ShOp = DAG.getConstant(0,
SDLoc(N), VT);
2773 AddToWorklist(NewNode.
getNode());
2774 return DAG.getVectorShuffle(VT,
SDLoc(N), ShOp, NewNode,
2793 return DAG.getConstant(0,
SDLoc(LocReference), VT);
2795 SDValue LL, LR, RL, RR, CC0, CC1;
2796 if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){
2800 if (LR == RR && isa<ConstantSDNode>(LR) && Op0 == Op1 &&
2806 AddToWorklist(ORNode.
getNode());
2807 return DAG.getSetCC(
SDLoc(LocReference), VT, ORNode, LR, Op1);
2814 AddToWorklist(ANDNode.
getNode());
2815 return DAG.getSetCC(
SDLoc(LocReference), VT, ANDNode, LR, Op1);
2821 AddToWorklist(ORNode.
getNode());
2822 return DAG.getSetCC(
SDLoc(LocReference), VT, ORNode, LR, Op1);
2827 if (LL == RL && isa<ConstantSDNode>(LR) && isa<ConstantSDNode>(RR) &&
2833 LL, DAG.getConstant(1, DL,
2835 AddToWorklist(ADDNode.
getNode());
2836 return DAG.getSetCC(
SDLoc(LocReference), VT, ADDNode,
2841 if (LL == RR && LR == RL) {
2845 if (LL == RL && LR == RR) {
2849 (!LegalOperations ||
2869 SRLI->getZExtValue());
2870 if (DAG.MaskedValueIsZero(N0.
getOperand(1), Mask)) {
2876 N0.
getOperand(0), DAG.getConstant(ADDC, DL, VT));
2877 CombineTo(N0.
getNode(), NewAdd);
2879 return SDValue(LocReference, 0);
2897 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
2903 return DAG.getConstant(
2909 return DAG.getConstant(
2924 if (N0C && N1C && !N1C->
isOpaque())
2925 return DAG.FoldConstantArithmetic(
ISD::AND,
SDLoc(N), VT, N0C, N1C);
2935 if (N1C && DAG.MaskedValueIsZero(
SDValue(N, 0),
2937 return DAG.getConstant(0,
SDLoc(N), VT);
2951 if (DAG.MaskedValueIsZero(N0Op0, Mask)) {
2961 CombineTo(N0.
getNode(), Zext);
2982 Constant =
C->getAPIntValue();
2984 APInt SplatValue, SplatUndef;
2985 unsigned SplatBitSize;
2987 bool IsSplat = Vector->isConstantSplat(SplatValue, SplatUndef,
2988 SplatBitSize, HasAnyUndefs);
2992 SplatValue |= SplatUndef;
2998 EVT VT = Vector->getValueType(0);
3004 if (BitWidth > SplatBitSize)
3005 for (SplatValue = SplatValue.
zextOrTrunc(BitWidth);
3006 SplatBitSize < BitWidth;
3007 SplatBitSize = SplatBitSize * 2)
3008 SplatValue |= SplatValue.
shl(SplatBitSize);
3012 if (SplatBitSize % BitWidth == 0) {
3014 for (
unsigned i = 0, n = SplatBitSize/BitWidth; i < n; ++i)
3023 bool CanZextLoadProfitably = TLI.isLoadExtLegal(
ISD::ZEXTLOAD,
3035 default: B =
false;
break;
3041 if (B && Constant.isAllOnesValue()) {
3056 CombineTo(Load, To, 3,
true);
3058 CombineTo(Load, NewLoad.getValue(0), NewLoad.getValue(1));
3064 CombineTo(N, (N0.
getNode() ==
Load) ? NewLoad : N0);
3079 : cast<LoadSDNode>(N0);
3088 if (ExtVT == LoadedVT &&
3089 (!LegalOperations || TLI.isLoadExtLegal(
ISD::ZEXTLOAD, LoadResultTy,
3097 CombineTo(LN0, NewLoad, NewLoad.
getValue(1));
3105 (!LegalOperations || TLI.isLoadExtLegal(
ISD::ZEXTLOAD, LoadResultTy,
3115 if (DAG.getDataLayout().isBigEndian()) {
3116 unsigned LVTStoreBytes = LoadedVT.getStoreSize();
3118 unsigned PtrOff = LVTStoreBytes - EVTStoreBytes;
3121 NewPtr, DAG.getConstant(PtrOff, DL, PtrType));
3122 Alignment =
MinAlign(Alignment, PtrOff);
3125 AddToWorklist(NewPtr.
getNode());
3134 CombineTo(LN0, Load, Load.getValue(1));
3141 if (
SDValue Combined = visitANDLike(N0, N1, N))
3146 SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N);
3147 if (Tmp.
getNode())
return Tmp;
3164 BitWidth - MemVT.getScalarType().getSizeInBits())) &&
3184 BitWidth - MemVT.getScalarType().getSizeInBits())) &&
3208 bool DemandHighBits) {
3209 if (!LegalOperations)
3219 bool LookPassAnd0 =
false;
3220 bool LookPassAnd1 =
false;
3232 LookPassAnd0 =
true;
3242 LookPassAnd1 =
true;
3257 if (N01C->
getZExtValue() != 8 || N11C->getZExtValue() != 8)
3269 LookPassAnd0 =
true;
3280 LookPassAnd1 =
true;
3289 if (DemandHighBits && OpSizeInBits > 16) {
3298 if (!LookPassAnd1 &&
3299 !DAG.MaskedValueIsZero(
3305 if (OpSizeInBits > 16) {
3308 DAG.getConstant(OpSizeInBits - 16, DL,
3336 case 0xFF: Num = 0;
break;
3337 case 0xFF00: Num = 1;
break;
3338 case 0xFF0000: Num = 2;
break;
3339 case 0xFF000000: Num = 3;
break;
3345 if (Num == 0 || Num == 2) {
3365 if (Num != 0 && Num != 2)
3373 if (Num != 1 && Num != 3)
3394 if (!LegalOperations)
3445 if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
3455 if (TLI.isOperationLegalOrCustom(
ISD::ROTL, VT))
3457 if (TLI.isOperationLegalOrCustom(
ISD::ROTR, VT))
3458 return DAG.getNode(
ISD::ROTR, DL, VT, BSwap, ShAmt);
3459 return DAG.getNode(
ISD::OR, DL, VT,
3460 DAG.getNode(
ISD::SHL, DL, VT, BSwap, ShAmt),
3461 DAG.getNode(
ISD::SRL, DL, VT, BSwap, ShAmt));
3469 if (!LegalOperations &&
3473 SDLoc(LocReference), VT);
3476 SDValue LL, LR, RL, RR, CC0, CC1;
3477 if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){
3487 AddToWorklist(ORNode.
getNode());
3488 return DAG.getSetCC(
SDLoc(LocReference), VT, ORNode, LR, Op1);
3495 AddToWorklist(ANDNode.
getNode());
3496 return DAG.getSetCC(
SDLoc(LocReference), VT, ANDNode, LR, Op1);
3500 if (LL == RR && LR == RL) {
3504 if (LL == RL && LR == RR) {
3508 (!LegalOperations ||
3529 const APInt &LHSMask = N0O1C->getAPIntValue();
3530 const APInt &RHSMask = N1O1C->getAPIntValue();
3532 if (DAG.MaskedValueIsZero(N0.
getOperand(0), RHSMask&~LHSMask) &&
3533 DAG.MaskedValueIsZero(N1.
getOperand(0), LHSMask&~RHSMask)) {
3537 return DAG.getNode(
ISD::AND, DL, VT, X,
3538 DAG.getConstant(LHSMask | RHSMask, DL, VT));
3565 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
3577 return DAG.getConstant(
3583 return DAG.getConstant(
3591 if (isa<ShuffleVectorSDNode>(N0) &&
3592 isa<ShuffleVectorSDNode>(N1) &&
3594 TLI.isTypeLegal(VT) &&
3597 bool CanFold =
true;
3611 for (
unsigned i = 0; i != NumElts && CanFold; ++i) {
3613 int M1 = SV1->getMaskElt(i);
3616 if (M0 < 0 && M1 < 0) {
3617 Mask1.push_back(M0);
3622 if (M0 < 0 || M1 < 0 ||
3623 (M0 < (
int)NumElts && M1 < (
int)NumElts) ||
3624 (M0 >= (
int)NumElts && M1 >= (
int)NumElts)) {
3629 Mask1.push_back(M0 < (
int)NumElts ? M0 : M1 + NumElts);
3630 Mask2.
push_back(M1 < (
int)NumElts ? M1 : M0 + NumElts);
3635 if (TLI.isShuffleMaskLegal(Mask1, VT))
3638 if (TLI.isShuffleMaskLegal(Mask2, VT))
3648 if (N0C && N1C && !N1C->
isOpaque())
3649 return DAG.FoldConstantArithmetic(
ISD::OR,
SDLoc(N), VT, N0C, N1C);
3661 if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->
getAPIntValue()))
3664 if (
SDValue Combined = visitORLike(N0, N1, N))
3668 SDValue BSwap = MatchBSwapHWord(N, N0, N1);
3671 BSwap = MatchBSwapHWordLow(N, N0, N1);
3694 SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N);
3695 if (Tmp.
getNode())
return Tmp;
3770 unsigned MaskLoBits = 0;
3774 cast<ConstantSDNode>(Neg.
getOperand(1))->getAPIntValue() == OpSize - 1) {
3792 cast<ConstantSDNode>(Pos.
getOperand(1))->getAPIntValue() == OpSize - 1)
3819 Width = (cast<ConstantSDNode>(Pos.
getOperand(1))->getAPIntValue() +
3827 return Width.
getLoBits(MaskLoBits) == 0;
3828 return Width == OpSize;
3838 SDValue InnerNeg,
unsigned PosOpcode,
3839 unsigned NegOpcode,
SDLoc DL) {
3849 bool HasPos = TLI.isOperationLegalOrCustom(PosOpcode, VT);
3850 return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, Shifted,
3851 HasPos ? Pos : Neg).getNode();
3863 if (!TLI.isTypeLegal(VT))
return nullptr;
3866 bool HasROTL = TLI.isOperationLegalOrCustom(
ISD::ROTL, VT);
3867 bool HasROTR = TLI.isOperationLegalOrCustom(
ISD::ROTR, VT);
3868 if (!HasROTL && !HasROTR)
return nullptr;
3904 uint64_t LShVal = cast<ConstantSDNode>(LHSShiftAmt)->getZExtValue();
3905 uint64_t RShVal = cast<ConstantSDNode>(RHSShiftAmt)->getZExtValue();
3906 if ((LShVal + RShVal) != OpSizeInBits)
3910 LHSShiftArg, HasROTL ? LHSShiftAmt : RHSShiftAmt);
3918 Mask &= cast<ConstantSDNode>(LHSMask)->getAPIntValue() | RHSBits;
3922 Mask &= cast<ConstantSDNode>(RHSMask)->getAPIntValue() | LHSBits;
3925 Rot = DAG.
getNode(
ISD::AND, DL, VT, Rot, DAG.getConstant(Mask, DL, VT));
3937 SDValue LExtOp0 = LHSShiftAmt;
3938 SDValue RExtOp0 = RHSShiftAmt;
3951 SDNode *TryL = MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt,
3952 LExtOp0, RExtOp0,
ISD::ROTL, ISD::ROTR, DL);
3956 SDNode *TryR = MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
3957 RExtOp0, LExtOp0, ISD::ROTR,
ISD::ROTL, DL);
3971 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
3983 return DAG.getConstant(0,
SDLoc(N), VT);
3993 return DAG.FoldConstantArithmetic(
ISD::XOR,
SDLoc(N), VT, N0C, N1C);
4007 if (TLI.isConstTrueVal(N1.
getNode()) && isSetCCEquivalent(N0, LHS, RHS, CC)) {
4012 if (!LegalOperations ||
4018 return DAG.getSetCC(
SDLoc(N), VT, LHS, RHS, NotCC);
4029 isSetCCEquivalent(N0.
getOperand(0), LHS, RHS, CC)){
4042 if (isOneUseSetCC(RHS) || isOneUseSetCC(LHS)) {
4046 AddToWorklist(LHS.
getNode()); AddToWorklist(RHS.getNode());
4047 return DAG.getNode(NewOpcode,
SDLoc(N), VT, LHS, RHS);
4054 if (isa<ConstantSDNode>(RHS) || isa<ConstantSDNode>(LHS)) {
4058 AddToWorklist(LHS.
getNode()); AddToWorklist(RHS.getNode());
4059 return DAG.getNode(NewOpcode,
SDLoc(N), VT, LHS, RHS);
4067 AddToWorklist(NotX.
getNode());
4076 N00C->getAPIntValue(),
DL, VT));
4110 return DAG.getNode(
ISD::ROTL, DL, VT, DAG.getConstant(~1, DL, VT),
4116 SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N);
4117 if (Tmp.
getNode())
return Tmp;
4138 bool HighBitSet =
false;
4158 if (!BinOpCst)
return SDValue();
4169 !isa<ConstantSDNode>(BinOpLHSVal->
getOperand(1)))
4180 if (BinOpRHSSignSet != HighBitSet)
4184 if (!TLI.isDesirableToCommuteWithShift(LHS))
4191 assert(isa<ConstantSDNode>(NewRHS) &&
"Folding was not successful!");
4199 return DAG.getNode(LHS->
getOpcode(),
SDLoc(N), VT, NewShift, NewRHS);
4202 SDValue DAGCombiner::distributeTruncateThroughAnd(
SDNode *N) {
4218 return DAG.getNode(
ISD::AND, DL, TruncVT,
4220 DAG.getConstant(TruncC, DL, TruncVT));
4249 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
4276 if (N0C && N1C && !N1C->
isOpaque())
4277 return DAG.FoldConstantArithmetic(
ISD::SHL,
SDLoc(N), VT, N0C, N1C);
4283 return DAG.getUNDEF(VT);
4289 return DAG.getConstant(0,
SDLoc(N), VT);
4291 if (DAG.MaskedValueIsZero(
SDValue(N, 0),
4293 return DAG.getConstant(0,
SDLoc(N), VT);
4308 uint64_t c1 = N0C1->getZExtValue();
4311 if (c1 + c2 >= OpSizeInBits)
4312 return DAG.getConstant(0, DL, VT);
4329 uint64_t c1 = N0Op0C1->getZExtValue();
4333 if (c2 >= OpSizeInBits - InnerShiftSize) {
4335 if (c1 + c2 >= OpSizeInBits)
4336 return DAG.getConstant(0, DL, VT);
4337 return DAG.getNode(
ISD::SHL, DL, VT,
4352 uint64_t c1 = N0Op0C1->getZExtValue();
4361 DAG.getConstant(c2, DL, CountVT));
4362 AddToWorklist(NewSHL.
getNode());
4372 cast<BinaryWithFlagsSDNode>(N0)->Flags.hasExact()) {
4391 uint64_t c1 = N0C1->getZExtValue();
4392 if (c1 < OpSizeInBits) {
4397 Mask = Mask.
shl(c2 - c1);
4402 Mask = Mask.
lshr(c1 - c2);
4408 return DAG.getNode(
ISD::AND, DL, VT, Shift,
4409 DAG.getConstant(Mask, DL, VT));
4415 unsigned BitSize = VT.getScalarSizeInBits();
4438 SDValue NewSHL = visitShiftByConstant(N, N1C);
4455 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
4463 if (N0C && N1C && !N1C->
isOpaque())
4464 return DAG.FoldConstantArithmetic(
ISD::SRA,
SDLoc(N), VT, N0C, N1C);
4473 return DAG.getUNDEF(VT);
4485 if ((!LegalOperations ||
4494 unsigned Sum = N1C->
getZExtValue() + C1->getZExtValue();
4495 if (Sum >= OpSizeInBits)
4496 Sum = OpSizeInBits - 1;
4499 DAG.getConstant(Sum, DL, N1.getValueType()));
4526 if ((ShiftAmt > 0) &&
4529 TLI.isTruncateFree(VT, TruncVT)) {
4532 SDValue Amt = DAG.getConstant(ShiftAmt, DL,
4546 N1.getOperand(0).getOpcode() ==
ISD::AND) {
4547 SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode());
4562 unsigned LargeShiftVal = LargeShift->getZExtValue();
4583 if (DAG.SignBitIsZero(N0))
4587 SDValue NewSRA = visitShiftByConstant(N, N1C);
4604 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
4612 if (N0C && N1C && !N1C->
isOpaque())
4613 return DAG.FoldConstantArithmetic(
ISD::SRL,
SDLoc(N), VT, N0C, N1C);
4619 return DAG.getUNDEF(VT);
4624 if (N1C && DAG.MaskedValueIsZero(
SDValue(N, 0),
4626 return DAG.getConstant(0,
SDLoc(N), VT);
4634 if (c1 + c2 >= OpSizeInBits)
4635 return DAG.getConstant(0, DL, VT);
4652 if (c1 + OpSizeInBits == InnerShiftSize) {
4654 if (c1 + c2 >= InnerShiftSize)
4655 return DAG.getConstant(0, DL, VT);
4657 DAG.getNode(
ISD::SRL, DL, InnerShiftVT,
4659 DAG.getConstant(c1 + c2, DL,
4667 if (BitSize <= 64) {
4671 DAG.getConstant(~0ULL >> ShAmt, DL, VT));
4681 return DAG.getUNDEF(VT);
4683 if (!LegalTypes || TLI.isTypeDesirableForOp(
ISD::SRL, SmallVT)) {
4688 DAG.getConstant(ShiftAmt, DL0,
4690 AddToWorklist(SmallShift.
getNode());
4693 return DAG.getNode(
ISD::AND, DL, VT,
4695 DAG.getConstant(Mask, DL, VT));
4709 APInt KnownZero, KnownOne;
4710 DAG.computeKnownBits(N0.
getOperand(0), KnownZero, KnownOne);
4718 APInt UnknownBits = ~KnownZero;
4719 if (UnknownBits == 0)
return DAG.getConstant(1,
SDLoc(N0), VT);
4722 if ((UnknownBits & (UnknownBits - 1)) == 0) {
4733 DAG.getConstant(ShAmt, DL,
4739 return DAG.getNode(
ISD::XOR, DL, VT,
4740 Op, DAG.getConstant(1, DL, VT));
4758 SDValue NewSRL = visitShiftByConstant(N, N1C);
4764 SDValue NarrowLoad = ReduceLoadWidth(N);
4869 if (!(LHS == True && RHS == False) && !(LHS == False && RHS == True))
4881 return DAG.
getNode(Opcode, DL, VT, LHS, RHS);
4892 return DAG.
getNode(Opcode, DL, VT, LHS, RHS);
4910 if (
const ConstantSDNode *N0C = dyn_cast<const ConstantSDNode>(N0)) {
4929 TLI.getBooleanContents(
false,
false) ==
4930 TLI.getBooleanContents(
false,
true) &&
4931 TLI.getBooleanContents(
false,
false) ==
4937 return DAG.getNode(
ISD::XOR, DL, VT0,
4938 N0, DAG.getConstant(1, DL, VT0));
4942 N0, DAG.getConstant(1, DL0, VT0));
4943 AddToWorklist(XORNode.
getNode());
4951 AddToWorklist(NOTNode.
getNode());
4957 AddToWorklist(NOTNode.
getNode());
4973 if (SimplifySelectOps(N, N1, N2))
4990 DAG.isKnownNeverNaN(N1) && DAG.isKnownNeverNaN(N2)) {
4995 N1, N2, CC, TLI, DAG);
5000 if ((!LegalOperations &&
5006 return SimplifySelect(
SDLoc(N), N0, N1, N2);
5010 if (TLI.shouldNormalizeToSelectSequence(*DAG.getContext(), VT)) {
5026 N1.getValueType(), Cond1, N1, N2);
5039 if (!TLI.shouldNormalizeToSelectSequence(*DAG.getContext(), VT)) {
5046 if (
SDValue Combined = visitANDLike(N0, N1_0, N))
5058 if (!TLI.shouldNormalizeToSelectSequence(*DAG.getContext(), VT)) {
5065 if (
SDValue Combined = visitORLike(N0, N2_0, N))
5089 return std::make_pair(Lo, Hi);
5116 for (
int i = 0; i < NumElems / 2; ++i) {
5120 if (BottomHalf ==
nullptr)
5121 BottomHalf = cast<ConstantSDNode>(Cond.
getOperand(i));
5128 for (
int i = NumElems / 2; i < NumElems; ++i) {
5132 if (TopHalf ==
nullptr)
5133 TopHalf = cast<ConstantSDNode>(Cond.
getOperand(i));
5138 assert(TopHalf && BottomHalf &&
5139 "One half of the selector was all UNDEFs and the other was all the "
5140 "same value. This should have been addressed before this function.");
5165 if (TLI.getTypeAction(*DAG.getContext(), Data.
getValueType()) !=
5169 std::tie(MaskLo, MaskHi) =
SplitVSETCC(Mask.getNode(), DAG);
5172 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MSC->
getValueType(0));
5179 EVT LoMemVT, HiMemVT;
5180 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
5183 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
5187 std::tie(IndexLo, IndexHi) = DAG.SplitVector(MSC->
getIndex(),
DL);
5194 SDValue OpsLo[] = { Chain, DataLo, MaskLo, BasePtr, IndexLo };
5198 SDValue OpsHi[] = {Chain, DataHi, MaskHi, BasePtr, IndexHi};
5225 if (TLI.getTypeAction(*DAG.getContext(), Data.
getValueType()) !=
5230 std::tie(MaskLo, MaskHi) =
SplitVSETCC(Mask.getNode(), DAG);
5233 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MST->
getValueType(0));
5243 unsigned SecondHalfAlignment =
5245 Alignment/2 : Alignment;
5247 EVT LoMemVT, HiMemVT;
5248 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
5251 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
5258 Lo = DAG.getMaskedStore(Chain, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
5263 DAG.getConstant(IncrementSize, DL, Ptr.
getValueType()));
5265 MMO = DAG.getMachineFunction().
5271 Hi = DAG.getMaskedStore(Chain, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
5274 AddToWorklist(Lo.getNode());
5275 AddToWorklist(Hi.getNode());
5302 if (TLI.getTypeAction(*DAG.getContext(), VT) !=
5307 std::tie(MaskLo, MaskHi) =
SplitVSETCC(Mask.getNode(), DAG);
5311 std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, DL);
5314 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
5320 EVT LoMemVT, HiMemVT;
5321 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
5326 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
5333 SDValue OpsLo[] = { Chain, Src0Lo, MaskLo, BasePtr, IndexLo };
5334 Lo = DAG.getMaskedGather(DAG.getVTList(LoVT,
MVT::Other), LoVT,
DL, OpsLo,
5337 SDValue OpsHi[] = {Chain, Src0Hi, MaskHi, BasePtr, IndexHi};
5338 Hi = DAG.getMaskedGather(DAG.getVTList(HiVT,
MVT::Other), HiVT,
DL, OpsHi,
5351 DAG.ReplaceAllUsesOfValueWith(
SDValue(MGT, 1), Chain);
5355 SDValue RetOps[] = { GatherRes, Chain };
5356 return DAG.getMergeValues(RetOps, DL);
5377 if (TLI.getTypeAction(*DAG.getContext(), VT) !=
5382 std::tie(MaskLo, MaskHi) =
SplitVSETCC(Mask.getNode(), DAG);
5386 std::tie(Src0Lo, Src0Hi) = DAG.SplitVector(Src0, DL);
5389 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->
getValueType(0));
5398 unsigned SecondHalfAlignment =
5400 Alignment/2 : Alignment;
5402 EVT LoMemVT, HiMemVT;
5403 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
5410 Lo = DAG.getMaskedLoad(LoVT, DL, Chain, Ptr, MaskLo, Src0Lo, LoMemVT, MMO,
5415 DAG.getConstant(IncrementSize, DL, Ptr.
getValueType()));
5417 MMO = DAG.getMachineFunction().
5422 Hi = DAG.getMaskedLoad(HiVT, DL, Chain, Ptr, MaskHi, Src0Hi, HiMemVT, MMO,
5425 AddToWorklist(Lo.getNode());
5426 AddToWorklist(Hi.getNode());
5435 DAG.ReplaceAllUsesOfValueWith(
SDValue(MLD, 1), Chain);
5439 SDValue RetOps[] = { LoadRes, Chain };
5440 return DAG.getMergeValues(RetOps, DL);
5476 AddToWorklist(Shift.
getNode());
5478 return DAG.getNode(
ISD::XOR, DL, VT, Add, Shift);
5482 if (SimplifySelectOps(N, N1, N2))
5493 if (TLI.getTypeAction(*DAG.getContext(), VT) !=
5499 std::tie(LL, LH) = DAG.SplitVectorOperand(N, 1);
5500 std::tie(RL, RH) = DAG.SplitVectorOperand(N, 2);
5548 N0, N1, CC,
SDLoc(N),
false);
5553 if (!SCCC->isNullValue())
5570 if (SimplifySelectOps(N, N2, N3))
5574 return SimplifySelectCC(
SDLoc(N), N0, N1, N2, N3, CC);
5579 cast<CondCodeSDNode>(N->
getOperand(2))->
get(),
5591 bool LegalOperations) {
5598 &&
"Expected EXTEND dag node in input!");
5603 if (isa<ConstantSDNode>(N0))
5611 (!LegalTypes || (!LegalOperations && TLI.
isTypeLegal(SVT))) &&
5622 for (
unsigned i=0; i != NumElts; ++i) {
5632 APInt C = cast<ConstantSDNode>(Op)->getAPIntValue().zextOrTrunc(EVTBits);
5650 bool HasCopyToRegUses =
false;
5658 if (UI.getUse().getResNo() != N0.
getResNo())
5667 for (
unsigned i = 0; i != 2; ++i) {
5671 if (!isa<ConstantSDNode>(UseOp))
5685 HasCopyToRegUses =
true;
5688 if (HasCopyToRegUses) {
5689 bool BothLiveOut =
false;
5692 SDUse &Use = UI.getUse();
5701 return ExtendNodes.
size();
5710 for (
unsigned i = 0, e = SetCCs.
size(); i != e; ++i) {
5711 SDNode *SetCC = SetCCs[i];
5714 for (
unsigned j = 0; j != 2; ++j) {
5735 "Unexpected node type (not an extend)!");
5772 EVT SplitSrcVT = SrcVT;
5773 EVT SplitDstVT = DstVT;
5774 while (!TLI.isLoadExtLegalOrCustom(ExtType, SplitDstVT, SplitSrcVT) &&
5776 SplitDstVT = DAG.GetSplitDestVTs(SplitDstVT).first;
5777 SplitSrcVT = DAG.GetSplitDestVTs(SplitSrcVT).first;
5780 if (!TLI.isLoadExtLegalOrCustom(ExtType, SplitDstVT, SplitSrcVT))
5784 const unsigned NumSplits =
5791 for (
unsigned Idx = 0; Idx < NumSplits; Idx++) {
5792 const unsigned Offset = Idx * Stride;
5795 SDValue SplitLoad = DAG.getExtLoad(
5796 ExtType, DL, SplitDstVT, LN0->
getChain(), BasePtr,
5805 Chains.
push_back(SplitLoad.getValue(1));
5811 CombineTo(N, NewValue);
5817 CombineTo(N0.
getNode(), Trunc, NewChain);
5818 ExtendSetCCUses(SetCCs, Trunc, NewValue, DL,
5844 CombineTo(N0.
getNode(), NarrowLoad);
5857 unsigned NumSignBits = DAG.ComputeNumSignBits(Op);
5859 if (OpBits == DestBits) {
5862 if (NumSignBits > DestBits-MidBits)
5864 }
else if (OpBits < DestBits) {
5867 if (NumSignBits > OpBits-MidBits)
5872 if (NumSignBits > OpBits-MidBits)
5879 if (OpBits < DestBits)
5881 else if (OpBits > DestBits)
5892 ((!LegalOperations && !VT.
isVector() &&
5895 bool DoXform =
true;
5900 DoXform &= TLI.isVectorLoadExtDesirable(
SDValue(N, 0));
5907 CombineTo(N, ExtLoad);
5911 ExtendSetCCUses(SetCCs, Trunc, ExtLoad,
SDLoc(N),
5919 if (
SDValue ExtLoad = CombineExtLoad(N))
5928 if ((!LegalOperations && !LN0->
isVolatile()) ||
5934 CombineTo(N, ExtLoad);
5950 (!LegalOperations && TLI.isOperationLegal(N0.
getOpcode(), VT))) {
5953 bool DoXform =
true;
5967 ExtLoad, DAG.getConstant(Mask, DL, VT));
5973 ExtendSetCCUses(SetCCs, Trunc, ExtLoad, DL,
5984 if (VT.isVector() && !LegalOperations &&
5985 TLI.getBooleanContents(N0VT) ==
6000 cast<CondCodeSDNode>(N0.
getOperand(2))->
get());
6006 if (SVT == MatchingVectorType) {
6007 SDValue VsetCC = DAG.getSetCC(
SDLoc(N), MatchingVectorType,
6009 cast<CondCodeSDNode>(N0.
getOperand(2))->
get());
6010 return DAG.getSExtOrTrunc(VsetCC,
SDLoc(N), VT);
6015 unsigned ElementWidth = VT.getScalarType().getSizeInBits();
6021 NegOne, DAG.getConstant(0, DL, VT),
6022 cast<CondCodeSDNode>(N0.
getOperand(2))->
get(),
true);
6023 if (SCC.
getNode())
return SCC;
6025 if (!VT.isVector()) {
6027 if (!LegalOperations || TLI.isOperationLegal(
ISD::SETCC, SetCCVT)) {
6030 SDValue SetCC = DAG.getSetCC(DL, SetCCVT,
6032 return DAG.getSelect(DL, VT, SetCC,
6033 NegOne, DAG.getConstant(0, DL, VT));
6040 DAG.SignBitIsZero(N0))
6103 APInt TruncatedBits =
6110 if (TruncatedBits == (KnownZero & TruncatedBits)) {
6127 CombineTo(N0.
getNode(), NarrowLoad);
6137 (!LegalOperations || TLI.isOperationLegal(
ISD::AND, VT))) {
6145 CombineTo(N0.
getNode(), NarrowLoad);
6160 return DAG.getZeroExtendInReg(Op,
SDLoc(N),
6181 return DAG.getNode(
ISD::AND, DL, VT,
6182 X, DAG.getConstant(Mask, DL, VT));
6189 ((!LegalOperations && !VT.
isVector() &&
6192 bool DoXform =
true;
6197 DoXform &= TLI.isVectorLoadExtDesirable(
SDValue(N, 0));
6204 CombineTo(N, ExtLoad);
6209 ExtendSetCCUses(SetCCs, Trunc, ExtLoad,
SDLoc(N),
6217 if (
SDValue ExtLoad = CombineExtLoad(N))
6227 (!LegalOperations && TLI.isOperationLegal(N0.
getOpcode(), VT))) {
6230 bool DoXform =
true;
6244 ExtLoad, DAG.getConstant(Mask, DL, VT));
6250 ExtendSetCCUses(SetCCs, Trunc, ExtLoad, DL,
6263 if ((!LegalOperations && !LN0->
isVolatile()) ||
6269 CombineTo(N, ExtLoad);
6279 if (!LegalOperations && VT.isVector() &&
6290 DAG.getConstant(1, DL, EltVT));
6297 return DAG.getNode(
ISD::AND, DL, VT,
6300 cast<CondCodeSDNode>(N0.
getOperand(2))->
get()),
6307 EVT MatchingElementType =
6310 EVT MatchingVectorType =
6314 DAG.getSetCC(DL, MatchingVectorType, N0.
getOperand(0),
6316 cast<CondCodeSDNode>(N0.
getOperand(2))->
get());
6318 DAG.getSExtOrTrunc(VsetCC, DL, VT),
6326 DAG.getConstant(1, DL, VT), DAG.getConstant(0, DL, VT),
6327 cast<CondCodeSDNode>(N0.
getOperand(2))->
get(),
true);
6328 if (SCC.
getNode())
return SCC;
6337 unsigned ShAmtVal = cast<ConstantSDNode>(ShAmt)->getZExtValue();
6344 if (ShAmtVal > KnownZeroBits)
6351 if (VT.getSizeInBits() >= 256)
6385 CombineTo(N0.
getNode(), NarrowLoad);
6419 return DAG.getNode(
ISD::AND, DL, VT,
6420 X, DAG.getConstant(Mask, DL, VT));
6430 bool DoXform =
true;
6440 CombineTo(N, ExtLoad);
6444 ExtendSetCCUses(SetCCs, Trunc, ExtLoad,
SDLoc(N),
6459 if (!LegalOperations || TLI.isLoadExtLegal(ExtType, VT, MemVT)) {
6463 CombineTo(N, ExtLoad);
6478 if (VT.
isVector() && !LegalOperations) {
6488 cast<CondCodeSDNode>(N0.
getOperand(2))->
get());
6497 cast<CondCodeSDNode>(N0.
getOperand(2))->
get());
6498 return DAG.getAnyExtOrTrunc(VsetCC,
SDLoc(N), VT);
6506 DAG.getConstant(1, DL, VT), DAG.getConstant(0, DL, VT),
6507 cast<CondCodeSDNode>(N0.
getOperand(2))->
get(),
true);
6523 assert(CV &&
"Const value should be ConstSDNode.");
6525 APInt NewVal = CVal & Mask;
6533 if (DAG.MaskedValueIsZero(V.
getOperand(0), Mask))
6535 if (DAG.MaskedValueIsZero(V.
getOperand(1), Mask))
6544 unsigned Amt = RHSC->getZExtValue();
6547 if (Amt >= Mask.getBitWidth())
break;
6548 APInt NewMask = Mask << Amt;
6579 ExtVT = cast<VTSDNode>(N->
getOperand(1))->getVT();
6589 if (LegalOperations && !TLI.isLoadExtLegal(ExtType, VT, ExtVT))
6604 if ((ShAmt & (EVTBits-1)) == 0) {
6612 if (!isa<LoadSDNode>(N0))
return SDValue();
6617 if (cast<LoadSDNode>(N0)->getExtensionType() ==
ISD::SEXTLOAD)
6623 if (ShAmt >= cast<LoadSDNode>(N0)->getMemoryVT().getSizeInBits())
6630 unsigned ShLeftAmt = 0;
6632 ExtVT == VT && TLI.isNarrowingProfitable(N0.
getValueType(), VT)) {
6641 if (!isa<LoadSDNode>(N0) || !N0.
hasOneUse())
6668 if (!TLI.shouldReduceLoadWidth(LN0, ExtType, ExtVT))
6679 if (DAG.getDataLayout().isBigEndian()) {
6682 ShAmt = LVTStoreBits - EVTStoreBits - ShAmt;
6685 uint64_t PtrOff = ShAmt / 8;
6690 DAG.getConstant(PtrOff, DL, PtrType));
6691 AddToWorklist(NewPtr.
getNode());
6700 Load = DAG.getExtLoad(ExtType,
SDLoc(N0), VT, LN0->
getChain(),NewPtr,
6706 WorklistRemover DeadNodes(*
this);
6711 if (ShLeftAmt != 0) {
6721 Result = DAG.getConstant(0, DL, VT);
6724 Result, DAG.getConstant(ShLeftAmt, DL, ShImmTy));
6735 EVT EVT = cast<VTSDNode>(N1)->getVT();
6744 if (DAG.ComputeNumSignBits(N0) >= VTBits-EVTBits+1)
6765 return DAG.getZeroExtendInReg(N0,
SDLoc(N), EVT);
6774 SDValue NarrowLoad = ReduceLoadWidth(N);
6783 if (ShAmt->getZExtValue()+EVTBits <= VTBits) {
6786 unsigned InSignBits = DAG.ComputeNumSignBits(N0.
getOperand(0));
6787 if (VTBits-(ShAmt->getZExtValue()+EVTBits) < InSignBits)
6796 EVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
6797 ((!LegalOperations && !cast<LoadSDNode>(N0)->
isVolatile()) ||
6804 CombineTo(N, ExtLoad);
6805 CombineTo(N0.
getNode(), ExtLoad, ExtLoad.getValue(1));
6806 AddToWorklist(ExtLoad.getNode());
6812 EVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
6813 ((!LegalOperations && !cast<LoadSDNode>(N0)->
isVolatile()) ||
6820 CombineTo(N, ExtLoad);
6821 CombineTo(N0.
getNode(), ExtLoad, ExtLoad.getValue(1));
6839 unsigned ShAmt = VTBits - EVTBits;
6841 for (
unsigned i = 0; i != NumElts; ++i) {
6850 Elts.
push_back(DAG.getConstant(C.shl(ShAmt).ashr(ShAmt).getZExtValue(),
6860 SDValue DAGCombiner::visitSIGN_EXTEND_VECTOR_INREG(
SDNode *N) {
6865 return DAG.getUNDEF(VT);
6877 bool isLE = DAG.getDataLayout().isLittleEndian();
6928 if (isa<ConstantSDNode>(EltNo) &&
isTypeLegal(NVT)) {
6929 int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
6930 EVT IndexTy = TLI.getVectorIdxTy(DAG.getDataLayout());
6931 int Index = isLE ? (Elt*SizeRatio) : (Elt*SizeRatio + (SizeRatio-1));
6939 DAG.getConstant(Index, DL, IndexTy));
6946 if ((!LegalOperations || TLI.isOperationLegal(
ISD::SELECT, SrcVT)) &&
6947 TLI.isTruncateFree(SrcVT, VT)) {
6970 if (BuildVectEltTy == TruncVecEltTy) {
6974 unsigned TruncEltOffset = BuildVecNumElts / TruncVecNumElts;
6976 assert((BuildVecNumElts % TruncVecNumElts) == 0 &&
6977 "Invalid number of elements");
6980 for (
unsigned i = 0, e = BuildVecNumElts; i != e; i += TruncEltOffset)
7001 if (!LegalTypes || TLI.isTypeDesirableForOp(N0.
getOpcode(), VT)) {
7002 SDValue Reduced = ReduceLoadWidth(N);
7026 unsigned NumDefs = 0;
7044 return DAG.getUNDEF(VT);
7047 assert(V.
getNode() &&
"The single defined operand is empty!");
7049 for (
unsigned i = 0, e = VTs.
size(); i != e; ++i) {
7056 Opnds.push_back(NV);
7079 SDValue DAGCombiner::CombineConsecutiveLoads(
SDNode *N, EVT VT) {
7085 LD1->getAddressSpace() != LD2->getAddressSpace())
7087 EVT LD1VT = LD1->getValueType(0);
7093 !LD1->isVolatile() &&
7094 !LD2->isVolatile() &&
7095 DAG.isConsecutiveLoad(LD2, LD1, LD1VT.
getSizeInBits()/8, 1)) {
7096 unsigned Align = LD1->getAlignment();
7097 unsigned NewAlign = DAG.getDataLayout().getABITypeAlignment(
7100 if (NewAlign <= Align &&
7101 (!LegalOperations || TLI.isOperationLegal(
ISD::LOAD, VT)))
7102 return DAG.getLoad(VT,
SDLoc(N), LD1->getChain(),
7103 LD1->getBasePtr(), LD1->getPointerInfo(),
7104 false,
false,
false,
Align);
7121 bool isSimple = cast<BuildVectorSDNode>(N0)->isConstant();
7125 "Element type of vector ValueType must not be vector!");
7127 return ConstantFoldBITCASTofBUILD_VECTOR(N0.
getNode(), DestEltVT);
7131 if (isa<ConstantSDNode>(N0) || isa<ConstantFPSDNode>(N0)) {
7135 if (!LegalOperations ||
7154 TLI.hasBigEndianPartOrdering(N0.
getValueType(), DAG.getDataLayout()) ==
7155 TLI.hasBigEndianPartOrdering(VT, DAG.getDataLayout()) &&
7156 (!LegalOperations || TLI.isOperationLegal(
ISD::LOAD, VT)) &&
7159 unsigned Align = DAG.getDataLayout().getABITypeAlignment(
7163 if (Align <= OrigAlign) {
7183 AddToWorklist(NewConv.
getNode());
7188 return DAG.getNode(
ISD::XOR, DL, VT,
7189 NewConv, DAG.getConstant(SignBit, DL, VT));
7191 return DAG.getNode(
ISD::AND, DL, VT,
7192 NewConv, DAG.getConstant(~SignBit, DL, VT));
7211 if (OrigXWidth < VTWidth) {
7214 }
else if (OrigXWidth > VTWidth) {
7220 DAG.getConstant(OrigXWidth-VTWidth, DL,
7224 AddToWorklist(X.getNode());
7229 X, DAG.getConstant(SignBit,
SDLoc(X), VT));
7230 AddToWorklist(X.getNode());
7235 Cst, DAG.getConstant(~SignBit,
SDLoc(Cst), VT));
7236 AddToWorklist(Cst.getNode());
7261 auto PeekThroughBitcast = [&](
SDValue Op) {
7280 for (
int i = 0; i != MaskScale; ++i)
7281 NewMask.
push_back(M < 0 ? -1 : M * MaskScale + i);
7283 bool LegalMask = TLI.isShuffleMaskLegal(NewMask, VT);
7287 LegalMask = TLI.isShuffleMaskLegal(NewMask, VT);
7291 return DAG.getVectorShuffle(VT,
SDLoc(N), SV0, SV1, NewMask);
7299 return CombineConsecutiveLoads(N, VT);
7305 ConstantFoldBITCASTofBUILD_VECTOR(
SDNode *BV, EVT DstEltVT) {
7309 if (SrcEltVT == DstEltVT)
return SDValue(BV, 0);
7316 if (SrcBitSize == DstBitSize) {
7347 BV = ConstantFoldBITCASTofBUILD_VECTOR(BV, IntVT).getNode();
7355 SDNode *Tmp = ConstantFoldBITCASTofBUILD_VECTOR(BV, TmpVT).
getNode();
7358 return ConstantFoldBITCASTofBUILD_VECTOR(Tmp, DstEltVT);
7366 if (SrcBitSize < DstBitSize) {
7367 unsigned NumInputsPerOutput = DstBitSize/SrcBitSize;
7370 for (
unsigned i = 0, e = BV->getNumOperands(); i != e;
7371 i += NumInputsPerOutput) {
7372 bool isLE = DAG.getDataLayout().isLittleEndian();
7374 bool EltIsUndef =
true;
7375 for (
unsigned j = 0; j != NumInputsPerOutput; ++j) {
7377 NewBits <<= SrcBitSize;
7382 NewBits |= cast<ConstantSDNode>(Op)->getAPIntValue().
7383 zextOrTrunc(SrcBitSize).
zext(DstBitSize);
7389 Ops.
push_back(DAG.getConstant(NewBits, DL, DstEltVT));
7398 unsigned NumOutputsPerInput = SrcBitSize/DstBitSize;
7400 NumOutputsPerInput*BV->getNumOperands());
7405 Ops.
append(NumOutputsPerInput, DAG.getUNDEF(DstEltVT));
7409 APInt OpVal = cast<ConstantSDNode>(Op)->
7412 for (
unsigned j = 0; j != NumOutputsPerInput; ++j) {
7414 Ops.push_back(DAG.getConstant(ThisVal, DL, DstEltVT));
7415 OpVal = OpVal.
lshr(DstBitSize);
7419 if (DAG.getDataLayout().isBigEndian())
7420 std::reverse(Ops.end()-NumOutputsPerInput, Ops.end());
7438 bool HasFMAD = (LegalOperations &&
7442 bool HasFMA = ((!LegalOperations ||
7443 TLI.isOperationLegalOrCustom(
ISD::FMA, VT)) &&
7444 TLI.isFMAFasterThanFMulAndFAdd(VT) &&
7448 if (!HasFMAD && !HasFMA)
7453 bool Aggressive = TLI.enableAggressiveFMAFusion(VT);
7454 bool LookThroughFPExt = TLI.isFPExtFree(VT);
7459 return DAG.getNode(PreferredFusedOpcode, SL, VT,
7467 return DAG.getNode(PreferredFusedOpcode, SL, VT,
7472 if (UnsafeFPMath && LookThroughFPExt) {
7477 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
7489 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
7498 if ((UnsafeFPMath || HasFMAD) &&
Aggressive) {
7500 if (N0.
getOpcode() == PreferredFusedOpcode &&
7502 return DAG.getNode(PreferredFusedOpcode, SL, VT,
7504 DAG.
getNode(PreferredFusedOpcode, SL, VT,
7511 if (N1->
getOpcode() == PreferredFusedOpcode &&
7513 return DAG.getNode(PreferredFusedOpcode, SL, VT,
7515 DAG.
getNode(PreferredFusedOpcode, SL, VT,
7521 if (UnsafeFPMath && LookThroughFPExt) {
7524 auto FoldFAddFMAFPExtFMul = [&] (
7526 return DAG.getNode(PreferredFusedOpcode, SL, VT, X, Y,
7527 DAG.
getNode(PreferredFusedOpcode, SL, VT,
7532 if (N0.
getOpcode() == PreferredFusedOpcode) {
7548 auto FoldFAddFPExtFMAFMul = [&] (
7550 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
7553 DAG.getNode(PreferredFusedOpcode, SL, VT,
7560 if (N00.
getOpcode() == PreferredFusedOpcode) {
7571 if (N1.
getOpcode() == PreferredFusedOpcode) {
7589 if (N10.
getOpcode() == PreferredFusedOpcode) {
7615 bool HasFMAD = (LegalOperations &&
7619 bool HasFMA = ((!LegalOperations ||
7620 TLI.isOperationLegalOrCustom(
ISD::FMA, VT)) &&
7621 TLI.isFMAFasterThanFMulAndFAdd(VT) &&
7625 if (!HasFMAD && !HasFMA)
7630 bool Aggressive = TLI.enableAggressiveFMAFusion(VT);
7631 bool LookThroughFPExt = TLI.isFPExtFree(VT);
7636 return DAG.getNode(PreferredFusedOpcode, SL, VT,
7645 return DAG.getNode(PreferredFusedOpcode, SL, VT,
7656 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
7657 DAG.getNode(
ISD::FNEG, SL, VT, N00), N01,
7662 if (UnsafeFPMath && LookThroughFPExt) {
7668 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
7682 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
7703 DAG.getNode(PreferredFusedOpcode, SL, VT,
7725 DAG.getNode(PreferredFusedOpcode, SL, VT,
7738 if ((UnsafeFPMath || HasFMAD) &&
Aggressive) {
7741 if (N0.
getOpcode() == PreferredFusedOpcode &&
7743 return DAG.getNode(PreferredFusedOpcode, SL, VT,
7745 DAG.
getNode(PreferredFusedOpcode, SL, VT,
7754 if (N1.
getOpcode() == PreferredFusedOpcode &&
7758 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
7762 DAG.
getNode(PreferredFusedOpcode, SL, VT,
7768 if (UnsafeFPMath && LookThroughFPExt) {
7771 if (N0.
getOpcode() == PreferredFusedOpcode) {
7776 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
7778 DAG.
getNode(PreferredFusedOpcode, SL, VT,
7796 if (N00.
getOpcode() == PreferredFusedOpcode) {
7799 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
7804 DAG.getNode(PreferredFusedOpcode, SL, VT,
7816 if (N1.
getOpcode() == PreferredFusedOpcode &&
7822 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
7825 DAG.
getNode(PreferredFusedOpcode, SL, VT,
7849 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
7854 DAG.getNode(PreferredFusedOpcode, SL, VT,
7880 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
7885 return DAG.getNode(
ISD::FADD, DL, VT, N0, N1);
7888 if (N0CFP && !N1CFP)
7889 return DAG.getNode(
ISD::FADD, DL, VT, N1, N0);
7892 if ((!LegalOperations || TLI.isOperationLegalOrCustom(
ISD::FSUB, VT)) &&
7894 return DAG.getNode(
ISD::FSUB, DL, VT, N0,
7898 if ((!LegalOperations || TLI.isOperationLegalOrCustom(
ISD::FSUB, VT)) &&
7900 return DAG.getNode(
ISD::FSUB, DL, VT, N1,
7910 if (N1CFP && N1CFP->isZero())
7921 return DAG.getConstantFP(0.0, DL, VT);
7925 return DAG.getConstantFP(0.0, DL, VT);
7930 if (TLI.isOperationLegalOrCustom(
ISD::FMUL, VT) && !N0CFP && !N1CFP) {
7936 if (CFP01 && !CFP00 && N0.
getOperand(0) == N1) {
7938 DAG.getConstantFP(1.0, DL, VT));
7939 return DAG.getNode(
ISD::FMUL, DL, VT, N1, NewCFP);
7947 DAG.getConstantFP(2.0, DL, VT));
7957 if (CFP11 && !CFP10 && N1.
getOperand(0) == N0) {
7959 DAG.getConstantFP(1.0, DL, VT));
7960 return DAG.getNode(
ISD::FMUL, DL, VT, N0, NewCFP);
7968 DAG.getConstantFP(2.0, DL, VT));
7979 N1, DAG.getConstantFP(3.0, DL, VT));
7989 N0, DAG.getConstantFP(3.0, DL, VT));
7994 if (AllowNewConst &&
8000 N0.
getOperand(0), DAG.getConstantFP(4.0, DL, VT));
8006 SDValue Fused = visitFADDForFMACombine(N);
8008 AddToWorklist(Fused.
getNode());
8026 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
8031 return DAG.getNode(
ISD::FSUB, dl, VT, N0, N1);
8035 return DAG.getNode(
ISD::FADD, dl, VT, N0,
8041 if (N1CFP && N1CFP->
isZero())
8045 if (N0CFP && N0CFP->
isZero()) {
8048 if (!LegalOperations || TLI.isOperationLegal(
ISD::FNEG, VT))
8054 return DAG.getConstantFP(0.0f, dl, VT);
8071 SDValue Fused = visitFSUBForFMACombine(N);
8073 AddToWorklist(Fused.
getNode());
8092 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
8098 return DAG.getNode(
ISD::FMUL, DL, VT, N0, N1);
8103 return DAG.getNode(
ISD::FMUL, DL, VT, N1, N0);
8111 if (N1CFP && N1CFP->
isZero())
8131 (BV1 && BV01 && BV1->isConstant() && BV01->isConstant())) {
8133 return DAG.getNode(
ISD::FMUL, DL, VT, N00, MulConsts);
8143 const SDValue Two = DAG.getConstantFP(2.0, DL, VT);
8151 return DAG.getNode(
ISD::FADD, DL, VT, N0, N0);
8155 if (!LegalOperations || TLI.isOperationLegal(
ISD::FNEG, VT))
8156 return DAG.getNode(
ISD::FNEG, DL, VT, N0);
8163 if (LHSNeg == 2 || RHSNeg == 2)
8184 if (isa<ConstantFPSDNode>(N0) &&
8185 isa<ConstantFPSDNode>(N1) &&
8186 isa<ConstantFPSDNode>(N2)) {
8187 return DAG.getNode(
ISD::FMA, dl, VT, N0, N1, N2);
8191 if (N0CFP && N0CFP->
isZero())
8193 if (N1CFP && N1CFP->isZero())
8198 if (N1CFP && N1CFP->isExactlyValue(1.0))
8202 if (N0CFP && !N1CFP)
8210 return DAG.getNode(
ISD::FMUL, dl, VT, N0,
8219 return DAG.getNode(
ISD::FMA, dl, VT,
8228 if (N1CFP->isExactlyValue(1.0))
8229 return DAG.getNode(
ISD::FADD, dl, VT, N0, N2);
8231 if (N1CFP->isExactlyValue(-1.0) &&
8232 (!LegalOperations || TLI.isOperationLegal(
ISD::FNEG, VT))) {
8234 AddToWorklist(RHSNeg.
getNode());
8235 return DAG.getNode(
ISD::FADD, dl, VT, N2, RHSNeg);
8241 return DAG.getNode(
ISD::FMUL, dl, VT, N0,
8243 N1, DAG.getConstantFP(1.0, dl, VT)));
8248 return DAG.getNode(
ISD::FMUL, dl, VT, N0,
8250 N1, DAG.getConstantFP(-1.0, dl, VT)));
8267 if (
SDValue FoldedVOp = SimplifyVBinOp(N))
8278 APFloat N1APF = N1CFP->getValueAPF();
8284 (!LegalOperations ||
8289 TLI.isFPImmLegal(Recip, VT)))
8290 return DAG.getNode(
ISD::FMUL, DL, VT, N0,
8291 DAG.getConstantFP(Recip, DL, VT));
8298 return DAG.getNode(
ISD::FMUL, DL, VT, N0, RV);
8305 return DAG.getNode(
ISD::FMUL, DL, VT, N0, RV);
8312 return DAG.getNode(
ISD::FMUL, DL, VT, N0, RV);
8332 return DAG.getNode(
ISD::FMUL, DL, VT, N0, RV);
8338 if (
SDValue RV = BuildReciprocalEstimate(N1)) {
8340 return DAG.getNode(
ISD::FMUL, DL, VT, N0, RV);
8349 if (LHSNeg == 2 || RHSNeg == 2)
8371 for (
auto *U : N1->
uses())
8372 if (U->getOpcode() ==
ISD::FDIV && U->getOperand(1) == N1)
8375 if (TLI.combineRepeatedFPDivisors(Users.
size())) {
8376 SDValue FPOne = DAG.getConstantFP(1.0, DL, VT);
8383 for (
auto *U : Users) {
8385 if (Dividend != FPOne) {
8388 CombineTo(U, NewNode);
8389 }
else if (U != Reciprocal.
getNode()) {
8392 CombineTo(U, Reciprocal);
8417 if (!DAG.getTarget().Options.UnsafeFPMath || TLI.isFsqrtCheap())
8432 SDValue Zero = DAG.getConstantFP(0.0, DL, VT);
8433 EVT CCVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
8435 AddToWorklist(ZeroCmp.
getNode());
8453 const APFloat& V = N1CFP->getValueAPF();
8457 if (!LegalOperations || TLI.isOperationLegal(
ISD::FABS, VT))
8460 if (!LegalOperations || TLI.isOperationLegal(
ISD::FNEG, VT))
8500 (!LegalOperations ||
8509 if (DAG.SignBitIsZero(N0))
8514 if (TLI.isOperationLegalOrCustom(
ISD::SELECT_CC, VT) || !LegalOperations) {
8518 (!LegalOperations ||
8523 DAG.getConstantFP(-1.0, DL, VT), DAG.getConstantFP(0.0, DL, VT),
8532 (!LegalOperations ||
8537 DAG.getConstantFP(1.0, DL, VT), DAG.getConstantFP(0.0, DL, VT),
8554 (!LegalOperations ||
8563 if (DAG.SignBitIsZero(N0))
8568 if (TLI.isOperationLegalOrCustom(
ISD::SELECT_CC, VT) || !LegalOperations) {
8572 (!LegalOperations ||
8577 DAG.getConstantFP(1.0, DL, VT), DAG.getConstantFP(0.0, DL, VT),
8610 unsigned ActualSize =
std::min(InputSize, OutputSize);
8675 if (DAG.getTarget().Options.UnsafeFPMath || N0IsTrunc) {
8678 DAG.getIntPtrConstant(NIsTrunc && N0IsTrunc, DL));
8697 EVT EVT = cast<VTSDNode>(N->
getOperand(1))->getVT();
8748 CombineTo(N, ExtLoad);
8752 DAG.getIntPtrConstant(1,
SDLoc(N0))),
8753 ExtLoad.getValue(1));
8803 &DAG.getTarget().Options))
8808 if (!TLI.isFNegFree(VT) &&
8826 DAG.getConstant(SignMask, DL0, IntVT));
8857 if (N0CFP && N1CFP) {
8859 const APFloat &C1 = N1CFP->getValueAPF();
8878 if (N0CFP && N1CFP) {
8880 const APFloat &C1 = N1CFP->getValueAPF();
8912 if (!TLI.isFAbsFree(VT) &&
8930 DAG.getConstant(SignMask, DL, IntVT));
8996 const APInt &AndConst = cast<ConstantSDNode>(AndOp1)->getAPIntValue();
8999 cast<ConstantSDNode>(Op1)->getAPIntValue()==AndConst.
logBase2()) {
9011 CombineTo(N, NewBRCond,
false);
9014 deleteAndRecombine(Trunc);
9016 WorklistRemover DeadNodes(*
this);
9017 DAG.ReplaceAllUsesOfValueWith(N1, SetCC);
9018 deleteAndRecombine(N1.
getNode());
9037 SDValue Tmp = visitXOR(TheXor);
9039 if (Tmp.
getNode() != TheXor) {
9042 dbgs() <<
"\nWith: ";
9045 WorklistRemover DeadNodes(*
this);
9046 DAG.ReplaceAllUsesOfValueWith(N1, Tmp);
9047 deleteAndRecombine(TheXor);
9074 WorklistRemover DeadNodes(*
this);
9075 DAG.ReplaceAllUsesOfValueWith(N1, SetCC);
9076 deleteAndRecombine(N1.
getNode());
9099 CondLHS, CondRHS, CC->
get(),
SDLoc(N),
9121 if (
LoadSDNode *LD = dyn_cast<LoadSDNode>(Use)) {
9127 if (
ST->isIndexed() ||
ST->getBasePtr().getNode() !=
N)
9129 VT =
ST->getMemoryVT();
9130 AS =
ST->getAddressSpace();
9163 bool DAGCombiner::CombineToPreIndexedLoadStore(
SDNode *N) {
9170 if (
LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
9179 if (
ST->isIndexed())
9181 VT =
ST->getMemoryVT();
9185 Ptr =
ST->getBasePtr();
9201 if (!TLI.getPreIndexedAddressParts(N, BasePtr, Offset, AM, DAG))
9207 bool Swapped =
false;
9208 if (isa<ConstantSDNode>(BasePtr)) {
9227 if (isa<FrameIndexSDNode>(BasePtr) || isa<RegisterSDNode>(BasePtr))
9232 SDValue Val = cast<StoreSDNode>(
N)->getValue();
9241 if (isa<ConstantSDNode>(Offset))
9245 SDUse &Use = UI.getUse();
9261 if (!isa<ConstantSDNode>(Op1)) {
9279 bool RealUse =
false;
9303 BasePtr, Offset, AM);
9306 BasePtr, Offset, AM);
9311 dbgs() <<
"\nWith: ";
9314 WorklistRemover DeadNodes(*
this);
9323 deleteAndRecombine(N);
9329 for (
unsigned i = 0, e = OtherUses.
size(); i != e; ++i) {
9330 unsigned OffsetIdx = 1;
9331 if (OtherUses[i]->getOperand(OffsetIdx).getNode() == BasePtr.
getNode())
9333 assert(OtherUses[i]->getOperand(!OffsetIdx).getNode() ==
9334 BasePtr.
getNode() &&
"Expected BasePtr operand");
9348 cast<ConstantSDNode>(OtherUses[i]->getOperand(OffsetIdx));
9350 APInt Offset0 = CN->getAPIntValue();
9351 APInt Offset1 = cast<ConstantSDNode>(Offset)->getAPIntValue();
9353 X0 = (OtherUses[i]->getOpcode() ==
ISD::SUB && OffsetIdx == 1) ? -1 : 1;
9354 Y0 = (OtherUses[i]->getOpcode() ==
ISD::SUB && OffsetIdx == 0) ? -1 : 1;
9360 APInt CNV = Offset0;
9361 if (X0 < 0) CNV = -CNV;
9362 if (X1 * Y0 * Y1 < 0) CNV = CNV + Offset1;
9363 else CNV = CNV - Offset1;
9368 SDValue NewOp1 = DAG.getConstant(CNV, DL, CN->getValueType(0));
9374 DAG.ReplaceAllUsesOfValueWith(
SDValue(OtherUses[i], 0), NewUse);
9375 deleteAndRecombine(OtherUses[i]);
9379 DAG.ReplaceAllUsesOfValueWith(Ptr, Result.
getValue(isLoad ? 1 : 0));
9380 deleteAndRecombine(Ptr.
getNode());
9389 bool DAGCombiner::CombineToPostIndexedLoadStore(
SDNode *N) {
9396 if (
LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
9405 if (
ST->isIndexed())
9407 VT =
ST->getMemoryVT();
9411 Ptr =
ST->getBasePtr();
9428 if (TLI.getPostIndexedAddressParts(N, Op, BasePtr, Offset, AM, DAG)) {
9440 if (isa<FrameIndexSDNode>(BasePtr) || isa<RegisterSDNode>(BasePtr))
9444 bool TryNext =
false;
9452 bool RealUse =
false;
9453 for (
SDNode *UseUse : Use->uses()) {
9472 BasePtr, Offset, AM)
9474 BasePtr, Offset, AM);
9479 dbgs() <<
"\nWith: ";
9482 WorklistRemover DeadNodes(*
this);
9491 deleteAndRecombine(N);
9494 DAG.ReplaceAllUsesOfValueWith(
SDValue(Op, 0),
9496 deleteAndRecombine(Op);
9516 !cast<ConstantSDNode>(Inc)->isOpaque()) &&
9517 "Cannot split out indexing using opaque target constants");
9549 dbgs() <<
"\nWith chain: ";
9550 Chain.getNode()->dump(&DAG);
9552 WorklistRemover DeadNodes(*
this);
9553 DAG.ReplaceAllUsesOfValueWith(
SDValue(N, 1), Chain);
9556 deleteAndRecombine(N);
9569 cast<ConstantSDNode>(LD->
getOperand(2))->isOpaque();
9576 Index = SplitIndexingFromLoad(LD);
9579 AddUsersToWorklist(N);
9584 dbgs() <<
"\nWith: ";
9586 dbgs() <<
" and 2 other values\n");
9587 WorklistRemover DeadNodes(*
this);
9588 DAG.ReplaceAllUsesOfValueWith(
SDValue(N, 0), Undef);
9589 DAG.ReplaceAllUsesOfValueWith(
SDValue(N, 1), Index);
9590 DAG.ReplaceAllUsesOfValueWith(
SDValue(N, 2), Chain);
9591 deleteAndRecombine(N);
9606 return CombineTo(N, Chain.
getOperand(1), Chain);
9612 if (
unsigned Align = DAG.InferPtrAlignment(Ptr)) {
9622 return CombineTo(N, NewLoad,
SDValue(NewLoad.
getNode(), 1),
true);
9627 bool UseAA = CombinerAA.getNumOccurrences() > 0 ? CombinerAA
9628 : DAG.getSubtarget().useAA();
9630 if (CombinerAAOnlyFunc.getNumOccurrences() &&
9631 CombinerAAOnlyFunc != DAG.getMachineFunction().getName())
9636 SDValue BetterChain = FindBetterChain(N, Chain);
9639 if (Chain != BetterChain) {
9658 AddToWorklist(Token.
getNode());
9662 return CombineTo(N, ReplLoad.
getValue(0), Token,
false);
9667 if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
9691 struct LoadedSlice {
9699 unsigned CrossRegisterBanksCopies;
9703 Cost(
bool ForCodeSize =
false)
9704 : ForCodeSize(ForCodeSize), Loads(0), Truncates(0),
9705 CrossRegisterBanksCopies(0), ZExts(0), Shift(0) {}
9708 Cost(
const LoadedSlice &
LS,
bool ForCodeSize =
false)
9709 : ForCodeSize(ForCodeSize), Loads(1), Truncates(0),
9710 CrossRegisterBanksCopies(0), ZExts(0), Shift(0) {
9711 EVT TruncType = LS.Inst->getValueType(0);
9712 EVT LoadedType = LS.getLoadedType();
9713 if (TruncType != LoadedType &&
9714 !LS.DAG->getTargetLoweringInfo().isZExtFree(LoadedType, TruncType))
9722 void addSliceGain(
const LoadedSlice &LS) {
9726 LS.Inst->getOperand(0).getValueType()))
9732 if (LS.canMergeExpensiveCrossRegisterBankCopy())
9733 ++CrossRegisterBanksCopies;
9738 Truncates += RHS.Truncates;
9739 CrossRegisterBanksCopies += RHS.CrossRegisterBanksCopies;
9746 return Loads == RHS.Loads && Truncates == RHS.Truncates &&
9747 CrossRegisterBanksCopies == RHS.CrossRegisterBanksCopies &&
9748 ZExts == RHS.ZExts && Shift == RHS.Shift;
9751 bool operator!=(
const Cost &RHS)
const {
return !(*
this == RHS); }
9756 unsigned ExpensiveOpsLHS = Loads + CrossRegisterBanksCopies;
9757 unsigned ExpensiveOpsRHS = RHS.Loads + RHS.CrossRegisterBanksCopies;
9760 if (!ForCodeSize && ExpensiveOpsLHS != ExpensiveOpsRHS)
9761 return ExpensiveOpsLHS < ExpensiveOpsRHS;
9762 return (Truncates + ZExts + Shift + ExpensiveOpsLHS) <
9763 (RHS.Truncates + RHS.ZExts + RHS.Shift + ExpensiveOpsRHS);
9766 bool operator>(
const Cost &RHS)
const {
return RHS < *
this; }
9768 bool operator<=(
const Cost &RHS)
const {
return !(RHS < *
this); }
9770 bool operator>=(
const Cost &RHS)
const {
return !(*
this < RHS); }
9785 : Inst(Inst), Origin(Origin), Shift(Shift), DAG(DAG) {}
9790 APInt getUsedBits()
const {
9795 assert(Origin &&
"No original load to compare against.");
9796 unsigned BitWidth = Origin->getValueSizeInBits(0);
9797 assert(Inst &&
"This slice is not bound to an instruction");
9798 assert(Inst->getValueSizeInBits(0) <= BitWidth &&
9799 "Extracted slice is bigger than the whole type!");
9800 APInt UsedBits(Inst->getValueSizeInBits(0), 0);
9802 UsedBits = UsedBits.zext(BitWidth);
9808 unsigned getLoadedSize()
const {
9809 unsigned SliceSize = getUsedBits().countPopulation();
9810 assert(!(SliceSize & 0x7) &&
"Size is not a multiple of a byte.");
9811 return SliceSize / 8;
9816 EVT getLoadedType()
const {
9817 assert(DAG &&
"Missing context");
9823 unsigned getAlignment()
const {
9824 unsigned Alignment = Origin->getAlignment();
9825 unsigned Offset = getOffsetFromBase();
9827 Alignment =
MinAlign(Alignment, Alignment + Offset);
9832 bool isLegal()
const {
9834 if (!Origin || !Inst || !DAG)
9838 if (Origin->getOffset().getOpcode() !=
ISD::UNDEF)
9844 EVT SliceType = getLoadedType();
9854 EVT PtrType = Origin->getBasePtr().getValueType();
9867 EVT TruncateType = Inst->getValueType(0);
9868 if (TruncateType != SliceType &&
9878 uint64_t getOffsetFromBase()
const {
9879 assert(DAG &&
"Missing context.");
9880 bool IsBigEndian = DAG->getDataLayout().isBigEndian();
9881 assert(!(Shift & 0x7) &&
"Shifts not aligned on Bytes are not supported.");
9882 uint64_t Offset = Shift / 8;
9884 assert(!(Origin->getValueSizeInBits(0) & 0x7) &&
9885 "The size of the original loaded type is not a multiple of a"
9889 assert(TySizeInBytes > Offset &&
9890 "Invalid shift amount for given loaded size");
9892 Offset = TySizeInBytes - Offset - getLoadedSize();
9903 assert(Inst && Origin &&
"Unable to replace a non-existing slice.");
9904 const SDValue &OldBaseAddr = Origin->getBasePtr();
9905 SDValue BaseAddr = OldBaseAddr;
9907 int64_t Offset =
static_cast<int64_t
>(getOffsetFromBase());
9908 assert(Offset >= 0 &&
"Offset too big to fit in int64_t!");
9913 BaseAddr = DAG->getNode(
ISD::ADD, DL, ArithType, BaseAddr,
9914 DAG->getConstant(Offset, DL, ArithType));
9918 EVT SliceType = getLoadedType();
9921 SDValue LastInst = DAG->getLoad(
9922 SliceType,
SDLoc(Origin), Origin->getChain(), BaseAddr,
9923 Origin->getPointerInfo().getWithOffset(Offset), Origin->isVolatile(),
9924 Origin->isNonTemporal(), Origin->isInvariant(), getAlignment());
9927 EVT FinalType = Inst->getValueType(0);
9928 if (SliceType != FinalType)
9938 bool canMergeExpensiveCrossRegisterBankCopy()
const {
9939 if (!Inst || !Inst->hasOneUse())
9941 SDNode *Use = *Inst->use_begin();
9944 assert(DAG &&
"Missing context");
9963 unsigned RequiredAlignment = DAG->getDataLayout().getABITypeAlignment(
9966 if (RequiredAlignment > getAlignment())
9974 if (Inst->getValueType(0) != getLoadedType())
10002 const LoadedSlice &Second) {
10003 assert(First.Origin == Second.Origin && First.Origin &&
10004 "Unable to match different memory origins.");
10005 APInt UsedBits = First.getUsedBits();
10006 assert((UsedBits & Second.getUsedBits()) == 0 &&
10007 "Slices are not supposed to overlap.");
10008 UsedBits |= Second.getUsedBits();
10017 LoadedSlice::Cost &GlobalLSCost) {
10018 unsigned NumberOfSlices = LoadedSlices.
size();
10020 if (NumberOfSlices < 2)
10025 std::sort(LoadedSlices.
begin(), LoadedSlices.
end(),
10026 [](
const LoadedSlice &LHS,
const LoadedSlice &RHS) {
10027 assert(LHS.Origin == RHS.Origin &&
"Different bases not implemented.");
10028 return LHS.getOffsetFromBase() < RHS.getOffsetFromBase();
10030 const TargetLowering &TLI = LoadedSlices[0].DAG->getTargetLoweringInfo();
10033 const LoadedSlice *First =
nullptr;
10034 const LoadedSlice *Second =
nullptr;
10035 for (
unsigned CurrSlice = 0; CurrSlice < NumberOfSlices; ++CurrSlice,
10039 Second = &LoadedSlices[CurrSlice];
10046 EVT LoadedType = First->getLoadedType();
10049 if (LoadedType != Second->getLoadedType())
10053 unsigned RequiredAlignment = 0;
10054 if (!TLI.hasPairedLoad(LoadedType, RequiredAlignment)) {
10060 if (RequiredAlignment > First->getAlignment())
10067 assert(GlobalLSCost.Loads > 0 &&
"We save more loads than we created!");
10068 --GlobalLSCost.Loads;
10085 const APInt &UsedBits,
bool ForCodeSize) {
10086 unsigned NumberOfSlices = LoadedSlices.
size();
10087 if (StressLoadSlicing)
10088 return NumberOfSlices > 1;
10091 if (NumberOfSlices != 2)
10099 LoadedSlice::Cost OrigCost(ForCodeSize), GlobalSlicingCost(ForCodeSize);
10101 OrigCost.Loads = 1;
10102 for (
unsigned CurrSlice = 0; CurrSlice < NumberOfSlices; ++CurrSlice) {
10103 const LoadedSlice &LS = LoadedSlices[CurrSlice];
10105 LoadedSlice::Cost SliceCost(LS, ForCodeSize);
10106 GlobalSlicingCost += SliceCost;
10110 OrigCost.addSliceGain(LS);
10115 return OrigCost > GlobalSlicingCost;
10124 bool DAGCombiner::SliceUpLoad(
SDNode *N) {
10143 UI != UIEnd; ++UI) {
10145 if (UI.getUse().getResNo() != 0)
10149 unsigned Shift = 0;
10154 Shift = cast<ConstantSDNode>(User->
getOperand(1))->getZExtValue();
10172 LoadedSlice
LS(User, LD, Shift, &DAG);
10173 APInt CurrentUsedBits = LS.getUsedBits();
10176 if ((CurrentUsedBits & UsedBits) != 0)
10179 UsedBits |= CurrentUsedBits;
10186 LoadedSlices.push_back(LS);
10201 LSIt = LoadedSlices.
begin(),
10202 LSItEnd = LoadedSlices.end();
10203 LSIt != LSItEnd; ++LSIt) {
10204 SDValue SliceInst = LSIt->loadSlice();
10205 CombineTo(LSIt->Inst, SliceInst,
true);
10209 "It takes more than a zext to get to the loaded slice!!");
10215 DAG.ReplaceAllUsesOfValueWith(
SDValue(N, 1), Chain);
10222 static std::pair<unsigned, unsigned>
10224 std::pair<unsigned, unsigned> Result(0, 0);
10245 if (ChainOp.getNode() ==
LD) {
10249 if (!isOk)
return Result;
10261 uint64_t NotMask = ~cast<ConstantSDNode>(V->
getOperand(1))->getSExtValue();
10263 if (NotMaskLZ & 7)
return Result;
10265 if (NotMaskTZ & 7)
return Result;
10266 if (NotMaskLZ == 64)
return Result;
10277 switch (MaskedBytes) {
10281 default:
return Result;
10286 if (NotMaskTZ && NotMaskTZ/8 % MaskedBytes)
return Result;
10288 Result.first = MaskedBytes;
10289 Result.second = NotMaskTZ/8;
10301 unsigned NumBytes = MaskInfo.first;
10302 unsigned ByteShift = MaskInfo.second;
10308 ByteShift*8, (ByteShift+NumBytes)*8);
10315 if (!DC->isTypeLegal(VT))
10332 StOffset = ByteShift;
10341 NewAlign =
MinAlign(NewAlign, StOffset);
10350 false,
false, NewAlign).getNode();
10379 std::pair<unsigned, unsigned> MaskedLoad;
10381 if (MaskedLoad.first)
10388 if (MaskedLoad.first)
10410 APInt Imm = cast<ConstantSDNode>(N1)->getAPIntValue();
10421 while (NewBW < BitWidth &&
10428 if (NewBW >= BitWidth)
10434 ShAmt = (((ShAmt + NewBW - 1) / NewBW) * NewBW) - NewBW;
10436 std::min(BitWidth, ShAmt + NewBW));
10437 if ((Imm & Mask) == Imm) {
10441 uint64_t PtrOff = ShAmt / 8;
10444 if (DAG.getDataLayout().isBigEndian())
10445 PtrOff = (BitWidth + 7 - NewBW) / 8 - PtrOff;
10449 if (NewAlign < DAG.getDataLayout().getABITypeAlignment(NewVTTy))
10454 DAG.getConstant(PtrOff,
SDLoc(LD),
10462 SDValue NewVal = DAG.getNode(Opc,
SDLoc(Value), NewVT, NewLD,
10463 DAG.getConstant(NewImm,
SDLoc(Value),
10468 false,
false, NewAlign);
10470 AddToWorklist(NewPtr.
getNode());
10471 AddToWorklist(NewLD.
getNode());
10472 AddToWorklist(NewVal.
getNode());
10473 WorklistRemover DeadNodes(*
this);
10495 if (!VT.isFloatingPoint() ||
10513 unsigned ABIAlign = DAG.getDataLayout().getABITypeAlignment(IntVTTy);
10514 if (LDAlign < ABIAlign || STAlign < ABIAlign)
10520 false,
false,
false, LDAlign);
10525 false,
false, STAlign);
10527 AddToWorklist(NewLD.
getNode());
10528 AddToWorklist(NewST.
getNode());
10529 WorklistRemover DeadNodes(*
this);
10552 struct BaseIndexOffset {
10556 bool IsIndexSignExt;
10558 BaseIndexOffset() : Offset(0), IsIndexSignExt(
false) {}
10561 bool IsIndexSignExt) :
10562 Base(Base), Index(Index), Offset(Offset), IsIndexSignExt(IsIndexSignExt) {}
10564 bool equalBaseIndex(
const BaseIndexOffset &
Other) {
10565 return Other.Base == Base && Other.Index == Index &&
10566 Other.IsIndexSignExt == IsIndexSignExt;
10571 bool IsIndexSignExt =
false;
10577 return BaseIndexOffset(Ptr,
SDValue(), 0, IsIndexSignExt);
10581 if (isa<ConstantSDNode>(Ptr->
getOperand(1))) {
10582 int64_t Offset = cast<ConstantSDNode>(Ptr->
getOperand(1))->getSExtValue();
10593 return BaseIndexOffset(Ptr,
SDValue(), 0, IsIndexSignExt);
10602 IsIndexSignExt =
true;
10607 return BaseIndexOffset(Base, IndexOffset, 0, IsIndexSignExt);
10613 if (!isa<ConstantSDNode>(Offset))
10614 return BaseIndexOffset(Ptr,
SDValue(), 0, IsIndexSignExt);
10619 IsIndexSignExt =
true;
10620 }
else IsIndexSignExt =
false;
10622 int64_t Off = cast<ConstantSDNode>(Offset)->getSExtValue();
10623 return BaseIndexOffset(Base, Index, Off, IsIndexSignExt);
10635 BuildVector.
push_back(cast<StoreSDNode>(Stores[
I].MemNode)->getValue());
10640 bool DAGCombiner::MergeStoresOfConstantsOrVecElts(
10642 unsigned NumElem,
bool IsConstantSrc,
bool UseVector) {
10649 unsigned LatestNodeUsed = 0;
10651 for (
unsigned i=0; i < NumElem; ++i) {
10656 if (StoreNodes[i].SequenceNum < StoreNodes[LatestNodeUsed].SequenceNum)
10657 LatestNodeUsed = i;
10661 LSBaseSDNode *LatestOp = StoreNodes[LatestNodeUsed].MemNode;
10662 SDLoc DL(StoreNodes[0].MemNode);
10668 assert(TLI.
isTypeLegal(Ty) &&
"Illegal vector store");
10669 if (IsConstantSrc) {
10670 StoredVal = getMergedConstantVectorStore(DAG, DL, StoreNodes, Ty);
10673 for (
unsigned i = 0; i < NumElem ; ++i) {
10674 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
10677 if (Val.getValueType() != MemVT)
10688 assert(IsConstantSrc &&
"Merged vector elements should use vector store");
10690 unsigned SizeInBits = NumElem * ElementSizeBytes * 8;
10691 APInt StoreInt(SizeInBits, 0);
10696 for (
unsigned i = 0; i < NumElem ; ++i) {
10697 unsigned Idx = IsLE ? (NumElem - 1 - i) : i;
10698 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[Idx].MemNode);
10699 SDValue Val = St->getValue();
10700 StoreInt <<= ElementSizeBytes * 8;
10702 StoreInt |= C->getAPIntValue().zext(SizeInBits);
10704 StoreInt |= C->getValueAPF().bitcastToAPInt().zext(SizeInBits);
10712 StoredVal = DAG.
getConstant(StoreInt, DL, StoreTy);
10722 CombineTo(LatestOp, NewStore);
10724 for (
unsigned i = 0; i < NumElem ; ++i) {
10725 if (StoreNodes[i].MemNode == LatestOp)
10727 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
10740 deleteAndRecombine(St);
10748 unsigned AS,
unsigned Align) {
10754 return (Align >= ABIAlignment);
10757 void DAGCombiner::getStoreMergeAndAliasCandidates(
10765 if (!BasePtr.Base.getNode())
10780 if (Index != St && !
SDValue(Index, 0)->hasOneUse())
10787 if (!Ptr.equalBaseIndex(BasePtr))
10795 if (
StoreSDNode *St = dyn_cast<StoreSDNode>(Index))
10804 StoreNodes.
push_back(MemOpLink(Index, Ptr.Offset, Seq++));
10812 if (
StoreSDNode *STn = dyn_cast<StoreSDNode>(NextInChain)) {
10816 }
else if (
LoadSDNode *Ldn = dyn_cast<LoadSDNode>(NextInChain)) {
10817 if (Ldn->isVolatile()) {
10824 NextInChain = Ldn->getChain().getNode();
10834 bool DAGCombiner::MergeConsecutiveStores(
StoreSDNode* St) {
10854 bool IsLoadSrc = isa<LoadSDNode>(StoredVal);
10855 bool IsConstantSrc = isa<ConstantSDNode>(StoredVal) ||
10856 isa<ConstantFPSDNode>(StoredVal);
10859 if (!IsConstantSrc && !IsLoadSrc && !IsExtractVecEltSrc)
10875 getStoreMergeAndAliasCandidates(St, StoreNodes, AliasLoadNodes);
10878 if (StoreNodes.
size() < 2)
10882 std::sort(StoreNodes.
begin(), StoreNodes.
end(),
10883 [](MemOpLink LHS, MemOpLink RHS) {
10884 return LHS.OffsetFromBase < RHS.OffsetFromBase ||
10885 (LHS.OffsetFromBase == RHS.OffsetFromBase &&
10886 LHS.SequenceNum > RHS.SequenceNum);
10891 unsigned LastConsecutiveStore = 0;
10892 int64_t StartAddress = StoreNodes[0].OffsetFromBase;
10893 for (
unsigned i = 0, e = StoreNodes.
size(); i < e; ++i) {
10898 int64_t CurrAddress = StoreNodes[i].OffsetFromBase;
10899 if (CurrAddress - StartAddress != (ElementSizeBytes * i))
10903 bool Alias =
false;
10905 for (
unsigned ld = 0, lde = AliasLoadNodes.
size(); ld < lde; ++ld)
10906 if (isAlias(AliasLoadNodes[ld], StoreNodes[i].MemNode)) {
10915 LastConsecutiveStore = i;
10921 unsigned FirstStoreAlign = FirstInChain->
getAlignment();
10924 if (IsConstantSrc) {
10925 unsigned LastLegalType = 0;
10926 unsigned LastLegalVectorType = 0;
10927 bool NonZero =
false;
10928 for (
unsigned i=0; i<LastConsecutiveStore+1; ++i) {
10929 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
10933 NonZero |= !C->isNullValue();
10935 NonZero |= !C->getConstantFPValue()->isNullValue();
10942 unsigned SizeInBits = (i+1) * ElementSizeBytes * 8;
10946 FirstStoreAlign)) {
10947 LastLegalType = i+1;
10951 EVT LegalizedStoredValueTy =
10955 FirstStoreAlign)) {
10956 LastLegalType = i + 1;
10964 LastLegalVectorType = i + 1;
10973 LastLegalVectorType = 0;
10975 LastLegalVectorType,
10977 LastLegalVectorType = 0;
10981 if (LastLegalType == 0 && LastLegalVectorType == 0)
10984 bool UseVector = (LastLegalVectorType > LastLegalType) && !NoVectors;
10985 unsigned NumElem = UseVector ? LastLegalVectorType : LastLegalType;
10987 return MergeStoresOfConstantsOrVecElts(StoreNodes, MemVT, NumElem,
10993 if (IsExtractVecEltSrc) {
10994 unsigned NumElem = 0;
10995 for (
unsigned i = 0; i < LastConsecutiveStore + 1; ++i) {
10996 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
11013 return MergeStoresOfConstantsOrVecElts(StoreNodes, MemVT, NumElem,
11026 BaseIndexOffset LdBasePtr;
11027 for (
unsigned i=0; i<LastConsecutiveStore+1; ++i) {
11028 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
11033 if (!Ld->hasNUsesOfValue(1, 0))
11037 if (Ld->isVolatile() || Ld->isIndexed())
11045 if (Ld->getMemoryVT() != MemVT)
11050 if (LdBasePtr.Base.getNode()) {
11052 if (!LdPtr.equalBaseIndex(LdBasePtr))
11060 LoadNodes.
push_back(MemOpLink(Ld, LdPtr.Offset, 0));
11063 if (LoadNodes.
size() < 2)
11068 unsigned RequiredAlignment;
11073 LoadSDNode *FirstLoad = cast<LoadSDNode>(LoadNodes[0].MemNode);
11080 unsigned LastConsecutiveLoad = 0;
11082 unsigned LastLegalVectorType = 0;
11083 unsigned LastLegalIntegerType = 0;
11084 StartAddress = LoadNodes[0].OffsetFromBase;
11086 for (
unsigned i = 1; i < LoadNodes.
size(); ++i) {
11088 if (LoadNodes[i].MemNode->getChain() != FirstChain)
11091 int64_t CurrAddress = LoadNodes[i].OffsetFromBase;
11092 if (CurrAddress - StartAddress != (ElementSizeBytes * i))
11094 LastConsecutiveLoad = i;
11101 LastLegalVectorType = i + 1;
11105 unsigned SizeInBits = (i+1) * ElementSizeBytes * 8;
11110 LastLegalIntegerType = i + 1;
11114 EVT LegalizedStoredValueTy =
11121 FirstStoreAlign) &&
11124 LastLegalIntegerType = i+1;
11130 bool UseVectorTy = LastLegalVectorType > LastLegalIntegerType && !NoVectors;
11131 unsigned LastLegalType = std::max(LastLegalVectorType, LastLegalIntegerType);
11135 unsigned NumElem =
std::min(LastConsecutiveStore, LastConsecutiveLoad) + 1;
11136 NumElem =
std::min(LastLegalType, NumElem);
11142 unsigned LatestNodeUsed = 0;
11143 for (
unsigned i=1; i<NumElem; ++i) {
11148 if (StoreNodes[i].SequenceNum < StoreNodes[LatestNodeUsed].SequenceNum)
11149 LatestNodeUsed = i;
11152 LSBaseSDNode *LatestOp = StoreNodes[LatestNodeUsed].MemNode;
11160 unsigned SizeInBits = NumElem * ElementSizeBytes * 8;
11164 SDLoc LoadDL(LoadNodes[0].MemNode);
11165 SDLoc StoreDL(StoreNodes[0].MemNode);
11169 FirstLoad->
getPointerInfo(),
false,
false,
false, FirstLoadAlign);
11176 LoadSDNode *Ld = cast<LoadSDNode>(LoadNodes[0].MemNode);
11178 SDValue(NewLoad.getNode(), 1));
11181 for (
unsigned i = 1; i < NumElem ; ++i) {
11184 LoadSDNode *Ld = cast<LoadSDNode>(LoadNodes[i].MemNode);
11189 CombineTo(LatestOp, NewStore);
11191 for (
unsigned i = 0; i < NumElem ; ++i) {
11193 if (StoreNodes[i].MemNode == LatestOp)
11195 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
11197 deleteAndRecombine(St);
11217 if (Align <= OrigAlign &&
11250 bitcastToAPInt().getZExtValue(),
SDLoc(CFP),
11291 Alignment =
MinAlign(Alignment, 4U);
11295 Alignment, AAInfo);
11315 return CombineTo(ST, NewStore,
true);
11322 SDValue NewST = TransformFPLoadStorePair(N);
11326 bool UseAA = CombinerAA.getNumOccurrences() > 0 ? CombinerAA
11329 if (CombinerAAOnlyFunc.getNumOccurrences() &&
11335 SDValue BetterChain = FindBetterChain(N, Chain);
11338 if (Chain != BetterChain) {
11346 ReplStore = DAG.
getStore(BetterChain,
SDLoc(N), Value, Ptr,
11355 AddToWorklist(Token.
getNode());
11358 return CombineTo(N, Token,
false);
11363 if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
11373 GetDemandedBits(Value,
11377 AddToWorklist(Value.
getNode());
11393 if (
LoadSDNode *Ld = dyn_cast<LoadSDNode>(Value)) {
11398 Chain.reachesChainWithoutSideEffects(
SDValue(Ld, 1))) {
11406 if (
StoreSDNode *ST1 = dyn_cast<StoreSDNode>(Chain)) {
11407 if (ST1->getBasePtr() == Ptr && ST->
getMemoryVT() == ST1->getMemoryVT() &&
11409 ST1->isUnindexed() && !ST1->isVolatile()) {
11428 bool EverChanged =
false;
11434 bool Changed = MergeConsecutiveStores(ST);
11435 EverChanged |= Changed;
11436 if (!Changed)
break;
11443 return ReduceLoadOpStoreWidth(N);
11463 if (!isa<ConstantSDNode>(EltNo))
11465 unsigned Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
11475 && isa<ConstantSDNode>(InVec.
getOperand(2))) {
11476 unsigned OtherElt =
11477 cast<ConstantSDNode>(InVec.
getOperand(2))->getZExtValue();
11478 if (Elt < OtherElt) {
11482 AddToWorklist(NewOp.
getNode());
11505 if (Elt < Ops.
size()) {
11508 EVT OpVT = Ops[0].getValueType();
11520 SDValue DAGCombiner::ReplaceExtractVectorEltOfLoadWithNarrowedLoad(
11538 if (
auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo)) {
11539 int Elt = ConstEltNo->getZExtValue();
11560 if (ResultVT.
bitsGT(VecEltVT)) {
11568 ExtType,
SDLoc(EVE), ResultVT, OriginalLoad->
getChain(), NewPtr, MPI,
11574 VecEltVT,
SDLoc(EVE), OriginalLoad->
getChain(), NewPtr, MPI,
11578 if (ResultVT.
bitsLT(VecEltVT))
11583 WorklistRemover DeadNodes(*
this);
11589 AddToWorklist(Load.
getNode());
11590 AddUsersToWorklist(Load.
getNode());
11592 AddToWorklist(EVE);
11616 bool ConstEltNo = isa<ConstantSDNode>(EltNo);
11626 int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
11638 if (OrigElt < NumElem) {
11642 OrigElt -= NumElem;
11658 if (!LegalOperations) {
11665 bool BCNumEltsChanged =
false;
11683 BCNumEltsChanged =
true;
11689 if (!LegalOperations && !ConstEltNo && InVec.
hasOneUse() &&
11693 if (
LoadSDNode *OrigLoad = dyn_cast<LoadSDNode>(InVec))
11694 return ReplaceExtractVectorEltOfLoadWithNarrowedLoad(N, VT, Index,
11700 if (!LegalOperations)
return SDValue();
11707 int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
11712 LN0 = cast<LoadSDNode>(InVec);
11720 LN0 = cast<LoadSDNode>(InVec.
getOperand(0));
11721 }
else if ((SVN = dyn_cast<ShuffleVectorSDNode>(InVec))) {
11732 if (BCNumEltsChanged)
11737 int Idx = (Elt > (
int)NumElems) ? -1 : SVN->
getMaskElt(Elt);
11748 LN0 = cast<LoadSDNode>(InVec);
11749 Elt = (Idx < (
int)NumElems) ? Idx : Idx - (
int)NumElems;
11763 return ReplaceExtractVectorEltOfLoadWithNarrowedLoad(N, VT, EltNo, LN0);
11770 SDValue DAGCombiner::reduceBuildVecExtToExtBuildVec(
SDNode *N) {
11790 bool AllAnyExt =
true;
11792 for (
unsigned i = 0; i != NumInScalars; ++i) {
11801 if (!ZeroExt && !AnyExt) {
11813 else if (InTy != SourceType) {
11820 AllAnyExt &= AnyExt;
11827 bool ValidTypes = SourceType !=
MVT::Other &&
11838 assert(ElemRatio > 1 &&
"Invalid element size ratio");
11856 unsigned Index = isLE ? (i * ElemRatio) :
11857 (i * ElemRatio + (ElemRatio - 1));
11859 assert(Index < Ops.
size() &&
"Invalid index");
11866 "Invalid vector size");
11879 SDValue DAGCombiner::reduceBuildVecConvertToConvertBuildVec(
SDNode *N) {
11887 unsigned NumDefs = 0;
11889 for (
unsigned i = 0; i != NumInScalars; ++i) {
11922 &&
"Should only handle conversion from integer to float.");
11923 assert(SrcVT !=
MVT::Other &&
"Cannot determine source type!");
11936 for (
unsigned i = 0; i != NumInScalars; ++i) {
11947 return DAG.
getNode(Opcode, dl, VT, BV);
11959 if (
SDValue V = reduceBuildVecExtToExtBuildVec(N))
11962 if (
SDValue V = reduceBuildVecConvertToConvertBuildVec(N))
11978 bool UsesZeroVector =
false;
11979 for (
unsigned i = 0; i != NumInScalars; ++i) {
11986 UsesZeroVector =
true;
11994 VecIn1 = VecIn2 =
SDValue(
nullptr, 0);
12000 if (ExtractedFromVec == VecIn1 || ExtractedFromVec == VecIn2)
12004 VecIn1 = ExtractedFromVec;
12005 }
else if (!VecIn2.
getNode() && !UsesZeroVector) {
12006 VecIn2 = ExtractedFromVec;
12009 VecIn1 = VecIn2 =
SDValue(
nullptr, 0);
12018 for (
unsigned i = 0; i != NumInScalars; ++i) {
12027 assert(UsesZeroVector &&
12029 "Unexpected node found!");
12037 unsigned ExtIndex = cast<ConstantSDNode>(ExtVal)->getZExtValue();
12044 Mask.
push_back(InNumElements + ExtIndex);
12074 VecIn2 =
SDValue(
nullptr, 0);
12082 if (UsesZeroVector || VecIn2.
getNode())
12101 if (UsesZeroVector)
12140 bool AnyInteger =
false;
12141 bool AnyFP =
false;
12241 auto IsBuildVectorOrUndef = [](
const SDValue &Op) {
12244 bool AllBuildVectorsOrUndefs =
12246 if (AllBuildVectorsOrUndefs) {
12254 bool FoundMinVT =
false;
12258 MinVT = (!FoundMinVT || OpSVT.
bitsLE(MinVT)) ? OpSVT : MinVT;
12261 assert(FoundMinVT &&
"Concat vector type mismatch");
12273 assert(SVT == OpVT.
getScalarType() &&
"Concat vector type mismatch");
12276 for (
unsigned i = 0; i != NumElts; ++i)
12284 "Concat vector type mismatch");
12310 if (SingleSource.
getNode()) {
12323 unsigned IdentityIndex = i * PartNumElem;
12335 return SingleSource;
12355 assert((Idx % NumElems) == 0 &&
12356 "IDX in concat is not a multiple of the result vector length.");
12377 if (InsIdx && ExtIdx &&
12411 bool FoundSimplification =
false;
12414 for (
int i = 0, NumOps = V->
getNumOperands(); i < NumOps; ++i) {
12416 bool OpUsed =
false;
12417 for (
int j = 0; j < OpSize; ++j)
12418 if (UsedElements[i * OpSize + j]) {
12419 OpUsedElements[j] =
true;
12425 FoundSimplification |= Op == NewOps.
back();
12426 OpUsedElements.
reset();
12428 if (FoundSimplification)
12441 int Idx = IdxN->getZExtValue();
12442 bool SubVectorUsed =
false;
12444 for (
int i = 0; i < SubSize; ++i)
12445 if (UsedElements[i + Idx]) {
12446 SubVectorUsed =
true;
12447 SubUsedElements[i] =
true;
12448 UsedElements[i + Idx] =
false;
12457 if (SimplifiedSubV != SubV || SimplifiedBaseV != BaseV)
12459 SimplifiedBaseV, SimplifiedSubV, V->
getOperand(2));
12469 SmallBitVector N0UsedElements(NumElts,
false), N1UsedElements(NumElts,
false);
12471 if (M >= 0 && M < NumElts)
12472 N0UsedElements[M] =
true;
12473 else if (M >= NumElts)
12474 N1UsedElements[M - NumElts] =
true;
12478 if (S0 == N0 && S1 == N1)
12497 unsigned NumConcats = NumElts / NumElemsPerConcat;
12504 SVN->
getMask().end(), [](
int i) {
return i == -1; })) {
12513 for (
unsigned I = 0; I != NumConcats; ++
I) {
12515 unsigned Begin = I * NumElemsPerConcat;
12516 bool AllUndef =
true, NoUndef =
true;
12517 for (
unsigned J = Begin; J != Begin + NumElemsPerConcat; ++J) {
12525 if (SVN->
getMaskElt(Begin) % NumElemsPerConcat != 0)
12528 for (
unsigned J = 1; J != NumElemsPerConcat; ++J)
12532 unsigned FirstElt = SVN->
getMaskElt(Begin) / NumElemsPerConcat;
12538 }
else if (AllUndef) {
12555 assert(N0.
getValueType() == VT &&
"Vector shuffle must be normalized in DAG");
12566 for (
unsigned i = 0; i != NumElts; ++i) {
12568 if (Idx >= (
int)NumElts) Idx -= NumElts;
12578 for (
unsigned i = 0; i != NumElts; ++i) {
12581 if (Idx >= (
int)NumElts)
12594 bool Changed =
false;
12596 for (
unsigned i = 0; i != NumElts; ++i) {
12598 if (Idx >= (
int)NumElts) {
12625 "BUILD_VECTOR has wrong number of operands");
12627 bool AllSame =
true;
12628 for (
unsigned i = 0; i != NumElts; ++i) {
12635 if (!Base.getNode())
12637 for (
unsigned i = 0; i != NumElts; ++i) {
12682 for (
int M : SVN->
getMask()) {
12685 int Idx = M % NumElts;
12730 auto ScaleShuffleMask = [](
ArrayRef<int> Mask,
int Scale) {
12736 for (
int s = 0; s != Scale; ++s)
12737 NewMask.
push_back(M < 0 ? -1 : Scale * M + s);
12747 EVT ScaleVT = SVT.
bitsLT(InnerSVT) ? VT : InnerVT;
12760 ScaleShuffleMask(InnerSVN->
getMask(), InnerScale);
12762 ScaleShuffleMask(SVN->
getMask(), OuterScale);
12766 for (
int M : OuterMask)
12767 NewMask.
push_back(M < 0 ? -1 : InnerMask[M]);
12799 assert(N1->getOperand(0).getValueType() == VT &&
12800 "Shuffle types don't match");
12804 bool HasSameOp0 = N0 == SV0;
12806 if (HasSameOp0 || IsSV1Undef || N0 == SV1)
12825 "Shuffle types don't match");
12831 for (
unsigned i = 0; i != NumElts; ++i) {
12840 if (Idx < (
int)NumElts) {
12850 CurrentVec = (Idx < (
int) NumElts) ? OtherSV->
getOperand(0)
12865 Idx = Idx % NumElts;
12866 if (!SV0.getNode() || SV0 == CurrentVec) {
12875 if (SV1.getNode() && SV1 != CurrentVec)
12885 bool isUndefMask =
true;
12886 for (
unsigned i = 0; i != NumElts && isUndefMask; ++i)
12887 isUndefMask &= Mask[i] < 0;
12892 if (!SV0.getNode())
12894 if (!SV1.getNode())
12955 APInt InsIdx = cast<ConstantSDNode>(N2)->getAPIntValue();
12996 if (LegalOperations)
13009 for (
unsigned i = 0; i != NumElts; ++i) {
13040 "SimplifyVBinOp only works on vectors!");
13045 if (
SDValue Shuffle = XformToShuffleWithZero(N))
13053 if (!(cast<BuildVectorSDNode>(LHS)->isConstant() &&
13054 cast<BuildVectorSDNode>(RHS)->isConstant()))
13058 for (
unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
13091 AddToWorklist(FoldOp.
getNode());
13094 if (Ops.
size() == LHS.getNumOperands())
13101 if (LegalTypes && isa<ShuffleVectorSDNode>(LHS) &&
13102 isa<ShuffleVectorSDNode>(RHS) && LHS.hasOneUse() && RHS.
hasOneUse() &&
13103 LHS.getOperand(1).getOpcode() ==
ISD::UNDEF &&
13108 if (SVN0->
getMask().equals(SVN1->getMask())) {
13113 AddUsersToWorklist(N);
13127 cast<CondCodeSDNode>(N0.
getOperand(2))->
get());
13132 if (SCC.getNode()) {
13138 SCC.getOperand(0), SCC.getOperand(1),
13139 SCC.getOperand(4));
13140 AddToWorklist(SETCC.
getNode());
13142 SCC.getOperand(2), SCC.getOperand(3));
13155 bool DAGCombiner::SimplifySelectOps(
SDNode *TheSelect,
SDValue LHS,
13185 CombineTo(TheSelect, Sqrt);
13213 LLD->
isIndexed() || RLD->isIndexed() ||
13227 RLD->getPointerInfo().getAddrSpace() != 0 ||
13239 (RLD->hasAnyUseOfValue(1) && RLD->isPredecessorOf(CondNode)))
13243 RLD->isPredecessorOf(LLD))
13248 RLD->getBasePtr());
13255 (RLD->hasAnyUseOfValue(1) &&
13256 (RLD->isPredecessorOf(CondLHS) || RLD->isPredecessorOf(CondRHS))))
13271 bool isInvariant = LLD->
isInvariant() & RLD->isInvariant();
13279 isInvariant, Alignment);
13292 CombineTo(TheSelect, Load);
13310 if (N2 == N3)
return N2;
13318 N0, N1, CC, DL,
false);
13319 if (SCC.getNode()) AddToWorklist(SCC.getNode());
13321 if (
ConstantSDNode *SCCC = dyn_cast_or_null<ConstantSDNode>(SCC.getNode())) {
13324 return !SCCC->isNullValue() ? N2 : N3;
13340 N2.getOperand(0) == N3)
13358 !TLI.
isFPImmLegal(TV->getValueAPF(), TV->getValueType(0)) &&
13359 !TLI.
isFPImmLegal(FV->getValueAPF(), FV->getValueType(0))) &&
13362 (TV->hasOneUse() || FV->hasOneUse())) {
13363 Constant *Elts[] = {
13364 const_cast<ConstantFP*
>(FV->getConstantFPValue()),
13365 const_cast<ConstantFP*>(TV->getConstantFPValue())
13375 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
13386 AddToWorklist(Cond.
getNode());
13389 AddToWorklist(CstOffset.getNode());
13392 AddToWorklist(CPIdx.getNode());
13395 false,
false, Alignment);
13405 EVT AType = N2.getValueType();
13406 if (XType.
bitsGE(AType)) {
13409 if (N2C && ((N2C->getAPIntValue() & (N2C->getAPIntValue() - 1)) == 0)) {
13410 unsigned ShCtV = N2C->getAPIntValue().logBase2();
13416 AddToWorklist(Shift.
getNode());
13418 if (XType.
bitsGT(AType)) {
13420 AddToWorklist(Shift.
getNode());
13431 AddToWorklist(Shift.
getNode());
13433 if (XType.
bitsGT(AType)) {
13435 AddToWorklist(Shift.
getNode());
13472 if (N2C &&
isNullConstant(N3) && N2C->getAPIntValue().isPowerOf2() &&
13478 if (NotExtCompare && N2C->isOne())
13483 if (!LegalOperations ||
13493 N2.getValueType());
13496 N2.getValueType(), SCC);
13500 N2.getValueType(), SCC);
13503 AddToWorklist(SCC.
getNode());
13504 AddToWorklist(Temp.
getNode());
13511 ISD::SHL, DL, N2.getValueType(), Temp,
13522 if (!LegalOperations ||
13532 (!LegalOperations ||
13573 SubC = dyn_cast<ConstantSDNode>(N3.
getOperand(0));
13577 SubC = dyn_cast<ConstantSDNode>(N2.getOperand(0));
13579 EVT XType = N0.getValueType();
13588 AddToWorklist(Shift.
getNode());
13589 AddToWorklist(Add.
getNode());
13600 SDLoc DL,
bool foldBooleans) {
13602 DagCombineInfo(DAG,
Level,
false,
this);
13603 return TLI.
SimplifySetCC(VT, N0, N1, Cond, foldBooleans, DagCombineInfo, DL);
13619 std::vector<SDNode*> Built;
13639 std::vector<SDNode *> Built;
13660 std::vector<SDNode*> Built;
13676 unsigned Iterations = 0;
13689 AddToWorklist(Est.getNode());
13692 for (
unsigned i = 0; i < Iterations; ++i) {
13694 AddToWorklist(NewEst.
getNode());
13697 AddToWorklist(NewEst.getNode());
13700 AddToWorklist(NewEst.getNode());
13703 AddToWorklist(Est.getNode());
13719 unsigned Iterations) {
13727 AddToWorklist(HalfArg.
getNode());
13730 AddToWorklist(HalfArg.getNode());
13733 for (
unsigned i = 0; i < Iterations; ++i) {
13735 AddToWorklist(NewEst.
getNode());
13738 AddToWorklist(NewEst.getNode());
13741 AddToWorklist(NewEst.getNode());
13744 AddToWorklist(Est.getNode());
13755 unsigned Iterations) {
13762 for (
unsigned i = 0; i < Iterations; ++i) {
13764 AddToWorklist(HalfEst.
getNode());
13767 AddToWorklist(Est.getNode());
13770 AddToWorklist(Est.getNode());
13773 AddToWorklist(Est.getNode());
13776 AddToWorklist(Est.getNode());
13787 unsigned Iterations = 0;
13788 bool UseOneConstNR =
false;
13790 AddToWorklist(Est.getNode());
13792 Est = UseOneConstNR ?
13793 BuildRsqrtNROneConst(Op, Est, Iterations) :
13794 BuildRsqrtNRTwoConst(Op, Est, Iterations);
13807 Base = Ptr; Offset = 0; GV =
nullptr; CV =
nullptr;
13821 GV =
G->getGlobal();
13822 Offset +=
G->getOffset();
13830 CV = C->isMachineConstantPoolEntry() ? (
const void *)C->getMachineCPVal()
13831 : (
const void *)C->getConstVal();
13832 Offset += C->getOffset();
13836 return isa<FrameIndexSDNode>(Base);
13858 int64_t Offset1, Offset2;
13860 const void *CV1, *CV2;
13862 Base1, Offset1, GV1, CV1);
13864 Base2, Offset2, GV2, CV2);
13867 if (Base1 == Base2 || (GV1 && (GV1 == GV2)) || (CV1 && (CV1 == CV2)))
13875 if (isFrameIndex1 && isFrameIndex2) {
13877 Offset1 += MFI->
getObjectOffset(cast<FrameIndexSDNode>(Base1)->getIndex());
13878 Offset2 += MFI->
getObjectOffset(cast<FrameIndexSDNode>(Base2)->getIndex());
13885 if ((isFrameIndex1 || CV1 || GV1) && (isFrameIndex2 || CV2 || GV2))
13907 bool UseAA = CombinerGlobalAA.getNumOccurrences() > 0
13911 if (CombinerAAOnlyFunc.getNumOccurrences() &&
13939 void DAGCombiner::GatherAllAliases(
SDNode *N,
SDValue OriginalChain,
13945 bool IsLoad = isa<LoadSDNode>(
N) && !cast<LSBaseSDNode>(N)->isVolatile();
13949 unsigned Depth = 0;
13954 while (!Chains.
empty()) {
13965 if (Depth > 6 || Aliases.
size() == 2) {
13983 bool IsOpLoad = isa<LoadSDNode>(Chain.
getNode()) &&
13987 if (!(IsLoad && IsOpLoad) &&
13988 isAlias(cast<LSBaseSDNode>(N), cast<LSBaseSDNode>(Chain.
getNode()))) {
14047 for (
const SDNode *N : Visited)
14048 if (N != OriginalChain.
getNode())
14051 while (!Worklist.
empty()) {
14059 UIE = M->
use_end(); UI != UIE; ++UI)
14060 if (UI.getUse().getValueType() ==
MVT::Other &&
14061 Visited.insert(*UI).second) {
14062 if (isa<MemSDNode>(*UI)) {
14083 GatherAllAliases(N, OldChain, Aliases);
14086 if (Aliases.
size() == 0)
14090 if (Aliases.
size() == 1)
14101 DAGCombiner(*
this, AA, OptLevel).Run(Level);
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
SDValue getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT TVT, bool isNonTemporal, bool isVolatile, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes())
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
unsigned getValueSizeInBits(unsigned ResNo) const
Returns MVT::getSizeInBits(getValueType(ResNo)).
bool use_empty() const
Return true if there are no uses of this node.
opStatus divide(const APFloat &, roundingMode)
static MVT getIntegerVT(unsigned BitWidth)
std::string & operator+=(std::string &buffer, StringRef string)
void push_back(const T &Elt)
BUILTIN_OP_END - This must be the last enum value in this list.
The memory access reads data.
A parsed version of the target data layout string in and methods for querying it. ...
SDValue getValue(unsigned R) const
const SDValue & getValue() const
static SDValue FoldIntToFPToInt(SDNode *N, SelectionDAG &DAG)
static APInt getSignBit(unsigned BitWidth)
Get the SignBit for a specific bit width.
The memory access writes data.
bool isTruncatingStore() const
Return true if the op does a truncation before store.
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
static bool areSlicesNextToEachOther(const LoadedSlice &First, const LoadedSlice &Second)
Check whether or not First and Second are next to each other in memory.
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
LLVMContext * getContext() const
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef...
uint64_t getZExtValue() const
Get zero extended value.
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
SmallBitVector - This is a 'bitvector' (really, a variable-sized bit array), optimized for the case w...
void dump() const
Dump this node, for debugging.
DELETED_NODE - This is an illegal value that is used to catch errors.
STATISTIC(NumFunctions,"Total number of functions")
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
BR_CC - Conditional branch.
bool isNON_TRUNCStore(const SDNode *N)
Returns true if the specified node is a non-truncating store.
bool none() const
none - Returns true if none of the bits are set.
static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned OpSize)
bool hasOneUse() const
Return true if there is exactly one use of this node.
virtual bool storeOfVectorConstantIsCheap(EVT MemVT, unsigned NumElem, unsigned AddrSpace) const
Return true if it is expected to be cheaper to do a store of a non-zero vector constant with the give...
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDVTList getVTList() const
static bool MatchRotateHalf(SDValue Op, SDValue &Shift, SDValue &Mask)
Match "(X shl/srl V1) & V2" where V2 may not be present.
int getSplatIndex() const
Carry-setting nodes for multiple precision addition and subtraction.
const TargetMachine & getTarget() const
bool isExtended() const
isExtended - Test if the given EVT is extended (as opposed to being simple).
unsigned getPrefTypeAlignment(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Clients of various APIs that cause global effects on the DAG can optionally implement this interface...
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
bool operator>(int64_t V1, const APSInt &V2)
static SDValue simplifyShuffleOperandRecursively(SmallBitVector &UsedElements, SDValue V, SelectionDAG &DAG)
const TargetSubtargetInfo & getSubtarget() const
unsigned InferPtrAlignment(SDValue Ptr) const
Infer alignment of a load / store address.
void computeKnownBits(SDValue Op, APInt &KnownZero, APInt &KnownOne, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in the KnownZero/KnownO...
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
CondCode getSetCCAndOperation(CondCode Op1, CondCode Op2, bool isInteger)
getSetCCAndOperation - Return the result of a logical AND between different comparisons of identical ...
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
const SDValue & getSrc0() const
Type * getTypeForEVT(LLVMContext &Context) const
getTypeForEVT - This method returns an LLVM type corresponding to the specified EVT.
unsigned getSizeInBits() const
opStatus
IEEE-754R 7: Default exception handling.
SDValue getLoad(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, bool isVolatile, bool isNonTemporal, bool isInvariant, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
SDValue getZeroExtendInReg(SDValue Op, SDLoc DL, EVT SrcTy)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value...
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
bool isMask(unsigned numBits, const APInt &APIVal)
unsigned getNumOperands() const
Return the number of values used by this operation.
void setAllBits()
Set every bit to 1.
const TargetRegisterClass * getCommonSubClass(const TargetRegisterClass *A, const TargetRegisterClass *B) const
getCommonSubClass - find the largest common subclass of A and B.
unsigned getNumOperands() const
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
virtual bool isZExtFree(Type *, Type *) const
Return true if any actual instruction that defines a value of type Ty1 implicitly zero-extends the va...
The two locations do not alias at all.
const SDValue & getOperand(unsigned Num) const
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, unsigned Align=1, bool *=nullptr) const
Determine if the target supports unaligned memory accesses.
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
static MachinePointerInfo getConstantPool()
getConstantPool - Return a MachinePointerInfo record that refers to the constant pool.
static ConstantSDNode * isConstOrConstSplat(SDValue N)
static bool isCommutativeBinOp(unsigned Opcode)
Returns true if the opcode is a commutative binary operation.
iv Induction Variable Users
void reserve(size_type N)
static bool FindBaseOffset(SDValue Ptr, SDValue &Base, int64_t &Offset, const GlobalValue *&GV, const void *&CV)
Return true if base is a frame index, which is known not to alias with anything but itself...
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
const SDValue & getBasePtr() const
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1 at the ...
SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align=0, int Offs=0, bool isT=false, unsigned char TargetFlags=0)
APInt LLVM_ATTRIBUTE_UNUSED_RESULT zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
SDValue BuildUDIV(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, bool IsAfterLegalization, std::vector< SDNode * > *Created) const
Given an ISD::UDIV node expressing a divide by constant, return a DAG expression to select that will ...
std::size_t countLeadingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the most significant bit to the least stopping at the first 1...
static SDNode * ShrinkLoadReplaceStoreWithStore(const std::pair< unsigned, unsigned > &MaskInfo, SDValue IVal, StoreSDNode *St, DAGCombiner *DC)
Check to see if IVal is something that provides a value as specified by MaskInfo. ...
virtual bool isFPImmLegal(const APFloat &, EVT) const
Returns true if the target can instruction select the specified FP immediate natively.
size_type size() const
Determine the number of elements in the SetVector.
EntryToken - This is the marker used to indicate the start of a region.
static bool isOneConstant(SDValue V)
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
unsigned getResNo() const
get the index which selects a specific result in the SDNode
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
bool bitsLT(EVT VT) const
bitsLT - Return true if this has less bits than VT.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedMask, APInt &KnownZero, APInt &KnownOne, TargetLoweringOpt &TLO, unsigned Depth=0) const
Look at Op.
int64_t getSrcValueOffset() const
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
bool isAllOnesValue() const
static bool isAllOnesConstant(SDValue V)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool operator<=(int64_t V1, const APSInt &V2)
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
bool isNegative() const
Determine sign of this APInt.
bool isVector() const
isVector - Return true if this is a vector value type.
bool match(Val *V, const Pattern &P)
SDValue getStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, bool isVolatile, bool isNonTemporal, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
bool isNegative() const
Return true if the value is negative.
MachineMemOperand - A description of a memory reference used in the backend.
bool isRound() const
isRound - Return true if the size is a power-of-two number of bytes.
bool operator>=(int64_t V1, const APSInt &V2)
Shift and rotation operations.
virtual bool isLegalAddImmediate(int64_t) const
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
std::size_t countTrailingOnes(T Value, ZeroBehavior ZB=ZB_Width)
Count the number of ones from the least significant bit to the first zero bit.
T LLVM_ATTRIBUTE_UNUSED_RESULT pop_back_val()
static cl::opt< bool > Aggressive("aggressive-ext-opt", cl::Hidden, cl::desc("Aggressive extension optimization"))
Base class for LoadSDNode and StoreSDNode.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef...
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
A Use represents the edge between a Value definition and its users.
bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist) const
Return true if N is a predecessor of this node.
MachineFunction & getMachineFunction() const
bool HonorSignDependentRoundingFPMath() const
HonorSignDependentRoundingFPMath - Return true if the codegen must assume that the rounding mode of t...
APInt LLVM_ATTRIBUTE_UNUSED_RESULT lshr(unsigned shiftAmt) const
Logical right-shift function.
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
CopyToReg - This node has three operands: a chain, a register number to set to this value...
unsigned logBase2(const APInt &APIVal)
Returns the floor log base 2 of the specified APInt value.
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
MachinePointerInfo getWithOffset(int64_t O) const
EVT getScalarType() const
getScalarType - If this is a vector type, return the element type, otherwise return this...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
Number of individual test Apply this number of consecutive mutations to each input exit after the first new interesting input is found the minimized corpus is saved into the first input directory Number of jobs to run If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
bool bitsGE(EVT VT) const
bitsGE - Return true if this has no less bits than VT.
int getMaskElt(unsigned Idx) const
This class is used to represent EVT's, which are used to parameterize some operations.
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, SDLoc dl) const
Try to simplify a setcc built with the specified operands and cc.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
bool getBoolValue() const
Convert APInt to a boolean value.
This class is used to represent an MSTORE node.
APInt lshr(const APInt &LHS, unsigned shiftAmt)
Logical right-shift function.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getVectorElementType() const
getVectorElementType - Given a vector type, return the type of each element.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector...
load Combine Adjacent Loads
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
bool insert(const value_type &X)
Insert a new element into the SetVector.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Simple integer binary arithmetic operators.
SDValue getVectorShuffle(EVT VT, SDLoc dl, SDValue N1, SDValue N2, const int *MaskElts)
Return an ISD::VECTOR_SHUFFLE node.
bool empty() const
Determine if the SetVector is empty or not.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
unsigned getActiveBits() const
Compute the number of active bits in the value.
static ConstantFPSDNode * isConstOrConstSplatFP(SDValue N)
static unsigned int semanticsPrecision(const fltSemantics &)
bool isLittleEndian() const
Layout endianness...
const SDValue & getBasePtr() const
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
static bool isConstantSplatVector(SDNode *N, APInt &SplatValue)
Returns true if N is a BUILD_VECTOR node whose elements are all the same constant or undefined...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
const APInt & getAPIntValue() const
APInt LLVM_ATTRIBUTE_UNUSED_RESULT shl(unsigned shiftAmt) const
Left-shift function.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification, or lowering of the constant.
virtual bool isDesirableToTransformToIntegerOp(unsigned, EVT) const
Return true if it is profitable for dag combiner to transform a floating point op of specified opcode...
EVT getMemoryVT() const
Return the type of the in-memory value.
const ConstantInt * getConstantIntValue() const
T LLVM_ATTRIBUTE_UNUSED_RESULT pop_back_val()
bool isSignedIntSetCC(CondCode Code)
isSignedIntSetCC - Return true if this is a setcc instruction that performs a signed comparison when ...
size_t size() const
size - Get the array size.
const DataLayout & getDataLayout() const
bool bitsLE(EVT VT) const
bitsLE - Return true if this has no more bits than VT.
bool isPow2VectorType() const
isPow2VectorType - Returns true if the given vector is a power of 2.
static bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op, APInt &KnownZero)
UNDEF - An undefined node.
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
SDNode * getNode() const
get the SDNode which holds the desired result
bool isTruncStoreLegal(EVT ValVT, EVT MemVT) const
Return true if the specified store with truncation is legal on this target.
A self-contained host- and target-independent arbitrary-precision floating-point software implementat...
bool hasPredecessor(const SDNode *N) const
Return true if N is a predecessor of this node.
unsigned getScalarSizeInBits() const
unsigned getStoreSize() const
getStoreSize - Return the number of bytes overwritten by a store of the specified value type...
static bool ExtendUsesToFormExtLoad(SDNode *N, SDValue N0, unsigned ExtOpc, SmallVectorImpl< SDNode * > &ExtendNodes, const TargetLowering &TLI)
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
initializer< Ty > init(const Ty &Val)
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
unsigned getStoreSizeInBits() const
getStoreSizeInBits - Return the number of bits overwritten by a store of the specified value type...
FPOpFusion::FPOpFusionMode AllowFPOpFusion
AllowFPOpFusion - This flag is set by the -fuse-fp-ops=xxx option.
virtual bool isShuffleMaskLegal(const SmallVectorImpl< int > &, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations, those with specific masks.
SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV)
Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to the shuffle node in input but with swa...
const SDValue & getBasePtr() const
bool isZero() const
Return true if the value is positive or negative zero.
std::size_t countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1...
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
APInt LLVM_ATTRIBUTE_UNUSED_RESULT trunc(unsigned width) const
Truncate to new width.
MVT - Machine Value Type.
static SDValue ConvertSelectToConcatVector(SDNode *N, SelectionDAG &DAG)
const SDValue & getOperand(unsigned i) const
The instances of the Type class are immutable: once they are created, they are never changed...
This is an important class for using LLVM in a threaded context.
Simple binary floating point operators.
bool isNonTemporal() const
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
void Combine(CombineLevel Level, AliasAnalysis &AA, CodeGenOpt::Level OptLevel)
This iterates over the nodes in the SelectionDAG, folding certain types of nodes together, or eliminating superfluous nodes.
This is an important base class in LLVM.
static SDNode * isConstantIntBuildVectorOrConstantInt(SDValue N)
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
int64_t getSExtValue() const
Get sign extended value.
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Carry-using nodes for multiple precision addition and subtraction.
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
ConstantFP - Floating Point Values [float, double].
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
unsigned getScalarValueSizeInBits() const
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
void AddToWorklist(SDNode *N)
APInt LLVM_ATTRIBUTE_UNUSED_RESULT sext(unsigned width) const
Sign extend to a new width.
AliasResult
The possible results of an alias query.
APInt getLoBits(unsigned numBits) const
Compute an APInt containing numBits lowbits from this APInt.
virtual bool isTruncateFree(Type *, Type *) const
Return true if it's free to truncate a value of type Ty1 to type Ty2.
This class provides iterator support for SDUse operands that use a specific SDNode.
unsigned getOriginalAlignment() const
Returns alignment and volatility of the memory access.
SDValue getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang","erlang-compatible garbage collector")
static EVT getFloatingPointVT(unsigned BitWidth)
getFloatingPointVT - Returns the EVT that represents a floating point type with the given number of b...
unsigned getBitWidth() const
Return the number of bits in the APInt.
static bool allowableAlignment(const SelectionDAG &DAG, const TargetLowering &TLI, EVT EVTTy, unsigned AS, unsigned Align)
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
unsigned getOpcode() const
const SDValue & getBasePtr() const
virtual bool isNarrowingProfitable(EVT, EVT) const
Return true if it's profitable to narrow operations of type VT1 to VT2.
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
virtual SDValue getRecipEstimate(SDValue Operand, DAGCombinerInfo &DCI, unsigned &RefinementSteps) const
Return a reciprocal estimate value for the input operand.
CondCode getSetCCSwappedOperands(CondCode Operation)
getSetCCSwappedOperands - Return the operation corresponding to (Y op X) when given the operation for...
unsigned countPopulation() const
Count the number of bits set.
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
const SDValue & getValue() const
SDValue getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, bool isVolatile, bool isNonTemporal, bool isInvariant, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes())
Bit counting operators with an undefined result for zero inputs.
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue BuildSDIV(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, bool IsAfterLegalization, std::vector< SDNode * > *Created) const
Given an ISD::SDIV node expressing a divide by constant, return a DAG expression to select that will ...
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
EVT - Extended Value Type.
uint64_t NextPowerOf2(uint64_t A)
NextPowerOf2 - Returns the next power of two (in 64-bits) that is strictly greater than A...
static SDValue tryFoldToZero(SDLoc DL, const TargetLowering &TLI, EVT VT, SelectionDAG &DAG, bool LegalOperations, bool LegalTypes)
const SDValue & getMask() const
const APFloat & getValueAPF() const
bool bitsEq(EVT VT) const
bitsEq - Return true if this has the same number of bits as VT.
const ConstantFP * getConstantFPValue() const
static bool areUsedBitsDense(const APInt &UsedBits)
Check that all bits set in UsedBits form a dense region, i.e., UsedBits looks like 0...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
static void adjustCostForPairing(SmallVectorImpl< LoadedSlice > &LoadedSlices, LoadedSlice::Cost &GlobalLSCost)
Adjust the GlobalLSCost according to the target paring capabilities and the layout of the slices...
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc, or post-dec.
bool isLoadExtLegal(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return true if the specified load with extension is legal on this target.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements)
getVectorVT - Returns the EVT that represents a vector NumElements in length, where each element is o...
MachinePointerInfo - This class contains a discriminated union of information about pointers in memor...
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
HANDLENODE node - Used as a handle for various purposes.
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE maxNum semantics.
static char isNegatibleForFree(SDValue Op, bool LegalOperations, const TargetLowering &TLI, const TargetOptions *Options, unsigned Depth=0)
Return 1 if we can compute the negated form of the specified expression for the same cost as the expr...
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
const MachinePointerInfo & getPointerInfo() const
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
const MDNode * getRanges() const
Returns the Ranges that describes the dereference.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
const SDValue & getOffset() const
bool bitsGT(EVT VT) const
bitsGT - Return true if this has more bits than VT.
ArrayRef< int > getMask() const
static void commuteMask(SmallVectorImpl< int > &Mask)
Change values in a shuffle permute mask assuming the two vector operands have swapped position...
unsigned countTrailingZeros() const
Count the number of trailing zero bits.
Representation for a specific memory location.
TokenFactor - This node takes multiple tokens as input and produces a single token result...
unsigned getABITypeAlignment(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
bool isBuildVectorAllOnes(const SDNode *N)
Node predicates.
static bool canFoldInAddressingMode(SDNode *N, SDNode *Use, SelectionDAG &DAG, const TargetLowering &TLI)
Return true if 'Use' is a load or a store that uses N as its base pointer and that N may be folded in...
A SetVector that performs no allocations if smaller than a certain size.
SDValue getNOT(SDLoc DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
virtual bool hasPairedLoad(Type *, unsigned &) const
Return true if the target supplies and combines to a paired load two loaded values of type LoadedType...
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
virtual SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, std::vector< SDNode * > *Created) const
void RemoveFromWorklist(SDNode *N)
bool isNegative() const
IEEE-754R isSignMinus: Returns true if and only if the current value is negative. ...
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
virtual bool isVectorClearMaskLegal(const SmallVectorImpl< int > &, EVT) const
Similar to isShuffleMaskLegal.
static bool isNullFPConstant(SDValue V)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
X = FP_ROUND_INREG(Y, VT) - This operator takes an FP register, and rounds it to a floating point val...
unsigned logBase2() const
bool isPredecessorOf(const SDNode *N) const
Return true if this node is a predecessor of N.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Type * getType() const
All values are typed, get the type of this value.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
bool allOperandsUndef(const SDNode *N)
Return true if the node has at least one operand and all operands of the specified node are ISD::UNDE...
BRCOND - Conditional branch.
const SDValue & getChain() const
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Byte Swap and Counting operators.
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
unsigned getAddrSpace() const
getAddrSpace - Return the LLVM IR address space number that this pointer points into.
MachineFrameInfo * getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Represents one node in the SelectionDAG.
static SDNode * tryToFoldExtendOfConstant(SDNode *N, const TargetLowering &TLI, SelectionDAG &DAG, bool LegalTypes, bool LegalOperations)
Try to fold a sext/zext/aext dag node into a ConstantSDNode or a build_vector of constants.
CondCode getSetCCInverse(CondCode Operation, bool isInteger)
getSetCCInverse - Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operat...
static cl::opt< AlignMode > Align(cl::desc("Load/store alignment support"), cl::Hidden, cl::init(NoStrictAlign), cl::values(clEnumValN(StrictAlign,"aarch64-strict-align","Disallow all unaligned memory accesses"), clEnumValN(NoStrictAlign,"aarch64-no-strict-align","Allow unaligned memory accesses"), clEnumValEnd))
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
static std::pair< SDValue, SDValue > SplitVSETCC(const SDNode *N, SelectionDAG &DAG)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
unsigned Log2_32(uint32_t Value)
Log2_32 - This function returns the floor log base 2 of the specified value, -1 if the value is zero...
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
const SDValue & getValue() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Class for arbitrary precision integers.
const Value * getValue() const
getValue - Return the base address of the memory access.
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef...
iterator_range< use_iterator > uses()
static SDNode * getBuildPairElt(SDNode *N, unsigned i)
A "pseudo-class" with methods for operating on BUILD_VECTORs.
Select(COND, TRUEVAL, FALSEVAL).
int64_t getSExtValue() const
op_iterator op_begin() const
static SDValue simplifyShuffleOperands(ShuffleVectorSDNode *SVN, SDValue N0, SDValue N1, SelectionDAG &DAG)
virtual const TargetRegisterClass * getRegClassFor(MVT VT) const
Return the register class that should be used for the specified value type.
static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOperations, unsigned Depth=0)
If isNegatibleForFree returns true, return the newly negated expression.
static use_iterator use_end()
ZERO_EXTEND - Used for integer types, zeroing the new bits.
AddrMode
ARM Addressing Modes.
bool isPowerOf2_64(uint64_t Value)
isPowerOf2_64 - This function returns true if the argument is a power of two 0 (64 bit edition...
APInt bitcastToAPInt() const
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
ANY_EXTEND - Used for integer types. The high bits are undefined.
static std::pair< unsigned, unsigned > CheckForMaskedLoad(SDValue V, SDValue Ptr, SDValue Chain)
Check to see if V is (and load (ptr), imm), where the load is having specific bytes cleared out...
CondCode getSetCCOrOperation(CondCode Op1, CondCode Op2, bool isInteger)
getSetCCOrOperation - Return the result of a logical OR between different comparisons of identical va...
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
static SDValue partitionShuffleOfConcats(SDNode *N, SelectionDAG &DAG)
iterator_range< value_op_iterator > op_values() const
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
const SDValue & getMask() const
bool operator!=(uint64_t V1, const APInt &V2)
bool isAllOnesValue() const
Determine if all bits are set.
uint64_t MinAlign(uint64_t A, uint64_t B)
MinAlign - A and B are either alignments or offsets.
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Represents a use of a SDNode.
Bitwise operators - logical and, logical or, logical xor.
static cl::opt< bool > UseTBAA("use-tbaa-in-sched-mi", cl::Hidden, cl::init(true), cl::desc("Enable use of TBAA during MI DAG construction"))
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
static bool isNullConstant(SDValue V)
static SDValue combineMinNumMaxNum(SDLoc DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode CC, const TargetLowering &TLI, SelectionDAG &DAG)
Generate Min/Max node.
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AddrSpace) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
void ReplaceAllUsesWith(SDValue From, SDValue Op)
Modify anything using 'From' to use 'To' instead.
ArrayRef< SDUse > ops() const
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
op_iterator op_end() const
static ArrayType * get(Type *ElementType, uint64_t NumElements)
ArrayType::get - This static method is the primary way to construct an ArrayType. ...
static SDValue combineConcatVectorOfScalars(SDNode *N, SelectionDAG &DAG)
static SDValue getInputChainForNode(SDNode *N)
Given a node, return its input chain if it has one, otherwise return a null sd operand.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
virtual SDValue getRsqrtEstimate(SDValue Operand, DAGCombinerInfo &DCI, unsigned &RefinementSteps, bool &UseOneConstNR) const
Hooks for building estimates in place of slower divisions and square roots.
virtual bool isExtractSubvectorCheap(EVT ResVT, unsigned Index) const
Return true if EXTRACT_SUBVECTOR is cheap for this result type with this index.
This class is used to represent an MSCATTER node.
const SDValue & getIndex() const
EVT getValueType() const
Return the ValueType of the referenced return value.
bool all_of(R &&Range, UnaryPredicate &&P)
Provide wrappers to std::all_of which take ranges instead of having to pass being/end explicitly...
SDValue getConstant(uint64_t Val, SDLoc DL, EVT VT, bool isTarget=false, bool isOpaque=false)
bool isByteSized() const
isByteSized - Return true if the bit size is a multiple of 8.
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
SDValue getSelect(SDLoc DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
bool isFloatingPoint() const
isFloatingPoint - Return true if this is a FP, or a vector FP type.
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent an MLOAD node.
bool isSimple() const
isSimple - Test if the given EVT is simple (as opposed to being extended).
bool operator<(int64_t V1, const APSInt &V2)
static ConstantSDNode * getAsNonOpaqueConstant(SDValue N)
If N is a ContantSDNode with isOpaque() == false return it casted to a ContantSDNode pointer else nul...
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
SDValue getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
LLVM Value Representation.
FMA - Perform a * b + c with no intermediate rounding step.
static bool isSlicingProfitable(SmallVectorImpl< LoadedSlice > &LoadedSlices, const APInt &UsedBits, bool ForCodeSize)
Check the profitability of all involved LoadedSlice.
const SDValue & getBasePtr() const
A vector that has set insertion semantics.
bool isTruncatingStore() const
Return true if the op does a truncation before store.
This class is used to represent an MGATHER node.
Disable implicit floating point insts.
const TargetLowering & getTargetLoweringInfo() const
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.Val alone...
bool isInt(int64_t x)
isInt - Checks if an integer fits into the given bit width.
unsigned countLeadingZeros() const
The APInt version of the countLeadingZeros functions in MathExtras.h.
bool isPowerOf2_32(uint32_t Value)
isPowerOf2_32 - This function returns true if the argument is a power of two > 0. ...
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations...
APInt LLVM_ATTRIBUTE_UNUSED_RESULT zext(unsigned width) const
Zero extend to a new width.
SetCC operator - This evaluates to a true value iff the condition is true.
static SDNode * isConstantFPBuildVectorOrConstantFP(SDValue N)
SDValue getConstantFP(double Val, SDLoc DL, EVT VT, bool isTarget=false)
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml","ocaml 3.10-compatible collector")
SDValue getSetCC(SDLoc DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
bool operator==(uint64_t V1, const APInt &V2)
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
static bool isVolatile(Instruction *Inst)
static APInt getNullValue(unsigned numBits)
Get the '0' value.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDNode * getUser()
This returns the SDNode that contains this Use.
TRUNCATE - Completely drop the high bits.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
bool isUIntN(unsigned N, uint64_t x)
isUIntN - Checks if an unsigned integer fits into the given (dynamic) bit width.
unsigned getAlignment() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR - Perform various unary floating point operations.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
getIntegerVT - Returns the EVT that represents an integer with the given number of bits...
const fltSemantics & getSemantics() const
unsigned Log2_64(uint64_t Value)
Log2_64 - This function returns the floor log base 2 of the specified value, -1 if the value is zero...
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
EVT changeVectorElementTypeToInteger() const
changeVectorElementTypeToInteger - Return a vector with the same number of elements as this vector...
unsigned getResNo() const
Convenience function for get().getResNo().
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE minNum semantics.
MVT getSimpleVT() const
getSimpleVT - Return the SimpleValueType held in the specified simple EVT.
CARRY_FALSE - This node is used when folding other nodes, like ADDC/SUBC, which indicate the carry re...
SDValue getIntPtrConstant(uint64_t Val, SDLoc DL, bool isTarget=false)
static bool isBSwapHWordElement(SDValue N, MutableArrayRef< SDNode * > Parts)
Return true if the specified node is an element that makes up a 32-bit packed halfword byteswap...
This file describes how to lower LLVM code to machine code.
void CommitTargetLoweringOpt(const TargetLoweringOpt &TLO)
uint64_t getBaseAlignment() const
getBaseAlignment - Return the minimum known alignment in bytes of the base address, without the offset.
virtual bool useAA() const
Enable use of alias analysis during code generation (during MI scheduling, DAGCombine, etc.).
ISD::CondCode get() const
uint64_t getZExtValue() const
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.
This class is used to represent ISD::LOAD nodes.
Function must be optimized for size first.