LLVM  6.0.0svn
LegalizeIntegerTypes.cpp
Go to the documentation of this file.
1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements integer type expansion and promotion for LegalizeTypes.
11 // Promotion is the act of changing a computation in an illegal type into a
12 // computation in a larger type. For example, implementing i8 arithmetic in an
13 // i32 register (often needed on powerpc).
14 // Expansion is the act of changing a computation in an illegal type into a
15 // computation in two identical registers of a smaller type. For example,
16 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
17 // targets).
18 //
19 //===----------------------------------------------------------------------===//
20 
21 #include "LegalizeTypes.h"
22 #include "llvm/IR/DerivedTypes.h"
24 #include "llvm/Support/KnownBits.h"
26 using namespace llvm;
27 
28 #define DEBUG_TYPE "legalize-types"
29 
30 //===----------------------------------------------------------------------===//
31 // Integer Result Promotion
32 //===----------------------------------------------------------------------===//
33 
34 /// PromoteIntegerResult - This method is called when a result of a node is
35 /// found to be in need of promotion to a larger type. At this point, the node
36 /// may also have invalid operands or may have other results that need
37 /// expansion, we just know that (at least) one result needs promotion.
38 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
39  DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); dbgs() << "\n");
40  SDValue Res = SDValue();
41 
42  // See if the target wants to custom expand this node.
43  if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
44  DEBUG(dbgs() << "Node has been custom expanded, done\n");
45  return;
46  }
47 
48  switch (N->getOpcode()) {
49  default:
50 #ifndef NDEBUG
51  dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
52  N->dump(&DAG); dbgs() << "\n";
53 #endif
54  llvm_unreachable("Do not know how to promote this operator!");
55  case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
56  case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
57  case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
58  case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
59  case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
60  case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
61  case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
62  case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
64  case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
65  case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break;
67  case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
69  Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
70  case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
71  case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
72  break;
73  case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
74  break;
75  case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
76  case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
77  case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
78  case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
79  case ISD::SMIN:
80  case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
81  case ISD::UMIN:
82  case ISD::UMAX: Res = PromoteIntRes_ZExtIntBinOp(N); break;
83 
84  case ISD::SHL: Res = PromoteIntRes_SHL(N); break;
86  Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
87  case ISD::SRA: Res = PromoteIntRes_SRA(N); break;
88  case ISD::SRL: Res = PromoteIntRes_SRL(N); break;
89  case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
90  case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
91  case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
92 
94  Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
96  Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
98  Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
99  case ISD::BUILD_VECTOR:
100  Res = PromoteIntRes_BUILD_VECTOR(N); break;
102  Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
103  case ISD::CONCAT_VECTORS:
104  Res = PromoteIntRes_CONCAT_VECTORS(N); break;
105 
109  Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
110 
111  case ISD::SIGN_EXTEND:
112  case ISD::ZERO_EXTEND:
113  case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
114 
115  case ISD::FP_TO_SINT:
116  case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
117 
118  case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
119 
120  case ISD::AND:
121  case ISD::OR:
122  case ISD::XOR:
123  case ISD::ADD:
124  case ISD::SUB:
125  case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
126 
127  case ISD::SDIV:
128  case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
129 
130  case ISD::UDIV:
131  case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
132 
133  case ISD::SADDO:
134  case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
135  case ISD::UADDO:
136  case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
137  case ISD::SMULO:
138  case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
139 
140  case ISD::ADDCARRY:
141  case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
142 
143  case ISD::ATOMIC_LOAD:
144  Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
145 
149  case ISD::ATOMIC_LOAD_OR:
156  case ISD::ATOMIC_SWAP:
157  Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
158 
161  Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
162  break;
163  }
164 
165  // If the result is null then the sub-method took care of registering it.
166  if (Res.getNode())
167  SetPromotedInteger(SDValue(N, ResNo), Res);
168 }
169 
170 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
171  unsigned ResNo) {
172  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
173  return GetPromotedInteger(Op);
174 }
175 
176 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
177  // Sign-extend the new bits, and continue the assertion.
178  SDValue Op = SExtPromotedInteger(N->getOperand(0));
179  return DAG.getNode(ISD::AssertSext, SDLoc(N),
180  Op.getValueType(), Op, N->getOperand(1));
181 }
182 
183 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
184  // Zero the new bits, and continue the assertion.
185  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
186  return DAG.getNode(ISD::AssertZext, SDLoc(N),
187  Op.getValueType(), Op, N->getOperand(1));
188 }
189 
190 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
191  EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
192  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
193  N->getMemoryVT(), ResVT,
194  N->getChain(), N->getBasePtr(),
195  N->getMemOperand());
196  // Legalize the chain result - switch anything that used the old chain to
197  // use the new one.
198  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
199  return Res;
200 }
201 
202 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
203  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
204  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
205  N->getMemoryVT(),
206  N->getChain(), N->getBasePtr(),
207  Op2, N->getMemOperand());
208  // Legalize the chain result - switch anything that used the old chain to
209  // use the new one.
210  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
211  return Res;
212 }
213 
214 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
215  unsigned ResNo) {
216  if (ResNo == 1) {
218  EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
219  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
220 
221  // Only use the result of getSetCCResultType if it is legal,
222  // otherwise just use the promoted result type (NVT).
223  if (!TLI.isTypeLegal(SVT))
224  SVT = NVT;
225 
226  SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
227  SDValue Res = DAG.getAtomicCmpSwap(
229  N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
230  N->getMemOperand());
231  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
232  ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
233  return Res.getValue(1);
234  }
235 
236  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
237  SDValue Op3 = GetPromotedInteger(N->getOperand(3));
238  SDVTList VTs =
239  DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
240  SDValue Res = DAG.getAtomicCmpSwap(
241  N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
242  N->getBasePtr(), Op2, Op3, N->getMemOperand());
243  // Update the use to N with the newly created Res.
244  for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
245  ReplaceValueWith(SDValue(N, i), Res.getValue(i));
246  return Res;
247 }
248 
249 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
250  SDValue InOp = N->getOperand(0);
251  EVT InVT = InOp.getValueType();
252  EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
253  EVT OutVT = N->getValueType(0);
254  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
255  SDLoc dl(N);
256 
257  switch (getTypeAction(InVT)) {
259  break;
261  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
262  // The input promotes to the same size. Convert the promoted value.
263  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
264  break;
266  // Promote the integer operand by hand.
267  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
269  // Convert the promoted float by hand.
270  SDValue PromotedOp = GetPromotedFloat(InOp);
271  return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, PromotedOp);
272  break;
273  }
276  break;
278  // Convert the element to an integer and promote it by hand.
279  if (!NOutVT.isVector())
280  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
281  BitConvertToInteger(GetScalarizedVector(InOp)));
282  break;
284  // For example, i32 = BITCAST v2i16 on alpha. Convert the split
285  // pieces of the input into integers and reassemble in the final type.
286  SDValue Lo, Hi;
287  GetSplitVector(N->getOperand(0), Lo, Hi);
288  Lo = BitConvertToInteger(Lo);
289  Hi = BitConvertToInteger(Hi);
290 
291  if (DAG.getDataLayout().isBigEndian())
292  std::swap(Lo, Hi);
293 
294  InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
296  NOutVT.getSizeInBits()),
297  JoinIntegers(Lo, Hi));
298  return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
299  }
301  // The input is widened to the same size. Convert to the widened value.
302  // Make sure that the outgoing value is not a vector, because this would
303  // make us bitcast between two vectors which are legalized in different ways.
304  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector())
305  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
306  }
307 
308  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
309  CreateStackStoreLoad(InOp, OutVT));
310 }
311 
312 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
313  SDValue Op = GetPromotedInteger(N->getOperand(0));
314  EVT OVT = N->getValueType(0);
315  EVT NVT = Op.getValueType();
316  SDLoc dl(N);
317 
318  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
319  return DAG.getNode(
320  ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
321  DAG.getConstant(DiffBits, dl,
322  TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
323 }
324 
325 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
326  SDValue Op = GetPromotedInteger(N->getOperand(0));
327  EVT OVT = N->getValueType(0);
328  EVT NVT = Op.getValueType();
329  SDLoc dl(N);
330 
331  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
332  return DAG.getNode(
333  ISD::SRL, dl, NVT, DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
334  DAG.getConstant(DiffBits, dl,
335  TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
336 }
337 
338 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
339  // The pair element type may be legal, or may not promote to the same type as
340  // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
341  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
342  TLI.getTypeToTransformTo(*DAG.getContext(),
343  N->getValueType(0)), JoinIntegers(N->getOperand(0),
344  N->getOperand(1)));
345 }
346 
347 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
348  EVT VT = N->getValueType(0);
349  // FIXME there is no actual debug info here
350  SDLoc dl(N);
351  // Zero extend things like i1, sign extend everything else. It shouldn't
352  // matter in theory which one we pick, but this tends to give better code?
353  unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
354  SDValue Result = DAG.getNode(Opc, dl,
355  TLI.getTypeToTransformTo(*DAG.getContext(), VT),
356  SDValue(N, 0));
357  assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
358  return Result;
359 }
360 
361 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
362  // Zero extend to the promoted type and do the count there.
363  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
364  SDLoc dl(N);
365  EVT OVT = N->getValueType(0);
366  EVT NVT = Op.getValueType();
367  Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
368  // Subtract off the extra leading bits in the bigger type.
369  return DAG.getNode(
370  ISD::SUB, dl, NVT, Op,
371  DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
372  NVT));
373 }
374 
375 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
376  // Zero extend to the promoted type and do the count there.
377  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
378  return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op);
379 }
380 
381 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
382  SDValue Op = GetPromotedInteger(N->getOperand(0));
383  EVT OVT = N->getValueType(0);
384  EVT NVT = Op.getValueType();
385  SDLoc dl(N);
386  if (N->getOpcode() == ISD::CTTZ) {
387  // The count is the same in the promoted type except if the original
388  // value was zero. This can be handled by setting the bit just off
389  // the top of the original type.
390  auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
391  OVT.getScalarSizeInBits());
392  Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
393  }
394  return DAG.getNode(N->getOpcode(), dl, NVT, Op);
395 }
396 
397 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
398  SDLoc dl(N);
399  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
400  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0),
401  N->getOperand(1));
402 }
403 
404 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
405  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
406  unsigned NewOpc = N->getOpcode();
407  SDLoc dl(N);
408 
409  // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
410  // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
411  // and SINT conversions are Custom, there is no way to tell which is
412  // preferable. We choose SINT because that's the right thing on PPC.)
413  if (N->getOpcode() == ISD::FP_TO_UINT &&
414  !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
416  NewOpc = ISD::FP_TO_SINT;
417 
418  SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
419 
420  // Assert that the converted value fits in the original type. If it doesn't
421  // (eg: because the value being converted is too big), then the result of the
422  // original operation was undefined anyway, so the assert is still correct.
423  //
424  // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
425  // before legalization: fp-to-uint16, 65534. -> 0xfffe
426  // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
427  return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ?
428  ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
430 }
431 
432 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
433  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
434  SDLoc dl(N);
435 
436  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
437 }
438 
439 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
440  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
441  SDLoc dl(N);
442 
443  if (getTypeAction(N->getOperand(0).getValueType())
445  SDValue Res = GetPromotedInteger(N->getOperand(0));
446  assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
447 
448  // If the result and operand types are the same after promotion, simplify
449  // to an in-register extension.
450  if (NVT == Res.getValueType()) {
451  // The high bits are not guaranteed to be anything. Insert an extend.
452  if (N->getOpcode() == ISD::SIGN_EXTEND)
453  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
454  DAG.getValueType(N->getOperand(0).getValueType()));
455  if (N->getOpcode() == ISD::ZERO_EXTEND)
456  return DAG.getZeroExtendInReg(Res, dl,
458  assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
459  return Res;
460  }
461  }
462 
463  // Otherwise, just extend the original operand all the way to the larger type.
464  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
465 }
466 
467 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
468  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
469  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
470  ISD::LoadExtType ExtType =
472  SDLoc dl(N);
473  SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
474  N->getMemoryVT(), N->getMemOperand());
475 
476  // Legalize the chain result - switch anything that used the old chain to
477  // use the new one.
478  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
479  return Res;
480 }
481 
482 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
483  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
484  SDValue ExtSrc0 = GetPromotedInteger(N->getSrc0());
485 
486  SDLoc dl(N);
487  SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
488  N->getMask(), ExtSrc0, N->getMemoryVT(),
490  // Legalize the chain result - switch anything that used the old chain to
491  // use the new one.
492  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
493  return Res;
494 }
495 
496 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
497  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
498  SDValue ExtSrc0 = GetPromotedInteger(N->getValue());
499  assert(NVT == ExtSrc0.getValueType() &&
500  "Gather result type and the passThru agrument type should be the same");
501 
502  SDLoc dl(N);
503  SDValue Ops[] = {N->getChain(), ExtSrc0, N->getMask(), N->getBasePtr(),
504  N->getIndex()};
505  SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
506  N->getMemoryVT(), dl, Ops,
507  N->getMemOperand());
508  // Legalize the chain result - switch anything that used the old chain to
509  // use the new one.
510  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
511  return Res;
512 }
513 
514 /// Promote the overflow flag of an overflowing arithmetic node.
515 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
516  // Simply change the return type of the boolean result.
517  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
518  EVT ValueVTs[] = { N->getValueType(0), NVT };
519  SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
520  unsigned NumOps = N->getNumOperands();
521  assert(NumOps <= 3 && "Too many operands");
522  if (NumOps == 3)
523  Ops[2] = N->getOperand(2);
524 
525  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
526  DAG.getVTList(ValueVTs), makeArrayRef(Ops, NumOps));
527 
528  // Modified the sum result - switch anything that used the old sum to use
529  // the new one.
530  ReplaceValueWith(SDValue(N, 0), Res);
531 
532  return SDValue(Res.getNode(), 1);
533 }
534 
535 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
536  if (ResNo == 1)
537  return PromoteIntRes_Overflow(N);
538 
539  // The operation overflowed iff the result in the larger type is not the
540  // sign extension of its truncation to the original type.
541  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
542  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
543  EVT OVT = N->getOperand(0).getValueType();
544  EVT NVT = LHS.getValueType();
545  SDLoc dl(N);
546 
547  // Do the arithmetic in the larger type.
548  unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
549  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
550 
551  // Calculate the overflow flag: sign extend the arithmetic result from
552  // the original type.
553  SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
554  DAG.getValueType(OVT));
555  // Overflowed if and only if this is not equal to Res.
556  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
557 
558  // Use the calculated overflow everywhere.
559  ReplaceValueWith(SDValue(N, 1), Ofl);
560 
561  return Res;
562 }
563 
564 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
565  SDValue LHS = GetPromotedInteger(N->getOperand(1));
566  SDValue RHS = GetPromotedInteger(N->getOperand(2));
567  return DAG.getSelect(SDLoc(N),
568  LHS.getValueType(), N->getOperand(0), LHS, RHS);
569 }
570 
571 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
572  SDValue Mask = N->getOperand(0);
573 
574  SDValue LHS = GetPromotedInteger(N->getOperand(1));
575  SDValue RHS = GetPromotedInteger(N->getOperand(2));
576  // Promote all the way up to the canonical SetCC type.
577  Mask = PromoteTargetBoolean(Mask, LHS.getValueType());
578  return DAG.getNode(ISD::VSELECT, SDLoc(N),
579  LHS.getValueType(), Mask, LHS, RHS);
580 }
581 
582 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
583  SDValue LHS = GetPromotedInteger(N->getOperand(2));
584  SDValue RHS = GetPromotedInteger(N->getOperand(3));
585  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
586  LHS.getValueType(), N->getOperand(0),
587  N->getOperand(1), LHS, RHS, N->getOperand(4));
588 }
589 
590 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
591  EVT SVT = getSetCCResultType(N->getOperand(0).getValueType());
592 
593  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
594 
595  // Only use the result of getSetCCResultType if it is legal,
596  // otherwise just use the promoted result type (NVT).
597  if (!TLI.isTypeLegal(SVT))
598  SVT = NVT;
599 
600  SDLoc dl(N);
601  assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() &&
602  "Vector compare must return a vector result!");
603 
604  SDValue LHS = N->getOperand(0);
605  SDValue RHS = N->getOperand(1);
606  if (LHS.getValueType() != RHS.getValueType()) {
607  if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger &&
608  !LHS.getValueType().isVector())
609  LHS = GetPromotedInteger(LHS);
610  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger &&
611  !RHS.getValueType().isVector())
612  RHS = GetPromotedInteger(RHS);
613  }
614 
615  // Get the SETCC result using the canonical SETCC type.
616  SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, LHS, RHS,
617  N->getOperand(2));
618 
619  // Convert to the expected type.
620  return DAG.getSExtOrTrunc(SetCC, dl, NVT);
621 }
622 
623 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
624  SDValue LHS = N->getOperand(0);
625  SDValue RHS = N->getOperand(1);
626  if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
627  LHS = GetPromotedInteger(LHS);
628  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
629  RHS = ZExtPromotedInteger(RHS);
630  return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
631 }
632 
633 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
634  SDValue Op = GetPromotedInteger(N->getOperand(0));
635  return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
636  Op.getValueType(), Op, N->getOperand(1));
637 }
638 
639 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
640  // The input may have strange things in the top bits of the registers, but
641  // these operations don't care. They may have weird bits going out, but
642  // that too is okay if they are integer operations.
643  SDValue LHS = GetPromotedInteger(N->getOperand(0));
644  SDValue RHS = GetPromotedInteger(N->getOperand(1));
645  return DAG.getNode(N->getOpcode(), SDLoc(N),
646  LHS.getValueType(), LHS, RHS);
647 }
648 
649 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
650  // Sign extend the input.
651  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
652  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
653  return DAG.getNode(N->getOpcode(), SDLoc(N),
654  LHS.getValueType(), LHS, RHS);
655 }
656 
657 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
658  // Zero extend the input.
659  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
660  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
661  return DAG.getNode(N->getOpcode(), SDLoc(N),
662  LHS.getValueType(), LHS, RHS);
663 }
664 
665 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
666  SDValue LHS = N->getOperand(0);
667  SDValue RHS = N->getOperand(1);
668  // The input value must be properly sign extended.
669  if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
670  LHS = SExtPromotedInteger(LHS);
671  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
672  RHS = ZExtPromotedInteger(RHS);
673  return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
674 }
675 
676 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
677  SDValue LHS = N->getOperand(0);
678  SDValue RHS = N->getOperand(1);
679  // The input value must be properly zero extended.
680  if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
681  LHS = ZExtPromotedInteger(LHS);
682  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
683  RHS = ZExtPromotedInteger(RHS);
684  return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
685 }
686 
687 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
688  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
689  SDValue Res;
690  SDValue InOp = N->getOperand(0);
691  SDLoc dl(N);
692 
693  switch (getTypeAction(InOp.getValueType())) {
694  default: llvm_unreachable("Unknown type action!");
697  Res = InOp;
698  break;
700  Res = GetPromotedInteger(InOp);
701  break;
703  EVT InVT = InOp.getValueType();
704  assert(InVT.isVector() && "Cannot split scalar types");
705  unsigned NumElts = InVT.getVectorNumElements();
706  assert(NumElts == NVT.getVectorNumElements() &&
707  "Dst and Src must have the same number of elements");
708  assert(isPowerOf2_32(NumElts) &&
709  "Promoted vector type must be a power of two");
710 
711  SDValue EOp1, EOp2;
712  GetSplitVector(InOp, EOp1, EOp2);
713 
714  EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
715  NumElts/2);
716  EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
717  EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
718 
719  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
720  }
722  SDValue WideInOp = GetWidenedVector(InOp);
723 
724  // Truncate widened InOp.
725  unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
726  EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
727  N->getValueType(0).getScalarType(), NumElem);
728  SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
729 
730  // Zero extend so that the elements are of same type as those of NVT
731  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
732  NumElem);
733  SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
734 
735  // Extract the low NVT subvector.
736  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
737  SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy);
738  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
739  }
740  }
741 
742  // Truncate to NVT instead of VT
743  return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
744 }
745 
746 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
747  if (ResNo == 1)
748  return PromoteIntRes_Overflow(N);
749 
750  // The operation overflowed iff the result in the larger type is not the
751  // zero extension of its truncation to the original type.
752  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
753  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
754  EVT OVT = N->getOperand(0).getValueType();
755  EVT NVT = LHS.getValueType();
756  SDLoc dl(N);
757 
758  // Do the arithmetic in the larger type.
759  unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
760  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
761 
762  // Calculate the overflow flag: zero extend the arithmetic result from
763  // the original type.
764  SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
765  // Overflowed if and only if this is not equal to Res.
766  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
767 
768  // Use the calculated overflow everywhere.
769  ReplaceValueWith(SDValue(N, 1), Ofl);
770 
771  return Res;
772 }
773 
774 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
775  if (ResNo == 1)
776  return PromoteIntRes_Overflow(N);
777  llvm_unreachable("Not implemented");
778 }
779 
780 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
781  // Promote the overflow bit trivially.
782  if (ResNo == 1)
783  return PromoteIntRes_Overflow(N);
784 
785  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
786  SDLoc DL(N);
787  EVT SmallVT = LHS.getValueType();
788 
789  // To determine if the result overflowed in a larger type, we extend the
790  // input to the larger type, do the multiply (checking if it overflows),
791  // then also check the high bits of the result to see if overflow happened
792  // there.
793  if (N->getOpcode() == ISD::SMULO) {
794  LHS = SExtPromotedInteger(LHS);
795  RHS = SExtPromotedInteger(RHS);
796  } else {
797  LHS = ZExtPromotedInteger(LHS);
798  RHS = ZExtPromotedInteger(RHS);
799  }
800  SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
801  SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
802 
803  // Overflow occurred if it occurred in the larger type, or if the high part
804  // of the result does not zero/sign-extend the low part. Check this second
805  // possibility first.
806  SDValue Overflow;
807  if (N->getOpcode() == ISD::UMULO) {
808  // Unsigned overflow occurred if the high part is non-zero.
809  SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
810  DAG.getIntPtrConstant(SmallVT.getSizeInBits(),
811  DL));
812  Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
813  DAG.getConstant(0, DL, Hi.getValueType()),
814  ISD::SETNE);
815  } else {
816  // Signed overflow occurred if the high part does not sign extend the low.
817  SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
818  Mul, DAG.getValueType(SmallVT));
819  Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
820  }
821 
822  // The only other way for overflow to occur is if the multiplication in the
823  // larger type itself overflowed.
824  Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
825  SDValue(Mul.getNode(), 1));
826 
827  // Use the calculated overflow everywhere.
828  ReplaceValueWith(SDValue(N, 1), Overflow);
829  return Mul;
830 }
831 
832 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
833  return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
834  N->getValueType(0)));
835 }
836 
837 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
838  SDValue Chain = N->getOperand(0); // Get the chain.
839  SDValue Ptr = N->getOperand(1); // Get the pointer.
840  EVT VT = N->getValueType(0);
841  SDLoc dl(N);
842 
843  MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
844  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
845  // The argument is passed as NumRegs registers of type RegVT.
846 
847  SmallVector<SDValue, 8> Parts(NumRegs);
848  for (unsigned i = 0; i < NumRegs; ++i) {
849  Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
850  N->getConstantOperandVal(3));
851  Chain = Parts[i].getValue(1);
852  }
853 
854  // Handle endianness of the load.
855  if (DAG.getDataLayout().isBigEndian())
856  std::reverse(Parts.begin(), Parts.end());
857 
858  // Assemble the parts in the promoted type.
859  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
860  SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
861  for (unsigned i = 1; i < NumRegs; ++i) {
862  SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
863  // Shift it to the right position and "or" it in.
864  Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
865  DAG.getConstant(i * RegVT.getSizeInBits(), dl,
866  TLI.getPointerTy(DAG.getDataLayout())));
867  Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
868  }
869 
870  // Modified the chain result - switch anything that used the old chain to
871  // use the new one.
872  ReplaceValueWith(SDValue(N, 1), Chain);
873 
874  return Res;
875 }
876 
877 //===----------------------------------------------------------------------===//
878 // Integer Operand Promotion
879 //===----------------------------------------------------------------------===//
880 
881 /// PromoteIntegerOperand - This method is called when the specified operand of
882 /// the specified node is found to need promotion. At this point, all of the
883 /// result types of the node are known to be legal, but other operands of the
884 /// node may need promotion or expansion as well as the specified one.
885 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
886  DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); dbgs() << "\n");
887  SDValue Res = SDValue();
888 
889  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
890  DEBUG(dbgs() << "Node has been custom lowered, done\n");
891  return false;
892  }
893 
894  switch (N->getOpcode()) {
895  default:
896  #ifndef NDEBUG
897  dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
898  N->dump(&DAG); dbgs() << "\n";
899  #endif
900  llvm_unreachable("Do not know how to promote this operator's operand!");
901 
902  case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
903  case ISD::ATOMIC_STORE:
904  Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
905  break;
906  case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
907  case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
908  case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
909  case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
910  case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
911  case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
912  case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
914  Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
916  Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
917  case ISD::VSELECT:
918  case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
919  case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
920  case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
921  case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
922  case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
923  case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
924  OpNo); break;
925  case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
926  OpNo); break;
927  case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
928  OpNo); break;
929  case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
930  OpNo); break;
931  case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
932  OpNo); break;
933  case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
934  case ISD::FP16_TO_FP:
935  case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
936  case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
937  case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
938 
939  case ISD::SHL:
940  case ISD::SRA:
941  case ISD::SRL:
942  case ISD::ROTL:
943  case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
944 
945  case ISD::ADDCARRY:
946  case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
947  }
948 
949  // If the result is null, the sub-method took care of registering results etc.
950  if (!Res.getNode()) return false;
951 
952  // If the result is N, the sub-method updated N in place. Tell the legalizer
953  // core about this.
954  if (Res.getNode() == N)
955  return true;
956 
957  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
958  "Invalid operand expansion");
959 
960  ReplaceValueWith(SDValue(N, 0), Res);
961  return false;
962 }
963 
964 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
965 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
966 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
967  ISD::CondCode CCCode) {
968  // We have to insert explicit sign or zero extends. Note that we could
969  // insert sign extends for ALL conditions, but zero extend is cheaper on
970  // many machines (an AND instead of two shifts), so prefer it.
971  switch (CCCode) {
972  default: llvm_unreachable("Unknown integer comparison!");
973  case ISD::SETEQ:
974  case ISD::SETNE: {
975  SDValue OpL = GetPromotedInteger(NewLHS);
976  SDValue OpR = GetPromotedInteger(NewRHS);
977 
978  // We would prefer to promote the comparison operand with sign extension.
979  // If the width of OpL/OpR excluding the duplicated sign bits is no greater
980  // than the width of NewLHS/NewRH, we can avoid inserting real truncate
981  // instruction, which is redudant eventually.
982  unsigned OpLEffectiveBits =
983  OpL.getValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
984  unsigned OpREffectiveBits =
985  OpR.getValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
986  if (OpLEffectiveBits <= NewLHS.getValueSizeInBits() &&
987  OpREffectiveBits <= NewRHS.getValueSizeInBits()) {
988  NewLHS = OpL;
989  NewRHS = OpR;
990  } else {
991  NewLHS = ZExtPromotedInteger(NewLHS);
992  NewRHS = ZExtPromotedInteger(NewRHS);
993  }
994  break;
995  }
996  case ISD::SETUGE:
997  case ISD::SETUGT:
998  case ISD::SETULE:
999  case ISD::SETULT:
1000  // ALL of these operations will work if we either sign or zero extend
1001  // the operands (including the unsigned comparisons!). Zero extend is
1002  // usually a simpler/cheaper operation, so prefer it.
1003  NewLHS = ZExtPromotedInteger(NewLHS);
1004  NewRHS = ZExtPromotedInteger(NewRHS);
1005  break;
1006  case ISD::SETGE:
1007  case ISD::SETGT:
1008  case ISD::SETLT:
1009  case ISD::SETLE:
1010  NewLHS = SExtPromotedInteger(NewLHS);
1011  NewRHS = SExtPromotedInteger(NewRHS);
1012  break;
1013  }
1014 }
1015 
1016 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1017  SDValue Op = GetPromotedInteger(N->getOperand(0));
1018  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1019 }
1020 
1021 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1022  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1023  return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1024  N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1025 }
1026 
1027 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1028  // This should only occur in unusual situations like bitcasting to an
1029  // x86_fp80, so just turn it into a store+load
1030  return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1031 }
1032 
1033 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1034  assert(OpNo == 2 && "Don't know how to promote this operand!");
1035 
1036  SDValue LHS = N->getOperand(2);
1037  SDValue RHS = N->getOperand(3);
1038  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1039 
1040  // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1041  // legal types.
1042  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1043  N->getOperand(1), LHS, RHS, N->getOperand(4)),
1044  0);
1045 }
1046 
1047 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1048  assert(OpNo == 1 && "only know how to promote condition");
1049 
1050  // Promote all the way up to the canonical SetCC type.
1051  SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1052 
1053  // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1054  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1055  N->getOperand(2)), 0);
1056 }
1057 
1058 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1059  // Since the result type is legal, the operands must promote to it.
1060  EVT OVT = N->getOperand(0).getValueType();
1061  SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1062  SDValue Hi = GetPromotedInteger(N->getOperand(1));
1063  assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1064  SDLoc dl(N);
1065 
1066  Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1067  DAG.getConstant(OVT.getSizeInBits(), dl,
1068  TLI.getPointerTy(DAG.getDataLayout())));
1069  return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1070 }
1071 
1072 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1073  // The vector type is legal but the element type is not. This implies
1074  // that the vector is a power-of-two in length and that the element
1075  // type does not have a strange size (eg: it is not i1).
1076  EVT VecVT = N->getValueType(0);
1077  unsigned NumElts = VecVT.getVectorNumElements();
1078  assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1079  "Legal vector of one illegal element?");
1080 
1081  // Promote the inserted value. The type does not need to match the
1082  // vector element type. Check that any extra bits introduced will be
1083  // truncated away.
1086  "Type of inserted value narrower than vector element type!");
1087 
1088  SmallVector<SDValue, 16> NewOps;
1089  for (unsigned i = 0; i < NumElts; ++i)
1090  NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1091 
1092  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1093 }
1094 
1095 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1096  unsigned OpNo) {
1097  if (OpNo == 1) {
1098  // Promote the inserted value. This is valid because the type does not
1099  // have to match the vector element type.
1100 
1101  // Check that any extra bits introduced will be truncated away.
1104  "Type of inserted value narrower than vector element type!");
1105  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1106  GetPromotedInteger(N->getOperand(1)),
1107  N->getOperand(2)),
1108  0);
1109  }
1110 
1111  assert(OpNo == 2 && "Different operand and result vector types?");
1112 
1113  // Promote the index.
1114  SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1115  TLI.getVectorIdxTy(DAG.getDataLayout()));
1116  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1117  N->getOperand(1), Idx), 0);
1118 }
1119 
1120 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1121  // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1122  // the operand in place.
1123  return SDValue(DAG.UpdateNodeOperands(N,
1124  GetPromotedInteger(N->getOperand(0))), 0);
1125 }
1126 
1127 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1128  assert(OpNo == 0 && "Only know how to promote the condition!");
1129  SDValue Cond = N->getOperand(0);
1130  EVT OpTy = N->getOperand(1).getValueType();
1131 
1132  if (N->getOpcode() == ISD::VSELECT)
1133  if (SDValue Res = WidenVSELECTAndMask(N))
1134  return Res;
1135 
1136  // Promote all the way up to the canonical SetCC type.
1137  EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1138  Cond = PromoteTargetBoolean(Cond, OpVT);
1139 
1140  return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1141  N->getOperand(2)), 0);
1142 }
1143 
1144 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1145  assert(OpNo == 0 && "Don't know how to promote this operand!");
1146 
1147  SDValue LHS = N->getOperand(0);
1148  SDValue RHS = N->getOperand(1);
1149  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1150 
1151  // The CC (#4) and the possible return values (#2 and #3) have legal types.
1152  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1153  N->getOperand(3), N->getOperand(4)), 0);
1154 }
1155 
1156 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1157  assert(OpNo == 0 && "Don't know how to promote this operand!");
1158 
1159  SDValue LHS = N->getOperand(0);
1160  SDValue RHS = N->getOperand(1);
1161  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1162 
1163  // The CC (#2) is always legal.
1164  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1165 }
1166 
1167 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1168  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1169  ZExtPromotedInteger(N->getOperand(1))), 0);
1170 }
1171 
1172 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1173  SDValue Op = GetPromotedInteger(N->getOperand(0));
1174  SDLoc dl(N);
1175  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1176  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1177  Op, DAG.getValueType(N->getOperand(0).getValueType()));
1178 }
1179 
1180 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1181  return SDValue(DAG.UpdateNodeOperands(N,
1182  SExtPromotedInteger(N->getOperand(0))), 0);
1183 }
1184 
1185 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1186  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1187  SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1188  SDLoc dl(N);
1189 
1190  SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1191 
1192  // Truncate the value and store the result.
1193  return DAG.getTruncStore(Ch, dl, Val, Ptr,
1194  N->getMemoryVT(), N->getMemOperand());
1195 }
1196 
1197 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1198  unsigned OpNo) {
1199 
1200  SDValue DataOp = N->getValue();
1201  EVT DataVT = DataOp.getValueType();
1202  SDValue Mask = N->getMask();
1203  SDLoc dl(N);
1204 
1205  bool TruncateStore = false;
1206  if (OpNo == 2) {
1207  // Mask comes before the data operand. If the data operand is legal, we just
1208  // promote the mask.
1209  // When the data operand has illegal type, we should legalize the data
1210  // operand first. The mask will be promoted/splitted/widened according to
1211  // the data operand type.
1212  if (TLI.isTypeLegal(DataVT))
1213  Mask = PromoteTargetBoolean(Mask, DataVT);
1214  else {
1215  if (getTypeAction(DataVT) == TargetLowering::TypePromoteInteger)
1216  return PromoteIntOp_MSTORE(N, 3);
1217 
1218  else if (getTypeAction(DataVT) == TargetLowering::TypeWidenVector)
1219  return WidenVecOp_MSTORE(N, 3);
1220 
1221  else {
1222  assert (getTypeAction(DataVT) == TargetLowering::TypeSplitVector);
1223  return SplitVecOp_MSTORE(N, 3);
1224  }
1225  }
1226  } else { // Data operand
1227  assert(OpNo == 3 && "Unexpected operand for promotion");
1228  DataOp = GetPromotedInteger(DataOp);
1229  Mask = PromoteTargetBoolean(Mask, DataOp.getValueType());
1230  TruncateStore = true;
1231  }
1232 
1233  return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask,
1234  N->getMemoryVT(), N->getMemOperand(),
1235  TruncateStore, N->isCompressingStore());
1236 }
1237 
1238 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1239  unsigned OpNo) {
1240  assert(OpNo == 2 && "Only know how to promote the mask!");
1241  EVT DataVT = N->getValueType(0);
1242  SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1243  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1244  NewOps[OpNo] = Mask;
1245  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1246 }
1247 
1248 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1249  unsigned OpNo) {
1250 
1251  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1252  if (OpNo == 2) {
1253  // The Mask
1254  EVT DataVT = N->getValueType(0);
1255  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1256  } else
1257  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1258 
1259  SDValue Res = SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1260  // updated in place.
1261  if (Res.getNode() == N)
1262  return Res;
1263 
1264  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
1265  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1266  return SDValue();
1267 }
1268 
1269 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1270  unsigned OpNo) {
1271  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1272  if (OpNo == 2) {
1273  // The Mask
1274  EVT DataVT = N->getValue().getValueType();
1275  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1276  } else
1277  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1278  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1279 }
1280 
1281 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
1282  SDValue Op = GetPromotedInteger(N->getOperand(0));
1283  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
1284 }
1285 
1286 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
1287  return SDValue(DAG.UpdateNodeOperands(N,
1288  ZExtPromotedInteger(N->getOperand(0))), 0);
1289 }
1290 
1291 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
1292  SDLoc dl(N);
1293  SDValue Op = GetPromotedInteger(N->getOperand(0));
1294  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1295  return DAG.getZeroExtendInReg(Op, dl,
1297 }
1298 
1299 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
1300  assert(OpNo == 2 && "Don't know how to promote this operand!");
1301 
1302  SDValue LHS = N->getOperand(0);
1303  SDValue RHS = N->getOperand(1);
1304  SDValue Carry = N->getOperand(2);
1305  SDLoc DL(N);
1306 
1307  auto VT = getSetCCResultType(LHS.getValueType());
1309  switch (BoolType) {
1311  Carry = DAG.getAnyExtOrTrunc(Carry, DL, VT);
1312  break;
1314  Carry = DAG.getZExtOrTrunc(Carry, DL, VT);
1315  break;
1317  Carry = DAG.getSExtOrTrunc(Carry, DL, VT);
1318  break;
1319  }
1320 
1321  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
1322 }
1323 
1324 //===----------------------------------------------------------------------===//
1325 // Integer Result Expansion
1326 //===----------------------------------------------------------------------===//
1327 
1328 /// ExpandIntegerResult - This method is called when the specified result of the
1329 /// specified node is found to need expansion. At this point, the node may also
1330 /// have invalid operands or may have other results that need promotion, we just
1331 /// know that (at least) one result needs expansion.
1332 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
1333  DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); dbgs() << "\n");
1334  SDValue Lo, Hi;
1335  Lo = Hi = SDValue();
1336 
1337  // See if the target wants to custom expand this node.
1338  if (CustomLowerNode(N, N->getValueType(ResNo), true))
1339  return;
1340 
1341  switch (N->getOpcode()) {
1342  default:
1343 #ifndef NDEBUG
1344  dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
1345  N->dump(&DAG); dbgs() << "\n";
1346 #endif
1347  llvm_unreachable("Do not know how to expand the result of this operator!");
1348 
1349  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1350  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
1351  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1352  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1353 
1354  case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1355  case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1356  case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1357  case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1358  case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1359 
1360  case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
1361  case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
1362  case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
1363  case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
1364  case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
1365  case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
1366  case ISD::CTLZ_ZERO_UNDEF:
1367  case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
1368  case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
1369  case ISD::CTTZ_ZERO_UNDEF:
1370  case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
1371  case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
1372  case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
1373  case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
1374  case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
1375  case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
1376  case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
1377  case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
1378  case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
1379  case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
1380  case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
1381  case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
1382  case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
1383  case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
1384  case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
1385  case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
1386 
1387  case ISD::ATOMIC_LOAD_ADD:
1388  case ISD::ATOMIC_LOAD_SUB:
1389  case ISD::ATOMIC_LOAD_AND:
1390  case ISD::ATOMIC_LOAD_OR:
1391  case ISD::ATOMIC_LOAD_XOR:
1392  case ISD::ATOMIC_LOAD_NAND:
1393  case ISD::ATOMIC_LOAD_MIN:
1394  case ISD::ATOMIC_LOAD_MAX:
1395  case ISD::ATOMIC_LOAD_UMIN:
1396  case ISD::ATOMIC_LOAD_UMAX:
1397  case ISD::ATOMIC_SWAP:
1398  case ISD::ATOMIC_CMP_SWAP: {
1399  std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
1400  SplitInteger(Tmp.first, Lo, Hi);
1401  ReplaceValueWith(SDValue(N, 1), Tmp.second);
1402  break;
1403  }
1405  AtomicSDNode *AN = cast<AtomicSDNode>(N);
1406  SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
1407  SDValue Tmp = DAG.getAtomicCmpSwap(
1408  ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
1409  N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
1410  AN->getMemOperand());
1411 
1412  // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
1413  // success simply by comparing the loaded value against the ingoing
1414  // comparison.
1415  SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
1416  N->getOperand(2), ISD::SETEQ);
1417 
1418  SplitInteger(Tmp, Lo, Hi);
1419  ReplaceValueWith(SDValue(N, 1), Success);
1420  ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
1421  break;
1422  }
1423 
1424  case ISD::AND:
1425  case ISD::OR:
1426  case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
1427 
1428  case ISD::UMAX:
1429  case ISD::SMAX:
1430  case ISD::UMIN:
1431  case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
1432 
1433  case ISD::ADD:
1434  case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
1435 
1436  case ISD::ADDC:
1437  case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
1438 
1439  case ISD::ADDE:
1440  case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
1441 
1442  case ISD::ADDCARRY:
1443  case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
1444 
1445  case ISD::SHL:
1446  case ISD::SRA:
1447  case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
1448 
1449  case ISD::SADDO:
1450  case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
1451  case ISD::UADDO:
1452  case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
1453  case ISD::UMULO:
1454  case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
1455  }
1456 
1457  // If Lo/Hi is null, the sub-method took care of registering results etc.
1458  if (Lo.getNode())
1459  SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
1460 }
1461 
1462 /// Lower an atomic node to the appropriate builtin call.
1463 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
1464  unsigned Opc = Node->getOpcode();
1465  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
1466  RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
1467  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
1468 
1469  return ExpandChainLibCall(LC, Node, false);
1470 }
1471 
1472 /// N is a shift by a value that needs to be expanded,
1473 /// and the shift amount is a constant 'Amt'. Expand the operation.
1474 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
1475  SDValue &Lo, SDValue &Hi) {
1476  SDLoc DL(N);
1477  // Expand the incoming operand to be shifted, so that we have its parts
1478  SDValue InL, InH;
1479  GetExpandedInteger(N->getOperand(0), InL, InH);
1480 
1481  // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
1482  // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
1483  if (!Amt) {
1484  Lo = InL;
1485  Hi = InH;
1486  return;
1487  }
1488 
1489  EVT NVT = InL.getValueType();
1490  unsigned VTBits = N->getValueType(0).getSizeInBits();
1491  unsigned NVTBits = NVT.getSizeInBits();
1492  EVT ShTy = N->getOperand(1).getValueType();
1493 
1494  if (N->getOpcode() == ISD::SHL) {
1495  if (Amt.ugt(VTBits)) {
1496  Lo = Hi = DAG.getConstant(0, DL, NVT);
1497  } else if (Amt.ugt(NVTBits)) {
1498  Lo = DAG.getConstant(0, DL, NVT);
1499  Hi = DAG.getNode(ISD::SHL, DL,
1500  NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1501  } else if (Amt == NVTBits) {
1502  Lo = DAG.getConstant(0, DL, NVT);
1503  Hi = InL;
1504  } else {
1505  Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
1506  Hi = DAG.getNode(ISD::OR, DL, NVT,
1507  DAG.getNode(ISD::SHL, DL, NVT, InH,
1508  DAG.getConstant(Amt, DL, ShTy)),
1509  DAG.getNode(ISD::SRL, DL, NVT, InL,
1510  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1511  }
1512  return;
1513  }
1514 
1515  if (N->getOpcode() == ISD::SRL) {
1516  if (Amt.ugt(VTBits)) {
1517  Lo = Hi = DAG.getConstant(0, DL, NVT);
1518  } else if (Amt.ugt(NVTBits)) {
1519  Lo = DAG.getNode(ISD::SRL, DL,
1520  NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1521  Hi = DAG.getConstant(0, DL, NVT);
1522  } else if (Amt == NVTBits) {
1523  Lo = InH;
1524  Hi = DAG.getConstant(0, DL, NVT);
1525  } else {
1526  Lo = DAG.getNode(ISD::OR, DL, NVT,
1527  DAG.getNode(ISD::SRL, DL, NVT, InL,
1528  DAG.getConstant(Amt, DL, ShTy)),
1529  DAG.getNode(ISD::SHL, DL, NVT, InH,
1530  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1531  Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1532  }
1533  return;
1534  }
1535 
1536  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
1537  if (Amt.ugt(VTBits)) {
1538  Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1539  DAG.getConstant(NVTBits - 1, DL, ShTy));
1540  } else if (Amt.ugt(NVTBits)) {
1541  Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1542  DAG.getConstant(Amt - NVTBits, DL, ShTy));
1543  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1544  DAG.getConstant(NVTBits - 1, DL, ShTy));
1545  } else if (Amt == NVTBits) {
1546  Lo = InH;
1547  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1548  DAG.getConstant(NVTBits - 1, DL, ShTy));
1549  } else {
1550  Lo = DAG.getNode(ISD::OR, DL, NVT,
1551  DAG.getNode(ISD::SRL, DL, NVT, InL,
1552  DAG.getConstant(Amt, DL, ShTy)),
1553  DAG.getNode(ISD::SHL, DL, NVT, InH,
1554  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1555  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1556  }
1557 }
1558 
1559 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
1560 /// this shift based on knowledge of the high bit of the shift amount. If we
1561 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
1562 /// shift amount.
1563 bool DAGTypeLegalizer::
1564 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1565  SDValue Amt = N->getOperand(1);
1566  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1567  EVT ShTy = Amt.getValueType();
1568  unsigned ShBits = ShTy.getScalarSizeInBits();
1569  unsigned NVTBits = NVT.getScalarSizeInBits();
1570  assert(isPowerOf2_32(NVTBits) &&
1571  "Expanded integer type size not a power of two!");
1572  SDLoc dl(N);
1573 
1574  APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
1575  KnownBits Known;
1576  DAG.computeKnownBits(N->getOperand(1), Known);
1577 
1578  // If we don't know anything about the high bits, exit.
1579  if (((Known.Zero|Known.One) & HighBitMask) == 0)
1580  return false;
1581 
1582  // Get the incoming operand to be shifted.
1583  SDValue InL, InH;
1584  GetExpandedInteger(N->getOperand(0), InL, InH);
1585 
1586  // If we know that any of the high bits of the shift amount are one, then we
1587  // can do this as a couple of simple shifts.
1588  if (Known.One.intersects(HighBitMask)) {
1589  // Mask out the high bit, which we know is set.
1590  Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
1591  DAG.getConstant(~HighBitMask, dl, ShTy));
1592 
1593  switch (N->getOpcode()) {
1594  default: llvm_unreachable("Unknown shift");
1595  case ISD::SHL:
1596  Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
1597  Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
1598  return true;
1599  case ISD::SRL:
1600  Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1601  Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
1602  return true;
1603  case ISD::SRA:
1604  Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
1605  DAG.getConstant(NVTBits - 1, dl, ShTy));
1606  Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
1607  return true;
1608  }
1609  }
1610 
1611  // If we know that all of the high bits of the shift amount are zero, then we
1612  // can do this as a couple of simple shifts.
1613  if (HighBitMask.isSubsetOf(Known.Zero)) {
1614  // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
1615  // shift if x is zero. We can use XOR here because x is known to be smaller
1616  // than 32.
1617  SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
1618  DAG.getConstant(NVTBits - 1, dl, ShTy));
1619 
1620  unsigned Op1, Op2;
1621  switch (N->getOpcode()) {
1622  default: llvm_unreachable("Unknown shift");
1623  case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
1624  case ISD::SRL:
1625  case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
1626  }
1627 
1628  // When shifting right the arithmetic for Lo and Hi is swapped.
1629  if (N->getOpcode() != ISD::SHL)
1630  std::swap(InL, InH);
1631 
1632  // Use a little trick to get the bits that move from Lo to Hi. First
1633  // shift by one bit.
1634  SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
1635  // Then compute the remaining shift with amount-1.
1636  SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
1637 
1638  Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
1639  Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
1640 
1641  if (N->getOpcode() != ISD::SHL)
1642  std::swap(Hi, Lo);
1643  return true;
1644  }
1645 
1646  return false;
1647 }
1648 
1649 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
1650 /// of any size.
1651 bool DAGTypeLegalizer::
1652 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1653  SDValue Amt = N->getOperand(1);
1654  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1655  EVT ShTy = Amt.getValueType();
1656  unsigned NVTBits = NVT.getSizeInBits();
1657  assert(isPowerOf2_32(NVTBits) &&
1658  "Expanded integer type size not a power of two!");
1659  SDLoc dl(N);
1660 
1661  // Get the incoming operand to be shifted.
1662  SDValue InL, InH;
1663  GetExpandedInteger(N->getOperand(0), InL, InH);
1664 
1665  SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
1666  SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
1667  SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
1668  SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1669  Amt, NVBitsNode, ISD::SETULT);
1670  SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1671  Amt, DAG.getConstant(0, dl, ShTy),
1672  ISD::SETEQ);
1673 
1674  SDValue LoS, HiS, LoL, HiL;
1675  switch (N->getOpcode()) {
1676  default: llvm_unreachable("Unknown shift");
1677  case ISD::SHL:
1678  // Short: ShAmt < NVTBits
1679  LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
1680  HiS = DAG.getNode(ISD::OR, dl, NVT,
1681  DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
1682  DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
1683 
1684  // Long: ShAmt >= NVTBits
1685  LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
1686  HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
1687 
1688  Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
1689  Hi = DAG.getSelect(dl, NVT, isZero, InH,
1690  DAG.getSelect(dl, NVT, isShort, HiS, HiL));
1691  return true;
1692  case ISD::SRL:
1693  // Short: ShAmt < NVTBits
1694  HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
1695  LoS = DAG.getNode(ISD::OR, dl, NVT,
1696  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1697  // FIXME: If Amt is zero, the following shift generates an undefined result
1698  // on some architectures.
1699  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1700 
1701  // Long: ShAmt >= NVTBits
1702  HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1703  LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1704 
1705  Lo = DAG.getSelect(dl, NVT, isZero, InL,
1706  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1707  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1708  return true;
1709  case ISD::SRA:
1710  // Short: ShAmt < NVTBits
1711  HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
1712  LoS = DAG.getNode(ISD::OR, dl, NVT,
1713  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1714  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1715 
1716  // Long: ShAmt >= NVTBits
1717  HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
1718  DAG.getConstant(NVTBits - 1, dl, ShTy));
1719  LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1720 
1721  Lo = DAG.getSelect(dl, NVT, isZero, InL,
1722  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1723  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1724  return true;
1725  }
1726 }
1727 
1728 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
1729 
1730  switch (Op) {
1731  default: llvm_unreachable("invalid min/max opcode");
1732  case ISD::SMAX:
1733  return std::make_pair(ISD::SETGT, ISD::UMAX);
1734  case ISD::UMAX:
1735  return std::make_pair(ISD::SETUGT, ISD::UMAX);
1736  case ISD::SMIN:
1737  return std::make_pair(ISD::SETLT, ISD::UMIN);
1738  case ISD::UMIN:
1739  return std::make_pair(ISD::SETULT, ISD::UMIN);
1740  }
1741 }
1742 
1743 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
1744  SDValue &Lo, SDValue &Hi) {
1745  SDLoc DL(N);
1746  ISD::NodeType LoOpc;
1747  ISD::CondCode CondC;
1748  std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
1749 
1750  // Expand the subcomponents.
1751  SDValue LHSL, LHSH, RHSL, RHSH;
1752  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1753  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1754 
1755  // Value types
1756  EVT NVT = LHSL.getValueType();
1757  EVT CCT = getSetCCResultType(NVT);
1758 
1759  // Hi part is always the same op
1760  Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
1761 
1762  // We need to know whether to select Lo part that corresponds to 'winning'
1763  // Hi part or if Hi parts are equal.
1764  SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
1765  SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
1766 
1767  // Lo part corresponding to the 'winning' Hi part
1768  SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
1769 
1770  // Recursed Lo part if Hi parts are equal, this uses unsigned version
1771  SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
1772 
1773  Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
1774 }
1775 
1776 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
1777  SDValue &Lo, SDValue &Hi) {
1778  SDLoc dl(N);
1779  // Expand the subcomponents.
1780  SDValue LHSL, LHSH, RHSL, RHSH;
1781  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1782  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1783 
1784  EVT NVT = LHSL.getValueType();
1785  SDValue LoOps[2] = { LHSL, RHSL };
1786  SDValue HiOps[3] = { LHSH, RHSH };
1787 
1788  bool HasOpCarry = TLI.isOperationLegalOrCustom(
1790  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1791  if (HasOpCarry) {
1792  SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
1793  if (N->getOpcode() == ISD::ADD) {
1794  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
1795  HiOps[2] = Lo.getValue(1);
1796  Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
1797  } else {
1798  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
1799  HiOps[2] = Lo.getValue(1);
1800  Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
1801  }
1802  return;
1803  }
1804 
1805  // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
1806  // them. TODO: Teach operation legalization how to expand unsupported
1807  // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
1808  // a carry of type MVT::Glue, but there doesn't seem to be any way to
1809  // generate a value of this type in the expanded code sequence.
1810  bool hasCarry =
1812  ISD::ADDC : ISD::SUBC,
1813  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1814 
1815  if (hasCarry) {
1816  SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
1817  if (N->getOpcode() == ISD::ADD) {
1818  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
1819  HiOps[2] = Lo.getValue(1);
1820  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
1821  } else {
1822  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
1823  HiOps[2] = Lo.getValue(1);
1824  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
1825  }
1826  return;
1827  }
1828 
1829  bool hasOVF =
1832  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1834 
1835  if (hasOVF) {
1836  EVT OvfVT = getSetCCResultType(NVT);
1837  SDVTList VTList = DAG.getVTList(NVT, OvfVT);
1838  int RevOpc;
1839  if (N->getOpcode() == ISD::ADD) {
1840  RevOpc = ISD::SUB;
1841  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
1842  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
1843  } else {
1844  RevOpc = ISD::ADD;
1845  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
1846  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
1847  }
1848  SDValue OVF = Lo.getValue(1);
1849 
1850  switch (BoolType) {
1852  OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
1855  OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
1856  Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
1857  break;
1859  OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
1860  Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
1861  }
1862  return;
1863  }
1864 
1865  if (N->getOpcode() == ISD::ADD) {
1866  Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
1867  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
1868  SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
1869  ISD::SETULT);
1870 
1872  SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
1873  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
1874  return;
1875  }
1876 
1877  SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
1878  DAG.getConstant(1, dl, NVT),
1879  DAG.getConstant(0, dl, NVT));
1880  SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
1881  ISD::SETULT);
1882  SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
1883  DAG.getConstant(1, dl, NVT), Carry1);
1884  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
1885  } else {
1886  Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
1887  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
1888  SDValue Cmp =
1889  DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
1890  LoOps[0], LoOps[1], ISD::SETULT);
1891 
1892  SDValue Borrow;
1894  Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
1895  else
1896  Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
1897  DAG.getConstant(0, dl, NVT));
1898 
1899  Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
1900  }
1901 }
1902 
1903 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
1904  SDValue &Lo, SDValue &Hi) {
1905  // Expand the subcomponents.
1906  SDValue LHSL, LHSH, RHSL, RHSH;
1907  SDLoc dl(N);
1908  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1909  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1910  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
1911  SDValue LoOps[2] = { LHSL, RHSL };
1912  SDValue HiOps[3] = { LHSH, RHSH };
1913 
1914  if (N->getOpcode() == ISD::ADDC) {
1915  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
1916  HiOps[2] = Lo.getValue(1);
1917  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
1918  } else {
1919  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
1920  HiOps[2] = Lo.getValue(1);
1921  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
1922  }
1923 
1924  // Legalized the flag result - switch anything that used the old flag to
1925  // use the new one.
1926  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
1927 }
1928 
1929 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
1930  SDValue &Lo, SDValue &Hi) {
1931  // Expand the subcomponents.
1932  SDValue LHSL, LHSH, RHSL, RHSH;
1933  SDLoc dl(N);
1934  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1935  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1936  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
1937  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
1938  SDValue HiOps[3] = { LHSH, RHSH };
1939 
1940  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
1941  HiOps[2] = Lo.getValue(1);
1942  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
1943 
1944  // Legalized the flag result - switch anything that used the old flag to
1945  // use the new one.
1946  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
1947 }
1948 
1949 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
1950  SDValue &Lo, SDValue &Hi) {
1951  SDValue LHS = N->getOperand(0);
1952  SDValue RHS = N->getOperand(1);
1953  SDLoc dl(N);
1954 
1955  SDValue Ovf;
1956 
1957  bool HasOpCarry = TLI.isOperationLegalOrCustom(
1959  TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
1960 
1961  if (HasOpCarry) {
1962  // Expand the subcomponents.
1963  SDValue LHSL, LHSH, RHSL, RHSH;
1964  GetExpandedInteger(LHS, LHSL, LHSH);
1965  GetExpandedInteger(RHS, RHSL, RHSH);
1966  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
1967  SDValue LoOps[2] = { LHSL, RHSL };
1968  SDValue HiOps[3] = { LHSH, RHSH };
1969 
1970  unsigned Opc = N->getOpcode() == ISD::UADDO ? ISD::ADDCARRY : ISD::SUBCARRY;
1971  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
1972  HiOps[2] = Lo.getValue(1);
1973  Hi = DAG.getNode(Opc, dl, VTList, HiOps);
1974 
1975  Ovf = Hi.getValue(1);
1976  } else {
1977  // Expand the result by simply replacing it with the equivalent
1978  // non-overflow-checking operation.
1979  auto Opc = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
1980  SDValue Sum = DAG.getNode(Opc, dl, LHS.getValueType(), LHS, RHS);
1981  SplitInteger(Sum, Lo, Hi);
1982 
1983  // Calculate the overflow: addition overflows iff a + b < a, and subtraction
1984  // overflows iff a - b > a.
1985  auto Cond = N->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
1986  Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
1987  }
1988 
1989  // Legalized the flag result - switch anything that used the old flag to
1990  // use the new one.
1991  ReplaceValueWith(SDValue(N, 1), Ovf);
1992 }
1993 
1994 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
1995  SDValue &Lo, SDValue &Hi) {
1996  // Expand the subcomponents.
1997  SDValue LHSL, LHSH, RHSL, RHSH;
1998  SDLoc dl(N);
1999  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2000  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2001  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2002  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2003  SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
2004 
2005  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2006  HiOps[2] = Lo.getValue(1);
2007  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2008 
2009  // Legalized the flag result - switch anything that used the old flag to
2010  // use the new one.
2011  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2012 }
2013 
2014 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
2015  SDValue &Lo, SDValue &Hi) {
2016  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2017  SDLoc dl(N);
2018  SDValue Op = N->getOperand(0);
2019  if (Op.getValueType().bitsLE(NVT)) {
2020  // The low part is any extension of the input (which degenerates to a copy).
2021  Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
2022  Hi = DAG.getUNDEF(NVT); // The high part is undefined.
2023  } else {
2024  // For example, extension of an i48 to an i64. The operand type necessarily
2025  // promotes to the result type, so will end up being expanded too.
2026  assert(getTypeAction(Op.getValueType()) ==
2028  "Only know how to promote this result!");
2029  SDValue Res = GetPromotedInteger(Op);
2030  assert(Res.getValueType() == N->getValueType(0) &&
2031  "Operand over promoted?");
2032  // Split the promoted operand. This will simplify when it is expanded.
2033  SplitInteger(Res, Lo, Hi);
2034  }
2035 }
2036 
2037 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
2038  SDValue &Lo, SDValue &Hi) {
2039  SDLoc dl(N);
2040  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2041  EVT NVT = Lo.getValueType();
2042  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2043  unsigned NVTBits = NVT.getSizeInBits();
2044  unsigned EVTBits = EVT.getSizeInBits();
2045 
2046  if (NVTBits < EVTBits) {
2047  Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
2049  EVTBits - NVTBits)));
2050  } else {
2051  Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
2052  // The high part replicates the sign bit of Lo, make it explicit.
2053  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2054  DAG.getConstant(NVTBits - 1, dl,
2055  TLI.getPointerTy(DAG.getDataLayout())));
2056  }
2057 }
2058 
2059 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
2060  SDValue &Lo, SDValue &Hi) {
2061  SDLoc dl(N);
2062  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2063  EVT NVT = Lo.getValueType();
2064  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2065  unsigned NVTBits = NVT.getSizeInBits();
2066  unsigned EVTBits = EVT.getSizeInBits();
2067 
2068  if (NVTBits < EVTBits) {
2069  Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
2071  EVTBits - NVTBits)));
2072  } else {
2073  Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
2074  // The high part must be zero, make it explicit.
2075  Hi = DAG.getConstant(0, dl, NVT);
2076  }
2077 }
2078 
2079 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
2080  SDValue &Lo, SDValue &Hi) {
2081  SDLoc dl(N);
2082  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2083  Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
2084  Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
2085 }
2086 
2087 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
2088  SDValue &Lo, SDValue &Hi) {
2089  SDLoc dl(N);
2090  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2091  Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
2092  Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
2093 }
2094 
2095 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
2096  SDValue &Lo, SDValue &Hi) {
2097  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2098  unsigned NBitWidth = NVT.getSizeInBits();
2099  auto Constant = cast<ConstantSDNode>(N);
2100  const APInt &Cst = Constant->getAPIntValue();
2101  bool IsTarget = Constant->isTargetOpcode();
2102  bool IsOpaque = Constant->isOpaque();
2103  SDLoc dl(N);
2104  Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
2105  Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
2106  IsOpaque);
2107 }
2108 
2109 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
2110  SDValue &Lo, SDValue &Hi) {
2111  SDLoc dl(N);
2112  // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
2113  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2114  EVT NVT = Lo.getValueType();
2115 
2116  SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
2117  DAG.getConstant(0, dl, NVT), ISD::SETNE);
2118 
2119  SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
2120  SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
2121 
2122  Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
2123  DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
2124  DAG.getConstant(NVT.getSizeInBits(), dl,
2125  NVT)));
2126  Hi = DAG.getConstant(0, dl, NVT);
2127 }
2128 
2129 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
2130  SDValue &Lo, SDValue &Hi) {
2131  SDLoc dl(N);
2132  // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
2133  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2134  EVT NVT = Lo.getValueType();
2135  Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
2136  DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
2137  Hi = DAG.getConstant(0, dl, NVT);
2138 }
2139 
2140 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
2141  SDValue &Lo, SDValue &Hi) {
2142  SDLoc dl(N);
2143  // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
2144  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2145  EVT NVT = Lo.getValueType();
2146 
2147  SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
2148  DAG.getConstant(0, dl, NVT), ISD::SETNE);
2149 
2150  SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
2151  SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
2152 
2153  Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
2154  DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
2155  DAG.getConstant(NVT.getSizeInBits(), dl,
2156  NVT)));
2157  Hi = DAG.getConstant(0, dl, NVT);
2158 }
2159 
2160 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
2161  SDValue &Hi) {
2162  SDLoc dl(N);
2163  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2164  unsigned NBitWidth = NVT.getSizeInBits();
2165 
2166  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2167  Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT);
2168  // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
2169  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2170  DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
2171 }
2172 
2173 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
2174  SDValue &Hi) {
2175  SDLoc dl(N);
2176  EVT VT = N->getValueType(0);
2177 
2178  SDValue Op = N->getOperand(0);
2179  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2180  Op = GetPromotedFloat(Op);
2181 
2183  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
2184  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, true/*irrelevant*/, dl).first,
2185  Lo, Hi);
2186 }
2187 
2188 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
2189  SDValue &Hi) {
2190  SDLoc dl(N);
2191  EVT VT = N->getValueType(0);
2192 
2193  SDValue Op = N->getOperand(0);
2194  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2195  Op = GetPromotedFloat(Op);
2196 
2198  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
2199  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, false/*irrelevant*/, dl).first,
2200  Lo, Hi);
2201 }
2202 
2203 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
2204  SDValue &Lo, SDValue &Hi) {
2205  if (ISD::isNormalLoad(N)) {
2206  ExpandRes_NormalLoad(N, Lo, Hi);
2207  return;
2208  }
2209 
2210  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2211 
2212  EVT VT = N->getValueType(0);
2213  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2214  SDValue Ch = N->getChain();
2215  SDValue Ptr = N->getBasePtr();
2216  ISD::LoadExtType ExtType = N->getExtensionType();
2217  unsigned Alignment = N->getAlignment();
2218  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2219  AAMDNodes AAInfo = N->getAAInfo();
2220  SDLoc dl(N);
2221 
2222  assert(NVT.isByteSized() && "Expanded type not byte sized!");
2223 
2224  if (N->getMemoryVT().bitsLE(NVT)) {
2225  EVT MemVT = N->getMemoryVT();
2226 
2227  Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
2228  Alignment, MMOFlags, AAInfo);
2229 
2230  // Remember the chain.
2231  Ch = Lo.getValue(1);
2232 
2233  if (ExtType == ISD::SEXTLOAD) {
2234  // The high part is obtained by SRA'ing all but one of the bits of the
2235  // lo part.
2236  unsigned LoSize = Lo.getValueSizeInBits();
2237  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2238  DAG.getConstant(LoSize - 1, dl,
2239  TLI.getPointerTy(DAG.getDataLayout())));
2240  } else if (ExtType == ISD::ZEXTLOAD) {
2241  // The high part is just a zero.
2242  Hi = DAG.getConstant(0, dl, NVT);
2243  } else {
2244  assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
2245  // The high part is undefined.
2246  Hi = DAG.getUNDEF(NVT);
2247  }
2248  } else if (DAG.getDataLayout().isLittleEndian()) {
2249  // Little-endian - low bits are at low addresses.
2250  Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2251  AAInfo);
2252 
2253  unsigned ExcessBits =
2254  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
2255  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
2256 
2257  // Increment the pointer to the other half.
2258  unsigned IncrementSize = NVT.getSizeInBits()/8;
2259  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2260  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2261  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
2262  N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
2263  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2264 
2265  // Build a factor node to remember that this load is independent of the
2266  // other one.
2267  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2268  Hi.getValue(1));
2269  } else {
2270  // Big-endian - high bits are at low addresses. Favor aligned loads at
2271  // the cost of some bit-fiddling.
2272  EVT MemVT = N->getMemoryVT();
2273  unsigned EBytes = MemVT.getStoreSize();
2274  unsigned IncrementSize = NVT.getSizeInBits()/8;
2275  unsigned ExcessBits = (EBytes - IncrementSize)*8;
2276 
2277  // Load both the high bits and maybe some of the low bits.
2278  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
2280  MemVT.getSizeInBits() - ExcessBits),
2281  Alignment, MMOFlags, AAInfo);
2282 
2283  // Increment the pointer to the other half.
2284  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2285  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2286  // Load the rest of the low bits.
2287  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
2288  N->getPointerInfo().getWithOffset(IncrementSize),
2289  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
2290  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2291 
2292  // Build a factor node to remember that this load is independent of the
2293  // other one.
2294  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2295  Hi.getValue(1));
2296 
2297  if (ExcessBits < NVT.getSizeInBits()) {
2298  // Transfer low bits from the bottom of Hi to the top of Lo.
2299  Lo = DAG.getNode(
2300  ISD::OR, dl, NVT, Lo,
2301  DAG.getNode(ISD::SHL, dl, NVT, Hi,
2302  DAG.getConstant(ExcessBits, dl,
2303  TLI.getPointerTy(DAG.getDataLayout()))));
2304  // Move high bits to the right position in Hi.
2305  Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
2306  Hi,
2307  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
2308  TLI.getPointerTy(DAG.getDataLayout())));
2309  }
2310  }
2311 
2312  // Legalize the chain result - switch anything that used the old chain to
2313  // use the new one.
2314  ReplaceValueWith(SDValue(N, 1), Ch);
2315 }
2316 
2317 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
2318  SDValue &Lo, SDValue &Hi) {
2319  SDLoc dl(N);
2320  SDValue LL, LH, RL, RH;
2321  GetExpandedInteger(N->getOperand(0), LL, LH);
2322  GetExpandedInteger(N->getOperand(1), RL, RH);
2323  Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
2324  Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
2325 }
2326 
2327 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
2328  SDValue &Lo, SDValue &Hi) {
2329  EVT VT = N->getValueType(0);
2330  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2331  SDLoc dl(N);
2332 
2333  SDValue LL, LH, RL, RH;
2334  GetExpandedInteger(N->getOperand(0), LL, LH);
2335  GetExpandedInteger(N->getOperand(1), RL, RH);
2336 
2337  if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
2339  LL, LH, RL, RH))
2340  return;
2341 
2342  // If nothing else, we can make a libcall.
2343  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2344  if (VT == MVT::i16)
2345  LC = RTLIB::MUL_I16;
2346  else if (VT == MVT::i32)
2347  LC = RTLIB::MUL_I32;
2348  else if (VT == MVT::i64)
2349  LC = RTLIB::MUL_I64;
2350  else if (VT == MVT::i128)
2351  LC = RTLIB::MUL_I128;
2352 
2353  if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
2354  // We'll expand the multiplication by brute force because we have no other
2355  // options. This is a trivially-generalized version of the code from
2356  // Hacker's Delight (itself derived from Knuth's Algorithm M from section
2357  // 4.3.1).
2358  unsigned Bits = NVT.getSizeInBits();
2359  unsigned HalfBits = Bits >> 1;
2360  SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
2361  NVT);
2362  SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
2363  SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
2364 
2365  SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
2366  SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
2367 
2368  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2369  if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
2370  // The type from TLI is too small to fit the shift amount we want.
2371  // Override it with i32. The shift will have to be legalized.
2372  ShiftAmtTy = MVT::i32;
2373  }
2374  SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
2375  SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
2376  SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
2377  SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
2378 
2379  SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
2380  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
2381  SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
2382  SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
2383 
2384  SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
2385  DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
2386  SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
2387 
2388  SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
2389  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
2390  DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
2391  Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
2392  DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
2393 
2394  Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
2395  DAG.getNode(ISD::ADD, dl, NVT,
2396  DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
2397  DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
2398  return;
2399  }
2400 
2401  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2402  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true/*irrelevant*/, dl).first,
2403  Lo, Hi);
2404 }
2405 
2406 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
2407  SDValue &Hi) {
2408  SDLoc DL(N);
2409  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2410  SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
2411  SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
2412  Lo = R.getValue(0);
2413  Hi = R.getValue(1);
2414  ReplaceValueWith(SDValue(N, 1), R.getValue(2));
2415 }
2416 
2417 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
2418  SDValue &Lo, SDValue &Hi) {
2419  SDValue LHS = Node->getOperand(0);
2420  SDValue RHS = Node->getOperand(1);
2421  SDLoc dl(Node);
2422 
2423  // Expand the result by simply replacing it with the equivalent
2424  // non-overflow-checking operation.
2425  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
2426  ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
2427  LHS, RHS);
2428  SplitInteger(Sum, Lo, Hi);
2429 
2430  // Compute the overflow.
2431  //
2432  // LHSSign -> LHS >= 0
2433  // RHSSign -> RHS >= 0
2434  // SumSign -> Sum >= 0
2435  //
2436  // Add:
2437  // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
2438  // Sub:
2439  // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
2440  //
2441  EVT OType = Node->getValueType(1);
2442  SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
2443 
2444  SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
2445  SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
2446  SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
2447  Node->getOpcode() == ISD::SADDO ?
2449 
2450  SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
2451  SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
2452 
2453  SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
2454 
2455  // Use the calculated overflow everywhere.
2456  ReplaceValueWith(SDValue(Node, 1), Cmp);
2457 }
2458 
2459 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
2460  SDValue &Lo, SDValue &Hi) {
2461  EVT VT = N->getValueType(0);
2462  SDLoc dl(N);
2463  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2464 
2466  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2467  SplitInteger(Res.getValue(0), Lo, Hi);
2468  return;
2469  }
2470 
2471  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2472  if (VT == MVT::i16)
2473  LC = RTLIB::SDIV_I16;
2474  else if (VT == MVT::i32)
2475  LC = RTLIB::SDIV_I32;
2476  else if (VT == MVT::i64)
2477  LC = RTLIB::SDIV_I64;
2478  else if (VT == MVT::i128)
2479  LC = RTLIB::SDIV_I128;
2480  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
2481 
2482  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
2483 }
2484 
2485 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
2486  SDValue &Lo, SDValue &Hi) {
2487  EVT VT = N->getValueType(0);
2488  SDLoc dl(N);
2489 
2490  // If we can emit an efficient shift operation, do so now. Check to see if
2491  // the RHS is a constant.
2492  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
2493  return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
2494 
2495  // If we can determine that the high bit of the shift is zero or one, even if
2496  // the low bits are variable, emit this shift in an optimized form.
2497  if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
2498  return;
2499 
2500  // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
2501  unsigned PartsOpc;
2502  if (N->getOpcode() == ISD::SHL) {
2503  PartsOpc = ISD::SHL_PARTS;
2504  } else if (N->getOpcode() == ISD::SRL) {
2505  PartsOpc = ISD::SRL_PARTS;
2506  } else {
2507  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2508  PartsOpc = ISD::SRA_PARTS;
2509  }
2510 
2511  // Next check to see if the target supports this SHL_PARTS operation or if it
2512  // will custom expand it.
2513  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2514  TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
2515  if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
2516  Action == TargetLowering::Custom) {
2517  // Expand the subcomponents.
2518  SDValue LHSL, LHSH;
2519  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2520  EVT VT = LHSL.getValueType();
2521 
2522  // If the shift amount operand is coming from a vector legalization it may
2523  // have an illegal type. Fix that first by casting the operand, otherwise
2524  // the new SHL_PARTS operation would need further legalization.
2525  SDValue ShiftOp = N->getOperand(1);
2526  EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2527  assert(ShiftTy.getScalarSizeInBits() >=
2528  Log2_32_Ceil(VT.getScalarSizeInBits()) &&
2529  "ShiftAmountTy is too small to cover the range of this type!");
2530  if (ShiftOp.getValueType() != ShiftTy)
2531  ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
2532 
2533  SDValue Ops[] = { LHSL, LHSH, ShiftOp };
2534  Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
2535  Hi = Lo.getValue(1);
2536  return;
2537  }
2538 
2539  // Otherwise, emit a libcall.
2540  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2541  bool isSigned;
2542  if (N->getOpcode() == ISD::SHL) {
2543  isSigned = false; /*sign irrelevant*/
2544  if (VT == MVT::i16)
2545  LC = RTLIB::SHL_I16;
2546  else if (VT == MVT::i32)
2547  LC = RTLIB::SHL_I32;
2548  else if (VT == MVT::i64)
2549  LC = RTLIB::SHL_I64;
2550  else if (VT == MVT::i128)
2551  LC = RTLIB::SHL_I128;
2552  } else if (N->getOpcode() == ISD::SRL) {
2553  isSigned = false;
2554  if (VT == MVT::i16)
2555  LC = RTLIB::SRL_I16;
2556  else if (VT == MVT::i32)
2557  LC = RTLIB::SRL_I32;
2558  else if (VT == MVT::i64)
2559  LC = RTLIB::SRL_I64;
2560  else if (VT == MVT::i128)
2561  LC = RTLIB::SRL_I128;
2562  } else {
2563  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2564  isSigned = true;
2565  if (VT == MVT::i16)
2566  LC = RTLIB::SRA_I16;
2567  else if (VT == MVT::i32)
2568  LC = RTLIB::SRA_I32;
2569  else if (VT == MVT::i64)
2570  LC = RTLIB::SRA_I64;
2571  else if (VT == MVT::i128)
2572  LC = RTLIB::SRA_I128;
2573  }
2574 
2575  if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
2576  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2577  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, isSigned, dl).first, Lo, Hi);
2578  return;
2579  }
2580 
2581  if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
2582  llvm_unreachable("Unsupported shift!");
2583 }
2584 
2585 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
2586  SDValue &Lo, SDValue &Hi) {
2587  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2588  SDLoc dl(N);
2589  SDValue Op = N->getOperand(0);
2590  if (Op.getValueType().bitsLE(NVT)) {
2591  // The low part is sign extension of the input (degenerates to a copy).
2592  Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
2593  // The high part is obtained by SRA'ing all but one of the bits of low part.
2594  unsigned LoSize = NVT.getSizeInBits();
2595  Hi = DAG.getNode(
2596  ISD::SRA, dl, NVT, Lo,
2597  DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
2598  } else {
2599  // For example, extension of an i48 to an i64. The operand type necessarily
2600  // promotes to the result type, so will end up being expanded too.
2601  assert(getTypeAction(Op.getValueType()) ==
2603  "Only know how to promote this result!");
2604  SDValue Res = GetPromotedInteger(Op);
2605  assert(Res.getValueType() == N->getValueType(0) &&
2606  "Operand over promoted?");
2607  // Split the promoted operand. This will simplify when it is expanded.
2608  SplitInteger(Res, Lo, Hi);
2609  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
2610  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2612  ExcessBits)));
2613  }
2614 }
2615 
2616 void DAGTypeLegalizer::
2617 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2618  SDLoc dl(N);
2619  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2620  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2621 
2622  if (EVT.bitsLE(Lo.getValueType())) {
2623  // sext_inreg the low part if needed.
2624  Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
2625  N->getOperand(1));
2626 
2627  // The high part gets the sign extension from the lo-part. This handles
2628  // things like sextinreg V:i64 from i8.
2629  Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
2630  DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
2631  TLI.getPointerTy(DAG.getDataLayout())));
2632  } else {
2633  // For example, extension of an i48 to an i64. Leave the low part alone,
2634  // sext_inreg the high part.
2635  unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
2636  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2638  ExcessBits)));
2639  }
2640 }
2641 
2642 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
2643  SDValue &Lo, SDValue &Hi) {
2644  EVT VT = N->getValueType(0);
2645  SDLoc dl(N);
2646  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2647 
2649  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2650  SplitInteger(Res.getValue(1), Lo, Hi);
2651  return;
2652  }
2653 
2654  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2655  if (VT == MVT::i16)
2656  LC = RTLIB::SREM_I16;
2657  else if (VT == MVT::i32)
2658  LC = RTLIB::SREM_I32;
2659  else if (VT == MVT::i64)
2660  LC = RTLIB::SREM_I64;
2661  else if (VT == MVT::i128)
2662  LC = RTLIB::SREM_I128;
2663  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
2664 
2665  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
2666 }
2667 
2668 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
2669  SDValue &Lo, SDValue &Hi) {
2670  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2671  SDLoc dl(N);
2672  Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
2673  Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
2674  N->getOperand(0),
2675  DAG.getConstant(NVT.getSizeInBits(), dl,
2676  TLI.getPointerTy(DAG.getDataLayout())));
2677  Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
2678 }
2679 
2680 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
2681  SDValue &Lo, SDValue &Hi) {
2682  EVT VT = N->getValueType(0);
2683  SDLoc dl(N);
2684 
2685  // A divide for UMULO should be faster than a function call.
2686  if (N->getOpcode() == ISD::UMULO) {
2687  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
2688 
2689  SDValue MUL = DAG.getNode(ISD::MUL, dl, LHS.getValueType(), LHS, RHS);
2690  SplitInteger(MUL, Lo, Hi);
2691 
2692  // A divide for UMULO will be faster than a function call. Select to
2693  // make sure we aren't using 0.
2694  SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(VT),
2695  RHS, DAG.getConstant(0, dl, VT), ISD::SETEQ);
2696  SDValue NotZero = DAG.getSelect(dl, VT, isZero,
2697  DAG.getConstant(1, dl, VT), RHS);
2698  SDValue DIV = DAG.getNode(ISD::UDIV, dl, VT, MUL, NotZero);
2699  SDValue Overflow = DAG.getSetCC(dl, N->getValueType(1), DIV, LHS,
2700  ISD::SETNE);
2701  Overflow = DAG.getSelect(dl, N->getValueType(1), isZero,
2702  DAG.getConstant(0, dl, N->getValueType(1)),
2703  Overflow);
2704  ReplaceValueWith(SDValue(N, 1), Overflow);
2705  return;
2706  }
2707 
2708  Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
2709  EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
2710  Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
2711 
2712  // Replace this with a libcall that will check overflow.
2713  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2714  if (VT == MVT::i32)
2715  LC = RTLIB::MULO_I32;
2716  else if (VT == MVT::i64)
2717  LC = RTLIB::MULO_I64;
2718  else if (VT == MVT::i128)
2719  LC = RTLIB::MULO_I128;
2720  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
2721 
2722  SDValue Temp = DAG.CreateStackTemporary(PtrVT);
2723  // Temporary for the overflow value, default it to zero.
2724  SDValue Chain =
2725  DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
2726  MachinePointerInfo());
2727 
2729  TargetLowering::ArgListEntry Entry;
2730  for (const SDValue &Op : N->op_values()) {
2731  EVT ArgVT = Op.getValueType();
2732  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2733  Entry.Node = Op;
2734  Entry.Ty = ArgTy;
2735  Entry.IsSExt = true;
2736  Entry.IsZExt = false;
2737  Args.push_back(Entry);
2738  }
2739 
2740  // Also pass the address of the overflow check.
2741  Entry.Node = Temp;
2742  Entry.Ty = PtrTy->getPointerTo();
2743  Entry.IsSExt = true;
2744  Entry.IsZExt = false;
2745  Args.push_back(Entry);
2746 
2747  SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
2748 
2750  CLI.setDebugLoc(dl)
2751  .setChain(Chain)
2752  .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
2753  .setSExtResult();
2754 
2755  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2756 
2757  SplitInteger(CallInfo.first, Lo, Hi);
2758  SDValue Temp2 =
2759  DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
2760  SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
2761  DAG.getConstant(0, dl, PtrVT),
2762  ISD::SETNE);
2763  // Use the overflow from the libcall everywhere.
2764  ReplaceValueWith(SDValue(N, 1), Ofl);
2765 }
2766 
2767 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
2768  SDValue &Lo, SDValue &Hi) {
2769  EVT VT = N->getValueType(0);
2770  SDLoc dl(N);
2771  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2772 
2774  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2775  SplitInteger(Res.getValue(0), Lo, Hi);
2776  return;
2777  }
2778 
2779  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2780  if (VT == MVT::i16)
2781  LC = RTLIB::UDIV_I16;
2782  else if (VT == MVT::i32)
2783  LC = RTLIB::UDIV_I32;
2784  else if (VT == MVT::i64)
2785  LC = RTLIB::UDIV_I64;
2786  else if (VT == MVT::i128)
2787  LC = RTLIB::UDIV_I128;
2788  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
2789 
2790  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
2791 }
2792 
2793 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
2794  SDValue &Lo, SDValue &Hi) {
2795  EVT VT = N->getValueType(0);
2796  SDLoc dl(N);
2797  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2798 
2800  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2801  SplitInteger(Res.getValue(1), Lo, Hi);
2802  return;
2803  }
2804 
2805  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2806  if (VT == MVT::i16)
2807  LC = RTLIB::UREM_I16;
2808  else if (VT == MVT::i32)
2809  LC = RTLIB::UREM_I32;
2810  else if (VT == MVT::i64)
2811  LC = RTLIB::UREM_I64;
2812  else if (VT == MVT::i128)
2813  LC = RTLIB::UREM_I128;
2814  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
2815 
2816  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
2817 }
2818 
2819 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
2820  SDValue &Lo, SDValue &Hi) {
2821  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2822  SDLoc dl(N);
2823  SDValue Op = N->getOperand(0);
2824  if (Op.getValueType().bitsLE(NVT)) {
2825  // The low part is zero extension of the input (degenerates to a copy).
2826  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
2827  Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
2828  } else {
2829  // For example, extension of an i48 to an i64. The operand type necessarily
2830  // promotes to the result type, so will end up being expanded too.
2831  assert(getTypeAction(Op.getValueType()) ==
2833  "Only know how to promote this result!");
2834  SDValue Res = GetPromotedInteger(Op);
2835  assert(Res.getValueType() == N->getValueType(0) &&
2836  "Operand over promoted?");
2837  // Split the promoted operand. This will simplify when it is expanded.
2838  SplitInteger(Res, Lo, Hi);
2839  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
2840  Hi = DAG.getZeroExtendInReg(Hi, dl,
2842  ExcessBits));
2843  }
2844 }
2845 
2846 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
2847  SDValue &Lo, SDValue &Hi) {
2848  SDLoc dl(N);
2849  EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
2850  SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
2851  SDValue Zero = DAG.getConstant(0, dl, VT);
2852  SDValue Swap = DAG.getAtomicCmpSwap(
2854  cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
2855  N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
2856 
2857  ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
2858  ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
2859 }
2860 
2861 //===----------------------------------------------------------------------===//
2862 // Integer Operand Expansion
2863 //===----------------------------------------------------------------------===//
2864 
2865 /// ExpandIntegerOperand - This method is called when the specified operand of
2866 /// the specified node is found to need expansion. At this point, all of the
2867 /// result types of the node are known to be legal, but other operands of the
2868 /// node may need promotion or expansion as well as the specified one.
2869 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
2870  DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); dbgs() << "\n");
2871  SDValue Res = SDValue();
2872 
2873  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2874  return false;
2875 
2876  switch (N->getOpcode()) {
2877  default:
2878  #ifndef NDEBUG
2879  dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
2880  N->dump(&DAG); dbgs() << "\n";
2881  #endif
2882  llvm_unreachable("Do not know how to expand this operator's operand!");
2883 
2884  case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
2885  case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
2886  case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
2887  case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
2888  case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
2889  case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
2890  case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
2891  case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
2892  case ISD::SETCCE: Res = ExpandIntOp_SETCCE(N); break;
2893  case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
2894  case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break;
2895  case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
2896  case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
2897  case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break;
2898 
2899  case ISD::SHL:
2900  case ISD::SRA:
2901  case ISD::SRL:
2902  case ISD::ROTL:
2903  case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
2904  case ISD::RETURNADDR:
2905  case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
2906 
2907  case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
2908  }
2909 
2910  // If the result is null, the sub-method took care of registering results etc.
2911  if (!Res.getNode()) return false;
2912 
2913  // If the result is N, the sub-method updated N in place. Tell the legalizer
2914  // core about this.
2915  if (Res.getNode() == N)
2916  return true;
2917 
2918  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2919  "Invalid operand expansion");
2920 
2921  ReplaceValueWith(SDValue(N, 0), Res);
2922  return false;
2923 }
2924 
2925 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
2926 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
2927 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
2928  SDValue &NewRHS,
2929  ISD::CondCode &CCCode,
2930  const SDLoc &dl) {
2931  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2932  GetExpandedInteger(NewLHS, LHSLo, LHSHi);
2933  GetExpandedInteger(NewRHS, RHSLo, RHSHi);
2934 
2935  if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
2936  if (RHSLo == RHSHi) {
2937  if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
2938  if (RHSCST->isAllOnesValue()) {
2939  // Equality comparison to -1.
2940  NewLHS = DAG.getNode(ISD::AND, dl,
2941  LHSLo.getValueType(), LHSLo, LHSHi);
2942  NewRHS = RHSLo;
2943  return;
2944  }
2945  }
2946  }
2947 
2948  NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
2949  NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
2950  NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
2951  NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
2952  return;
2953  }
2954 
2955  // If this is a comparison of the sign bit, just look at the top part.
2956  // X > -1, x < 0
2957  if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
2958  if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0
2959  (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1
2960  NewLHS = LHSHi;
2961  NewRHS = RHSHi;
2962  return;
2963  }
2964 
2965  // FIXME: This generated code sucks.
2966  ISD::CondCode LowCC;
2967  switch (CCCode) {
2968  default: llvm_unreachable("Unknown integer setcc!");
2969  case ISD::SETLT:
2970  case ISD::SETULT: LowCC = ISD::SETULT; break;
2971  case ISD::SETGT:
2972  case ISD::SETUGT: LowCC = ISD::SETUGT; break;
2973  case ISD::SETLE:
2974  case ISD::SETULE: LowCC = ISD::SETULE; break;
2975  case ISD::SETGE:
2976  case ISD::SETUGE: LowCC = ISD::SETUGE; break;
2977  }
2978 
2979  // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
2980  // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
2981  // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
2982 
2983  // NOTE: on targets without efficient SELECT of bools, we can always use
2984  // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
2985  TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
2986  nullptr);
2987  SDValue LoCmp, HiCmp;
2988  if (TLI.isTypeLegal(LHSLo.getValueType()) &&
2989  TLI.isTypeLegal(RHSLo.getValueType()))
2990  LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
2991  RHSLo, LowCC, false, DagCombineInfo, dl);
2992  if (!LoCmp.getNode())
2993  LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
2994  RHSLo, LowCC);
2995  if (TLI.isTypeLegal(LHSHi.getValueType()) &&
2996  TLI.isTypeLegal(RHSHi.getValueType()))
2997  HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
2998  RHSHi, CCCode, false, DagCombineInfo, dl);
2999  if (!HiCmp.getNode())
3000  HiCmp =
3001  DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
3002  LHSHi, RHSHi, DAG.getCondCode(CCCode));
3003 
3004  ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
3005  ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
3006 
3007  bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
3008  CCCode == ISD::SETUGE || CCCode == ISD::SETULE);
3009 
3010  if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) ||
3011  (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) ||
3012  (LoCmpC && LoCmpC->isNullValue())))) {
3013  // For LE / GE, if high part is known false, ignore the low part.
3014  // For LT / GT: if low part is known false, return the high part.
3015  // if high part is known true, ignore the low part.
3016  NewLHS = HiCmp;
3017  NewRHS = SDValue();
3018  return;
3019  }
3020 
3021  if (LHSHi == RHSHi) {
3022  // Comparing the low bits is enough.
3023  NewLHS = LoCmp;
3024  NewRHS = SDValue();
3025  return;
3026  }
3027 
3028  // Lower with SETCCE or SETCCCARRY if the target supports it.
3029  EVT HiVT = LHSHi.getValueType();
3030  EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
3031  bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
3032 
3033  // FIXME: Make all targets support this, then remove the other lowering.
3034  if (HasSETCCCARRY ||
3036  // SETCCE/SETCCCARRY can detect < and >= directly. For > and <=, flip
3037  // operands and condition code.
3038  bool FlipOperands = false;
3039  switch (CCCode) {
3040  case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
3041  case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
3042  case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
3043  case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
3044  default: break;
3045  }
3046  if (FlipOperands) {
3047  std::swap(LHSLo, RHSLo);
3048  std::swap(LHSHi, RHSHi);
3049  }
3050  // Perform a wide subtraction, feeding the carry from the low part into
3051  // SETCCE/SETCCCARRY. The SETCCE/SETCCCARRY operation is essentially
3052  // looking at the high part of the result of LHS - RHS. It is negative
3053  // iff LHS < RHS. It is zero or positive iff LHS >= RHS.
3054  EVT LoVT = LHSLo.getValueType();
3055  SDVTList VTList = DAG.getVTList(
3056  LoVT, HasSETCCCARRY ? getSetCCResultType(LoVT) : MVT::Glue);
3057  SDValue LowCmp = DAG.getNode(HasSETCCCARRY ? ISD::USUBO : ISD::SUBC, dl,
3058  VTList, LHSLo, RHSLo);
3059  SDValue Res = DAG.getNode(HasSETCCCARRY ? ISD::SETCCCARRY : ISD::SETCCE, dl,
3060  getSetCCResultType(HiVT), LHSHi, RHSHi,
3061  LowCmp.getValue(1), DAG.getCondCode(CCCode));
3062  NewLHS = Res;
3063  NewRHS = SDValue();
3064  return;
3065  }
3066 
3067  NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
3068  false, DagCombineInfo, dl);
3069  if (!NewLHS.getNode())
3070  NewLHS =
3071  DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
3072  NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
3073  NewRHS = SDValue();
3074 }
3075 
3076 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
3077  SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
3078  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
3079  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3080 
3081  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3082  // against zero to select between true and false values.
3083  if (!NewRHS.getNode()) {
3084  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3085  CCCode = ISD::SETNE;
3086  }
3087 
3088  // Update N to have the operands specified.
3089  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
3090  DAG.getCondCode(CCCode), NewLHS, NewRHS,
3091  N->getOperand(4)), 0);
3092 }
3093 
3094 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
3095  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3096  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
3097  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3098 
3099  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3100  // against zero to select between true and false values.
3101  if (!NewRHS.getNode()) {
3102  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3103  CCCode = ISD::SETNE;
3104  }
3105 
3106  // Update N to have the operands specified.
3107  return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
3108  N->getOperand(2), N->getOperand(3),
3109  DAG.getCondCode(CCCode)), 0);
3110 }
3111 
3112 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
3113  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3114  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3115  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3116 
3117  // If ExpandSetCCOperands returned a scalar, use it.
3118  if (!NewRHS.getNode()) {
3119  assert(NewLHS.getValueType() == N->getValueType(0) &&
3120  "Unexpected setcc expansion!");
3121  return NewLHS;
3122  }
3123 
3124  // Otherwise, update N to have the operands specified.
3125  return SDValue(
3126  DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
3127 }
3128 
3129 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCE(SDNode *N) {
3130  SDValue LHS = N->getOperand(0);
3131  SDValue RHS = N->getOperand(1);
3132  SDValue Carry = N->getOperand(2);
3133  SDValue Cond = N->getOperand(3);
3134  SDLoc dl = SDLoc(N);
3135 
3136  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3137  GetExpandedInteger(LHS, LHSLo, LHSHi);
3138  GetExpandedInteger(RHS, RHSLo, RHSHi);
3139 
3140  // Expand to a SUBE for the low part and a smaller SETCCE for the high.
3141  SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), MVT::Glue);
3142  SDValue LowCmp = DAG.getNode(ISD::SUBE, dl, VTList, LHSLo, RHSLo, Carry);
3143  return DAG.getNode(ISD::SETCCE, dl, N->getValueType(0), LHSHi, RHSHi,
3144  LowCmp.getValue(1), Cond);
3145 }
3146 
3147 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
3148  SDValue LHS = N->getOperand(0);
3149  SDValue RHS = N->getOperand(1);
3150  SDValue Carry = N->getOperand(2);
3151  SDValue Cond = N->getOperand(3);
3152  SDLoc dl = SDLoc(N);
3153 
3154  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3155  GetExpandedInteger(LHS, LHSLo, LHSHi);
3156  GetExpandedInteger(RHS, RHSLo, RHSHi);
3157 
3158  // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
3159  SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
3160  SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
3161  return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
3162  LowCmp.getValue(1), Cond);
3163 }
3164 
3165 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
3166  // The value being shifted is legal, but the shift amount is too big.
3167  // It follows that either the result of the shift is undefined, or the
3168  // upper half of the shift amount is zero. Just use the lower half.
3169  SDValue Lo, Hi;
3170  GetExpandedInteger(N->getOperand(1), Lo, Hi);
3171  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
3172 }
3173 
3174 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
3175  // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
3176  // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
3177  // constant to valid type.
3178  SDValue Lo, Hi;
3179  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3180  return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
3181 }
3182 
3183 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
3184  SDValue Op = N->getOperand(0);
3185  EVT DstVT = N->getValueType(0);
3187  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3188  "Don't know how to expand this SINT_TO_FP!");
3189  return TLI.makeLibCall(DAG, LC, DstVT, Op, true, SDLoc(N)).first;
3190 }
3191 
3192 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
3193  if (ISD::isNormalStore(N))
3194  return ExpandOp_NormalStore(N, OpNo);
3195 
3196  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
3197  assert(OpNo == 1 && "Can only expand the stored value so far");
3198 
3199  EVT VT = N->getOperand(1).getValueType();
3200  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3201  SDValue Ch = N->getChain();
3202  SDValue Ptr = N->getBasePtr();
3203  unsigned Alignment = N->getAlignment();
3204  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3205  AAMDNodes AAInfo = N->getAAInfo();
3206  SDLoc dl(N);
3207  SDValue Lo, Hi;
3208 
3209  assert(NVT.isByteSized() && "Expanded type not byte sized!");
3210 
3211  if (N->getMemoryVT().bitsLE(NVT)) {
3212  GetExpandedInteger(N->getValue(), Lo, Hi);
3213  return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
3214  N->getMemoryVT(), Alignment, MMOFlags, AAInfo);
3215  }
3216 
3217  if (DAG.getDataLayout().isLittleEndian()) {
3218  // Little-endian - low bits are at low addresses.
3219  GetExpandedInteger(N->getValue(), Lo, Hi);
3220 
3221  Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3222  AAInfo);
3223 
3224  unsigned ExcessBits =
3225  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3226  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3227 
3228  // Increment the pointer to the other half.
3229  unsigned IncrementSize = NVT.getSizeInBits()/8;
3230  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
3231  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
3232  Hi = DAG.getTruncStore(
3233  Ch, dl, Hi, Ptr, N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3234  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3235  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3236  }
3237 
3238  // Big-endian - high bits are at low addresses. Favor aligned stores at
3239  // the cost of some bit-fiddling.
3240  GetExpandedInteger(N->getValue(), Lo, Hi);
3241 
3242  EVT ExtVT = N->getMemoryVT();
3243  unsigned EBytes = ExtVT.getStoreSize();
3244  unsigned IncrementSize = NVT.getSizeInBits()/8;
3245  unsigned ExcessBits = (EBytes - IncrementSize)*8;
3246  EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
3247  ExtVT.getSizeInBits() - ExcessBits);
3248 
3249  if (ExcessBits < NVT.getSizeInBits()) {
3250  // Transfer high bits from the top of Lo to the bottom of Hi.
3251  Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
3252  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3253  TLI.getPointerTy(DAG.getDataLayout())));
3254  Hi = DAG.getNode(
3255  ISD::OR, dl, NVT, Hi,
3256  DAG.getNode(ISD::SRL, dl, NVT, Lo,
3257  DAG.getConstant(ExcessBits, dl,
3258  TLI.getPointerTy(DAG.getDataLayout()))));
3259  }
3260 
3261  // Store both the high bits and maybe some of the low bits.
3262  Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment,
3263  MMOFlags, AAInfo);
3264 
3265  // Increment the pointer to the other half.
3266  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
3267  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
3268  // Store the lowest ExcessBits bits in the second half.
3269  Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
3270  N->getPointerInfo().getWithOffset(IncrementSize),
3271  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3272  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3273  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3274 }
3275 
3276 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
3277  SDValue InL, InH;
3278  GetExpandedInteger(N->getOperand(0), InL, InH);
3279  // Just truncate the low part of the source.
3280  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
3281 }
3282 
3283 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
3284  SDValue Op = N->getOperand(0);
3285  EVT SrcVT = Op.getValueType();
3286  EVT DstVT = N->getValueType(0);
3287  SDLoc dl(N);
3288 
3289  // The following optimization is valid only if every value in SrcVT (when
3290  // treated as signed) is representable in DstVT. Check that the mantissa
3291  // size of DstVT is >= than the number of bits in SrcVT -1.
3292  const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT);
3293  if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 &&
3295  // Do a signed conversion then adjust the result.
3296  SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op);
3297  SignedConv = TLI.LowerOperation(SignedConv, DAG);
3298 
3299  // The result of the signed conversion needs adjusting if the 'sign bit' of
3300  // the incoming integer was set. To handle this, we dynamically test to see
3301  // if it is set, and, if so, add a fudge factor.
3302 
3303  const uint64_t F32TwoE32 = 0x4F800000ULL;
3304  const uint64_t F32TwoE64 = 0x5F800000ULL;
3305  const uint64_t F32TwoE128 = 0x7F800000ULL;
3306 
3307  APInt FF(32, 0);
3308  if (SrcVT == MVT::i32)
3309  FF = APInt(32, F32TwoE32);
3310  else if (SrcVT == MVT::i64)
3311  FF = APInt(32, F32TwoE64);
3312  else if (SrcVT == MVT::i128)
3313  FF = APInt(32, F32TwoE128);
3314  else
3315  llvm_unreachable("Unsupported UINT_TO_FP!");
3316 
3317  // Check whether the sign bit is set.
3318  SDValue Lo, Hi;
3319  GetExpandedInteger(Op, Lo, Hi);
3320  SDValue SignSet = DAG.getSetCC(dl,
3321  getSetCCResultType(Hi.getValueType()),
3322  Hi,
3323  DAG.getConstant(0, dl, Hi.getValueType()),
3324  ISD::SETLT);
3325 
3326  // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
3327  SDValue FudgePtr =
3328  DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)),
3329  TLI.getPointerTy(DAG.getDataLayout()));
3330 
3331  // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
3332  SDValue Zero = DAG.getIntPtrConstant(0, dl);
3333  SDValue Four = DAG.getIntPtrConstant(4, dl);
3334  if (DAG.getDataLayout().isBigEndian())
3335  std::swap(Zero, Four);
3336  SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet,
3337  Zero, Four);
3338  unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment();
3339  FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(),
3340  FudgePtr, Offset);
3341  Alignment = std::min(Alignment, 4u);
3342 
3343  // Load the value out, extending it from f32 to the destination float type.
3344  // FIXME: Avoid the extend by constructing the right constant pool?
3345  SDValue Fudge = DAG.getExtLoad(
3346  ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), FudgePtr,
3348  Alignment);
3349  return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge);
3350  }
3351 
3352  // Otherwise, use a libcall.
3353  RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT);
3354  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3355  "Don't know how to expand this UINT_TO_FP!");
3356  return TLI.makeLibCall(DAG, LC, DstVT, Op, true, dl).first;
3357 }
3358 
3359 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
3360  SDLoc dl(N);
3361  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
3362  cast<AtomicSDNode>(N)->getMemoryVT(),
3363  N->getOperand(0),
3364  N->getOperand(1), N->getOperand(2),
3365  cast<AtomicSDNode>(N)->getMemOperand());
3366  return Swap.getValue(1);
3367 }
3368 
3369 
3370 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
3371  SDValue InOp0 = N->getOperand(0);
3372  EVT InVT = InOp0.getValueType();
3373 
3374  EVT OutVT = N->getValueType(0);
3375  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3376  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3377  unsigned OutNumElems = OutVT.getVectorNumElements();
3378  EVT NOutVTElem = NOutVT.getVectorElementType();
3379 
3380  SDLoc dl(N);
3381  SDValue BaseIdx = N->getOperand(1);
3382 
3384  Ops.reserve(OutNumElems);
3385  for (unsigned i = 0; i != OutNumElems; ++i) {
3386 
3387  // Extract the element from the original vector.
3388  SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
3389  BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
3391  InVT.getVectorElementType(), N->getOperand(0), Index);
3392 
3393  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext);
3394  // Insert the converted element to the new vector.
3395  Ops.push_back(Op);
3396  }
3397 
3398  return DAG.getBuildVector(NOutVT, dl, Ops);
3399 }
3400 
3401 
3402 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
3403  ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
3404  EVT VT = N->getValueType(0);
3405  SDLoc dl(N);
3406 
3407  ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
3408 
3409  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3410  SDValue V1 = GetPromotedInteger(N->getOperand(1));
3411  EVT OutVT = V0.getValueType();
3412 
3413  return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
3414 }
3415 
3416 
3417 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
3418  EVT OutVT = N->getValueType(0);
3419  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3420  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3421  unsigned NumElems = N->getNumOperands();
3422  EVT NOutVTElem = NOutVT.getVectorElementType();
3423 
3424  SDLoc dl(N);
3425 
3427  Ops.reserve(NumElems);
3428  for (unsigned i = 0; i != NumElems; ++i) {
3429  SDValue Op;
3430  // BUILD_VECTOR integer operand types are allowed to be larger than the
3431  // result's element type. This may still be true after the promotion. For
3432  // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
3433  // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
3434  if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
3435  Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
3436  else
3437  Op = N->getOperand(i);
3438  Ops.push_back(Op);
3439  }
3440 
3441  return DAG.getBuildVector(NOutVT, dl, Ops);
3442 }
3443 
3444 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
3445 
3446  SDLoc dl(N);
3447 
3448  assert(!N->getOperand(0).getValueType().isVector() &&
3449  "Input must be a scalar");
3450 
3451  EVT OutVT = N->getValueType(0);
3452  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3453  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3454  EVT NOutVTElem = NOutVT.getVectorElementType();
3455 
3456  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
3457 
3458  return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
3459 }
3460 
3461 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
3462  SDLoc dl(N);
3463 
3464  EVT OutVT = N->getValueType(0);
3465  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3466  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3467 
3468  EVT InElemTy = OutVT.getVectorElementType();
3469  EVT OutElemTy = NOutVT.getVectorElementType();
3470 
3471  unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
3472  unsigned NumOutElem = NOutVT.getVectorNumElements();
3473  unsigned NumOperands = N->getNumOperands();
3474  assert(NumElem * NumOperands == NumOutElem &&
3475  "Unexpected number of elements");
3476 
3477  // Take the elements from the first vector.
3478  SmallVector<SDValue, 8> Ops(NumOutElem);
3479  for (unsigned i = 0; i < NumOperands; ++i) {
3480  SDValue Op = N->getOperand(i);
3481  for (unsigned j = 0; j < NumElem; ++j) {
3482  SDValue Ext = DAG.getNode(
3483  ISD::EXTRACT_VECTOR_ELT, dl, InElemTy, Op,
3484  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3485  Ops[i * NumElem + j] = DAG.getNode(ISD::ANY_EXTEND, dl, OutElemTy, Ext);
3486  }
3487  }
3488 
3489  return DAG.getBuildVector(NOutVT, dl, Ops);
3490 }
3491 
3492 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
3493  EVT VT = N->getValueType(0);
3494  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3495  assert(NVT.isVector() && "This type must be promoted to a vector type");
3496 
3497  SDLoc dl(N);
3498 
3499  // For operands whose TypeAction is to promote, extend the promoted node
3500  // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
3501  // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
3502  // type..
3503  if (getTypeAction(N->getOperand(0).getValueType())
3505  SDValue Promoted;
3506 
3507  switch(N->getOpcode()) {
3509  Promoted = SExtPromotedInteger(N->getOperand(0));
3510  break;
3512  Promoted = ZExtPromotedInteger(N->getOperand(0));
3513  break;
3515  Promoted = GetPromotedInteger(N->getOperand(0));
3516  break;
3517  default:
3518  llvm_unreachable("Node has unexpected Opcode");
3519  }
3520  return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
3521  }
3522 
3523  // Directly extend to the appropriate transform-to type.
3524  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
3525 }
3526 
3527 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
3528  EVT OutVT = N->getValueType(0);
3529  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3530  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3531 
3532  EVT NOutVTElem = NOutVT.getVectorElementType();
3533 
3534  SDLoc dl(N);
3535  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3536 
3537  SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
3538  NOutVTElem, N->getOperand(1));
3539  return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
3540  V0, ConvElem, N->getOperand(2));
3541 }
3542 
3543 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3544  SDLoc dl(N);
3545  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3546  SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
3547  TLI.getVectorIdxTy(DAG.getDataLayout()));
3549  V0->getValueType(0).getScalarType(), V0, V1);
3550 
3551  // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
3552  // element types. If this is the case then we need to expand the outgoing
3553  // value and not truncate it.
3554  return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
3555 }
3556 
3557 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
3558  SDLoc dl(N);
3559  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3560  MVT InVT = V0.getValueType().getSimpleVT();
3561  MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
3563  SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
3564  return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
3565 }
3566 
3567 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
3568  SDLoc dl(N);
3569  unsigned NumElems = N->getNumOperands();
3570 
3571  EVT RetSclrTy = N->getValueType(0).getVectorElementType();
3572 
3573  SmallVector<SDValue, 8> NewOps;
3574  NewOps.reserve(NumElems);
3575 
3576  // For each incoming vector
3577  for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
3578  SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
3579  EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
3580  unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
3581 
3582  for (unsigned i=0; i<NumElem; ++i) {
3583  // Extract element from incoming vector
3584  SDValue Ex = DAG.getNode(
3585  ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
3586  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3587  SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
3588  NewOps.push_back(Tr);
3589  }
3590  }
3591 
3592  return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
3593 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:545
void push_back(const T &Elt)
Definition: SmallVector.h:212
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:544
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getValueType() const
Return the ValueType of the referenced return value.
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant, which is required to be operand #1) half of the integer or float value specified as operand #0.
Definition: ISDOpcodes.h:184
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC&#39;s if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:898
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:333
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:617
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
static MVT getVectorVT(MVT VT, unsigned NumElements)
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:342
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:260
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:494
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none...
const SDValue & getBasePtr() const
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:222
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:865
const SDValue & getValue() const
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
Definition: APInt.h:641
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:253
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
const SDValue & getBasePtr() const
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:250
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1308
const SDValue & getBasePtr() const
const SDValue & getChain() const
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
unsigned getAlignment() const
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:769
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select&#39;s if you just have operands and don&#39;t want to check...
Definition: SelectionDAG.h:911
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:485
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT TVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:359
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Same for subtraction.
Definition: ISDOpcodes.h:253
void reserve(size_type N)
Definition: SmallVector.h:380
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:212
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align=0, int Offs=0, bool isT=false, unsigned char TargetFlags=0)
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:404
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:39
const SDValue & getValue() const
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
SDValue getExternalSymbol(const char *Sym, EVT VT)
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:209
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations...
Definition: ISDOpcodes.h:433
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
static uint32_t getAlignment(const MCSectionCOFF &Sec)
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:451
Shift and rotation operations.
Definition: ISDOpcodes.h:379
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:205
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:639
CallLoweringInfo & setChain(SDValue InChain)
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:190
op_iterator op_end() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
FLT_ROUNDS_ - Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest 2 Round to ...
Definition: ISDOpcodes.h:520
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:304
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt) For double-word atomic operations: ValLo, ValHi, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amtLo, amtHi) ValLo, ValHi, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amtLo, amtHi) These correspond to the atomicrmw instruction.
Definition: ISDOpcodes.h:777
const SDValue & getValue() const
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:446
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:388
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG...
Definition: ISDOpcodes.h:73
This is an SDNode representing atomic operations.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
This represents a list of ValueType&#39;s that has been intern&#39;d by a SelectionDAG.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
This class is used to represent an MSTORE node.
unsigned getSizeInBits() const
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:232
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:298
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:292
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:385
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:455
SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:398
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:216
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:830
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTruncating=false, bool IsCompressing=false)
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:916
op_iterator op_begin() const
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:714
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:472
UNDEF - An undefined node.
Definition: ISDOpcodes.h:178
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:498
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:302
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
Definition: APInt.h:629
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits...
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
Definition: MathExtras.h:602
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const SDValue & getBasePtr() const
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:421
Machine Value Type.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
Simple binary floating point operators.
Definition: ISDOpcodes.h:259
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:273
This is an important base class in LLVM.
Definition: Constant.h:42
iterator_range< value_op_iterator > op_values() const
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
Definition: SmallVector.h:116
const SDValue & getOperand(unsigned Num) const
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:891
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:307
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:247
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:231
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:581
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:57
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:686
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType, bool IsExpanding=false)
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:385
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo...
Definition: ISDOpcodes.h:763
std::vector< ArgListEntry > ArgListTy
const SDValue & getIndex() const
Extended Value Type.
Definition: ValueTypes.h:34
const SDValue & getBasePtr() const
This structure contains all information that is necessary for lowering calls.
const SDValue & getSrc0() const
This class contains a discriminated union of information about pointers in memory operands...
unsigned getNumOperands() const
Return the number of values used by this operation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, unsigned Alignment, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SyncScope::ID SSID)
Gets a node for an atomic cmpxchg op.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
const SDValue & getMask() const
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:50
void dump() const
Dump this node, for debugging.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:314
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:265
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
Definition: ISDOpcodes.h:428
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:864
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:712
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:611
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:642
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:382
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:555
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:560
Represents one node in the SelectionDAG.
static unsigned int semanticsPrecision(const fltSemantics &)
Definition: APFloat.cpp:154
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
MachinePointerInfo getWithOffset(int64_t O) const
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:531
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:923
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT...
Definition: ValueTypes.h:73
EVT getMemoryVT() const
Return the type of the in-memory value.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
Class for arbitrary precision integers.
Definition: APInt.h:69
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:389
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:241
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:445
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:448
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition: APInt.h:523
Flags
Flags values. These may be or&#39;d together.
#define Success
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool ugt(const APInt &RHS) const
Unsigned greather than comparison.
Definition: APInt.h:1234
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:538
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Definition: SmallVector.h:120
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:151
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
Definition: APInt.h:1300
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:362
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, bool isSigned, const SDLoc &dl, bool doesNotReturn=false, bool isReturnValueUsed=true) const
Returns a pair of (return value, chain).
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:463
void computeKnownBits(SDValue Op, KnownBits &Known, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:581
#define N
Flags getFlags() const
Return the raw flags of the source value,.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:323
Same for multiplication.
Definition: ISDOpcodes.h:256
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
SDValue getValue(unsigned R) const
This class is used to represent an MSCATTER node.
unsigned getNumRegisters(LLVMContext &Context, EVT VT) const
Return the number of registers that this ValueType will eventually require.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getCondCode(ISD::CondCode Cond)
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, and op #2 is a carry value...
Definition: ISDOpcodes.h:421
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, const Value *PtrVal, unsigned Alignment, AtomicOrdering Ordering, SyncScope::ID SSID)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands...
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input...
const MachinePointerInfo & getPointerInfo() const
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:320
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This class is used to represent an MLOAD node.
ArrayRef< int > getMask() const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:235
This class is used to represent an MGATHER node.
SDValue getValueType(EVT)
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:81
#define DEBUG(X)
Definition: Debug.h:118
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:412
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Definition: ValueTypes.h:223
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:197
Conversion operators.
Definition: ISDOpcodes.h:442
BooleanContent
Enum that describes how the target represents true/false values.
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:756
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT SrcTy)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value...
bool isBigEndian() const
Definition: DataLayout.h:217
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:451
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:752
LLVMContext * getContext() const
Definition: SelectionDAG.h:393
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:350
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:241
const SDValue & getMask() const
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
This class is used to represent ISD::LOAD nodes.