LLVM  7.0.0svn
LegalizeIntegerTypes.cpp
Go to the documentation of this file.
1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements integer type expansion and promotion for LegalizeTypes.
11 // Promotion is the act of changing a computation in an illegal type into a
12 // computation in a larger type. For example, implementing i8 arithmetic in an
13 // i32 register (often needed on powerpc).
14 // Expansion is the act of changing a computation in an illegal type into a
15 // computation in two identical registers of a smaller type. For example,
16 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
17 // targets).
18 //
19 //===----------------------------------------------------------------------===//
20 
21 #include "LegalizeTypes.h"
22 #include "llvm/IR/DerivedTypes.h"
24 #include "llvm/Support/KnownBits.h"
26 using namespace llvm;
27 
28 #define DEBUG_TYPE "legalize-types"
29 
30 //===----------------------------------------------------------------------===//
31 // Integer Result Promotion
32 //===----------------------------------------------------------------------===//
33 
34 /// PromoteIntegerResult - This method is called when a result of a node is
35 /// found to be in need of promotion to a larger type. At this point, the node
36 /// may also have invalid operands or may have other results that need
37 /// expansion, we just know that (at least) one result needs promotion.
38 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
39  DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); dbgs() << "\n");
40  SDValue Res = SDValue();
41 
42  // See if the target wants to custom expand this node.
43  if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
44  DEBUG(dbgs() << "Node has been custom expanded, done\n");
45  return;
46  }
47 
48  switch (N->getOpcode()) {
49  default:
50 #ifndef NDEBUG
51  dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
52  N->dump(&DAG); dbgs() << "\n";
53 #endif
54  llvm_unreachable("Do not know how to promote this operator!");
55  case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
56  case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
57  case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
58  case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
59  case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
60  case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
61  case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
62  case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
64  case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
65  case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break;
67  case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
69  Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
70  case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
71  case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
72  break;
73  case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
74  break;
75  case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
76  case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
77  case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
78  case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
79  case ISD::SMIN:
80  case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
81  case ISD::UMIN:
82  case ISD::UMAX: Res = PromoteIntRes_ZExtIntBinOp(N); break;
83 
84  case ISD::SHL: Res = PromoteIntRes_SHL(N); break;
86  Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
87  case ISD::SRA: Res = PromoteIntRes_SRA(N); break;
88  case ISD::SRL: Res = PromoteIntRes_SRL(N); break;
89  case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
90  case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
91  case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
92 
94  Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
96  Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
98  Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
99  case ISD::BUILD_VECTOR:
100  Res = PromoteIntRes_BUILD_VECTOR(N); break;
102  Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
103  case ISD::CONCAT_VECTORS:
104  Res = PromoteIntRes_CONCAT_VECTORS(N); break;
105 
109  Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
110 
111  case ISD::SIGN_EXTEND:
112  case ISD::ZERO_EXTEND:
113  case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
114 
115  case ISD::FP_TO_SINT:
116  case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
117 
118  case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
119 
120  case ISD::AND:
121  case ISD::OR:
122  case ISD::XOR:
123  case ISD::ADD:
124  case ISD::SUB:
125  case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
126 
127  case ISD::SDIV:
128  case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
129 
130  case ISD::UDIV:
131  case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
132 
133  case ISD::SADDO:
134  case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
135  case ISD::UADDO:
136  case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
137  case ISD::SMULO:
138  case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
139 
140  case ISD::ADDCARRY:
141  case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
142 
143  case ISD::ATOMIC_LOAD:
144  Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
145 
150  case ISD::ATOMIC_LOAD_OR:
157  case ISD::ATOMIC_SWAP:
158  Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
159 
162  Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
163  break;
164  }
165 
166  // If the result is null then the sub-method took care of registering it.
167  if (Res.getNode())
168  SetPromotedInteger(SDValue(N, ResNo), Res);
169 }
170 
171 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
172  unsigned ResNo) {
173  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
174  return GetPromotedInteger(Op);
175 }
176 
177 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
178  // Sign-extend the new bits, and continue the assertion.
179  SDValue Op = SExtPromotedInteger(N->getOperand(0));
180  return DAG.getNode(ISD::AssertSext, SDLoc(N),
181  Op.getValueType(), Op, N->getOperand(1));
182 }
183 
184 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
185  // Zero the new bits, and continue the assertion.
186  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
187  return DAG.getNode(ISD::AssertZext, SDLoc(N),
188  Op.getValueType(), Op, N->getOperand(1));
189 }
190 
191 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
192  EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
193  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
194  N->getMemoryVT(), ResVT,
195  N->getChain(), N->getBasePtr(),
196  N->getMemOperand());
197  // Legalize the chain result - switch anything that used the old chain to
198  // use the new one.
199  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
200  return Res;
201 }
202 
203 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
204  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
205  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
206  N->getMemoryVT(),
207  N->getChain(), N->getBasePtr(),
208  Op2, N->getMemOperand());
209  // Legalize the chain result - switch anything that used the old chain to
210  // use the new one.
211  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
212  return Res;
213 }
214 
215 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
216  unsigned ResNo) {
217  if (ResNo == 1) {
219  EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
220  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
221 
222  // Only use the result of getSetCCResultType if it is legal,
223  // otherwise just use the promoted result type (NVT).
224  if (!TLI.isTypeLegal(SVT))
225  SVT = NVT;
226 
227  SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
228  SDValue Res = DAG.getAtomicCmpSwap(
230  N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
231  N->getMemOperand());
232  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
233  ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
234  return Res.getValue(1);
235  }
236 
237  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
238  SDValue Op3 = GetPromotedInteger(N->getOperand(3));
239  SDVTList VTs =
240  DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
241  SDValue Res = DAG.getAtomicCmpSwap(
242  N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
243  N->getBasePtr(), Op2, Op3, N->getMemOperand());
244  // Update the use to N with the newly created Res.
245  for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
246  ReplaceValueWith(SDValue(N, i), Res.getValue(i));
247  return Res;
248 }
249 
250 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
251  SDValue InOp = N->getOperand(0);
252  EVT InVT = InOp.getValueType();
253  EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
254  EVT OutVT = N->getValueType(0);
255  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
256  SDLoc dl(N);
257 
258  switch (getTypeAction(InVT)) {
260  break;
262  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
263  // The input promotes to the same size. Convert the promoted value.
264  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
265  break;
267  // Promote the integer operand by hand.
268  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
270  // Convert the promoted float by hand.
271  SDValue PromotedOp = GetPromotedFloat(InOp);
272  return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, PromotedOp);
273  break;
274  }
277  break;
279  // Convert the element to an integer and promote it by hand.
280  if (!NOutVT.isVector())
281  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
282  BitConvertToInteger(GetScalarizedVector(InOp)));
283  break;
285  // For example, i32 = BITCAST v2i16 on alpha. Convert the split
286  // pieces of the input into integers and reassemble in the final type.
287  SDValue Lo, Hi;
288  GetSplitVector(N->getOperand(0), Lo, Hi);
289  Lo = BitConvertToInteger(Lo);
290  Hi = BitConvertToInteger(Hi);
291 
292  if (DAG.getDataLayout().isBigEndian())
293  std::swap(Lo, Hi);
294 
295  InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
297  NOutVT.getSizeInBits()),
298  JoinIntegers(Lo, Hi));
299  return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
300  }
302  // The input is widened to the same size. Convert to the widened value.
303  // Make sure that the outgoing value is not a vector, because this would
304  // make us bitcast between two vectors which are legalized in different ways.
305  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector())
306  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
307  }
308 
309  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
310  CreateStackStoreLoad(InOp, OutVT));
311 }
312 
313 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
314  SDValue Op = GetPromotedInteger(N->getOperand(0));
315  EVT OVT = N->getValueType(0);
316  EVT NVT = Op.getValueType();
317  SDLoc dl(N);
318 
319  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
320  return DAG.getNode(
321  ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
322  DAG.getConstant(DiffBits, dl,
323  TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
324 }
325 
326 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
327  SDValue Op = GetPromotedInteger(N->getOperand(0));
328  EVT OVT = N->getValueType(0);
329  EVT NVT = Op.getValueType();
330  SDLoc dl(N);
331 
332  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
333  return DAG.getNode(
334  ISD::SRL, dl, NVT, DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
335  DAG.getConstant(DiffBits, dl,
336  TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
337 }
338 
339 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
340  // The pair element type may be legal, or may not promote to the same type as
341  // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
342  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
343  TLI.getTypeToTransformTo(*DAG.getContext(),
344  N->getValueType(0)), JoinIntegers(N->getOperand(0),
345  N->getOperand(1)));
346 }
347 
348 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
349  EVT VT = N->getValueType(0);
350  // FIXME there is no actual debug info here
351  SDLoc dl(N);
352  // Zero extend things like i1, sign extend everything else. It shouldn't
353  // matter in theory which one we pick, but this tends to give better code?
354  unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
355  SDValue Result = DAG.getNode(Opc, dl,
356  TLI.getTypeToTransformTo(*DAG.getContext(), VT),
357  SDValue(N, 0));
358  assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
359  return Result;
360 }
361 
362 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
363  // Zero extend to the promoted type and do the count there.
364  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
365  SDLoc dl(N);
366  EVT OVT = N->getValueType(0);
367  EVT NVT = Op.getValueType();
368  Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
369  // Subtract off the extra leading bits in the bigger type.
370  return DAG.getNode(
371  ISD::SUB, dl, NVT, Op,
372  DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
373  NVT));
374 }
375 
376 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
377  // Zero extend to the promoted type and do the count there.
378  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
379  return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op);
380 }
381 
382 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
383  SDValue Op = GetPromotedInteger(N->getOperand(0));
384  EVT OVT = N->getValueType(0);
385  EVT NVT = Op.getValueType();
386  SDLoc dl(N);
387  if (N->getOpcode() == ISD::CTTZ) {
388  // The count is the same in the promoted type except if the original
389  // value was zero. This can be handled by setting the bit just off
390  // the top of the original type.
391  auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
392  OVT.getScalarSizeInBits());
393  Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
394  }
395  return DAG.getNode(N->getOpcode(), dl, NVT, Op);
396 }
397 
398 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
399  SDLoc dl(N);
400  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
401  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0),
402  N->getOperand(1));
403 }
404 
405 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
406  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
407  unsigned NewOpc = N->getOpcode();
408  SDLoc dl(N);
409 
410  // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
411  // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
412  // and SINT conversions are Custom, there is no way to tell which is
413  // preferable. We choose SINT because that's the right thing on PPC.)
414  if (N->getOpcode() == ISD::FP_TO_UINT &&
415  !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
417  NewOpc = ISD::FP_TO_SINT;
418 
419  SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
420 
421  // Assert that the converted value fits in the original type. If it doesn't
422  // (eg: because the value being converted is too big), then the result of the
423  // original operation was undefined anyway, so the assert is still correct.
424  //
425  // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
426  // before legalization: fp-to-uint16, 65534. -> 0xfffe
427  // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
428  return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ?
429  ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
431 }
432 
433 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
434  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
435  SDLoc dl(N);
436 
437  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
438 }
439 
440 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
441  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
442  SDLoc dl(N);
443 
444  if (getTypeAction(N->getOperand(0).getValueType())
446  SDValue Res = GetPromotedInteger(N->getOperand(0));
447  assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
448 
449  // If the result and operand types are the same after promotion, simplify
450  // to an in-register extension.
451  if (NVT == Res.getValueType()) {
452  // The high bits are not guaranteed to be anything. Insert an extend.
453  if (N->getOpcode() == ISD::SIGN_EXTEND)
454  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
455  DAG.getValueType(N->getOperand(0).getValueType()));
456  if (N->getOpcode() == ISD::ZERO_EXTEND)
457  return DAG.getZeroExtendInReg(Res, dl,
459  assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
460  return Res;
461  }
462  }
463 
464  // Otherwise, just extend the original operand all the way to the larger type.
465  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
466 }
467 
468 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
469  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
470  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
471  ISD::LoadExtType ExtType =
473  SDLoc dl(N);
474  SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
475  N->getMemoryVT(), N->getMemOperand());
476 
477  // Legalize the chain result - switch anything that used the old chain to
478  // use the new one.
479  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
480  return Res;
481 }
482 
483 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
484  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
485  SDValue ExtSrc0 = GetPromotedInteger(N->getSrc0());
486 
487  SDLoc dl(N);
488  SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
489  N->getMask(), ExtSrc0, N->getMemoryVT(),
491  // Legalize the chain result - switch anything that used the old chain to
492  // use the new one.
493  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
494  return Res;
495 }
496 
497 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
498  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
499  SDValue ExtSrc0 = GetPromotedInteger(N->getValue());
500  assert(NVT == ExtSrc0.getValueType() &&
501  "Gather result type and the passThru agrument type should be the same");
502 
503  SDLoc dl(N);
504  SDValue Ops[] = {N->getChain(), ExtSrc0, N->getMask(), N->getBasePtr(),
505  N->getIndex(), N->getScale() };
506  SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
507  N->getMemoryVT(), dl, Ops,
508  N->getMemOperand());
509  // Legalize the chain result - switch anything that used the old chain to
510  // use the new one.
511  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
512  return Res;
513 }
514 
515 /// Promote the overflow flag of an overflowing arithmetic node.
516 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
517  // Simply change the return type of the boolean result.
518  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
519  EVT ValueVTs[] = { N->getValueType(0), NVT };
520  SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
521  unsigned NumOps = N->getNumOperands();
522  assert(NumOps <= 3 && "Too many operands");
523  if (NumOps == 3)
524  Ops[2] = N->getOperand(2);
525 
526  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
527  DAG.getVTList(ValueVTs), makeArrayRef(Ops, NumOps));
528 
529  // Modified the sum result - switch anything that used the old sum to use
530  // the new one.
531  ReplaceValueWith(SDValue(N, 0), Res);
532 
533  return SDValue(Res.getNode(), 1);
534 }
535 
536 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
537  if (ResNo == 1)
538  return PromoteIntRes_Overflow(N);
539 
540  // The operation overflowed iff the result in the larger type is not the
541  // sign extension of its truncation to the original type.
542  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
543  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
544  EVT OVT = N->getOperand(0).getValueType();
545  EVT NVT = LHS.getValueType();
546  SDLoc dl(N);
547 
548  // Do the arithmetic in the larger type.
549  unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
550  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
551 
552  // Calculate the overflow flag: sign extend the arithmetic result from
553  // the original type.
554  SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
555  DAG.getValueType(OVT));
556  // Overflowed if and only if this is not equal to Res.
557  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
558 
559  // Use the calculated overflow everywhere.
560  ReplaceValueWith(SDValue(N, 1), Ofl);
561 
562  return Res;
563 }
564 
565 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
566  SDValue LHS = GetPromotedInteger(N->getOperand(1));
567  SDValue RHS = GetPromotedInteger(N->getOperand(2));
568  return DAG.getSelect(SDLoc(N),
569  LHS.getValueType(), N->getOperand(0), LHS, RHS);
570 }
571 
572 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
573  SDValue Mask = N->getOperand(0);
574 
575  SDValue LHS = GetPromotedInteger(N->getOperand(1));
576  SDValue RHS = GetPromotedInteger(N->getOperand(2));
577  return DAG.getNode(ISD::VSELECT, SDLoc(N),
578  LHS.getValueType(), Mask, LHS, RHS);
579 }
580 
581 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
582  SDValue LHS = GetPromotedInteger(N->getOperand(2));
583  SDValue RHS = GetPromotedInteger(N->getOperand(3));
584  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
585  LHS.getValueType(), N->getOperand(0),
586  N->getOperand(1), LHS, RHS, N->getOperand(4));
587 }
588 
589 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
590  EVT SVT = getSetCCResultType(N->getOperand(0).getValueType());
591 
592  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
593 
594  // Only use the result of getSetCCResultType if it is legal,
595  // otherwise just use the promoted result type (NVT).
596  if (!TLI.isTypeLegal(SVT))
597  SVT = NVT;
598 
599  SDLoc dl(N);
600  assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() &&
601  "Vector compare must return a vector result!");
602 
603  // Get the SETCC result using the canonical SETCC type.
604  SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
605  N->getOperand(1), N->getOperand(2));
606 
607  // Convert to the expected type.
608  return DAG.getSExtOrTrunc(SetCC, dl, NVT);
609 }
610 
611 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
612  SDValue LHS = N->getOperand(0);
613  SDValue RHS = N->getOperand(1);
614  if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
615  LHS = GetPromotedInteger(LHS);
616  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
617  RHS = ZExtPromotedInteger(RHS);
618  return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
619 }
620 
621 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
622  SDValue Op = GetPromotedInteger(N->getOperand(0));
623  return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
624  Op.getValueType(), Op, N->getOperand(1));
625 }
626 
627 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
628  // The input may have strange things in the top bits of the registers, but
629  // these operations don't care. They may have weird bits going out, but
630  // that too is okay if they are integer operations.
631  SDValue LHS = GetPromotedInteger(N->getOperand(0));
632  SDValue RHS = GetPromotedInteger(N->getOperand(1));
633  return DAG.getNode(N->getOpcode(), SDLoc(N),
634  LHS.getValueType(), LHS, RHS);
635 }
636 
637 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
638  // Sign extend the input.
639  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
640  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
641  return DAG.getNode(N->getOpcode(), SDLoc(N),
642  LHS.getValueType(), LHS, RHS);
643 }
644 
645 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
646  // Zero extend the input.
647  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
648  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
649  return DAG.getNode(N->getOpcode(), SDLoc(N),
650  LHS.getValueType(), LHS, RHS);
651 }
652 
653 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
654  SDValue LHS = N->getOperand(0);
655  SDValue RHS = N->getOperand(1);
656  // The input value must be properly sign extended.
657  if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
658  LHS = SExtPromotedInteger(LHS);
659  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
660  RHS = ZExtPromotedInteger(RHS);
661  return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
662 }
663 
664 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
665  SDValue LHS = N->getOperand(0);
666  SDValue RHS = N->getOperand(1);
667  // The input value must be properly zero extended.
668  if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
669  LHS = ZExtPromotedInteger(LHS);
670  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
671  RHS = ZExtPromotedInteger(RHS);
672  return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
673 }
674 
675 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
676  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
677  SDValue Res;
678  SDValue InOp = N->getOperand(0);
679  SDLoc dl(N);
680 
681  switch (getTypeAction(InOp.getValueType())) {
682  default: llvm_unreachable("Unknown type action!");
685  Res = InOp;
686  break;
688  Res = GetPromotedInteger(InOp);
689  break;
691  EVT InVT = InOp.getValueType();
692  assert(InVT.isVector() && "Cannot split scalar types");
693  unsigned NumElts = InVT.getVectorNumElements();
694  assert(NumElts == NVT.getVectorNumElements() &&
695  "Dst and Src must have the same number of elements");
696  assert(isPowerOf2_32(NumElts) &&
697  "Promoted vector type must be a power of two");
698 
699  SDValue EOp1, EOp2;
700  GetSplitVector(InOp, EOp1, EOp2);
701 
702  EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
703  NumElts/2);
704  EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
705  EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
706 
707  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
708  }
710  SDValue WideInOp = GetWidenedVector(InOp);
711 
712  // Truncate widened InOp.
713  unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
714  EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
715  N->getValueType(0).getScalarType(), NumElem);
716  SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
717 
718  // Zero extend so that the elements are of same type as those of NVT
719  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
720  NumElem);
721  SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
722 
723  // Extract the low NVT subvector.
724  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
725  SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy);
726  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
727  }
728  }
729 
730  // Truncate to NVT instead of VT
731  return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
732 }
733 
734 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
735  if (ResNo == 1)
736  return PromoteIntRes_Overflow(N);
737 
738  // The operation overflowed iff the result in the larger type is not the
739  // zero extension of its truncation to the original type.
740  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
741  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
742  EVT OVT = N->getOperand(0).getValueType();
743  EVT NVT = LHS.getValueType();
744  SDLoc dl(N);
745 
746  // Do the arithmetic in the larger type.
747  unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
748  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
749 
750  // Calculate the overflow flag: zero extend the arithmetic result from
751  // the original type.
752  SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
753  // Overflowed if and only if this is not equal to Res.
754  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
755 
756  // Use the calculated overflow everywhere.
757  ReplaceValueWith(SDValue(N, 1), Ofl);
758 
759  return Res;
760 }
761 
762 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
763  if (ResNo == 1)
764  return PromoteIntRes_Overflow(N);
765 
766  // We need to sign-extend the operands so the carry value computed by the
767  // wide operation will be equivalent to the carry value computed by the
768  // narrow operation.
769  // An ADDCARRY can generate carry only if any of the operands has its
770  // most significant bit set. Sign extension propagates the most significant
771  // bit into the higher bits which means the extra bit that the narrow
772  // addition would need (i.e. the carry) will be propagated through the higher
773  // bits of the wide addition.
774  // A SUBCARRY can generate borrow only if LHS < RHS and this property will be
775  // preserved by sign extension.
776  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
777  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
778 
779  EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
780 
781  // Do the arithmetic in the wide type.
782  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
783  LHS, RHS, N->getOperand(2));
784 
785  // Update the users of the original carry/borrow value.
786  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
787 
788  return SDValue(Res.getNode(), 0);
789 }
790 
791 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
792  // Promote the overflow bit trivially.
793  if (ResNo == 1)
794  return PromoteIntRes_Overflow(N);
795 
796  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
797  SDLoc DL(N);
798  EVT SmallVT = LHS.getValueType();
799 
800  // To determine if the result overflowed in a larger type, we extend the
801  // input to the larger type, do the multiply (checking if it overflows),
802  // then also check the high bits of the result to see if overflow happened
803  // there.
804  if (N->getOpcode() == ISD::SMULO) {
805  LHS = SExtPromotedInteger(LHS);
806  RHS = SExtPromotedInteger(RHS);
807  } else {
808  LHS = ZExtPromotedInteger(LHS);
809  RHS = ZExtPromotedInteger(RHS);
810  }
811  SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
812  SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
813 
814  // Overflow occurred if it occurred in the larger type, or if the high part
815  // of the result does not zero/sign-extend the low part. Check this second
816  // possibility first.
817  SDValue Overflow;
818  if (N->getOpcode() == ISD::UMULO) {
819  // Unsigned overflow occurred if the high part is non-zero.
820  SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
821  DAG.getIntPtrConstant(SmallVT.getSizeInBits(),
822  DL));
823  Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
824  DAG.getConstant(0, DL, Hi.getValueType()),
825  ISD::SETNE);
826  } else {
827  // Signed overflow occurred if the high part does not sign extend the low.
828  SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
829  Mul, DAG.getValueType(SmallVT));
830  Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
831  }
832 
833  // The only other way for overflow to occur is if the multiplication in the
834  // larger type itself overflowed.
835  Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
836  SDValue(Mul.getNode(), 1));
837 
838  // Use the calculated overflow everywhere.
839  ReplaceValueWith(SDValue(N, 1), Overflow);
840  return Mul;
841 }
842 
843 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
844  return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
845  N->getValueType(0)));
846 }
847 
848 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
849  SDValue Chain = N->getOperand(0); // Get the chain.
850  SDValue Ptr = N->getOperand(1); // Get the pointer.
851  EVT VT = N->getValueType(0);
852  SDLoc dl(N);
853 
854  MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
855  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
856  // The argument is passed as NumRegs registers of type RegVT.
857 
858  SmallVector<SDValue, 8> Parts(NumRegs);
859  for (unsigned i = 0; i < NumRegs; ++i) {
860  Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
861  N->getConstantOperandVal(3));
862  Chain = Parts[i].getValue(1);
863  }
864 
865  // Handle endianness of the load.
866  if (DAG.getDataLayout().isBigEndian())
867  std::reverse(Parts.begin(), Parts.end());
868 
869  // Assemble the parts in the promoted type.
870  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
871  SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
872  for (unsigned i = 1; i < NumRegs; ++i) {
873  SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
874  // Shift it to the right position and "or" it in.
875  Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
876  DAG.getConstant(i * RegVT.getSizeInBits(), dl,
877  TLI.getPointerTy(DAG.getDataLayout())));
878  Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
879  }
880 
881  // Modified the chain result - switch anything that used the old chain to
882  // use the new one.
883  ReplaceValueWith(SDValue(N, 1), Chain);
884 
885  return Res;
886 }
887 
888 //===----------------------------------------------------------------------===//
889 // Integer Operand Promotion
890 //===----------------------------------------------------------------------===//
891 
892 /// PromoteIntegerOperand - This method is called when the specified operand of
893 /// the specified node is found to need promotion. At this point, all of the
894 /// result types of the node are known to be legal, but other operands of the
895 /// node may need promotion or expansion as well as the specified one.
896 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
897  DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); dbgs() << "\n");
898  SDValue Res = SDValue();
899 
900  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
901  DEBUG(dbgs() << "Node has been custom lowered, done\n");
902  return false;
903  }
904 
905  switch (N->getOpcode()) {
906  default:
907  #ifndef NDEBUG
908  dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
909  N->dump(&DAG); dbgs() << "\n";
910  #endif
911  llvm_unreachable("Do not know how to promote this operator's operand!");
912 
913  case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
914  case ISD::ATOMIC_STORE:
915  Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
916  break;
917  case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
918  case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
919  case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
920  case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
921  case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
922  case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
923  case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
925  Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
927  Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
928  case ISD::VSELECT:
929  case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
930  case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
931  case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
932  case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
933  case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
934  case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
935  OpNo); break;
936  case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
937  OpNo); break;
938  case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
939  OpNo); break;
940  case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
941  OpNo); break;
942  case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
943  OpNo); break;
944  case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
945  case ISD::FP16_TO_FP:
946  case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
947  case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
948  case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
949 
950  case ISD::SHL:
951  case ISD::SRA:
952  case ISD::SRL:
953  case ISD::ROTL:
954  case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
955 
956  case ISD::ADDCARRY:
957  case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
958  }
959 
960  // If the result is null, the sub-method took care of registering results etc.
961  if (!Res.getNode()) return false;
962 
963  // If the result is N, the sub-method updated N in place. Tell the legalizer
964  // core about this.
965  if (Res.getNode() == N)
966  return true;
967 
968  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
969  "Invalid operand expansion");
970 
971  ReplaceValueWith(SDValue(N, 0), Res);
972  return false;
973 }
974 
975 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
976 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
977 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
978  ISD::CondCode CCCode) {
979  // We have to insert explicit sign or zero extends. Note that we could
980  // insert sign extends for ALL conditions, but zero extend is cheaper on
981  // many machines (an AND instead of two shifts), so prefer it.
982  switch (CCCode) {
983  default: llvm_unreachable("Unknown integer comparison!");
984  case ISD::SETEQ:
985  case ISD::SETNE: {
986  SDValue OpL = GetPromotedInteger(NewLHS);
987  SDValue OpR = GetPromotedInteger(NewRHS);
988 
989  // We would prefer to promote the comparison operand with sign extension.
990  // If the width of OpL/OpR excluding the duplicated sign bits is no greater
991  // than the width of NewLHS/NewRH, we can avoid inserting real truncate
992  // instruction, which is redudant eventually.
993  unsigned OpLEffectiveBits =
994  OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
995  unsigned OpREffectiveBits =
996  OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
997  if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() &&
998  OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()) {
999  NewLHS = OpL;
1000  NewRHS = OpR;
1001  } else {
1002  NewLHS = ZExtPromotedInteger(NewLHS);
1003  NewRHS = ZExtPromotedInteger(NewRHS);
1004  }
1005  break;
1006  }
1007  case ISD::SETUGE:
1008  case ISD::SETUGT:
1009  case ISD::SETULE:
1010  case ISD::SETULT:
1011  // ALL of these operations will work if we either sign or zero extend
1012  // the operands (including the unsigned comparisons!). Zero extend is
1013  // usually a simpler/cheaper operation, so prefer it.
1014  NewLHS = ZExtPromotedInteger(NewLHS);
1015  NewRHS = ZExtPromotedInteger(NewRHS);
1016  break;
1017  case ISD::SETGE:
1018  case ISD::SETGT:
1019  case ISD::SETLT:
1020  case ISD::SETLE:
1021  NewLHS = SExtPromotedInteger(NewLHS);
1022  NewRHS = SExtPromotedInteger(NewRHS);
1023  break;
1024  }
1025 }
1026 
1027 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1028  SDValue Op = GetPromotedInteger(N->getOperand(0));
1029  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1030 }
1031 
1032 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1033  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1034  return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1035  N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1036 }
1037 
1038 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1039  // This should only occur in unusual situations like bitcasting to an
1040  // x86_fp80, so just turn it into a store+load
1041  return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1042 }
1043 
1044 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1045  assert(OpNo == 2 && "Don't know how to promote this operand!");
1046 
1047  SDValue LHS = N->getOperand(2);
1048  SDValue RHS = N->getOperand(3);
1049  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1050 
1051  // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1052  // legal types.
1053  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1054  N->getOperand(1), LHS, RHS, N->getOperand(4)),
1055  0);
1056 }
1057 
1058 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1059  assert(OpNo == 1 && "only know how to promote condition");
1060 
1061  // Promote all the way up to the canonical SetCC type.
1062  SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1063 
1064  // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1065  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1066  N->getOperand(2)), 0);
1067 }
1068 
1069 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1070  // Since the result type is legal, the operands must promote to it.
1071  EVT OVT = N->getOperand(0).getValueType();
1072  SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1073  SDValue Hi = GetPromotedInteger(N->getOperand(1));
1074  assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1075  SDLoc dl(N);
1076 
1077  Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1078  DAG.getConstant(OVT.getSizeInBits(), dl,
1079  TLI.getPointerTy(DAG.getDataLayout())));
1080  return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1081 }
1082 
1083 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1084  // The vector type is legal but the element type is not. This implies
1085  // that the vector is a power-of-two in length and that the element
1086  // type does not have a strange size (eg: it is not i1).
1087  EVT VecVT = N->getValueType(0);
1088  unsigned NumElts = VecVT.getVectorNumElements();
1089  assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1090  "Legal vector of one illegal element?");
1091 
1092  // Promote the inserted value. The type does not need to match the
1093  // vector element type. Check that any extra bits introduced will be
1094  // truncated away.
1097  "Type of inserted value narrower than vector element type!");
1098 
1099  SmallVector<SDValue, 16> NewOps;
1100  for (unsigned i = 0; i < NumElts; ++i)
1101  NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1102 
1103  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1104 }
1105 
1106 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1107  unsigned OpNo) {
1108  if (OpNo == 1) {
1109  // Promote the inserted value. This is valid because the type does not
1110  // have to match the vector element type.
1111 
1112  // Check that any extra bits introduced will be truncated away.
1115  "Type of inserted value narrower than vector element type!");
1116  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1117  GetPromotedInteger(N->getOperand(1)),
1118  N->getOperand(2)),
1119  0);
1120  }
1121 
1122  assert(OpNo == 2 && "Different operand and result vector types?");
1123 
1124  // Promote the index.
1125  SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1126  TLI.getVectorIdxTy(DAG.getDataLayout()));
1127  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1128  N->getOperand(1), Idx), 0);
1129 }
1130 
1131 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1132  // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1133  // the operand in place.
1134  return SDValue(DAG.UpdateNodeOperands(N,
1135  GetPromotedInteger(N->getOperand(0))), 0);
1136 }
1137 
1138 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1139  assert(OpNo == 0 && "Only know how to promote the condition!");
1140  SDValue Cond = N->getOperand(0);
1141  EVT OpTy = N->getOperand(1).getValueType();
1142 
1143  if (N->getOpcode() == ISD::VSELECT)
1144  if (SDValue Res = WidenVSELECTAndMask(N))
1145  return Res;
1146 
1147  // Promote all the way up to the canonical SetCC type.
1148  EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1149  Cond = PromoteTargetBoolean(Cond, OpVT);
1150 
1151  return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1152  N->getOperand(2)), 0);
1153 }
1154 
1155 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1156  assert(OpNo == 0 && "Don't know how to promote this operand!");
1157 
1158  SDValue LHS = N->getOperand(0);
1159  SDValue RHS = N->getOperand(1);
1160  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1161 
1162  // The CC (#4) and the possible return values (#2 and #3) have legal types.
1163  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1164  N->getOperand(3), N->getOperand(4)), 0);
1165 }
1166 
1167 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1168  assert(OpNo == 0 && "Don't know how to promote this operand!");
1169 
1170  SDValue LHS = N->getOperand(0);
1171  SDValue RHS = N->getOperand(1);
1172  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1173 
1174  // The CC (#2) is always legal.
1175  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1176 }
1177 
1178 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1179  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1180  ZExtPromotedInteger(N->getOperand(1))), 0);
1181 }
1182 
1183 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1184  SDValue Op = GetPromotedInteger(N->getOperand(0));
1185  SDLoc dl(N);
1186  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1187  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1188  Op, DAG.getValueType(N->getOperand(0).getValueType()));
1189 }
1190 
1191 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1192  return SDValue(DAG.UpdateNodeOperands(N,
1193  SExtPromotedInteger(N->getOperand(0))), 0);
1194 }
1195 
1196 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1197  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1198  SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1199  SDLoc dl(N);
1200 
1201  SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1202 
1203  // Truncate the value and store the result.
1204  return DAG.getTruncStore(Ch, dl, Val, Ptr,
1205  N->getMemoryVT(), N->getMemOperand());
1206 }
1207 
1208 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1209  unsigned OpNo) {
1210 
1211  SDValue DataOp = N->getValue();
1212  EVT DataVT = DataOp.getValueType();
1213  SDValue Mask = N->getMask();
1214  SDLoc dl(N);
1215 
1216  bool TruncateStore = false;
1217  if (OpNo == 2) {
1218  // Mask comes before the data operand. If the data operand is legal, we just
1219  // promote the mask.
1220  // When the data operand has illegal type, we should legalize the data
1221  // operand first. The mask will be promoted/splitted/widened according to
1222  // the data operand type.
1223  if (TLI.isTypeLegal(DataVT)) {
1224  Mask = PromoteTargetBoolean(Mask, DataVT);
1225  // Update in place.
1226  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1227  NewOps[2] = Mask;
1228  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1229  }
1230 
1231  if (getTypeAction(DataVT) == TargetLowering::TypePromoteInteger)
1232  return PromoteIntOp_MSTORE(N, 3);
1233  if (getTypeAction(DataVT) == TargetLowering::TypeWidenVector)
1234  return WidenVecOp_MSTORE(N, 3);
1235  assert (getTypeAction(DataVT) == TargetLowering::TypeSplitVector);
1236  return SplitVecOp_MSTORE(N, 3);
1237  } else { // Data operand
1238  assert(OpNo == 3 && "Unexpected operand for promotion");
1239  DataOp = GetPromotedInteger(DataOp);
1240  TruncateStore = true;
1241  }
1242 
1243  return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask,
1244  N->getMemoryVT(), N->getMemOperand(),
1245  TruncateStore, N->isCompressingStore());
1246 }
1247 
1248 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1249  unsigned OpNo) {
1250  assert(OpNo == 2 && "Only know how to promote the mask!");
1251  EVT DataVT = N->getValueType(0);
1252  SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1253  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1254  NewOps[OpNo] = Mask;
1255  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1256 }
1257 
1258 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1259  unsigned OpNo) {
1260 
1261  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1262  if (OpNo == 2) {
1263  // The Mask
1264  EVT DataVT = N->getValueType(0);
1265  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1266  } else if (OpNo == 4) {
1267  // Need to sign extend the index since the bits will likely be used.
1268  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1269  } else
1270  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1271 
1272  SDValue Res = SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1273  // updated in place.
1274  if (Res.getNode() == N)
1275  return Res;
1276 
1277  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
1278  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1279  return SDValue();
1280 }
1281 
1282 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1283  unsigned OpNo) {
1284  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1285  if (OpNo == 2) {
1286  // The Mask
1287  EVT DataVT = N->getValue().getValueType();
1288  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1289  } else if (OpNo == 4) {
1290  // Need to sign extend the index since the bits will likely be used.
1291  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1292  } else
1293  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1294  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1295 }
1296 
1297 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
1298  SDValue Op = GetPromotedInteger(N->getOperand(0));
1299  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
1300 }
1301 
1302 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
1303  return SDValue(DAG.UpdateNodeOperands(N,
1304  ZExtPromotedInteger(N->getOperand(0))), 0);
1305 }
1306 
1307 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
1308  SDLoc dl(N);
1309  SDValue Op = GetPromotedInteger(N->getOperand(0));
1310  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1311  return DAG.getZeroExtendInReg(Op, dl,
1313 }
1314 
1315 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
1316  assert(OpNo == 2 && "Don't know how to promote this operand!");
1317 
1318  SDValue LHS = N->getOperand(0);
1319  SDValue RHS = N->getOperand(1);
1320  SDValue Carry = N->getOperand(2);
1321  SDLoc DL(N);
1322 
1323  auto VT = getSetCCResultType(LHS.getValueType());
1325  switch (BoolType) {
1327  Carry = DAG.getAnyExtOrTrunc(Carry, DL, VT);
1328  break;
1330  Carry = DAG.getZExtOrTrunc(Carry, DL, VT);
1331  break;
1333  Carry = DAG.getSExtOrTrunc(Carry, DL, VT);
1334  break;
1335  }
1336 
1337  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
1338 }
1339 
1340 //===----------------------------------------------------------------------===//
1341 // Integer Result Expansion
1342 //===----------------------------------------------------------------------===//
1343 
1344 /// ExpandIntegerResult - This method is called when the specified result of the
1345 /// specified node is found to need expansion. At this point, the node may also
1346 /// have invalid operands or may have other results that need promotion, we just
1347 /// know that (at least) one result needs expansion.
1348 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
1349  DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); dbgs() << "\n");
1350  SDValue Lo, Hi;
1351  Lo = Hi = SDValue();
1352 
1353  // See if the target wants to custom expand this node.
1354  if (CustomLowerNode(N, N->getValueType(ResNo), true))
1355  return;
1356 
1357  switch (N->getOpcode()) {
1358  default:
1359 #ifndef NDEBUG
1360  dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
1361  N->dump(&DAG); dbgs() << "\n";
1362 #endif
1363  llvm_unreachable("Do not know how to expand the result of this operator!");
1364 
1365  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1366  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
1367  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1368  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1369 
1370  case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1371  case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1372  case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1373  case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1374  case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1375 
1376  case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
1377  case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
1378  case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
1379  case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
1380  case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
1381  case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
1382  case ISD::CTLZ_ZERO_UNDEF:
1383  case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
1384  case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
1385  case ISD::CTTZ_ZERO_UNDEF:
1386  case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
1387  case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
1388  case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
1389  case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
1390  case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
1391  case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
1392  case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
1393  case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
1394  case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
1395  case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
1396  case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
1397  case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
1398  case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
1399  case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
1400  case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
1401  case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
1402 
1403  case ISD::ATOMIC_LOAD_ADD:
1404  case ISD::ATOMIC_LOAD_SUB:
1405  case ISD::ATOMIC_LOAD_AND:
1406  case ISD::ATOMIC_LOAD_CLR:
1407  case ISD::ATOMIC_LOAD_OR:
1408  case ISD::ATOMIC_LOAD_XOR:
1409  case ISD::ATOMIC_LOAD_NAND:
1410  case ISD::ATOMIC_LOAD_MIN:
1411  case ISD::ATOMIC_LOAD_MAX:
1412  case ISD::ATOMIC_LOAD_UMIN:
1413  case ISD::ATOMIC_LOAD_UMAX:
1414  case ISD::ATOMIC_SWAP:
1415  case ISD::ATOMIC_CMP_SWAP: {
1416  std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
1417  SplitInteger(Tmp.first, Lo, Hi);
1418  ReplaceValueWith(SDValue(N, 1), Tmp.second);
1419  break;
1420  }
1422  AtomicSDNode *AN = cast<AtomicSDNode>(N);
1423  SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
1424  SDValue Tmp = DAG.getAtomicCmpSwap(
1425  ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
1426  N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
1427  AN->getMemOperand());
1428 
1429  // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
1430  // success simply by comparing the loaded value against the ingoing
1431  // comparison.
1432  SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
1433  N->getOperand(2), ISD::SETEQ);
1434 
1435  SplitInteger(Tmp, Lo, Hi);
1436  ReplaceValueWith(SDValue(N, 1), Success);
1437  ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
1438  break;
1439  }
1440 
1441  case ISD::AND:
1442  case ISD::OR:
1443  case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
1444 
1445  case ISD::UMAX:
1446  case ISD::SMAX:
1447  case ISD::UMIN:
1448  case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
1449 
1450  case ISD::ADD:
1451  case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
1452 
1453  case ISD::ADDC:
1454  case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
1455 
1456  case ISD::ADDE:
1457  case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
1458 
1459  case ISD::ADDCARRY:
1460  case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
1461 
1462  case ISD::SHL:
1463  case ISD::SRA:
1464  case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
1465 
1466  case ISD::SADDO:
1467  case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
1468  case ISD::UADDO:
1469  case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
1470  case ISD::UMULO:
1471  case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
1472  }
1473 
1474  // If Lo/Hi is null, the sub-method took care of registering results etc.
1475  if (Lo.getNode())
1476  SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
1477 }
1478 
1479 /// Lower an atomic node to the appropriate builtin call.
1480 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
1481  unsigned Opc = Node->getOpcode();
1482  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
1483  RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
1484  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
1485 
1486  return ExpandChainLibCall(LC, Node, false);
1487 }
1488 
1489 /// N is a shift by a value that needs to be expanded,
1490 /// and the shift amount is a constant 'Amt'. Expand the operation.
1491 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
1492  SDValue &Lo, SDValue &Hi) {
1493  SDLoc DL(N);
1494  // Expand the incoming operand to be shifted, so that we have its parts
1495  SDValue InL, InH;
1496  GetExpandedInteger(N->getOperand(0), InL, InH);
1497 
1498  // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
1499  // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
1500  if (!Amt) {
1501  Lo = InL;
1502  Hi = InH;
1503  return;
1504  }
1505 
1506  EVT NVT = InL.getValueType();
1507  unsigned VTBits = N->getValueType(0).getSizeInBits();
1508  unsigned NVTBits = NVT.getSizeInBits();
1509  EVT ShTy = N->getOperand(1).getValueType();
1510 
1511  if (N->getOpcode() == ISD::SHL) {
1512  if (Amt.ugt(VTBits)) {
1513  Lo = Hi = DAG.getConstant(0, DL, NVT);
1514  } else if (Amt.ugt(NVTBits)) {
1515  Lo = DAG.getConstant(0, DL, NVT);
1516  Hi = DAG.getNode(ISD::SHL, DL,
1517  NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1518  } else if (Amt == NVTBits) {
1519  Lo = DAG.getConstant(0, DL, NVT);
1520  Hi = InL;
1521  } else {
1522  Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
1523  Hi = DAG.getNode(ISD::OR, DL, NVT,
1524  DAG.getNode(ISD::SHL, DL, NVT, InH,
1525  DAG.getConstant(Amt, DL, ShTy)),
1526  DAG.getNode(ISD::SRL, DL, NVT, InL,
1527  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1528  }
1529  return;
1530  }
1531 
1532  if (N->getOpcode() == ISD::SRL) {
1533  if (Amt.ugt(VTBits)) {
1534  Lo = Hi = DAG.getConstant(0, DL, NVT);
1535  } else if (Amt.ugt(NVTBits)) {
1536  Lo = DAG.getNode(ISD::SRL, DL,
1537  NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1538  Hi = DAG.getConstant(0, DL, NVT);
1539  } else if (Amt == NVTBits) {
1540  Lo = InH;
1541  Hi = DAG.getConstant(0, DL, NVT);
1542  } else {
1543  Lo = DAG.getNode(ISD::OR, DL, NVT,
1544  DAG.getNode(ISD::SRL, DL, NVT, InL,
1545  DAG.getConstant(Amt, DL, ShTy)),
1546  DAG.getNode(ISD::SHL, DL, NVT, InH,
1547  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1548  Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1549  }
1550  return;
1551  }
1552 
1553  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
1554  if (Amt.ugt(VTBits)) {
1555  Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1556  DAG.getConstant(NVTBits - 1, DL, ShTy));
1557  } else if (Amt.ugt(NVTBits)) {
1558  Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1559  DAG.getConstant(Amt - NVTBits, DL, ShTy));
1560  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1561  DAG.getConstant(NVTBits - 1, DL, ShTy));
1562  } else if (Amt == NVTBits) {
1563  Lo = InH;
1564  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1565  DAG.getConstant(NVTBits - 1, DL, ShTy));
1566  } else {
1567  Lo = DAG.getNode(ISD::OR, DL, NVT,
1568  DAG.getNode(ISD::SRL, DL, NVT, InL,
1569  DAG.getConstant(Amt, DL, ShTy)),
1570  DAG.getNode(ISD::SHL, DL, NVT, InH,
1571  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1572  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1573  }
1574 }
1575 
1576 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
1577 /// this shift based on knowledge of the high bit of the shift amount. If we
1578 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
1579 /// shift amount.
1580 bool DAGTypeLegalizer::
1581 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1582  SDValue Amt = N->getOperand(1);
1583  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1584  EVT ShTy = Amt.getValueType();
1585  unsigned ShBits = ShTy.getScalarSizeInBits();
1586  unsigned NVTBits = NVT.getScalarSizeInBits();
1587  assert(isPowerOf2_32(NVTBits) &&
1588  "Expanded integer type size not a power of two!");
1589  SDLoc dl(N);
1590 
1591  APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
1592  KnownBits Known;
1593  DAG.computeKnownBits(N->getOperand(1), Known);
1594 
1595  // If we don't know anything about the high bits, exit.
1596  if (((Known.Zero|Known.One) & HighBitMask) == 0)
1597  return false;
1598 
1599  // Get the incoming operand to be shifted.
1600  SDValue InL, InH;
1601  GetExpandedInteger(N->getOperand(0), InL, InH);
1602 
1603  // If we know that any of the high bits of the shift amount are one, then we
1604  // can do this as a couple of simple shifts.
1605  if (Known.One.intersects(HighBitMask)) {
1606  // Mask out the high bit, which we know is set.
1607  Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
1608  DAG.getConstant(~HighBitMask, dl, ShTy));
1609 
1610  switch (N->getOpcode()) {
1611  default: llvm_unreachable("Unknown shift");
1612  case ISD::SHL:
1613  Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
1614  Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
1615  return true;
1616  case ISD::SRL:
1617  Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1618  Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
1619  return true;
1620  case ISD::SRA:
1621  Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
1622  DAG.getConstant(NVTBits - 1, dl, ShTy));
1623  Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
1624  return true;
1625  }
1626  }
1627 
1628  // If we know that all of the high bits of the shift amount are zero, then we
1629  // can do this as a couple of simple shifts.
1630  if (HighBitMask.isSubsetOf(Known.Zero)) {
1631  // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
1632  // shift if x is zero. We can use XOR here because x is known to be smaller
1633  // than 32.
1634  SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
1635  DAG.getConstant(NVTBits - 1, dl, ShTy));
1636 
1637  unsigned Op1, Op2;
1638  switch (N->getOpcode()) {
1639  default: llvm_unreachable("Unknown shift");
1640  case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
1641  case ISD::SRL:
1642  case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
1643  }
1644 
1645  // When shifting right the arithmetic for Lo and Hi is swapped.
1646  if (N->getOpcode() != ISD::SHL)
1647  std::swap(InL, InH);
1648 
1649  // Use a little trick to get the bits that move from Lo to Hi. First
1650  // shift by one bit.
1651  SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
1652  // Then compute the remaining shift with amount-1.
1653  SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
1654 
1655  Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
1656  Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
1657 
1658  if (N->getOpcode() != ISD::SHL)
1659  std::swap(Hi, Lo);
1660  return true;
1661  }
1662 
1663  return false;
1664 }
1665 
1666 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
1667 /// of any size.
1668 bool DAGTypeLegalizer::
1669 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1670  SDValue Amt = N->getOperand(1);
1671  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1672  EVT ShTy = Amt.getValueType();
1673  unsigned NVTBits = NVT.getSizeInBits();
1674  assert(isPowerOf2_32(NVTBits) &&
1675  "Expanded integer type size not a power of two!");
1676  SDLoc dl(N);
1677 
1678  // Get the incoming operand to be shifted.
1679  SDValue InL, InH;
1680  GetExpandedInteger(N->getOperand(0), InL, InH);
1681 
1682  SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
1683  SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
1684  SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
1685  SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1686  Amt, NVBitsNode, ISD::SETULT);
1687  SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1688  Amt, DAG.getConstant(0, dl, ShTy),
1689  ISD::SETEQ);
1690 
1691  SDValue LoS, HiS, LoL, HiL;
1692  switch (N->getOpcode()) {
1693  default: llvm_unreachable("Unknown shift");
1694  case ISD::SHL:
1695  // Short: ShAmt < NVTBits
1696  LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
1697  HiS = DAG.getNode(ISD::OR, dl, NVT,
1698  DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
1699  DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
1700 
1701  // Long: ShAmt >= NVTBits
1702  LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
1703  HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
1704 
1705  Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
1706  Hi = DAG.getSelect(dl, NVT, isZero, InH,
1707  DAG.getSelect(dl, NVT, isShort, HiS, HiL));
1708  return true;
1709  case ISD::SRL:
1710  // Short: ShAmt < NVTBits
1711  HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
1712  LoS = DAG.getNode(ISD::OR, dl, NVT,
1713  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1714  // FIXME: If Amt is zero, the following shift generates an undefined result
1715  // on some architectures.
1716  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1717 
1718  // Long: ShAmt >= NVTBits
1719  HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1720  LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1721 
1722  Lo = DAG.getSelect(dl, NVT, isZero, InL,
1723  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1724  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1725  return true;
1726  case ISD::SRA:
1727  // Short: ShAmt < NVTBits
1728  HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
1729  LoS = DAG.getNode(ISD::OR, dl, NVT,
1730  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1731  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1732 
1733  // Long: ShAmt >= NVTBits
1734  HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
1735  DAG.getConstant(NVTBits - 1, dl, ShTy));
1736  LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1737 
1738  Lo = DAG.getSelect(dl, NVT, isZero, InL,
1739  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1740  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1741  return true;
1742  }
1743 }
1744 
1745 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
1746 
1747  switch (Op) {
1748  default: llvm_unreachable("invalid min/max opcode");
1749  case ISD::SMAX:
1750  return std::make_pair(ISD::SETGT, ISD::UMAX);
1751  case ISD::UMAX:
1752  return std::make_pair(ISD::SETUGT, ISD::UMAX);
1753  case ISD::SMIN:
1754  return std::make_pair(ISD::SETLT, ISD::UMIN);
1755  case ISD::UMIN:
1756  return std::make_pair(ISD::SETULT, ISD::UMIN);
1757  }
1758 }
1759 
1760 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
1761  SDValue &Lo, SDValue &Hi) {
1762  SDLoc DL(N);
1763  ISD::NodeType LoOpc;
1764  ISD::CondCode CondC;
1765  std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
1766 
1767  // Expand the subcomponents.
1768  SDValue LHSL, LHSH, RHSL, RHSH;
1769  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1770  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1771 
1772  // Value types
1773  EVT NVT = LHSL.getValueType();
1774  EVT CCT = getSetCCResultType(NVT);
1775 
1776  // Hi part is always the same op
1777  Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
1778 
1779  // We need to know whether to select Lo part that corresponds to 'winning'
1780  // Hi part or if Hi parts are equal.
1781  SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
1782  SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
1783 
1784  // Lo part corresponding to the 'winning' Hi part
1785  SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
1786 
1787  // Recursed Lo part if Hi parts are equal, this uses unsigned version
1788  SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
1789 
1790  Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
1791 }
1792 
1793 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
1794  SDValue &Lo, SDValue &Hi) {
1795  SDLoc dl(N);
1796  // Expand the subcomponents.
1797  SDValue LHSL, LHSH, RHSL, RHSH;
1798  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1799  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1800 
1801  EVT NVT = LHSL.getValueType();
1802  SDValue LoOps[2] = { LHSL, RHSL };
1803  SDValue HiOps[3] = { LHSH, RHSH };
1804 
1805  bool HasOpCarry = TLI.isOperationLegalOrCustom(
1807  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1808  if (HasOpCarry) {
1809  SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
1810  if (N->getOpcode() == ISD::ADD) {
1811  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
1812  HiOps[2] = Lo.getValue(1);
1813  Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
1814  } else {
1815  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
1816  HiOps[2] = Lo.getValue(1);
1817  Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
1818  }
1819  return;
1820  }
1821 
1822  // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
1823  // them. TODO: Teach operation legalization how to expand unsupported
1824  // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
1825  // a carry of type MVT::Glue, but there doesn't seem to be any way to
1826  // generate a value of this type in the expanded code sequence.
1827  bool hasCarry =
1829  ISD::ADDC : ISD::SUBC,
1830  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1831 
1832  if (hasCarry) {
1833  SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
1834  if (N->getOpcode() == ISD::ADD) {
1835  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
1836  HiOps[2] = Lo.getValue(1);
1837  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
1838  } else {
1839  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
1840  HiOps[2] = Lo.getValue(1);
1841  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
1842  }
1843  return;
1844  }
1845 
1846  bool hasOVF =
1849  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1851 
1852  if (hasOVF) {
1853  EVT OvfVT = getSetCCResultType(NVT);
1854  SDVTList VTList = DAG.getVTList(NVT, OvfVT);
1855  int RevOpc;
1856  if (N->getOpcode() == ISD::ADD) {
1857  RevOpc = ISD::SUB;
1858  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
1859  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
1860  } else {
1861  RevOpc = ISD::ADD;
1862  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
1863  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
1864  }
1865  SDValue OVF = Lo.getValue(1);
1866 
1867  switch (BoolType) {
1869  OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
1872  OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
1873  Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
1874  break;
1876  OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
1877  Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
1878  }
1879  return;
1880  }
1881 
1882  if (N->getOpcode() == ISD::ADD) {
1883  Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
1884  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
1885  SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
1886  ISD::SETULT);
1887 
1889  SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
1890  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
1891  return;
1892  }
1893 
1894  SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
1895  DAG.getConstant(1, dl, NVT),
1896  DAG.getConstant(0, dl, NVT));
1897  SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
1898  ISD::SETULT);
1899  SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
1900  DAG.getConstant(1, dl, NVT), Carry1);
1901  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
1902  } else {
1903  Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
1904  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
1905  SDValue Cmp =
1906  DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
1907  LoOps[0], LoOps[1], ISD::SETULT);
1908 
1909  SDValue Borrow;
1911  Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
1912  else
1913  Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
1914  DAG.getConstant(0, dl, NVT));
1915 
1916  Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
1917  }
1918 }
1919 
1920 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
1921  SDValue &Lo, SDValue &Hi) {
1922  // Expand the subcomponents.
1923  SDValue LHSL, LHSH, RHSL, RHSH;
1924  SDLoc dl(N);
1925  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1926  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1927  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
1928  SDValue LoOps[2] = { LHSL, RHSL };
1929  SDValue HiOps[3] = { LHSH, RHSH };
1930 
1931  if (N->getOpcode() == ISD::ADDC) {
1932  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
1933  HiOps[2] = Lo.getValue(1);
1934  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
1935  } else {
1936  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
1937  HiOps[2] = Lo.getValue(1);
1938  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
1939  }
1940 
1941  // Legalized the flag result - switch anything that used the old flag to
1942  // use the new one.
1943  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
1944 }
1945 
1946 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
1947  SDValue &Lo, SDValue &Hi) {
1948  // Expand the subcomponents.
1949  SDValue LHSL, LHSH, RHSL, RHSH;
1950  SDLoc dl(N);
1951  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1952  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1953  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
1954  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
1955  SDValue HiOps[3] = { LHSH, RHSH };
1956 
1957  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
1958  HiOps[2] = Lo.getValue(1);
1959  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
1960 
1961  // Legalized the flag result - switch anything that used the old flag to
1962  // use the new one.
1963  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
1964 }
1965 
1966 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
1967  SDValue &Lo, SDValue &Hi) {
1968  SDValue LHS = N->getOperand(0);
1969  SDValue RHS = N->getOperand(1);
1970  SDLoc dl(N);
1971 
1972  SDValue Ovf;
1973 
1974  bool HasOpCarry = TLI.isOperationLegalOrCustom(
1976  TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
1977 
1978  if (HasOpCarry) {
1979  // Expand the subcomponents.
1980  SDValue LHSL, LHSH, RHSL, RHSH;
1981  GetExpandedInteger(LHS, LHSL, LHSH);
1982  GetExpandedInteger(RHS, RHSL, RHSH);
1983  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
1984  SDValue LoOps[2] = { LHSL, RHSL };
1985  SDValue HiOps[3] = { LHSH, RHSH };
1986 
1987  unsigned Opc = N->getOpcode() == ISD::UADDO ? ISD::ADDCARRY : ISD::SUBCARRY;
1988  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
1989  HiOps[2] = Lo.getValue(1);
1990  Hi = DAG.getNode(Opc, dl, VTList, HiOps);
1991 
1992  Ovf = Hi.getValue(1);
1993  } else {
1994  // Expand the result by simply replacing it with the equivalent
1995  // non-overflow-checking operation.
1996  auto Opc = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
1997  SDValue Sum = DAG.getNode(Opc, dl, LHS.getValueType(), LHS, RHS);
1998  SplitInteger(Sum, Lo, Hi);
1999 
2000  // Calculate the overflow: addition overflows iff a + b < a, and subtraction
2001  // overflows iff a - b > a.
2002  auto Cond = N->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
2003  Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
2004  }
2005 
2006  // Legalized the flag result - switch anything that used the old flag to
2007  // use the new one.
2008  ReplaceValueWith(SDValue(N, 1), Ovf);
2009 }
2010 
2011 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
2012  SDValue &Lo, SDValue &Hi) {
2013  // Expand the subcomponents.
2014  SDValue LHSL, LHSH, RHSL, RHSH;
2015  SDLoc dl(N);
2016  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2017  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2018  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2019  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2020  SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
2021 
2022  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2023  HiOps[2] = Lo.getValue(1);
2024  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2025 
2026  // Legalized the flag result - switch anything that used the old flag to
2027  // use the new one.
2028  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2029 }
2030 
2031 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
2032  SDValue &Lo, SDValue &Hi) {
2033  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2034  SDLoc dl(N);
2035  SDValue Op = N->getOperand(0);
2036  if (Op.getValueType().bitsLE(NVT)) {
2037  // The low part is any extension of the input (which degenerates to a copy).
2038  Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
2039  Hi = DAG.getUNDEF(NVT); // The high part is undefined.
2040  } else {
2041  // For example, extension of an i48 to an i64. The operand type necessarily
2042  // promotes to the result type, so will end up being expanded too.
2043  assert(getTypeAction(Op.getValueType()) ==
2045  "Only know how to promote this result!");
2046  SDValue Res = GetPromotedInteger(Op);
2047  assert(Res.getValueType() == N->getValueType(0) &&
2048  "Operand over promoted?");
2049  // Split the promoted operand. This will simplify when it is expanded.
2050  SplitInteger(Res, Lo, Hi);
2051  }
2052 }
2053 
2054 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
2055  SDValue &Lo, SDValue &Hi) {
2056  SDLoc dl(N);
2057  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2058  EVT NVT = Lo.getValueType();
2059  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2060  unsigned NVTBits = NVT.getSizeInBits();
2061  unsigned EVTBits = EVT.getSizeInBits();
2062 
2063  if (NVTBits < EVTBits) {
2064  Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
2066  EVTBits - NVTBits)));
2067  } else {
2068  Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
2069  // The high part replicates the sign bit of Lo, make it explicit.
2070  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2071  DAG.getConstant(NVTBits - 1, dl,
2072  TLI.getPointerTy(DAG.getDataLayout())));
2073  }
2074 }
2075 
2076 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
2077  SDValue &Lo, SDValue &Hi) {
2078  SDLoc dl(N);
2079  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2080  EVT NVT = Lo.getValueType();
2081  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2082  unsigned NVTBits = NVT.getSizeInBits();
2083  unsigned EVTBits = EVT.getSizeInBits();
2084 
2085  if (NVTBits < EVTBits) {
2086  Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
2088  EVTBits - NVTBits)));
2089  } else {
2090  Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
2091  // The high part must be zero, make it explicit.
2092  Hi = DAG.getConstant(0, dl, NVT);
2093  }
2094 }
2095 
2096 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
2097  SDValue &Lo, SDValue &Hi) {
2098  SDLoc dl(N);
2099  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2100  Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
2101  Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
2102 }
2103 
2104 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
2105  SDValue &Lo, SDValue &Hi) {
2106  SDLoc dl(N);
2107  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2108  Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
2109  Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
2110 }
2111 
2112 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
2113  SDValue &Lo, SDValue &Hi) {
2114  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2115  unsigned NBitWidth = NVT.getSizeInBits();
2116  auto Constant = cast<ConstantSDNode>(N);
2117  const APInt &Cst = Constant->getAPIntValue();
2118  bool IsTarget = Constant->isTargetOpcode();
2119  bool IsOpaque = Constant->isOpaque();
2120  SDLoc dl(N);
2121  Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
2122  Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
2123  IsOpaque);
2124 }
2125 
2126 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
2127  SDValue &Lo, SDValue &Hi) {
2128  SDLoc dl(N);
2129  // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
2130  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2131  EVT NVT = Lo.getValueType();
2132 
2133  SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
2134  DAG.getConstant(0, dl, NVT), ISD::SETNE);
2135 
2136  SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
2137  SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
2138 
2139  Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
2140  DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
2141  DAG.getConstant(NVT.getSizeInBits(), dl,
2142  NVT)));
2143  Hi = DAG.getConstant(0, dl, NVT);
2144 }
2145 
2146 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
2147  SDValue &Lo, SDValue &Hi) {
2148  SDLoc dl(N);
2149  // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
2150  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2151  EVT NVT = Lo.getValueType();
2152  Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
2153  DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
2154  Hi = DAG.getConstant(0, dl, NVT);
2155 }
2156 
2157 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
2158  SDValue &Lo, SDValue &Hi) {
2159  SDLoc dl(N);
2160  // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
2161  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2162  EVT NVT = Lo.getValueType();
2163 
2164  SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
2165  DAG.getConstant(0, dl, NVT), ISD::SETNE);
2166 
2167  SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
2168  SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
2169 
2170  Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
2171  DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
2172  DAG.getConstant(NVT.getSizeInBits(), dl,
2173  NVT)));
2174  Hi = DAG.getConstant(0, dl, NVT);
2175 }
2176 
2177 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
2178  SDValue &Hi) {
2179  SDLoc dl(N);
2180  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2181  unsigned NBitWidth = NVT.getSizeInBits();
2182 
2183  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2184  Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT);
2185  // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
2186  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2187  DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
2188 }
2189 
2190 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
2191  SDValue &Hi) {
2192  SDLoc dl(N);
2193  EVT VT = N->getValueType(0);
2194 
2195  SDValue Op = N->getOperand(0);
2196  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2197  Op = GetPromotedFloat(Op);
2198 
2200  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
2201  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, true/*irrelevant*/, dl).first,
2202  Lo, Hi);
2203 }
2204 
2205 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
2206  SDValue &Hi) {
2207  SDLoc dl(N);
2208  EVT VT = N->getValueType(0);
2209 
2210  SDValue Op = N->getOperand(0);
2211  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2212  Op = GetPromotedFloat(Op);
2213 
2215  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
2216  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, false/*irrelevant*/, dl).first,
2217  Lo, Hi);
2218 }
2219 
2220 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
2221  SDValue &Lo, SDValue &Hi) {
2222  if (ISD::isNormalLoad(N)) {
2223  ExpandRes_NormalLoad(N, Lo, Hi);
2224  return;
2225  }
2226 
2227  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2228 
2229  EVT VT = N->getValueType(0);
2230  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2231  SDValue Ch = N->getChain();
2232  SDValue Ptr = N->getBasePtr();
2233  ISD::LoadExtType ExtType = N->getExtensionType();
2234  unsigned Alignment = N->getAlignment();
2235  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2236  AAMDNodes AAInfo = N->getAAInfo();
2237  SDLoc dl(N);
2238 
2239  assert(NVT.isByteSized() && "Expanded type not byte sized!");
2240 
2241  if (N->getMemoryVT().bitsLE(NVT)) {
2242  EVT MemVT = N->getMemoryVT();
2243 
2244  Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
2245  Alignment, MMOFlags, AAInfo);
2246 
2247  // Remember the chain.
2248  Ch = Lo.getValue(1);
2249 
2250  if (ExtType == ISD::SEXTLOAD) {
2251  // The high part is obtained by SRA'ing all but one of the bits of the
2252  // lo part.
2253  unsigned LoSize = Lo.getValueSizeInBits();
2254  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2255  DAG.getConstant(LoSize - 1, dl,
2256  TLI.getPointerTy(DAG.getDataLayout())));
2257  } else if (ExtType == ISD::ZEXTLOAD) {
2258  // The high part is just a zero.
2259  Hi = DAG.getConstant(0, dl, NVT);
2260  } else {
2261  assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
2262  // The high part is undefined.
2263  Hi = DAG.getUNDEF(NVT);
2264  }
2265  } else if (DAG.getDataLayout().isLittleEndian()) {
2266  // Little-endian - low bits are at low addresses.
2267  Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2268  AAInfo);
2269 
2270  unsigned ExcessBits =
2271  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
2272  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
2273 
2274  // Increment the pointer to the other half.
2275  unsigned IncrementSize = NVT.getSizeInBits()/8;
2276  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2277  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2278  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
2279  N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
2280  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2281 
2282  // Build a factor node to remember that this load is independent of the
2283  // other one.
2284  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2285  Hi.getValue(1));
2286  } else {
2287  // Big-endian - high bits are at low addresses. Favor aligned loads at
2288  // the cost of some bit-fiddling.
2289  EVT MemVT = N->getMemoryVT();
2290  unsigned EBytes = MemVT.getStoreSize();
2291  unsigned IncrementSize = NVT.getSizeInBits()/8;
2292  unsigned ExcessBits = (EBytes - IncrementSize)*8;
2293 
2294  // Load both the high bits and maybe some of the low bits.
2295  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
2297  MemVT.getSizeInBits() - ExcessBits),
2298  Alignment, MMOFlags, AAInfo);
2299 
2300  // Increment the pointer to the other half.
2301  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2302  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2303  // Load the rest of the low bits.
2304  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
2305  N->getPointerInfo().getWithOffset(IncrementSize),
2306  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
2307  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2308 
2309  // Build a factor node to remember that this load is independent of the
2310  // other one.
2311  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2312  Hi.getValue(1));
2313 
2314  if (ExcessBits < NVT.getSizeInBits()) {
2315  // Transfer low bits from the bottom of Hi to the top of Lo.
2316  Lo = DAG.getNode(
2317  ISD::OR, dl, NVT, Lo,
2318  DAG.getNode(ISD::SHL, dl, NVT, Hi,
2319  DAG.getConstant(ExcessBits, dl,
2320  TLI.getPointerTy(DAG.getDataLayout()))));
2321  // Move high bits to the right position in Hi.
2322  Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
2323  Hi,
2324  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
2325  TLI.getPointerTy(DAG.getDataLayout())));
2326  }
2327  }
2328 
2329  // Legalize the chain result - switch anything that used the old chain to
2330  // use the new one.
2331  ReplaceValueWith(SDValue(N, 1), Ch);
2332 }
2333 
2334 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
2335  SDValue &Lo, SDValue &Hi) {
2336  SDLoc dl(N);
2337  SDValue LL, LH, RL, RH;
2338  GetExpandedInteger(N->getOperand(0), LL, LH);
2339  GetExpandedInteger(N->getOperand(1), RL, RH);
2340  Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
2341  Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
2342 }
2343 
2344 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
2345  SDValue &Lo, SDValue &Hi) {
2346  EVT VT = N->getValueType(0);
2347  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2348  SDLoc dl(N);
2349 
2350  SDValue LL, LH, RL, RH;
2351  GetExpandedInteger(N->getOperand(0), LL, LH);
2352  GetExpandedInteger(N->getOperand(1), RL, RH);
2353 
2354  if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
2356  LL, LH, RL, RH))
2357  return;
2358 
2359  // If nothing else, we can make a libcall.
2360  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2361  if (VT == MVT::i16)
2362  LC = RTLIB::MUL_I16;
2363  else if (VT == MVT::i32)
2364  LC = RTLIB::MUL_I32;
2365  else if (VT == MVT::i64)
2366  LC = RTLIB::MUL_I64;
2367  else if (VT == MVT::i128)
2368  LC = RTLIB::MUL_I128;
2369 
2370  if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
2371  // We'll expand the multiplication by brute force because we have no other
2372  // options. This is a trivially-generalized version of the code from
2373  // Hacker's Delight (itself derived from Knuth's Algorithm M from section
2374  // 4.3.1).
2375  unsigned Bits = NVT.getSizeInBits();
2376  unsigned HalfBits = Bits >> 1;
2377  SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
2378  NVT);
2379  SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
2380  SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
2381 
2382  SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
2383  SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
2384 
2385  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2386  if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
2387  // The type from TLI is too small to fit the shift amount we want.
2388  // Override it with i32. The shift will have to be legalized.
2389  ShiftAmtTy = MVT::i32;
2390  }
2391  SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
2392  SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
2393  SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
2394  SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
2395 
2396  SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
2397  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
2398  SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
2399  SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
2400 
2401  SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
2402  DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
2403  SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
2404 
2405  SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
2406  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
2407  DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
2408  Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
2409  DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
2410 
2411  Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
2412  DAG.getNode(ISD::ADD, dl, NVT,
2413  DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
2414  DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
2415  return;
2416  }
2417 
2418  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2419  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true/*irrelevant*/, dl).first,
2420  Lo, Hi);
2421 }
2422 
2423 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
2424  SDValue &Hi) {
2425  SDLoc DL(N);
2426  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2427  SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
2428  SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
2429  Lo = R.getValue(0);
2430  Hi = R.getValue(1);
2431  ReplaceValueWith(SDValue(N, 1), R.getValue(2));
2432 }
2433 
2434 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
2435  SDValue &Lo, SDValue &Hi) {
2436  SDValue LHS = Node->getOperand(0);
2437  SDValue RHS = Node->getOperand(1);
2438  SDLoc dl(Node);
2439 
2440  // Expand the result by simply replacing it with the equivalent
2441  // non-overflow-checking operation.
2442  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
2443  ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
2444  LHS, RHS);
2445  SplitInteger(Sum, Lo, Hi);
2446 
2447  // Compute the overflow.
2448  //
2449  // LHSSign -> LHS >= 0
2450  // RHSSign -> RHS >= 0
2451  // SumSign -> Sum >= 0
2452  //
2453  // Add:
2454  // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
2455  // Sub:
2456  // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
2457  //
2458  EVT OType = Node->getValueType(1);
2459  SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
2460 
2461  SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
2462  SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
2463  SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
2464  Node->getOpcode() == ISD::SADDO ?
2466 
2467  SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
2468  SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
2469 
2470  SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
2471 
2472  // Use the calculated overflow everywhere.
2473  ReplaceValueWith(SDValue(Node, 1), Cmp);
2474 }
2475 
2476 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
2477  SDValue &Lo, SDValue &Hi) {
2478  EVT VT = N->getValueType(0);
2479  SDLoc dl(N);
2480  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2481 
2483  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2484  SplitInteger(Res.getValue(0), Lo, Hi);
2485  return;
2486  }
2487 
2488  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2489  if (VT == MVT::i16)
2490  LC = RTLIB::SDIV_I16;
2491  else if (VT == MVT::i32)
2492  LC = RTLIB::SDIV_I32;
2493  else if (VT == MVT::i64)
2494  LC = RTLIB::SDIV_I64;
2495  else if (VT == MVT::i128)
2496  LC = RTLIB::SDIV_I128;
2497  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
2498 
2499  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
2500 }
2501 
2502 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
2503  SDValue &Lo, SDValue &Hi) {
2504  EVT VT = N->getValueType(0);
2505  SDLoc dl(N);
2506 
2507  // If we can emit an efficient shift operation, do so now. Check to see if
2508  // the RHS is a constant.
2509  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
2510  return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
2511 
2512  // If we can determine that the high bit of the shift is zero or one, even if
2513  // the low bits are variable, emit this shift in an optimized form.
2514  if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
2515  return;
2516 
2517  // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
2518  unsigned PartsOpc;
2519  if (N->getOpcode() == ISD::SHL) {
2520  PartsOpc = ISD::SHL_PARTS;
2521  } else if (N->getOpcode() == ISD::SRL) {
2522  PartsOpc = ISD::SRL_PARTS;
2523  } else {
2524  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2525  PartsOpc = ISD::SRA_PARTS;
2526  }
2527 
2528  // Next check to see if the target supports this SHL_PARTS operation or if it
2529  // will custom expand it.
2530  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2531  TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
2532  if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
2533  Action == TargetLowering::Custom) {
2534  // Expand the subcomponents.
2535  SDValue LHSL, LHSH;
2536  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2537  EVT VT = LHSL.getValueType();
2538 
2539  // If the shift amount operand is coming from a vector legalization it may
2540  // have an illegal type. Fix that first by casting the operand, otherwise
2541  // the new SHL_PARTS operation would need further legalization.
2542  SDValue ShiftOp = N->getOperand(1);
2543  EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2544  assert(ShiftTy.getScalarSizeInBits() >=
2545  Log2_32_Ceil(VT.getScalarSizeInBits()) &&
2546  "ShiftAmountTy is too small to cover the range of this type!");
2547  if (ShiftOp.getValueType() != ShiftTy)
2548  ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
2549 
2550  SDValue Ops[] = { LHSL, LHSH, ShiftOp };
2551  Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
2552  Hi = Lo.getValue(1);
2553  return;
2554  }
2555 
2556  // Otherwise, emit a libcall.
2557  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2558  bool isSigned;
2559  if (N->getOpcode() == ISD::SHL) {
2560  isSigned = false; /*sign irrelevant*/
2561  if (VT == MVT::i16)
2562  LC = RTLIB::SHL_I16;
2563  else if (VT == MVT::i32)
2564  LC = RTLIB::SHL_I32;
2565  else if (VT == MVT::i64)
2566  LC = RTLIB::SHL_I64;
2567  else if (VT == MVT::i128)
2568  LC = RTLIB::SHL_I128;
2569  } else if (N->getOpcode() == ISD::SRL) {
2570  isSigned = false;
2571  if (VT == MVT::i16)
2572  LC = RTLIB::SRL_I16;
2573  else if (VT == MVT::i32)
2574  LC = RTLIB::SRL_I32;
2575  else if (VT == MVT::i64)
2576  LC = RTLIB::SRL_I64;
2577  else if (VT == MVT::i128)
2578  LC = RTLIB::SRL_I128;
2579  } else {
2580  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2581  isSigned = true;
2582  if (VT == MVT::i16)
2583  LC = RTLIB::SRA_I16;
2584  else if (VT == MVT::i32)
2585  LC = RTLIB::SRA_I32;
2586  else if (VT == MVT::i64)
2587  LC = RTLIB::SRA_I64;
2588  else if (VT == MVT::i128)
2589  LC = RTLIB::SRA_I128;
2590  }
2591 
2592  if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
2593  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2594  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, isSigned, dl).first, Lo, Hi);
2595  return;
2596  }
2597 
2598  if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
2599  llvm_unreachable("Unsupported shift!");
2600 }
2601 
2602 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
2603  SDValue &Lo, SDValue &Hi) {
2604  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2605  SDLoc dl(N);
2606  SDValue Op = N->getOperand(0);
2607  if (Op.getValueType().bitsLE(NVT)) {
2608  // The low part is sign extension of the input (degenerates to a copy).
2609  Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
2610  // The high part is obtained by SRA'ing all but one of the bits of low part.
2611  unsigned LoSize = NVT.getSizeInBits();
2612  Hi = DAG.getNode(
2613  ISD::SRA, dl, NVT, Lo,
2614  DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
2615  } else {
2616  // For example, extension of an i48 to an i64. The operand type necessarily
2617  // promotes to the result type, so will end up being expanded too.
2618  assert(getTypeAction(Op.getValueType()) ==
2620  "Only know how to promote this result!");
2621  SDValue Res = GetPromotedInteger(Op);
2622  assert(Res.getValueType() == N->getValueType(0) &&
2623  "Operand over promoted?");
2624  // Split the promoted operand. This will simplify when it is expanded.
2625  SplitInteger(Res, Lo, Hi);
2626  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
2627  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2629  ExcessBits)));
2630  }
2631 }
2632 
2633 void DAGTypeLegalizer::
2634 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2635  SDLoc dl(N);
2636  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2637  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2638 
2639  if (EVT.bitsLE(Lo.getValueType())) {
2640  // sext_inreg the low part if needed.
2641  Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
2642  N->getOperand(1));
2643 
2644  // The high part gets the sign extension from the lo-part. This handles
2645  // things like sextinreg V:i64 from i8.
2646  Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
2647  DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
2648  TLI.getPointerTy(DAG.getDataLayout())));
2649  } else {
2650  // For example, extension of an i48 to an i64. Leave the low part alone,
2651  // sext_inreg the high part.
2652  unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
2653  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2655  ExcessBits)));
2656  }
2657 }
2658 
2659 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
2660  SDValue &Lo, SDValue &Hi) {
2661  EVT VT = N->getValueType(0);
2662  SDLoc dl(N);
2663  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2664 
2666  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2667  SplitInteger(Res.getValue(1), Lo, Hi);
2668  return;
2669  }
2670 
2671  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2672  if (VT == MVT::i16)
2673  LC = RTLIB::SREM_I16;
2674  else if (VT == MVT::i32)
2675  LC = RTLIB::SREM_I32;
2676  else if (VT == MVT::i64)
2677  LC = RTLIB::SREM_I64;
2678  else if (VT == MVT::i128)
2679  LC = RTLIB::SREM_I128;
2680  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
2681 
2682  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
2683 }
2684 
2685 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
2686  SDValue &Lo, SDValue &Hi) {
2687  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2688  SDLoc dl(N);
2689  Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
2690  Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
2691  N->getOperand(0),
2692  DAG.getConstant(NVT.getSizeInBits(), dl,
2693  TLI.getPointerTy(DAG.getDataLayout())));
2694  Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
2695 }
2696 
2697 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
2698  SDValue &Lo, SDValue &Hi) {
2699  EVT VT = N->getValueType(0);
2700  SDLoc dl(N);
2701 
2702  // A divide for UMULO should be faster than a function call.
2703  if (N->getOpcode() == ISD::UMULO) {
2704  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
2705 
2706  SDValue MUL = DAG.getNode(ISD::MUL, dl, LHS.getValueType(), LHS, RHS);
2707  SplitInteger(MUL, Lo, Hi);
2708 
2709  // A divide for UMULO will be faster than a function call. Select to
2710  // make sure we aren't using 0.
2711  SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(VT),
2712  RHS, DAG.getConstant(0, dl, VT), ISD::SETEQ);
2713  SDValue NotZero = DAG.getSelect(dl, VT, isZero,
2714  DAG.getConstant(1, dl, VT), RHS);
2715  SDValue DIV = DAG.getNode(ISD::UDIV, dl, VT, MUL, NotZero);
2716  SDValue Overflow = DAG.getSetCC(dl, N->getValueType(1), DIV, LHS,
2717  ISD::SETNE);
2718  Overflow = DAG.getSelect(dl, N->getValueType(1), isZero,
2719  DAG.getConstant(0, dl, N->getValueType(1)),
2720  Overflow);
2721  ReplaceValueWith(SDValue(N, 1), Overflow);
2722  return;
2723  }
2724 
2725  Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
2726  EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
2727  Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
2728 
2729  // Replace this with a libcall that will check overflow.
2730  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2731  if (VT == MVT::i32)
2732  LC = RTLIB::MULO_I32;
2733  else if (VT == MVT::i64)
2734  LC = RTLIB::MULO_I64;
2735  else if (VT == MVT::i128)
2736  LC = RTLIB::MULO_I128;
2737  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
2738 
2739  SDValue Temp = DAG.CreateStackTemporary(PtrVT);
2740  // Temporary for the overflow value, default it to zero.
2741  SDValue Chain =
2742  DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
2743  MachinePointerInfo());
2744 
2746  TargetLowering::ArgListEntry Entry;
2747  for (const SDValue &Op : N->op_values()) {
2748  EVT ArgVT = Op.getValueType();
2749  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2750  Entry.Node = Op;
2751  Entry.Ty = ArgTy;
2752  Entry.IsSExt = true;
2753  Entry.IsZExt = false;
2754  Args.push_back(Entry);
2755  }
2756 
2757  // Also pass the address of the overflow check.
2758  Entry.Node = Temp;
2759  Entry.Ty = PtrTy->getPointerTo();
2760  Entry.IsSExt = true;
2761  Entry.IsZExt = false;
2762  Args.push_back(Entry);
2763 
2764  SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
2765 
2767  CLI.setDebugLoc(dl)
2768  .setChain(Chain)
2769  .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
2770  .setSExtResult();
2771 
2772  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2773 
2774  SplitInteger(CallInfo.first, Lo, Hi);
2775  SDValue Temp2 =
2776  DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
2777  SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
2778  DAG.getConstant(0, dl, PtrVT),
2779  ISD::SETNE);
2780  // Use the overflow from the libcall everywhere.
2781  ReplaceValueWith(SDValue(N, 1), Ofl);
2782 }
2783 
2784 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
2785  SDValue &Lo, SDValue &Hi) {
2786  EVT VT = N->getValueType(0);
2787  SDLoc dl(N);
2788  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2789 
2791  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2792  SplitInteger(Res.getValue(0), Lo, Hi);
2793  return;
2794  }
2795 
2796  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2797  if (VT == MVT::i16)
2798  LC = RTLIB::UDIV_I16;
2799  else if (VT == MVT::i32)
2800  LC = RTLIB::UDIV_I32;
2801  else if (VT == MVT::i64)
2802  LC = RTLIB::UDIV_I64;
2803  else if (VT == MVT::i128)
2804  LC = RTLIB::UDIV_I128;
2805  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
2806 
2807  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
2808 }
2809 
2810 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
2811  SDValue &Lo, SDValue &Hi) {
2812  EVT VT = N->getValueType(0);
2813  SDLoc dl(N);
2814  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2815 
2817  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2818  SplitInteger(Res.getValue(1), Lo, Hi);
2819  return;
2820  }
2821 
2822  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2823  if (VT == MVT::i16)
2824  LC = RTLIB::UREM_I16;
2825  else if (VT == MVT::i32)
2826  LC = RTLIB::UREM_I32;
2827  else if (VT == MVT::i64)
2828  LC = RTLIB::UREM_I64;
2829  else if (VT == MVT::i128)
2830  LC = RTLIB::UREM_I128;
2831  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
2832 
2833  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
2834 }
2835 
2836 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
2837  SDValue &Lo, SDValue &Hi) {
2838  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2839  SDLoc dl(N);
2840  SDValue Op = N->getOperand(0);
2841  if (Op.getValueType().bitsLE(NVT)) {
2842  // The low part is zero extension of the input (degenerates to a copy).
2843  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
2844  Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
2845  } else {
2846  // For example, extension of an i48 to an i64. The operand type necessarily
2847  // promotes to the result type, so will end up being expanded too.
2848  assert(getTypeAction(Op.getValueType()) ==
2850  "Only know how to promote this result!");
2851  SDValue Res = GetPromotedInteger(Op);
2852  assert(Res.getValueType() == N->getValueType(0) &&
2853  "Operand over promoted?");
2854  // Split the promoted operand. This will simplify when it is expanded.
2855  SplitInteger(Res, Lo, Hi);
2856  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
2857  Hi = DAG.getZeroExtendInReg(Hi, dl,
2859  ExcessBits));
2860  }
2861 }
2862 
2863 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
2864  SDValue &Lo, SDValue &Hi) {
2865  SDLoc dl(N);
2866  EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
2867  SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
2868  SDValue Zero = DAG.getConstant(0, dl, VT);
2869  SDValue Swap = DAG.getAtomicCmpSwap(
2871  cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
2872  N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
2873 
2874  ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
2875  ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
2876 }
2877 
2878 //===----------------------------------------------------------------------===//
2879 // Integer Operand Expansion
2880 //===----------------------------------------------------------------------===//
2881 
2882 /// ExpandIntegerOperand - This method is called when the specified operand of
2883 /// the specified node is found to need expansion. At this point, all of the
2884 /// result types of the node are known to be legal, but other operands of the
2885 /// node may need promotion or expansion as well as the specified one.
2886 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
2887  DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); dbgs() << "\n");
2888  SDValue Res = SDValue();
2889 
2890  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2891  return false;
2892 
2893  switch (N->getOpcode()) {
2894  default:
2895  #ifndef NDEBUG
2896  dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
2897  N->dump(&DAG); dbgs() << "\n";
2898  #endif
2899  llvm_unreachable("Do not know how to expand this operator's operand!");
2900 
2901  case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
2902  case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
2903  case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
2904  case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
2905  case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
2906  case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
2907  case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
2908  case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
2909  case ISD::SETCCE: Res = ExpandIntOp_SETCCE(N); break;
2910  case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
2911  case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break;
2912  case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
2913  case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
2914  case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break;
2915 
2916  case ISD::SHL:
2917  case ISD::SRA:
2918  case ISD::SRL:
2919  case ISD::ROTL:
2920  case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
2921  case ISD::RETURNADDR:
2922  case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
2923 
2924  case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
2925  }
2926 
2927  // If the result is null, the sub-method took care of registering results etc.
2928  if (!Res.getNode()) return false;
2929 
2930  // If the result is N, the sub-method updated N in place. Tell the legalizer
2931  // core about this.
2932  if (Res.getNode() == N)
2933  return true;
2934 
2935  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2936  "Invalid operand expansion");
2937 
2938  ReplaceValueWith(SDValue(N, 0), Res);
2939  return false;
2940 }
2941 
2942 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
2943 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
2944 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
2945  SDValue &NewRHS,
2946  ISD::CondCode &CCCode,
2947  const SDLoc &dl) {
2948  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2949  GetExpandedInteger(NewLHS, LHSLo, LHSHi);
2950  GetExpandedInteger(NewRHS, RHSLo, RHSHi);
2951 
2952  if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
2953  if (RHSLo == RHSHi) {
2954  if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
2955  if (RHSCST->isAllOnesValue()) {
2956  // Equality comparison to -1.
2957  NewLHS = DAG.getNode(ISD::AND, dl,
2958  LHSLo.getValueType(), LHSLo, LHSHi);
2959  NewRHS = RHSLo;
2960  return;
2961  }
2962  }
2963  }
2964 
2965  NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
2966  NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
2967  NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
2968  NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
2969  return;
2970  }
2971 
2972  // If this is a comparison of the sign bit, just look at the top part.
2973  // X > -1, x < 0
2974  if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
2975  if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0
2976  (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1
2977  NewLHS = LHSHi;
2978  NewRHS = RHSHi;
2979  return;
2980  }
2981 
2982  // FIXME: This generated code sucks.
2983  ISD::CondCode LowCC;
2984  switch (CCCode) {
2985  default: llvm_unreachable("Unknown integer setcc!");
2986  case ISD::SETLT:
2987  case ISD::SETULT: LowCC = ISD::SETULT; break;
2988  case ISD::SETGT:
2989  case ISD::SETUGT: LowCC = ISD::SETUGT; break;
2990  case ISD::SETLE:
2991  case ISD::SETULE: LowCC = ISD::SETULE; break;
2992  case ISD::SETGE:
2993  case ISD::SETUGE: LowCC = ISD::SETUGE; break;
2994  }
2995 
2996  // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
2997  // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
2998  // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
2999 
3000  // NOTE: on targets without efficient SELECT of bools, we can always use
3001  // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
3002  TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
3003  nullptr);
3004  SDValue LoCmp, HiCmp;
3005  if (TLI.isTypeLegal(LHSLo.getValueType()) &&
3006  TLI.isTypeLegal(RHSLo.getValueType()))
3007  LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
3008  RHSLo, LowCC, false, DagCombineInfo, dl);
3009  if (!LoCmp.getNode())
3010  LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
3011  RHSLo, LowCC);
3012  if (TLI.isTypeLegal(LHSHi.getValueType()) &&
3013  TLI.isTypeLegal(RHSHi.getValueType()))
3014  HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
3015  RHSHi, CCCode, false, DagCombineInfo, dl);
3016  if (!HiCmp.getNode())
3017  HiCmp =
3018  DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
3019  LHSHi, RHSHi, DAG.getCondCode(CCCode));
3020 
3021  ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
3022  ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
3023 
3024  bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
3025  CCCode == ISD::SETUGE || CCCode == ISD::SETULE);
3026 
3027  if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) ||
3028  (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) ||
3029  (LoCmpC && LoCmpC->isNullValue())))) {
3030  // For LE / GE, if high part is known false, ignore the low part.
3031  // For LT / GT: if low part is known false, return the high part.
3032  // if high part is known true, ignore the low part.
3033  NewLHS = HiCmp;
3034  NewRHS = SDValue();
3035  return;
3036  }
3037 
3038  if (LHSHi == RHSHi) {
3039  // Comparing the low bits is enough.
3040  NewLHS = LoCmp;
3041  NewRHS = SDValue();
3042  return;
3043  }
3044 
3045  // Lower with SETCCE or SETCCCARRY if the target supports it.
3046  EVT HiVT = LHSHi.getValueType();
3047  EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
3048  bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
3049 
3050  // FIXME: Make all targets support this, then remove the other lowering.
3051  if (HasSETCCCARRY ||
3053  // SETCCE/SETCCCARRY can detect < and >= directly. For > and <=, flip
3054  // operands and condition code.
3055  bool FlipOperands = false;
3056  switch (CCCode) {
3057  case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
3058  case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
3059  case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
3060  case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
3061  default: break;
3062  }
3063  if (FlipOperands) {
3064  std::swap(LHSLo, RHSLo);
3065  std::swap(LHSHi, RHSHi);
3066  }
3067  // Perform a wide subtraction, feeding the carry from the low part into
3068  // SETCCE/SETCCCARRY. The SETCCE/SETCCCARRY operation is essentially
3069  // looking at the high part of the result of LHS - RHS. It is negative
3070  // iff LHS < RHS. It is zero or positive iff LHS >= RHS.
3071  EVT LoVT = LHSLo.getValueType();
3072  SDVTList VTList = DAG.getVTList(
3073  LoVT, HasSETCCCARRY ? getSetCCResultType(LoVT) : MVT::Glue);
3074  SDValue LowCmp = DAG.getNode(HasSETCCCARRY ? ISD::USUBO : ISD::SUBC, dl,
3075  VTList, LHSLo, RHSLo);
3076  SDValue Res = DAG.getNode(HasSETCCCARRY ? ISD::SETCCCARRY : ISD::SETCCE, dl,
3077  getSetCCResultType(HiVT), LHSHi, RHSHi,
3078  LowCmp.getValue(1), DAG.getCondCode(CCCode));
3079  NewLHS = Res;
3080  NewRHS = SDValue();
3081  return;
3082  }
3083 
3084  NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
3085  false, DagCombineInfo, dl);
3086  if (!NewLHS.getNode())
3087  NewLHS =
3088  DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
3089  NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
3090  NewRHS = SDValue();
3091 }
3092 
3093 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
3094  SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
3095  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
3096  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3097 
3098  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3099  // against zero to select between true and false values.
3100  if (!NewRHS.getNode()) {
3101  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3102  CCCode = ISD::SETNE;
3103  }
3104 
3105  // Update N to have the operands specified.
3106  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
3107  DAG.getCondCode(CCCode), NewLHS, NewRHS,
3108  N->getOperand(4)), 0);
3109 }
3110 
3111 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
3112  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3113  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
3114  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3115 
3116  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3117  // against zero to select between true and false values.
3118  if (!NewRHS.getNode()) {
3119  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3120  CCCode = ISD::SETNE;
3121  }
3122 
3123  // Update N to have the operands specified.
3124  return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
3125  N->getOperand(2), N->getOperand(3),
3126  DAG.getCondCode(CCCode)), 0);
3127 }
3128 
3129 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
3130  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3131  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3132  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3133 
3134  // If ExpandSetCCOperands returned a scalar, use it.
3135  if (!NewRHS.getNode()) {
3136  assert(NewLHS.getValueType() == N->getValueType(0) &&
3137  "Unexpected setcc expansion!");
3138  return NewLHS;
3139  }
3140 
3141  // Otherwise, update N to have the operands specified.
3142  return SDValue(
3143  DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
3144 }
3145 
3146 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCE(SDNode *N) {
3147  SDValue LHS = N->getOperand(0);
3148  SDValue RHS = N->getOperand(1);
3149  SDValue Carry = N->getOperand(2);
3150  SDValue Cond = N->getOperand(3);
3151  SDLoc dl = SDLoc(N);
3152 
3153  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3154  GetExpandedInteger(LHS, LHSLo, LHSHi);
3155  GetExpandedInteger(RHS, RHSLo, RHSHi);
3156 
3157  // Expand to a SUBE for the low part and a smaller SETCCE for the high.
3158  SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), MVT::Glue);
3159  SDValue LowCmp = DAG.getNode(ISD::SUBE, dl, VTList, LHSLo, RHSLo, Carry);
3160  return DAG.getNode(ISD::SETCCE, dl, N->getValueType(0), LHSHi, RHSHi,
3161  LowCmp.getValue(1), Cond);
3162 }
3163 
3164 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
3165  SDValue LHS = N->getOperand(0);
3166  SDValue RHS = N->getOperand(1);
3167  SDValue Carry = N->getOperand(2);
3168  SDValue Cond = N->getOperand(3);
3169  SDLoc dl = SDLoc(N);
3170 
3171  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3172  GetExpandedInteger(LHS, LHSLo, LHSHi);
3173  GetExpandedInteger(RHS, RHSLo, RHSHi);
3174 
3175  // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
3176  SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
3177  SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
3178  return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
3179  LowCmp.getValue(1), Cond);
3180 }
3181 
3182 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
3183  // The value being shifted is legal, but the shift amount is too big.
3184  // It follows that either the result of the shift is undefined, or the
3185  // upper half of the shift amount is zero. Just use the lower half.
3186  SDValue Lo, Hi;
3187  GetExpandedInteger(N->getOperand(1), Lo, Hi);
3188  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
3189 }
3190 
3191 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
3192  // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
3193  // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
3194  // constant to valid type.
3195  SDValue Lo, Hi;
3196  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3197  return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
3198 }
3199 
3200 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
3201  SDValue Op = N->getOperand(0);
3202  EVT DstVT = N->getValueType(0);
3204  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3205  "Don't know how to expand this SINT_TO_FP!");
3206  return TLI.makeLibCall(DAG, LC, DstVT, Op, true, SDLoc(N)).first;
3207 }
3208 
3209 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
3210  if (ISD::isNormalStore(N))
3211  return ExpandOp_NormalStore(N, OpNo);
3212 
3213  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
3214  assert(OpNo == 1 && "Can only expand the stored value so far");
3215 
3216  EVT VT = N->getOperand(1).getValueType();
3217  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3218  SDValue Ch = N->getChain();
3219  SDValue Ptr = N->getBasePtr();
3220  unsigned Alignment = N->getAlignment();
3221  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3222  AAMDNodes AAInfo = N->getAAInfo();
3223  SDLoc dl(N);
3224  SDValue Lo, Hi;
3225 
3226  assert(NVT.isByteSized() && "Expanded type not byte sized!");
3227 
3228  if (N->getMemoryVT().bitsLE(NVT)) {
3229  GetExpandedInteger(N->getValue(), Lo, Hi);
3230  return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
3231  N->getMemoryVT(), Alignment, MMOFlags, AAInfo);
3232  }
3233 
3234  if (DAG.getDataLayout().isLittleEndian()) {
3235  // Little-endian - low bits are at low addresses.
3236  GetExpandedInteger(N->getValue(), Lo, Hi);
3237 
3238  Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3239  AAInfo);
3240 
3241  unsigned ExcessBits =
3242  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3243  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3244 
3245  // Increment the pointer to the other half.
3246  unsigned IncrementSize = NVT.getSizeInBits()/8;
3247  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3248  Hi = DAG.getTruncStore(
3249  Ch, dl, Hi, Ptr, N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3250  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3251  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3252  }
3253 
3254  // Big-endian - high bits are at low addresses. Favor aligned stores at
3255  // the cost of some bit-fiddling.
3256  GetExpandedInteger(N->getValue(), Lo, Hi);
3257 
3258  EVT ExtVT = N->getMemoryVT();
3259  unsigned EBytes = ExtVT.getStoreSize();
3260  unsigned IncrementSize = NVT.getSizeInBits()/8;
3261  unsigned ExcessBits = (EBytes - IncrementSize)*8;
3262  EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
3263  ExtVT.getSizeInBits() - ExcessBits);
3264 
3265  if (ExcessBits < NVT.getSizeInBits()) {
3266  // Transfer high bits from the top of Lo to the bottom of Hi.
3267  Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
3268  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3269  TLI.getPointerTy(DAG.getDataLayout())));
3270  Hi = DAG.getNode(
3271  ISD::OR, dl, NVT, Hi,
3272  DAG.getNode(ISD::SRL, dl, NVT, Lo,
3273  DAG.getConstant(ExcessBits, dl,
3274  TLI.getPointerTy(DAG.getDataLayout()))));
3275  }
3276 
3277  // Store both the high bits and maybe some of the low bits.
3278  Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment,
3279  MMOFlags, AAInfo);
3280 
3281  // Increment the pointer to the other half.
3282  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3283  // Store the lowest ExcessBits bits in the second half.
3284  Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
3285  N->getPointerInfo().getWithOffset(IncrementSize),
3286  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3287  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3288  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3289 }
3290 
3291 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
3292  SDValue InL, InH;
3293  GetExpandedInteger(N->getOperand(0), InL, InH);
3294  // Just truncate the low part of the source.
3295  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
3296 }
3297 
3298 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
3299  SDValue Op = N->getOperand(0);
3300  EVT SrcVT = Op.getValueType();
3301  EVT DstVT = N->getValueType(0);
3302  SDLoc dl(N);
3303 
3304  // The following optimization is valid only if every value in SrcVT (when
3305  // treated as signed) is representable in DstVT. Check that the mantissa
3306  // size of DstVT is >= than the number of bits in SrcVT -1.
3307  const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT);
3308  if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 &&
3310  // Do a signed conversion then adjust the result.
3311  SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op);
3312  SignedConv = TLI.LowerOperation(SignedConv, DAG);
3313 
3314  // The result of the signed conversion needs adjusting if the 'sign bit' of
3315  // the incoming integer was set. To handle this, we dynamically test to see
3316  // if it is set, and, if so, add a fudge factor.
3317 
3318  const uint64_t F32TwoE32 = 0x4F800000ULL;
3319  const uint64_t F32TwoE64 = 0x5F800000ULL;
3320  const uint64_t F32TwoE128 = 0x7F800000ULL;
3321 
3322  APInt FF(32, 0);
3323  if (SrcVT == MVT::i32)
3324  FF = APInt(32, F32TwoE32);
3325  else if (SrcVT == MVT::i64)
3326  FF = APInt(32, F32TwoE64);
3327  else if (SrcVT == MVT::i128)
3328  FF = APInt(32, F32TwoE128);
3329  else
3330  llvm_unreachable("Unsupported UINT_TO_FP!");
3331 
3332  // Check whether the sign bit is set.
3333  SDValue Lo, Hi;
3334  GetExpandedInteger(Op, Lo, Hi);
3335  SDValue SignSet = DAG.getSetCC(dl,
3336  getSetCCResultType(Hi.getValueType()),
3337  Hi,
3338  DAG.getConstant(0, dl, Hi.getValueType()),
3339  ISD::SETLT);
3340 
3341  // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
3342  SDValue FudgePtr =
3343  DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)),
3344  TLI.getPointerTy(DAG.getDataLayout()));
3345 
3346  // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
3347  SDValue Zero = DAG.getIntPtrConstant(0, dl);
3348  SDValue Four = DAG.getIntPtrConstant(4, dl);
3349  if (DAG.getDataLayout().isBigEndian())
3350  std::swap(Zero, Four);
3351  SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet,
3352  Zero, Four);
3353  unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment();
3354  FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(),
3355  FudgePtr, Offset);
3356  Alignment = std::min(Alignment, 4u);
3357 
3358  // Load the value out, extending it from f32 to the destination float type.
3359  // FIXME: Avoid the extend by constructing the right constant pool?
3360  SDValue Fudge = DAG.getExtLoad(
3361  ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), FudgePtr,
3363  Alignment);
3364  return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge);
3365  }
3366 
3367  // Otherwise, use a libcall.
3368  RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT);
3369  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3370  "Don't know how to expand this UINT_TO_FP!");
3371  return TLI.makeLibCall(DAG, LC, DstVT, Op, true, dl).first;
3372 }
3373 
3374 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
3375  SDLoc dl(N);
3376  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
3377  cast<AtomicSDNode>(N)->getMemoryVT(),
3378  N->getOperand(0),
3379  N->getOperand(1), N->getOperand(2),
3380  cast<AtomicSDNode>(N)->getMemOperand());
3381  return Swap.getValue(1);
3382 }
3383 
3384 
3385 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
3386  SDValue InOp0 = N->getOperand(0);
3387  EVT InVT = InOp0.getValueType();
3388 
3389  EVT OutVT = N->getValueType(0);
3390  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3391  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3392  unsigned OutNumElems = OutVT.getVectorNumElements();
3393  EVT NOutVTElem = NOutVT.getVectorElementType();
3394 
3395  SDLoc dl(N);
3396  SDValue BaseIdx = N->getOperand(1);
3397 
3399  Ops.reserve(OutNumElems);
3400  for (unsigned i = 0; i != OutNumElems; ++i) {
3401 
3402  // Extract the element from the original vector.
3403  SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
3404  BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
3406  InVT.getVectorElementType(), N->getOperand(0), Index);
3407 
3408  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext);
3409  // Insert the converted element to the new vector.
3410  Ops.push_back(Op);
3411  }
3412 
3413  return DAG.getBuildVector(NOutVT, dl, Ops);
3414 }
3415 
3416 
3417 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
3418  ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
3419  EVT VT = N->getValueType(0);
3420  SDLoc dl(N);
3421 
3422  ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
3423 
3424  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3425  SDValue V1 = GetPromotedInteger(N->getOperand(1));
3426  EVT OutVT = V0.getValueType();
3427 
3428  return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
3429 }
3430 
3431 
3432 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
3433  EVT OutVT = N->getValueType(0);
3434  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3435  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3436  unsigned NumElems = N->getNumOperands();
3437  EVT NOutVTElem = NOutVT.getVectorElementType();
3438 
3439  SDLoc dl(N);
3440 
3442  Ops.reserve(NumElems);
3443  for (unsigned i = 0; i != NumElems; ++i) {
3444  SDValue Op;
3445  // BUILD_VECTOR integer operand types are allowed to be larger than the
3446  // result's element type. This may still be true after the promotion. For
3447  // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
3448  // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
3449  if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
3450  Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
3451  else
3452  Op = N->getOperand(i);
3453  Ops.push_back(Op);
3454  }
3455 
3456  return DAG.getBuildVector(NOutVT, dl, Ops);
3457 }
3458 
3459 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
3460 
3461  SDLoc dl(N);
3462 
3463  assert(!N->getOperand(0).getValueType().isVector() &&
3464  "Input must be a scalar");
3465 
3466  EVT OutVT = N->getValueType(0);
3467  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3468  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3469  EVT NOutVTElem = NOutVT.getVectorElementType();
3470 
3471  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
3472 
3473  return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
3474 }
3475 
3476 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
3477  SDLoc dl(N);
3478 
3479  EVT OutVT = N->getValueType(0);
3480  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3481  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3482 
3483  EVT OutElemTy = NOutVT.getVectorElementType();
3484 
3485  unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
3486  unsigned NumOutElem = NOutVT.getVectorNumElements();
3487  unsigned NumOperands = N->getNumOperands();
3488  assert(NumElem * NumOperands == NumOutElem &&
3489  "Unexpected number of elements");
3490 
3491  // If the input type is legal and we can promote it to a legal type with the
3492  // same element size, go ahead do that to create a new concat.
3493  if (getTypeAction(N->getOperand(0).getValueType()) ==
3495  EVT InPromotedTy = EVT::getVectorVT(*DAG.getContext(), OutElemTy, NumElem);
3496  if (TLI.isTypeLegal(InPromotedTy)) {
3497  SmallVector<SDValue, 8> Ops(NumOperands);
3498  for (unsigned i = 0; i < NumOperands; ++i) {
3499  Ops[i] = DAG.getNode(ISD::ANY_EXTEND, dl, InPromotedTy,
3500  N->getOperand(i));
3501  }
3502  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NOutVT, Ops);
3503  }
3504  }
3505 
3506  // Take the elements from the first vector.
3507  SmallVector<SDValue, 8> Ops(NumOutElem);
3508  for (unsigned i = 0; i < NumOperands; ++i) {
3509  SDValue Op = N->getOperand(i);
3510  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
3511  Op = GetPromotedInteger(Op);
3512  EVT SclrTy = Op.getValueType().getVectorElementType();
3513  assert(NumElem == Op.getValueType().getVectorNumElements() &&
3514  "Unexpected number of elements");
3515 
3516  for (unsigned j = 0; j < NumElem; ++j) {
3517  SDValue Ext = DAG.getNode(
3518  ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
3519  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3520  Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
3521  }
3522  }
3523 
3524  return DAG.getBuildVector(NOutVT, dl, Ops);
3525 }
3526 
3527 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
3528  EVT VT = N->getValueType(0);
3529  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3530  assert(NVT.isVector() && "This type must be promoted to a vector type");
3531 
3532  SDLoc dl(N);
3533 
3534  // For operands whose TypeAction is to promote, extend the promoted node
3535  // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
3536  // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
3537  // type..
3538  if (getTypeAction(N->getOperand(0).getValueType())
3540  SDValue Promoted;
3541 
3542  switch(N->getOpcode()) {
3544  Promoted = SExtPromotedInteger(N->getOperand(0));
3545  break;
3547  Promoted = ZExtPromotedInteger(N->getOperand(0));
3548  break;
3550  Promoted = GetPromotedInteger(N->getOperand(0));
3551  break;
3552  default:
3553  llvm_unreachable("Node has unexpected Opcode");
3554  }
3555  return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
3556  }
3557 
3558  // Directly extend to the appropriate transform-to type.
3559  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
3560 }
3561 
3562 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
3563  EVT OutVT = N->getValueType(0);
3564  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3565  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3566 
3567  EVT NOutVTElem = NOutVT.getVectorElementType();
3568 
3569  SDLoc dl(N);
3570  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3571 
3572  SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
3573  NOutVTElem, N->getOperand(1));
3574  return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
3575  V0, ConvElem, N->getOperand(2));
3576 }
3577 
3578 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3579  SDLoc dl(N);
3580  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3581  SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
3582  TLI.getVectorIdxTy(DAG.getDataLayout()));
3584  V0->getValueType(0).getScalarType(), V0, V1);
3585 
3586  // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
3587  // element types. If this is the case then we need to expand the outgoing
3588  // value and not truncate it.
3589  return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
3590 }
3591 
3592 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
3593  SDLoc dl(N);
3594  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3595  MVT InVT = V0.getValueType().getSimpleVT();
3596  MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
3598  SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
3599  return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
3600 }
3601 
3602 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
3603  SDLoc dl(N);
3604  unsigned NumElems = N->getNumOperands();
3605 
3606  EVT RetSclrTy = N->getValueType(0).getVectorElementType();
3607 
3608  SmallVector<SDValue, 8> NewOps;
3609  NewOps.reserve(NumElems);
3610 
3611  // For each incoming vector
3612  for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
3613  SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
3614  EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
3615  unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
3616 
3617  for (unsigned i=0; i<NumElem; ++i) {
3618  // Extract element from incoming vector
3619  SDValue Ex = DAG.getNode(
3620  ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
3621  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3622  SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
3623  NewOps.push_back(Tr);
3624  }
3625  }
3626 
3627  return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
3628 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:546
void push_back(const T &Elt)
Definition: SmallVector.h:212
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:544
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getValueType() const
Return the ValueType of the referenced return value.
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant, which is required to be operand #1) half of the integer or float value specified as operand #0.
Definition: ISDOpcodes.h:184
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC&#39;s if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:923
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:334
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:618
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
static MVT getVectorVT(MVT VT, unsigned NumElements)
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:343
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:260
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:495
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none...
const SDValue & getBasePtr() const
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:223
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:866
const SDValue & getValue() const
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
Definition: APInt.h:641
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:253
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
const SDValue & getBasePtr() const
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:251
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1308
const SDValue & getBasePtr() const
const SDValue & getChain() const
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
unsigned getAlignment() const
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:770
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select&#39;s if you just have operands and don&#39;t want to check...
Definition: SelectionDAG.h:936
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:486
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT TVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:360
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Same for subtraction.
Definition: ISDOpcodes.h:254
void reserve(size_type N)
Definition: SmallVector.h:378
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:212
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align=0, int Offs=0, bool isT=false, unsigned char TargetFlags=0)
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:405
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:39
const SDValue & getValue() const
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
SDValue getExternalSymbol(const char *Sym, EVT VT)
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:210
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations...
Definition: ISDOpcodes.h:434
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
static uint32_t getAlignment(const MCSectionCOFF &Sec)
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:451
Shift and rotation operations.
Definition: ISDOpcodes.h:380
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:205
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:639
CallLoweringInfo & setChain(SDValue InChain)
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:191
op_iterator op_end() const
unsigned getScalarValueSizeInBits() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
FLT_ROUNDS_ - Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest 2 Round to ...
Definition: ISDOpcodes.h:521
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:304
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt) For double-word atomic operations: ValLo, ValHi, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amtLo, amtHi) ValLo, ValHi, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amtLo, amtHi) These correspond to the atomicrmw instruction.
Definition: ISDOpcodes.h:778
const SDValue & getValue() const
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:449
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:390
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG...
Definition: ISDOpcodes.h:73
This is an SDNode representing atomic operations.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
This represents a list of ValueType&#39;s that has been intern&#39;d by a SelectionDAG.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
This class is used to represent an MSTORE node.
unsigned getSizeInBits() const
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:233
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:298
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:292
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:387
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:456
SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
const SDValue & getScale() const
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:399
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:201
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:221
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:855
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTruncating=false, bool IsCompressing=false)
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:918
op_iterator op_begin() const
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:715
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:473
UNDEF - An undefined node.
Definition: ISDOpcodes.h:178
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:499
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:303
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
Definition: APInt.h:629
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits...
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
Definition: MathExtras.h:602
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const SDValue & getBasePtr() const
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:421
Machine Value Type.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
Simple binary floating point operators.
Definition: ISDOpcodes.h:260
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:273
This is an important base class in LLVM.
Definition: Constant.h:42
iterator_range< value_op_iterator > op_values() const
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
Definition: SmallVector.h:116
const SDValue & getOperand(unsigned Num) const
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:893
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:308
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:247
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:232
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:581
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:57
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:687
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType, bool IsExpanding=false)
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:386
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo...
Definition: ISDOpcodes.h:764
std::vector< ArgListEntry > ArgListTy
const SDValue & getIndex() const
Extended Value Type.
Definition: ValueTypes.h:34
const SDValue & getBasePtr() const
This structure contains all information that is necessary for lowering calls.
const SDValue & getSrc0() const
This class contains a discriminated union of information about pointers in memory operands...
unsigned getNumOperands() const
Return the number of values used by this operation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, unsigned Alignment, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SyncScope::ID SSID)
Gets a node for an atomic cmpxchg op.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
const SDValue & getMask() const
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:50
void dump() const
Dump this node, for debugging.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:315
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:265
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
Definition: ISDOpcodes.h:429
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:862
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:719
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:612
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:642
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:383
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:556
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:598
Represents one node in the SelectionDAG.
static unsigned int semanticsPrecision(const fltSemantics &)
Definition: APFloat.cpp:154
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
MachinePointerInfo getWithOffset(int64_t O) const
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:531
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:924
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT...
Definition: ValueTypes.h:73
EVT getMemoryVT() const
Return the type of the in-memory value.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
Class for arbitrary precision integers.
Definition: APInt.h:69
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:390
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:241
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:446
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:449
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition: APInt.h:523
Flags
Flags values. These may be or&#39;d together.
#define Success
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool ugt(const APInt &RHS) const
Unsigned greather than comparison.
Definition: APInt.h:1234
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:546
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Definition: SmallVector.h:120
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:151
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
Definition: APInt.h:1300
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:363
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, bool isSigned, const SDLoc &dl, bool doesNotReturn=false, bool isReturnValueUsed=true) const
Returns a pair of (return value, chain).
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:464
void computeKnownBits(SDValue Op, KnownBits &Known, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:582
#define N
Flags getFlags() const
Return the raw flags of the source value,.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:323
Same for multiplication.
Definition: ISDOpcodes.h:257
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
SDValue getValue(unsigned R) const
This class is used to represent an MSCATTER node.
unsigned getNumRegisters(LLVMContext &Context, EVT VT) const
Return the number of registers that this ValueType will eventually require.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getCondCode(ISD::CondCode Cond)
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, and op #2 is a carry value...
Definition: ISDOpcodes.h:422
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, const Value *PtrVal, unsigned Alignment, AtomicOrdering Ordering, SyncScope::ID SSID)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands...
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input...
const MachinePointerInfo & getPointerInfo() const
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:321
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This class is used to represent an MLOAD node.
ArrayRef< int > getMask() const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:235
This class is used to represent an MGATHER node.
SDValue getValueType(EVT)
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:81
#define DEBUG(X)
Definition: Debug.h:118
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:413
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Definition: ValueTypes.h:223
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:198
Conversion operators.
Definition: ISDOpcodes.h:443
BooleanContent
Enum that describes how the target represents true/false values.
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:757
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT SrcTy)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value...
bool isBigEndian() const
Definition: DataLayout.h:222
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:452
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Op, int64_t Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object...
Definition: SelectionDAG.h:809
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:753
LLVMContext * getContext() const
Definition: SelectionDAG.h:396
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:351
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:242
const SDValue & getMask() const
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
This class is used to represent ISD::LOAD nodes.