LLVM  3.7.0
LegalizeIntegerTypes.cpp
Go to the documentation of this file.
1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements integer type expansion and promotion for LegalizeTypes.
11 // Promotion is the act of changing a computation in an illegal type into a
12 // computation in a larger type. For example, implementing i8 arithmetic in an
13 // i32 register (often needed on powerpc).
14 // Expansion is the act of changing a computation in an illegal type into a
15 // computation in two identical registers of a smaller type. For example,
16 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
17 // targets).
18 //
19 //===----------------------------------------------------------------------===//
20 
21 #include "LegalizeTypes.h"
22 #include "llvm/IR/DerivedTypes.h"
25 using namespace llvm;
26 
27 #define DEBUG_TYPE "legalize-types"
28 
29 //===----------------------------------------------------------------------===//
30 // Integer Result Promotion
31 //===----------------------------------------------------------------------===//
32 
33 /// PromoteIntegerResult - This method is called when a result of a node is
34 /// found to be in need of promotion to a larger type. At this point, the node
35 /// may also have invalid operands or may have other results that need
36 /// expansion, we just know that (at least) one result needs promotion.
37 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
38  DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); dbgs() << "\n");
39  SDValue Res = SDValue();
40 
41  // See if the target wants to custom expand this node.
42  if (CustomLowerNode(N, N->getValueType(ResNo), true))
43  return;
44 
45  switch (N->getOpcode()) {
46  default:
47 #ifndef NDEBUG
48  dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
49  N->dump(&DAG); dbgs() << "\n";
50 #endif
51  llvm_unreachable("Do not know how to promote this operator!");
52  case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
53  case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
54  case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
55  case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
56  case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
57  case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
58  case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
60  Res = PromoteIntRes_CONVERT_RNDSAT(N); break;
62  case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
63  case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break;
65  case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
67  Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
68  case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N));break;
69  case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));break;
70  case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
71  case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
72  case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
73  case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
74  case ISD::SMIN:
75  case ISD::SMAX:
76  case ISD::UMIN:
77  case ISD::UMAX: Res = PromoteIntRes_SimpleIntBinOp(N); break;
78  case ISD::SHL: Res = PromoteIntRes_SHL(N); break;
80  Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
81  case ISD::SRA: Res = PromoteIntRes_SRA(N); break;
82  case ISD::SRL: Res = PromoteIntRes_SRL(N); break;
83  case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
84  case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
85  case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
86 
88  Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
90  Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
92  Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
93  case ISD::BUILD_VECTOR:
94  Res = PromoteIntRes_BUILD_VECTOR(N); break;
96  Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
98  Res = PromoteIntRes_CONCAT_VECTORS(N); break;
99 
100  case ISD::SIGN_EXTEND:
101  case ISD::ZERO_EXTEND:
102  case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
103 
104  case ISD::FP_TO_SINT:
105  case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
106 
107  case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
108 
109  case ISD::AND:
110  case ISD::OR:
111  case ISD::XOR:
112  case ISD::ADD:
113  case ISD::SUB:
114  case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
115 
116  case ISD::SDIV:
117  case ISD::SREM: Res = PromoteIntRes_SDIV(N); break;
118 
119  case ISD::UDIV:
120  case ISD::UREM: Res = PromoteIntRes_UDIV(N); break;
121 
122  case ISD::SADDO:
123  case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
124  case ISD::UADDO:
125  case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
126  case ISD::SMULO:
127  case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
128 
129  case ISD::ATOMIC_LOAD:
130  Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
131 
135  case ISD::ATOMIC_LOAD_OR:
142  case ISD::ATOMIC_SWAP:
143  Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
144 
147  Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
148  break;
149  }
150 
151  // If the result is null then the sub-method took care of registering it.
152  if (Res.getNode())
153  SetPromotedInteger(SDValue(N, ResNo), Res);
154 }
155 
156 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
157  unsigned ResNo) {
158  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
159  return GetPromotedInteger(Op);
160 }
161 
162 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
163  // Sign-extend the new bits, and continue the assertion.
164  SDValue Op = SExtPromotedInteger(N->getOperand(0));
165  return DAG.getNode(ISD::AssertSext, SDLoc(N),
166  Op.getValueType(), Op, N->getOperand(1));
167 }
168 
169 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
170  // Zero the new bits, and continue the assertion.
171  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
172  return DAG.getNode(ISD::AssertZext, SDLoc(N),
173  Op.getValueType(), Op, N->getOperand(1));
174 }
175 
176 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
177  EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
178  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
179  N->getMemoryVT(), ResVT,
180  N->getChain(), N->getBasePtr(),
181  N->getMemOperand(), N->getOrdering(),
182  N->getSynchScope());
183  // Legalized the chain result - switch anything that used the old chain to
184  // use the new one.
185  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
186  return Res;
187 }
188 
189 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
190  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
191  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
192  N->getMemoryVT(),
193  N->getChain(), N->getBasePtr(),
194  Op2, N->getMemOperand(), N->getOrdering(),
195  N->getSynchScope());
196  // Legalized the chain result - switch anything that used the old chain to
197  // use the new one.
198  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
199  return Res;
200 }
201 
202 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
203  unsigned ResNo) {
204  if (ResNo == 1) {
206  EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
207  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
208 
209  // Only use the result of getSetCCResultType if it is legal,
210  // otherwise just use the promoted result type (NVT).
211  if (!TLI.isTypeLegal(SVT))
212  SVT = NVT;
213 
214  SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
215  SDValue Res = DAG.getAtomicCmpSwap(
217  N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
219  N->getSynchScope());
220  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
221  ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
222  return Res.getValue(1);
223  }
224 
225  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
226  SDValue Op3 = GetPromotedInteger(N->getOperand(3));
227  SDVTList VTs =
228  DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
229  SDValue Res = DAG.getAtomicCmpSwap(
230  N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
231  N->getBasePtr(), Op2, Op3, N->getMemOperand(), N->getSuccessOrdering(),
232  N->getFailureOrdering(), N->getSynchScope());
233  // Update the use to N with the newly created Res.
234  for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
235  ReplaceValueWith(SDValue(N, i), Res.getValue(i));
236  return Res;
237 }
238 
239 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
240  SDValue InOp = N->getOperand(0);
241  EVT InVT = InOp.getValueType();
242  EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
243  EVT OutVT = N->getValueType(0);
244  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
245  SDLoc dl(N);
246 
247  switch (getTypeAction(InVT)) {
249  break;
251  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
252  // The input promotes to the same size. Convert the promoted value.
253  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
254  break;
256  // Promote the integer operand by hand.
257  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
259  // Convert the promoted float by hand.
260  if (NOutVT.bitsEq(NInVT)) {
261  SDValue PromotedOp = GetPromotedFloat(InOp);
262  SDValue Trunc = DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, PromotedOp);
263  return DAG.getNode(ISD::AssertZext, dl, NOutVT, Trunc,
264  DAG.getValueType(OutVT));
265  }
266  break;
267  }
270  break;
272  // Convert the element to an integer and promote it by hand.
273  if (!NOutVT.isVector())
274  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
275  BitConvertToInteger(GetScalarizedVector(InOp)));
276  break;
278  // For example, i32 = BITCAST v2i16 on alpha. Convert the split
279  // pieces of the input into integers and reassemble in the final type.
280  SDValue Lo, Hi;
281  GetSplitVector(N->getOperand(0), Lo, Hi);
282  Lo = BitConvertToInteger(Lo);
283  Hi = BitConvertToInteger(Hi);
284 
285  if (DAG.getDataLayout().isBigEndian())
286  std::swap(Lo, Hi);
287 
288  InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
290  NOutVT.getSizeInBits()),
291  JoinIntegers(Lo, Hi));
292  return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
293  }
295  // The input is widened to the same size. Convert to the widened value.
296  // Make sure that the outgoing value is not a vector, because this would
297  // make us bitcast between two vectors which are legalized in different ways.
298  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector())
299  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
300  }
301 
302  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
303  CreateStackStoreLoad(InOp, OutVT));
304 }
305 
306 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
307  SDValue Op = GetPromotedInteger(N->getOperand(0));
308  EVT OVT = N->getValueType(0);
309  EVT NVT = Op.getValueType();
310  SDLoc dl(N);
311 
312  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
313  return DAG.getNode(
314  ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
315  DAG.getConstant(DiffBits, dl,
316  TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
317 }
318 
319 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
320  // The pair element type may be legal, or may not promote to the same type as
321  // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
322  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
323  TLI.getTypeToTransformTo(*DAG.getContext(),
324  N->getValueType(0)), JoinIntegers(N->getOperand(0),
325  N->getOperand(1)));
326 }
327 
328 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
329  EVT VT = N->getValueType(0);
330  // FIXME there is no actual debug info here
331  SDLoc dl(N);
332  // Zero extend things like i1, sign extend everything else. It shouldn't
333  // matter in theory which one we pick, but this tends to give better code?
334  unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
335  SDValue Result = DAG.getNode(Opc, dl,
336  TLI.getTypeToTransformTo(*DAG.getContext(), VT),
337  SDValue(N, 0));
338  assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
339  return Result;
340 }
341 
342 SDValue DAGTypeLegalizer::PromoteIntRes_CONVERT_RNDSAT(SDNode *N) {
343  ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode();
344  assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU ||
345  CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU ||
346  CvtCode == ISD::CVT_SF || CvtCode == ISD::CVT_UF) &&
347  "can only promote integers");
348  EVT OutVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
349  return DAG.getConvertRndSat(OutVT, SDLoc(N), N->getOperand(0),
350  N->getOperand(1), N->getOperand(2),
351  N->getOperand(3), N->getOperand(4), CvtCode);
352 }
353 
354 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
355  // Zero extend to the promoted type and do the count there.
356  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
357  SDLoc dl(N);
358  EVT OVT = N->getValueType(0);
359  EVT NVT = Op.getValueType();
360  Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
361  // Subtract off the extra leading bits in the bigger type.
362  return DAG.getNode(
363  ISD::SUB, dl, NVT, Op,
364  DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
365  NVT));
366 }
367 
368 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
369  // Zero extend to the promoted type and do the count there.
370  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
371  return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op);
372 }
373 
374 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
375  SDValue Op = GetPromotedInteger(N->getOperand(0));
376  EVT OVT = N->getValueType(0);
377  EVT NVT = Op.getValueType();
378  SDLoc dl(N);
379  if (N->getOpcode() == ISD::CTTZ) {
380  // The count is the same in the promoted type except if the original
381  // value was zero. This can be handled by setting the bit just off
382  // the top of the original type.
383  auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
384  OVT.getScalarSizeInBits());
385  Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
386  }
387  return DAG.getNode(N->getOpcode(), dl, NVT, Op);
388 }
389 
390 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
391  SDLoc dl(N);
392  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
393  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0),
394  N->getOperand(1));
395 }
396 
397 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
398  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
399  unsigned NewOpc = N->getOpcode();
400  SDLoc dl(N);
401 
402  // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
403  // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
404  // and SINT conversions are Custom, there is no way to tell which is
405  // preferable. We choose SINT because that's the right thing on PPC.)
406  if (N->getOpcode() == ISD::FP_TO_UINT &&
407  !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
409  NewOpc = ISD::FP_TO_SINT;
410 
411  SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
412 
413  // Assert that the converted value fits in the original type. If it doesn't
414  // (eg: because the value being converted is too big), then the result of the
415  // original operation was undefined anyway, so the assert is still correct.
416  return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ?
417  ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
419 }
420 
421 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
422  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
423  SDLoc dl(N);
424 
425  SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
426 
427  return DAG.getNode(ISD::AssertZext, dl,
428  NVT, Res, DAG.getValueType(N->getValueType(0)));
429 }
430 
431 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
432  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
433  SDLoc dl(N);
434 
435  if (getTypeAction(N->getOperand(0).getValueType())
437  SDValue Res = GetPromotedInteger(N->getOperand(0));
438  assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
439 
440  // If the result and operand types are the same after promotion, simplify
441  // to an in-register extension.
442  if (NVT == Res.getValueType()) {
443  // The high bits are not guaranteed to be anything. Insert an extend.
444  if (N->getOpcode() == ISD::SIGN_EXTEND)
445  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
446  DAG.getValueType(N->getOperand(0).getValueType()));
447  if (N->getOpcode() == ISD::ZERO_EXTEND)
448  return DAG.getZeroExtendInReg(Res, dl,
450  assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
451  return Res;
452  }
453  }
454 
455  // Otherwise, just extend the original operand all the way to the larger type.
456  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
457 }
458 
459 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
460  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
461  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
462  ISD::LoadExtType ExtType =
464  SDLoc dl(N);
465  SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
466  N->getMemoryVT(), N->getMemOperand());
467 
468  // Legalized the chain result - switch anything that used the old chain to
469  // use the new one.
470  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
471  return Res;
472 }
473 
474 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
475  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
476  SDValue ExtSrc0 = GetPromotedInteger(N->getSrc0());
477 
478  SDValue Mask = N->getMask();
479  EVT NewMaskVT = getSetCCResultType(NVT);
480  if (NewMaskVT != N->getMask().getValueType())
481  Mask = PromoteTargetBoolean(Mask, NewMaskVT);
482  SDLoc dl(N);
483 
484  SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
485  Mask, ExtSrc0, N->getMemoryVT(),
487  // Legalized the chain result - switch anything that used the old chain to
488  // use the new one.
489  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
490  return Res;
491 }
492 /// Promote the overflow flag of an overflowing arithmetic node.
493 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
494  // Simply change the return type of the boolean result.
495  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
496  EVT ValueVTs[] = { N->getValueType(0), NVT };
497  SDValue Ops[] = { N->getOperand(0), N->getOperand(1) };
498  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
499  DAG.getVTList(ValueVTs), Ops);
500 
501  // Modified the sum result - switch anything that used the old sum to use
502  // the new one.
503  ReplaceValueWith(SDValue(N, 0), Res);
504 
505  return SDValue(Res.getNode(), 1);
506 }
507 
508 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
509  if (ResNo == 1)
510  return PromoteIntRes_Overflow(N);
511 
512  // The operation overflowed iff the result in the larger type is not the
513  // sign extension of its truncation to the original type.
514  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
515  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
516  EVT OVT = N->getOperand(0).getValueType();
517  EVT NVT = LHS.getValueType();
518  SDLoc dl(N);
519 
520  // Do the arithmetic in the larger type.
521  unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
522  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
523 
524  // Calculate the overflow flag: sign extend the arithmetic result from
525  // the original type.
526  SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
527  DAG.getValueType(OVT));
528  // Overflowed if and only if this is not equal to Res.
529  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
530 
531  // Use the calculated overflow everywhere.
532  ReplaceValueWith(SDValue(N, 1), Ofl);
533 
534  return Res;
535 }
536 
537 SDValue DAGTypeLegalizer::PromoteIntRes_SDIV(SDNode *N) {
538  // Sign extend the input.
539  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
540  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
541  return DAG.getNode(N->getOpcode(), SDLoc(N),
542  LHS.getValueType(), LHS, RHS);
543 }
544 
545 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
546  SDValue LHS = GetPromotedInteger(N->getOperand(1));
547  SDValue RHS = GetPromotedInteger(N->getOperand(2));
548  return DAG.getSelect(SDLoc(N),
549  LHS.getValueType(), N->getOperand(0), LHS, RHS);
550 }
551 
552 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
553  SDValue Mask = N->getOperand(0);
554  EVT OpTy = N->getOperand(1).getValueType();
555 
556  // Promote all the way up to the canonical SetCC type.
557  Mask = PromoteTargetBoolean(Mask, OpTy);
558  SDValue LHS = GetPromotedInteger(N->getOperand(1));
559  SDValue RHS = GetPromotedInteger(N->getOperand(2));
560  return DAG.getNode(ISD::VSELECT, SDLoc(N),
561  LHS.getValueType(), Mask, LHS, RHS);
562 }
563 
564 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
565  SDValue LHS = GetPromotedInteger(N->getOperand(2));
566  SDValue RHS = GetPromotedInteger(N->getOperand(3));
567  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
568  LHS.getValueType(), N->getOperand(0),
569  N->getOperand(1), LHS, RHS, N->getOperand(4));
570 }
571 
572 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
573  EVT SVT = getSetCCResultType(N->getOperand(0).getValueType());
574 
575  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
576 
577  // Only use the result of getSetCCResultType if it is legal,
578  // otherwise just use the promoted result type (NVT).
579  if (!TLI.isTypeLegal(SVT))
580  SVT = NVT;
581 
582  SDLoc dl(N);
583  assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() &&
584  "Vector compare must return a vector result!");
585 
586  SDValue LHS = N->getOperand(0);
587  SDValue RHS = N->getOperand(1);
588  if (LHS.getValueType() != RHS.getValueType()) {
589  if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger &&
590  !LHS.getValueType().isVector())
591  LHS = GetPromotedInteger(LHS);
592  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger &&
593  !RHS.getValueType().isVector())
594  RHS = GetPromotedInteger(RHS);
595  }
596 
597  // Get the SETCC result using the canonical SETCC type.
598  SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, LHS, RHS,
599  N->getOperand(2));
600 
601  assert(NVT.bitsLE(SVT) && "Integer type overpromoted?");
602  // Convert to the expected type.
603  return DAG.getNode(ISD::TRUNCATE, dl, NVT, SetCC);
604 }
605 
606 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
607  SDValue LHS = N->getOperand(0);
608  SDValue RHS = N->getOperand(1);
609  if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
610  LHS = GetPromotedInteger(LHS);
611  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
612  RHS = ZExtPromotedInteger(RHS);
613  return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
614 }
615 
616 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
617  SDValue Op = GetPromotedInteger(N->getOperand(0));
618  return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
619  Op.getValueType(), Op, N->getOperand(1));
620 }
621 
622 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
623  // The input may have strange things in the top bits of the registers, but
624  // these operations don't care. They may have weird bits going out, but
625  // that too is okay if they are integer operations.
626  SDValue LHS = GetPromotedInteger(N->getOperand(0));
627  SDValue RHS = GetPromotedInteger(N->getOperand(1));
628  return DAG.getNode(N->getOpcode(), SDLoc(N),
629  LHS.getValueType(), LHS, RHS);
630 }
631 
632 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
633  SDValue LHS = N->getOperand(0);
634  SDValue RHS = N->getOperand(1);
635  // The input value must be properly sign extended.
636  if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
637  LHS = SExtPromotedInteger(LHS);
638  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
639  RHS = ZExtPromotedInteger(RHS);
640  return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
641 }
642 
643 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
644  SDValue LHS = N->getOperand(0);
645  SDValue RHS = N->getOperand(1);
646  // The input value must be properly zero extended.
647  if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
648  LHS = ZExtPromotedInteger(LHS);
649  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
650  RHS = ZExtPromotedInteger(RHS);
651  return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
652 }
653 
654 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
655  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
656  SDValue Res;
657  SDValue InOp = N->getOperand(0);
658  SDLoc dl(N);
659 
660  switch (getTypeAction(InOp.getValueType())) {
661  default: llvm_unreachable("Unknown type action!");
664  Res = InOp;
665  break;
667  Res = GetPromotedInteger(InOp);
668  break;
670  EVT InVT = InOp.getValueType();
671  assert(InVT.isVector() && "Cannot split scalar types");
672  unsigned NumElts = InVT.getVectorNumElements();
673  assert(NumElts == NVT.getVectorNumElements() &&
674  "Dst and Src must have the same number of elements");
675  assert(isPowerOf2_32(NumElts) &&
676  "Promoted vector type must be a power of two");
677 
678  SDValue EOp1, EOp2;
679  GetSplitVector(InOp, EOp1, EOp2);
680 
681  EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
682  NumElts/2);
683  EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
684  EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
685 
686  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
687  }
688 
689  // Truncate to NVT instead of VT
690  return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
691 }
692 
693 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
694  if (ResNo == 1)
695  return PromoteIntRes_Overflow(N);
696 
697  // The operation overflowed iff the result in the larger type is not the
698  // zero extension of its truncation to the original type.
699  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
700  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
701  EVT OVT = N->getOperand(0).getValueType();
702  EVT NVT = LHS.getValueType();
703  SDLoc dl(N);
704 
705  // Do the arithmetic in the larger type.
706  unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
707  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
708 
709  // Calculate the overflow flag: zero extend the arithmetic result from
710  // the original type.
711  SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
712  // Overflowed if and only if this is not equal to Res.
713  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
714 
715  // Use the calculated overflow everywhere.
716  ReplaceValueWith(SDValue(N, 1), Ofl);
717 
718  return Res;
719 }
720 
721 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
722  // Promote the overflow bit trivially.
723  if (ResNo == 1)
724  return PromoteIntRes_Overflow(N);
725 
726  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
727  SDLoc DL(N);
728  EVT SmallVT = LHS.getValueType();
729 
730  // To determine if the result overflowed in a larger type, we extend the
731  // input to the larger type, do the multiply (checking if it overflows),
732  // then also check the high bits of the result to see if overflow happened
733  // there.
734  if (N->getOpcode() == ISD::SMULO) {
735  LHS = SExtPromotedInteger(LHS);
736  RHS = SExtPromotedInteger(RHS);
737  } else {
738  LHS = ZExtPromotedInteger(LHS);
739  RHS = ZExtPromotedInteger(RHS);
740  }
741  SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
742  SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
743 
744  // Overflow occurred if it occurred in the larger type, or if the high part
745  // of the result does not zero/sign-extend the low part. Check this second
746  // possibility first.
747  SDValue Overflow;
748  if (N->getOpcode() == ISD::UMULO) {
749  // Unsigned overflow occurred if the high part is non-zero.
750  SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
751  DAG.getIntPtrConstant(SmallVT.getSizeInBits(),
752  DL));
753  Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
754  DAG.getConstant(0, DL, Hi.getValueType()),
755  ISD::SETNE);
756  } else {
757  // Signed overflow occurred if the high part does not sign extend the low.
759  Mul, DAG.getValueType(SmallVT));
760  Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
761  }
762 
763  // The only other way for overflow to occur is if the multiplication in the
764  // larger type itself overflowed.
765  Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
766  SDValue(Mul.getNode(), 1));
767 
768  // Use the calculated overflow everywhere.
769  ReplaceValueWith(SDValue(N, 1), Overflow);
770  return Mul;
771 }
772 
773 SDValue DAGTypeLegalizer::PromoteIntRes_UDIV(SDNode *N) {
774  // Zero extend the input.
775  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
776  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
777  return DAG.getNode(N->getOpcode(), SDLoc(N),
778  LHS.getValueType(), LHS, RHS);
779 }
780 
781 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
782  return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
783  N->getValueType(0)));
784 }
785 
786 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
787  SDValue Chain = N->getOperand(0); // Get the chain.
788  SDValue Ptr = N->getOperand(1); // Get the pointer.
789  EVT VT = N->getValueType(0);
790  SDLoc dl(N);
791 
792  MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
793  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
794  // The argument is passed as NumRegs registers of type RegVT.
795 
796  SmallVector<SDValue, 8> Parts(NumRegs);
797  for (unsigned i = 0; i < NumRegs; ++i) {
798  Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
799  N->getConstantOperandVal(3));
800  Chain = Parts[i].getValue(1);
801  }
802 
803  // Handle endianness of the load.
804  if (DAG.getDataLayout().isBigEndian())
805  std::reverse(Parts.begin(), Parts.end());
806 
807  // Assemble the parts in the promoted type.
808  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
809  SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
810  for (unsigned i = 1; i < NumRegs; ++i) {
811  SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
812  // Shift it to the right position and "or" it in.
813  Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
814  DAG.getConstant(i * RegVT.getSizeInBits(), dl,
815  TLI.getPointerTy(DAG.getDataLayout())));
816  Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
817  }
818 
819  // Modified the chain result - switch anything that used the old chain to
820  // use the new one.
821  ReplaceValueWith(SDValue(N, 1), Chain);
822 
823  return Res;
824 }
825 
826 //===----------------------------------------------------------------------===//
827 // Integer Operand Promotion
828 //===----------------------------------------------------------------------===//
829 
830 /// PromoteIntegerOperand - This method is called when the specified operand of
831 /// the specified node is found to need promotion. At this point, all of the
832 /// result types of the node are known to be legal, but other operands of the
833 /// node may need promotion or expansion as well as the specified one.
834 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
835  DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); dbgs() << "\n");
836  SDValue Res = SDValue();
837 
838  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
839  return false;
840 
841  switch (N->getOpcode()) {
842  default:
843  #ifndef NDEBUG
844  dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
845  N->dump(&DAG); dbgs() << "\n";
846  #endif
847  llvm_unreachable("Do not know how to promote this operator's operand!");
848 
849  case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
850  case ISD::ATOMIC_STORE:
851  Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
852  break;
853  case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
854  case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
855  case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
856  case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
857  case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
858  case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
859  case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
860  case ISD::CONVERT_RNDSAT:
861  Res = PromoteIntOp_CONVERT_RNDSAT(N); break;
863  Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
865  Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
866  case ISD::VSELECT:
867  case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
868  case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
869  case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
870  case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
871  case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
872  case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
873  OpNo); break;
874  case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
875  OpNo); break;
876  case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
877  OpNo); break;
878  case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
879  case ISD::FP16_TO_FP:
880  case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
881  case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
882  case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
883 
884  case ISD::SHL:
885  case ISD::SRA:
886  case ISD::SRL:
887  case ISD::ROTL:
888  case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
889  }
890 
891  // If the result is null, the sub-method took care of registering results etc.
892  if (!Res.getNode()) return false;
893 
894  // If the result is N, the sub-method updated N in place. Tell the legalizer
895  // core about this.
896  if (Res.getNode() == N)
897  return true;
898 
899  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
900  "Invalid operand expansion");
901 
902  ReplaceValueWith(SDValue(N, 0), Res);
903  return false;
904 }
905 
906 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
907 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
908 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
909  ISD::CondCode CCCode) {
910  // We have to insert explicit sign or zero extends. Note that we could
911  // insert sign extends for ALL conditions, but zero extend is cheaper on
912  // many machines (an AND instead of two shifts), so prefer it.
913  switch (CCCode) {
914  default: llvm_unreachable("Unknown integer comparison!");
915  case ISD::SETEQ:
916  case ISD::SETNE: {
917  SDValue OpL = GetPromotedInteger(NewLHS);
918  SDValue OpR = GetPromotedInteger(NewRHS);
919 
920  // We would prefer to promote the comparison operand with sign extension,
921  // if we find the operand is actually to truncate an AssertSext. With this
922  // optimization, we can avoid inserting real truncate instruction, which
923  // is redudant eventually.
924  if (OpL->getOpcode() == ISD::AssertSext &&
925  cast<VTSDNode>(OpL->getOperand(1))->getVT() == NewLHS.getValueType() &&
926  OpR->getOpcode() == ISD::AssertSext &&
927  cast<VTSDNode>(OpR->getOperand(1))->getVT() == NewRHS.getValueType()) {
928  NewLHS = OpL;
929  NewRHS = OpR;
930  } else {
931  NewLHS = ZExtPromotedInteger(NewLHS);
932  NewRHS = ZExtPromotedInteger(NewRHS);
933  }
934  break;
935  }
936  case ISD::SETUGE:
937  case ISD::SETUGT:
938  case ISD::SETULE:
939  case ISD::SETULT:
940  // ALL of these operations will work if we either sign or zero extend
941  // the operands (including the unsigned comparisons!). Zero extend is
942  // usually a simpler/cheaper operation, so prefer it.
943  NewLHS = ZExtPromotedInteger(NewLHS);
944  NewRHS = ZExtPromotedInteger(NewRHS);
945  break;
946  case ISD::SETGE:
947  case ISD::SETGT:
948  case ISD::SETLT:
949  case ISD::SETLE:
950  NewLHS = SExtPromotedInteger(NewLHS);
951  NewRHS = SExtPromotedInteger(NewRHS);
952  break;
953  }
954 }
955 
956 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
957  SDValue Op = GetPromotedInteger(N->getOperand(0));
958  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
959 }
960 
961 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
962  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
963  return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
964  N->getChain(), N->getBasePtr(), Op2, N->getMemOperand(),
965  N->getOrdering(), N->getSynchScope());
966 }
967 
968 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
969  // This should only occur in unusual situations like bitcasting to an
970  // x86_fp80, so just turn it into a store+load
971  return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
972 }
973 
974 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
975  assert(OpNo == 2 && "Don't know how to promote this operand!");
976 
977  SDValue LHS = N->getOperand(2);
978  SDValue RHS = N->getOperand(3);
979  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
980 
981  // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
982  // legal types.
983  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
984  N->getOperand(1), LHS, RHS, N->getOperand(4)),
985  0);
986 }
987 
988 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
989  assert(OpNo == 1 && "only know how to promote condition");
990 
991  // Promote all the way up to the canonical SetCC type.
992  SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
993 
994  // The chain (Op#0) and basic block destination (Op#2) are always legal types.
995  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
996  N->getOperand(2)), 0);
997 }
998 
999 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1000  // Since the result type is legal, the operands must promote to it.
1001  EVT OVT = N->getOperand(0).getValueType();
1002  SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1003  SDValue Hi = GetPromotedInteger(N->getOperand(1));
1004  assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1005  SDLoc dl(N);
1006 
1007  Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1008  DAG.getConstant(OVT.getSizeInBits(), dl,
1009  TLI.getPointerTy(DAG.getDataLayout())));
1010  return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1011 }
1012 
1013 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1014  // The vector type is legal but the element type is not. This implies
1015  // that the vector is a power-of-two in length and that the element
1016  // type does not have a strange size (eg: it is not i1).
1017  EVT VecVT = N->getValueType(0);
1018  unsigned NumElts = VecVT.getVectorNumElements();
1019  assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1020  "Legal vector of one illegal element?");
1021 
1022  // Promote the inserted value. The type does not need to match the
1023  // vector element type. Check that any extra bits introduced will be
1024  // truncated away.
1025  assert(N->getOperand(0).getValueType().getSizeInBits() >=
1027  "Type of inserted value narrower than vector element type!");
1028 
1029  SmallVector<SDValue, 16> NewOps;
1030  for (unsigned i = 0; i < NumElts; ++i)
1031  NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1032 
1033  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1034 }
1035 
1036 SDValue DAGTypeLegalizer::PromoteIntOp_CONVERT_RNDSAT(SDNode *N) {
1037  ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode();
1038  assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU ||
1039  CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU ||
1040  CvtCode == ISD::CVT_FS || CvtCode == ISD::CVT_FU) &&
1041  "can only promote integer arguments");
1042  SDValue InOp = GetPromotedInteger(N->getOperand(0));
1043  return DAG.getConvertRndSat(N->getValueType(0), SDLoc(N), InOp,
1044  N->getOperand(1), N->getOperand(2),
1045  N->getOperand(3), N->getOperand(4), CvtCode);
1046 }
1047 
1048 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1049  unsigned OpNo) {
1050  if (OpNo == 1) {
1051  // Promote the inserted value. This is valid because the type does not
1052  // have to match the vector element type.
1053 
1054  // Check that any extra bits introduced will be truncated away.
1055  assert(N->getOperand(1).getValueType().getSizeInBits() >=
1057  "Type of inserted value narrower than vector element type!");
1058  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1059  GetPromotedInteger(N->getOperand(1)),
1060  N->getOperand(2)),
1061  0);
1062  }
1063 
1064  assert(OpNo == 2 && "Different operand and result vector types?");
1065 
1066  // Promote the index.
1067  SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1068  TLI.getVectorIdxTy(DAG.getDataLayout()));
1069  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1070  N->getOperand(1), Idx), 0);
1071 }
1072 
1073 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1074  // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1075  // the operand in place.
1076  return SDValue(DAG.UpdateNodeOperands(N,
1077  GetPromotedInteger(N->getOperand(0))), 0);
1078 }
1079 
1080 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1081  assert(OpNo == 0 && "Only know how to promote the condition!");
1082  SDValue Cond = N->getOperand(0);
1083  EVT OpTy = N->getOperand(1).getValueType();
1084 
1085  // Promote all the way up to the canonical SetCC type.
1086  EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1087  Cond = PromoteTargetBoolean(Cond, OpVT);
1088 
1089  return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1090  N->getOperand(2)), 0);
1091 }
1092 
1093 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1094  assert(OpNo == 0 && "Don't know how to promote this operand!");
1095 
1096  SDValue LHS = N->getOperand(0);
1097  SDValue RHS = N->getOperand(1);
1098  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1099 
1100  // The CC (#4) and the possible return values (#2 and #3) have legal types.
1101  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1102  N->getOperand(3), N->getOperand(4)), 0);
1103 }
1104 
1105 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1106  assert(OpNo == 0 && "Don't know how to promote this operand!");
1107 
1108  SDValue LHS = N->getOperand(0);
1109  SDValue RHS = N->getOperand(1);
1110  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1111 
1112  // The CC (#2) is always legal.
1113  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1114 }
1115 
1116 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1117  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1118  ZExtPromotedInteger(N->getOperand(1))), 0);
1119 }
1120 
1121 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1122  SDValue Op = GetPromotedInteger(N->getOperand(0));
1123  SDLoc dl(N);
1124  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1125  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1126  Op, DAG.getValueType(N->getOperand(0).getValueType()));
1127 }
1128 
1129 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1130  return SDValue(DAG.UpdateNodeOperands(N,
1131  SExtPromotedInteger(N->getOperand(0))), 0);
1132 }
1133 
1134 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1135  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1136  SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1137  SDLoc dl(N);
1138 
1139  SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1140 
1141  // Truncate the value and store the result.
1142  return DAG.getTruncStore(Ch, dl, Val, Ptr,
1143  N->getMemoryVT(), N->getMemOperand());
1144 }
1145 
1146 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo){
1147 
1148  SDValue DataOp = N->getValue();
1149  EVT DataVT = DataOp.getValueType();
1150  SDValue Mask = N->getMask();
1151  EVT MaskVT = Mask.getValueType();
1152  SDLoc dl(N);
1153 
1154  bool TruncateStore = false;
1155  if (!TLI.isTypeLegal(DataVT)) {
1156  if (getTypeAction(DataVT) == TargetLowering::TypePromoteInteger) {
1157  DataOp = GetPromotedInteger(DataOp);
1158  if (!TLI.isTypeLegal(MaskVT))
1159  Mask = PromoteTargetBoolean(Mask, DataOp.getValueType());
1160  TruncateStore = true;
1161  }
1162  else {
1163  assert(getTypeAction(DataVT) == TargetLowering::TypeWidenVector &&
1164  "Unexpected data legalization in MSTORE");
1165  DataOp = GetWidenedVector(DataOp);
1166 
1167  if (getTypeAction(MaskVT) == TargetLowering::TypeWidenVector)
1168  Mask = GetWidenedVector(Mask);
1169  else {
1170  EVT BoolVT = getSetCCResultType(DataOp.getValueType());
1171 
1172  // We can't use ModifyToType() because we should fill the mask with
1173  // zeroes
1174  unsigned WidenNumElts = BoolVT.getVectorNumElements();
1175  unsigned MaskNumElts = MaskVT.getVectorNumElements();
1176 
1177  unsigned NumConcat = WidenNumElts / MaskNumElts;
1178  SmallVector<SDValue, 16> Ops(NumConcat);
1179  SDValue ZeroVal = DAG.getConstant(0, dl, MaskVT);
1180  Ops[0] = Mask;
1181  for (unsigned i = 1; i != NumConcat; ++i)
1182  Ops[i] = ZeroVal;
1183 
1184  Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, BoolVT, Ops);
1185  }
1186  }
1187  }
1188  else
1189  Mask = PromoteTargetBoolean(N->getMask(), DataOp.getValueType());
1190  return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask,
1191  N->getMemoryVT(), N->getMemOperand(),
1192  TruncateStore);
1193 }
1194 
1195 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N, unsigned OpNo){
1196  assert(OpNo == 2 && "Only know how to promote the mask!");
1197  EVT DataVT = N->getValueType(0);
1198  SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1199  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1200  NewOps[OpNo] = Mask;
1201  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1202 }
1203 
1204 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
1205  SDValue Op = GetPromotedInteger(N->getOperand(0));
1206  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
1207 }
1208 
1209 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
1210  return SDValue(DAG.UpdateNodeOperands(N,
1211  ZExtPromotedInteger(N->getOperand(0))), 0);
1212 }
1213 
1214 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
1215  SDLoc dl(N);
1216  SDValue Op = GetPromotedInteger(N->getOperand(0));
1217  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1218  return DAG.getZeroExtendInReg(Op, dl,
1220 }
1221 
1222 
1223 //===----------------------------------------------------------------------===//
1224 // Integer Result Expansion
1225 //===----------------------------------------------------------------------===//
1226 
1227 /// ExpandIntegerResult - This method is called when the specified result of the
1228 /// specified node is found to need expansion. At this point, the node may also
1229 /// have invalid operands or may have other results that need promotion, we just
1230 /// know that (at least) one result needs expansion.
1231 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
1232  DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); dbgs() << "\n");
1233  SDValue Lo, Hi;
1234  Lo = Hi = SDValue();
1235 
1236  // See if the target wants to custom expand this node.
1237  if (CustomLowerNode(N, N->getValueType(ResNo), true))
1238  return;
1239 
1240  switch (N->getOpcode()) {
1241  default:
1242 #ifndef NDEBUG
1243  dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
1244  N->dump(&DAG); dbgs() << "\n";
1245 #endif
1246  llvm_unreachable("Do not know how to expand the result of this operator!");
1247 
1248  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1249  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
1250  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1251  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1252 
1253  case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1254  case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1255  case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1256  case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1257  case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1258 
1259  case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
1260  case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
1261  case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
1262  case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
1263  case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
1264  case ISD::CTLZ_ZERO_UNDEF:
1265  case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
1266  case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
1267  case ISD::CTTZ_ZERO_UNDEF:
1268  case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
1269  case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
1270  case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
1271  case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
1272  case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
1273  case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
1274  case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
1275  case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
1276  case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
1277  case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
1278  case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
1279  case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
1280  case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
1281  case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
1282 
1283  case ISD::ATOMIC_LOAD_ADD:
1284  case ISD::ATOMIC_LOAD_SUB:
1285  case ISD::ATOMIC_LOAD_AND:
1286  case ISD::ATOMIC_LOAD_OR:
1287  case ISD::ATOMIC_LOAD_XOR:
1288  case ISD::ATOMIC_LOAD_NAND:
1289  case ISD::ATOMIC_LOAD_MIN:
1290  case ISD::ATOMIC_LOAD_MAX:
1291  case ISD::ATOMIC_LOAD_UMIN:
1292  case ISD::ATOMIC_LOAD_UMAX:
1293  case ISD::ATOMIC_SWAP:
1294  case ISD::ATOMIC_CMP_SWAP: {
1295  std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
1296  SplitInteger(Tmp.first, Lo, Hi);
1297  ReplaceValueWith(SDValue(N, 1), Tmp.second);
1298  break;
1299  }
1301  AtomicSDNode *AN = cast<AtomicSDNode>(N);
1302  SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
1303  SDValue Tmp = DAG.getAtomicCmpSwap(
1304  ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
1305  N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
1307  AN->getSynchScope());
1308 
1309  // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
1310  // success simply by comparing the loaded value against the ingoing
1311  // comparison.
1312  SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
1313  N->getOperand(2), ISD::SETEQ);
1314 
1315  SplitInteger(Tmp, Lo, Hi);
1316  ReplaceValueWith(SDValue(N, 1), Success);
1317  ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
1318  break;
1319  }
1320 
1321  case ISD::AND:
1322  case ISD::OR:
1323  case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
1324 
1325  case ISD::ADD:
1326  case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
1327 
1328  case ISD::ADDC:
1329  case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
1330 
1331  case ISD::ADDE:
1332  case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
1333 
1334  case ISD::SHL:
1335  case ISD::SRA:
1336  case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
1337 
1338  case ISD::SADDO:
1339  case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
1340  case ISD::UADDO:
1341  case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
1342  case ISD::UMULO:
1343  case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
1344  }
1345 
1346  // If Lo/Hi is null, the sub-method took care of registering results etc.
1347  if (Lo.getNode())
1348  SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
1349 }
1350 
1351 /// Lower an atomic node to the appropriate builtin call.
1352 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
1353  unsigned Opc = Node->getOpcode();
1354  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
1355  RTLIB::Libcall LC = RTLIB::getATOMIC(Opc, VT);
1356  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
1357 
1358  return ExpandChainLibCall(LC, Node, false);
1359 }
1360 
1361 /// N is a shift by a value that needs to be expanded,
1362 /// and the shift amount is a constant 'Amt'. Expand the operation.
1363 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
1364  SDValue &Lo, SDValue &Hi) {
1365  SDLoc DL(N);
1366  // Expand the incoming operand to be shifted, so that we have its parts
1367  SDValue InL, InH;
1368  GetExpandedInteger(N->getOperand(0), InL, InH);
1369 
1370  // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
1371  // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
1372  if (!Amt) {
1373  Lo = InL;
1374  Hi = InH;
1375  return;
1376  }
1377 
1378  EVT NVT = InL.getValueType();
1379  unsigned VTBits = N->getValueType(0).getSizeInBits();
1380  unsigned NVTBits = NVT.getSizeInBits();
1381  EVT ShTy = N->getOperand(1).getValueType();
1382 
1383  if (N->getOpcode() == ISD::SHL) {
1384  if (Amt.ugt(VTBits)) {
1385  Lo = Hi = DAG.getConstant(0, DL, NVT);
1386  } else if (Amt.ugt(NVTBits)) {
1387  Lo = DAG.getConstant(0, DL, NVT);
1388  Hi = DAG.getNode(ISD::SHL, DL,
1389  NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1390  } else if (Amt == NVTBits) {
1391  Lo = DAG.getConstant(0, DL, NVT);
1392  Hi = InL;
1393  } else if (Amt == 1 &&
1395  TLI.getTypeToExpandTo(*DAG.getContext(), NVT))) {
1396  // Emit this X << 1 as X+X.
1397  SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
1398  SDValue LoOps[2] = { InL, InL };
1399  Lo = DAG.getNode(ISD::ADDC, DL, VTList, LoOps);
1400  SDValue HiOps[3] = { InH, InH, Lo.getValue(1) };
1401  Hi = DAG.getNode(ISD::ADDE, DL, VTList, HiOps);
1402  } else {
1403  Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
1404  Hi = DAG.getNode(ISD::OR, DL, NVT,
1405  DAG.getNode(ISD::SHL, DL, NVT, InH,
1406  DAG.getConstant(Amt, DL, ShTy)),
1407  DAG.getNode(ISD::SRL, DL, NVT, InL,
1408  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1409  }
1410  return;
1411  }
1412 
1413  if (N->getOpcode() == ISD::SRL) {
1414  if (Amt.ugt(VTBits)) {
1415  Lo = Hi = DAG.getConstant(0, DL, NVT);
1416  } else if (Amt.ugt(NVTBits)) {
1417  Lo = DAG.getNode(ISD::SRL, DL,
1418  NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1419  Hi = DAG.getConstant(0, DL, NVT);
1420  } else if (Amt == NVTBits) {
1421  Lo = InH;
1422  Hi = DAG.getConstant(0, DL, NVT);
1423  } else {
1424  Lo = DAG.getNode(ISD::OR, DL, NVT,
1425  DAG.getNode(ISD::SRL, DL, NVT, InL,
1426  DAG.getConstant(Amt, DL, ShTy)),
1427  DAG.getNode(ISD::SHL, DL, NVT, InH,
1428  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1429  Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1430  }
1431  return;
1432  }
1433 
1434  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
1435  if (Amt.ugt(VTBits)) {
1436  Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1437  DAG.getConstant(NVTBits - 1, DL, ShTy));
1438  } else if (Amt.ugt(NVTBits)) {
1439  Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1440  DAG.getConstant(Amt - NVTBits, DL, ShTy));
1441  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1442  DAG.getConstant(NVTBits - 1, DL, ShTy));
1443  } else if (Amt == NVTBits) {
1444  Lo = InH;
1445  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1446  DAG.getConstant(NVTBits - 1, DL, ShTy));
1447  } else {
1448  Lo = DAG.getNode(ISD::OR, DL, NVT,
1449  DAG.getNode(ISD::SRL, DL, NVT, InL,
1450  DAG.getConstant(Amt, DL, ShTy)),
1451  DAG.getNode(ISD::SHL, DL, NVT, InH,
1452  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1453  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1454  }
1455 }
1456 
1457 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
1458 /// this shift based on knowledge of the high bit of the shift amount. If we
1459 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
1460 /// shift amount.
1461 bool DAGTypeLegalizer::
1462 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1463  SDValue Amt = N->getOperand(1);
1464  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1465  EVT ShTy = Amt.getValueType();
1466  unsigned ShBits = ShTy.getScalarType().getSizeInBits();
1467  unsigned NVTBits = NVT.getScalarType().getSizeInBits();
1468  assert(isPowerOf2_32(NVTBits) &&
1469  "Expanded integer type size not a power of two!");
1470  SDLoc dl(N);
1471 
1472  APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
1473  APInt KnownZero, KnownOne;
1474  DAG.computeKnownBits(N->getOperand(1), KnownZero, KnownOne);
1475 
1476  // If we don't know anything about the high bits, exit.
1477  if (((KnownZero|KnownOne) & HighBitMask) == 0)
1478  return false;
1479 
1480  // Get the incoming operand to be shifted.
1481  SDValue InL, InH;
1482  GetExpandedInteger(N->getOperand(0), InL, InH);
1483 
1484  // If we know that any of the high bits of the shift amount are one, then we
1485  // can do this as a couple of simple shifts.
1486  if (KnownOne.intersects(HighBitMask)) {
1487  // Mask out the high bit, which we know is set.
1488  Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
1489  DAG.getConstant(~HighBitMask, dl, ShTy));
1490 
1491  switch (N->getOpcode()) {
1492  default: llvm_unreachable("Unknown shift");
1493  case ISD::SHL:
1494  Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
1495  Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
1496  return true;
1497  case ISD::SRL:
1498  Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1499  Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
1500  return true;
1501  case ISD::SRA:
1502  Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
1503  DAG.getConstant(NVTBits - 1, dl, ShTy));
1504  Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
1505  return true;
1506  }
1507  }
1508 
1509  // If we know that all of the high bits of the shift amount are zero, then we
1510  // can do this as a couple of simple shifts.
1511  if ((KnownZero & HighBitMask) == HighBitMask) {
1512  // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
1513  // shift if x is zero. We can use XOR here because x is known to be smaller
1514  // than 32.
1515  SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
1516  DAG.getConstant(NVTBits - 1, dl, ShTy));
1517 
1518  unsigned Op1, Op2;
1519  switch (N->getOpcode()) {
1520  default: llvm_unreachable("Unknown shift");
1521  case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
1522  case ISD::SRL:
1523  case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
1524  }
1525 
1526  // When shifting right the arithmetic for Lo and Hi is swapped.
1527  if (N->getOpcode() != ISD::SHL)
1528  std::swap(InL, InH);
1529 
1530  // Use a little trick to get the bits that move from Lo to Hi. First
1531  // shift by one bit.
1532  SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
1533  // Then compute the remaining shift with amount-1.
1534  SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
1535 
1536  Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
1537  Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
1538 
1539  if (N->getOpcode() != ISD::SHL)
1540  std::swap(Hi, Lo);
1541  return true;
1542  }
1543 
1544  return false;
1545 }
1546 
1547 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
1548 /// of any size.
1549 bool DAGTypeLegalizer::
1550 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1551  SDValue Amt = N->getOperand(1);
1552  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1553  EVT ShTy = Amt.getValueType();
1554  unsigned NVTBits = NVT.getSizeInBits();
1555  assert(isPowerOf2_32(NVTBits) &&
1556  "Expanded integer type size not a power of two!");
1557  SDLoc dl(N);
1558 
1559  // Get the incoming operand to be shifted.
1560  SDValue InL, InH;
1561  GetExpandedInteger(N->getOperand(0), InL, InH);
1562 
1563  SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
1564  SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
1565  SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
1566  SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1567  Amt, NVBitsNode, ISD::SETULT);
1568  SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1569  Amt, DAG.getConstant(0, dl, ShTy),
1570  ISD::SETEQ);
1571 
1572  SDValue LoS, HiS, LoL, HiL;
1573  switch (N->getOpcode()) {
1574  default: llvm_unreachable("Unknown shift");
1575  case ISD::SHL:
1576  // Short: ShAmt < NVTBits
1577  LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
1578  HiS = DAG.getNode(ISD::OR, dl, NVT,
1579  DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
1580  DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
1581 
1582  // Long: ShAmt >= NVTBits
1583  LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
1584  HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
1585 
1586  Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
1587  Hi = DAG.getSelect(dl, NVT, isZero, InH,
1588  DAG.getSelect(dl, NVT, isShort, HiS, HiL));
1589  return true;
1590  case ISD::SRL:
1591  // Short: ShAmt < NVTBits
1592  HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
1593  LoS = DAG.getNode(ISD::OR, dl, NVT,
1594  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1595  // FIXME: If Amt is zero, the following shift generates an undefined result
1596  // on some architectures.
1597  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1598 
1599  // Long: ShAmt >= NVTBits
1600  HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1601  LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1602 
1603  Lo = DAG.getSelect(dl, NVT, isZero, InL,
1604  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1605  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1606  return true;
1607  case ISD::SRA:
1608  // Short: ShAmt < NVTBits
1609  HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
1610  LoS = DAG.getNode(ISD::OR, dl, NVT,
1611  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1612  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1613 
1614  // Long: ShAmt >= NVTBits
1615  HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
1616  DAG.getConstant(NVTBits - 1, dl, ShTy));
1617  LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1618 
1619  Lo = DAG.getSelect(dl, NVT, isZero, InL,
1620  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1621  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1622  return true;
1623  }
1624 }
1625 
1626 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
1627  SDValue &Lo, SDValue &Hi) {
1628  SDLoc dl(N);
1629  // Expand the subcomponents.
1630  SDValue LHSL, LHSH, RHSL, RHSH;
1631  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1632  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1633 
1634  EVT NVT = LHSL.getValueType();
1635  SDValue LoOps[2] = { LHSL, RHSL };
1636  SDValue HiOps[3] = { LHSH, RHSH };
1637 
1638  // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
1639  // them. TODO: Teach operation legalization how to expand unsupported
1640  // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
1641  // a carry of type MVT::Glue, but there doesn't seem to be any way to
1642  // generate a value of this type in the expanded code sequence.
1643  bool hasCarry =
1645  ISD::ADDC : ISD::SUBC,
1646  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1647 
1648  if (hasCarry) {
1649  SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
1650  if (N->getOpcode() == ISD::ADD) {
1651  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
1652  HiOps[2] = Lo.getValue(1);
1653  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
1654  } else {
1655  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
1656  HiOps[2] = Lo.getValue(1);
1657  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
1658  }
1659  return;
1660  }
1661 
1662  bool hasOVF =
1665  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1666  if (hasOVF) {
1667  SDVTList VTList = DAG.getVTList(NVT, NVT);
1669  int RevOpc;
1670  if (N->getOpcode() == ISD::ADD) {
1671  RevOpc = ISD::SUB;
1672  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
1673  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
1674  } else {
1675  RevOpc = ISD::ADD;
1676  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
1677  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
1678  }
1679  SDValue OVF = Lo.getValue(1);
1680 
1681  switch (BoolType) {
1683  OVF = DAG.getNode(ISD::AND, dl, NVT, DAG.getConstant(1, dl, NVT), OVF);
1684  // Fallthrough
1686  Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
1687  break;
1689  Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
1690  }
1691  return;
1692  }
1693 
1694  if (N->getOpcode() == ISD::ADD) {
1695  Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
1696  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
1697  SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
1698  ISD::SETULT);
1699  SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
1700  DAG.getConstant(1, dl, NVT),
1701  DAG.getConstant(0, dl, NVT));
1702  SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
1703  ISD::SETULT);
1704  SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
1705  DAG.getConstant(1, dl, NVT), Carry1);
1706  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
1707  } else {
1708  Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
1709  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
1710  SDValue Cmp =
1711  DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
1712  LoOps[0], LoOps[1], ISD::SETULT);
1713  SDValue Borrow = DAG.getSelect(dl, NVT, Cmp,
1714  DAG.getConstant(1, dl, NVT),
1715  DAG.getConstant(0, dl, NVT));
1716  Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
1717  }
1718 }
1719 
1720 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
1721  SDValue &Lo, SDValue &Hi) {
1722  // Expand the subcomponents.
1723  SDValue LHSL, LHSH, RHSL, RHSH;
1724  SDLoc dl(N);
1725  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1726  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1727  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
1728  SDValue LoOps[2] = { LHSL, RHSL };
1729  SDValue HiOps[3] = { LHSH, RHSH };
1730 
1731  if (N->getOpcode() == ISD::ADDC) {
1732  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
1733  HiOps[2] = Lo.getValue(1);
1734  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
1735  } else {
1736  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
1737  HiOps[2] = Lo.getValue(1);
1738  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
1739  }
1740 
1741  // Legalized the flag result - switch anything that used the old flag to
1742  // use the new one.
1743  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
1744 }
1745 
1746 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
1747  SDValue &Lo, SDValue &Hi) {
1748  // Expand the subcomponents.
1749  SDValue LHSL, LHSH, RHSL, RHSH;
1750  SDLoc dl(N);
1751  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1752  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1753  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
1754  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
1755  SDValue HiOps[3] = { LHSH, RHSH };
1756 
1757  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
1758  HiOps[2] = Lo.getValue(1);
1759  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
1760 
1761  // Legalized the flag result - switch anything that used the old flag to
1762  // use the new one.
1763  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
1764 }
1765 
1766 void DAGTypeLegalizer::ExpandIntRes_MERGE_VALUES(SDNode *N, unsigned ResNo,
1767  SDValue &Lo, SDValue &Hi) {
1768  SDValue Res = DisintegrateMERGE_VALUES(N, ResNo);
1769  SplitInteger(Res, Lo, Hi);
1770 }
1771 
1772 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
1773  SDValue &Lo, SDValue &Hi) {
1774  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1775  SDLoc dl(N);
1776  SDValue Op = N->getOperand(0);
1777  if (Op.getValueType().bitsLE(NVT)) {
1778  // The low part is any extension of the input (which degenerates to a copy).
1779  Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
1780  Hi = DAG.getUNDEF(NVT); // The high part is undefined.
1781  } else {
1782  // For example, extension of an i48 to an i64. The operand type necessarily
1783  // promotes to the result type, so will end up being expanded too.
1784  assert(getTypeAction(Op.getValueType()) ==
1786  "Only know how to promote this result!");
1787  SDValue Res = GetPromotedInteger(Op);
1788  assert(Res.getValueType() == N->getValueType(0) &&
1789  "Operand over promoted?");
1790  // Split the promoted operand. This will simplify when it is expanded.
1791  SplitInteger(Res, Lo, Hi);
1792  }
1793 }
1794 
1795 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
1796  SDValue &Lo, SDValue &Hi) {
1797  SDLoc dl(N);
1798  GetExpandedInteger(N->getOperand(0), Lo, Hi);
1799  EVT NVT = Lo.getValueType();
1800  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
1801  unsigned NVTBits = NVT.getSizeInBits();
1802  unsigned EVTBits = EVT.getSizeInBits();
1803 
1804  if (NVTBits < EVTBits) {
1805  Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
1807  EVTBits - NVTBits)));
1808  } else {
1809  Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
1810  // The high part replicates the sign bit of Lo, make it explicit.
1811  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
1812  DAG.getConstant(NVTBits - 1, dl,
1813  TLI.getPointerTy(DAG.getDataLayout())));
1814  }
1815 }
1816 
1817 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
1818  SDValue &Lo, SDValue &Hi) {
1819  SDLoc dl(N);
1820  GetExpandedInteger(N->getOperand(0), Lo, Hi);
1821  EVT NVT = Lo.getValueType();
1822  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
1823  unsigned NVTBits = NVT.getSizeInBits();
1824  unsigned EVTBits = EVT.getSizeInBits();
1825 
1826  if (NVTBits < EVTBits) {
1827  Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
1829  EVTBits - NVTBits)));
1830  } else {
1831  Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
1832  // The high part must be zero, make it explicit.
1833  Hi = DAG.getConstant(0, dl, NVT);
1834  }
1835 }
1836 
1837 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
1838  SDValue &Lo, SDValue &Hi) {
1839  SDLoc dl(N);
1840  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
1841  Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
1842  Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
1843 }
1844 
1845 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
1846  SDValue &Lo, SDValue &Hi) {
1847  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1848  unsigned NBitWidth = NVT.getSizeInBits();
1849  auto Constant = cast<ConstantSDNode>(N);
1850  const APInt &Cst = Constant->getAPIntValue();
1851  bool IsTarget = Constant->isTargetOpcode();
1852  bool IsOpaque = Constant->isOpaque();
1853  SDLoc dl(N);
1854  Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
1855  Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
1856  IsOpaque);
1857 }
1858 
1859 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
1860  SDValue &Lo, SDValue &Hi) {
1861  SDLoc dl(N);
1862  // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
1863  GetExpandedInteger(N->getOperand(0), Lo, Hi);
1864  EVT NVT = Lo.getValueType();
1865 
1866  SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
1867  DAG.getConstant(0, dl, NVT), ISD::SETNE);
1868 
1869  SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
1870  SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
1871 
1872  Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
1873  DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
1874  DAG.getConstant(NVT.getSizeInBits(), dl,
1875  NVT)));
1876  Hi = DAG.getConstant(0, dl, NVT);
1877 }
1878 
1879 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
1880  SDValue &Lo, SDValue &Hi) {
1881  SDLoc dl(N);
1882  // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
1883  GetExpandedInteger(N->getOperand(0), Lo, Hi);
1884  EVT NVT = Lo.getValueType();
1885  Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
1886  DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
1887  Hi = DAG.getConstant(0, dl, NVT);
1888 }
1889 
1890 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
1891  SDValue &Lo, SDValue &Hi) {
1892  SDLoc dl(N);
1893  // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
1894  GetExpandedInteger(N->getOperand(0), Lo, Hi);
1895  EVT NVT = Lo.getValueType();
1896 
1897  SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
1898  DAG.getConstant(0, dl, NVT), ISD::SETNE);
1899 
1900  SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
1901  SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
1902 
1903  Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
1904  DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
1905  DAG.getConstant(NVT.getSizeInBits(), dl,
1906  NVT)));
1907  Hi = DAG.getConstant(0, dl, NVT);
1908 }
1909 
1910 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
1911  SDValue &Hi) {
1912  SDLoc dl(N);
1913  EVT VT = N->getValueType(0);
1914 
1915  SDValue Op = N->getOperand(0);
1916  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
1917  Op = GetPromotedFloat(Op);
1918 
1920  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
1921  SplitInteger(TLI.makeLibCall(DAG, LC, VT, &Op, 1, true/*irrelevant*/,
1922  dl).first,
1923  Lo, Hi);
1924 }
1925 
1926 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
1927  SDValue &Hi) {
1928  SDLoc dl(N);
1929  EVT VT = N->getValueType(0);
1930 
1931  SDValue Op = N->getOperand(0);
1932  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
1933  Op = GetPromotedFloat(Op);
1934 
1936  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
1937  SplitInteger(TLI.makeLibCall(DAG, LC, VT, &Op, 1, false/*irrelevant*/,
1938  dl).first,
1939  Lo, Hi);
1940 }
1941 
1942 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
1943  SDValue &Lo, SDValue &Hi) {
1944  if (ISD::isNormalLoad(N)) {
1945  ExpandRes_NormalLoad(N, Lo, Hi);
1946  return;
1947  }
1948 
1949  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1950 
1951  EVT VT = N->getValueType(0);
1952  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1953  SDValue Ch = N->getChain();
1954  SDValue Ptr = N->getBasePtr();
1955  ISD::LoadExtType ExtType = N->getExtensionType();
1956  unsigned Alignment = N->getAlignment();
1957  bool isVolatile = N->isVolatile();
1958  bool isNonTemporal = N->isNonTemporal();
1959  bool isInvariant = N->isInvariant();
1960  AAMDNodes AAInfo = N->getAAInfo();
1961  SDLoc dl(N);
1962 
1963  assert(NVT.isByteSized() && "Expanded type not byte sized!");
1964 
1965  if (N->getMemoryVT().bitsLE(NVT)) {
1966  EVT MemVT = N->getMemoryVT();
1967 
1968  Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
1969  MemVT, isVolatile, isNonTemporal, isInvariant,
1970  Alignment, AAInfo);
1971 
1972  // Remember the chain.
1973  Ch = Lo.getValue(1);
1974 
1975  if (ExtType == ISD::SEXTLOAD) {
1976  // The high part is obtained by SRA'ing all but one of the bits of the
1977  // lo part.
1978  unsigned LoSize = Lo.getValueType().getSizeInBits();
1979  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
1980  DAG.getConstant(LoSize - 1, dl,
1981  TLI.getPointerTy(DAG.getDataLayout())));
1982  } else if (ExtType == ISD::ZEXTLOAD) {
1983  // The high part is just a zero.
1984  Hi = DAG.getConstant(0, dl, NVT);
1985  } else {
1986  assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
1987  // The high part is undefined.
1988  Hi = DAG.getUNDEF(NVT);
1989  }
1990  } else if (DAG.getDataLayout().isLittleEndian()) {
1991  // Little-endian - low bits are at low addresses.
1992  Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(),
1993  isVolatile, isNonTemporal, isInvariant, Alignment,
1994  AAInfo);
1995 
1996  unsigned ExcessBits =
1997  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
1998  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
1999 
2000  // Increment the pointer to the other half.
2001  unsigned IncrementSize = NVT.getSizeInBits()/8;
2002  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2003  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2004  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
2005  N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
2006  isVolatile, isNonTemporal, isInvariant,
2007  MinAlign(Alignment, IncrementSize), AAInfo);
2008 
2009  // Build a factor node to remember that this load is independent of the
2010  // other one.
2011  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2012  Hi.getValue(1));
2013  } else {
2014  // Big-endian - high bits are at low addresses. Favor aligned loads at
2015  // the cost of some bit-fiddling.
2016  EVT MemVT = N->getMemoryVT();
2017  unsigned EBytes = MemVT.getStoreSize();
2018  unsigned IncrementSize = NVT.getSizeInBits()/8;
2019  unsigned ExcessBits = (EBytes - IncrementSize)*8;
2020 
2021  // Load both the high bits and maybe some of the low bits.
2022  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
2024  MemVT.getSizeInBits() - ExcessBits),
2025  isVolatile, isNonTemporal, isInvariant, Alignment,
2026  AAInfo);
2027 
2028  // Increment the pointer to the other half.
2029  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2030  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2031  // Load the rest of the low bits.
2032  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
2033  N->getPointerInfo().getWithOffset(IncrementSize),
2034  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
2035  isVolatile, isNonTemporal, isInvariant,
2036  MinAlign(Alignment, IncrementSize), AAInfo);
2037 
2038  // Build a factor node to remember that this load is independent of the
2039  // other one.
2040  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2041  Hi.getValue(1));
2042 
2043  if (ExcessBits < NVT.getSizeInBits()) {
2044  // Transfer low bits from the bottom of Hi to the top of Lo.
2045  Lo = DAG.getNode(
2046  ISD::OR, dl, NVT, Lo,
2047  DAG.getNode(ISD::SHL, dl, NVT, Hi,
2048  DAG.getConstant(ExcessBits, dl,
2049  TLI.getPointerTy(DAG.getDataLayout()))));
2050  // Move high bits to the right position in Hi.
2051  Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
2052  Hi,
2053  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
2054  TLI.getPointerTy(DAG.getDataLayout())));
2055  }
2056  }
2057 
2058  // Legalized the chain result - switch anything that used the old chain to
2059  // use the new one.
2060  ReplaceValueWith(SDValue(N, 1), Ch);
2061 }
2062 
2063 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
2064  SDValue &Lo, SDValue &Hi) {
2065  SDLoc dl(N);
2066  SDValue LL, LH, RL, RH;
2067  GetExpandedInteger(N->getOperand(0), LL, LH);
2068  GetExpandedInteger(N->getOperand(1), RL, RH);
2069  Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
2070  Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
2071 }
2072 
2073 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
2074  SDValue &Lo, SDValue &Hi) {
2075  EVT VT = N->getValueType(0);
2076  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2077  SDLoc dl(N);
2078 
2079  SDValue LL, LH, RL, RH;
2080  GetExpandedInteger(N->getOperand(0), LL, LH);
2081  GetExpandedInteger(N->getOperand(1), RL, RH);
2082 
2083  if (TLI.expandMUL(N, Lo, Hi, NVT, DAG, LL, LH, RL, RH))
2084  return;
2085 
2086  // If nothing else, we can make a libcall.
2088  if (VT == MVT::i16)
2089  LC = RTLIB::MUL_I16;
2090  else if (VT == MVT::i32)
2091  LC = RTLIB::MUL_I32;
2092  else if (VT == MVT::i64)
2093  LC = RTLIB::MUL_I64;
2094  else if (VT == MVT::i128)
2095  LC = RTLIB::MUL_I128;
2096  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported MUL!");
2097 
2098  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2099  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, true/*irrelevant*/,
2100  dl).first,
2101  Lo, Hi);
2102 }
2103 
2104 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
2105  SDValue &Lo, SDValue &Hi) {
2106  SDValue LHS = Node->getOperand(0);
2107  SDValue RHS = Node->getOperand(1);
2108  SDLoc dl(Node);
2109 
2110  // Expand the result by simply replacing it with the equivalent
2111  // non-overflow-checking operation.
2112  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
2113  ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
2114  LHS, RHS);
2115  SplitInteger(Sum, Lo, Hi);
2116 
2117  // Compute the overflow.
2118  //
2119  // LHSSign -> LHS >= 0
2120  // RHSSign -> RHS >= 0
2121  // SumSign -> Sum >= 0
2122  //
2123  // Add:
2124  // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
2125  // Sub:
2126  // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
2127  //
2128  EVT OType = Node->getValueType(1);
2129  SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
2130 
2131  SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
2132  SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
2133  SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
2134  Node->getOpcode() == ISD::SADDO ?
2136 
2137  SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
2138  SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
2139 
2140  SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
2141 
2142  // Use the calculated overflow everywhere.
2143  ReplaceValueWith(SDValue(Node, 1), Cmp);
2144 }
2145 
2146 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
2147  SDValue &Lo, SDValue &Hi) {
2148  EVT VT = N->getValueType(0);
2149  SDLoc dl(N);
2150  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2151 
2153  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2154  SplitInteger(Res.getValue(0), Lo, Hi);
2155  return;
2156  }
2157 
2159  if (VT == MVT::i16)
2160  LC = RTLIB::SDIV_I16;
2161  else if (VT == MVT::i32)
2162  LC = RTLIB::SDIV_I32;
2163  else if (VT == MVT::i64)
2164  LC = RTLIB::SDIV_I64;
2165  else if (VT == MVT::i128)
2166  LC = RTLIB::SDIV_I128;
2167  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
2168 
2169  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, true, dl).first, Lo, Hi);
2170 }
2171 
2172 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
2173  SDValue &Lo, SDValue &Hi) {
2174  EVT VT = N->getValueType(0);
2175  SDLoc dl(N);
2176 
2177  // If we can emit an efficient shift operation, do so now. Check to see if
2178  // the RHS is a constant.
2179  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
2180  return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
2181 
2182  // If we can determine that the high bit of the shift is zero or one, even if
2183  // the low bits are variable, emit this shift in an optimized form.
2184  if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
2185  return;
2186 
2187  // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
2188  unsigned PartsOpc;
2189  if (N->getOpcode() == ISD::SHL) {
2190  PartsOpc = ISD::SHL_PARTS;
2191  } else if (N->getOpcode() == ISD::SRL) {
2192  PartsOpc = ISD::SRL_PARTS;
2193  } else {
2194  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2195  PartsOpc = ISD::SRA_PARTS;
2196  }
2197 
2198  // Next check to see if the target supports this SHL_PARTS operation or if it
2199  // will custom expand it.
2200  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2201  TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
2202  if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
2203  Action == TargetLowering::Custom) {
2204  // Expand the subcomponents.
2205  SDValue LHSL, LHSH;
2206  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2207  EVT VT = LHSL.getValueType();
2208 
2209  // If the shift amount operand is coming from a vector legalization it may
2210  // have an illegal type. Fix that first by casting the operand, otherwise
2211  // the new SHL_PARTS operation would need further legalization.
2212  SDValue ShiftOp = N->getOperand(1);
2213  EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2214  assert(ShiftTy.getScalarType().getSizeInBits() >=
2215  Log2_32_Ceil(VT.getScalarType().getSizeInBits()) &&
2216  "ShiftAmountTy is too small to cover the range of this type!");
2217  if (ShiftOp.getValueType() != ShiftTy)
2218  ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
2219 
2220  SDValue Ops[] = { LHSL, LHSH, ShiftOp };
2221  Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
2222  Hi = Lo.getValue(1);
2223  return;
2224  }
2225 
2226  // Otherwise, emit a libcall.
2228  bool isSigned;
2229  if (N->getOpcode() == ISD::SHL) {
2230  isSigned = false; /*sign irrelevant*/
2231  if (VT == MVT::i16)
2232  LC = RTLIB::SHL_I16;
2233  else if (VT == MVT::i32)
2234  LC = RTLIB::SHL_I32;
2235  else if (VT == MVT::i64)
2236  LC = RTLIB::SHL_I64;
2237  else if (VT == MVT::i128)
2238  LC = RTLIB::SHL_I128;
2239  } else if (N->getOpcode() == ISD::SRL) {
2240  isSigned = false;
2241  if (VT == MVT::i16)
2242  LC = RTLIB::SRL_I16;
2243  else if (VT == MVT::i32)
2244  LC = RTLIB::SRL_I32;
2245  else if (VT == MVT::i64)
2246  LC = RTLIB::SRL_I64;
2247  else if (VT == MVT::i128)
2248  LC = RTLIB::SRL_I128;
2249  } else {
2250  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2251  isSigned = true;
2252  if (VT == MVT::i16)
2253  LC = RTLIB::SRA_I16;
2254  else if (VT == MVT::i32)
2255  LC = RTLIB::SRA_I32;
2256  else if (VT == MVT::i64)
2257  LC = RTLIB::SRA_I64;
2258  else if (VT == MVT::i128)
2259  LC = RTLIB::SRA_I128;
2260  }
2261 
2262  if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
2263  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2264  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, isSigned, dl).first, Lo,
2265  Hi);
2266  return;
2267  }
2268 
2269  if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
2270  llvm_unreachable("Unsupported shift!");
2271 }
2272 
2273 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
2274  SDValue &Lo, SDValue &Hi) {
2275  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2276  SDLoc dl(N);
2277  SDValue Op = N->getOperand(0);
2278  if (Op.getValueType().bitsLE(NVT)) {
2279  // The low part is sign extension of the input (degenerates to a copy).
2280  Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
2281  // The high part is obtained by SRA'ing all but one of the bits of low part.
2282  unsigned LoSize = NVT.getSizeInBits();
2283  Hi = DAG.getNode(
2284  ISD::SRA, dl, NVT, Lo,
2285  DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
2286  } else {
2287  // For example, extension of an i48 to an i64. The operand type necessarily
2288  // promotes to the result type, so will end up being expanded too.
2289  assert(getTypeAction(Op.getValueType()) ==
2291  "Only know how to promote this result!");
2292  SDValue Res = GetPromotedInteger(Op);
2293  assert(Res.getValueType() == N->getValueType(0) &&
2294  "Operand over promoted?");
2295  // Split the promoted operand. This will simplify when it is expanded.
2296  SplitInteger(Res, Lo, Hi);
2297  unsigned ExcessBits =
2298  Op.getValueType().getSizeInBits() - NVT.getSizeInBits();
2299  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2301  ExcessBits)));
2302  }
2303 }
2304 
2305 void DAGTypeLegalizer::
2306 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2307  SDLoc dl(N);
2308  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2309  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2310 
2311  if (EVT.bitsLE(Lo.getValueType())) {
2312  // sext_inreg the low part if needed.
2313  Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
2314  N->getOperand(1));
2315 
2316  // The high part gets the sign extension from the lo-part. This handles
2317  // things like sextinreg V:i64 from i8.
2318  Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
2319  DAG.getConstant(Hi.getValueType().getSizeInBits() - 1, dl,
2320  TLI.getPointerTy(DAG.getDataLayout())));
2321  } else {
2322  // For example, extension of an i48 to an i64. Leave the low part alone,
2323  // sext_inreg the high part.
2324  unsigned ExcessBits =
2325  EVT.getSizeInBits() - Lo.getValueType().getSizeInBits();
2326  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2328  ExcessBits)));
2329  }
2330 }
2331 
2332 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
2333  SDValue &Lo, SDValue &Hi) {
2334  EVT VT = N->getValueType(0);
2335  SDLoc dl(N);
2336  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2337 
2339  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2340  SplitInteger(Res.getValue(1), Lo, Hi);
2341  return;
2342  }
2343 
2345  if (VT == MVT::i16)
2346  LC = RTLIB::SREM_I16;
2347  else if (VT == MVT::i32)
2348  LC = RTLIB::SREM_I32;
2349  else if (VT == MVT::i64)
2350  LC = RTLIB::SREM_I64;
2351  else if (VT == MVT::i128)
2352  LC = RTLIB::SREM_I128;
2353  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
2354 
2355  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, true, dl).first, Lo, Hi);
2356 }
2357 
2358 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
2359  SDValue &Lo, SDValue &Hi) {
2360  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2361  SDLoc dl(N);
2362  Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
2363  Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
2364  N->getOperand(0),
2365  DAG.getConstant(NVT.getSizeInBits(), dl,
2366  TLI.getPointerTy(DAG.getDataLayout())));
2367  Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
2368 }
2369 
2370 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
2371  SDValue &Lo, SDValue &Hi) {
2372  SDValue LHS = N->getOperand(0);
2373  SDValue RHS = N->getOperand(1);
2374  SDLoc dl(N);
2375 
2376  // Expand the result by simply replacing it with the equivalent
2377  // non-overflow-checking operation.
2378  SDValue Sum = DAG.getNode(N->getOpcode() == ISD::UADDO ?
2379  ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
2380  LHS, RHS);
2381  SplitInteger(Sum, Lo, Hi);
2382 
2383  // Calculate the overflow: addition overflows iff a + b < a, and subtraction
2384  // overflows iff a - b > a.
2385  SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS,
2386  N->getOpcode () == ISD::UADDO ?
2388 
2389  // Use the calculated overflow everywhere.
2390  ReplaceValueWith(SDValue(N, 1), Ofl);
2391 }
2392 
2393 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
2394  SDValue &Lo, SDValue &Hi) {
2395  EVT VT = N->getValueType(0);
2396  SDLoc dl(N);
2397 
2398  // A divide for UMULO should be faster than a function call.
2399  if (N->getOpcode() == ISD::UMULO) {
2400  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
2401 
2402  SDValue MUL = DAG.getNode(ISD::MUL, dl, LHS.getValueType(), LHS, RHS);
2403  SplitInteger(MUL, Lo, Hi);
2404 
2405  // A divide for UMULO will be faster than a function call. Select to
2406  // make sure we aren't using 0.
2407  SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(VT),
2408  RHS, DAG.getConstant(0, dl, VT), ISD::SETEQ);
2409  SDValue NotZero = DAG.getSelect(dl, VT, isZero,
2410  DAG.getConstant(1, dl, VT), RHS);
2411  SDValue DIV = DAG.getNode(ISD::UDIV, dl, VT, MUL, NotZero);
2412  SDValue Overflow = DAG.getSetCC(dl, N->getValueType(1), DIV, LHS,
2413  ISD::SETNE);
2414  Overflow = DAG.getSelect(dl, N->getValueType(1), isZero,
2415  DAG.getConstant(0, dl, N->getValueType(1)),
2416  Overflow);
2417  ReplaceValueWith(SDValue(N, 1), Overflow);
2418  return;
2419  }
2420 
2421  Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
2422  EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
2423  Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
2424 
2425  // Replace this with a libcall that will check overflow.
2427  if (VT == MVT::i32)
2428  LC = RTLIB::MULO_I32;
2429  else if (VT == MVT::i64)
2430  LC = RTLIB::MULO_I64;
2431  else if (VT == MVT::i128)
2432  LC = RTLIB::MULO_I128;
2433  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
2434 
2435  SDValue Temp = DAG.CreateStackTemporary(PtrVT);
2436  // Temporary for the overflow value, default it to zero.
2437  SDValue Chain = DAG.getStore(DAG.getEntryNode(), dl,
2438  DAG.getConstant(0, dl, PtrVT), Temp,
2439  MachinePointerInfo(), false, false, 0);
2440 
2443  for (const SDValue &Op : N->op_values()) {
2444  EVT ArgVT = Op.getValueType();
2445  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2446  Entry.Node = Op;
2447  Entry.Ty = ArgTy;
2448  Entry.isSExt = true;
2449  Entry.isZExt = false;
2450  Args.push_back(Entry);
2451  }
2452 
2453  // Also pass the address of the overflow check.
2454  Entry.Node = Temp;
2455  Entry.Ty = PtrTy->getPointerTo();
2456  Entry.isSExt = true;
2457  Entry.isZExt = false;
2458  Args.push_back(Entry);
2459 
2460  SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
2461 
2463  CLI.setDebugLoc(dl).setChain(Chain)
2464  .setCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args), 0)
2465  .setSExtResult();
2466 
2467  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2468 
2469  SplitInteger(CallInfo.first, Lo, Hi);
2470  SDValue Temp2 = DAG.getLoad(PtrVT, dl, CallInfo.second, Temp,
2471  MachinePointerInfo(), false, false, false, 0);
2472  SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
2473  DAG.getConstant(0, dl, PtrVT),
2474  ISD::SETNE);
2475  // Use the overflow from the libcall everywhere.
2476  ReplaceValueWith(SDValue(N, 1), Ofl);
2477 }
2478 
2479 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
2480  SDValue &Lo, SDValue &Hi) {
2481  EVT VT = N->getValueType(0);
2482  SDLoc dl(N);
2483  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2484 
2486  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2487  SplitInteger(Res.getValue(0), Lo, Hi);
2488  return;
2489  }
2490 
2492  if (VT == MVT::i16)
2493  LC = RTLIB::UDIV_I16;
2494  else if (VT == MVT::i32)
2495  LC = RTLIB::UDIV_I32;
2496  else if (VT == MVT::i64)
2497  LC = RTLIB::UDIV_I64;
2498  else if (VT == MVT::i128)
2499  LC = RTLIB::UDIV_I128;
2500  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
2501 
2502  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, false, dl).first, Lo, Hi);
2503 }
2504 
2505 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
2506  SDValue &Lo, SDValue &Hi) {
2507  EVT VT = N->getValueType(0);
2508  SDLoc dl(N);
2509  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2510 
2512  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2513  SplitInteger(Res.getValue(1), Lo, Hi);
2514  return;
2515  }
2516 
2518  if (VT == MVT::i16)
2519  LC = RTLIB::UREM_I16;
2520  else if (VT == MVT::i32)
2521  LC = RTLIB::UREM_I32;
2522  else if (VT == MVT::i64)
2523  LC = RTLIB::UREM_I64;
2524  else if (VT == MVT::i128)
2525  LC = RTLIB::UREM_I128;
2526  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
2527 
2528  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, false, dl).first, Lo, Hi);
2529 }
2530 
2531 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
2532  SDValue &Lo, SDValue &Hi) {
2533  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2534  SDLoc dl(N);
2535  SDValue Op = N->getOperand(0);
2536  if (Op.getValueType().bitsLE(NVT)) {
2537  // The low part is zero extension of the input (degenerates to a copy).
2538  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
2539  Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
2540  } else {
2541  // For example, extension of an i48 to an i64. The operand type necessarily
2542  // promotes to the result type, so will end up being expanded too.
2543  assert(getTypeAction(Op.getValueType()) ==
2545  "Only know how to promote this result!");
2546  SDValue Res = GetPromotedInteger(Op);
2547  assert(Res.getValueType() == N->getValueType(0) &&
2548  "Operand over promoted?");
2549  // Split the promoted operand. This will simplify when it is expanded.
2550  SplitInteger(Res, Lo, Hi);
2551  unsigned ExcessBits =
2552  Op.getValueType().getSizeInBits() - NVT.getSizeInBits();
2553  Hi = DAG.getZeroExtendInReg(Hi, dl,
2555  ExcessBits));
2556  }
2557 }
2558 
2559 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
2560  SDValue &Lo, SDValue &Hi) {
2561  SDLoc dl(N);
2562  EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
2563  SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
2564  SDValue Zero = DAG.getConstant(0, dl, VT);
2565  SDValue Swap = DAG.getAtomicCmpSwap(
2567  cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
2568  N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand(),
2569  cast<AtomicSDNode>(N)->getOrdering(),
2570  cast<AtomicSDNode>(N)->getOrdering(),
2571  cast<AtomicSDNode>(N)->getSynchScope());
2572 
2573  ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
2574  ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
2575 }
2576 
2577 //===----------------------------------------------------------------------===//
2578 // Integer Operand Expansion
2579 //===----------------------------------------------------------------------===//
2580 
2581 /// ExpandIntegerOperand - This method is called when the specified operand of
2582 /// the specified node is found to need expansion. At this point, all of the
2583 /// result types of the node are known to be legal, but other operands of the
2584 /// node may need promotion or expansion as well as the specified one.
2585 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
2586  DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); dbgs() << "\n");
2587  SDValue Res = SDValue();
2588 
2589  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2590  return false;
2591 
2592  switch (N->getOpcode()) {
2593  default:
2594  #ifndef NDEBUG
2595  dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
2596  N->dump(&DAG); dbgs() << "\n";
2597  #endif
2598  llvm_unreachable("Do not know how to expand this operator's operand!");
2599 
2600  case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
2601  case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
2602  case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
2603  case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
2604  case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
2605  case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
2606  case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
2607  case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
2608  case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break;
2609  case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
2610  case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
2611  case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break;
2612 
2613  case ISD::SHL:
2614  case ISD::SRA:
2615  case ISD::SRL:
2616  case ISD::ROTL:
2617  case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
2618  case ISD::RETURNADDR:
2619  case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
2620 
2621  case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
2622  }
2623 
2624  // If the result is null, the sub-method took care of registering results etc.
2625  if (!Res.getNode()) return false;
2626 
2627  // If the result is N, the sub-method updated N in place. Tell the legalizer
2628  // core about this.
2629  if (Res.getNode() == N)
2630  return true;
2631 
2632  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2633  "Invalid operand expansion");
2634 
2635  ReplaceValueWith(SDValue(N, 0), Res);
2636  return false;
2637 }
2638 
2639 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
2640 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
2641 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
2642  SDValue &NewRHS,
2643  ISD::CondCode &CCCode,
2644  SDLoc dl) {
2645  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2646  GetExpandedInteger(NewLHS, LHSLo, LHSHi);
2647  GetExpandedInteger(NewRHS, RHSLo, RHSHi);
2648 
2649  if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
2650  if (RHSLo == RHSHi) {
2651  if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
2652  if (RHSCST->isAllOnesValue()) {
2653  // Equality comparison to -1.
2654  NewLHS = DAG.getNode(ISD::AND, dl,
2655  LHSLo.getValueType(), LHSLo, LHSHi);
2656  NewRHS = RHSLo;
2657  return;
2658  }
2659  }
2660  }
2661 
2662  NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
2663  NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
2664  NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
2665  NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
2666  return;
2667  }
2668 
2669  // If this is a comparison of the sign bit, just look at the top part.
2670  // X > -1, x < 0
2671  if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
2672  if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0
2673  (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1
2674  NewLHS = LHSHi;
2675  NewRHS = RHSHi;
2676  return;
2677  }
2678 
2679  // FIXME: This generated code sucks.
2680  ISD::CondCode LowCC;
2681  switch (CCCode) {
2682  default: llvm_unreachable("Unknown integer setcc!");
2683  case ISD::SETLT:
2684  case ISD::SETULT: LowCC = ISD::SETULT; break;
2685  case ISD::SETGT:
2686  case ISD::SETUGT: LowCC = ISD::SETUGT; break;
2687  case ISD::SETLE:
2688  case ISD::SETULE: LowCC = ISD::SETULE; break;
2689  case ISD::SETGE:
2690  case ISD::SETUGE: LowCC = ISD::SETUGE; break;
2691  }
2692 
2693  // Tmp1 = lo(op1) < lo(op2) // Always unsigned comparison
2694  // Tmp2 = hi(op1) < hi(op2) // Signedness depends on operands
2695  // dest = hi(op1) == hi(op2) ? Tmp1 : Tmp2;
2696 
2697  // NOTE: on targets without efficient SELECT of bools, we can always use
2698  // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
2699  TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
2700  nullptr);
2701  SDValue Tmp1, Tmp2;
2702  if (TLI.isTypeLegal(LHSLo.getValueType()) &&
2703  TLI.isTypeLegal(RHSLo.getValueType()))
2704  Tmp1 = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()),
2705  LHSLo, RHSLo, LowCC, false, DagCombineInfo, dl);
2706  if (!Tmp1.getNode())
2707  Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()),
2708  LHSLo, RHSLo, LowCC);
2709  if (TLI.isTypeLegal(LHSHi.getValueType()) &&
2710  TLI.isTypeLegal(RHSHi.getValueType()))
2711  Tmp2 = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()),
2712  LHSHi, RHSHi, CCCode, false, DagCombineInfo, dl);
2713  if (!Tmp2.getNode())
2714  Tmp2 = DAG.getNode(ISD::SETCC, dl,
2715  getSetCCResultType(LHSHi.getValueType()),
2716  LHSHi, RHSHi, DAG.getCondCode(CCCode));
2717 
2718  ConstantSDNode *Tmp1C = dyn_cast<ConstantSDNode>(Tmp1.getNode());
2719  ConstantSDNode *Tmp2C = dyn_cast<ConstantSDNode>(Tmp2.getNode());
2720  if ((Tmp1C && Tmp1C->isNullValue()) ||
2721  (Tmp2C && Tmp2C->isNullValue() &&
2722  (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
2723  CCCode == ISD::SETUGE || CCCode == ISD::SETULE)) ||
2724  (Tmp2C && Tmp2C->getAPIntValue() == 1 &&
2725  (CCCode == ISD::SETLT || CCCode == ISD::SETGT ||
2726  CCCode == ISD::SETUGT || CCCode == ISD::SETULT))) {
2727  // low part is known false, returns high part.
2728  // For LE / GE, if high part is known false, ignore the low part.
2729  // For LT / GT, if high part is known true, ignore the low part.
2730  NewLHS = Tmp2;
2731  NewRHS = SDValue();
2732  return;
2733  }
2734 
2735  NewLHS = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()),
2736  LHSHi, RHSHi, ISD::SETEQ, false,
2737  DagCombineInfo, dl);
2738  if (!NewLHS.getNode())
2739  NewLHS = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()),
2740  LHSHi, RHSHi, ISD::SETEQ);
2741  NewLHS = DAG.getSelect(dl, Tmp1.getValueType(),
2742  NewLHS, Tmp1, Tmp2);
2743  NewRHS = SDValue();
2744 }
2745 
2746 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
2747  SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
2748  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
2749  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
2750 
2751  // If ExpandSetCCOperands returned a scalar, we need to compare the result
2752  // against zero to select between true and false values.
2753  if (!NewRHS.getNode()) {
2754  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2755  CCCode = ISD::SETNE;
2756  }
2757 
2758  // Update N to have the operands specified.
2759  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2760  DAG.getCondCode(CCCode), NewLHS, NewRHS,
2761  N->getOperand(4)), 0);
2762 }
2763 
2764 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
2765  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
2766  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
2767  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
2768 
2769  // If ExpandSetCCOperands returned a scalar, we need to compare the result
2770  // against zero to select between true and false values.
2771  if (!NewRHS.getNode()) {
2772  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2773  CCCode = ISD::SETNE;
2774  }
2775 
2776  // Update N to have the operands specified.
2777  return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
2778  N->getOperand(2), N->getOperand(3),
2779  DAG.getCondCode(CCCode)), 0);
2780 }
2781 
2782 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
2783  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
2784  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2785  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
2786 
2787  // If ExpandSetCCOperands returned a scalar, use it.
2788  if (!NewRHS.getNode()) {
2789  assert(NewLHS.getValueType() == N->getValueType(0) &&
2790  "Unexpected setcc expansion!");
2791  return NewLHS;
2792  }
2793 
2794  // Otherwise, update N to have the operands specified.
2795  return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
2796  DAG.getCondCode(CCCode)), 0);
2797 }
2798 
2799 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
2800  // The value being shifted is legal, but the shift amount is too big.
2801  // It follows that either the result of the shift is undefined, or the
2802  // upper half of the shift amount is zero. Just use the lower half.
2803  SDValue Lo, Hi;
2804  GetExpandedInteger(N->getOperand(1), Lo, Hi);
2805  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
2806 }
2807 
2808 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
2809  // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
2810  // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
2811  // constant to valid type.
2812  SDValue Lo, Hi;
2813  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2814  return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
2815 }
2816 
2817 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
2818  SDValue Op = N->getOperand(0);
2819  EVT DstVT = N->getValueType(0);
2821  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
2822  "Don't know how to expand this SINT_TO_FP!");
2823  return TLI.makeLibCall(DAG, LC, DstVT, &Op, 1, true, SDLoc(N)).first;
2824 }
2825 
2826 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
2827  if (ISD::isNormalStore(N))
2828  return ExpandOp_NormalStore(N, OpNo);
2829 
2830  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2831  assert(OpNo == 1 && "Can only expand the stored value so far");
2832 
2833  EVT VT = N->getOperand(1).getValueType();
2834  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2835  SDValue Ch = N->getChain();
2836  SDValue Ptr = N->getBasePtr();
2837  unsigned Alignment = N->getAlignment();
2838  bool isVolatile = N->isVolatile();
2839  bool isNonTemporal = N->isNonTemporal();
2840  AAMDNodes AAInfo = N->getAAInfo();
2841  SDLoc dl(N);
2842  SDValue Lo, Hi;
2843 
2844  assert(NVT.isByteSized() && "Expanded type not byte sized!");
2845 
2846  if (N->getMemoryVT().bitsLE(NVT)) {
2847  GetExpandedInteger(N->getValue(), Lo, Hi);
2848  return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
2849  N->getMemoryVT(), isVolatile, isNonTemporal,
2850  Alignment, AAInfo);
2851  }
2852 
2853  if (DAG.getDataLayout().isLittleEndian()) {
2854  // Little-endian - low bits are at low addresses.
2855  GetExpandedInteger(N->getValue(), Lo, Hi);
2856 
2857  Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
2858  isVolatile, isNonTemporal, Alignment, AAInfo);
2859 
2860  unsigned ExcessBits =
2861  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
2862  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
2863 
2864  // Increment the pointer to the other half.
2865  unsigned IncrementSize = NVT.getSizeInBits()/8;
2866  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2867  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2868  Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr,
2869  N->getPointerInfo().getWithOffset(IncrementSize),
2870  NEVT, isVolatile, isNonTemporal,
2871  MinAlign(Alignment, IncrementSize), AAInfo);
2872  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
2873  }
2874 
2875  // Big-endian - high bits are at low addresses. Favor aligned stores at
2876  // the cost of some bit-fiddling.
2877  GetExpandedInteger(N->getValue(), Lo, Hi);
2878 
2879  EVT ExtVT = N->getMemoryVT();
2880  unsigned EBytes = ExtVT.getStoreSize();
2881  unsigned IncrementSize = NVT.getSizeInBits()/8;
2882  unsigned ExcessBits = (EBytes - IncrementSize)*8;
2883  EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
2884  ExtVT.getSizeInBits() - ExcessBits);
2885 
2886  if (ExcessBits < NVT.getSizeInBits()) {
2887  // Transfer high bits from the top of Lo to the bottom of Hi.
2888  Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
2889  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
2890  TLI.getPointerTy(DAG.getDataLayout())));
2891  Hi = DAG.getNode(
2892  ISD::OR, dl, NVT, Hi,
2893  DAG.getNode(ISD::SRL, dl, NVT, Lo,
2894  DAG.getConstant(ExcessBits, dl,
2895  TLI.getPointerTy(DAG.getDataLayout()))));
2896  }
2897 
2898  // Store both the high bits and maybe some of the low bits.
2899  Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(),
2900  HiVT, isVolatile, isNonTemporal, Alignment, AAInfo);
2901 
2902  // Increment the pointer to the other half.
2903  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2904  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2905  // Store the lowest ExcessBits bits in the second half.
2906  Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
2907  N->getPointerInfo().getWithOffset(IncrementSize),
2908  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
2909  isVolatile, isNonTemporal,
2910  MinAlign(Alignment, IncrementSize), AAInfo);
2911  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
2912 }
2913 
2914 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
2915  SDValue InL, InH;
2916  GetExpandedInteger(N->getOperand(0), InL, InH);
2917  // Just truncate the low part of the source.
2918  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
2919 }
2920 
2921 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
2922  SDValue Op = N->getOperand(0);
2923  EVT SrcVT = Op.getValueType();
2924  EVT DstVT = N->getValueType(0);
2925  SDLoc dl(N);
2926 
2927  // The following optimization is valid only if every value in SrcVT (when
2928  // treated as signed) is representable in DstVT. Check that the mantissa
2929  // size of DstVT is >= than the number of bits in SrcVT -1.
2930  const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT);
2931  if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 &&
2933  // Do a signed conversion then adjust the result.
2934  SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op);
2935  SignedConv = TLI.LowerOperation(SignedConv, DAG);
2936 
2937  // The result of the signed conversion needs adjusting if the 'sign bit' of
2938  // the incoming integer was set. To handle this, we dynamically test to see
2939  // if it is set, and, if so, add a fudge factor.
2940 
2941  const uint64_t F32TwoE32 = 0x4F800000ULL;
2942  const uint64_t F32TwoE64 = 0x5F800000ULL;
2943  const uint64_t F32TwoE128 = 0x7F800000ULL;
2944 
2945  APInt FF(32, 0);
2946  if (SrcVT == MVT::i32)
2947  FF = APInt(32, F32TwoE32);
2948  else if (SrcVT == MVT::i64)
2949  FF = APInt(32, F32TwoE64);
2950  else if (SrcVT == MVT::i128)
2951  FF = APInt(32, F32TwoE128);
2952  else
2953  llvm_unreachable("Unsupported UINT_TO_FP!");
2954 
2955  // Check whether the sign bit is set.
2956  SDValue Lo, Hi;
2957  GetExpandedInteger(Op, Lo, Hi);
2958  SDValue SignSet = DAG.getSetCC(dl,
2959  getSetCCResultType(Hi.getValueType()),
2960  Hi,
2961  DAG.getConstant(0, dl, Hi.getValueType()),
2962  ISD::SETLT);
2963 
2964  // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
2965  SDValue FudgePtr =
2966  DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)),
2967  TLI.getPointerTy(DAG.getDataLayout()));
2968 
2969  // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
2970  SDValue Zero = DAG.getIntPtrConstant(0, dl);
2971  SDValue Four = DAG.getIntPtrConstant(4, dl);
2972  if (DAG.getDataLayout().isBigEndian())
2973  std::swap(Zero, Four);
2974  SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet,
2975  Zero, Four);
2976  unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment();
2977  FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(),
2978  FudgePtr, Offset);
2979  Alignment = std::min(Alignment, 4u);
2980 
2981  // Load the value out, extending it from f32 to the destination float type.
2982  // FIXME: Avoid the extend by constructing the right constant pool?
2983  SDValue Fudge = DAG.getExtLoad(ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(),
2984  FudgePtr,
2986  MVT::f32,
2987  false, false, false, Alignment);
2988  return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge);
2989  }
2990 
2991  // Otherwise, use a libcall.
2992  RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT);
2993  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
2994  "Don't know how to expand this UINT_TO_FP!");
2995  return TLI.makeLibCall(DAG, LC, DstVT, &Op, 1, true, dl).first;
2996 }
2997 
2998 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
2999  SDLoc dl(N);
3000  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
3001  cast<AtomicSDNode>(N)->getMemoryVT(),
3002  N->getOperand(0),
3003  N->getOperand(1), N->getOperand(2),
3004  cast<AtomicSDNode>(N)->getMemOperand(),
3005  cast<AtomicSDNode>(N)->getOrdering(),
3006  cast<AtomicSDNode>(N)->getSynchScope());
3007  return Swap.getValue(1);
3008 }
3009 
3010 
3011 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
3012  SDValue InOp0 = N->getOperand(0);
3013  EVT InVT = InOp0.getValueType();
3014 
3015  EVT OutVT = N->getValueType(0);
3016  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3017  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3018  unsigned OutNumElems = OutVT.getVectorNumElements();
3019  EVT NOutVTElem = NOutVT.getVectorElementType();
3020 
3021  SDLoc dl(N);
3022  SDValue BaseIdx = N->getOperand(1);
3023 
3025  Ops.reserve(OutNumElems);
3026  for (unsigned i = 0; i != OutNumElems; ++i) {
3027 
3028  // Extract the element from the original vector.
3029  SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
3030  BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
3032  InVT.getVectorElementType(), N->getOperand(0), Index);
3033 
3034  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext);
3035  // Insert the converted element to the new vector.
3036  Ops.push_back(Op);
3037  }
3038 
3039  return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, Ops);
3040 }
3041 
3042 
3043 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
3044  ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
3045  EVT VT = N->getValueType(0);
3046  SDLoc dl(N);
3047 
3048  ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
3049 
3050  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3051  SDValue V1 = GetPromotedInteger(N->getOperand(1));
3052  EVT OutVT = V0.getValueType();
3053 
3054  return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
3055 }
3056 
3057 
3058 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
3059  EVT OutVT = N->getValueType(0);
3060  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3061  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3062  unsigned NumElems = N->getNumOperands();
3063  EVT NOutVTElem = NOutVT.getVectorElementType();
3064 
3065  SDLoc dl(N);
3066 
3068  Ops.reserve(NumElems);
3069  for (unsigned i = 0; i != NumElems; ++i) {
3070  SDValue Op;
3071  // BUILD_VECTOR integer operand types are allowed to be larger than the
3072  // result's element type. This may still be true after the promotion. For
3073  // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
3074  // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
3075  if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
3076  Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
3077  else
3078  Op = N->getOperand(i);
3079  Ops.push_back(Op);
3080  }
3081 
3082  return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, Ops);
3083 }
3084 
3085 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
3086 
3087  SDLoc dl(N);
3088 
3089  assert(!N->getOperand(0).getValueType().isVector() &&
3090  "Input must be a scalar");
3091 
3092  EVT OutVT = N->getValueType(0);
3093  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3094  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3095  EVT NOutVTElem = NOutVT.getVectorElementType();
3096 
3097  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
3098 
3099  return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
3100 }
3101 
3102 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
3103  SDLoc dl(N);
3104 
3105  EVT OutVT = N->getValueType(0);
3106  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3107  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3108 
3109  EVT InElemTy = OutVT.getVectorElementType();
3110  EVT OutElemTy = NOutVT.getVectorElementType();
3111 
3112  unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
3113  unsigned NumOutElem = NOutVT.getVectorNumElements();
3114  unsigned NumOperands = N->getNumOperands();
3115  assert(NumElem * NumOperands == NumOutElem &&
3116  "Unexpected number of elements");
3117 
3118  // Take the elements from the first vector.
3119  SmallVector<SDValue, 8> Ops(NumOutElem);
3120  for (unsigned i = 0; i < NumOperands; ++i) {
3121  SDValue Op = N->getOperand(i);
3122  for (unsigned j = 0; j < NumElem; ++j) {
3123  SDValue Ext = DAG.getNode(
3124  ISD::EXTRACT_VECTOR_ELT, dl, InElemTy, Op,
3125  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3126  Ops[i * NumElem + j] = DAG.getNode(ISD::ANY_EXTEND, dl, OutElemTy, Ext);
3127  }
3128  }
3129 
3130  return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, Ops);
3131 }
3132 
3133 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
3134  EVT OutVT = N->getValueType(0);
3135  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3136  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3137 
3138  EVT NOutVTElem = NOutVT.getVectorElementType();
3139 
3140  SDLoc dl(N);
3141  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3142 
3143  SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
3144  NOutVTElem, N->getOperand(1));
3145  return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
3146  V0, ConvElem, N->getOperand(2));
3147 }
3148 
3149 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3150  SDLoc dl(N);
3151  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3152  SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
3153  TLI.getVectorIdxTy(DAG.getDataLayout()));
3154  SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
3155  V0->getValueType(0).getScalarType(), V0, V1);
3156 
3157  // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
3158  // element types. If this is the case then we need to expand the outgoing
3159  // value and not truncate it.
3160  return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
3161 }
3162 
3163 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
3164  SDLoc dl(N);
3165  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3166  MVT InVT = V0.getValueType().getSimpleVT();
3167  MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
3169  SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
3170  return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
3171 }
3172 
3173 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
3174  SDLoc dl(N);
3175  unsigned NumElems = N->getNumOperands();
3176 
3177  EVT RetSclrTy = N->getValueType(0).getVectorElementType();
3178 
3179  SmallVector<SDValue, 8> NewOps;
3180  NewOps.reserve(NumElems);
3181 
3182  // For each incoming vector
3183  for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
3184  SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
3185  EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
3186  unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
3187 
3188  for (unsigned i=0; i<NumElem; ++i) {
3189  // Extract element from incoming vector
3190  SDValue Ex = DAG.getNode(
3191  ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
3192  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3193  SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
3194  NewOps.push_back(Tr);
3195  }
3196  }
3197 
3198  return DAG.getNode(ISD::BUILD_VECTOR, dl, N->getValueType(0), NewOps);
3199 }
SDValue getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT TVT, bool isNonTemporal, bool isVolatile, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes())
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:477
void push_back(const T &Elt)
Definition: SmallVector.h:222
unsigned Log2_32_Ceil(uint32_t Value)
Log2_32_Ceil - This function returns the ceil log base 2 of the specified value, 32 if the value is z...
Definition: MathExtras.h:481
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
SDValue getValue(unsigned R) const
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant, which is required to be operand #1) half of the integer or float value specified as operand #0.
Definition: ISDOpcodes.h:175
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
LLVMContext * getContext() const
Definition: SelectionDAG.h:289
void dump() const
Dump this node, for debugging.
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:292
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:554
unsigned getNumRegisters(LLVMContext &Context, EVT VT) const
Return the number of registers that this ValueType will eventually require.
static MVT getVectorVT(MVT VT, unsigned NumElements)
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:301
SDValue getVAArg(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input...
AtomicOrdering getSuccessOrdering() const
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:210
void computeKnownBits(SDValue Op, APInt &KnownZero, APInt &KnownOne, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in the KnownZero/KnownO...
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:228
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
const SDValue & getSrc0() const
Type * getTypeForEVT(LLVMContext &Context) const
getTypeForEVT - This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:181
unsigned getSizeInBits() const
SDValue getLoad(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, bool isVolatile, bool isNonTemporal, bool isInvariant, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
SDValue getZeroExtendInReg(SDValue Op, SDLoc DL, EVT SrcTy)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value...
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:679
unsigned getNumOperands() const
Return the number of values used by this operation.
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const SDValue & getOperand(unsigned Num) const
static MachinePointerInfo getConstantPool()
getConstantPool - Return a MachinePointerInfo record that refers to the constant pool.
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:318
CvtCode
CvtCode enum - This enum defines the various converts CONVERT_RNDSAT supports.
Definition: ISDOpcodes.h:885
Same for subtraction.
Definition: ISDOpcodes.h:231
void reserve(size_type N)
Definition: SmallVector.h:401
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
const SDValue & getBasePtr() const
SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align=0, int Offs=0, bool isT=false, unsigned char TargetFlags=0)
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:357
bool bitsLT(EVT VT) const
bitsLT - Return true if this has less bits than VT.
Definition: ValueTypes.h:189
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
SDValue getExternalSymbol(const char *Sym, EVT VT)
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:200
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations...
Definition: ISDOpcodes.h:371
bool isVector() const
isVector - Return true if this is a vector value type.
Definition: ValueTypes.h:115
SDValue getStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, bool isVolatile, bool isNonTemporal, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:308
Shift and rotation operations.
Definition: ISDOpcodes.h:332
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:98
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops, unsigned NumOps, bool isSigned, SDLoc dl, bool doesNotReturn=false, bool isReturnValueUsed=true) const
Returns a pair of (return value, chain).
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:181
MachinePointerInfo getWithOffset(int64_t O) const
EVT getScalarType() const
getScalarType - If this is a vector type, return the element type, otherwise return this...
Definition: ValueTypes.h:210
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt) For double-word atomic operations: ValLo, ValHi, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amtLo, amtHi) ValLo, ValHi, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amtLo, amtHi) These correspond to the atomicrmw instruction.
Definition: ISDOpcodes.h:687
Number of individual test Apply this number of consecutive mutations to each input exit after the first new interesting input is found the minimized corpus is saved into the first input directory Number of jobs to run If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG...
Definition: ISDOpcodes.h:73
This is an SDNode representing atomic operations.
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, SDLoc dl) const
Try to simplify a setcc built with the specified operands and cc.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This class is used to represent an MSTORE node.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getVectorElementType() const
getVectorElementType - Given a vector type, return the type of each element.
Definition: ValueTypes.h:216
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:393
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:351
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:191
SDValue getVectorShuffle(EVT VT, SDLoc dl, SDValue N1, SDValue N2, const int *MaskElts)
Return an ISD::VECTOR_SHUFFLE node.
AtomicOrdering getOrdering() const
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: ArrayRef.h:31
static unsigned int semanticsPrecision(const fltSemantics &)
Definition: APFloat.cpp:846
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:217
const SDValue & getBasePtr() const
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:659
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:804
EVT getMemoryVT() const
Return the type of the in-memory value.
SDValue getAnyExtOrTrunc(SDValue Op, SDLoc DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:284
bool bitsLE(EVT VT) const
bitsLE - Return true if this has no more bits than VT.
Definition: ValueTypes.h:195
UNDEF - An undefined node.
Definition: ISDOpcodes.h:169
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:436
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:262
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
Definition: APInt.h:513
SDNode * getNode() const
get the SDNode which holds the desired result
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:239
unsigned getStoreSize() const
getStoreSize - Return the number of bytes overwritten by a store of the specified value type...
Definition: ValueTypes.h:245
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Float from Float.
Definition: ISDOpcodes.h:887
SynchronizationScope getSynchScope() const
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
Definition: APInt.h:1165
MVT - Machine Value Type.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
Simple binary floating point operators.
Definition: ISDOpcodes.h:237
bool isNonTemporal() const
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
This is an important base class in LLVM.
Definition: Constant.h:41
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:780
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:267
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:219
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
AtomicOrdering getFailureOrdering() const
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:479
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:57
Signed from Signed.
Definition: ISDOpcodes.h:892
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:598
bool isVolatile() const
const SDValue & getValue() const
SDValue getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, bool isVolatile, bool isNonTemporal, bool isInvariant, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes())
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:338
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo...
Definition: ISDOpcodes.h:673
EVT - Extended Value Type.
Definition: ValueTypes.h:31
std::vector< ArgListEntry > ArgListTy
const SDValue & getMask() const
bool bitsEq(EVT VT) const
bitsEq - Return true if this has the same number of bits as VT.
Definition: ValueTypes.h:171
SDValue getMaskedLoad(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType)
This structure contains all information that is necessary for lowering calls.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
PointerType * getPointerTo(unsigned AddrSpace=0)
getPointerTo - Return a pointer to the current type.
Definition: Type.cpp:764
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements)
getVectorVT - Returns the EVT that represents a vector NumElements in length, where each element is o...
Definition: ValueTypes.h:70
MachinePointerInfo - This class contains a discriminated union of information about pointers in memor...
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
SDValue getMaskedStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTrunc)
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
const MachinePointerInfo & getPointerInfo() const
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
ArrayRef< int > getMask() const
bool ugt(const APInt &RHS) const
Unsigned greather than comparison.
Definition: APInt.h:1101
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:50
SDValue getConvertRndSat(EVT VT, SDLoc dl, SDValue Val, SDValue DTy, SDValue STy, SDValue Rnd, SDValue Sat, ISD::CvtCode Code)
Returns the ConvertRndSat Note: Avoid using this node because it may disappear in the future and most...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:273
bool isInvariant() const
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:548
Signed from Float.
Definition: ISDOpcodes.h:890
const SDValue & getChain() const
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:548
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:335
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:500
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Unsigned from Signed.
Definition: ISDOpcodes.h:894
SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, const Value *PtrVal, unsigned Alignment, AtomicOrdering Ordering, SynchronizationScope SynchScope)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:582
Represents one node in the SelectionDAG.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:123
unsigned Log2_32(uint32_t Value)
Log2_32 - This function returns the floor log base 2 of the specified value, -1 if the value is zero...
Definition: MathExtras.h:468
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:576
const SDValue & getValue() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Class for arbitrary precision integers.
Definition: APInt.h:73
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:342
op_iterator op_begin() const
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:383
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:285
SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:386
iterator_range< value_op_iterator > op_values() const
uint64_t MinAlign(uint64_t A, uint64_t B)
MinAlign - A and B are either alignments or offsets.
Definition: MathExtras.h:552
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:697
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:321
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:401
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:518
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
Definition: ValueTypes.h:233
#define N
op_iterator op_end() const
Same for multiplication.
Definition: ISDOpcodes.h:234
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
EVT getValueType() const
Return the ValueType of the referenced return value.
SDValue getCondCode(ISD::CondCode Cond)
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
SDValue getConstant(uint64_t Val, SDLoc DL, EVT VT, bool isTarget=false, bool isOpaque=false)
CONVERT_RNDSAT - This operator is used to support various conversions between various types (float...
Definition: ISDOpcodes.h:494
Libcall getATOMIC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none...
bool isByteSized() const
isByteSized - Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:160
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:279
SDValue getSelect(SDLoc DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
Definition: SelectionDAG.h:739
This class is used to represent an MLOAD node.
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
SDValue getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const SDValue & getBasePtr() const
SDValue getValueType(EVT)
#define DEBUG(X)
Definition: Debug.h:92
bool isPowerOf2_32(uint32_t Value)
isPowerOf2_32 - This function returns true if the argument is a power of two > 0. ...
Definition: MathExtras.h:354
Signed from Unsigned.
Definition: ISDOpcodes.h:893
Float from Signed.
Definition: ISDOpcodes.h:888
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:365
SDValue getSetCC(SDLoc DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:726
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:188
static bool isVolatile(Instruction *Inst)
Conversion operators.
Definition: ISDOpcodes.h:380
BooleanContent
Enum that describes how the target represents true/false values.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:338
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:666
Unsigned from Float.
Definition: ISDOpcodes.h:891
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:389
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
unsigned getAlignment() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
bool isBigEndian() const
Definition: DataLayout.h:218
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
getIntegerVT - Returns the EVT that represents an integer with the given number of bits...
Definition: ValueTypes.h:61
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
SDValue getAtomicCmpSwap(unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, unsigned Alignment, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SynchronizationScope SynchScope)
Gets a node for an atomic cmpxchg op.
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:662
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:309
Float from Unsigned.
Definition: ISDOpcodes.h:889
MVT getSimpleVT() const
getSimpleVT - Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:203
SDValue getIntPtrConstant(uint64_t Val, SDLoc DL, bool isTarget=false)
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:225
This class is used to represent ISD::LOAD nodes.