LLVM  7.0.0svn
LegalizeIntegerTypes.cpp
Go to the documentation of this file.
1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements integer type expansion and promotion for LegalizeTypes.
11 // Promotion is the act of changing a computation in an illegal type into a
12 // computation in a larger type. For example, implementing i8 arithmetic in an
13 // i32 register (often needed on powerpc).
14 // Expansion is the act of changing a computation in an illegal type into a
15 // computation in two identical registers of a smaller type. For example,
16 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
17 // targets).
18 //
19 //===----------------------------------------------------------------------===//
20 
21 #include "LegalizeTypes.h"
22 #include "llvm/IR/DerivedTypes.h"
24 #include "llvm/Support/KnownBits.h"
26 using namespace llvm;
27 
28 #define DEBUG_TYPE "legalize-types"
29 
30 //===----------------------------------------------------------------------===//
31 // Integer Result Promotion
32 //===----------------------------------------------------------------------===//
33 
34 /// PromoteIntegerResult - This method is called when a result of a node is
35 /// found to be in need of promotion to a larger type. At this point, the node
36 /// may also have invalid operands or may have other results that need
37 /// expansion, we just know that (at least) one result needs promotion.
38 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
39  LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG);
40  dbgs() << "\n");
41  SDValue Res = SDValue();
42 
43  // See if the target wants to custom expand this node.
44  if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
45  LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
46  return;
47  }
48 
49  switch (N->getOpcode()) {
50  default:
51 #ifndef NDEBUG
52  dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
53  N->dump(&DAG); dbgs() << "\n";
54 #endif
55  llvm_unreachable("Do not know how to promote this operator!");
56  case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
57  case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
58  case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
59  case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
60  case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
61  case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
62  case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
63  case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
65  case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
66  case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break;
68  case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
70  Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
71  case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
72  case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
73  break;
74  case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
75  break;
76  case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
77  case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
78  case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
79  case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
80  case ISD::SMIN:
81  case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
82  case ISD::UMIN:
83  case ISD::UMAX: Res = PromoteIntRes_ZExtIntBinOp(N); break;
84 
85  case ISD::SHL: Res = PromoteIntRes_SHL(N); break;
87  Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
88  case ISD::SRA: Res = PromoteIntRes_SRA(N); break;
89  case ISD::SRL: Res = PromoteIntRes_SRL(N); break;
90  case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
91  case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
92  case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
93 
95  Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
97  Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
99  Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
100  case ISD::BUILD_VECTOR:
101  Res = PromoteIntRes_BUILD_VECTOR(N); break;
103  Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
104  case ISD::CONCAT_VECTORS:
105  Res = PromoteIntRes_CONCAT_VECTORS(N); break;
106 
110  Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
111 
112  case ISD::SIGN_EXTEND:
113  case ISD::ZERO_EXTEND:
114  case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
115 
116  case ISD::FP_TO_SINT:
117  case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
118 
119  case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
120 
121  case ISD::AND:
122  case ISD::OR:
123  case ISD::XOR:
124  case ISD::ADD:
125  case ISD::SUB:
126  case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
127 
128  case ISD::SDIV:
129  case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
130 
131  case ISD::UDIV:
132  case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
133 
134  case ISD::SADDO:
135  case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
136  case ISD::UADDO:
137  case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
138  case ISD::SMULO:
139  case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
140 
141  case ISD::ADDCARRY:
142  case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
143 
144  case ISD::ATOMIC_LOAD:
145  Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
146 
151  case ISD::ATOMIC_LOAD_OR:
158  case ISD::ATOMIC_SWAP:
159  Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
160 
163  Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
164  break;
165  }
166 
167  // If the result is null then the sub-method took care of registering it.
168  if (Res.getNode())
169  SetPromotedInteger(SDValue(N, ResNo), Res);
170 }
171 
172 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
173  unsigned ResNo) {
174  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
175  return GetPromotedInteger(Op);
176 }
177 
178 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
179  // Sign-extend the new bits, and continue the assertion.
180  SDValue Op = SExtPromotedInteger(N->getOperand(0));
181  return DAG.getNode(ISD::AssertSext, SDLoc(N),
182  Op.getValueType(), Op, N->getOperand(1));
183 }
184 
185 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
186  // Zero the new bits, and continue the assertion.
187  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
188  return DAG.getNode(ISD::AssertZext, SDLoc(N),
189  Op.getValueType(), Op, N->getOperand(1));
190 }
191 
192 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
193  EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
194  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
195  N->getMemoryVT(), ResVT,
196  N->getChain(), N->getBasePtr(),
197  N->getMemOperand());
198  // Legalize the chain result - switch anything that used the old chain to
199  // use the new one.
200  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
201  return Res;
202 }
203 
204 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
205  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
206  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
207  N->getMemoryVT(),
208  N->getChain(), N->getBasePtr(),
209  Op2, N->getMemOperand());
210  // Legalize the chain result - switch anything that used the old chain to
211  // use the new one.
212  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
213  return Res;
214 }
215 
216 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
217  unsigned ResNo) {
218  if (ResNo == 1) {
220  EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
221  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
222 
223  // Only use the result of getSetCCResultType if it is legal,
224  // otherwise just use the promoted result type (NVT).
225  if (!TLI.isTypeLegal(SVT))
226  SVT = NVT;
227 
228  SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
229  SDValue Res = DAG.getAtomicCmpSwap(
231  N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
232  N->getMemOperand());
233  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
234  ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
235  return Res.getValue(1);
236  }
237 
238  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
239  SDValue Op3 = GetPromotedInteger(N->getOperand(3));
240  SDVTList VTs =
241  DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
242  SDValue Res = DAG.getAtomicCmpSwap(
243  N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
244  N->getBasePtr(), Op2, Op3, N->getMemOperand());
245  // Update the use to N with the newly created Res.
246  for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
247  ReplaceValueWith(SDValue(N, i), Res.getValue(i));
248  return Res;
249 }
250 
251 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
252  SDValue InOp = N->getOperand(0);
253  EVT InVT = InOp.getValueType();
254  EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
255  EVT OutVT = N->getValueType(0);
256  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
257  SDLoc dl(N);
258 
259  switch (getTypeAction(InVT)) {
261  break;
263  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
264  // The input promotes to the same size. Convert the promoted value.
265  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
266  break;
268  // Promote the integer operand by hand.
269  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
271  // Convert the promoted float by hand.
272  SDValue PromotedOp = GetPromotedFloat(InOp);
273  return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, PromotedOp);
274  break;
275  }
278  break;
280  // Convert the element to an integer and promote it by hand.
281  if (!NOutVT.isVector())
282  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
283  BitConvertToInteger(GetScalarizedVector(InOp)));
284  break;
286  // For example, i32 = BITCAST v2i16 on alpha. Convert the split
287  // pieces of the input into integers and reassemble in the final type.
288  SDValue Lo, Hi;
289  GetSplitVector(N->getOperand(0), Lo, Hi);
290  Lo = BitConvertToInteger(Lo);
291  Hi = BitConvertToInteger(Hi);
292 
293  if (DAG.getDataLayout().isBigEndian())
294  std::swap(Lo, Hi);
295 
296  InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
298  NOutVT.getSizeInBits()),
299  JoinIntegers(Lo, Hi));
300  return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
301  }
303  // The input is widened to the same size. Convert to the widened value.
304  // Make sure that the outgoing value is not a vector, because this would
305  // make us bitcast between two vectors which are legalized in different ways.
306  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector())
307  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
308  }
309 
310  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
311  CreateStackStoreLoad(InOp, OutVT));
312 }
313 
314 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
315  SDValue Op = GetPromotedInteger(N->getOperand(0));
316  EVT OVT = N->getValueType(0);
317  EVT NVT = Op.getValueType();
318  SDLoc dl(N);
319 
320  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
321  return DAG.getNode(
322  ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
323  DAG.getConstant(DiffBits, dl,
324  TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
325 }
326 
327 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
328  SDValue Op = GetPromotedInteger(N->getOperand(0));
329  EVT OVT = N->getValueType(0);
330  EVT NVT = Op.getValueType();
331  SDLoc dl(N);
332 
333  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
334  return DAG.getNode(
335  ISD::SRL, dl, NVT, DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
336  DAG.getConstant(DiffBits, dl,
337  TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
338 }
339 
340 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
341  // The pair element type may be legal, or may not promote to the same type as
342  // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
343  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
344  TLI.getTypeToTransformTo(*DAG.getContext(),
345  N->getValueType(0)), JoinIntegers(N->getOperand(0),
346  N->getOperand(1)));
347 }
348 
349 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
350  EVT VT = N->getValueType(0);
351  // FIXME there is no actual debug info here
352  SDLoc dl(N);
353  // Zero extend things like i1, sign extend everything else. It shouldn't
354  // matter in theory which one we pick, but this tends to give better code?
355  unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
356  SDValue Result = DAG.getNode(Opc, dl,
357  TLI.getTypeToTransformTo(*DAG.getContext(), VT),
358  SDValue(N, 0));
359  assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
360  return Result;
361 }
362 
363 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
364  // Zero extend to the promoted type and do the count there.
365  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
366  SDLoc dl(N);
367  EVT OVT = N->getValueType(0);
368  EVT NVT = Op.getValueType();
369  Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
370  // Subtract off the extra leading bits in the bigger type.
371  return DAG.getNode(
372  ISD::SUB, dl, NVT, Op,
373  DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
374  NVT));
375 }
376 
377 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
378  // Zero extend to the promoted type and do the count there.
379  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
380  return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op);
381 }
382 
383 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
384  SDValue Op = GetPromotedInteger(N->getOperand(0));
385  EVT OVT = N->getValueType(0);
386  EVT NVT = Op.getValueType();
387  SDLoc dl(N);
388  if (N->getOpcode() == ISD::CTTZ) {
389  // The count is the same in the promoted type except if the original
390  // value was zero. This can be handled by setting the bit just off
391  // the top of the original type.
392  auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
393  OVT.getScalarSizeInBits());
394  Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
395  }
396  return DAG.getNode(N->getOpcode(), dl, NVT, Op);
397 }
398 
399 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
400  SDLoc dl(N);
401  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
402  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0),
403  N->getOperand(1));
404 }
405 
406 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
407  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
408  unsigned NewOpc = N->getOpcode();
409  SDLoc dl(N);
410 
411  // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
412  // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
413  // and SINT conversions are Custom, there is no way to tell which is
414  // preferable. We choose SINT because that's the right thing on PPC.)
415  if (N->getOpcode() == ISD::FP_TO_UINT &&
416  !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
418  NewOpc = ISD::FP_TO_SINT;
419 
420  SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
421 
422  // Assert that the converted value fits in the original type. If it doesn't
423  // (eg: because the value being converted is too big), then the result of the
424  // original operation was undefined anyway, so the assert is still correct.
425  //
426  // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
427  // before legalization: fp-to-uint16, 65534. -> 0xfffe
428  // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
429  return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ?
430  ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
432 }
433 
434 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
435  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
436  SDLoc dl(N);
437 
438  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
439 }
440 
441 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
442  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
443  SDLoc dl(N);
444 
445  if (getTypeAction(N->getOperand(0).getValueType())
447  SDValue Res = GetPromotedInteger(N->getOperand(0));
448  assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
449 
450  // If the result and operand types are the same after promotion, simplify
451  // to an in-register extension.
452  if (NVT == Res.getValueType()) {
453  // The high bits are not guaranteed to be anything. Insert an extend.
454  if (N->getOpcode() == ISD::SIGN_EXTEND)
455  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
456  DAG.getValueType(N->getOperand(0).getValueType()));
457  if (N->getOpcode() == ISD::ZERO_EXTEND)
458  return DAG.getZeroExtendInReg(Res, dl,
460  assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
461  return Res;
462  }
463  }
464 
465  // Otherwise, just extend the original operand all the way to the larger type.
466  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
467 }
468 
469 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
470  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
471  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
472  ISD::LoadExtType ExtType =
474  SDLoc dl(N);
475  SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
476  N->getMemoryVT(), N->getMemOperand());
477 
478  // Legalize the chain result - switch anything that used the old chain to
479  // use the new one.
480  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
481  return Res;
482 }
483 
484 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
485  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
486  SDValue ExtSrc0 = GetPromotedInteger(N->getSrc0());
487 
488  SDLoc dl(N);
489  SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
490  N->getMask(), ExtSrc0, N->getMemoryVT(),
492  // Legalize the chain result - switch anything that used the old chain to
493  // use the new one.
494  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
495  return Res;
496 }
497 
498 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
499  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
500  SDValue ExtSrc0 = GetPromotedInteger(N->getValue());
501  assert(NVT == ExtSrc0.getValueType() &&
502  "Gather result type and the passThru agrument type should be the same");
503 
504  SDLoc dl(N);
505  SDValue Ops[] = {N->getChain(), ExtSrc0, N->getMask(), N->getBasePtr(),
506  N->getIndex(), N->getScale() };
507  SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
508  N->getMemoryVT(), dl, Ops,
509  N->getMemOperand());
510  // Legalize the chain result - switch anything that used the old chain to
511  // use the new one.
512  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
513  return Res;
514 }
515 
516 /// Promote the overflow flag of an overflowing arithmetic node.
517 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
518  // Simply change the return type of the boolean result.
519  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
520  EVT ValueVTs[] = { N->getValueType(0), NVT };
521  SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
522  unsigned NumOps = N->getNumOperands();
523  assert(NumOps <= 3 && "Too many operands");
524  if (NumOps == 3)
525  Ops[2] = N->getOperand(2);
526 
527  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
528  DAG.getVTList(ValueVTs), makeArrayRef(Ops, NumOps));
529 
530  // Modified the sum result - switch anything that used the old sum to use
531  // the new one.
532  ReplaceValueWith(SDValue(N, 0), Res);
533 
534  return SDValue(Res.getNode(), 1);
535 }
536 
537 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
538  if (ResNo == 1)
539  return PromoteIntRes_Overflow(N);
540 
541  // The operation overflowed iff the result in the larger type is not the
542  // sign extension of its truncation to the original type.
543  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
544  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
545  EVT OVT = N->getOperand(0).getValueType();
546  EVT NVT = LHS.getValueType();
547  SDLoc dl(N);
548 
549  // Do the arithmetic in the larger type.
550  unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
551  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
552 
553  // Calculate the overflow flag: sign extend the arithmetic result from
554  // the original type.
555  SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
556  DAG.getValueType(OVT));
557  // Overflowed if and only if this is not equal to Res.
558  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
559 
560  // Use the calculated overflow everywhere.
561  ReplaceValueWith(SDValue(N, 1), Ofl);
562 
563  return Res;
564 }
565 
566 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
567  SDValue LHS = GetPromotedInteger(N->getOperand(1));
568  SDValue RHS = GetPromotedInteger(N->getOperand(2));
569  return DAG.getSelect(SDLoc(N),
570  LHS.getValueType(), N->getOperand(0), LHS, RHS);
571 }
572 
573 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
574  SDValue Mask = N->getOperand(0);
575 
576  SDValue LHS = GetPromotedInteger(N->getOperand(1));
577  SDValue RHS = GetPromotedInteger(N->getOperand(2));
578  return DAG.getNode(ISD::VSELECT, SDLoc(N),
579  LHS.getValueType(), Mask, LHS, RHS);
580 }
581 
582 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
583  SDValue LHS = GetPromotedInteger(N->getOperand(2));
584  SDValue RHS = GetPromotedInteger(N->getOperand(3));
585  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
586  LHS.getValueType(), N->getOperand(0),
587  N->getOperand(1), LHS, RHS, N->getOperand(4));
588 }
589 
590 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
591  EVT InVT = N->getOperand(0).getValueType();
592  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
593 
594  EVT SVT = getSetCCResultType(InVT);
595 
596  // If we got back a type that needs to be promoted, this likely means the
597  // the input type also needs to be promoted. So get the promoted type for
598  // the input and try the query again.
599  if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
600  if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
601  InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
602  SVT = getSetCCResultType(InVT);
603  } else {
604  // Input type isn't promoted, just use the default promoted type.
605  SVT = NVT;
606  }
607  }
608 
609  SDLoc dl(N);
610  assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() &&
611  "Vector compare must return a vector result!");
612 
613  // Get the SETCC result using the canonical SETCC type.
614  SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
615  N->getOperand(1), N->getOperand(2));
616 
617  // Convert to the expected type.
618  return DAG.getSExtOrTrunc(SetCC, dl, NVT);
619 }
620 
621 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
622  SDValue LHS = GetPromotedInteger(N->getOperand(0));
623  SDValue RHS = N->getOperand(1);
624  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
625  RHS = ZExtPromotedInteger(RHS);
626  return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
627 }
628 
629 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
630  SDValue Op = GetPromotedInteger(N->getOperand(0));
631  return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
632  Op.getValueType(), Op, N->getOperand(1));
633 }
634 
635 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
636  // The input may have strange things in the top bits of the registers, but
637  // these operations don't care. They may have weird bits going out, but
638  // that too is okay if they are integer operations.
639  SDValue LHS = GetPromotedInteger(N->getOperand(0));
640  SDValue RHS = GetPromotedInteger(N->getOperand(1));
641  return DAG.getNode(N->getOpcode(), SDLoc(N),
642  LHS.getValueType(), LHS, RHS);
643 }
644 
645 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
646  // Sign extend the input.
647  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
648  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
649  return DAG.getNode(N->getOpcode(), SDLoc(N),
650  LHS.getValueType(), LHS, RHS);
651 }
652 
653 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
654  // Zero extend the input.
655  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
656  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
657  return DAG.getNode(N->getOpcode(), SDLoc(N),
658  LHS.getValueType(), LHS, RHS);
659 }
660 
661 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
662  // The input value must be properly sign extended.
663  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
664  SDValue RHS = N->getOperand(1);
665  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
666  RHS = ZExtPromotedInteger(RHS);
667  return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
668 }
669 
670 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
671  // The input value must be properly zero extended.
672  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
673  SDValue RHS = N->getOperand(1);
674  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
675  RHS = ZExtPromotedInteger(RHS);
676  return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
677 }
678 
679 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
680  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
681  SDValue Res;
682  SDValue InOp = N->getOperand(0);
683  SDLoc dl(N);
684 
685  switch (getTypeAction(InOp.getValueType())) {
686  default: llvm_unreachable("Unknown type action!");
689  Res = InOp;
690  break;
692  Res = GetPromotedInteger(InOp);
693  break;
695  EVT InVT = InOp.getValueType();
696  assert(InVT.isVector() && "Cannot split scalar types");
697  unsigned NumElts = InVT.getVectorNumElements();
698  assert(NumElts == NVT.getVectorNumElements() &&
699  "Dst and Src must have the same number of elements");
700  assert(isPowerOf2_32(NumElts) &&
701  "Promoted vector type must be a power of two");
702 
703  SDValue EOp1, EOp2;
704  GetSplitVector(InOp, EOp1, EOp2);
705 
706  EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
707  NumElts/2);
708  EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
709  EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
710 
711  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
712  }
714  SDValue WideInOp = GetWidenedVector(InOp);
715 
716  // Truncate widened InOp.
717  unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
718  EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
719  N->getValueType(0).getScalarType(), NumElem);
720  SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
721 
722  // Zero extend so that the elements are of same type as those of NVT
723  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
724  NumElem);
725  SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
726 
727  // Extract the low NVT subvector.
728  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
729  SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy);
730  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
731  }
732  }
733 
734  // Truncate to NVT instead of VT
735  return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
736 }
737 
738 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
739  if (ResNo == 1)
740  return PromoteIntRes_Overflow(N);
741 
742  // The operation overflowed iff the result in the larger type is not the
743  // zero extension of its truncation to the original type.
744  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
745  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
746  EVT OVT = N->getOperand(0).getValueType();
747  EVT NVT = LHS.getValueType();
748  SDLoc dl(N);
749 
750  // Do the arithmetic in the larger type.
751  unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
752  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
753 
754  // Calculate the overflow flag: zero extend the arithmetic result from
755  // the original type.
756  SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
757  // Overflowed if and only if this is not equal to Res.
758  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
759 
760  // Use the calculated overflow everywhere.
761  ReplaceValueWith(SDValue(N, 1), Ofl);
762 
763  return Res;
764 }
765 
766 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
767  if (ResNo == 1)
768  return PromoteIntRes_Overflow(N);
769 
770  // We need to sign-extend the operands so the carry value computed by the
771  // wide operation will be equivalent to the carry value computed by the
772  // narrow operation.
773  // An ADDCARRY can generate carry only if any of the operands has its
774  // most significant bit set. Sign extension propagates the most significant
775  // bit into the higher bits which means the extra bit that the narrow
776  // addition would need (i.e. the carry) will be propagated through the higher
777  // bits of the wide addition.
778  // A SUBCARRY can generate borrow only if LHS < RHS and this property will be
779  // preserved by sign extension.
780  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
781  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
782 
783  EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
784 
785  // Do the arithmetic in the wide type.
786  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
787  LHS, RHS, N->getOperand(2));
788 
789  // Update the users of the original carry/borrow value.
790  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
791 
792  return SDValue(Res.getNode(), 0);
793 }
794 
795 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
796  // Promote the overflow bit trivially.
797  if (ResNo == 1)
798  return PromoteIntRes_Overflow(N);
799 
800  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
801  SDLoc DL(N);
802  EVT SmallVT = LHS.getValueType();
803 
804  // To determine if the result overflowed in a larger type, we extend the
805  // input to the larger type, do the multiply (checking if it overflows),
806  // then also check the high bits of the result to see if overflow happened
807  // there.
808  if (N->getOpcode() == ISD::SMULO) {
809  LHS = SExtPromotedInteger(LHS);
810  RHS = SExtPromotedInteger(RHS);
811  } else {
812  LHS = ZExtPromotedInteger(LHS);
813  RHS = ZExtPromotedInteger(RHS);
814  }
815  SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
816  SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
817 
818  // Overflow occurred if it occurred in the larger type, or if the high part
819  // of the result does not zero/sign-extend the low part. Check this second
820  // possibility first.
821  SDValue Overflow;
822  if (N->getOpcode() == ISD::UMULO) {
823  // Unsigned overflow occurred if the high part is non-zero.
824  SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
825  DAG.getIntPtrConstant(SmallVT.getSizeInBits(),
826  DL));
827  Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
828  DAG.getConstant(0, DL, Hi.getValueType()),
829  ISD::SETNE);
830  } else {
831  // Signed overflow occurred if the high part does not sign extend the low.
832  SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
833  Mul, DAG.getValueType(SmallVT));
834  Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
835  }
836 
837  // The only other way for overflow to occur is if the multiplication in the
838  // larger type itself overflowed.
839  Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
840  SDValue(Mul.getNode(), 1));
841 
842  // Use the calculated overflow everywhere.
843  ReplaceValueWith(SDValue(N, 1), Overflow);
844  return Mul;
845 }
846 
847 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
848  return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
849  N->getValueType(0)));
850 }
851 
852 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
853  SDValue Chain = N->getOperand(0); // Get the chain.
854  SDValue Ptr = N->getOperand(1); // Get the pointer.
855  EVT VT = N->getValueType(0);
856  SDLoc dl(N);
857 
858  MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
859  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
860  // The argument is passed as NumRegs registers of type RegVT.
861 
862  SmallVector<SDValue, 8> Parts(NumRegs);
863  for (unsigned i = 0; i < NumRegs; ++i) {
864  Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
865  N->getConstantOperandVal(3));
866  Chain = Parts[i].getValue(1);
867  }
868 
869  // Handle endianness of the load.
870  if (DAG.getDataLayout().isBigEndian())
871  std::reverse(Parts.begin(), Parts.end());
872 
873  // Assemble the parts in the promoted type.
874  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
875  SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
876  for (unsigned i = 1; i < NumRegs; ++i) {
877  SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
878  // Shift it to the right position and "or" it in.
879  Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
880  DAG.getConstant(i * RegVT.getSizeInBits(), dl,
881  TLI.getPointerTy(DAG.getDataLayout())));
882  Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
883  }
884 
885  // Modified the chain result - switch anything that used the old chain to
886  // use the new one.
887  ReplaceValueWith(SDValue(N, 1), Chain);
888 
889  return Res;
890 }
891 
892 //===----------------------------------------------------------------------===//
893 // Integer Operand Promotion
894 //===----------------------------------------------------------------------===//
895 
896 /// PromoteIntegerOperand - This method is called when the specified operand of
897 /// the specified node is found to need promotion. At this point, all of the
898 /// result types of the node are known to be legal, but other operands of the
899 /// node may need promotion or expansion as well as the specified one.
900 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
901  LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG);
902  dbgs() << "\n");
903  SDValue Res = SDValue();
904 
905  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
906  LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
907  return false;
908  }
909 
910  switch (N->getOpcode()) {
911  default:
912  #ifndef NDEBUG
913  dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
914  N->dump(&DAG); dbgs() << "\n";
915  #endif
916  llvm_unreachable("Do not know how to promote this operator's operand!");
917 
918  case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
919  case ISD::ATOMIC_STORE:
920  Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
921  break;
922  case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
923  case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
924  case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
925  case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
926  case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
927  case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
928  case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
930  Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
932  Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
933  case ISD::VSELECT:
934  case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
935  case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
936  case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
937  case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
938  case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
939  case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
940  OpNo); break;
941  case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
942  OpNo); break;
943  case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
944  OpNo); break;
945  case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
946  OpNo); break;
947  case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
948  OpNo); break;
949  case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
950  case ISD::FP16_TO_FP:
951  case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
952  case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
953  case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
954 
955  case ISD::SHL:
956  case ISD::SRA:
957  case ISD::SRL:
958  case ISD::ROTL:
959  case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
960 
961  case ISD::ADDCARRY:
962  case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
963  }
964 
965  // If the result is null, the sub-method took care of registering results etc.
966  if (!Res.getNode()) return false;
967 
968  // If the result is N, the sub-method updated N in place. Tell the legalizer
969  // core about this.
970  if (Res.getNode() == N)
971  return true;
972 
973  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
974  "Invalid operand expansion");
975 
976  ReplaceValueWith(SDValue(N, 0), Res);
977  return false;
978 }
979 
980 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
981 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
982 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
983  ISD::CondCode CCCode) {
984  // We have to insert explicit sign or zero extends. Note that we could
985  // insert sign extends for ALL conditions, but zero extend is cheaper on
986  // many machines (an AND instead of two shifts), so prefer it.
987  switch (CCCode) {
988  default: llvm_unreachable("Unknown integer comparison!");
989  case ISD::SETEQ:
990  case ISD::SETNE: {
991  SDValue OpL = GetPromotedInteger(NewLHS);
992  SDValue OpR = GetPromotedInteger(NewRHS);
993 
994  // We would prefer to promote the comparison operand with sign extension.
995  // If the width of OpL/OpR excluding the duplicated sign bits is no greater
996  // than the width of NewLHS/NewRH, we can avoid inserting real truncate
997  // instruction, which is redudant eventually.
998  unsigned OpLEffectiveBits =
999  OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
1000  unsigned OpREffectiveBits =
1001  OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
1002  if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() &&
1003  OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()) {
1004  NewLHS = OpL;
1005  NewRHS = OpR;
1006  } else {
1007  NewLHS = ZExtPromotedInteger(NewLHS);
1008  NewRHS = ZExtPromotedInteger(NewRHS);
1009  }
1010  break;
1011  }
1012  case ISD::SETUGE:
1013  case ISD::SETUGT:
1014  case ISD::SETULE:
1015  case ISD::SETULT:
1016  // ALL of these operations will work if we either sign or zero extend
1017  // the operands (including the unsigned comparisons!). Zero extend is
1018  // usually a simpler/cheaper operation, so prefer it.
1019  NewLHS = ZExtPromotedInteger(NewLHS);
1020  NewRHS = ZExtPromotedInteger(NewRHS);
1021  break;
1022  case ISD::SETGE:
1023  case ISD::SETGT:
1024  case ISD::SETLT:
1025  case ISD::SETLE:
1026  NewLHS = SExtPromotedInteger(NewLHS);
1027  NewRHS = SExtPromotedInteger(NewRHS);
1028  break;
1029  }
1030 }
1031 
1032 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1033  SDValue Op = GetPromotedInteger(N->getOperand(0));
1034  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1035 }
1036 
1037 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1038  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1039  return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1040  N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1041 }
1042 
1043 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1044  // This should only occur in unusual situations like bitcasting to an
1045  // x86_fp80, so just turn it into a store+load
1046  return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1047 }
1048 
1049 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1050  assert(OpNo == 2 && "Don't know how to promote this operand!");
1051 
1052  SDValue LHS = N->getOperand(2);
1053  SDValue RHS = N->getOperand(3);
1054  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1055 
1056  // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1057  // legal types.
1058  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1059  N->getOperand(1), LHS, RHS, N->getOperand(4)),
1060  0);
1061 }
1062 
1063 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1064  assert(OpNo == 1 && "only know how to promote condition");
1065 
1066  // Promote all the way up to the canonical SetCC type.
1067  SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1068 
1069  // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1070  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1071  N->getOperand(2)), 0);
1072 }
1073 
1074 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1075  // Since the result type is legal, the operands must promote to it.
1076  EVT OVT = N->getOperand(0).getValueType();
1077  SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1078  SDValue Hi = GetPromotedInteger(N->getOperand(1));
1079  assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1080  SDLoc dl(N);
1081 
1082  Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1083  DAG.getConstant(OVT.getSizeInBits(), dl,
1084  TLI.getPointerTy(DAG.getDataLayout())));
1085  return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1086 }
1087 
1088 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1089  // The vector type is legal but the element type is not. This implies
1090  // that the vector is a power-of-two in length and that the element
1091  // type does not have a strange size (eg: it is not i1).
1092  EVT VecVT = N->getValueType(0);
1093  unsigned NumElts = VecVT.getVectorNumElements();
1094  assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1095  "Legal vector of one illegal element?");
1096 
1097  // Promote the inserted value. The type does not need to match the
1098  // vector element type. Check that any extra bits introduced will be
1099  // truncated away.
1102  "Type of inserted value narrower than vector element type!");
1103 
1104  SmallVector<SDValue, 16> NewOps;
1105  for (unsigned i = 0; i < NumElts; ++i)
1106  NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1107 
1108  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1109 }
1110 
1111 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1112  unsigned OpNo) {
1113  if (OpNo == 1) {
1114  // Promote the inserted value. This is valid because the type does not
1115  // have to match the vector element type.
1116 
1117  // Check that any extra bits introduced will be truncated away.
1120  "Type of inserted value narrower than vector element type!");
1121  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1122  GetPromotedInteger(N->getOperand(1)),
1123  N->getOperand(2)),
1124  0);
1125  }
1126 
1127  assert(OpNo == 2 && "Different operand and result vector types?");
1128 
1129  // Promote the index.
1130  SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1131  TLI.getVectorIdxTy(DAG.getDataLayout()));
1132  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1133  N->getOperand(1), Idx), 0);
1134 }
1135 
1136 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1137  // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1138  // the operand in place.
1139  return SDValue(DAG.UpdateNodeOperands(N,
1140  GetPromotedInteger(N->getOperand(0))), 0);
1141 }
1142 
1143 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1144  assert(OpNo == 0 && "Only know how to promote the condition!");
1145  SDValue Cond = N->getOperand(0);
1146  EVT OpTy = N->getOperand(1).getValueType();
1147 
1148  if (N->getOpcode() == ISD::VSELECT)
1149  if (SDValue Res = WidenVSELECTAndMask(N))
1150  return Res;
1151 
1152  // Promote all the way up to the canonical SetCC type.
1153  EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1154  Cond = PromoteTargetBoolean(Cond, OpVT);
1155 
1156  return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1157  N->getOperand(2)), 0);
1158 }
1159 
1160 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1161  assert(OpNo == 0 && "Don't know how to promote this operand!");
1162 
1163  SDValue LHS = N->getOperand(0);
1164  SDValue RHS = N->getOperand(1);
1165  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1166 
1167  // The CC (#4) and the possible return values (#2 and #3) have legal types.
1168  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1169  N->getOperand(3), N->getOperand(4)), 0);
1170 }
1171 
1172 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1173  assert(OpNo == 0 && "Don't know how to promote this operand!");
1174 
1175  SDValue LHS = N->getOperand(0);
1176  SDValue RHS = N->getOperand(1);
1177  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1178 
1179  // The CC (#2) is always legal.
1180  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1181 }
1182 
1183 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1184  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1185  ZExtPromotedInteger(N->getOperand(1))), 0);
1186 }
1187 
1188 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1189  SDValue Op = GetPromotedInteger(N->getOperand(0));
1190  SDLoc dl(N);
1191  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1192  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1193  Op, DAG.getValueType(N->getOperand(0).getValueType()));
1194 }
1195 
1196 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1197  return SDValue(DAG.UpdateNodeOperands(N,
1198  SExtPromotedInteger(N->getOperand(0))), 0);
1199 }
1200 
1201 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1202  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1203  SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1204  SDLoc dl(N);
1205 
1206  SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1207 
1208  // Truncate the value and store the result.
1209  return DAG.getTruncStore(Ch, dl, Val, Ptr,
1210  N->getMemoryVT(), N->getMemOperand());
1211 }
1212 
1213 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1214  unsigned OpNo) {
1215 
1216  SDValue DataOp = N->getValue();
1217  EVT DataVT = DataOp.getValueType();
1218  SDValue Mask = N->getMask();
1219  SDLoc dl(N);
1220 
1221  bool TruncateStore = false;
1222  if (OpNo == 2) {
1223  // Mask comes before the data operand. If the data operand is legal, we just
1224  // promote the mask.
1225  // When the data operand has illegal type, we should legalize the data
1226  // operand first. The mask will be promoted/splitted/widened according to
1227  // the data operand type.
1228  if (TLI.isTypeLegal(DataVT)) {
1229  Mask = PromoteTargetBoolean(Mask, DataVT);
1230  // Update in place.
1231  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1232  NewOps[2] = Mask;
1233  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1234  }
1235 
1236  if (getTypeAction(DataVT) == TargetLowering::TypePromoteInteger)
1237  return PromoteIntOp_MSTORE(N, 3);
1238  if (getTypeAction(DataVT) == TargetLowering::TypeWidenVector)
1239  return WidenVecOp_MSTORE(N, 3);
1240  assert (getTypeAction(DataVT) == TargetLowering::TypeSplitVector);
1241  return SplitVecOp_MSTORE(N, 3);
1242  } else { // Data operand
1243  assert(OpNo == 3 && "Unexpected operand for promotion");
1244  DataOp = GetPromotedInteger(DataOp);
1245  TruncateStore = true;
1246  }
1247 
1248  return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask,
1249  N->getMemoryVT(), N->getMemOperand(),
1250  TruncateStore, N->isCompressingStore());
1251 }
1252 
1253 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1254  unsigned OpNo) {
1255  assert(OpNo == 2 && "Only know how to promote the mask!");
1256  EVT DataVT = N->getValueType(0);
1257  SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1258  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1259  NewOps[OpNo] = Mask;
1260  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1261 }
1262 
1263 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1264  unsigned OpNo) {
1265 
1266  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1267  if (OpNo == 2) {
1268  // The Mask
1269  EVT DataVT = N->getValueType(0);
1270  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1271  } else if (OpNo == 4) {
1272  // Need to sign extend the index since the bits will likely be used.
1273  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1274  } else
1275  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1276 
1277  SDValue Res = SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1278  // updated in place.
1279  if (Res.getNode() == N)
1280  return Res;
1281 
1282  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
1283  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1284  return SDValue();
1285 }
1286 
1287 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1288  unsigned OpNo) {
1289  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1290  if (OpNo == 2) {
1291  // The Mask
1292  EVT DataVT = N->getValue().getValueType();
1293  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1294  } else if (OpNo == 4) {
1295  // Need to sign extend the index since the bits will likely be used.
1296  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1297  } else
1298  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1299  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1300 }
1301 
1302 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
1303  SDValue Op = GetPromotedInteger(N->getOperand(0));
1304  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
1305 }
1306 
1307 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
1308  return SDValue(DAG.UpdateNodeOperands(N,
1309  ZExtPromotedInteger(N->getOperand(0))), 0);
1310 }
1311 
1312 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
1313  SDLoc dl(N);
1314  SDValue Op = GetPromotedInteger(N->getOperand(0));
1315  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1316  return DAG.getZeroExtendInReg(Op, dl,
1318 }
1319 
1320 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
1321  assert(OpNo == 2 && "Don't know how to promote this operand!");
1322 
1323  SDValue LHS = N->getOperand(0);
1324  SDValue RHS = N->getOperand(1);
1325  SDValue Carry = N->getOperand(2);
1326  SDLoc DL(N);
1327 
1328  auto VT = getSetCCResultType(LHS.getValueType());
1330  switch (BoolType) {
1332  Carry = DAG.getAnyExtOrTrunc(Carry, DL, VT);
1333  break;
1335  Carry = DAG.getZExtOrTrunc(Carry, DL, VT);
1336  break;
1338  Carry = DAG.getSExtOrTrunc(Carry, DL, VT);
1339  break;
1340  }
1341 
1342  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
1343 }
1344 
1345 //===----------------------------------------------------------------------===//
1346 // Integer Result Expansion
1347 //===----------------------------------------------------------------------===//
1348 
1349 /// ExpandIntegerResult - This method is called when the specified result of the
1350 /// specified node is found to need expansion. At this point, the node may also
1351 /// have invalid operands or may have other results that need promotion, we just
1352 /// know that (at least) one result needs expansion.
1353 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
1354  LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG);
1355  dbgs() << "\n");
1356  SDValue Lo, Hi;
1357  Lo = Hi = SDValue();
1358 
1359  // See if the target wants to custom expand this node.
1360  if (CustomLowerNode(N, N->getValueType(ResNo), true))
1361  return;
1362 
1363  switch (N->getOpcode()) {
1364  default:
1365 #ifndef NDEBUG
1366  dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
1367  N->dump(&DAG); dbgs() << "\n";
1368 #endif
1369  llvm_unreachable("Do not know how to expand the result of this operator!");
1370 
1371  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1372  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
1373  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1374  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1375 
1376  case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1377  case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1378  case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1379  case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1380  case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1381 
1382  case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
1383  case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
1384  case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
1385  case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
1386  case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
1387  case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
1388  case ISD::CTLZ_ZERO_UNDEF:
1389  case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
1390  case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
1391  case ISD::CTTZ_ZERO_UNDEF:
1392  case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
1393  case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
1394  case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
1395  case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
1396  case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
1397  case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
1398  case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
1399  case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
1400  case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
1401  case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
1402  case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
1403  case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
1404  case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
1405  case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
1406  case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
1407  case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
1408 
1409  case ISD::ATOMIC_LOAD_ADD:
1410  case ISD::ATOMIC_LOAD_SUB:
1411  case ISD::ATOMIC_LOAD_AND:
1412  case ISD::ATOMIC_LOAD_CLR:
1413  case ISD::ATOMIC_LOAD_OR:
1414  case ISD::ATOMIC_LOAD_XOR:
1415  case ISD::ATOMIC_LOAD_NAND:
1416  case ISD::ATOMIC_LOAD_MIN:
1417  case ISD::ATOMIC_LOAD_MAX:
1418  case ISD::ATOMIC_LOAD_UMIN:
1419  case ISD::ATOMIC_LOAD_UMAX:
1420  case ISD::ATOMIC_SWAP:
1421  case ISD::ATOMIC_CMP_SWAP: {
1422  std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
1423  SplitInteger(Tmp.first, Lo, Hi);
1424  ReplaceValueWith(SDValue(N, 1), Tmp.second);
1425  break;
1426  }
1428  AtomicSDNode *AN = cast<AtomicSDNode>(N);
1429  SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
1430  SDValue Tmp = DAG.getAtomicCmpSwap(
1431  ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
1432  N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
1433  AN->getMemOperand());
1434 
1435  // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
1436  // success simply by comparing the loaded value against the ingoing
1437  // comparison.
1438  SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
1439  N->getOperand(2), ISD::SETEQ);
1440 
1441  SplitInteger(Tmp, Lo, Hi);
1442  ReplaceValueWith(SDValue(N, 1), Success);
1443  ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
1444  break;
1445  }
1446 
1447  case ISD::AND:
1448  case ISD::OR:
1449  case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
1450 
1451  case ISD::UMAX:
1452  case ISD::SMAX:
1453  case ISD::UMIN:
1454  case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
1455 
1456  case ISD::ADD:
1457  case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
1458 
1459  case ISD::ADDC:
1460  case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
1461 
1462  case ISD::ADDE:
1463  case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
1464 
1465  case ISD::ADDCARRY:
1466  case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
1467 
1468  case ISD::SHL:
1469  case ISD::SRA:
1470  case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
1471 
1472  case ISD::SADDO:
1473  case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
1474  case ISD::UADDO:
1475  case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
1476  case ISD::UMULO:
1477  case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
1478  }
1479 
1480  // If Lo/Hi is null, the sub-method took care of registering results etc.
1481  if (Lo.getNode())
1482  SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
1483 }
1484 
1485 /// Lower an atomic node to the appropriate builtin call.
1486 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
1487  unsigned Opc = Node->getOpcode();
1488  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
1489  RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
1490  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
1491 
1492  return ExpandChainLibCall(LC, Node, false);
1493 }
1494 
1495 /// N is a shift by a value that needs to be expanded,
1496 /// and the shift amount is a constant 'Amt'. Expand the operation.
1497 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
1498  SDValue &Lo, SDValue &Hi) {
1499  SDLoc DL(N);
1500  // Expand the incoming operand to be shifted, so that we have its parts
1501  SDValue InL, InH;
1502  GetExpandedInteger(N->getOperand(0), InL, InH);
1503 
1504  // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
1505  // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
1506  if (!Amt) {
1507  Lo = InL;
1508  Hi = InH;
1509  return;
1510  }
1511 
1512  EVT NVT = InL.getValueType();
1513  unsigned VTBits = N->getValueType(0).getSizeInBits();
1514  unsigned NVTBits = NVT.getSizeInBits();
1515  EVT ShTy = N->getOperand(1).getValueType();
1516 
1517  if (N->getOpcode() == ISD::SHL) {
1518  if (Amt.ugt(VTBits)) {
1519  Lo = Hi = DAG.getConstant(0, DL, NVT);
1520  } else if (Amt.ugt(NVTBits)) {
1521  Lo = DAG.getConstant(0, DL, NVT);
1522  Hi = DAG.getNode(ISD::SHL, DL,
1523  NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1524  } else if (Amt == NVTBits) {
1525  Lo = DAG.getConstant(0, DL, NVT);
1526  Hi = InL;
1527  } else {
1528  Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
1529  Hi = DAG.getNode(ISD::OR, DL, NVT,
1530  DAG.getNode(ISD::SHL, DL, NVT, InH,
1531  DAG.getConstant(Amt, DL, ShTy)),
1532  DAG.getNode(ISD::SRL, DL, NVT, InL,
1533  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1534  }
1535  return;
1536  }
1537 
1538  if (N->getOpcode() == ISD::SRL) {
1539  if (Amt.ugt(VTBits)) {
1540  Lo = Hi = DAG.getConstant(0, DL, NVT);
1541  } else if (Amt.ugt(NVTBits)) {
1542  Lo = DAG.getNode(ISD::SRL, DL,
1543  NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1544  Hi = DAG.getConstant(0, DL, NVT);
1545  } else if (Amt == NVTBits) {
1546  Lo = InH;
1547  Hi = DAG.getConstant(0, DL, NVT);
1548  } else {
1549  Lo = DAG.getNode(ISD::OR, DL, NVT,
1550  DAG.getNode(ISD::SRL, DL, NVT, InL,
1551  DAG.getConstant(Amt, DL, ShTy)),
1552  DAG.getNode(ISD::SHL, DL, NVT, InH,
1553  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1554  Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1555  }
1556  return;
1557  }
1558 
1559  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
1560  if (Amt.ugt(VTBits)) {
1561  Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1562  DAG.getConstant(NVTBits - 1, DL, ShTy));
1563  } else if (Amt.ugt(NVTBits)) {
1564  Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1565  DAG.getConstant(Amt - NVTBits, DL, ShTy));
1566  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1567  DAG.getConstant(NVTBits - 1, DL, ShTy));
1568  } else if (Amt == NVTBits) {
1569  Lo = InH;
1570  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1571  DAG.getConstant(NVTBits - 1, DL, ShTy));
1572  } else {
1573  Lo = DAG.getNode(ISD::OR, DL, NVT,
1574  DAG.getNode(ISD::SRL, DL, NVT, InL,
1575  DAG.getConstant(Amt, DL, ShTy)),
1576  DAG.getNode(ISD::SHL, DL, NVT, InH,
1577  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1578  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1579  }
1580 }
1581 
1582 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
1583 /// this shift based on knowledge of the high bit of the shift amount. If we
1584 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
1585 /// shift amount.
1586 bool DAGTypeLegalizer::
1587 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1588  SDValue Amt = N->getOperand(1);
1589  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1590  EVT ShTy = Amt.getValueType();
1591  unsigned ShBits = ShTy.getScalarSizeInBits();
1592  unsigned NVTBits = NVT.getScalarSizeInBits();
1593  assert(isPowerOf2_32(NVTBits) &&
1594  "Expanded integer type size not a power of two!");
1595  SDLoc dl(N);
1596 
1597  APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
1598  KnownBits Known;
1599  DAG.computeKnownBits(N->getOperand(1), Known);
1600 
1601  // If we don't know anything about the high bits, exit.
1602  if (((Known.Zero|Known.One) & HighBitMask) == 0)
1603  return false;
1604 
1605  // Get the incoming operand to be shifted.
1606  SDValue InL, InH;
1607  GetExpandedInteger(N->getOperand(0), InL, InH);
1608 
1609  // If we know that any of the high bits of the shift amount are one, then we
1610  // can do this as a couple of simple shifts.
1611  if (Known.One.intersects(HighBitMask)) {
1612  // Mask out the high bit, which we know is set.
1613  Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
1614  DAG.getConstant(~HighBitMask, dl, ShTy));
1615 
1616  switch (N->getOpcode()) {
1617  default: llvm_unreachable("Unknown shift");
1618  case ISD::SHL:
1619  Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
1620  Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
1621  return true;
1622  case ISD::SRL:
1623  Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1624  Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
1625  return true;
1626  case ISD::SRA:
1627  Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
1628  DAG.getConstant(NVTBits - 1, dl, ShTy));
1629  Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
1630  return true;
1631  }
1632  }
1633 
1634  // If we know that all of the high bits of the shift amount are zero, then we
1635  // can do this as a couple of simple shifts.
1636  if (HighBitMask.isSubsetOf(Known.Zero)) {
1637  // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
1638  // shift if x is zero. We can use XOR here because x is known to be smaller
1639  // than 32.
1640  SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
1641  DAG.getConstant(NVTBits - 1, dl, ShTy));
1642 
1643  unsigned Op1, Op2;
1644  switch (N->getOpcode()) {
1645  default: llvm_unreachable("Unknown shift");
1646  case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
1647  case ISD::SRL:
1648  case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
1649  }
1650 
1651  // When shifting right the arithmetic for Lo and Hi is swapped.
1652  if (N->getOpcode() != ISD::SHL)
1653  std::swap(InL, InH);
1654 
1655  // Use a little trick to get the bits that move from Lo to Hi. First
1656  // shift by one bit.
1657  SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
1658  // Then compute the remaining shift with amount-1.
1659  SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
1660 
1661  Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
1662  Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
1663 
1664  if (N->getOpcode() != ISD::SHL)
1665  std::swap(Hi, Lo);
1666  return true;
1667  }
1668 
1669  return false;
1670 }
1671 
1672 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
1673 /// of any size.
1674 bool DAGTypeLegalizer::
1675 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1676  SDValue Amt = N->getOperand(1);
1677  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1678  EVT ShTy = Amt.getValueType();
1679  unsigned NVTBits = NVT.getSizeInBits();
1680  assert(isPowerOf2_32(NVTBits) &&
1681  "Expanded integer type size not a power of two!");
1682  SDLoc dl(N);
1683 
1684  // Get the incoming operand to be shifted.
1685  SDValue InL, InH;
1686  GetExpandedInteger(N->getOperand(0), InL, InH);
1687 
1688  SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
1689  SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
1690  SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
1691  SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1692  Amt, NVBitsNode, ISD::SETULT);
1693  SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1694  Amt, DAG.getConstant(0, dl, ShTy),
1695  ISD::SETEQ);
1696 
1697  SDValue LoS, HiS, LoL, HiL;
1698  switch (N->getOpcode()) {
1699  default: llvm_unreachable("Unknown shift");
1700  case ISD::SHL:
1701  // Short: ShAmt < NVTBits
1702  LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
1703  HiS = DAG.getNode(ISD::OR, dl, NVT,
1704  DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
1705  DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
1706 
1707  // Long: ShAmt >= NVTBits
1708  LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
1709  HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
1710 
1711  Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
1712  Hi = DAG.getSelect(dl, NVT, isZero, InH,
1713  DAG.getSelect(dl, NVT, isShort, HiS, HiL));
1714  return true;
1715  case ISD::SRL:
1716  // Short: ShAmt < NVTBits
1717  HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
1718  LoS = DAG.getNode(ISD::OR, dl, NVT,
1719  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1720  // FIXME: If Amt is zero, the following shift generates an undefined result
1721  // on some architectures.
1722  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1723 
1724  // Long: ShAmt >= NVTBits
1725  HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1726  LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1727 
1728  Lo = DAG.getSelect(dl, NVT, isZero, InL,
1729  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1730  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1731  return true;
1732  case ISD::SRA:
1733  // Short: ShAmt < NVTBits
1734  HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
1735  LoS = DAG.getNode(ISD::OR, dl, NVT,
1736  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1737  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1738 
1739  // Long: ShAmt >= NVTBits
1740  HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
1741  DAG.getConstant(NVTBits - 1, dl, ShTy));
1742  LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1743 
1744  Lo = DAG.getSelect(dl, NVT, isZero, InL,
1745  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1746  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1747  return true;
1748  }
1749 }
1750 
1751 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
1752 
1753  switch (Op) {
1754  default: llvm_unreachable("invalid min/max opcode");
1755  case ISD::SMAX:
1756  return std::make_pair(ISD::SETGT, ISD::UMAX);
1757  case ISD::UMAX:
1758  return std::make_pair(ISD::SETUGT, ISD::UMAX);
1759  case ISD::SMIN:
1760  return std::make_pair(ISD::SETLT, ISD::UMIN);
1761  case ISD::UMIN:
1762  return std::make_pair(ISD::SETULT, ISD::UMIN);
1763  }
1764 }
1765 
1766 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
1767  SDValue &Lo, SDValue &Hi) {
1768  SDLoc DL(N);
1769  ISD::NodeType LoOpc;
1770  ISD::CondCode CondC;
1771  std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
1772 
1773  // Expand the subcomponents.
1774  SDValue LHSL, LHSH, RHSL, RHSH;
1775  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1776  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1777 
1778  // Value types
1779  EVT NVT = LHSL.getValueType();
1780  EVT CCT = getSetCCResultType(NVT);
1781 
1782  // Hi part is always the same op
1783  Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
1784 
1785  // We need to know whether to select Lo part that corresponds to 'winning'
1786  // Hi part or if Hi parts are equal.
1787  SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
1788  SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
1789 
1790  // Lo part corresponding to the 'winning' Hi part
1791  SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
1792 
1793  // Recursed Lo part if Hi parts are equal, this uses unsigned version
1794  SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
1795 
1796  Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
1797 }
1798 
1799 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
1800  SDValue &Lo, SDValue &Hi) {
1801  SDLoc dl(N);
1802  // Expand the subcomponents.
1803  SDValue LHSL, LHSH, RHSL, RHSH;
1804  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1805  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1806 
1807  EVT NVT = LHSL.getValueType();
1808  SDValue LoOps[2] = { LHSL, RHSL };
1809  SDValue HiOps[3] = { LHSH, RHSH };
1810 
1811  bool HasOpCarry = TLI.isOperationLegalOrCustom(
1813  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1814  if (HasOpCarry) {
1815  SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
1816  if (N->getOpcode() == ISD::ADD) {
1817  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
1818  HiOps[2] = Lo.getValue(1);
1819  Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
1820  } else {
1821  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
1822  HiOps[2] = Lo.getValue(1);
1823  Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
1824  }
1825  return;
1826  }
1827 
1828  // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
1829  // them. TODO: Teach operation legalization how to expand unsupported
1830  // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
1831  // a carry of type MVT::Glue, but there doesn't seem to be any way to
1832  // generate a value of this type in the expanded code sequence.
1833  bool hasCarry =
1835  ISD::ADDC : ISD::SUBC,
1836  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1837 
1838  if (hasCarry) {
1839  SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
1840  if (N->getOpcode() == ISD::ADD) {
1841  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
1842  HiOps[2] = Lo.getValue(1);
1843  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
1844  } else {
1845  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
1846  HiOps[2] = Lo.getValue(1);
1847  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
1848  }
1849  return;
1850  }
1851 
1852  bool hasOVF =
1855  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1857 
1858  if (hasOVF) {
1859  EVT OvfVT = getSetCCResultType(NVT);
1860  SDVTList VTList = DAG.getVTList(NVT, OvfVT);
1861  int RevOpc;
1862  if (N->getOpcode() == ISD::ADD) {
1863  RevOpc = ISD::SUB;
1864  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
1865  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
1866  } else {
1867  RevOpc = ISD::ADD;
1868  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
1869  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
1870  }
1871  SDValue OVF = Lo.getValue(1);
1872 
1873  switch (BoolType) {
1875  OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
1878  OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
1879  Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
1880  break;
1882  OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
1883  Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
1884  }
1885  return;
1886  }
1887 
1888  if (N->getOpcode() == ISD::ADD) {
1889  Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
1890  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
1891  SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
1892  ISD::SETULT);
1893 
1895  SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
1896  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
1897  return;
1898  }
1899 
1900  SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
1901  DAG.getConstant(1, dl, NVT),
1902  DAG.getConstant(0, dl, NVT));
1903  SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
1904  ISD::SETULT);
1905  SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
1906  DAG.getConstant(1, dl, NVT), Carry1);
1907  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
1908  } else {
1909  Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
1910  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
1911  SDValue Cmp =
1912  DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
1913  LoOps[0], LoOps[1], ISD::SETULT);
1914 
1915  SDValue Borrow;
1917  Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
1918  else
1919  Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
1920  DAG.getConstant(0, dl, NVT));
1921 
1922  Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
1923  }
1924 }
1925 
1926 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
1927  SDValue &Lo, SDValue &Hi) {
1928  // Expand the subcomponents.
1929  SDValue LHSL, LHSH, RHSL, RHSH;
1930  SDLoc dl(N);
1931  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1932  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1933  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
1934  SDValue LoOps[2] = { LHSL, RHSL };
1935  SDValue HiOps[3] = { LHSH, RHSH };
1936 
1937  if (N->getOpcode() == ISD::ADDC) {
1938  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
1939  HiOps[2] = Lo.getValue(1);
1940  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
1941  } else {
1942  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
1943  HiOps[2] = Lo.getValue(1);
1944  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
1945  }
1946 
1947  // Legalized the flag result - switch anything that used the old flag to
1948  // use the new one.
1949  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
1950 }
1951 
1952 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
1953  SDValue &Lo, SDValue &Hi) {
1954  // Expand the subcomponents.
1955  SDValue LHSL, LHSH, RHSL, RHSH;
1956  SDLoc dl(N);
1957  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1958  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1959  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
1960  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
1961  SDValue HiOps[3] = { LHSH, RHSH };
1962 
1963  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
1964  HiOps[2] = Lo.getValue(1);
1965  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
1966 
1967  // Legalized the flag result - switch anything that used the old flag to
1968  // use the new one.
1969  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
1970 }
1971 
1972 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
1973  SDValue &Lo, SDValue &Hi) {
1974  SDValue LHS = N->getOperand(0);
1975  SDValue RHS = N->getOperand(1);
1976  SDLoc dl(N);
1977 
1978  SDValue Ovf;
1979 
1980  bool HasOpCarry = TLI.isOperationLegalOrCustom(
1982  TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
1983 
1984  if (HasOpCarry) {
1985  // Expand the subcomponents.
1986  SDValue LHSL, LHSH, RHSL, RHSH;
1987  GetExpandedInteger(LHS, LHSL, LHSH);
1988  GetExpandedInteger(RHS, RHSL, RHSH);
1989  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
1990  SDValue LoOps[2] = { LHSL, RHSL };
1991  SDValue HiOps[3] = { LHSH, RHSH };
1992 
1993  unsigned Opc = N->getOpcode() == ISD::UADDO ? ISD::ADDCARRY : ISD::SUBCARRY;
1994  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
1995  HiOps[2] = Lo.getValue(1);
1996  Hi = DAG.getNode(Opc, dl, VTList, HiOps);
1997 
1998  Ovf = Hi.getValue(1);
1999  } else {
2000  // Expand the result by simply replacing it with the equivalent
2001  // non-overflow-checking operation.
2002  auto Opc = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
2003  SDValue Sum = DAG.getNode(Opc, dl, LHS.getValueType(), LHS, RHS);
2004  SplitInteger(Sum, Lo, Hi);
2005 
2006  // Calculate the overflow: addition overflows iff a + b < a, and subtraction
2007  // overflows iff a - b > a.
2008  auto Cond = N->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
2009  Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
2010  }
2011 
2012  // Legalized the flag result - switch anything that used the old flag to
2013  // use the new one.
2014  ReplaceValueWith(SDValue(N, 1), Ovf);
2015 }
2016 
2017 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
2018  SDValue &Lo, SDValue &Hi) {
2019  // Expand the subcomponents.
2020  SDValue LHSL, LHSH, RHSL, RHSH;
2021  SDLoc dl(N);
2022  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2023  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2024  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2025  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2026  SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
2027 
2028  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2029  HiOps[2] = Lo.getValue(1);
2030  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2031 
2032  // Legalized the flag result - switch anything that used the old flag to
2033  // use the new one.
2034  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2035 }
2036 
2037 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
2038  SDValue &Lo, SDValue &Hi) {
2039  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2040  SDLoc dl(N);
2041  SDValue Op = N->getOperand(0);
2042  if (Op.getValueType().bitsLE(NVT)) {
2043  // The low part is any extension of the input (which degenerates to a copy).
2044  Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
2045  Hi = DAG.getUNDEF(NVT); // The high part is undefined.
2046  } else {
2047  // For example, extension of an i48 to an i64. The operand type necessarily
2048  // promotes to the result type, so will end up being expanded too.
2049  assert(getTypeAction(Op.getValueType()) ==
2051  "Only know how to promote this result!");
2052  SDValue Res = GetPromotedInteger(Op);
2053  assert(Res.getValueType() == N->getValueType(0) &&
2054  "Operand over promoted?");
2055  // Split the promoted operand. This will simplify when it is expanded.
2056  SplitInteger(Res, Lo, Hi);
2057  }
2058 }
2059 
2060 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
2061  SDValue &Lo, SDValue &Hi) {
2062  SDLoc dl(N);
2063  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2064  EVT NVT = Lo.getValueType();
2065  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2066  unsigned NVTBits = NVT.getSizeInBits();
2067  unsigned EVTBits = EVT.getSizeInBits();
2068 
2069  if (NVTBits < EVTBits) {
2070  Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
2072  EVTBits - NVTBits)));
2073  } else {
2074  Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
2075  // The high part replicates the sign bit of Lo, make it explicit.
2076  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2077  DAG.getConstant(NVTBits - 1, dl,
2078  TLI.getPointerTy(DAG.getDataLayout())));
2079  }
2080 }
2081 
2082 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
2083  SDValue &Lo, SDValue &Hi) {
2084  SDLoc dl(N);
2085  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2086  EVT NVT = Lo.getValueType();
2087  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2088  unsigned NVTBits = NVT.getSizeInBits();
2089  unsigned EVTBits = EVT.getSizeInBits();
2090 
2091  if (NVTBits < EVTBits) {
2092  Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
2094  EVTBits - NVTBits)));
2095  } else {
2096  Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
2097  // The high part must be zero, make it explicit.
2098  Hi = DAG.getConstant(0, dl, NVT);
2099  }
2100 }
2101 
2102 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
2103  SDValue &Lo, SDValue &Hi) {
2104  SDLoc dl(N);
2105  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2106  Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
2107  Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
2108 }
2109 
2110 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
2111  SDValue &Lo, SDValue &Hi) {
2112  SDLoc dl(N);
2113  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2114  Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
2115  Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
2116 }
2117 
2118 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
2119  SDValue &Lo, SDValue &Hi) {
2120  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2121  unsigned NBitWidth = NVT.getSizeInBits();
2122  auto Constant = cast<ConstantSDNode>(N);
2123  const APInt &Cst = Constant->getAPIntValue();
2124  bool IsTarget = Constant->isTargetOpcode();
2125  bool IsOpaque = Constant->isOpaque();
2126  SDLoc dl(N);
2127  Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
2128  Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
2129  IsOpaque);
2130 }
2131 
2132 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
2133  SDValue &Lo, SDValue &Hi) {
2134  SDLoc dl(N);
2135  // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
2136  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2137  EVT NVT = Lo.getValueType();
2138 
2139  SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
2140  DAG.getConstant(0, dl, NVT), ISD::SETNE);
2141 
2142  SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
2143  SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
2144 
2145  Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
2146  DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
2147  DAG.getConstant(NVT.getSizeInBits(), dl,
2148  NVT)));
2149  Hi = DAG.getConstant(0, dl, NVT);
2150 }
2151 
2152 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
2153  SDValue &Lo, SDValue &Hi) {
2154  SDLoc dl(N);
2155  // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
2156  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2157  EVT NVT = Lo.getValueType();
2158  Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
2159  DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
2160  Hi = DAG.getConstant(0, dl, NVT);
2161 }
2162 
2163 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
2164  SDValue &Lo, SDValue &Hi) {
2165  SDLoc dl(N);
2166  // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
2167  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2168  EVT NVT = Lo.getValueType();
2169 
2170  SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
2171  DAG.getConstant(0, dl, NVT), ISD::SETNE);
2172 
2173  SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
2174  SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
2175 
2176  Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
2177  DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
2178  DAG.getConstant(NVT.getSizeInBits(), dl,
2179  NVT)));
2180  Hi = DAG.getConstant(0, dl, NVT);
2181 }
2182 
2183 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
2184  SDValue &Hi) {
2185  SDLoc dl(N);
2186  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2187  unsigned NBitWidth = NVT.getSizeInBits();
2188 
2189  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2190  Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT);
2191  // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
2192  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2193  DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
2194 }
2195 
2196 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
2197  SDValue &Hi) {
2198  SDLoc dl(N);
2199  EVT VT = N->getValueType(0);
2200 
2201  SDValue Op = N->getOperand(0);
2202  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2203  Op = GetPromotedFloat(Op);
2204 
2206  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
2207  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, true/*irrelevant*/, dl).first,
2208  Lo, Hi);
2209 }
2210 
2211 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
2212  SDValue &Hi) {
2213  SDLoc dl(N);
2214  EVT VT = N->getValueType(0);
2215 
2216  SDValue Op = N->getOperand(0);
2217  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2218  Op = GetPromotedFloat(Op);
2219 
2221  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
2222  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, false/*irrelevant*/, dl).first,
2223  Lo, Hi);
2224 }
2225 
2226 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
2227  SDValue &Lo, SDValue &Hi) {
2228  if (ISD::isNormalLoad(N)) {
2229  ExpandRes_NormalLoad(N, Lo, Hi);
2230  return;
2231  }
2232 
2233  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2234 
2235  EVT VT = N->getValueType(0);
2236  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2237  SDValue Ch = N->getChain();
2238  SDValue Ptr = N->getBasePtr();
2239  ISD::LoadExtType ExtType = N->getExtensionType();
2240  unsigned Alignment = N->getAlignment();
2241  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2242  AAMDNodes AAInfo = N->getAAInfo();
2243  SDLoc dl(N);
2244 
2245  assert(NVT.isByteSized() && "Expanded type not byte sized!");
2246 
2247  if (N->getMemoryVT().bitsLE(NVT)) {
2248  EVT MemVT = N->getMemoryVT();
2249 
2250  Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
2251  Alignment, MMOFlags, AAInfo);
2252 
2253  // Remember the chain.
2254  Ch = Lo.getValue(1);
2255 
2256  if (ExtType == ISD::SEXTLOAD) {
2257  // The high part is obtained by SRA'ing all but one of the bits of the
2258  // lo part.
2259  unsigned LoSize = Lo.getValueSizeInBits();
2260  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2261  DAG.getConstant(LoSize - 1, dl,
2262  TLI.getPointerTy(DAG.getDataLayout())));
2263  } else if (ExtType == ISD::ZEXTLOAD) {
2264  // The high part is just a zero.
2265  Hi = DAG.getConstant(0, dl, NVT);
2266  } else {
2267  assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
2268  // The high part is undefined.
2269  Hi = DAG.getUNDEF(NVT);
2270  }
2271  } else if (DAG.getDataLayout().isLittleEndian()) {
2272  // Little-endian - low bits are at low addresses.
2273  Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2274  AAInfo);
2275 
2276  unsigned ExcessBits =
2277  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
2278  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
2279 
2280  // Increment the pointer to the other half.
2281  unsigned IncrementSize = NVT.getSizeInBits()/8;
2282  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2283  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2284  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
2285  N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
2286  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2287 
2288  // Build a factor node to remember that this load is independent of the
2289  // other one.
2290  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2291  Hi.getValue(1));
2292  } else {
2293  // Big-endian - high bits are at low addresses. Favor aligned loads at
2294  // the cost of some bit-fiddling.
2295  EVT MemVT = N->getMemoryVT();
2296  unsigned EBytes = MemVT.getStoreSize();
2297  unsigned IncrementSize = NVT.getSizeInBits()/8;
2298  unsigned ExcessBits = (EBytes - IncrementSize)*8;
2299 
2300  // Load both the high bits and maybe some of the low bits.
2301  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
2303  MemVT.getSizeInBits() - ExcessBits),
2304  Alignment, MMOFlags, AAInfo);
2305 
2306  // Increment the pointer to the other half.
2307  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2308  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2309  // Load the rest of the low bits.
2310  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
2311  N->getPointerInfo().getWithOffset(IncrementSize),
2312  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
2313  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2314 
2315  // Build a factor node to remember that this load is independent of the
2316  // other one.
2317  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2318  Hi.getValue(1));
2319 
2320  if (ExcessBits < NVT.getSizeInBits()) {
2321  // Transfer low bits from the bottom of Hi to the top of Lo.
2322  Lo = DAG.getNode(
2323  ISD::OR, dl, NVT, Lo,
2324  DAG.getNode(ISD::SHL, dl, NVT, Hi,
2325  DAG.getConstant(ExcessBits, dl,
2326  TLI.getPointerTy(DAG.getDataLayout()))));
2327  // Move high bits to the right position in Hi.
2328  Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
2329  Hi,
2330  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
2331  TLI.getPointerTy(DAG.getDataLayout())));
2332  }
2333  }
2334 
2335  // Legalize the chain result - switch anything that used the old chain to
2336  // use the new one.
2337  ReplaceValueWith(SDValue(N, 1), Ch);
2338 }
2339 
2340 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
2341  SDValue &Lo, SDValue &Hi) {
2342  SDLoc dl(N);
2343  SDValue LL, LH, RL, RH;
2344  GetExpandedInteger(N->getOperand(0), LL, LH);
2345  GetExpandedInteger(N->getOperand(1), RL, RH);
2346  Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
2347  Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
2348 }
2349 
2350 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
2351  SDValue &Lo, SDValue &Hi) {
2352  EVT VT = N->getValueType(0);
2353  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2354  SDLoc dl(N);
2355 
2356  SDValue LL, LH, RL, RH;
2357  GetExpandedInteger(N->getOperand(0), LL, LH);
2358  GetExpandedInteger(N->getOperand(1), RL, RH);
2359 
2360  if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
2362  LL, LH, RL, RH))
2363  return;
2364 
2365  // If nothing else, we can make a libcall.
2366  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2367  if (VT == MVT::i16)
2368  LC = RTLIB::MUL_I16;
2369  else if (VT == MVT::i32)
2370  LC = RTLIB::MUL_I32;
2371  else if (VT == MVT::i64)
2372  LC = RTLIB::MUL_I64;
2373  else if (VT == MVT::i128)
2374  LC = RTLIB::MUL_I128;
2375 
2376  if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
2377  // We'll expand the multiplication by brute force because we have no other
2378  // options. This is a trivially-generalized version of the code from
2379  // Hacker's Delight (itself derived from Knuth's Algorithm M from section
2380  // 4.3.1).
2381  unsigned Bits = NVT.getSizeInBits();
2382  unsigned HalfBits = Bits >> 1;
2383  SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
2384  NVT);
2385  SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
2386  SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
2387 
2388  SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
2389  SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
2390 
2391  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2392  if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
2393  // The type from TLI is too small to fit the shift amount we want.
2394  // Override it with i32. The shift will have to be legalized.
2395  ShiftAmtTy = MVT::i32;
2396  }
2397  SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
2398  SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
2399  SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
2400  SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
2401 
2402  SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
2403  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
2404  SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
2405  SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
2406 
2407  SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
2408  DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
2409  SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
2410 
2411  SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
2412  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
2413  DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
2414  Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
2415  DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
2416 
2417  Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
2418  DAG.getNode(ISD::ADD, dl, NVT,
2419  DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
2420  DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
2421  return;
2422  }
2423 
2424  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2425  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true/*irrelevant*/, dl).first,
2426  Lo, Hi);
2427 }
2428 
2429 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
2430  SDValue &Hi) {
2431  SDLoc DL(N);
2432  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2433  SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
2434  SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
2435  Lo = R.getValue(0);
2436  Hi = R.getValue(1);
2437  ReplaceValueWith(SDValue(N, 1), R.getValue(2));
2438 }
2439 
2440 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
2441  SDValue &Lo, SDValue &Hi) {
2442  SDValue LHS = Node->getOperand(0);
2443  SDValue RHS = Node->getOperand(1);
2444  SDLoc dl(Node);
2445 
2446  // Expand the result by simply replacing it with the equivalent
2447  // non-overflow-checking operation.
2448  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
2449  ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
2450  LHS, RHS);
2451  SplitInteger(Sum, Lo, Hi);
2452 
2453  // Compute the overflow.
2454  //
2455  // LHSSign -> LHS >= 0
2456  // RHSSign -> RHS >= 0
2457  // SumSign -> Sum >= 0
2458  //
2459  // Add:
2460  // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
2461  // Sub:
2462  // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
2463  //
2464  EVT OType = Node->getValueType(1);
2465  SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
2466 
2467  SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
2468  SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
2469  SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
2470  Node->getOpcode() == ISD::SADDO ?
2472 
2473  SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
2474  SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
2475 
2476  SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
2477 
2478  // Use the calculated overflow everywhere.
2479  ReplaceValueWith(SDValue(Node, 1), Cmp);
2480 }
2481 
2482 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
2483  SDValue &Lo, SDValue &Hi) {
2484  EVT VT = N->getValueType(0);
2485  SDLoc dl(N);
2486  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2487 
2489  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2490  SplitInteger(Res.getValue(0), Lo, Hi);
2491  return;
2492  }
2493 
2494  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2495  if (VT == MVT::i16)
2496  LC = RTLIB::SDIV_I16;
2497  else if (VT == MVT::i32)
2498  LC = RTLIB::SDIV_I32;
2499  else if (VT == MVT::i64)
2500  LC = RTLIB::SDIV_I64;
2501  else if (VT == MVT::i128)
2502  LC = RTLIB::SDIV_I128;
2503  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
2504 
2505  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
2506 }
2507 
2508 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
2509  SDValue &Lo, SDValue &Hi) {
2510  EVT VT = N->getValueType(0);
2511  SDLoc dl(N);
2512 
2513  // If we can emit an efficient shift operation, do so now. Check to see if
2514  // the RHS is a constant.
2515  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
2516  return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
2517 
2518  // If we can determine that the high bit of the shift is zero or one, even if
2519  // the low bits are variable, emit this shift in an optimized form.
2520  if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
2521  return;
2522 
2523  // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
2524  unsigned PartsOpc;
2525  if (N->getOpcode() == ISD::SHL) {
2526  PartsOpc = ISD::SHL_PARTS;
2527  } else if (N->getOpcode() == ISD::SRL) {
2528  PartsOpc = ISD::SRL_PARTS;
2529  } else {
2530  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2531  PartsOpc = ISD::SRA_PARTS;
2532  }
2533 
2534  // Next check to see if the target supports this SHL_PARTS operation or if it
2535  // will custom expand it.
2536  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2537  TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
2538  if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
2539  Action == TargetLowering::Custom) {
2540  // Expand the subcomponents.
2541  SDValue LHSL, LHSH;
2542  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2543  EVT VT = LHSL.getValueType();
2544 
2545  // If the shift amount operand is coming from a vector legalization it may
2546  // have an illegal type. Fix that first by casting the operand, otherwise
2547  // the new SHL_PARTS operation would need further legalization.
2548  SDValue ShiftOp = N->getOperand(1);
2549  EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2550  assert(ShiftTy.getScalarSizeInBits() >=
2551  Log2_32_Ceil(VT.getScalarSizeInBits()) &&
2552  "ShiftAmountTy is too small to cover the range of this type!");
2553  if (ShiftOp.getValueType() != ShiftTy)
2554  ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
2555 
2556  SDValue Ops[] = { LHSL, LHSH, ShiftOp };
2557  Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
2558  Hi = Lo.getValue(1);
2559  return;
2560  }
2561 
2562  // Otherwise, emit a libcall.
2563  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2564  bool isSigned;
2565  if (N->getOpcode() == ISD::SHL) {
2566  isSigned = false; /*sign irrelevant*/
2567  if (VT == MVT::i16)
2568  LC = RTLIB::SHL_I16;
2569  else if (VT == MVT::i32)
2570  LC = RTLIB::SHL_I32;
2571  else if (VT == MVT::i64)
2572  LC = RTLIB::SHL_I64;
2573  else if (VT == MVT::i128)
2574  LC = RTLIB::SHL_I128;
2575  } else if (N->getOpcode() == ISD::SRL) {
2576  isSigned = false;
2577  if (VT == MVT::i16)
2578  LC = RTLIB::SRL_I16;
2579  else if (VT == MVT::i32)
2580  LC = RTLIB::SRL_I32;
2581  else if (VT == MVT::i64)
2582  LC = RTLIB::SRL_I64;
2583  else if (VT == MVT::i128)
2584  LC = RTLIB::SRL_I128;
2585  } else {
2586  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2587  isSigned = true;
2588  if (VT == MVT::i16)
2589  LC = RTLIB::SRA_I16;
2590  else if (VT == MVT::i32)
2591  LC = RTLIB::SRA_I32;
2592  else if (VT == MVT::i64)
2593  LC = RTLIB::SRA_I64;
2594  else if (VT == MVT::i128)
2595  LC = RTLIB::SRA_I128;
2596  }
2597 
2598  if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
2599  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2600  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, isSigned, dl).first, Lo, Hi);
2601  return;
2602  }
2603 
2604  if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
2605  llvm_unreachable("Unsupported shift!");
2606 }
2607 
2608 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
2609  SDValue &Lo, SDValue &Hi) {
2610  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2611  SDLoc dl(N);
2612  SDValue Op = N->getOperand(0);
2613  if (Op.getValueType().bitsLE(NVT)) {
2614  // The low part is sign extension of the input (degenerates to a copy).
2615  Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
2616  // The high part is obtained by SRA'ing all but one of the bits of low part.
2617  unsigned LoSize = NVT.getSizeInBits();
2618  Hi = DAG.getNode(
2619  ISD::SRA, dl, NVT, Lo,
2620  DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
2621  } else {
2622  // For example, extension of an i48 to an i64. The operand type necessarily
2623  // promotes to the result type, so will end up being expanded too.
2624  assert(getTypeAction(Op.getValueType()) ==
2626  "Only know how to promote this result!");
2627  SDValue Res = GetPromotedInteger(Op);
2628  assert(Res.getValueType() == N->getValueType(0) &&
2629  "Operand over promoted?");
2630  // Split the promoted operand. This will simplify when it is expanded.
2631  SplitInteger(Res, Lo, Hi);
2632  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
2633  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2635  ExcessBits)));
2636  }
2637 }
2638 
2639 void DAGTypeLegalizer::
2640 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2641  SDLoc dl(N);
2642  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2643  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2644 
2645  if (EVT.bitsLE(Lo.getValueType())) {
2646  // sext_inreg the low part if needed.
2647  Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
2648  N->getOperand(1));
2649 
2650  // The high part gets the sign extension from the lo-part. This handles
2651  // things like sextinreg V:i64 from i8.
2652  Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
2653  DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
2654  TLI.getPointerTy(DAG.getDataLayout())));
2655  } else {
2656  // For example, extension of an i48 to an i64. Leave the low part alone,
2657  // sext_inreg the high part.
2658  unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
2659  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2661  ExcessBits)));
2662  }
2663 }
2664 
2665 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
2666  SDValue &Lo, SDValue &Hi) {
2667  EVT VT = N->getValueType(0);
2668  SDLoc dl(N);
2669  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2670 
2672  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2673  SplitInteger(Res.getValue(1), Lo, Hi);
2674  return;
2675  }
2676 
2677  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2678  if (VT == MVT::i16)
2679  LC = RTLIB::SREM_I16;
2680  else if (VT == MVT::i32)
2681  LC = RTLIB::SREM_I32;
2682  else if (VT == MVT::i64)
2683  LC = RTLIB::SREM_I64;
2684  else if (VT == MVT::i128)
2685  LC = RTLIB::SREM_I128;
2686  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
2687 
2688  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
2689 }
2690 
2691 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
2692  SDValue &Lo, SDValue &Hi) {
2693  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2694  SDLoc dl(N);
2695  Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
2696  Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
2697  N->getOperand(0),
2698  DAG.getConstant(NVT.getSizeInBits(), dl,
2699  TLI.getPointerTy(DAG.getDataLayout())));
2700  Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
2701 }
2702 
2703 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
2704  SDValue &Lo, SDValue &Hi) {
2705  EVT VT = N->getValueType(0);
2706  SDLoc dl(N);
2707 
2708  // A divide for UMULO should be faster than a function call.
2709  if (N->getOpcode() == ISD::UMULO) {
2710  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
2711 
2712  SDValue MUL = DAG.getNode(ISD::MUL, dl, LHS.getValueType(), LHS, RHS);
2713  SplitInteger(MUL, Lo, Hi);
2714 
2715  // A divide for UMULO will be faster than a function call. Select to
2716  // make sure we aren't using 0.
2717  SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(VT),
2718  RHS, DAG.getConstant(0, dl, VT), ISD::SETEQ);
2719  SDValue NotZero = DAG.getSelect(dl, VT, isZero,
2720  DAG.getConstant(1, dl, VT), RHS);
2721  SDValue DIV = DAG.getNode(ISD::UDIV, dl, VT, MUL, NotZero);
2722  SDValue Overflow = DAG.getSetCC(dl, N->getValueType(1), DIV, LHS,
2723  ISD::SETNE);
2724  Overflow = DAG.getSelect(dl, N->getValueType(1), isZero,
2725  DAG.getConstant(0, dl, N->getValueType(1)),
2726  Overflow);
2727  ReplaceValueWith(SDValue(N, 1), Overflow);
2728  return;
2729  }
2730 
2731  Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
2732  EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
2733  Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
2734 
2735  // Replace this with a libcall that will check overflow.
2736  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2737  if (VT == MVT::i32)
2738  LC = RTLIB::MULO_I32;
2739  else if (VT == MVT::i64)
2740  LC = RTLIB::MULO_I64;
2741  else if (VT == MVT::i128)
2742  LC = RTLIB::MULO_I128;
2743  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
2744 
2745  SDValue Temp = DAG.CreateStackTemporary(PtrVT);
2746  // Temporary for the overflow value, default it to zero.
2747  SDValue Chain =
2748  DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
2749  MachinePointerInfo());
2750 
2752  TargetLowering::ArgListEntry Entry;
2753  for (const SDValue &Op : N->op_values()) {
2754  EVT ArgVT = Op.getValueType();
2755  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2756  Entry.Node = Op;
2757  Entry.Ty = ArgTy;
2758  Entry.IsSExt = true;
2759  Entry.IsZExt = false;
2760  Args.push_back(Entry);
2761  }
2762 
2763  // Also pass the address of the overflow check.
2764  Entry.Node = Temp;
2765  Entry.Ty = PtrTy->getPointerTo();
2766  Entry.IsSExt = true;
2767  Entry.IsZExt = false;
2768  Args.push_back(Entry);
2769 
2770  SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
2771 
2773  CLI.setDebugLoc(dl)
2774  .setChain(Chain)
2775  .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
2776  .setSExtResult();
2777 
2778  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2779 
2780  SplitInteger(CallInfo.first, Lo, Hi);
2781  SDValue Temp2 =
2782  DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
2783  SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
2784  DAG.getConstant(0, dl, PtrVT),
2785  ISD::SETNE);
2786  // Use the overflow from the libcall everywhere.
2787  ReplaceValueWith(SDValue(N, 1), Ofl);
2788 }
2789 
2790 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
2791  SDValue &Lo, SDValue &Hi) {
2792  EVT VT = N->getValueType(0);
2793  SDLoc dl(N);
2794  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2795 
2797  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2798  SplitInteger(Res.getValue(0), Lo, Hi);
2799  return;
2800  }
2801 
2802  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2803  if (VT == MVT::i16)
2804  LC = RTLIB::UDIV_I16;
2805  else if (VT == MVT::i32)
2806  LC = RTLIB::UDIV_I32;
2807  else if (VT == MVT::i64)
2808  LC = RTLIB::UDIV_I64;
2809  else if (VT == MVT::i128)
2810  LC = RTLIB::UDIV_I128;
2811  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
2812 
2813  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
2814 }
2815 
2816 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
2817  SDValue &Lo, SDValue &Hi) {
2818  EVT VT = N->getValueType(0);
2819  SDLoc dl(N);
2820  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2821 
2823  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2824  SplitInteger(Res.getValue(1), Lo, Hi);
2825  return;
2826  }
2827 
2828  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2829  if (VT == MVT::i16)
2830  LC = RTLIB::UREM_I16;
2831  else if (VT == MVT::i32)
2832  LC = RTLIB::UREM_I32;
2833  else if (VT == MVT::i64)
2834  LC = RTLIB::UREM_I64;
2835  else if (VT == MVT::i128)
2836  LC = RTLIB::UREM_I128;
2837  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
2838 
2839  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
2840 }
2841 
2842 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
2843  SDValue &Lo, SDValue &Hi) {
2844  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2845  SDLoc dl(N);
2846  SDValue Op = N->getOperand(0);
2847  if (Op.getValueType().bitsLE(NVT)) {
2848  // The low part is zero extension of the input (degenerates to a copy).
2849  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
2850  Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
2851  } else {
2852  // For example, extension of an i48 to an i64. The operand type necessarily
2853  // promotes to the result type, so will end up being expanded too.
2854  assert(getTypeAction(Op.getValueType()) ==
2856  "Only know how to promote this result!");
2857  SDValue Res = GetPromotedInteger(Op);
2858  assert(Res.getValueType() == N->getValueType(0) &&
2859  "Operand over promoted?");
2860  // Split the promoted operand. This will simplify when it is expanded.
2861  SplitInteger(Res, Lo, Hi);
2862  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
2863  Hi = DAG.getZeroExtendInReg(Hi, dl,
2865  ExcessBits));
2866  }
2867 }
2868 
2869 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
2870  SDValue &Lo, SDValue &Hi) {
2871  SDLoc dl(N);
2872  EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
2873  SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
2874  SDValue Zero = DAG.getConstant(0, dl, VT);
2875  SDValue Swap = DAG.getAtomicCmpSwap(
2877  cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
2878  N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
2879 
2880  ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
2881  ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
2882 }
2883 
2884 //===----------------------------------------------------------------------===//
2885 // Integer Operand Expansion
2886 //===----------------------------------------------------------------------===//
2887 
2888 /// ExpandIntegerOperand - This method is called when the specified operand of
2889 /// the specified node is found to need expansion. At this point, all of the
2890 /// result types of the node are known to be legal, but other operands of the
2891 /// node may need promotion or expansion as well as the specified one.
2892 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
2893  LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG);
2894  dbgs() << "\n");
2895  SDValue Res = SDValue();
2896 
2897  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2898  return false;
2899 
2900  switch (N->getOpcode()) {
2901  default:
2902  #ifndef NDEBUG
2903  dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
2904  N->dump(&DAG); dbgs() << "\n";
2905  #endif
2906  llvm_unreachable("Do not know how to expand this operator's operand!");
2907 
2908  case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
2909  case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
2910  case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
2911  case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
2912  case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
2913  case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
2914  case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
2915  case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
2916  case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
2917  case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break;
2918  case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
2919  case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
2920  case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break;
2921 
2922  case ISD::SHL:
2923  case ISD::SRA:
2924  case ISD::SRL:
2925  case ISD::ROTL:
2926  case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
2927  case ISD::RETURNADDR:
2928  case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
2929 
2930  case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
2931  }
2932 
2933  // If the result is null, the sub-method took care of registering results etc.
2934  if (!Res.getNode()) return false;
2935 
2936  // If the result is N, the sub-method updated N in place. Tell the legalizer
2937  // core about this.
2938  if (Res.getNode() == N)
2939  return true;
2940 
2941  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2942  "Invalid operand expansion");
2943 
2944  ReplaceValueWith(SDValue(N, 0), Res);
2945  return false;
2946 }
2947 
2948 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
2949 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
2950 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
2951  SDValue &NewRHS,
2952  ISD::CondCode &CCCode,
2953  const SDLoc &dl) {
2954  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2955  GetExpandedInteger(NewLHS, LHSLo, LHSHi);
2956  GetExpandedInteger(NewRHS, RHSLo, RHSHi);
2957 
2958  if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
2959  if (RHSLo == RHSHi) {
2960  if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
2961  if (RHSCST->isAllOnesValue()) {
2962  // Equality comparison to -1.
2963  NewLHS = DAG.getNode(ISD::AND, dl,
2964  LHSLo.getValueType(), LHSLo, LHSHi);
2965  NewRHS = RHSLo;
2966  return;
2967  }
2968  }
2969  }
2970 
2971  NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
2972  NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
2973  NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
2974  NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
2975  return;
2976  }
2977 
2978  // If this is a comparison of the sign bit, just look at the top part.
2979  // X > -1, x < 0
2980  if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
2981  if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0
2982  (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1
2983  NewLHS = LHSHi;
2984  NewRHS = RHSHi;
2985  return;
2986  }
2987 
2988  // FIXME: This generated code sucks.
2989  ISD::CondCode LowCC;
2990  switch (CCCode) {
2991  default: llvm_unreachable("Unknown integer setcc!");
2992  case ISD::SETLT:
2993  case ISD::SETULT: LowCC = ISD::SETULT; break;
2994  case ISD::SETGT:
2995  case ISD::SETUGT: LowCC = ISD::SETUGT; break;
2996  case ISD::SETLE:
2997  case ISD::SETULE: LowCC = ISD::SETULE; break;
2998  case ISD::SETGE:
2999  case ISD::SETUGE: LowCC = ISD::SETUGE; break;
3000  }
3001 
3002  // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
3003  // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
3004  // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
3005 
3006  // NOTE: on targets without efficient SELECT of bools, we can always use
3007  // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
3008  TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
3009  nullptr);
3010  SDValue LoCmp, HiCmp;
3011  if (TLI.isTypeLegal(LHSLo.getValueType()) &&
3012  TLI.isTypeLegal(RHSLo.getValueType()))
3013  LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
3014  RHSLo, LowCC, false, DagCombineInfo, dl);
3015  if (!LoCmp.getNode())
3016  LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
3017  RHSLo, LowCC);
3018  if (TLI.isTypeLegal(LHSHi.getValueType()) &&
3019  TLI.isTypeLegal(RHSHi.getValueType()))
3020  HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
3021  RHSHi, CCCode, false, DagCombineInfo, dl);
3022  if (!HiCmp.getNode())
3023  HiCmp =
3024  DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
3025  LHSHi, RHSHi, DAG.getCondCode(CCCode));
3026 
3027  ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
3028  ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
3029 
3030  bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
3031  CCCode == ISD::SETUGE || CCCode == ISD::SETULE);
3032 
3033  if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) ||
3034  (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) ||
3035  (LoCmpC && LoCmpC->isNullValue())))) {
3036  // For LE / GE, if high part is known false, ignore the low part.
3037  // For LT / GT: if low part is known false, return the high part.
3038  // if high part is known true, ignore the low part.
3039  NewLHS = HiCmp;
3040  NewRHS = SDValue();
3041  return;
3042  }
3043 
3044  if (LHSHi == RHSHi) {
3045  // Comparing the low bits is enough.
3046  NewLHS = LoCmp;
3047  NewRHS = SDValue();
3048  return;
3049  }
3050 
3051  // Lower with SETCCCARRY if the target supports it.
3052  EVT HiVT = LHSHi.getValueType();
3053  EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
3054  bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
3055 
3056  // FIXME: Make all targets support this, then remove the other lowering.
3057  if (HasSETCCCARRY) {
3058  // SETCCCARRY can detect < and >= directly. For > and <=, flip
3059  // operands and condition code.
3060  bool FlipOperands = false;
3061  switch (CCCode) {
3062  case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
3063  case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
3064  case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
3065  case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
3066  default: break;
3067  }
3068  if (FlipOperands) {
3069  std::swap(LHSLo, RHSLo);
3070  std::swap(LHSHi, RHSHi);
3071  }
3072  // Perform a wide subtraction, feeding the carry from the low part into
3073  // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
3074  // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
3075  // zero or positive iff LHS >= RHS.
3076  EVT LoVT = LHSLo.getValueType();
3077  SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
3078  SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
3079  SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
3080  LHSHi, RHSHi, LowCmp.getValue(1),
3081  DAG.getCondCode(CCCode));
3082  NewLHS = Res;
3083  NewRHS = SDValue();
3084  return;
3085  }
3086 
3087  NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
3088  false, DagCombineInfo, dl);
3089  if (!NewLHS.getNode())
3090  NewLHS =
3091  DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
3092  NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
3093  NewRHS = SDValue();
3094 }
3095 
3096 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
3097  SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
3098  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
3099  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3100 
3101  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3102  // against zero to select between true and false values.
3103  if (!NewRHS.getNode()) {
3104  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3105  CCCode = ISD::SETNE;
3106  }
3107 
3108  // Update N to have the operands specified.
3109  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
3110  DAG.getCondCode(CCCode), NewLHS, NewRHS,
3111  N->getOperand(4)), 0);
3112 }
3113 
3114 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
3115  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3116  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
3117  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3118 
3119  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3120  // against zero to select between true and false values.
3121  if (!NewRHS.getNode()) {
3122  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3123  CCCode = ISD::SETNE;
3124  }
3125 
3126  // Update N to have the operands specified.
3127  return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
3128  N->getOperand(2), N->getOperand(3),
3129  DAG.getCondCode(CCCode)), 0);
3130 }
3131 
3132 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
3133  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3134  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3135  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3136 
3137  // If ExpandSetCCOperands returned a scalar, use it.
3138  if (!NewRHS.getNode()) {
3139  assert(NewLHS.getValueType() == N->getValueType(0) &&
3140  "Unexpected setcc expansion!");
3141  return NewLHS;
3142  }
3143 
3144  // Otherwise, update N to have the operands specified.
3145  return SDValue(
3146  DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
3147 }
3148 
3149 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
3150  SDValue LHS = N->getOperand(0);
3151  SDValue RHS = N->getOperand(1);
3152  SDValue Carry = N->getOperand(2);
3153  SDValue Cond = N->getOperand(3);
3154  SDLoc dl = SDLoc(N);
3155 
3156  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3157  GetExpandedInteger(LHS, LHSLo, LHSHi);
3158  GetExpandedInteger(RHS, RHSLo, RHSHi);
3159 
3160  // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
3161  SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
3162  SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
3163  return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
3164  LowCmp.getValue(1), Cond);
3165 }
3166 
3167 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
3168  // The value being shifted is legal, but the shift amount is too big.
3169  // It follows that either the result of the shift is undefined, or the
3170  // upper half of the shift amount is zero. Just use the lower half.
3171  SDValue Lo, Hi;
3172  GetExpandedInteger(N->getOperand(1), Lo, Hi);
3173  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
3174 }
3175 
3176 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
3177  // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
3178  // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
3179  // constant to valid type.
3180  SDValue Lo, Hi;
3181  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3182  return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
3183 }
3184 
3185 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
3186  SDValue Op = N->getOperand(0);
3187  EVT DstVT = N->getValueType(0);
3189  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3190  "Don't know how to expand this SINT_TO_FP!");
3191  return TLI.makeLibCall(DAG, LC, DstVT, Op, true, SDLoc(N)).first;
3192 }
3193 
3194 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
3195  if (ISD::isNormalStore(N))
3196  return ExpandOp_NormalStore(N, OpNo);
3197 
3198  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
3199  assert(OpNo == 1 && "Can only expand the stored value so far");
3200 
3201  EVT VT = N->getOperand(1).getValueType();
3202  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3203  SDValue Ch = N->getChain();
3204  SDValue Ptr = N->getBasePtr();
3205  unsigned Alignment = N->getAlignment();
3206  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3207  AAMDNodes AAInfo = N->getAAInfo();
3208  SDLoc dl(N);
3209  SDValue Lo, Hi;
3210 
3211  assert(NVT.isByteSized() && "Expanded type not byte sized!");
3212 
3213  if (N->getMemoryVT().bitsLE(NVT)) {
3214  GetExpandedInteger(N->getValue(), Lo, Hi);
3215  return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
3216  N->getMemoryVT(), Alignment, MMOFlags, AAInfo);
3217  }
3218 
3219  if (DAG.getDataLayout().isLittleEndian()) {
3220  // Little-endian - low bits are at low addresses.
3221  GetExpandedInteger(N->getValue(), Lo, Hi);
3222 
3223  Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3224  AAInfo);
3225 
3226  unsigned ExcessBits =
3227  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3228  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3229 
3230  // Increment the pointer to the other half.
3231  unsigned IncrementSize = NVT.getSizeInBits()/8;
3232  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3233  Hi = DAG.getTruncStore(
3234  Ch, dl, Hi, Ptr, N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3235  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3236  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3237  }
3238 
3239  // Big-endian - high bits are at low addresses. Favor aligned stores at
3240  // the cost of some bit-fiddling.
3241  GetExpandedInteger(N->getValue(), Lo, Hi);
3242 
3243  EVT ExtVT = N->getMemoryVT();
3244  unsigned EBytes = ExtVT.getStoreSize();
3245  unsigned IncrementSize = NVT.getSizeInBits()/8;
3246  unsigned ExcessBits = (EBytes - IncrementSize)*8;
3247  EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
3248  ExtVT.getSizeInBits() - ExcessBits);
3249 
3250  if (ExcessBits < NVT.getSizeInBits()) {
3251  // Transfer high bits from the top of Lo to the bottom of Hi.
3252  Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
3253  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3254  TLI.getPointerTy(DAG.getDataLayout())));
3255  Hi = DAG.getNode(
3256  ISD::OR, dl, NVT, Hi,
3257  DAG.getNode(ISD::SRL, dl, NVT, Lo,
3258  DAG.getConstant(ExcessBits, dl,
3259  TLI.getPointerTy(DAG.getDataLayout()))));
3260  }
3261 
3262  // Store both the high bits and maybe some of the low bits.
3263  Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment,
3264  MMOFlags, AAInfo);
3265 
3266  // Increment the pointer to the other half.
3267  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3268  // Store the lowest ExcessBits bits in the second half.
3269  Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
3270  N->getPointerInfo().getWithOffset(IncrementSize),
3271  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3272  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3273  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3274 }
3275 
3276 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
3277  SDValue InL, InH;
3278  GetExpandedInteger(N->getOperand(0), InL, InH);
3279  // Just truncate the low part of the source.
3280  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
3281 }
3282 
3283 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
3284  SDValue Op = N->getOperand(0);
3285  EVT SrcVT = Op.getValueType();
3286  EVT DstVT = N->getValueType(0);
3287  SDLoc dl(N);
3288 
3289  // The following optimization is valid only if every value in SrcVT (when
3290  // treated as signed) is representable in DstVT. Check that the mantissa
3291  // size of DstVT is >= than the number of bits in SrcVT -1.
3292  const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT);
3293  if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 &&
3295  // Do a signed conversion then adjust the result.
3296  SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op);
3297  SignedConv = TLI.LowerOperation(SignedConv, DAG);
3298 
3299  // The result of the signed conversion needs adjusting if the 'sign bit' of
3300  // the incoming integer was set. To handle this, we dynamically test to see
3301  // if it is set, and, if so, add a fudge factor.
3302 
3303  const uint64_t F32TwoE32 = 0x4F800000ULL;
3304  const uint64_t F32TwoE64 = 0x5F800000ULL;
3305  const uint64_t F32TwoE128 = 0x7F800000ULL;
3306 
3307  APInt FF(32, 0);
3308  if (SrcVT == MVT::i32)
3309  FF = APInt(32, F32TwoE32);
3310  else if (SrcVT == MVT::i64)
3311  FF = APInt(32, F32TwoE64);
3312  else if (SrcVT == MVT::i128)
3313  FF = APInt(32, F32TwoE128);
3314  else
3315  llvm_unreachable("Unsupported UINT_TO_FP!");
3316 
3317  // Check whether the sign bit is set.
3318  SDValue Lo, Hi;
3319  GetExpandedInteger(Op, Lo, Hi);
3320  SDValue SignSet = DAG.getSetCC(dl,
3321  getSetCCResultType(Hi.getValueType()),
3322  Hi,
3323  DAG.getConstant(0, dl, Hi.getValueType()),
3324  ISD::SETLT);
3325 
3326  // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
3327  SDValue FudgePtr =
3328  DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)),
3329  TLI.getPointerTy(DAG.getDataLayout()));
3330 
3331  // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
3332  SDValue Zero = DAG.getIntPtrConstant(0, dl);
3333  SDValue Four = DAG.getIntPtrConstant(4, dl);
3334  if (DAG.getDataLayout().isBigEndian())
3335  std::swap(Zero, Four);
3336  SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet,
3337  Zero, Four);
3338  unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment();
3339  FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(),
3340  FudgePtr, Offset);
3341  Alignment = std::min(Alignment, 4u);
3342 
3343  // Load the value out, extending it from f32 to the destination float type.
3344  // FIXME: Avoid the extend by constructing the right constant pool?
3345  SDValue Fudge = DAG.getExtLoad(
3346  ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), FudgePtr,
3348  Alignment);
3349  return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge);
3350  }
3351 
3352  // Otherwise, use a libcall.
3353  RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT);
3354  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3355  "Don't know how to expand this UINT_TO_FP!");
3356  return TLI.makeLibCall(DAG, LC, DstVT, Op, true, dl).first;
3357 }
3358 
3359 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
3360  SDLoc dl(N);
3361  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
3362  cast<AtomicSDNode>(N)->getMemoryVT(),
3363  N->getOperand(0),
3364  N->getOperand(1), N->getOperand(2),
3365  cast<AtomicSDNode>(N)->getMemOperand());
3366  return Swap.getValue(1);
3367 }
3368 
3369 
3370 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
3371  SDValue InOp0 = N->getOperand(0);
3372  EVT InVT = InOp0.getValueType();
3373 
3374  EVT OutVT = N->getValueType(0);
3375  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3376  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3377  unsigned OutNumElems = OutVT.getVectorNumElements();
3378  EVT NOutVTElem = NOutVT.getVectorElementType();
3379 
3380  SDLoc dl(N);
3381  SDValue BaseIdx = N->getOperand(1);
3382 
3384  Ops.reserve(OutNumElems);
3385  for (unsigned i = 0; i != OutNumElems; ++i) {
3386 
3387  // Extract the element from the original vector.
3388  SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
3389  BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
3391  InVT.getVectorElementType(), N->getOperand(0), Index);
3392 
3393  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext);
3394  // Insert the converted element to the new vector.
3395  Ops.push_back(Op);
3396  }
3397 
3398  return DAG.getBuildVector(NOutVT, dl, Ops);
3399 }
3400 
3401 
3402 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
3403  ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
3404  EVT VT = N->getValueType(0);
3405  SDLoc dl(N);
3406 
3407  ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
3408 
3409  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3410  SDValue V1 = GetPromotedInteger(N->getOperand(1));
3411  EVT OutVT = V0.getValueType();
3412 
3413  return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
3414 }
3415 
3416 
3417 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
3418  EVT OutVT = N->getValueType(0);
3419  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3420  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3421  unsigned NumElems = N->getNumOperands();
3422  EVT NOutVTElem = NOutVT.getVectorElementType();
3423 
3424  SDLoc dl(N);
3425 
3427  Ops.reserve(NumElems);
3428  for (unsigned i = 0; i != NumElems; ++i) {
3429  SDValue Op;
3430  // BUILD_VECTOR integer operand types are allowed to be larger than the
3431  // result's element type. This may still be true after the promotion. For
3432  // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
3433  // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
3434  if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
3435  Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
3436  else
3437  Op = N->getOperand(i);
3438  Ops.push_back(Op);
3439  }
3440 
3441  return DAG.getBuildVector(NOutVT, dl, Ops);
3442 }
3443 
3444 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
3445 
3446  SDLoc dl(N);
3447 
3448  assert(!N->getOperand(0).getValueType().isVector() &&
3449  "Input must be a scalar");
3450 
3451  EVT OutVT = N->getValueType(0);
3452  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3453  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3454  EVT NOutVTElem = NOutVT.getVectorElementType();
3455 
3456  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
3457 
3458  return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
3459 }
3460 
3461 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
3462  SDLoc dl(N);
3463 
3464  EVT OutVT = N->getValueType(0);
3465  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3466  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3467 
3468  EVT OutElemTy = NOutVT.getVectorElementType();
3469 
3470  unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
3471  unsigned NumOutElem = NOutVT.getVectorNumElements();
3472  unsigned NumOperands = N->getNumOperands();
3473  assert(NumElem * NumOperands == NumOutElem &&
3474  "Unexpected number of elements");
3475 
3476  // Take the elements from the first vector.
3477  SmallVector<SDValue, 8> Ops(NumOutElem);
3478  for (unsigned i = 0; i < NumOperands; ++i) {
3479  SDValue Op = N->getOperand(i);
3480  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
3481  Op = GetPromotedInteger(Op);
3482  EVT SclrTy = Op.getValueType().getVectorElementType();
3483  assert(NumElem == Op.getValueType().getVectorNumElements() &&
3484  "Unexpected number of elements");
3485 
3486  for (unsigned j = 0; j < NumElem; ++j) {
3487  SDValue Ext = DAG.getNode(
3488  ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
3489  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3490  Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
3491  }
3492  }
3493 
3494  return DAG.getBuildVector(NOutVT, dl, Ops);
3495 }
3496 
3497 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
3498  EVT VT = N->getValueType(0);
3499  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3500  assert(NVT.isVector() && "This type must be promoted to a vector type");
3501 
3502  SDLoc dl(N);
3503 
3504  // For operands whose TypeAction is to promote, extend the promoted node
3505  // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
3506  // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
3507  // type..
3508  if (getTypeAction(N->getOperand(0).getValueType())
3510  SDValue Promoted;
3511 
3512  switch(N->getOpcode()) {
3514  Promoted = SExtPromotedInteger(N->getOperand(0));
3515  break;
3517  Promoted = ZExtPromotedInteger(N->getOperand(0));
3518  break;
3520  Promoted = GetPromotedInteger(N->getOperand(0));
3521  break;
3522  default:
3523  llvm_unreachable("Node has unexpected Opcode");
3524  }
3525  return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
3526  }
3527 
3528  // Directly extend to the appropriate transform-to type.
3529  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
3530 }
3531 
3532 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
3533  EVT OutVT = N->getValueType(0);
3534  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3535  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3536 
3537  EVT NOutVTElem = NOutVT.getVectorElementType();
3538 
3539  SDLoc dl(N);
3540  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3541 
3542  SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
3543  NOutVTElem, N->getOperand(1));
3544  return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
3545  V0, ConvElem, N->getOperand(2));
3546 }
3547 
3548 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3549  SDLoc dl(N);
3550  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3551  SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
3552  TLI.getVectorIdxTy(DAG.getDataLayout()));
3554  V0->getValueType(0).getScalarType(), V0, V1);
3555 
3556  // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
3557  // element types. If this is the case then we need to expand the outgoing
3558  // value and not truncate it.
3559  return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
3560 }
3561 
3562 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
3563  SDLoc dl(N);
3564  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3565  MVT InVT = V0.getValueType().getSimpleVT();
3566  MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
3568  SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
3569  return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
3570 }
3571 
3572 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
3573  SDLoc dl(N);
3574  unsigned NumElems = N->getNumOperands();
3575 
3576  EVT RetSclrTy = N->getValueType(0).getVectorElementType();
3577 
3578  SmallVector<SDValue, 8> NewOps;
3579  NewOps.reserve(NumElems);
3580 
3581  // For each incoming vector
3582  for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
3583  SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
3584  EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
3585  unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
3586 
3587  for (unsigned i=0; i<NumElem; ++i) {
3588  // Extract element from incoming vector
3589  SDValue Ex = DAG.getNode(
3590  ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
3591  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3592  SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
3593  NewOps.push_back(Tr);
3594  }
3595  }
3596 
3597  return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
3598 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:539
void push_back(const T &Elt)
Definition: SmallVector.h:213
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:544
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getValueType() const
Return the ValueType of the referenced return value.
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant, which is required to be operand #1) half of the integer or float value specified as operand #0.
Definition: ISDOpcodes.h:184
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC&#39;s if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:951
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:334
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:611
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
static MVT getVectorVT(MVT VT, unsigned NumElements)
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:343
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:260
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:487
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none...
const SDValue & getBasePtr() const
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:223
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:864
const SDValue & getValue() const
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
Definition: APInt.h:641
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:253
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
const SDValue & getBasePtr() const
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:251
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1313
const SDValue & getBasePtr() const
const SDValue & getChain() const
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
unsigned getAlignment() const
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:763
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select&#39;s if you just have operands and don&#39;t want to check...
Definition: SelectionDAG.h:964
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:478
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT TVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:360
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Same for subtraction.
Definition: ISDOpcodes.h:254
void reserve(size_type N)
Definition: SmallVector.h:377
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:212
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align=0, int Offs=0, bool isT=false, unsigned char TargetFlags=0)
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:405
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:39
const SDValue & getValue() const
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
SDValue getExternalSymbol(const char *Sym, EVT VT)
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:210
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations...
Definition: ISDOpcodes.h:426
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
static uint32_t getAlignment(const MCSectionCOFF &Sec)
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:451
Shift and rotation operations.
Definition: ISDOpcodes.h:380
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:202
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:639
CallLoweringInfo & setChain(SDValue InChain)
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:191
op_iterator op_end() const
unsigned getScalarValueSizeInBits() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
FLT_ROUNDS_ - Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest 2 Round to ...
Definition: ISDOpcodes.h:514
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:304
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt) For double-word atomic operations: ValLo, ValHi, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amtLo, amtHi) ValLo, ValHi, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amtLo, amtHi) These correspond to the atomicrmw instruction.
Definition: ISDOpcodes.h:771
const SDValue & getValue() const
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:457
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:398
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG...
Definition: ISDOpcodes.h:73
This is an SDNode representing atomic operations.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
This represents a list of ValueType&#39;s that has been intern&#39;d by a SelectionDAG.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
This class is used to represent an MSTORE node.
unsigned getSizeInBits() const
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:237
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:298
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:292
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:395
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:448
SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
const SDValue & getScale() const
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:399
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:201
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:221
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:865
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTruncating=false, bool IsCompressing=false)
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:911
op_iterator op_begin() const
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:708
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:465
UNDEF - An undefined node.
Definition: ISDOpcodes.h:178
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:492
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:303
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
Definition: APInt.h:629
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits...
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
Definition: MathExtras.h:602
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const SDValue & getBasePtr() const
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:421
Machine Value Type.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
Simple binary floating point operators.
Definition: ISDOpcodes.h:260
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:273
This is an important base class in LLVM.
Definition: Constant.h:42
iterator_range< value_op_iterator > op_values() const
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
Definition: SmallVector.h:117
const SDValue & getOperand(unsigned Num) const
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:886
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:308
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:247
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:232
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:581
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:57
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:680
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType, bool IsExpanding=false)
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:386
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo...
Definition: ISDOpcodes.h:757
std::vector< ArgListEntry > ArgListTy
const SDValue & getIndex() const
Extended Value Type.
Definition: ValueTypes.h:34
const SDValue & getBasePtr() const
This structure contains all information that is necessary for lowering calls.
const SDValue & getSrc0() const
This class contains a discriminated union of information about pointers in memory operands...
unsigned getNumOperands() const
Return the number of values used by this operation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, unsigned Alignment, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SyncScope::ID SSID)
Gets a node for an atomic cmpxchg op.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
const SDValue & getMask() const
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:50
void dump() const
Dump this node, for debugging.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:315
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:265
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
Definition: ISDOpcodes.h:421
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:729
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:605
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:642
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:383
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:549
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:611
Represents one node in the SelectionDAG.
static unsigned int semanticsPrecision(const fltSemantics &)
Definition: APFloat.cpp:155
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:133
MachinePointerInfo getWithOffset(int64_t O) const
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:531
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:924
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT...
Definition: ValueTypes.h:73
EVT getMemoryVT() const
Return the type of the in-memory value.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
Class for arbitrary precision integers.
Definition: APInt.h:69
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:390
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:241
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:438
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:441
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition: APInt.h:523
Flags
Flags values. These may be or&#39;d together.
#define Success
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool ugt(const APInt &RHS) const
Unsigned greather than comparison.
Definition: APInt.h:1239
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:546
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Definition: SmallVector.h:121
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:151
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
Definition: APInt.h:1305
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:363
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, bool isSigned, const SDLoc &dl, bool doesNotReturn=false, bool isReturnValueUsed=true) const
Returns a pair of (return value, chain).
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:456
void computeKnownBits(SDValue Op, KnownBits &Known, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:575
#define N
Flags getFlags() const
Return the raw flags of the source value,.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:323
Same for multiplication.
Definition: ISDOpcodes.h:257
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
SDValue getValue(unsigned R) const
This class is used to represent an MSCATTER node.
unsigned getNumRegisters(LLVMContext &Context, EVT VT) const
Return the number of registers that this ValueType will eventually require.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getCondCode(ISD::CondCode Cond)
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, const Value *PtrVal, unsigned Alignment, AtomicOrdering Ordering, SyncScope::ID SSID)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands...
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input...
const MachinePointerInfo & getPointerInfo() const
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:321
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This class is used to represent an MLOAD node.
ArrayRef< int > getMask() const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:235
This class is used to represent an MGATHER node.
SDValue getValueType(EVT)
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:81
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:413
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Definition: ValueTypes.h:223
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:198
Conversion operators.
Definition: ISDOpcodes.h:435
BooleanContent
Enum that describes how the target represents true/false values.
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:750
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT SrcTy)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value...
bool isBigEndian() const
Definition: DataLayout.h:222
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:444
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Op, int64_t Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object...
Definition: SelectionDAG.h:819
#define LLVM_DEBUG(X)
Definition: Debug.h:119
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:746
LLVMContext * getContext() const
Definition: SelectionDAG.h:404
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:351
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:242
const SDValue & getMask() const
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
This class is used to represent ISD::LOAD nodes.