LLVM  8.0.0svn
LegalizeIntegerTypes.cpp
Go to the documentation of this file.
1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements integer type expansion and promotion for LegalizeTypes.
11 // Promotion is the act of changing a computation in an illegal type into a
12 // computation in a larger type. For example, implementing i8 arithmetic in an
13 // i32 register (often needed on powerpc).
14 // Expansion is the act of changing a computation in an illegal type into a
15 // computation in two identical registers of a smaller type. For example,
16 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
17 // targets).
18 //
19 //===----------------------------------------------------------------------===//
20 
21 #include "LegalizeTypes.h"
22 #include "llvm/IR/DerivedTypes.h"
24 #include "llvm/Support/KnownBits.h"
26 using namespace llvm;
27 
28 #define DEBUG_TYPE "legalize-types"
29 
30 //===----------------------------------------------------------------------===//
31 // Integer Result Promotion
32 //===----------------------------------------------------------------------===//
33 
34 /// PromoteIntegerResult - This method is called when a result of a node is
35 /// found to be in need of promotion to a larger type. At this point, the node
36 /// may also have invalid operands or may have other results that need
37 /// expansion, we just know that (at least) one result needs promotion.
38 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
39  LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG);
40  dbgs() << "\n");
41  SDValue Res = SDValue();
42 
43  // See if the target wants to custom expand this node.
44  if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
45  LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
46  return;
47  }
48 
49  switch (N->getOpcode()) {
50  default:
51 #ifndef NDEBUG
52  dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
53  N->dump(&DAG); dbgs() << "\n";
54 #endif
55  llvm_unreachable("Do not know how to promote this operator!");
56  case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
57  case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
58  case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
59  case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
60  case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
61  case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
62  case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
63  case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
65  case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
66  case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break;
68  case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
70  Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
71  case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
72  case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
73  break;
74  case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
75  break;
76  case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
77  case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
78  case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
79  case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
80  case ISD::SMIN:
81  case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
82  case ISD::UMIN:
83  case ISD::UMAX: Res = PromoteIntRes_ZExtIntBinOp(N); break;
84 
85  case ISD::SHL: Res = PromoteIntRes_SHL(N); break;
87  Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
88  case ISD::SRA: Res = PromoteIntRes_SRA(N); break;
89  case ISD::SRL: Res = PromoteIntRes_SRL(N); break;
90  case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
91  case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
92  case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
93 
95  Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
97  Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
99  Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
100  case ISD::BUILD_VECTOR:
101  Res = PromoteIntRes_BUILD_VECTOR(N); break;
103  Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
104  case ISD::CONCAT_VECTORS:
105  Res = PromoteIntRes_CONCAT_VECTORS(N); break;
106 
110  Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
111 
112  case ISD::SIGN_EXTEND:
113  case ISD::ZERO_EXTEND:
114  case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
115 
116  case ISD::FP_TO_SINT:
117  case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
118 
119  case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
120 
121  case ISD::AND:
122  case ISD::OR:
123  case ISD::XOR:
124  case ISD::ADD:
125  case ISD::SUB:
126  case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
127 
128  case ISD::SDIV:
129  case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
130 
131  case ISD::UDIV:
132  case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
133 
134  case ISD::SADDO:
135  case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
136  case ISD::UADDO:
137  case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
138  case ISD::SMULO:
139  case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
140 
141  case ISD::ADDCARRY:
142  case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
143 
144  case ISD::ATOMIC_LOAD:
145  Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
146 
151  case ISD::ATOMIC_LOAD_OR:
158  case ISD::ATOMIC_SWAP:
159  Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
160 
163  Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
164  break;
165  }
166 
167  // If the result is null then the sub-method took care of registering it.
168  if (Res.getNode())
169  SetPromotedInteger(SDValue(N, ResNo), Res);
170 }
171 
172 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
173  unsigned ResNo) {
174  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
175  return GetPromotedInteger(Op);
176 }
177 
178 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
179  // Sign-extend the new bits, and continue the assertion.
180  SDValue Op = SExtPromotedInteger(N->getOperand(0));
181  return DAG.getNode(ISD::AssertSext, SDLoc(N),
182  Op.getValueType(), Op, N->getOperand(1));
183 }
184 
185 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
186  // Zero the new bits, and continue the assertion.
187  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
188  return DAG.getNode(ISD::AssertZext, SDLoc(N),
189  Op.getValueType(), Op, N->getOperand(1));
190 }
191 
192 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
193  EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
194  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
195  N->getMemoryVT(), ResVT,
196  N->getChain(), N->getBasePtr(),
197  N->getMemOperand());
198  // Legalize the chain result - switch anything that used the old chain to
199  // use the new one.
200  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
201  return Res;
202 }
203 
204 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
205  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
206  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
207  N->getMemoryVT(),
208  N->getChain(), N->getBasePtr(),
209  Op2, N->getMemOperand());
210  // Legalize the chain result - switch anything that used the old chain to
211  // use the new one.
212  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
213  return Res;
214 }
215 
216 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
217  unsigned ResNo) {
218  if (ResNo == 1) {
220  EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
221  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
222 
223  // Only use the result of getSetCCResultType if it is legal,
224  // otherwise just use the promoted result type (NVT).
225  if (!TLI.isTypeLegal(SVT))
226  SVT = NVT;
227 
228  SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
229  SDValue Res = DAG.getAtomicCmpSwap(
231  N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
232  N->getMemOperand());
233  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
234  ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
235  return Res.getValue(1);
236  }
237 
238  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
239  SDValue Op3 = GetPromotedInteger(N->getOperand(3));
240  SDVTList VTs =
241  DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
242  SDValue Res = DAG.getAtomicCmpSwap(
243  N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
244  N->getBasePtr(), Op2, Op3, N->getMemOperand());
245  // Update the use to N with the newly created Res.
246  for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
247  ReplaceValueWith(SDValue(N, i), Res.getValue(i));
248  return Res;
249 }
250 
251 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
252  SDValue InOp = N->getOperand(0);
253  EVT InVT = InOp.getValueType();
254  EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
255  EVT OutVT = N->getValueType(0);
256  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
257  SDLoc dl(N);
258 
259  switch (getTypeAction(InVT)) {
261  break;
263  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
264  // The input promotes to the same size. Convert the promoted value.
265  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
266  break;
268  // Promote the integer operand by hand.
269  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
271  // Convert the promoted float by hand.
272  if (!NOutVT.isVector())
273  return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
274  break;
275  }
278  break;
280  // Convert the element to an integer and promote it by hand.
281  if (!NOutVT.isVector())
282  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
283  BitConvertToInteger(GetScalarizedVector(InOp)));
284  break;
286  // For example, i32 = BITCAST v2i16 on alpha. Convert the split
287  // pieces of the input into integers and reassemble in the final type.
288  SDValue Lo, Hi;
289  GetSplitVector(N->getOperand(0), Lo, Hi);
290  Lo = BitConvertToInteger(Lo);
291  Hi = BitConvertToInteger(Hi);
292 
293  if (DAG.getDataLayout().isBigEndian())
294  std::swap(Lo, Hi);
295 
296  InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
298  NOutVT.getSizeInBits()),
299  JoinIntegers(Lo, Hi));
300  return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
301  }
303  // The input is widened to the same size. Convert to the widened value.
304  // Make sure that the outgoing value is not a vector, because this would
305  // make us bitcast between two vectors which are legalized in different ways.
306  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector())
307  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
308  }
309 
310  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
311  CreateStackStoreLoad(InOp, OutVT));
312 }
313 
314 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
315  SDValue Op = GetPromotedInteger(N->getOperand(0));
316  EVT OVT = N->getValueType(0);
317  EVT NVT = Op.getValueType();
318  SDLoc dl(N);
319 
320  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
321  return DAG.getNode(
322  ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
323  DAG.getConstant(DiffBits, dl,
324  TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
325 }
326 
327 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
328  SDValue Op = GetPromotedInteger(N->getOperand(0));
329  EVT OVT = N->getValueType(0);
330  EVT NVT = Op.getValueType();
331  SDLoc dl(N);
332 
333  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
334  return DAG.getNode(
335  ISD::SRL, dl, NVT, DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
336  DAG.getConstant(DiffBits, dl,
337  TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
338 }
339 
340 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
341  // The pair element type may be legal, or may not promote to the same type as
342  // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
343  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
344  TLI.getTypeToTransformTo(*DAG.getContext(),
345  N->getValueType(0)), JoinIntegers(N->getOperand(0),
346  N->getOperand(1)));
347 }
348 
349 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
350  EVT VT = N->getValueType(0);
351  // FIXME there is no actual debug info here
352  SDLoc dl(N);
353  // Zero extend things like i1, sign extend everything else. It shouldn't
354  // matter in theory which one we pick, but this tends to give better code?
355  unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
356  SDValue Result = DAG.getNode(Opc, dl,
357  TLI.getTypeToTransformTo(*DAG.getContext(), VT),
358  SDValue(N, 0));
359  assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
360  return Result;
361 }
362 
363 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
364  // Zero extend to the promoted type and do the count there.
365  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
366  SDLoc dl(N);
367  EVT OVT = N->getValueType(0);
368  EVT NVT = Op.getValueType();
369  Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
370  // Subtract off the extra leading bits in the bigger type.
371  return DAG.getNode(
372  ISD::SUB, dl, NVT, Op,
373  DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
374  NVT));
375 }
376 
377 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
378  // Zero extend to the promoted type and do the count there.
379  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
380  return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op);
381 }
382 
383 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
384  SDValue Op = GetPromotedInteger(N->getOperand(0));
385  EVT OVT = N->getValueType(0);
386  EVT NVT = Op.getValueType();
387  SDLoc dl(N);
388  if (N->getOpcode() == ISD::CTTZ) {
389  // The count is the same in the promoted type except if the original
390  // value was zero. This can be handled by setting the bit just off
391  // the top of the original type.
392  auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
393  OVT.getScalarSizeInBits());
394  Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
395  }
396  return DAG.getNode(N->getOpcode(), dl, NVT, Op);
397 }
398 
399 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
400  SDLoc dl(N);
401  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
402  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0),
403  N->getOperand(1));
404 }
405 
406 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
407  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
408  unsigned NewOpc = N->getOpcode();
409  SDLoc dl(N);
410 
411  // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
412  // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
413  // and SINT conversions are Custom, there is no way to tell which is
414  // preferable. We choose SINT because that's the right thing on PPC.)
415  if (N->getOpcode() == ISD::FP_TO_UINT &&
416  !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
418  NewOpc = ISD::FP_TO_SINT;
419 
420  SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
421 
422  // Assert that the converted value fits in the original type. If it doesn't
423  // (eg: because the value being converted is too big), then the result of the
424  // original operation was undefined anyway, so the assert is still correct.
425  //
426  // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
427  // before legalization: fp-to-uint16, 65534. -> 0xfffe
428  // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
429  return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ?
430  ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
432 }
433 
434 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
435  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
436  SDLoc dl(N);
437 
438  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
439 }
440 
441 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
442  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
443  SDLoc dl(N);
444 
445  if (getTypeAction(N->getOperand(0).getValueType())
447  SDValue Res = GetPromotedInteger(N->getOperand(0));
448  assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
449 
450  // If the result and operand types are the same after promotion, simplify
451  // to an in-register extension.
452  if (NVT == Res.getValueType()) {
453  // The high bits are not guaranteed to be anything. Insert an extend.
454  if (N->getOpcode() == ISD::SIGN_EXTEND)
455  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
456  DAG.getValueType(N->getOperand(0).getValueType()));
457  if (N->getOpcode() == ISD::ZERO_EXTEND)
458  return DAG.getZeroExtendInReg(Res, dl,
460  assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
461  return Res;
462  }
463  }
464 
465  // Otherwise, just extend the original operand all the way to the larger type.
466  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
467 }
468 
469 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
470  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
471  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
474  SDLoc dl(N);
475  SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
476  N->getMemoryVT(), N->getMemOperand());
477 
478  // Legalize the chain result - switch anything that used the old chain to
479  // use the new one.
480  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
481  return Res;
482 }
483 
484 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
485  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
486  SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
487 
488  SDLoc dl(N);
489  SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
490  N->getMask(), ExtPassThru, N->getMemoryVT(),
492  // Legalize the chain result - switch anything that used the old chain to
493  // use the new one.
494  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
495  return Res;
496 }
497 
498 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
499  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
500  SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
501  assert(NVT == ExtPassThru.getValueType() &&
502  "Gather result type and the passThru agrument type should be the same");
503 
504  SDLoc dl(N);
505  SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
506  N->getIndex(), N->getScale() };
507  SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
508  N->getMemoryVT(), dl, Ops,
509  N->getMemOperand());
510  // Legalize the chain result - switch anything that used the old chain to
511  // use the new one.
512  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
513  return Res;
514 }
515 
516 /// Promote the overflow flag of an overflowing arithmetic node.
517 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
518  // Simply change the return type of the boolean result.
519  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
520  EVT ValueVTs[] = { N->getValueType(0), NVT };
521  SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
522  unsigned NumOps = N->getNumOperands();
523  assert(NumOps <= 3 && "Too many operands");
524  if (NumOps == 3)
525  Ops[2] = N->getOperand(2);
526 
527  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
528  DAG.getVTList(ValueVTs), makeArrayRef(Ops, NumOps));
529 
530  // Modified the sum result - switch anything that used the old sum to use
531  // the new one.
532  ReplaceValueWith(SDValue(N, 0), Res);
533 
534  return SDValue(Res.getNode(), 1);
535 }
536 
537 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
538  if (ResNo == 1)
539  return PromoteIntRes_Overflow(N);
540 
541  // The operation overflowed iff the result in the larger type is not the
542  // sign extension of its truncation to the original type.
543  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
544  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
545  EVT OVT = N->getOperand(0).getValueType();
546  EVT NVT = LHS.getValueType();
547  SDLoc dl(N);
548 
549  // Do the arithmetic in the larger type.
550  unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
551  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
552 
553  // Calculate the overflow flag: sign extend the arithmetic result from
554  // the original type.
555  SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
556  DAG.getValueType(OVT));
557  // Overflowed if and only if this is not equal to Res.
558  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
559 
560  // Use the calculated overflow everywhere.
561  ReplaceValueWith(SDValue(N, 1), Ofl);
562 
563  return Res;
564 }
565 
566 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
567  SDValue LHS = GetPromotedInteger(N->getOperand(1));
568  SDValue RHS = GetPromotedInteger(N->getOperand(2));
569  return DAG.getSelect(SDLoc(N),
570  LHS.getValueType(), N->getOperand(0), LHS, RHS);
571 }
572 
573 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
574  SDValue Mask = N->getOperand(0);
575 
576  SDValue LHS = GetPromotedInteger(N->getOperand(1));
577  SDValue RHS = GetPromotedInteger(N->getOperand(2));
578  return DAG.getNode(ISD::VSELECT, SDLoc(N),
579  LHS.getValueType(), Mask, LHS, RHS);
580 }
581 
582 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
583  SDValue LHS = GetPromotedInteger(N->getOperand(2));
584  SDValue RHS = GetPromotedInteger(N->getOperand(3));
585  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
586  LHS.getValueType(), N->getOperand(0),
587  N->getOperand(1), LHS, RHS, N->getOperand(4));
588 }
589 
590 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
591  EVT InVT = N->getOperand(0).getValueType();
592  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
593 
594  EVT SVT = getSetCCResultType(InVT);
595 
596  // If we got back a type that needs to be promoted, this likely means the
597  // the input type also needs to be promoted. So get the promoted type for
598  // the input and try the query again.
599  if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
600  if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
601  InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
602  SVT = getSetCCResultType(InVT);
603  } else {
604  // Input type isn't promoted, just use the default promoted type.
605  SVT = NVT;
606  }
607  }
608 
609  SDLoc dl(N);
610  assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() &&
611  "Vector compare must return a vector result!");
612 
613  // Get the SETCC result using the canonical SETCC type.
614  SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
615  N->getOperand(1), N->getOperand(2));
616 
617  // Convert to the expected type.
618  return DAG.getSExtOrTrunc(SetCC, dl, NVT);
619 }
620 
621 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
622  SDValue LHS = GetPromotedInteger(N->getOperand(0));
623  SDValue RHS = N->getOperand(1);
624  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
625  RHS = ZExtPromotedInteger(RHS);
626  return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
627 }
628 
629 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
630  SDValue Op = GetPromotedInteger(N->getOperand(0));
631  return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
632  Op.getValueType(), Op, N->getOperand(1));
633 }
634 
635 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
636  // The input may have strange things in the top bits of the registers, but
637  // these operations don't care. They may have weird bits going out, but
638  // that too is okay if they are integer operations.
639  SDValue LHS = GetPromotedInteger(N->getOperand(0));
640  SDValue RHS = GetPromotedInteger(N->getOperand(1));
641  return DAG.getNode(N->getOpcode(), SDLoc(N),
642  LHS.getValueType(), LHS, RHS);
643 }
644 
645 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
646  // Sign extend the input.
647  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
648  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
649  return DAG.getNode(N->getOpcode(), SDLoc(N),
650  LHS.getValueType(), LHS, RHS);
651 }
652 
653 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
654  // Zero extend the input.
655  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
656  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
657  return DAG.getNode(N->getOpcode(), SDLoc(N),
658  LHS.getValueType(), LHS, RHS);
659 }
660 
661 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
662  // The input value must be properly sign extended.
663  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
664  SDValue RHS = N->getOperand(1);
665  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
666  RHS = ZExtPromotedInteger(RHS);
667  return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
668 }
669 
670 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
671  // The input value must be properly zero extended.
672  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
673  SDValue RHS = N->getOperand(1);
674  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
675  RHS = ZExtPromotedInteger(RHS);
676  return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
677 }
678 
679 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
680  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
681  SDValue Res;
682  SDValue InOp = N->getOperand(0);
683  SDLoc dl(N);
684 
685  switch (getTypeAction(InOp.getValueType())) {
686  default: llvm_unreachable("Unknown type action!");
689  Res = InOp;
690  break;
692  Res = GetPromotedInteger(InOp);
693  break;
695  EVT InVT = InOp.getValueType();
696  assert(InVT.isVector() && "Cannot split scalar types");
697  unsigned NumElts = InVT.getVectorNumElements();
698  assert(NumElts == NVT.getVectorNumElements() &&
699  "Dst and Src must have the same number of elements");
700  assert(isPowerOf2_32(NumElts) &&
701  "Promoted vector type must be a power of two");
702 
703  SDValue EOp1, EOp2;
704  GetSplitVector(InOp, EOp1, EOp2);
705 
706  EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
707  NumElts/2);
708  EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
709  EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
710 
711  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
712  }
714  SDValue WideInOp = GetWidenedVector(InOp);
715 
716  // Truncate widened InOp.
717  unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
718  EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
719  N->getValueType(0).getScalarType(), NumElem);
720  SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
721 
722  // Zero extend so that the elements are of same type as those of NVT
723  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
724  NumElem);
725  SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
726 
727  // Extract the low NVT subvector.
728  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
729  SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy);
730  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
731  }
732  }
733 
734  // Truncate to NVT instead of VT
735  return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
736 }
737 
738 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
739  if (ResNo == 1)
740  return PromoteIntRes_Overflow(N);
741 
742  // The operation overflowed iff the result in the larger type is not the
743  // zero extension of its truncation to the original type.
744  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
745  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
746  EVT OVT = N->getOperand(0).getValueType();
747  EVT NVT = LHS.getValueType();
748  SDLoc dl(N);
749 
750  // Do the arithmetic in the larger type.
751  unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
752  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
753 
754  // Calculate the overflow flag: zero extend the arithmetic result from
755  // the original type.
756  SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
757  // Overflowed if and only if this is not equal to Res.
758  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
759 
760  // Use the calculated overflow everywhere.
761  ReplaceValueWith(SDValue(N, 1), Ofl);
762 
763  return Res;
764 }
765 
766 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
767  if (ResNo == 1)
768  return PromoteIntRes_Overflow(N);
769 
770  // We need to sign-extend the operands so the carry value computed by the
771  // wide operation will be equivalent to the carry value computed by the
772  // narrow operation.
773  // An ADDCARRY can generate carry only if any of the operands has its
774  // most significant bit set. Sign extension propagates the most significant
775  // bit into the higher bits which means the extra bit that the narrow
776  // addition would need (i.e. the carry) will be propagated through the higher
777  // bits of the wide addition.
778  // A SUBCARRY can generate borrow only if LHS < RHS and this property will be
779  // preserved by sign extension.
780  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
781  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
782 
783  EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
784 
785  // Do the arithmetic in the wide type.
786  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
787  LHS, RHS, N->getOperand(2));
788 
789  // Update the users of the original carry/borrow value.
790  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
791 
792  return SDValue(Res.getNode(), 0);
793 }
794 
795 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
796  // Promote the overflow bit trivially.
797  if (ResNo == 1)
798  return PromoteIntRes_Overflow(N);
799 
800  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
801  SDLoc DL(N);
802  EVT SmallVT = LHS.getValueType();
803 
804  // To determine if the result overflowed in a larger type, we extend the
805  // input to the larger type, do the multiply (checking if it overflows),
806  // then also check the high bits of the result to see if overflow happened
807  // there.
808  if (N->getOpcode() == ISD::SMULO) {
809  LHS = SExtPromotedInteger(LHS);
810  RHS = SExtPromotedInteger(RHS);
811  } else {
812  LHS = ZExtPromotedInteger(LHS);
813  RHS = ZExtPromotedInteger(RHS);
814  }
815  SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
816  SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
817 
818  // Overflow occurred if it occurred in the larger type, or if the high part
819  // of the result does not zero/sign-extend the low part. Check this second
820  // possibility first.
821  SDValue Overflow;
822  if (N->getOpcode() == ISD::UMULO) {
823  // Unsigned overflow occurred if the high part is non-zero.
824  SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
825  DAG.getIntPtrConstant(SmallVT.getSizeInBits(),
826  DL));
827  Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
828  DAG.getConstant(0, DL, Hi.getValueType()),
829  ISD::SETNE);
830  } else {
831  // Signed overflow occurred if the high part does not sign extend the low.
832  SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
833  Mul, DAG.getValueType(SmallVT));
834  Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
835  }
836 
837  // The only other way for overflow to occur is if the multiplication in the
838  // larger type itself overflowed.
839  Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
840  SDValue(Mul.getNode(), 1));
841 
842  // Use the calculated overflow everywhere.
843  ReplaceValueWith(SDValue(N, 1), Overflow);
844  return Mul;
845 }
846 
847 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
848  return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
849  N->getValueType(0)));
850 }
851 
852 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
853  SDValue Chain = N->getOperand(0); // Get the chain.
854  SDValue Ptr = N->getOperand(1); // Get the pointer.
855  EVT VT = N->getValueType(0);
856  SDLoc dl(N);
857 
858  MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
859  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
860  // The argument is passed as NumRegs registers of type RegVT.
861 
862  SmallVector<SDValue, 8> Parts(NumRegs);
863  for (unsigned i = 0; i < NumRegs; ++i) {
864  Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
865  N->getConstantOperandVal(3));
866  Chain = Parts[i].getValue(1);
867  }
868 
869  // Handle endianness of the load.
870  if (DAG.getDataLayout().isBigEndian())
871  std::reverse(Parts.begin(), Parts.end());
872 
873  // Assemble the parts in the promoted type.
874  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
875  SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
876  for (unsigned i = 1; i < NumRegs; ++i) {
877  SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
878  // Shift it to the right position and "or" it in.
879  Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
880  DAG.getConstant(i * RegVT.getSizeInBits(), dl,
881  TLI.getPointerTy(DAG.getDataLayout())));
882  Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
883  }
884 
885  // Modified the chain result - switch anything that used the old chain to
886  // use the new one.
887  ReplaceValueWith(SDValue(N, 1), Chain);
888 
889  return Res;
890 }
891 
892 //===----------------------------------------------------------------------===//
893 // Integer Operand Promotion
894 //===----------------------------------------------------------------------===//
895 
896 /// PromoteIntegerOperand - This method is called when the specified operand of
897 /// the specified node is found to need promotion. At this point, all of the
898 /// result types of the node are known to be legal, but other operands of the
899 /// node may need promotion or expansion as well as the specified one.
900 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
901  LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG);
902  dbgs() << "\n");
903  SDValue Res = SDValue();
904 
905  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
906  LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
907  return false;
908  }
909 
910  switch (N->getOpcode()) {
911  default:
912  #ifndef NDEBUG
913  dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
914  N->dump(&DAG); dbgs() << "\n";
915  #endif
916  llvm_unreachable("Do not know how to promote this operator's operand!");
917 
918  case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
919  case ISD::ATOMIC_STORE:
920  Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
921  break;
922  case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
923  case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
924  case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
925  case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
926  case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
927  case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
928  case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
930  Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
932  Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
933  case ISD::VSELECT:
934  case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
935  case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
936  case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
937  case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
938  case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
939  case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
940  OpNo); break;
941  case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
942  OpNo); break;
943  case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
944  OpNo); break;
945  case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
946  OpNo); break;
947  case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
948  OpNo); break;
949  case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
950  case ISD::FP16_TO_FP:
951  case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
952  case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
953  case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
954 
955  case ISD::SHL:
956  case ISD::SRA:
957  case ISD::SRL:
958  case ISD::ROTL:
959  case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
960 
961  case ISD::ADDCARRY:
962  case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
963  }
964 
965  // If the result is null, the sub-method took care of registering results etc.
966  if (!Res.getNode()) return false;
967 
968  // If the result is N, the sub-method updated N in place. Tell the legalizer
969  // core about this.
970  if (Res.getNode() == N)
971  return true;
972 
973  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
974  "Invalid operand expansion");
975 
976  ReplaceValueWith(SDValue(N, 0), Res);
977  return false;
978 }
979 
980 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
981 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
982 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
983  ISD::CondCode CCCode) {
984  // We have to insert explicit sign or zero extends. Note that we could
985  // insert sign extends for ALL conditions, but zero extend is cheaper on
986  // many machines (an AND instead of two shifts), so prefer it.
987  switch (CCCode) {
988  default: llvm_unreachable("Unknown integer comparison!");
989  case ISD::SETEQ:
990  case ISD::SETNE: {
991  SDValue OpL = GetPromotedInteger(NewLHS);
992  SDValue OpR = GetPromotedInteger(NewRHS);
993 
994  // We would prefer to promote the comparison operand with sign extension.
995  // If the width of OpL/OpR excluding the duplicated sign bits is no greater
996  // than the width of NewLHS/NewRH, we can avoid inserting real truncate
997  // instruction, which is redudant eventually.
998  unsigned OpLEffectiveBits =
999  OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
1000  unsigned OpREffectiveBits =
1001  OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
1002  if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() &&
1003  OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()) {
1004  NewLHS = OpL;
1005  NewRHS = OpR;
1006  } else {
1007  NewLHS = ZExtPromotedInteger(NewLHS);
1008  NewRHS = ZExtPromotedInteger(NewRHS);
1009  }
1010  break;
1011  }
1012  case ISD::SETUGE:
1013  case ISD::SETUGT:
1014  case ISD::SETULE:
1015  case ISD::SETULT:
1016  // ALL of these operations will work if we either sign or zero extend
1017  // the operands (including the unsigned comparisons!). Zero extend is
1018  // usually a simpler/cheaper operation, so prefer it.
1019  NewLHS = ZExtPromotedInteger(NewLHS);
1020  NewRHS = ZExtPromotedInteger(NewRHS);
1021  break;
1022  case ISD::SETGE:
1023  case ISD::SETGT:
1024  case ISD::SETLT:
1025  case ISD::SETLE:
1026  NewLHS = SExtPromotedInteger(NewLHS);
1027  NewRHS = SExtPromotedInteger(NewRHS);
1028  break;
1029  }
1030 }
1031 
1032 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1033  SDValue Op = GetPromotedInteger(N->getOperand(0));
1034  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1035 }
1036 
1037 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1038  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1039  return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1040  N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1041 }
1042 
1043 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1044  // This should only occur in unusual situations like bitcasting to an
1045  // x86_fp80, so just turn it into a store+load
1046  return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1047 }
1048 
1049 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1050  assert(OpNo == 2 && "Don't know how to promote this operand!");
1051 
1052  SDValue LHS = N->getOperand(2);
1053  SDValue RHS = N->getOperand(3);
1054  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1055 
1056  // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1057  // legal types.
1058  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1059  N->getOperand(1), LHS, RHS, N->getOperand(4)),
1060  0);
1061 }
1062 
1063 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1064  assert(OpNo == 1 && "only know how to promote condition");
1065 
1066  // Promote all the way up to the canonical SetCC type.
1067  SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1068 
1069  // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1070  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1071  N->getOperand(2)), 0);
1072 }
1073 
1074 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1075  // Since the result type is legal, the operands must promote to it.
1076  EVT OVT = N->getOperand(0).getValueType();
1077  SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1078  SDValue Hi = GetPromotedInteger(N->getOperand(1));
1079  assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1080  SDLoc dl(N);
1081 
1082  Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1083  DAG.getConstant(OVT.getSizeInBits(), dl,
1084  TLI.getPointerTy(DAG.getDataLayout())));
1085  return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1086 }
1087 
1088 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1089  // The vector type is legal but the element type is not. This implies
1090  // that the vector is a power-of-two in length and that the element
1091  // type does not have a strange size (eg: it is not i1).
1092  EVT VecVT = N->getValueType(0);
1093  unsigned NumElts = VecVT.getVectorNumElements();
1094  assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1095  "Legal vector of one illegal element?");
1096 
1097  // Promote the inserted value. The type does not need to match the
1098  // vector element type. Check that any extra bits introduced will be
1099  // truncated away.
1102  "Type of inserted value narrower than vector element type!");
1103 
1104  SmallVector<SDValue, 16> NewOps;
1105  for (unsigned i = 0; i < NumElts; ++i)
1106  NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1107 
1108  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1109 }
1110 
1111 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1112  unsigned OpNo) {
1113  if (OpNo == 1) {
1114  // Promote the inserted value. This is valid because the type does not
1115  // have to match the vector element type.
1116 
1117  // Check that any extra bits introduced will be truncated away.
1120  "Type of inserted value narrower than vector element type!");
1121  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1122  GetPromotedInteger(N->getOperand(1)),
1123  N->getOperand(2)),
1124  0);
1125  }
1126 
1127  assert(OpNo == 2 && "Different operand and result vector types?");
1128 
1129  // Promote the index.
1130  SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1131  TLI.getVectorIdxTy(DAG.getDataLayout()));
1132  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1133  N->getOperand(1), Idx), 0);
1134 }
1135 
1136 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1137  // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1138  // the operand in place.
1139  return SDValue(DAG.UpdateNodeOperands(N,
1140  GetPromotedInteger(N->getOperand(0))), 0);
1141 }
1142 
1143 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1144  assert(OpNo == 0 && "Only know how to promote the condition!");
1145  SDValue Cond = N->getOperand(0);
1146  EVT OpTy = N->getOperand(1).getValueType();
1147 
1148  if (N->getOpcode() == ISD::VSELECT)
1149  if (SDValue Res = WidenVSELECTAndMask(N))
1150  return Res;
1151 
1152  // Promote all the way up to the canonical SetCC type.
1153  EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1154  Cond = PromoteTargetBoolean(Cond, OpVT);
1155 
1156  return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1157  N->getOperand(2)), 0);
1158 }
1159 
1160 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1161  assert(OpNo == 0 && "Don't know how to promote this operand!");
1162 
1163  SDValue LHS = N->getOperand(0);
1164  SDValue RHS = N->getOperand(1);
1165  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1166 
1167  // The CC (#4) and the possible return values (#2 and #3) have legal types.
1168  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1169  N->getOperand(3), N->getOperand(4)), 0);
1170 }
1171 
1172 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1173  assert(OpNo == 0 && "Don't know how to promote this operand!");
1174 
1175  SDValue LHS = N->getOperand(0);
1176  SDValue RHS = N->getOperand(1);
1177  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1178 
1179  // The CC (#2) is always legal.
1180  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1181 }
1182 
1183 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1184  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1185  ZExtPromotedInteger(N->getOperand(1))), 0);
1186 }
1187 
1188 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1189  SDValue Op = GetPromotedInteger(N->getOperand(0));
1190  SDLoc dl(N);
1191  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1192  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1193  Op, DAG.getValueType(N->getOperand(0).getValueType()));
1194 }
1195 
1196 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1197  return SDValue(DAG.UpdateNodeOperands(N,
1198  SExtPromotedInteger(N->getOperand(0))), 0);
1199 }
1200 
1201 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1202  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1203  SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1204  SDLoc dl(N);
1205 
1206  SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1207 
1208  // Truncate the value and store the result.
1209  return DAG.getTruncStore(Ch, dl, Val, Ptr,
1210  N->getMemoryVT(), N->getMemOperand());
1211 }
1212 
1213 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1214  unsigned OpNo) {
1215 
1216  SDValue DataOp = N->getValue();
1217  EVT DataVT = DataOp.getValueType();
1218  SDValue Mask = N->getMask();
1219  SDLoc dl(N);
1220 
1221  bool TruncateStore = false;
1222  if (OpNo == 3) {
1223  Mask = PromoteTargetBoolean(Mask, DataVT);
1224  // Update in place.
1225  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1226  NewOps[3] = Mask;
1227  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1228  } else { // Data operand
1229  assert(OpNo == 1 && "Unexpected operand for promotion");
1230  DataOp = GetPromotedInteger(DataOp);
1231  TruncateStore = true;
1232  }
1233 
1234  return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask,
1235  N->getMemoryVT(), N->getMemOperand(),
1236  TruncateStore, N->isCompressingStore());
1237 }
1238 
1239 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1240  unsigned OpNo) {
1241  assert(OpNo == 2 && "Only know how to promote the mask!");
1242  EVT DataVT = N->getValueType(0);
1243  SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1244  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1245  NewOps[OpNo] = Mask;
1246  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1247 }
1248 
1249 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1250  unsigned OpNo) {
1251 
1252  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1253  if (OpNo == 2) {
1254  // The Mask
1255  EVT DataVT = N->getValueType(0);
1256  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1257  } else if (OpNo == 4) {
1258  // Need to sign extend the index since the bits will likely be used.
1259  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1260  } else
1261  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1262 
1263  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1264 }
1265 
1266 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1267  unsigned OpNo) {
1268  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1269  if (OpNo == 2) {
1270  // The Mask
1271  EVT DataVT = N->getValue().getValueType();
1272  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1273  } else if (OpNo == 4) {
1274  // Need to sign extend the index since the bits will likely be used.
1275  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1276  } else
1277  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1278  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1279 }
1280 
1281 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
1282  SDValue Op = GetPromotedInteger(N->getOperand(0));
1283  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
1284 }
1285 
1286 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
1287  return SDValue(DAG.UpdateNodeOperands(N,
1288  ZExtPromotedInteger(N->getOperand(0))), 0);
1289 }
1290 
1291 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
1292  SDLoc dl(N);
1293  SDValue Op = GetPromotedInteger(N->getOperand(0));
1294  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1295  return DAG.getZeroExtendInReg(Op, dl,
1297 }
1298 
1299 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
1300  assert(OpNo == 2 && "Don't know how to promote this operand!");
1301 
1302  SDValue LHS = N->getOperand(0);
1303  SDValue RHS = N->getOperand(1);
1304  SDValue Carry = N->getOperand(2);
1305  SDLoc DL(N);
1306 
1307  auto VT = getSetCCResultType(LHS.getValueType());
1309  switch (BoolType) {
1311  Carry = DAG.getAnyExtOrTrunc(Carry, DL, VT);
1312  break;
1314  Carry = DAG.getZExtOrTrunc(Carry, DL, VT);
1315  break;
1317  Carry = DAG.getSExtOrTrunc(Carry, DL, VT);
1318  break;
1319  }
1320 
1321  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
1322 }
1323 
1324 //===----------------------------------------------------------------------===//
1325 // Integer Result Expansion
1326 //===----------------------------------------------------------------------===//
1327 
1328 /// ExpandIntegerResult - This method is called when the specified result of the
1329 /// specified node is found to need expansion. At this point, the node may also
1330 /// have invalid operands or may have other results that need promotion, we just
1331 /// know that (at least) one result needs expansion.
1332 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
1333  LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG);
1334  dbgs() << "\n");
1335  SDValue Lo, Hi;
1336  Lo = Hi = SDValue();
1337 
1338  // See if the target wants to custom expand this node.
1339  if (CustomLowerNode(N, N->getValueType(ResNo), true))
1340  return;
1341 
1342  switch (N->getOpcode()) {
1343  default:
1344 #ifndef NDEBUG
1345  dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
1346  N->dump(&DAG); dbgs() << "\n";
1347 #endif
1348  llvm_unreachable("Do not know how to expand the result of this operator!");
1349 
1350  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1351  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
1352  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1353  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1354 
1355  case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1356  case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1357  case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1358  case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1359  case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1360 
1361  case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
1362  case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
1363  case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
1364  case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
1365  case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
1366  case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
1367  case ISD::CTLZ_ZERO_UNDEF:
1368  case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
1369  case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
1370  case ISD::CTTZ_ZERO_UNDEF:
1371  case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
1372  case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
1373  case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
1374  case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
1375  case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
1376  case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
1377  case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
1378  case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
1379  case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
1380  case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
1381  case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
1382  case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
1383  case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
1384  case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
1385  case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
1386  case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
1387 
1388  case ISD::ATOMIC_LOAD_ADD:
1389  case ISD::ATOMIC_LOAD_SUB:
1390  case ISD::ATOMIC_LOAD_AND:
1391  case ISD::ATOMIC_LOAD_CLR:
1392  case ISD::ATOMIC_LOAD_OR:
1393  case ISD::ATOMIC_LOAD_XOR:
1394  case ISD::ATOMIC_LOAD_NAND:
1395  case ISD::ATOMIC_LOAD_MIN:
1396  case ISD::ATOMIC_LOAD_MAX:
1397  case ISD::ATOMIC_LOAD_UMIN:
1398  case ISD::ATOMIC_LOAD_UMAX:
1399  case ISD::ATOMIC_SWAP:
1400  case ISD::ATOMIC_CMP_SWAP: {
1401  std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
1402  SplitInteger(Tmp.first, Lo, Hi);
1403  ReplaceValueWith(SDValue(N, 1), Tmp.second);
1404  break;
1405  }
1407  AtomicSDNode *AN = cast<AtomicSDNode>(N);
1408  SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
1409  SDValue Tmp = DAG.getAtomicCmpSwap(
1410  ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
1411  N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
1412  AN->getMemOperand());
1413 
1414  // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
1415  // success simply by comparing the loaded value against the ingoing
1416  // comparison.
1417  SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
1418  N->getOperand(2), ISD::SETEQ);
1419 
1420  SplitInteger(Tmp, Lo, Hi);
1421  ReplaceValueWith(SDValue(N, 1), Success);
1422  ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
1423  break;
1424  }
1425 
1426  case ISD::AND:
1427  case ISD::OR:
1428  case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
1429 
1430  case ISD::UMAX:
1431  case ISD::SMAX:
1432  case ISD::UMIN:
1433  case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
1434 
1435  case ISD::ADD:
1436  case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
1437 
1438  case ISD::ADDC:
1439  case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
1440 
1441  case ISD::ADDE:
1442  case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
1443 
1444  case ISD::ADDCARRY:
1445  case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
1446 
1447  case ISD::SHL:
1448  case ISD::SRA:
1449  case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
1450 
1451  case ISD::SADDO:
1452  case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
1453  case ISD::UADDO:
1454  case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
1455  case ISD::UMULO:
1456  case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
1457  }
1458 
1459  // If Lo/Hi is null, the sub-method took care of registering results etc.
1460  if (Lo.getNode())
1461  SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
1462 }
1463 
1464 /// Lower an atomic node to the appropriate builtin call.
1465 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
1466  unsigned Opc = Node->getOpcode();
1467  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
1468  RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
1469  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
1470 
1471  return ExpandChainLibCall(LC, Node, false);
1472 }
1473 
1474 /// N is a shift by a value that needs to be expanded,
1475 /// and the shift amount is a constant 'Amt'. Expand the operation.
1476 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
1477  SDValue &Lo, SDValue &Hi) {
1478  SDLoc DL(N);
1479  // Expand the incoming operand to be shifted, so that we have its parts
1480  SDValue InL, InH;
1481  GetExpandedInteger(N->getOperand(0), InL, InH);
1482 
1483  // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
1484  // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
1485  if (!Amt) {
1486  Lo = InL;
1487  Hi = InH;
1488  return;
1489  }
1490 
1491  EVT NVT = InL.getValueType();
1492  unsigned VTBits = N->getValueType(0).getSizeInBits();
1493  unsigned NVTBits = NVT.getSizeInBits();
1494  EVT ShTy = N->getOperand(1).getValueType();
1495 
1496  if (N->getOpcode() == ISD::SHL) {
1497  if (Amt.ugt(VTBits)) {
1498  Lo = Hi = DAG.getConstant(0, DL, NVT);
1499  } else if (Amt.ugt(NVTBits)) {
1500  Lo = DAG.getConstant(0, DL, NVT);
1501  Hi = DAG.getNode(ISD::SHL, DL,
1502  NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1503  } else if (Amt == NVTBits) {
1504  Lo = DAG.getConstant(0, DL, NVT);
1505  Hi = InL;
1506  } else {
1507  Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
1508  Hi = DAG.getNode(ISD::OR, DL, NVT,
1509  DAG.getNode(ISD::SHL, DL, NVT, InH,
1510  DAG.getConstant(Amt, DL, ShTy)),
1511  DAG.getNode(ISD::SRL, DL, NVT, InL,
1512  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1513  }
1514  return;
1515  }
1516 
1517  if (N->getOpcode() == ISD::SRL) {
1518  if (Amt.ugt(VTBits)) {
1519  Lo = Hi = DAG.getConstant(0, DL, NVT);
1520  } else if (Amt.ugt(NVTBits)) {
1521  Lo = DAG.getNode(ISD::SRL, DL,
1522  NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1523  Hi = DAG.getConstant(0, DL, NVT);
1524  } else if (Amt == NVTBits) {
1525  Lo = InH;
1526  Hi = DAG.getConstant(0, DL, NVT);
1527  } else {
1528  Lo = DAG.getNode(ISD::OR, DL, NVT,
1529  DAG.getNode(ISD::SRL, DL, NVT, InL,
1530  DAG.getConstant(Amt, DL, ShTy)),
1531  DAG.getNode(ISD::SHL, DL, NVT, InH,
1532  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1533  Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1534  }
1535  return;
1536  }
1537 
1538  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
1539  if (Amt.ugt(VTBits)) {
1540  Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1541  DAG.getConstant(NVTBits - 1, DL, ShTy));
1542  } else if (Amt.ugt(NVTBits)) {
1543  Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1544  DAG.getConstant(Amt - NVTBits, DL, ShTy));
1545  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1546  DAG.getConstant(NVTBits - 1, DL, ShTy));
1547  } else if (Amt == NVTBits) {
1548  Lo = InH;
1549  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1550  DAG.getConstant(NVTBits - 1, DL, ShTy));
1551  } else {
1552  Lo = DAG.getNode(ISD::OR, DL, NVT,
1553  DAG.getNode(ISD::SRL, DL, NVT, InL,
1554  DAG.getConstant(Amt, DL, ShTy)),
1555  DAG.getNode(ISD::SHL, DL, NVT, InH,
1556  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1557  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1558  }
1559 }
1560 
1561 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
1562 /// this shift based on knowledge of the high bit of the shift amount. If we
1563 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
1564 /// shift amount.
1565 bool DAGTypeLegalizer::
1566 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1567  SDValue Amt = N->getOperand(1);
1568  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1569  EVT ShTy = Amt.getValueType();
1570  unsigned ShBits = ShTy.getScalarSizeInBits();
1571  unsigned NVTBits = NVT.getScalarSizeInBits();
1572  assert(isPowerOf2_32(NVTBits) &&
1573  "Expanded integer type size not a power of two!");
1574  SDLoc dl(N);
1575 
1576  APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
1577  KnownBits Known;
1578  DAG.computeKnownBits(N->getOperand(1), Known);
1579 
1580  // If we don't know anything about the high bits, exit.
1581  if (((Known.Zero|Known.One) & HighBitMask) == 0)
1582  return false;
1583 
1584  // Get the incoming operand to be shifted.
1585  SDValue InL, InH;
1586  GetExpandedInteger(N->getOperand(0), InL, InH);
1587 
1588  // If we know that any of the high bits of the shift amount are one, then we
1589  // can do this as a couple of simple shifts.
1590  if (Known.One.intersects(HighBitMask)) {
1591  // Mask out the high bit, which we know is set.
1592  Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
1593  DAG.getConstant(~HighBitMask, dl, ShTy));
1594 
1595  switch (N->getOpcode()) {
1596  default: llvm_unreachable("Unknown shift");
1597  case ISD::SHL:
1598  Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
1599  Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
1600  return true;
1601  case ISD::SRL:
1602  Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1603  Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
1604  return true;
1605  case ISD::SRA:
1606  Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
1607  DAG.getConstant(NVTBits - 1, dl, ShTy));
1608  Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
1609  return true;
1610  }
1611  }
1612 
1613  // If we know that all of the high bits of the shift amount are zero, then we
1614  // can do this as a couple of simple shifts.
1615  if (HighBitMask.isSubsetOf(Known.Zero)) {
1616  // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
1617  // shift if x is zero. We can use XOR here because x is known to be smaller
1618  // than 32.
1619  SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
1620  DAG.getConstant(NVTBits - 1, dl, ShTy));
1621 
1622  unsigned Op1, Op2;
1623  switch (N->getOpcode()) {
1624  default: llvm_unreachable("Unknown shift");
1625  case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
1626  case ISD::SRL:
1627  case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
1628  }
1629 
1630  // When shifting right the arithmetic for Lo and Hi is swapped.
1631  if (N->getOpcode() != ISD::SHL)
1632  std::swap(InL, InH);
1633 
1634  // Use a little trick to get the bits that move from Lo to Hi. First
1635  // shift by one bit.
1636  SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
1637  // Then compute the remaining shift with amount-1.
1638  SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
1639 
1640  Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
1641  Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
1642 
1643  if (N->getOpcode() != ISD::SHL)
1644  std::swap(Hi, Lo);
1645  return true;
1646  }
1647 
1648  return false;
1649 }
1650 
1651 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
1652 /// of any size.
1653 bool DAGTypeLegalizer::
1654 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1655  SDValue Amt = N->getOperand(1);
1656  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1657  EVT ShTy = Amt.getValueType();
1658  unsigned NVTBits = NVT.getSizeInBits();
1659  assert(isPowerOf2_32(NVTBits) &&
1660  "Expanded integer type size not a power of two!");
1661  SDLoc dl(N);
1662 
1663  // Get the incoming operand to be shifted.
1664  SDValue InL, InH;
1665  GetExpandedInteger(N->getOperand(0), InL, InH);
1666 
1667  SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
1668  SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
1669  SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
1670  SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1671  Amt, NVBitsNode, ISD::SETULT);
1672  SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1673  Amt, DAG.getConstant(0, dl, ShTy),
1674  ISD::SETEQ);
1675 
1676  SDValue LoS, HiS, LoL, HiL;
1677  switch (N->getOpcode()) {
1678  default: llvm_unreachable("Unknown shift");
1679  case ISD::SHL:
1680  // Short: ShAmt < NVTBits
1681  LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
1682  HiS = DAG.getNode(ISD::OR, dl, NVT,
1683  DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
1684  DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
1685 
1686  // Long: ShAmt >= NVTBits
1687  LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
1688  HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
1689 
1690  Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
1691  Hi = DAG.getSelect(dl, NVT, isZero, InH,
1692  DAG.getSelect(dl, NVT, isShort, HiS, HiL));
1693  return true;
1694  case ISD::SRL:
1695  // Short: ShAmt < NVTBits
1696  HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
1697  LoS = DAG.getNode(ISD::OR, dl, NVT,
1698  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1699  // FIXME: If Amt is zero, the following shift generates an undefined result
1700  // on some architectures.
1701  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1702 
1703  // Long: ShAmt >= NVTBits
1704  HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1705  LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1706 
1707  Lo = DAG.getSelect(dl, NVT, isZero, InL,
1708  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1709  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1710  return true;
1711  case ISD::SRA:
1712  // Short: ShAmt < NVTBits
1713  HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
1714  LoS = DAG.getNode(ISD::OR, dl, NVT,
1715  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1716  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1717 
1718  // Long: ShAmt >= NVTBits
1719  HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
1720  DAG.getConstant(NVTBits - 1, dl, ShTy));
1721  LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1722 
1723  Lo = DAG.getSelect(dl, NVT, isZero, InL,
1724  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1725  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1726  return true;
1727  }
1728 }
1729 
1730 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
1731 
1732  switch (Op) {
1733  default: llvm_unreachable("invalid min/max opcode");
1734  case ISD::SMAX:
1735  return std::make_pair(ISD::SETGT, ISD::UMAX);
1736  case ISD::UMAX:
1737  return std::make_pair(ISD::SETUGT, ISD::UMAX);
1738  case ISD::SMIN:
1739  return std::make_pair(ISD::SETLT, ISD::UMIN);
1740  case ISD::UMIN:
1741  return std::make_pair(ISD::SETULT, ISD::UMIN);
1742  }
1743 }
1744 
1745 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
1746  SDValue &Lo, SDValue &Hi) {
1747  SDLoc DL(N);
1748  ISD::NodeType LoOpc;
1749  ISD::CondCode CondC;
1750  std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
1751 
1752  // Expand the subcomponents.
1753  SDValue LHSL, LHSH, RHSL, RHSH;
1754  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1755  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1756 
1757  // Value types
1758  EVT NVT = LHSL.getValueType();
1759  EVT CCT = getSetCCResultType(NVT);
1760 
1761  // Hi part is always the same op
1762  Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
1763 
1764  // We need to know whether to select Lo part that corresponds to 'winning'
1765  // Hi part or if Hi parts are equal.
1766  SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
1767  SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
1768 
1769  // Lo part corresponding to the 'winning' Hi part
1770  SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
1771 
1772  // Recursed Lo part if Hi parts are equal, this uses unsigned version
1773  SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
1774 
1775  Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
1776 }
1777 
1778 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
1779  SDValue &Lo, SDValue &Hi) {
1780  SDLoc dl(N);
1781  // Expand the subcomponents.
1782  SDValue LHSL, LHSH, RHSL, RHSH;
1783  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1784  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1785 
1786  EVT NVT = LHSL.getValueType();
1787  SDValue LoOps[2] = { LHSL, RHSL };
1788  SDValue HiOps[3] = { LHSH, RHSH };
1789 
1790  bool HasOpCarry = TLI.isOperationLegalOrCustom(
1792  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1793  if (HasOpCarry) {
1794  SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
1795  if (N->getOpcode() == ISD::ADD) {
1796  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
1797  HiOps[2] = Lo.getValue(1);
1798  Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
1799  } else {
1800  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
1801  HiOps[2] = Lo.getValue(1);
1802  Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
1803  }
1804  return;
1805  }
1806 
1807  // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
1808  // them. TODO: Teach operation legalization how to expand unsupported
1809  // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
1810  // a carry of type MVT::Glue, but there doesn't seem to be any way to
1811  // generate a value of this type in the expanded code sequence.
1812  bool hasCarry =
1814  ISD::ADDC : ISD::SUBC,
1815  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1816 
1817  if (hasCarry) {
1818  SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
1819  if (N->getOpcode() == ISD::ADD) {
1820  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
1821  HiOps[2] = Lo.getValue(1);
1822  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
1823  } else {
1824  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
1825  HiOps[2] = Lo.getValue(1);
1826  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
1827  }
1828  return;
1829  }
1830 
1831  bool hasOVF =
1834  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1836 
1837  if (hasOVF) {
1838  EVT OvfVT = getSetCCResultType(NVT);
1839  SDVTList VTList = DAG.getVTList(NVT, OvfVT);
1840  int RevOpc;
1841  if (N->getOpcode() == ISD::ADD) {
1842  RevOpc = ISD::SUB;
1843  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
1844  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
1845  } else {
1846  RevOpc = ISD::ADD;
1847  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
1848  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
1849  }
1850  SDValue OVF = Lo.getValue(1);
1851 
1852  switch (BoolType) {
1854  OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
1857  OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
1858  Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
1859  break;
1861  OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
1862  Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
1863  }
1864  return;
1865  }
1866 
1867  if (N->getOpcode() == ISD::ADD) {
1868  Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
1869  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
1870  SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
1871  ISD::SETULT);
1872 
1874  SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
1875  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
1876  return;
1877  }
1878 
1879  SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
1880  DAG.getConstant(1, dl, NVT),
1881  DAG.getConstant(0, dl, NVT));
1882  SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
1883  ISD::SETULT);
1884  SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
1885  DAG.getConstant(1, dl, NVT), Carry1);
1886  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
1887  } else {
1888  Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
1889  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
1890  SDValue Cmp =
1891  DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
1892  LoOps[0], LoOps[1], ISD::SETULT);
1893 
1894  SDValue Borrow;
1896  Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
1897  else
1898  Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
1899  DAG.getConstant(0, dl, NVT));
1900 
1901  Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
1902  }
1903 }
1904 
1905 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
1906  SDValue &Lo, SDValue &Hi) {
1907  // Expand the subcomponents.
1908  SDValue LHSL, LHSH, RHSL, RHSH;
1909  SDLoc dl(N);
1910  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1911  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1912  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
1913  SDValue LoOps[2] = { LHSL, RHSL };
1914  SDValue HiOps[3] = { LHSH, RHSH };
1915 
1916  if (N->getOpcode() == ISD::ADDC) {
1917  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
1918  HiOps[2] = Lo.getValue(1);
1919  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
1920  } else {
1921  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
1922  HiOps[2] = Lo.getValue(1);
1923  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
1924  }
1925 
1926  // Legalized the flag result - switch anything that used the old flag to
1927  // use the new one.
1928  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
1929 }
1930 
1931 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
1932  SDValue &Lo, SDValue &Hi) {
1933  // Expand the subcomponents.
1934  SDValue LHSL, LHSH, RHSL, RHSH;
1935  SDLoc dl(N);
1936  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1937  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1938  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
1939  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
1940  SDValue HiOps[3] = { LHSH, RHSH };
1941 
1942  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
1943  HiOps[2] = Lo.getValue(1);
1944  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
1945 
1946  // Legalized the flag result - switch anything that used the old flag to
1947  // use the new one.
1948  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
1949 }
1950 
1951 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
1952  SDValue &Lo, SDValue &Hi) {
1953  SDValue LHS = N->getOperand(0);
1954  SDValue RHS = N->getOperand(1);
1955  SDLoc dl(N);
1956 
1957  SDValue Ovf;
1958 
1959  bool HasOpCarry = TLI.isOperationLegalOrCustom(
1961  TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
1962 
1963  if (HasOpCarry) {
1964  // Expand the subcomponents.
1965  SDValue LHSL, LHSH, RHSL, RHSH;
1966  GetExpandedInteger(LHS, LHSL, LHSH);
1967  GetExpandedInteger(RHS, RHSL, RHSH);
1968  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
1969  SDValue LoOps[2] = { LHSL, RHSL };
1970  SDValue HiOps[3] = { LHSH, RHSH };
1971 
1972  unsigned Opc = N->getOpcode() == ISD::UADDO ? ISD::ADDCARRY : ISD::SUBCARRY;
1973  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
1974  HiOps[2] = Lo.getValue(1);
1975  Hi = DAG.getNode(Opc, dl, VTList, HiOps);
1976 
1977  Ovf = Hi.getValue(1);
1978  } else {
1979  // Expand the result by simply replacing it with the equivalent
1980  // non-overflow-checking operation.
1981  auto Opc = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
1982  SDValue Sum = DAG.getNode(Opc, dl, LHS.getValueType(), LHS, RHS);
1983  SplitInteger(Sum, Lo, Hi);
1984 
1985  // Calculate the overflow: addition overflows iff a + b < a, and subtraction
1986  // overflows iff a - b > a.
1987  auto Cond = N->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
1988  Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
1989  }
1990 
1991  // Legalized the flag result - switch anything that used the old flag to
1992  // use the new one.
1993  ReplaceValueWith(SDValue(N, 1), Ovf);
1994 }
1995 
1996 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
1997  SDValue &Lo, SDValue &Hi) {
1998  // Expand the subcomponents.
1999  SDValue LHSL, LHSH, RHSL, RHSH;
2000  SDLoc dl(N);
2001  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2002  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2003  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2004  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2005  SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
2006 
2007  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2008  HiOps[2] = Lo.getValue(1);
2009  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2010 
2011  // Legalized the flag result - switch anything that used the old flag to
2012  // use the new one.
2013  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2014 }
2015 
2016 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
2017  SDValue &Lo, SDValue &Hi) {
2018  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2019  SDLoc dl(N);
2020  SDValue Op = N->getOperand(0);
2021  if (Op.getValueType().bitsLE(NVT)) {
2022  // The low part is any extension of the input (which degenerates to a copy).
2023  Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
2024  Hi = DAG.getUNDEF(NVT); // The high part is undefined.
2025  } else {
2026  // For example, extension of an i48 to an i64. The operand type necessarily
2027  // promotes to the result type, so will end up being expanded too.
2028  assert(getTypeAction(Op.getValueType()) ==
2030  "Only know how to promote this result!");
2031  SDValue Res = GetPromotedInteger(Op);
2032  assert(Res.getValueType() == N->getValueType(0) &&
2033  "Operand over promoted?");
2034  // Split the promoted operand. This will simplify when it is expanded.
2035  SplitInteger(Res, Lo, Hi);
2036  }
2037 }
2038 
2039 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
2040  SDValue &Lo, SDValue &Hi) {
2041  SDLoc dl(N);
2042  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2043  EVT NVT = Lo.getValueType();
2044  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2045  unsigned NVTBits = NVT.getSizeInBits();
2046  unsigned EVTBits = EVT.getSizeInBits();
2047 
2048  if (NVTBits < EVTBits) {
2049  Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
2051  EVTBits - NVTBits)));
2052  } else {
2053  Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
2054  // The high part replicates the sign bit of Lo, make it explicit.
2055  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2056  DAG.getConstant(NVTBits - 1, dl,
2057  TLI.getPointerTy(DAG.getDataLayout())));
2058  }
2059 }
2060 
2061 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
2062  SDValue &Lo, SDValue &Hi) {
2063  SDLoc dl(N);
2064  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2065  EVT NVT = Lo.getValueType();
2066  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2067  unsigned NVTBits = NVT.getSizeInBits();
2068  unsigned EVTBits = EVT.getSizeInBits();
2069 
2070  if (NVTBits < EVTBits) {
2071  Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
2073  EVTBits - NVTBits)));
2074  } else {
2075  Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
2076  // The high part must be zero, make it explicit.
2077  Hi = DAG.getConstant(0, dl, NVT);
2078  }
2079 }
2080 
2081 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
2082  SDValue &Lo, SDValue &Hi) {
2083  SDLoc dl(N);
2084  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2085  Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
2086  Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
2087 }
2088 
2089 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
2090  SDValue &Lo, SDValue &Hi) {
2091  SDLoc dl(N);
2092  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2093  Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
2094  Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
2095 }
2096 
2097 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
2098  SDValue &Lo, SDValue &Hi) {
2099  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2100  unsigned NBitWidth = NVT.getSizeInBits();
2101  auto Constant = cast<ConstantSDNode>(N);
2102  const APInt &Cst = Constant->getAPIntValue();
2103  bool IsTarget = Constant->isTargetOpcode();
2104  bool IsOpaque = Constant->isOpaque();
2105  SDLoc dl(N);
2106  Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
2107  Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
2108  IsOpaque);
2109 }
2110 
2111 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
2112  SDValue &Lo, SDValue &Hi) {
2113  SDLoc dl(N);
2114  // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
2115  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2116  EVT NVT = Lo.getValueType();
2117 
2118  SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
2119  DAG.getConstant(0, dl, NVT), ISD::SETNE);
2120 
2121  SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
2122  SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
2123 
2124  Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
2125  DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
2126  DAG.getConstant(NVT.getSizeInBits(), dl,
2127  NVT)));
2128  Hi = DAG.getConstant(0, dl, NVT);
2129 }
2130 
2131 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
2132  SDValue &Lo, SDValue &Hi) {
2133  SDLoc dl(N);
2134  // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
2135  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2136  EVT NVT = Lo.getValueType();
2137  Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
2138  DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
2139  Hi = DAG.getConstant(0, dl, NVT);
2140 }
2141 
2142 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
2143  SDValue &Lo, SDValue &Hi) {
2144  SDLoc dl(N);
2145  // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
2146  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2147  EVT NVT = Lo.getValueType();
2148 
2149  SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
2150  DAG.getConstant(0, dl, NVT), ISD::SETNE);
2151 
2152  SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
2153  SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
2154 
2155  Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
2156  DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
2157  DAG.getConstant(NVT.getSizeInBits(), dl,
2158  NVT)));
2159  Hi = DAG.getConstant(0, dl, NVT);
2160 }
2161 
2162 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
2163  SDValue &Hi) {
2164  SDLoc dl(N);
2165  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2166  unsigned NBitWidth = NVT.getSizeInBits();
2167 
2168  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2169  Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT);
2170  // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
2171  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2172  DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
2173 }
2174 
2175 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
2176  SDValue &Hi) {
2177  SDLoc dl(N);
2178  EVT VT = N->getValueType(0);
2179 
2180  SDValue Op = N->getOperand(0);
2181  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2182  Op = GetPromotedFloat(Op);
2183 
2185  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
2186  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, true/*irrelevant*/, dl).first,
2187  Lo, Hi);
2188 }
2189 
2190 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
2191  SDValue &Hi) {
2192  SDLoc dl(N);
2193  EVT VT = N->getValueType(0);
2194 
2195  SDValue Op = N->getOperand(0);
2196  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2197  Op = GetPromotedFloat(Op);
2198 
2200  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
2201  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, false/*irrelevant*/, dl).first,
2202  Lo, Hi);
2203 }
2204 
2205 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
2206  SDValue &Lo, SDValue &Hi) {
2207  if (ISD::isNormalLoad(N)) {
2208  ExpandRes_NormalLoad(N, Lo, Hi);
2209  return;
2210  }
2211 
2212  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2213 
2214  EVT VT = N->getValueType(0);
2215  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2216  SDValue Ch = N->getChain();
2217  SDValue Ptr = N->getBasePtr();
2219  unsigned Alignment = N->getAlignment();
2220  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2221  AAMDNodes AAInfo = N->getAAInfo();
2222  SDLoc dl(N);
2223 
2224  assert(NVT.isByteSized() && "Expanded type not byte sized!");
2225 
2226  if (N->getMemoryVT().bitsLE(NVT)) {
2227  EVT MemVT = N->getMemoryVT();
2228 
2229  Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
2230  Alignment, MMOFlags, AAInfo);
2231 
2232  // Remember the chain.
2233  Ch = Lo.getValue(1);
2234 
2235  if (ExtType == ISD::SEXTLOAD) {
2236  // The high part is obtained by SRA'ing all but one of the bits of the
2237  // lo part.
2238  unsigned LoSize = Lo.getValueSizeInBits();
2239  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2240  DAG.getConstant(LoSize - 1, dl,
2241  TLI.getPointerTy(DAG.getDataLayout())));
2242  } else if (ExtType == ISD::ZEXTLOAD) {
2243  // The high part is just a zero.
2244  Hi = DAG.getConstant(0, dl, NVT);
2245  } else {
2246  assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
2247  // The high part is undefined.
2248  Hi = DAG.getUNDEF(NVT);
2249  }
2250  } else if (DAG.getDataLayout().isLittleEndian()) {
2251  // Little-endian - low bits are at low addresses.
2252  Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2253  AAInfo);
2254 
2255  unsigned ExcessBits =
2256  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
2257  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
2258 
2259  // Increment the pointer to the other half.
2260  unsigned IncrementSize = NVT.getSizeInBits()/8;
2261  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2262  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2263  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
2264  N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
2265  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2266 
2267  // Build a factor node to remember that this load is independent of the
2268  // other one.
2269  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2270  Hi.getValue(1));
2271  } else {
2272  // Big-endian - high bits are at low addresses. Favor aligned loads at
2273  // the cost of some bit-fiddling.
2274  EVT MemVT = N->getMemoryVT();
2275  unsigned EBytes = MemVT.getStoreSize();
2276  unsigned IncrementSize = NVT.getSizeInBits()/8;
2277  unsigned ExcessBits = (EBytes - IncrementSize)*8;
2278 
2279  // Load both the high bits and maybe some of the low bits.
2280  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
2282  MemVT.getSizeInBits() - ExcessBits),
2283  Alignment, MMOFlags, AAInfo);
2284 
2285  // Increment the pointer to the other half.
2286  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2287  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2288  // Load the rest of the low bits.
2289  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
2290  N->getPointerInfo().getWithOffset(IncrementSize),
2291  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
2292  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2293 
2294  // Build a factor node to remember that this load is independent of the
2295  // other one.
2296  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2297  Hi.getValue(1));
2298 
2299  if (ExcessBits < NVT.getSizeInBits()) {
2300  // Transfer low bits from the bottom of Hi to the top of Lo.
2301  Lo = DAG.getNode(
2302  ISD::OR, dl, NVT, Lo,
2303  DAG.getNode(ISD::SHL, dl, NVT, Hi,
2304  DAG.getConstant(ExcessBits, dl,
2305  TLI.getPointerTy(DAG.getDataLayout()))));
2306  // Move high bits to the right position in Hi.
2307  Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
2308  Hi,
2309  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
2310  TLI.getPointerTy(DAG.getDataLayout())));
2311  }
2312  }
2313 
2314  // Legalize the chain result - switch anything that used the old chain to
2315  // use the new one.
2316  ReplaceValueWith(SDValue(N, 1), Ch);
2317 }
2318 
2319 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
2320  SDValue &Lo, SDValue &Hi) {
2321  SDLoc dl(N);
2322  SDValue LL, LH, RL, RH;
2323  GetExpandedInteger(N->getOperand(0), LL, LH);
2324  GetExpandedInteger(N->getOperand(1), RL, RH);
2325  Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
2326  Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
2327 }
2328 
2329 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
2330  SDValue &Lo, SDValue &Hi) {
2331  EVT VT = N->getValueType(0);
2332  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2333  SDLoc dl(N);
2334 
2335  SDValue LL, LH, RL, RH;
2336  GetExpandedInteger(N->getOperand(0), LL, LH);
2337  GetExpandedInteger(N->getOperand(1), RL, RH);
2338 
2339  if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
2341  LL, LH, RL, RH))
2342  return;
2343 
2344  // If nothing else, we can make a libcall.
2345  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2346  if (VT == MVT::i16)
2347  LC = RTLIB::MUL_I16;
2348  else if (VT == MVT::i32)
2349  LC = RTLIB::MUL_I32;
2350  else if (VT == MVT::i64)
2351  LC = RTLIB::MUL_I64;
2352  else if (VT == MVT::i128)
2353  LC = RTLIB::MUL_I128;
2354 
2355  if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
2356  // We'll expand the multiplication by brute force because we have no other
2357  // options. This is a trivially-generalized version of the code from
2358  // Hacker's Delight (itself derived from Knuth's Algorithm M from section
2359  // 4.3.1).
2360  unsigned Bits = NVT.getSizeInBits();
2361  unsigned HalfBits = Bits >> 1;
2362  SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
2363  NVT);
2364  SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
2365  SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
2366 
2367  SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
2368  SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
2369 
2370  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2371  if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
2372  // The type from TLI is too small to fit the shift amount we want.
2373  // Override it with i32. The shift will have to be legalized.
2374  ShiftAmtTy = MVT::i32;
2375  }
2376  SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
2377  SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
2378  SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
2379  SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
2380 
2381  SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
2382  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
2383  SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
2384  SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
2385 
2386  SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
2387  DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
2388  SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
2389 
2390  SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
2391  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
2392  DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
2393  Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
2394  DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
2395 
2396  Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
2397  DAG.getNode(ISD::ADD, dl, NVT,
2398  DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
2399  DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
2400  return;
2401  }
2402 
2403  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2404  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true/*irrelevant*/, dl).first,
2405  Lo, Hi);
2406 }
2407 
2408 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
2409  SDValue &Hi) {
2410  SDLoc DL(N);
2411  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2412  SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
2413  SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
2414  Lo = R.getValue(0);
2415  Hi = R.getValue(1);
2416  ReplaceValueWith(SDValue(N, 1), R.getValue(2));
2417 }
2418 
2419 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
2420  SDValue &Lo, SDValue &Hi) {
2421  SDValue LHS = Node->getOperand(0);
2422  SDValue RHS = Node->getOperand(1);
2423  SDLoc dl(Node);
2424 
2425  // Expand the result by simply replacing it with the equivalent
2426  // non-overflow-checking operation.
2427  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
2428  ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
2429  LHS, RHS);
2430  SplitInteger(Sum, Lo, Hi);
2431 
2432  // Compute the overflow.
2433  //
2434  // LHSSign -> LHS >= 0
2435  // RHSSign -> RHS >= 0
2436  // SumSign -> Sum >= 0
2437  //
2438  // Add:
2439  // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
2440  // Sub:
2441  // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
2442  //
2443  EVT OType = Node->getValueType(1);
2444  SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
2445 
2446  SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
2447  SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
2448  SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
2449  Node->getOpcode() == ISD::SADDO ?
2451 
2452  SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
2453  SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
2454 
2455  SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
2456 
2457  // Use the calculated overflow everywhere.
2458  ReplaceValueWith(SDValue(Node, 1), Cmp);
2459 }
2460 
2461 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
2462  SDValue &Lo, SDValue &Hi) {
2463  EVT VT = N->getValueType(0);
2464  SDLoc dl(N);
2465  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2466 
2468  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2469  SplitInteger(Res.getValue(0), Lo, Hi);
2470  return;
2471  }
2472 
2473  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2474  if (VT == MVT::i16)
2475  LC = RTLIB::SDIV_I16;
2476  else if (VT == MVT::i32)
2477  LC = RTLIB::SDIV_I32;
2478  else if (VT == MVT::i64)
2479  LC = RTLIB::SDIV_I64;
2480  else if (VT == MVT::i128)
2481  LC = RTLIB::SDIV_I128;
2482  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
2483 
2484  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
2485 }
2486 
2487 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
2488  SDValue &Lo, SDValue &Hi) {
2489  EVT VT = N->getValueType(0);
2490  SDLoc dl(N);
2491 
2492  // If we can emit an efficient shift operation, do so now. Check to see if
2493  // the RHS is a constant.
2494  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
2495  return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
2496 
2497  // If we can determine that the high bit of the shift is zero or one, even if
2498  // the low bits are variable, emit this shift in an optimized form.
2499  if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
2500  return;
2501 
2502  // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
2503  unsigned PartsOpc;
2504  if (N->getOpcode() == ISD::SHL) {
2505  PartsOpc = ISD::SHL_PARTS;
2506  } else if (N->getOpcode() == ISD::SRL) {
2507  PartsOpc = ISD::SRL_PARTS;
2508  } else {
2509  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2510  PartsOpc = ISD::SRA_PARTS;
2511  }
2512 
2513  // Next check to see if the target supports this SHL_PARTS operation or if it
2514  // will custom expand it.
2515  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2516  TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
2517  if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
2518  Action == TargetLowering::Custom) {
2519  // Expand the subcomponents.
2520  SDValue LHSL, LHSH;
2521  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2522  EVT VT = LHSL.getValueType();
2523 
2524  // If the shift amount operand is coming from a vector legalization it may
2525  // have an illegal type. Fix that first by casting the operand, otherwise
2526  // the new SHL_PARTS operation would need further legalization.
2527  SDValue ShiftOp = N->getOperand(1);
2528  EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2529  assert(ShiftTy.getScalarSizeInBits() >=
2530  Log2_32_Ceil(VT.getScalarSizeInBits()) &&
2531  "ShiftAmountTy is too small to cover the range of this type!");
2532  if (ShiftOp.getValueType() != ShiftTy)
2533  ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
2534 
2535  SDValue Ops[] = { LHSL, LHSH, ShiftOp };
2536  Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
2537  Hi = Lo.getValue(1);
2538  return;
2539  }
2540 
2541  // Otherwise, emit a libcall.
2542  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2543  bool isSigned;
2544  if (N->getOpcode() == ISD::SHL) {
2545  isSigned = false; /*sign irrelevant*/
2546  if (VT == MVT::i16)
2547  LC = RTLIB::SHL_I16;
2548  else if (VT == MVT::i32)
2549  LC = RTLIB::SHL_I32;
2550  else if (VT == MVT::i64)
2551  LC = RTLIB::SHL_I64;
2552  else if (VT == MVT::i128)
2553  LC = RTLIB::SHL_I128;
2554  } else if (N->getOpcode() == ISD::SRL) {
2555  isSigned = false;
2556  if (VT == MVT::i16)
2557  LC = RTLIB::SRL_I16;
2558  else if (VT == MVT::i32)
2559  LC = RTLIB::SRL_I32;
2560  else if (VT == MVT::i64)
2561  LC = RTLIB::SRL_I64;
2562  else if (VT == MVT::i128)
2563  LC = RTLIB::SRL_I128;
2564  } else {
2565  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2566  isSigned = true;
2567  if (VT == MVT::i16)
2568  LC = RTLIB::SRA_I16;
2569  else if (VT == MVT::i32)
2570  LC = RTLIB::SRA_I32;
2571  else if (VT == MVT::i64)
2572  LC = RTLIB::SRA_I64;
2573  else if (VT == MVT::i128)
2574  LC = RTLIB::SRA_I128;
2575  }
2576 
2577  if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
2578  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2579  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, isSigned, dl).first, Lo, Hi);
2580  return;
2581  }
2582 
2583  if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
2584  llvm_unreachable("Unsupported shift!");
2585 }
2586 
2587 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
2588  SDValue &Lo, SDValue &Hi) {
2589  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2590  SDLoc dl(N);
2591  SDValue Op = N->getOperand(0);
2592  if (Op.getValueType().bitsLE(NVT)) {
2593  // The low part is sign extension of the input (degenerates to a copy).
2594  Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
2595  // The high part is obtained by SRA'ing all but one of the bits of low part.
2596  unsigned LoSize = NVT.getSizeInBits();
2597  Hi = DAG.getNode(
2598  ISD::SRA, dl, NVT, Lo,
2599  DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
2600  } else {
2601  // For example, extension of an i48 to an i64. The operand type necessarily
2602  // promotes to the result type, so will end up being expanded too.
2603  assert(getTypeAction(Op.getValueType()) ==
2605  "Only know how to promote this result!");
2606  SDValue Res = GetPromotedInteger(Op);
2607  assert(Res.getValueType() == N->getValueType(0) &&
2608  "Operand over promoted?");
2609  // Split the promoted operand. This will simplify when it is expanded.
2610  SplitInteger(Res, Lo, Hi);
2611  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
2612  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2614  ExcessBits)));
2615  }
2616 }
2617 
2618 void DAGTypeLegalizer::
2619 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2620  SDLoc dl(N);
2621  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2622  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2623 
2624  if (EVT.bitsLE(Lo.getValueType())) {
2625  // sext_inreg the low part if needed.
2626  Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
2627  N->getOperand(1));
2628 
2629  // The high part gets the sign extension from the lo-part. This handles
2630  // things like sextinreg V:i64 from i8.
2631  Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
2632  DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
2633  TLI.getPointerTy(DAG.getDataLayout())));
2634  } else {
2635  // For example, extension of an i48 to an i64. Leave the low part alone,
2636  // sext_inreg the high part.
2637  unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
2638  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2640  ExcessBits)));
2641  }
2642 }
2643 
2644 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
2645  SDValue &Lo, SDValue &Hi) {
2646  EVT VT = N->getValueType(0);
2647  SDLoc dl(N);
2648  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2649 
2651  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2652  SplitInteger(Res.getValue(1), Lo, Hi);
2653  return;
2654  }
2655 
2656  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2657  if (VT == MVT::i16)
2658  LC = RTLIB::SREM_I16;
2659  else if (VT == MVT::i32)
2660  LC = RTLIB::SREM_I32;
2661  else if (VT == MVT::i64)
2662  LC = RTLIB::SREM_I64;
2663  else if (VT == MVT::i128)
2664  LC = RTLIB::SREM_I128;
2665  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
2666 
2667  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
2668 }
2669 
2670 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
2671  SDValue &Lo, SDValue &Hi) {
2672  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2673  SDLoc dl(N);
2674  Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
2675  Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
2676  N->getOperand(0),
2677  DAG.getConstant(NVT.getSizeInBits(), dl,
2678  TLI.getPointerTy(DAG.getDataLayout())));
2679  Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
2680 }
2681 
2682 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
2683  SDValue &Lo, SDValue &Hi) {
2684  EVT VT = N->getValueType(0);
2685  SDLoc dl(N);
2686 
2687  if (N->getOpcode() == ISD::UMULO) {
2688  // This section expands the operation into the following sequence of
2689  // instructions. `iNh` here refers to a type which has half the bit width of
2690  // the type the original operation operated on.
2691  //
2692  // %0 = %LHS.HI != 0 && %RHS.HI != 0
2693  // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
2694  // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
2695  // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
2696  // %4 = add iN (%1.0 as iN) << Nh, (%2.0 as iN) << Nh
2697  // %5 = { iN, i1 } @uadd.with.overflow.iN( %4, %3 )
2698  //
2699  // %res = { %5.0, %0 || %1.1 || %2.1 || %5.1 }
2700  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
2701  SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
2702  SplitInteger(LHS, LHSLow, LHSHigh);
2703  SplitInteger(RHS, RHSLow, RHSHigh);
2704  EVT HalfVT = LHSLow.getValueType()
2705  , BitVT = N->getValueType(1);
2706  SDVTList VTHalfMulO = DAG.getVTList(HalfVT, BitVT);
2707  SDVTList VTFullAddO = DAG.getVTList(VT, BitVT);
2708 
2709  SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
2710  SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
2711  DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
2712  DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));
2713 
2714  SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, LHSHigh, RHSLow);
2715  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));
2716  SDValue OneInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
2717  One.getValue(0));
2718 
2719  SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, RHSHigh, LHSLow);
2720  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));
2721  SDValue TwoInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
2722  Two.getValue(0));
2723 
2724  // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
2725  // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
2726  // operation recursively legalized?).
2727  //
2728  // Many backends understand this pattern and will convert into LOHI
2729  // themselves, if applicable.
2730  SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
2731  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
2732  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow));
2733  SDValue Four = DAG.getNode(ISD::ADD, dl, VT, OneInHigh, TwoInHigh);
2734  SDValue Five = DAG.getNode(ISD::UADDO, dl, VTFullAddO, Three, Four);
2735  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Five.getValue(1));
2736  SplitInteger(Five, Lo, Hi);
2737  ReplaceValueWith(SDValue(N, 1), Overflow);
2738  return;
2739  }
2740 
2741  Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
2742  EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
2743  Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
2744 
2745  // Replace this with a libcall that will check overflow.
2746  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2747  if (VT == MVT::i32)
2748  LC = RTLIB::MULO_I32;
2749  else if (VT == MVT::i64)
2750  LC = RTLIB::MULO_I64;
2751  else if (VT == MVT::i128)
2752  LC = RTLIB::MULO_I128;
2753  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
2754 
2755  SDValue Temp = DAG.CreateStackTemporary(PtrVT);
2756  // Temporary for the overflow value, default it to zero.
2757  SDValue Chain =
2758  DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
2759  MachinePointerInfo());
2760 
2762  TargetLowering::ArgListEntry Entry;
2763  for (const SDValue &Op : N->op_values()) {
2764  EVT ArgVT = Op.getValueType();
2765  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2766  Entry.Node = Op;
2767  Entry.Ty = ArgTy;
2768  Entry.IsSExt = true;
2769  Entry.IsZExt = false;
2770  Args.push_back(Entry);
2771  }
2772 
2773  // Also pass the address of the overflow check.
2774  Entry.Node = Temp;
2775  Entry.Ty = PtrTy->getPointerTo();
2776  Entry.IsSExt = true;
2777  Entry.IsZExt = false;
2778  Args.push_back(Entry);
2779 
2780  SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
2781 
2783  CLI.setDebugLoc(dl)
2784  .setChain(Chain)
2785  .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
2786  .setSExtResult();
2787 
2788  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2789 
2790  SplitInteger(CallInfo.first, Lo, Hi);
2791  SDValue Temp2 =
2792  DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
2793  SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
2794  DAG.getConstant(0, dl, PtrVT),
2795  ISD::SETNE);
2796  // Use the overflow from the libcall everywhere.
2797  ReplaceValueWith(SDValue(N, 1), Ofl);
2798 }
2799 
2800 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
2801  SDValue &Lo, SDValue &Hi) {
2802  EVT VT = N->getValueType(0);
2803  SDLoc dl(N);
2804  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2805 
2807  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2808  SplitInteger(Res.getValue(0), Lo, Hi);
2809  return;
2810  }
2811 
2812  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2813  if (VT == MVT::i16)
2814  LC = RTLIB::UDIV_I16;
2815  else if (VT == MVT::i32)
2816  LC = RTLIB::UDIV_I32;
2817  else if (VT == MVT::i64)
2818  LC = RTLIB::UDIV_I64;
2819  else if (VT == MVT::i128)
2820  LC = RTLIB::UDIV_I128;
2821  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
2822 
2823  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
2824 }
2825 
2826 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
2827  SDValue &Lo, SDValue &Hi) {
2828  EVT VT = N->getValueType(0);
2829  SDLoc dl(N);
2830  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2831 
2833  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2834  SplitInteger(Res.getValue(1), Lo, Hi);
2835  return;
2836  }
2837 
2838  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2839  if (VT == MVT::i16)
2840  LC = RTLIB::UREM_I16;
2841  else if (VT == MVT::i32)
2842  LC = RTLIB::UREM_I32;
2843  else if (VT == MVT::i64)
2844  LC = RTLIB::UREM_I64;
2845  else if (VT == MVT::i128)
2846  LC = RTLIB::UREM_I128;
2847  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
2848 
2849  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
2850 }
2851 
2852 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
2853  SDValue &Lo, SDValue &Hi) {
2854  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2855  SDLoc dl(N);
2856  SDValue Op = N->getOperand(0);
2857  if (Op.getValueType().bitsLE(NVT)) {
2858  // The low part is zero extension of the input (degenerates to a copy).
2859  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
2860  Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
2861  } else {
2862  // For example, extension of an i48 to an i64. The operand type necessarily
2863  // promotes to the result type, so will end up being expanded too.
2864  assert(getTypeAction(Op.getValueType()) ==
2866  "Only know how to promote this result!");
2867  SDValue Res = GetPromotedInteger(Op);
2868  assert(Res.getValueType() == N->getValueType(0) &&
2869  "Operand over promoted?");
2870  // Split the promoted operand. This will simplify when it is expanded.
2871  SplitInteger(Res, Lo, Hi);
2872  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
2873  Hi = DAG.getZeroExtendInReg(Hi, dl,
2875  ExcessBits));
2876  }
2877 }
2878 
2879 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
2880  SDValue &Lo, SDValue &Hi) {
2881  SDLoc dl(N);
2882  EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
2883  SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
2884  SDValue Zero = DAG.getConstant(0, dl, VT);
2885  SDValue Swap = DAG.getAtomicCmpSwap(
2887  cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
2888  N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
2889 
2890  ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
2891  ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
2892 }
2893 
2894 //===----------------------------------------------------------------------===//
2895 // Integer Operand Expansion
2896 //===----------------------------------------------------------------------===//
2897 
2898 /// ExpandIntegerOperand - This method is called when the specified operand of
2899 /// the specified node is found to need expansion. At this point, all of the
2900 /// result types of the node are known to be legal, but other operands of the
2901 /// node may need promotion or expansion as well as the specified one.
2902 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
2903  LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG);
2904  dbgs() << "\n");
2905  SDValue Res = SDValue();
2906 
2907  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2908  return false;
2909 
2910  switch (N->getOpcode()) {
2911  default:
2912  #ifndef NDEBUG
2913  dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
2914  N->dump(&DAG); dbgs() << "\n";
2915  #endif
2916  llvm_unreachable("Do not know how to expand this operator's operand!");
2917 
2918  case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
2919  case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
2920  case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
2921  case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
2922  case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
2923  case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
2924  case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
2925  case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
2926  case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
2927  case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break;
2928  case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
2929  case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
2930  case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break;
2931 
2932  case ISD::SHL:
2933  case ISD::SRA:
2934  case ISD::SRL:
2935  case ISD::ROTL:
2936  case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
2937  case ISD::RETURNADDR:
2938  case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
2939 
2940  case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
2941  }
2942 
2943  // If the result is null, the sub-method took care of registering results etc.
2944  if (!Res.getNode()) return false;
2945 
2946  // If the result is N, the sub-method updated N in place. Tell the legalizer
2947  // core about this.
2948  if (Res.getNode() == N)
2949  return true;
2950 
2951  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2952  "Invalid operand expansion");
2953 
2954  ReplaceValueWith(SDValue(N, 0), Res);
2955  return false;
2956 }
2957 
2958 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
2959 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
2960 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
2961  SDValue &NewRHS,
2962  ISD::CondCode &CCCode,
2963  const SDLoc &dl) {
2964  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2965  GetExpandedInteger(NewLHS, LHSLo, LHSHi);
2966  GetExpandedInteger(NewRHS, RHSLo, RHSHi);
2967 
2968  if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
2969  if (RHSLo == RHSHi) {
2970  if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
2971  if (RHSCST->isAllOnesValue()) {
2972  // Equality comparison to -1.
2973  NewLHS = DAG.getNode(ISD::AND, dl,
2974  LHSLo.getValueType(), LHSLo, LHSHi);
2975  NewRHS = RHSLo;
2976  return;
2977  }
2978  }
2979  }
2980 
2981  NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
2982  NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
2983  NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
2984  NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
2985  return;
2986  }
2987 
2988  // If this is a comparison of the sign bit, just look at the top part.
2989  // X > -1, x < 0
2990  if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
2991  if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0
2992  (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1
2993  NewLHS = LHSHi;
2994  NewRHS = RHSHi;
2995  return;
2996  }
2997 
2998  // FIXME: This generated code sucks.
2999  ISD::CondCode LowCC;
3000  switch (CCCode) {
3001  default: llvm_unreachable("Unknown integer setcc!");
3002  case ISD::SETLT:
3003  case ISD::SETULT: LowCC = ISD::SETULT; break;
3004  case ISD::SETGT:
3005  case ISD::SETUGT: LowCC = ISD::SETUGT; break;
3006  case ISD::SETLE:
3007  case ISD::SETULE: LowCC = ISD::SETULE; break;
3008  case ISD::SETGE:
3009  case ISD::SETUGE: LowCC = ISD::SETUGE; break;
3010  }
3011 
3012  // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
3013  // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
3014  // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
3015 
3016  // NOTE: on targets without efficient SELECT of bools, we can always use
3017  // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
3018  TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
3019  nullptr);
3020  SDValue LoCmp, HiCmp;
3021  if (TLI.isTypeLegal(LHSLo.getValueType()) &&
3022  TLI.isTypeLegal(RHSLo.getValueType()))
3023  LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
3024  RHSLo, LowCC, false, DagCombineInfo, dl);
3025  if (!LoCmp.getNode())
3026  LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
3027  RHSLo, LowCC);
3028  if (TLI.isTypeLegal(LHSHi.getValueType()) &&
3029  TLI.isTypeLegal(RHSHi.getValueType()))
3030  HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
3031  RHSHi, CCCode, false, DagCombineInfo, dl);
3032  if (!HiCmp.getNode())
3033  HiCmp =
3034  DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
3035  LHSHi, RHSHi, DAG.getCondCode(CCCode));
3036 
3037  ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
3038  ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
3039 
3040  bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
3041  CCCode == ISD::SETUGE || CCCode == ISD::SETULE);
3042 
3043  if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) ||
3044  (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) ||
3045  (LoCmpC && LoCmpC->isNullValue())))) {
3046  // For LE / GE, if high part is known false, ignore the low part.
3047  // For LT / GT: if low part is known false, return the high part.
3048  // if high part is known true, ignore the low part.
3049  NewLHS = HiCmp;
3050  NewRHS = SDValue();
3051  return;
3052  }
3053 
3054  if (LHSHi == RHSHi) {
3055  // Comparing the low bits is enough.
3056  NewLHS = LoCmp;
3057  NewRHS = SDValue();
3058  return;
3059  }
3060 
3061  // Lower with SETCCCARRY if the target supports it.
3062  EVT HiVT = LHSHi.getValueType();
3063  EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
3064  bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
3065 
3066  // FIXME: Make all targets support this, then remove the other lowering.
3067  if (HasSETCCCARRY) {
3068  // SETCCCARRY can detect < and >= directly. For > and <=, flip
3069  // operands and condition code.
3070  bool FlipOperands = false;
3071  switch (CCCode) {
3072  case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
3073  case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
3074  case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
3075  case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
3076  default: break;
3077  }
3078  if (FlipOperands) {
3079  std::swap(LHSLo, RHSLo);
3080  std::swap(LHSHi, RHSHi);
3081  }
3082  // Perform a wide subtraction, feeding the carry from the low part into
3083  // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
3084  // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
3085  // zero or positive iff LHS >= RHS.
3086  EVT LoVT = LHSLo.getValueType();
3087  SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
3088  SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
3089  SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
3090  LHSHi, RHSHi, LowCmp.getValue(1),
3091  DAG.getCondCode(CCCode));
3092  NewLHS = Res;
3093  NewRHS = SDValue();
3094  return;
3095  }
3096 
3097  NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
3098  false, DagCombineInfo, dl);
3099  if (!NewLHS.getNode())
3100  NewLHS =
3101  DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
3102  NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
3103  NewRHS = SDValue();
3104 }
3105 
3106 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
3107  SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
3108  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
3109  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3110 
3111  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3112  // against zero to select between true and false values.
3113  if (!NewRHS.getNode()) {
3114  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3115  CCCode = ISD::SETNE;
3116  }
3117 
3118  // Update N to have the operands specified.
3119  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
3120  DAG.getCondCode(CCCode), NewLHS, NewRHS,
3121  N->getOperand(4)), 0);
3122 }
3123 
3124 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
3125  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3126  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
3127  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3128 
3129  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3130  // against zero to select between true and false values.
3131  if (!NewRHS.getNode()) {
3132  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3133  CCCode = ISD::SETNE;
3134  }
3135 
3136  // Update N to have the operands specified.
3137  return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
3138  N->getOperand(2), N->getOperand(3),
3139  DAG.getCondCode(CCCode)), 0);
3140 }
3141 
3142 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
3143  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3144  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3145  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3146 
3147  // If ExpandSetCCOperands returned a scalar, use it.
3148  if (!NewRHS.getNode()) {
3149  assert(NewLHS.getValueType() == N->getValueType(0) &&
3150  "Unexpected setcc expansion!");
3151  return NewLHS;
3152  }
3153 
3154  // Otherwise, update N to have the operands specified.
3155  return SDValue(
3156  DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
3157 }
3158 
3159 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
3160  SDValue LHS = N->getOperand(0);
3161  SDValue RHS = N->getOperand(1);
3162  SDValue Carry = N->getOperand(2);
3163  SDValue Cond = N->getOperand(3);
3164  SDLoc dl = SDLoc(N);
3165 
3166  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3167  GetExpandedInteger(LHS, LHSLo, LHSHi);
3168  GetExpandedInteger(RHS, RHSLo, RHSHi);
3169 
3170  // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
3171  SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
3172  SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
3173  return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
3174  LowCmp.getValue(1), Cond);
3175 }
3176 
3177 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
3178  // The value being shifted is legal, but the shift amount is too big.
3179  // It follows that either the result of the shift is undefined, or the
3180  // upper half of the shift amount is zero. Just use the lower half.
3181  SDValue Lo, Hi;
3182  GetExpandedInteger(N->getOperand(1), Lo, Hi);
3183  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
3184 }
3185 
3186 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
3187  // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
3188  // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
3189  // constant to valid type.
3190  SDValue Lo, Hi;
3191  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3192  return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
3193 }
3194 
3195 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
3196  SDValue Op = N->getOperand(0);
3197  EVT DstVT = N->getValueType(0);
3199  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3200  "Don't know how to expand this SINT_TO_FP!");
3201  return TLI.makeLibCall(DAG, LC, DstVT, Op, true, SDLoc(N)).first;
3202 }
3203 
3204 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
3205  if (ISD::isNormalStore(N))
3206  return ExpandOp_NormalStore(N, OpNo);
3207 
3208  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
3209  assert(OpNo == 1 && "Can only expand the stored value so far");
3210 
3211  EVT VT = N->getOperand(1).getValueType();
3212  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3213  SDValue Ch = N->getChain();
3214  SDValue Ptr = N->getBasePtr();
3215  unsigned Alignment = N->getAlignment();
3216  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3217  AAMDNodes AAInfo = N->getAAInfo();
3218  SDLoc dl(N);
3219  SDValue Lo, Hi;
3220 
3221  assert(NVT.isByteSized() && "Expanded type not byte sized!");
3222 
3223  if (N->getMemoryVT().bitsLE(NVT)) {
3224  GetExpandedInteger(N->getValue(), Lo, Hi);
3225  return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
3226  N->getMemoryVT(), Alignment, MMOFlags, AAInfo);
3227  }
3228 
3229  if (DAG.getDataLayout().isLittleEndian()) {
3230  // Little-endian - low bits are at low addresses.
3231  GetExpandedInteger(N->getValue(), Lo, Hi);
3232 
3233  Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3234  AAInfo);
3235 
3236  unsigned ExcessBits =
3237  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3238  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3239 
3240  // Increment the pointer to the other half.
3241  unsigned IncrementSize = NVT.getSizeInBits()/8;
3242  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3243  Hi = DAG.getTruncStore(
3244  Ch, dl, Hi, Ptr, N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3245  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3246  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3247  }
3248 
3249  // Big-endian - high bits are at low addresses. Favor aligned stores at
3250  // the cost of some bit-fiddling.
3251  GetExpandedInteger(N->getValue(), Lo, Hi);
3252 
3253  EVT ExtVT = N->getMemoryVT();
3254  unsigned EBytes = ExtVT.getStoreSize();
3255  unsigned IncrementSize = NVT.getSizeInBits()/8;
3256  unsigned ExcessBits = (EBytes - IncrementSize)*8;
3257  EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
3258  ExtVT.getSizeInBits() - ExcessBits);
3259 
3260  if (ExcessBits < NVT.getSizeInBits()) {
3261  // Transfer high bits from the top of Lo to the bottom of Hi.
3262  Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
3263  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3264  TLI.getPointerTy(DAG.getDataLayout())));
3265  Hi = DAG.getNode(
3266  ISD::OR, dl, NVT, Hi,
3267  DAG.getNode(ISD::SRL, dl, NVT, Lo,
3268  DAG.getConstant(ExcessBits, dl,
3269  TLI.getPointerTy(DAG.getDataLayout()))));
3270  }
3271 
3272  // Store both the high bits and maybe some of the low bits.
3273  Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment,
3274  MMOFlags, AAInfo);
3275 
3276  // Increment the pointer to the other half.
3277  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3278  // Store the lowest ExcessBits bits in the second half.
3279  Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
3280  N->getPointerInfo().getWithOffset(IncrementSize),
3281  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3282  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3283  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3284 }
3285 
3286 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
3287  SDValue InL, InH;
3288  GetExpandedInteger(N->getOperand(0), InL, InH);
3289  // Just truncate the low part of the source.
3290  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
3291 }
3292 
3293 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
3294  SDValue Op = N->getOperand(0);
3295  EVT SrcVT = Op.getValueType();
3296  EVT DstVT = N->getValueType(0);
3297  SDLoc dl(N);
3298 
3299  // The following optimization is valid only if every value in SrcVT (when
3300  // treated as signed) is representable in DstVT. Check that the mantissa
3301  // size of DstVT is >= than the number of bits in SrcVT -1.
3302  const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT);
3303  if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 &&
3305  // Do a signed conversion then adjust the result.
3306  SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op);
3307  SignedConv = TLI.LowerOperation(SignedConv, DAG);
3308 
3309  // The result of the signed conversion needs adjusting if the 'sign bit' of
3310  // the incoming integer was set. To handle this, we dynamically test to see
3311  // if it is set, and, if so, add a fudge factor.
3312 
3313  const uint64_t F32TwoE32 = 0x4F800000ULL;
3314  const uint64_t F32TwoE64 = 0x5F800000ULL;
3315  const uint64_t F32TwoE128 = 0x7F800000ULL;
3316 
3317  APInt FF(32, 0);
3318  if (SrcVT == MVT::i32)
3319  FF = APInt(32, F32TwoE32);
3320  else if (SrcVT == MVT::i64)
3321  FF = APInt(32, F32TwoE64);
3322  else if (SrcVT == MVT::i128)
3323  FF = APInt(32, F32TwoE128);
3324  else
3325  llvm_unreachable("Unsupported UINT_TO_FP!");
3326 
3327  // Check whether the sign bit is set.
3328  SDValue Lo, Hi;
3329  GetExpandedInteger(Op, Lo, Hi);
3330  SDValue SignSet = DAG.getSetCC(dl,
3331  getSetCCResultType(Hi.getValueType()),
3332  Hi,
3333  DAG.getConstant(0, dl, Hi.getValueType()),
3334  ISD::SETLT);
3335 
3336  // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
3337  SDValue FudgePtr =
3338  DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)),
3339  TLI.getPointerTy(DAG.getDataLayout()));
3340 
3341  // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
3342  SDValue Zero = DAG.getIntPtrConstant(0, dl);
3343  SDValue Four = DAG.getIntPtrConstant(4, dl);
3344  if (DAG.getDataLayout().isBigEndian())
3345  std::swap(Zero, Four);
3346  SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet,
3347  Zero, Four);
3348  unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment();
3349  FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(),
3350  FudgePtr, Offset);
3351  Alignment = std::min(Alignment, 4u);
3352 
3353  // Load the value out, extending it from f32 to the destination float type.
3354  // FIXME: Avoid the extend by constructing the right constant pool?
3355  SDValue Fudge = DAG.getExtLoad(
3356  ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), FudgePtr,
3358  Alignment);
3359  return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge);
3360  }
3361 
3362  // Otherwise, use a libcall.
3363  RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT);
3364  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3365  "Don't know how to expand this UINT_TO_FP!");
3366  return TLI.makeLibCall(DAG, LC, DstVT, Op, true, dl).first;
3367 }
3368 
3369 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
3370  SDLoc dl(N);
3371  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
3372  cast<AtomicSDNode>(N)->getMemoryVT(),
3373  N->getOperand(0),
3374  N->getOperand(1), N->getOperand(2),
3375  cast<AtomicSDNode>(N)->getMemOperand());
3376  return Swap.getValue(1);
3377 }
3378 
3379 
3380 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
3381  SDValue InOp0 = N->getOperand(0);
3382  EVT InVT = InOp0.getValueType();
3383 
3384  EVT OutVT = N->getValueType(0);
3385  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3386  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3387  unsigned OutNumElems = OutVT.getVectorNumElements();
3388  EVT NOutVTElem = NOutVT.getVectorElementType();
3389 
3390  SDLoc dl(N);
3391  SDValue BaseIdx = N->getOperand(1);
3392 
3394  Ops.reserve(OutNumElems);
3395  for (unsigned i = 0; i != OutNumElems; ++i) {
3396 
3397  // Extract the element from the original vector.
3398  SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
3399  BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
3401  InVT.getVectorElementType(), N->getOperand(0), Index);
3402 
3403  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext);
3404  // Insert the converted element to the new vector.
3405  Ops.push_back(Op);
3406  }
3407 
3408  return DAG.getBuildVector(NOutVT, dl, Ops);
3409 }
3410 
3411 
3412 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
3413  ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
3414  EVT VT = N->getValueType(0);
3415  SDLoc dl(N);
3416 
3417  ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
3418 
3419  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3420  SDValue V1 = GetPromotedInteger(N->getOperand(1));
3421  EVT OutVT = V0.getValueType();
3422 
3423  return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
3424 }
3425 
3426 
3427 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
3428  EVT OutVT = N->getValueType(0);
3429  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3430  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3431  unsigned NumElems = N->getNumOperands();
3432  EVT NOutVTElem = NOutVT.getVectorElementType();
3433 
3434  SDLoc dl(N);
3435 
3437  Ops.reserve(NumElems);
3438  for (unsigned i = 0; i != NumElems; ++i) {
3439  SDValue Op;
3440  // BUILD_VECTOR integer operand types are allowed to be larger than the
3441  // result's element type. This may still be true after the promotion. For
3442  // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
3443  // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
3444  if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
3445  Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
3446  else
3447  Op = N->getOperand(i);
3448  Ops.push_back(Op);
3449  }
3450 
3451  return DAG.getBuildVector(NOutVT, dl, Ops);
3452 }
3453 
3454 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
3455 
3456  SDLoc dl(N);
3457 
3458  assert(!N->getOperand(0).getValueType().isVector() &&
3459  "Input must be a scalar");
3460 
3461  EVT OutVT = N->getValueType(0);
3462  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3463  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3464  EVT NOutVTElem = NOutVT.getVectorElementType();
3465 
3466  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
3467 
3468  return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
3469 }
3470 
3471 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
3472  SDLoc dl(N);
3473 
3474  EVT OutVT = N->getValueType(0);
3475  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3476  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3477 
3478  EVT OutElemTy = NOutVT.getVectorElementType();
3479 
3480  unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
3481  unsigned NumOutElem = NOutVT.getVectorNumElements();
3482  unsigned NumOperands = N->getNumOperands();
3483  assert(NumElem * NumOperands == NumOutElem &&
3484  "Unexpected number of elements");
3485 
3486  // Take the elements from the first vector.
3487  SmallVector<SDValue, 8> Ops(NumOutElem);
3488  for (unsigned i = 0; i < NumOperands; ++i) {
3489  SDValue Op = N->getOperand(i);
3490  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
3491  Op = GetPromotedInteger(Op);
3492  EVT SclrTy = Op.getValueType().getVectorElementType();
3493  assert(NumElem == Op.getValueType().getVectorNumElements() &&
3494  "Unexpected number of elements");
3495 
3496  for (unsigned j = 0; j < NumElem; ++j) {
3497  SDValue Ext = DAG.getNode(
3498  ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
3499  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3500  Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
3501  }
3502  }
3503 
3504  return DAG.getBuildVector(NOutVT, dl, Ops);
3505 }
3506 
3507 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
3508  EVT VT = N->getValueType(0);
3509  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3510  assert(NVT.isVector() && "This type must be promoted to a vector type");
3511 
3512  SDLoc dl(N);
3513 
3514  // For operands whose TypeAction is to promote, extend the promoted node
3515  // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
3516  // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
3517  // type..
3518  if (getTypeAction(N->getOperand(0).getValueType())
3520  SDValue Promoted;
3521 
3522  switch(N->getOpcode()) {
3524  Promoted = SExtPromotedInteger(N->getOperand(0));
3525  break;
3527  Promoted = ZExtPromotedInteger(N->getOperand(0));
3528  break;
3530  Promoted = GetPromotedInteger(N->getOperand(0));
3531  break;
3532  default:
3533  llvm_unreachable("Node has unexpected Opcode");
3534  }
3535  return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
3536  }
3537 
3538  // Directly extend to the appropriate transform-to type.
3539  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
3540 }
3541 
3542 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
3543  EVT OutVT = N->getValueType(0);
3544  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3545  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3546 
3547  EVT NOutVTElem = NOutVT.getVectorElementType();
3548 
3549  SDLoc dl(N);
3550  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3551 
3552  SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
3553  NOutVTElem, N->getOperand(1));
3554  return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
3555  V0, ConvElem, N->getOperand(2));
3556 }
3557 
3558 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3559  SDLoc dl(N);
3560  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3561  SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
3562  TLI.getVectorIdxTy(DAG.getDataLayout()));
3564  V0->getValueType(0).getScalarType(), V0, V1);
3565 
3566  // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
3567  // element types. If this is the case then we need to expand the outgoing
3568  // value and not truncate it.
3569  return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
3570 }
3571 
3572 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
3573  SDLoc dl(N);
3574  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3575  MVT InVT = V0.getValueType().getSimpleVT();
3576  MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
3578  SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
3579  return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
3580 }
3581 
3582 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
3583  SDLoc dl(N);
3584  unsigned NumElems = N->getNumOperands();
3585 
3586  EVT RetSclrTy = N->getValueType(0).getVectorElementType();
3587 
3588  SmallVector<SDValue, 8> NewOps;
3589  NewOps.reserve(NumElems);
3590 
3591  // For each incoming vector
3592  for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
3593  SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
3594  EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
3595  unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
3596 
3597  for (unsigned i=0; i<NumElem; ++i) {
3598  // Extract element from incoming vector
3599  SDValue Ex = DAG.getNode(
3600  ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
3601  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3602  SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
3603  NewOps.push_back(Tr);
3604  }
3605  }
3606 
3607  return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
3608 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:541
void push_back(const T &Elt)
Definition: SmallVector.h:218
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:552
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getValueType() const
Return the ValueType of the referenced return value.
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant, which is required to be operand #1) half of the integer or float value specified as operand #0.
Definition: ISDOpcodes.h:184
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC&#39;s if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:951
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:334
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:610
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
static MVT getVectorVT(MVT VT, unsigned NumElements)
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:343
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:260
#define LLVM_FALLTHROUGH
Definition: Compiler.h:86
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:489
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none...
const SDValue & getBasePtr() const
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:223
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:858
const SDValue & getValue() const
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
Definition: APInt.h:648
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:253
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:251
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1323
const SDValue & getBasePtr() const
const SDValue & getChain() const
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
unsigned getAlignment() const
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:762
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select&#39;s if you just have operands and don&#39;t want to check...
Definition: SelectionDAG.h:964
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:480
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:360
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Same for subtraction.
Definition: ISDOpcodes.h:254
void reserve(size_type N)
Definition: SmallVector.h:376
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:212
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align=0, int Offs=0, bool isT=false, unsigned char TargetFlags=0)
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:407
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:39
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
SDValue getExternalSymbol(const char *Sym, EVT VT)
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:210
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations...
Definition: ISDOpcodes.h:428
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
static uint32_t getAlignment(const MCSectionCOFF &Sec)
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:451
Shift and rotation operations.
Definition: ISDOpcodes.h:382
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:202
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:639
CallLoweringInfo & setChain(SDValue InChain)
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:191
const SDValue & getPassThru() const
op_iterator op_end() const
unsigned getScalarValueSizeInBits() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
FLT_ROUNDS_ - Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest 2 Round to ...
Definition: ISDOpcodes.h:516
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:304
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt) For double-word atomic operations: ValLo, ValHi, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amtLo, amtHi) ValLo, ValHi, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amtLo, amtHi) These correspond to the atomicrmw instruction.
Definition: ISDOpcodes.h:770
const SDValue & getValue() const
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:457
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:398
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG...
Definition: ISDOpcodes.h:73
This is an SDNode representing atomic operations.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
This represents a list of ValueType&#39;s that has been intern&#39;d by a SelectionDAG.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
This class is used to represent an MSTORE node.
unsigned getSizeInBits() const
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:251
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:298
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:292
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:395
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:450
SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
const SDValue & getScale() const
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:401
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:201
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:221
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:865
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTruncating=false, bool IsCompressing=false)
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:919
op_iterator op_begin() const
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:707
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:467
const SDValue & getMask() const
UNDEF - An undefined node.
Definition: ISDOpcodes.h:178
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:494
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:303
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
Definition: APInt.h:636
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits...
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
Definition: MathExtras.h:610
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const SDValue & getBasePtr() const
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:429
Machine Value Type.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
Simple binary floating point operators.
Definition: ISDOpcodes.h:260
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:273
This is an important base class in LLVM.
Definition: Constant.h:42
iterator_range< value_op_iterator > op_values() const
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
Definition: SmallVector.h:129
const SDValue & getOperand(unsigned Num) const
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:894
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:308
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:247
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:232
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:588
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:57
const SDValue & getValue() const
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:679
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType, bool IsExpanding=false)
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:388
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo...
Definition: ISDOpcodes.h:756
std::vector< ArgListEntry > ArgListTy
const SDValue & getIndex() const
Extended Value Type.
Definition: ValueTypes.h:34
const SDValue & getBasePtr() const
This structure contains all information that is necessary for lowering calls.
This class contains a discriminated union of information about pointers in memory operands...
unsigned getNumOperands() const
Return the number of values used by this operation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, unsigned Alignment, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SyncScope::ID SSID)
Gets a node for an atomic cmpxchg op.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:50
void dump() const
Dump this node, for debugging.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:315
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:265
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
Definition: ISDOpcodes.h:423
const SDValue & getMask() const
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:847
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:729
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:604
const SDValue & getPassThru() const
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:644
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:385
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:551
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:621
Represents one node in the SelectionDAG.
static unsigned int semanticsPrecision(const fltSemantics &)
Definition: APFloat.cpp:155
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:133
MachinePointerInfo getWithOffset(int64_t O) const
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:539
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:941
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT...
Definition: ValueTypes.h:73
EVT getMemoryVT() const
Return the type of the in-memory value.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
Class for arbitrary precision integers.
Definition: APInt.h:70
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:392
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:241
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:440
const SDValue & getBasePtr() const
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:443
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition: APInt.h:530
Flags
Flags values. These may be or&#39;d together.
#define Success
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool ugt(const APInt &RHS) const
Unsigned greather than comparison.
Definition: APInt.h:1249
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:546
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Definition: SmallVector.h:133
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:151
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
Definition: APInt.h:1315
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:363
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, bool isSigned, const SDLoc &dl, bool doesNotReturn=false, bool isReturnValueUsed=true) const
Returns a pair of (return value, chain).
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:458
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:574
#define N
Flags getFlags() const
Return the raw flags of the source value,.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:323
Same for multiplication.
Definition: ISDOpcodes.h:257
const SDValue & getBasePtr() const
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
SDValue getValue(unsigned R) const
This class is used to represent an MSCATTER node.
unsigned getNumRegisters(LLVMContext &Context, EVT VT) const
Return the number of registers that this ValueType will eventually require.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getCondCode(ISD::CondCode Cond)
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, const Value *PtrVal, unsigned Alignment, AtomicOrdering Ordering, SyncScope::ID SSID)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands...
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input...
const MachinePointerInfo & getPointerInfo() const
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:321
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
This class is used to represent an MLOAD node.
ArrayRef< int > getMask() const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
This class is used to represent an MGATHER node.
SDValue getValueType(EVT)
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:81
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:415
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Definition: ValueTypes.h:223
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:198
Conversion operators.
Definition: ISDOpcodes.h:437
BooleanContent
Enum that describes how the target represents true/false values.
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:749
bool isBigEndian() const
Definition: DataLayout.h:222
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:446
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Op, int64_t Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object...
Definition: SelectionDAG.h:819
#define LLVM_DEBUG(X)
Definition: Debug.h:123
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:745
LLVMContext * getContext() const
Definition: SelectionDAG.h:404
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:351
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:242
const SDValue & getMask() const
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
This class is used to represent ISD::LOAD nodes.