LLVM  8.0.0svn
LegalizeIntegerTypes.cpp
Go to the documentation of this file.
1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements integer type expansion and promotion for LegalizeTypes.
11 // Promotion is the act of changing a computation in an illegal type into a
12 // computation in a larger type. For example, implementing i8 arithmetic in an
13 // i32 register (often needed on powerpc).
14 // Expansion is the act of changing a computation in an illegal type into a
15 // computation in two identical registers of a smaller type. For example,
16 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
17 // targets).
18 //
19 //===----------------------------------------------------------------------===//
20 
21 #include "LegalizeTypes.h"
22 #include "llvm/IR/DerivedTypes.h"
24 #include "llvm/Support/KnownBits.h"
26 using namespace llvm;
27 
28 #define DEBUG_TYPE "legalize-types"
29 
30 //===----------------------------------------------------------------------===//
31 // Integer Result Promotion
32 //===----------------------------------------------------------------------===//
33 
34 /// PromoteIntegerResult - This method is called when a result of a node is
35 /// found to be in need of promotion to a larger type. At this point, the node
36 /// may also have invalid operands or may have other results that need
37 /// expansion, we just know that (at least) one result needs promotion.
38 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
39  LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG);
40  dbgs() << "\n");
41  SDValue Res = SDValue();
42 
43  // See if the target wants to custom expand this node.
44  if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
45  LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
46  return;
47  }
48 
49  switch (N->getOpcode()) {
50  default:
51 #ifndef NDEBUG
52  dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
53  N->dump(&DAG); dbgs() << "\n";
54 #endif
55  llvm_unreachable("Do not know how to promote this operator!");
56  case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
57  case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
58  case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
59  case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
60  case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
61  case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
62  case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
63  case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
65  case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
66  case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break;
68  case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
70  Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
71  case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
72  case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
73  break;
74  case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
75  break;
76  case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
77  case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
78  case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
79  case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
80  case ISD::SMIN:
81  case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
82  case ISD::UMIN:
83  case ISD::UMAX: Res = PromoteIntRes_ZExtIntBinOp(N); break;
84 
85  case ISD::SHL: Res = PromoteIntRes_SHL(N); break;
87  Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
88  case ISD::SRA: Res = PromoteIntRes_SRA(N); break;
89  case ISD::SRL: Res = PromoteIntRes_SRL(N); break;
90  case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
91  case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
92  case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
93 
95  Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
97  Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
99  Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
100  case ISD::BUILD_VECTOR:
101  Res = PromoteIntRes_BUILD_VECTOR(N); break;
103  Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
104  case ISD::CONCAT_VECTORS:
105  Res = PromoteIntRes_CONCAT_VECTORS(N); break;
106 
110  Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
111 
112  case ISD::SIGN_EXTEND:
113  case ISD::ZERO_EXTEND:
114  case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
115 
116  case ISD::FP_TO_SINT:
117  case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
118 
119  case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
120 
121  case ISD::FLT_ROUNDS_: Res = PromoteIntRes_FLT_ROUNDS(N); break;
122 
123  case ISD::AND:
124  case ISD::OR:
125  case ISD::XOR:
126  case ISD::ADD:
127  case ISD::SUB:
128  case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
129 
130  case ISD::SDIV:
131  case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
132 
133  case ISD::UDIV:
134  case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
135 
136  case ISD::SADDO:
137  case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
138  case ISD::UADDO:
139  case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
140  case ISD::SMULO:
141  case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
142 
143  case ISD::ADDCARRY:
144  case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
145 
146  case ISD::SADDSAT:
147  case ISD::UADDSAT:
148  case ISD::SSUBSAT:
149  case ISD::USUBSAT: Res = PromoteIntRes_ADDSUBSAT(N); break;
150  case ISD::SMULFIX: Res = PromoteIntRes_SMULFIX(N); break;
151 
152  case ISD::ATOMIC_LOAD:
153  Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
154 
159  case ISD::ATOMIC_LOAD_OR:
166  case ISD::ATOMIC_SWAP:
167  Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
168 
171  Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
172  break;
173  }
174 
175  // If the result is null then the sub-method took care of registering it.
176  if (Res.getNode())
177  SetPromotedInteger(SDValue(N, ResNo), Res);
178 }
179 
180 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
181  unsigned ResNo) {
182  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
183  return GetPromotedInteger(Op);
184 }
185 
186 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
187  // Sign-extend the new bits, and continue the assertion.
188  SDValue Op = SExtPromotedInteger(N->getOperand(0));
189  return DAG.getNode(ISD::AssertSext, SDLoc(N),
190  Op.getValueType(), Op, N->getOperand(1));
191 }
192 
193 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
194  // Zero the new bits, and continue the assertion.
195  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
196  return DAG.getNode(ISD::AssertZext, SDLoc(N),
197  Op.getValueType(), Op, N->getOperand(1));
198 }
199 
200 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
201  EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
202  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
203  N->getMemoryVT(), ResVT,
204  N->getChain(), N->getBasePtr(),
205  N->getMemOperand());
206  // Legalize the chain result - switch anything that used the old chain to
207  // use the new one.
208  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
209  return Res;
210 }
211 
212 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
213  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
214  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
215  N->getMemoryVT(),
216  N->getChain(), N->getBasePtr(),
217  Op2, N->getMemOperand());
218  // Legalize the chain result - switch anything that used the old chain to
219  // use the new one.
220  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
221  return Res;
222 }
223 
224 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
225  unsigned ResNo) {
226  if (ResNo == 1) {
228  EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
229  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
230 
231  // Only use the result of getSetCCResultType if it is legal,
232  // otherwise just use the promoted result type (NVT).
233  if (!TLI.isTypeLegal(SVT))
234  SVT = NVT;
235 
236  SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
237  SDValue Res = DAG.getAtomicCmpSwap(
239  N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
240  N->getMemOperand());
241  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
242  ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
243  return Res.getValue(1);
244  }
245 
246  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
247  SDValue Op3 = GetPromotedInteger(N->getOperand(3));
248  SDVTList VTs =
249  DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
250  SDValue Res = DAG.getAtomicCmpSwap(
251  N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
252  N->getBasePtr(), Op2, Op3, N->getMemOperand());
253  // Update the use to N with the newly created Res.
254  for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
255  ReplaceValueWith(SDValue(N, i), Res.getValue(i));
256  return Res;
257 }
258 
259 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
260  SDValue InOp = N->getOperand(0);
261  EVT InVT = InOp.getValueType();
262  EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
263  EVT OutVT = N->getValueType(0);
264  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
265  SDLoc dl(N);
266 
267  switch (getTypeAction(InVT)) {
269  break;
271  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
272  // The input promotes to the same size. Convert the promoted value.
273  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
274  break;
276  // Promote the integer operand by hand.
277  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
279  // Convert the promoted float by hand.
280  if (!NOutVT.isVector())
281  return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
282  break;
283  }
286  break;
288  // Convert the element to an integer and promote it by hand.
289  if (!NOutVT.isVector())
290  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
291  BitConvertToInteger(GetScalarizedVector(InOp)));
292  break;
294  // For example, i32 = BITCAST v2i16 on alpha. Convert the split
295  // pieces of the input into integers and reassemble in the final type.
296  SDValue Lo, Hi;
297  GetSplitVector(N->getOperand(0), Lo, Hi);
298  Lo = BitConvertToInteger(Lo);
299  Hi = BitConvertToInteger(Hi);
300 
301  if (DAG.getDataLayout().isBigEndian())
302  std::swap(Lo, Hi);
303 
304  InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
306  NOutVT.getSizeInBits()),
307  JoinIntegers(Lo, Hi));
308  return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
309  }
311  // The input is widened to the same size. Convert to the widened value.
312  // Make sure that the outgoing value is not a vector, because this would
313  // make us bitcast between two vectors which are legalized in different ways.
314  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector())
315  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
316  // If the output type is also a vector and widening it to the same size
317  // as the widened input type would be a legal type, we can widen the bitcast
318  // and handle the promotion after.
319  if (NOutVT.isVector()) {
320  unsigned WidenInSize = NInVT.getSizeInBits();
321  unsigned OutSize = OutVT.getSizeInBits();
322  if (WidenInSize % OutSize == 0) {
323  unsigned Scale = WidenInSize / OutSize;
324  EVT WideOutVT = EVT::getVectorVT(*DAG.getContext(),
325  OutVT.getVectorElementType(),
326  OutVT.getVectorNumElements() * Scale);
327  if (isTypeLegal(WideOutVT)) {
328  InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
329  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
330  InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
331  DAG.getConstant(0, dl, IdxTy));
332  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
333  }
334  }
335  }
336  }
337 
338  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
339  CreateStackStoreLoad(InOp, OutVT));
340 }
341 
342 // Helper for BSWAP/BITREVERSE promotion to ensure we can fit the shift amount
343 // in the VT returned by getShiftAmountTy and to return a safe VT if we can't.
344 static EVT getShiftAmountTyForConstant(unsigned Val, EVT VT,
345  const TargetLowering &TLI,
346  SelectionDAG &DAG) {
347  EVT ShiftVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
348  // If the value won't fit in the prefered type, just use something safe. It
349  // will be legalized when the shift is expanded.
350  if ((Log2_32(Val) + 1) > ShiftVT.getScalarSizeInBits())
351  ShiftVT = MVT::i32;
352  return ShiftVT;
353 }
354 
355 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
356  SDValue Op = GetPromotedInteger(N->getOperand(0));
357  EVT OVT = N->getValueType(0);
358  EVT NVT = Op.getValueType();
359  SDLoc dl(N);
360 
361  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
362  EVT ShiftVT = getShiftAmountTyForConstant(DiffBits, NVT, TLI, DAG);
363  return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
364  DAG.getConstant(DiffBits, dl, ShiftVT));
365 }
366 
367 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
368  SDValue Op = GetPromotedInteger(N->getOperand(0));
369  EVT OVT = N->getValueType(0);
370  EVT NVT = Op.getValueType();
371  SDLoc dl(N);
372 
373  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
374  EVT ShiftVT = getShiftAmountTyForConstant(DiffBits, NVT, TLI, DAG);
375  return DAG.getNode(ISD::SRL, dl, NVT,
376  DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
377  DAG.getConstant(DiffBits, dl, ShiftVT));
378 }
379 
380 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
381  // The pair element type may be legal, or may not promote to the same type as
382  // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
383  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
384  TLI.getTypeToTransformTo(*DAG.getContext(),
385  N->getValueType(0)), JoinIntegers(N->getOperand(0),
386  N->getOperand(1)));
387 }
388 
389 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
390  EVT VT = N->getValueType(0);
391  // FIXME there is no actual debug info here
392  SDLoc dl(N);
393  // Zero extend things like i1, sign extend everything else. It shouldn't
394  // matter in theory which one we pick, but this tends to give better code?
395  unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
396  SDValue Result = DAG.getNode(Opc, dl,
397  TLI.getTypeToTransformTo(*DAG.getContext(), VT),
398  SDValue(N, 0));
399  assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
400  return Result;
401 }
402 
403 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
404  // Zero extend to the promoted type and do the count there.
405  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
406  SDLoc dl(N);
407  EVT OVT = N->getValueType(0);
408  EVT NVT = Op.getValueType();
409  Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
410  // Subtract off the extra leading bits in the bigger type.
411  return DAG.getNode(
412  ISD::SUB, dl, NVT, Op,
413  DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
414  NVT));
415 }
416 
417 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
418  // Zero extend to the promoted type and do the count there.
419  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
420  return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op);
421 }
422 
423 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
424  SDValue Op = GetPromotedInteger(N->getOperand(0));
425  EVT OVT = N->getValueType(0);
426  EVT NVT = Op.getValueType();
427  SDLoc dl(N);
428  if (N->getOpcode() == ISD::CTTZ) {
429  // The count is the same in the promoted type except if the original
430  // value was zero. This can be handled by setting the bit just off
431  // the top of the original type.
432  auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
433  OVT.getScalarSizeInBits());
434  Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
435  }
436  return DAG.getNode(N->getOpcode(), dl, NVT, Op);
437 }
438 
439 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
440  SDLoc dl(N);
441  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
442  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0),
443  N->getOperand(1));
444 }
445 
446 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
447  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
448  unsigned NewOpc = N->getOpcode();
449  SDLoc dl(N);
450 
451  // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
452  // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
453  // and SINT conversions are Custom, there is no way to tell which is
454  // preferable. We choose SINT because that's the right thing on PPC.)
455  if (N->getOpcode() == ISD::FP_TO_UINT &&
456  !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
458  NewOpc = ISD::FP_TO_SINT;
459 
460  SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
461 
462  // Assert that the converted value fits in the original type. If it doesn't
463  // (eg: because the value being converted is too big), then the result of the
464  // original operation was undefined anyway, so the assert is still correct.
465  //
466  // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
467  // before legalization: fp-to-uint16, 65534. -> 0xfffe
468  // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
469  return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ?
470  ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
472 }
473 
474 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
475  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
476  SDLoc dl(N);
477 
478  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
479 }
480 
481 SDValue DAGTypeLegalizer::PromoteIntRes_FLT_ROUNDS(SDNode *N) {
482  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
483  SDLoc dl(N);
484 
485  return DAG.getNode(N->getOpcode(), dl, NVT);
486 }
487 
488 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
489  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
490  SDLoc dl(N);
491 
492  if (getTypeAction(N->getOperand(0).getValueType())
494  SDValue Res = GetPromotedInteger(N->getOperand(0));
495  assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
496 
497  // If the result and operand types are the same after promotion, simplify
498  // to an in-register extension.
499  if (NVT == Res.getValueType()) {
500  // The high bits are not guaranteed to be anything. Insert an extend.
501  if (N->getOpcode() == ISD::SIGN_EXTEND)
502  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
503  DAG.getValueType(N->getOperand(0).getValueType()));
504  if (N->getOpcode() == ISD::ZERO_EXTEND)
505  return DAG.getZeroExtendInReg(Res, dl,
507  assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
508  return Res;
509  }
510  }
511 
512  // Otherwise, just extend the original operand all the way to the larger type.
513  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
514 }
515 
516 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
517  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
518  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
521  SDLoc dl(N);
522  SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
523  N->getMemoryVT(), N->getMemOperand());
524 
525  // Legalize the chain result - switch anything that used the old chain to
526  // use the new one.
527  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
528  return Res;
529 }
530 
531 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
532  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
533  SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
534 
535  SDLoc dl(N);
536  SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
537  N->getMask(), ExtPassThru, N->getMemoryVT(),
539  // Legalize the chain result - switch anything that used the old chain to
540  // use the new one.
541  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
542  return Res;
543 }
544 
545 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
546  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
547  SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
548  assert(NVT == ExtPassThru.getValueType() &&
549  "Gather result type and the passThru agrument type should be the same");
550 
551  SDLoc dl(N);
552  SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
553  N->getIndex(), N->getScale() };
554  SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
555  N->getMemoryVT(), dl, Ops,
556  N->getMemOperand());
557  // Legalize the chain result - switch anything that used the old chain to
558  // use the new one.
559  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
560  return Res;
561 }
562 
563 /// Promote the overflow flag of an overflowing arithmetic node.
564 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
565  // Simply change the return type of the boolean result.
566  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
567  EVT ValueVTs[] = { N->getValueType(0), NVT };
568  SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
569  unsigned NumOps = N->getNumOperands();
570  assert(NumOps <= 3 && "Too many operands");
571  if (NumOps == 3)
572  Ops[2] = N->getOperand(2);
573 
574  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
575  DAG.getVTList(ValueVTs), makeArrayRef(Ops, NumOps));
576 
577  // Modified the sum result - switch anything that used the old sum to use
578  // the new one.
579  ReplaceValueWith(SDValue(N, 0), Res);
580 
581  return SDValue(Res.getNode(), 1);
582 }
583 
584 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSAT(SDNode *N) {
585  // For promoting iN -> iM, this can be expanded by
586  // 1. ANY_EXTEND iN to iM
587  // 2. SHL by M-N
588  // 3. [US][ADD|SUB]SAT
589  // 4. L/ASHR by M-N
590  SDLoc dl(N);
591  SDValue Op1 = N->getOperand(0);
592  SDValue Op2 = N->getOperand(1);
593  unsigned OldBits = Op1.getValueSizeInBits();
594 
595  unsigned Opcode = N->getOpcode();
596  unsigned ShiftOp;
597  switch (Opcode) {
598  case ISD::SADDSAT:
599  case ISD::SSUBSAT:
600  ShiftOp = ISD::SRA;
601  break;
602  case ISD::UADDSAT:
603  case ISD::USUBSAT:
604  ShiftOp = ISD::SRL;
605  break;
606  default:
607  llvm_unreachable("Expected opcode to be signed or unsigned saturation "
608  "addition or subtraction");
609  }
610 
611  SDValue Op1Promoted = GetPromotedInteger(Op1);
612  SDValue Op2Promoted = GetPromotedInteger(Op2);
613 
614  EVT PromotedType = Op1Promoted.getValueType();
615  unsigned NewBits = Op1Promoted.getValueSizeInBits();
616  unsigned SHLAmount = NewBits - OldBits;
617  EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
618  SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT);
619  Op1Promoted =
620  DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
621  Op2Promoted =
622  DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
623 
624  SDValue Result =
625  DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
626  return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
627 }
628 
629 SDValue DAGTypeLegalizer::PromoteIntRes_SMULFIX(SDNode *N) {
630  // Can just promote the operands then continue with operation.
631  SDLoc dl(N);
632  SDValue Op1Promoted = SExtPromotedInteger(N->getOperand(0));
633  SDValue Op2Promoted = SExtPromotedInteger(N->getOperand(1));
634  EVT PromotedType = Op1Promoted.getValueType();
635  return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
636  N->getOperand(2));
637 }
638 
639 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
640  if (ResNo == 1)
641  return PromoteIntRes_Overflow(N);
642 
643  // The operation overflowed iff the result in the larger type is not the
644  // sign extension of its truncation to the original type.
645  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
646  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
647  EVT OVT = N->getOperand(0).getValueType();
648  EVT NVT = LHS.getValueType();
649  SDLoc dl(N);
650 
651  // Do the arithmetic in the larger type.
652  unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
653  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
654 
655  // Calculate the overflow flag: sign extend the arithmetic result from
656  // the original type.
657  SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
658  DAG.getValueType(OVT));
659  // Overflowed if and only if this is not equal to Res.
660  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
661 
662  // Use the calculated overflow everywhere.
663  ReplaceValueWith(SDValue(N, 1), Ofl);
664 
665  return Res;
666 }
667 
668 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
669  SDValue LHS = GetPromotedInteger(N->getOperand(1));
670  SDValue RHS = GetPromotedInteger(N->getOperand(2));
671  return DAG.getSelect(SDLoc(N),
672  LHS.getValueType(), N->getOperand(0), LHS, RHS);
673 }
674 
675 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
676  SDValue Mask = N->getOperand(0);
677 
678  SDValue LHS = GetPromotedInteger(N->getOperand(1));
679  SDValue RHS = GetPromotedInteger(N->getOperand(2));
680  return DAG.getNode(ISD::VSELECT, SDLoc(N),
681  LHS.getValueType(), Mask, LHS, RHS);
682 }
683 
684 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
685  SDValue LHS = GetPromotedInteger(N->getOperand(2));
686  SDValue RHS = GetPromotedInteger(N->getOperand(3));
687  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
688  LHS.getValueType(), N->getOperand(0),
689  N->getOperand(1), LHS, RHS, N->getOperand(4));
690 }
691 
692 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
693  EVT InVT = N->getOperand(0).getValueType();
694  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
695 
696  EVT SVT = getSetCCResultType(InVT);
697 
698  // If we got back a type that needs to be promoted, this likely means the
699  // the input type also needs to be promoted. So get the promoted type for
700  // the input and try the query again.
701  if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
702  if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
703  InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
704  SVT = getSetCCResultType(InVT);
705  } else {
706  // Input type isn't promoted, just use the default promoted type.
707  SVT = NVT;
708  }
709  }
710 
711  SDLoc dl(N);
712  assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() &&
713  "Vector compare must return a vector result!");
714 
715  // Get the SETCC result using the canonical SETCC type.
716  SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
717  N->getOperand(1), N->getOperand(2));
718 
719  // Convert to the expected type.
720  return DAG.getSExtOrTrunc(SetCC, dl, NVT);
721 }
722 
723 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
724  SDValue LHS = GetPromotedInteger(N->getOperand(0));
725  SDValue RHS = N->getOperand(1);
726  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
727  RHS = ZExtPromotedInteger(RHS);
728  return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
729 }
730 
731 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
732  SDValue Op = GetPromotedInteger(N->getOperand(0));
733  return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
734  Op.getValueType(), Op, N->getOperand(1));
735 }
736 
737 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
738  // The input may have strange things in the top bits of the registers, but
739  // these operations don't care. They may have weird bits going out, but
740  // that too is okay if they are integer operations.
741  SDValue LHS = GetPromotedInteger(N->getOperand(0));
742  SDValue RHS = GetPromotedInteger(N->getOperand(1));
743  return DAG.getNode(N->getOpcode(), SDLoc(N),
744  LHS.getValueType(), LHS, RHS);
745 }
746 
747 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
748  // Sign extend the input.
749  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
750  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
751  return DAG.getNode(N->getOpcode(), SDLoc(N),
752  LHS.getValueType(), LHS, RHS);
753 }
754 
755 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
756  // Zero extend the input.
757  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
758  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
759  return DAG.getNode(N->getOpcode(), SDLoc(N),
760  LHS.getValueType(), LHS, RHS);
761 }
762 
763 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
764  // The input value must be properly sign extended.
765  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
766  SDValue RHS = N->getOperand(1);
767  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
768  RHS = ZExtPromotedInteger(RHS);
769  return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
770 }
771 
772 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
773  // The input value must be properly zero extended.
774  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
775  SDValue RHS = N->getOperand(1);
776  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
777  RHS = ZExtPromotedInteger(RHS);
778  return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
779 }
780 
781 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
782  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
783  SDValue Res;
784  SDValue InOp = N->getOperand(0);
785  SDLoc dl(N);
786 
787  switch (getTypeAction(InOp.getValueType())) {
788  default: llvm_unreachable("Unknown type action!");
791  Res = InOp;
792  break;
794  Res = GetPromotedInteger(InOp);
795  break;
797  EVT InVT = InOp.getValueType();
798  assert(InVT.isVector() && "Cannot split scalar types");
799  unsigned NumElts = InVT.getVectorNumElements();
800  assert(NumElts == NVT.getVectorNumElements() &&
801  "Dst and Src must have the same number of elements");
802  assert(isPowerOf2_32(NumElts) &&
803  "Promoted vector type must be a power of two");
804 
805  SDValue EOp1, EOp2;
806  GetSplitVector(InOp, EOp1, EOp2);
807 
808  EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
809  NumElts/2);
810  EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
811  EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
812 
813  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
814  }
816  SDValue WideInOp = GetWidenedVector(InOp);
817 
818  // Truncate widened InOp.
819  unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
820  EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
821  N->getValueType(0).getScalarType(), NumElem);
822  SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
823 
824  // Zero extend so that the elements are of same type as those of NVT
825  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
826  NumElem);
827  SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
828 
829  // Extract the low NVT subvector.
830  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
831  SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy);
832  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
833  }
834  }
835 
836  // Truncate to NVT instead of VT
837  return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
838 }
839 
840 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
841  if (ResNo == 1)
842  return PromoteIntRes_Overflow(N);
843 
844  // The operation overflowed iff the result in the larger type is not the
845  // zero extension of its truncation to the original type.
846  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
847  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
848  EVT OVT = N->getOperand(0).getValueType();
849  EVT NVT = LHS.getValueType();
850  SDLoc dl(N);
851 
852  // Do the arithmetic in the larger type.
853  unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
854  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
855 
856  // Calculate the overflow flag: zero extend the arithmetic result from
857  // the original type.
858  SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
859  // Overflowed if and only if this is not equal to Res.
860  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
861 
862  // Use the calculated overflow everywhere.
863  ReplaceValueWith(SDValue(N, 1), Ofl);
864 
865  return Res;
866 }
867 
868 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
869  if (ResNo == 1)
870  return PromoteIntRes_Overflow(N);
871 
872  // We need to sign-extend the operands so the carry value computed by the
873  // wide operation will be equivalent to the carry value computed by the
874  // narrow operation.
875  // An ADDCARRY can generate carry only if any of the operands has its
876  // most significant bit set. Sign extension propagates the most significant
877  // bit into the higher bits which means the extra bit that the narrow
878  // addition would need (i.e. the carry) will be propagated through the higher
879  // bits of the wide addition.
880  // A SUBCARRY can generate borrow only if LHS < RHS and this property will be
881  // preserved by sign extension.
882  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
883  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
884 
885  EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
886 
887  // Do the arithmetic in the wide type.
888  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
889  LHS, RHS, N->getOperand(2));
890 
891  // Update the users of the original carry/borrow value.
892  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
893 
894  return SDValue(Res.getNode(), 0);
895 }
896 
897 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
898  // Promote the overflow bit trivially.
899  if (ResNo == 1)
900  return PromoteIntRes_Overflow(N);
901 
902  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
903  SDLoc DL(N);
904  EVT SmallVT = LHS.getValueType();
905 
906  // To determine if the result overflowed in a larger type, we extend the
907  // input to the larger type, do the multiply (checking if it overflows),
908  // then also check the high bits of the result to see if overflow happened
909  // there.
910  if (N->getOpcode() == ISD::SMULO) {
911  LHS = SExtPromotedInteger(LHS);
912  RHS = SExtPromotedInteger(RHS);
913  } else {
914  LHS = ZExtPromotedInteger(LHS);
915  RHS = ZExtPromotedInteger(RHS);
916  }
917  SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
918  SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
919 
920  // Overflow occurred if it occurred in the larger type, or if the high part
921  // of the result does not zero/sign-extend the low part. Check this second
922  // possibility first.
923  SDValue Overflow;
924  if (N->getOpcode() == ISD::UMULO) {
925  // Unsigned overflow occurred if the high part is non-zero.
926  SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
927  DAG.getIntPtrConstant(SmallVT.getSizeInBits(),
928  DL));
929  Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
930  DAG.getConstant(0, DL, Hi.getValueType()),
931  ISD::SETNE);
932  } else {
933  // Signed overflow occurred if the high part does not sign extend the low.
934  SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
935  Mul, DAG.getValueType(SmallVT));
936  Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
937  }
938 
939  // The only other way for overflow to occur is if the multiplication in the
940  // larger type itself overflowed.
941  Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
942  SDValue(Mul.getNode(), 1));
943 
944  // Use the calculated overflow everywhere.
945  ReplaceValueWith(SDValue(N, 1), Overflow);
946  return Mul;
947 }
948 
949 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
950  return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
951  N->getValueType(0)));
952 }
953 
954 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
955  SDValue Chain = N->getOperand(0); // Get the chain.
956  SDValue Ptr = N->getOperand(1); // Get the pointer.
957  EVT VT = N->getValueType(0);
958  SDLoc dl(N);
959 
960  MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
961  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
962  // The argument is passed as NumRegs registers of type RegVT.
963 
964  SmallVector<SDValue, 8> Parts(NumRegs);
965  for (unsigned i = 0; i < NumRegs; ++i) {
966  Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
967  N->getConstantOperandVal(3));
968  Chain = Parts[i].getValue(1);
969  }
970 
971  // Handle endianness of the load.
972  if (DAG.getDataLayout().isBigEndian())
973  std::reverse(Parts.begin(), Parts.end());
974 
975  // Assemble the parts in the promoted type.
976  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
977  SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
978  for (unsigned i = 1; i < NumRegs; ++i) {
979  SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
980  // Shift it to the right position and "or" it in.
981  Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
982  DAG.getConstant(i * RegVT.getSizeInBits(), dl,
983  TLI.getPointerTy(DAG.getDataLayout())));
984  Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
985  }
986 
987  // Modified the chain result - switch anything that used the old chain to
988  // use the new one.
989  ReplaceValueWith(SDValue(N, 1), Chain);
990 
991  return Res;
992 }
993 
994 //===----------------------------------------------------------------------===//
995 // Integer Operand Promotion
996 //===----------------------------------------------------------------------===//
997 
998 /// PromoteIntegerOperand - This method is called when the specified operand of
999 /// the specified node is found to need promotion. At this point, all of the
1000 /// result types of the node are known to be legal, but other operands of the
1001 /// node may need promotion or expansion as well as the specified one.
1002 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
1003  LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG);
1004  dbgs() << "\n");
1005  SDValue Res = SDValue();
1006 
1007  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
1008  LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
1009  return false;
1010  }
1011 
1012  switch (N->getOpcode()) {
1013  default:
1014  #ifndef NDEBUG
1015  dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
1016  N->dump(&DAG); dbgs() << "\n";
1017  #endif
1018  llvm_unreachable("Do not know how to promote this operator's operand!");
1019 
1020  case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
1021  case ISD::ATOMIC_STORE:
1022  Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
1023  break;
1024  case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
1025  case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
1026  case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
1027  case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
1028  case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
1029  case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
1030  case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
1032  Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
1033  case ISD::SCALAR_TO_VECTOR:
1034  Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
1035  case ISD::VSELECT:
1036  case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
1037  case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
1038  case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
1039  case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
1040  case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
1041  case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
1042  OpNo); break;
1043  case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
1044  OpNo); break;
1045  case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
1046  OpNo); break;
1047  case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
1048  OpNo); break;
1049  case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
1050  OpNo); break;
1051  case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
1052  case ISD::FP16_TO_FP:
1053  case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
1054  case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
1055  case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
1056 
1057  case ISD::SHL:
1058  case ISD::SRA:
1059  case ISD::SRL:
1060  case ISD::ROTL:
1061  case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
1062 
1063  case ISD::ADDCARRY:
1064  case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
1065 
1066  case ISD::FRAMEADDR:
1067  case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
1068 
1069  case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break;
1070 
1071  case ISD::SMULFIX: Res = PromoteIntOp_SMULFIX(N); break;
1072  }
1073 
1074  // If the result is null, the sub-method took care of registering results etc.
1075  if (!Res.getNode()) return false;
1076 
1077  // If the result is N, the sub-method updated N in place. Tell the legalizer
1078  // core about this.
1079  if (Res.getNode() == N)
1080  return true;
1081 
1082  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1083  "Invalid operand expansion");
1084 
1085  ReplaceValueWith(SDValue(N, 0), Res);
1086  return false;
1087 }
1088 
1089 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
1090 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
1091 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
1092  ISD::CondCode CCCode) {
1093  // We have to insert explicit sign or zero extends. Note that we could
1094  // insert sign extends for ALL conditions. For those operations where either
1095  // zero or sign extension would be valid, use SExtOrZExtPromotedInteger
1096  // which will choose the cheapest for the target.
1097  switch (CCCode) {
1098  default: llvm_unreachable("Unknown integer comparison!");
1099  case ISD::SETEQ:
1100  case ISD::SETNE: {
1101  SDValue OpL = GetPromotedInteger(NewLHS);
1102  SDValue OpR = GetPromotedInteger(NewRHS);
1103 
1104  // We would prefer to promote the comparison operand with sign extension.
1105  // If the width of OpL/OpR excluding the duplicated sign bits is no greater
1106  // than the width of NewLHS/NewRH, we can avoid inserting real truncate
1107  // instruction, which is redundant eventually.
1108  unsigned OpLEffectiveBits =
1109  OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
1110  unsigned OpREffectiveBits =
1111  OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
1112  if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() &&
1113  OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()) {
1114  NewLHS = OpL;
1115  NewRHS = OpR;
1116  } else {
1117  NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1118  NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1119  }
1120  break;
1121  }
1122  case ISD::SETUGE:
1123  case ISD::SETUGT:
1124  case ISD::SETULE:
1125  case ISD::SETULT:
1126  NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1127  NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1128  break;
1129  case ISD::SETGE:
1130  case ISD::SETGT:
1131  case ISD::SETLT:
1132  case ISD::SETLE:
1133  NewLHS = SExtPromotedInteger(NewLHS);
1134  NewRHS = SExtPromotedInteger(NewRHS);
1135  break;
1136  }
1137 }
1138 
1139 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1140  SDValue Op = GetPromotedInteger(N->getOperand(0));
1141  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1142 }
1143 
1144 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1145  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1146  return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1147  N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1148 }
1149 
1150 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1151  // This should only occur in unusual situations like bitcasting to an
1152  // x86_fp80, so just turn it into a store+load
1153  return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1154 }
1155 
1156 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1157  assert(OpNo == 2 && "Don't know how to promote this operand!");
1158 
1159  SDValue LHS = N->getOperand(2);
1160  SDValue RHS = N->getOperand(3);
1161  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1162 
1163  // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1164  // legal types.
1165  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1166  N->getOperand(1), LHS, RHS, N->getOperand(4)),
1167  0);
1168 }
1169 
1170 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1171  assert(OpNo == 1 && "only know how to promote condition");
1172 
1173  // Promote all the way up to the canonical SetCC type.
1174  SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1175 
1176  // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1177  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1178  N->getOperand(2)), 0);
1179 }
1180 
1181 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1182  // Since the result type is legal, the operands must promote to it.
1183  EVT OVT = N->getOperand(0).getValueType();
1184  SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1185  SDValue Hi = GetPromotedInteger(N->getOperand(1));
1186  assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1187  SDLoc dl(N);
1188 
1189  Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1190  DAG.getConstant(OVT.getSizeInBits(), dl,
1191  TLI.getPointerTy(DAG.getDataLayout())));
1192  return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1193 }
1194 
1195 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1196  // The vector type is legal but the element type is not. This implies
1197  // that the vector is a power-of-two in length and that the element
1198  // type does not have a strange size (eg: it is not i1).
1199  EVT VecVT = N->getValueType(0);
1200  unsigned NumElts = VecVT.getVectorNumElements();
1201  assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1202  "Legal vector of one illegal element?");
1203 
1204  // Promote the inserted value. The type does not need to match the
1205  // vector element type. Check that any extra bits introduced will be
1206  // truncated away.
1209  "Type of inserted value narrower than vector element type!");
1210 
1211  SmallVector<SDValue, 16> NewOps;
1212  for (unsigned i = 0; i < NumElts; ++i)
1213  NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1214 
1215  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1216 }
1217 
1218 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1219  unsigned OpNo) {
1220  if (OpNo == 1) {
1221  // Promote the inserted value. This is valid because the type does not
1222  // have to match the vector element type.
1223 
1224  // Check that any extra bits introduced will be truncated away.
1227  "Type of inserted value narrower than vector element type!");
1228  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1229  GetPromotedInteger(N->getOperand(1)),
1230  N->getOperand(2)),
1231  0);
1232  }
1233 
1234  assert(OpNo == 2 && "Different operand and result vector types?");
1235 
1236  // Promote the index.
1237  SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1238  TLI.getVectorIdxTy(DAG.getDataLayout()));
1239  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1240  N->getOperand(1), Idx), 0);
1241 }
1242 
1243 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1244  // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1245  // the operand in place.
1246  return SDValue(DAG.UpdateNodeOperands(N,
1247  GetPromotedInteger(N->getOperand(0))), 0);
1248 }
1249 
1250 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1251  assert(OpNo == 0 && "Only know how to promote the condition!");
1252  SDValue Cond = N->getOperand(0);
1253  EVT OpTy = N->getOperand(1).getValueType();
1254 
1255  if (N->getOpcode() == ISD::VSELECT)
1256  if (SDValue Res = WidenVSELECTAndMask(N))
1257  return Res;
1258 
1259  // Promote all the way up to the canonical SetCC type.
1260  EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1261  Cond = PromoteTargetBoolean(Cond, OpVT);
1262 
1263  return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1264  N->getOperand(2)), 0);
1265 }
1266 
1267 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1268  assert(OpNo == 0 && "Don't know how to promote this operand!");
1269 
1270  SDValue LHS = N->getOperand(0);
1271  SDValue RHS = N->getOperand(1);
1272  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1273 
1274  // The CC (#4) and the possible return values (#2 and #3) have legal types.
1275  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1276  N->getOperand(3), N->getOperand(4)), 0);
1277 }
1278 
1279 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1280  assert(OpNo == 0 && "Don't know how to promote this operand!");
1281 
1282  SDValue LHS = N->getOperand(0);
1283  SDValue RHS = N->getOperand(1);
1284  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1285 
1286  // The CC (#2) is always legal.
1287  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1288 }
1289 
1290 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1291  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1292  ZExtPromotedInteger(N->getOperand(1))), 0);
1293 }
1294 
1295 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1296  SDValue Op = GetPromotedInteger(N->getOperand(0));
1297  SDLoc dl(N);
1298  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1299  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1300  Op, DAG.getValueType(N->getOperand(0).getValueType()));
1301 }
1302 
1303 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1304  return SDValue(DAG.UpdateNodeOperands(N,
1305  SExtPromotedInteger(N->getOperand(0))), 0);
1306 }
1307 
1308 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1309  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1310  SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1311  SDLoc dl(N);
1312 
1313  SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1314 
1315  // Truncate the value and store the result.
1316  return DAG.getTruncStore(Ch, dl, Val, Ptr,
1317  N->getMemoryVT(), N->getMemOperand());
1318 }
1319 
1320 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1321  unsigned OpNo) {
1322 
1323  SDValue DataOp = N->getValue();
1324  EVT DataVT = DataOp.getValueType();
1325  SDValue Mask = N->getMask();
1326  SDLoc dl(N);
1327 
1328  bool TruncateStore = false;
1329  if (OpNo == 3) {
1330  Mask = PromoteTargetBoolean(Mask, DataVT);
1331  // Update in place.
1332  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1333  NewOps[3] = Mask;
1334  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1335  } else { // Data operand
1336  assert(OpNo == 1 && "Unexpected operand for promotion");
1337  DataOp = GetPromotedInteger(DataOp);
1338  TruncateStore = true;
1339  }
1340 
1341  return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask,
1342  N->getMemoryVT(), N->getMemOperand(),
1343  TruncateStore, N->isCompressingStore());
1344 }
1345 
1346 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1347  unsigned OpNo) {
1348  assert(OpNo == 2 && "Only know how to promote the mask!");
1349  EVT DataVT = N->getValueType(0);
1350  SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1351  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1352  NewOps[OpNo] = Mask;
1353  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1354 }
1355 
1356 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1357  unsigned OpNo) {
1358 
1359  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1360  if (OpNo == 2) {
1361  // The Mask
1362  EVT DataVT = N->getValueType(0);
1363  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1364  } else if (OpNo == 4) {
1365  // Need to sign extend the index since the bits will likely be used.
1366  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1367  } else
1368  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1369 
1370  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1371 }
1372 
1373 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1374  unsigned OpNo) {
1375  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1376  if (OpNo == 2) {
1377  // The Mask
1378  EVT DataVT = N->getValue().getValueType();
1379  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1380  } else if (OpNo == 4) {
1381  // Need to sign extend the index since the bits will likely be used.
1382  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1383  } else
1384  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1385  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1386 }
1387 
1388 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
1389  SDValue Op = GetPromotedInteger(N->getOperand(0));
1390  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
1391 }
1392 
1393 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
1394  return SDValue(DAG.UpdateNodeOperands(N,
1395  ZExtPromotedInteger(N->getOperand(0))), 0);
1396 }
1397 
1398 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
1399  SDLoc dl(N);
1400  SDValue Op = GetPromotedInteger(N->getOperand(0));
1401  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1402  return DAG.getZeroExtendInReg(Op, dl,
1404 }
1405 
1406 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
1407  assert(OpNo == 2 && "Don't know how to promote this operand!");
1408 
1409  SDValue LHS = N->getOperand(0);
1410  SDValue RHS = N->getOperand(1);
1411  SDValue Carry = N->getOperand(2);
1412  SDLoc DL(N);
1413 
1414  auto VT = getSetCCResultType(LHS.getValueType());
1416  switch (BoolType) {
1418  Carry = DAG.getAnyExtOrTrunc(Carry, DL, VT);
1419  break;
1421  Carry = DAG.getZExtOrTrunc(Carry, DL, VT);
1422  break;
1424  Carry = DAG.getSExtOrTrunc(Carry, DL, VT);
1425  break;
1426  }
1427 
1428  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
1429 }
1430 
1431 SDValue DAGTypeLegalizer::PromoteIntOp_SMULFIX(SDNode *N) {
1432  SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1433  return SDValue(
1434  DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
1435 }
1436 
1437 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
1438  // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
1439  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
1440  return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
1441 }
1442 
1443 SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) {
1444  assert(OpNo > 1 && "Don't know how to promote this operand!");
1445  // Promote the rw, locality, and cache type arguments to a supported integer
1446  // width.
1447  SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1448  SDValue Op3 = ZExtPromotedInteger(N->getOperand(3));
1449  SDValue Op4 = ZExtPromotedInteger(N->getOperand(4));
1450  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
1451  Op2, Op3, Op4),
1452  0);
1453 }
1454 
1455 //===----------------------------------------------------------------------===//
1456 // Integer Result Expansion
1457 //===----------------------------------------------------------------------===//
1458 
1459 /// ExpandIntegerResult - This method is called when the specified result of the
1460 /// specified node is found to need expansion. At this point, the node may also
1461 /// have invalid operands or may have other results that need promotion, we just
1462 /// know that (at least) one result needs expansion.
1463 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
1464  LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG);
1465  dbgs() << "\n");
1466  SDValue Lo, Hi;
1467  Lo = Hi = SDValue();
1468 
1469  // See if the target wants to custom expand this node.
1470  if (CustomLowerNode(N, N->getValueType(ResNo), true))
1471  return;
1472 
1473  switch (N->getOpcode()) {
1474  default:
1475 #ifndef NDEBUG
1476  dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
1477  N->dump(&DAG); dbgs() << "\n";
1478 #endif
1479  llvm_unreachable("Do not know how to expand the result of this operator!");
1480 
1481  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1482  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
1483  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1484  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1485 
1486  case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1487  case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1488  case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1489  case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1490  case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1491 
1492  case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
1493  case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
1494  case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
1495  case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
1496  case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
1497  case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
1498  case ISD::CTLZ_ZERO_UNDEF:
1499  case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
1500  case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
1501  case ISD::CTTZ_ZERO_UNDEF:
1502  case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
1503  case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
1504  case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
1505  case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
1506  case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
1507  case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
1508  case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
1509  case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
1510  case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
1511  case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
1512  case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
1513  case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
1514  case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
1515  case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
1516  case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
1517  case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
1518 
1519  case ISD::ATOMIC_LOAD_ADD:
1520  case ISD::ATOMIC_LOAD_SUB:
1521  case ISD::ATOMIC_LOAD_AND:
1522  case ISD::ATOMIC_LOAD_CLR:
1523  case ISD::ATOMIC_LOAD_OR:
1524  case ISD::ATOMIC_LOAD_XOR:
1525  case ISD::ATOMIC_LOAD_NAND:
1526  case ISD::ATOMIC_LOAD_MIN:
1527  case ISD::ATOMIC_LOAD_MAX:
1528  case ISD::ATOMIC_LOAD_UMIN:
1529  case ISD::ATOMIC_LOAD_UMAX:
1530  case ISD::ATOMIC_SWAP:
1531  case ISD::ATOMIC_CMP_SWAP: {
1532  std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
1533  SplitInteger(Tmp.first, Lo, Hi);
1534  ReplaceValueWith(SDValue(N, 1), Tmp.second);
1535  break;
1536  }
1538  AtomicSDNode *AN = cast<AtomicSDNode>(N);
1539  SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
1540  SDValue Tmp = DAG.getAtomicCmpSwap(
1541  ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
1542  N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
1543  AN->getMemOperand());
1544 
1545  // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
1546  // success simply by comparing the loaded value against the ingoing
1547  // comparison.
1548  SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
1549  N->getOperand(2), ISD::SETEQ);
1550 
1551  SplitInteger(Tmp, Lo, Hi);
1552  ReplaceValueWith(SDValue(N, 1), Success);
1553  ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
1554  break;
1555  }
1556 
1557  case ISD::AND:
1558  case ISD::OR:
1559  case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
1560 
1561  case ISD::UMAX:
1562  case ISD::SMAX:
1563  case ISD::UMIN:
1564  case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
1565 
1566  case ISD::ADD:
1567  case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
1568 
1569  case ISD::ADDC:
1570  case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
1571 
1572  case ISD::ADDE:
1573  case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
1574 
1575  case ISD::ADDCARRY:
1576  case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
1577 
1578  case ISD::SHL:
1579  case ISD::SRA:
1580  case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
1581 
1582  case ISD::SADDO:
1583  case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
1584  case ISD::UADDO:
1585  case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
1586  case ISD::UMULO:
1587  case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
1588 
1589  case ISD::SADDSAT:
1590  case ISD::UADDSAT:
1591  case ISD::SSUBSAT:
1592  case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
1593  case ISD::SMULFIX: ExpandIntRes_SMULFIX(N, Lo, Hi); break;
1594  }
1595 
1596  // If Lo/Hi is null, the sub-method took care of registering results etc.
1597  if (Lo.getNode())
1598  SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
1599 }
1600 
1601 /// Lower an atomic node to the appropriate builtin call.
1602 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
1603  unsigned Opc = Node->getOpcode();
1604  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
1605  RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
1606  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
1607 
1608  return ExpandChainLibCall(LC, Node, false);
1609 }
1610 
1611 /// N is a shift by a value that needs to be expanded,
1612 /// and the shift amount is a constant 'Amt'. Expand the operation.
1613 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
1614  SDValue &Lo, SDValue &Hi) {
1615  SDLoc DL(N);
1616  // Expand the incoming operand to be shifted, so that we have its parts
1617  SDValue InL, InH;
1618  GetExpandedInteger(N->getOperand(0), InL, InH);
1619 
1620  // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
1621  // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
1622  if (!Amt) {
1623  Lo = InL;
1624  Hi = InH;
1625  return;
1626  }
1627 
1628  EVT NVT = InL.getValueType();
1629  unsigned VTBits = N->getValueType(0).getSizeInBits();
1630  unsigned NVTBits = NVT.getSizeInBits();
1631  EVT ShTy = N->getOperand(1).getValueType();
1632 
1633  if (N->getOpcode() == ISD::SHL) {
1634  if (Amt.ugt(VTBits)) {
1635  Lo = Hi = DAG.getConstant(0, DL, NVT);
1636  } else if (Amt.ugt(NVTBits)) {
1637  Lo = DAG.getConstant(0, DL, NVT);
1638  Hi = DAG.getNode(ISD::SHL, DL,
1639  NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1640  } else if (Amt == NVTBits) {
1641  Lo = DAG.getConstant(0, DL, NVT);
1642  Hi = InL;
1643  } else {
1644  Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
1645  Hi = DAG.getNode(ISD::OR, DL, NVT,
1646  DAG.getNode(ISD::SHL, DL, NVT, InH,
1647  DAG.getConstant(Amt, DL, ShTy)),
1648  DAG.getNode(ISD::SRL, DL, NVT, InL,
1649  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1650  }
1651  return;
1652  }
1653 
1654  if (N->getOpcode() == ISD::SRL) {
1655  if (Amt.ugt(VTBits)) {
1656  Lo = Hi = DAG.getConstant(0, DL, NVT);
1657  } else if (Amt.ugt(NVTBits)) {
1658  Lo = DAG.getNode(ISD::SRL, DL,
1659  NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1660  Hi = DAG.getConstant(0, DL, NVT);
1661  } else if (Amt == NVTBits) {
1662  Lo = InH;
1663  Hi = DAG.getConstant(0, DL, NVT);
1664  } else {
1665  Lo = DAG.getNode(ISD::OR, DL, NVT,
1666  DAG.getNode(ISD::SRL, DL, NVT, InL,
1667  DAG.getConstant(Amt, DL, ShTy)),
1668  DAG.getNode(ISD::SHL, DL, NVT, InH,
1669  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1670  Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1671  }
1672  return;
1673  }
1674 
1675  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
1676  if (Amt.ugt(VTBits)) {
1677  Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1678  DAG.getConstant(NVTBits - 1, DL, ShTy));
1679  } else if (Amt.ugt(NVTBits)) {
1680  Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1681  DAG.getConstant(Amt - NVTBits, DL, ShTy));
1682  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1683  DAG.getConstant(NVTBits - 1, DL, ShTy));
1684  } else if (Amt == NVTBits) {
1685  Lo = InH;
1686  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1687  DAG.getConstant(NVTBits - 1, DL, ShTy));
1688  } else {
1689  Lo = DAG.getNode(ISD::OR, DL, NVT,
1690  DAG.getNode(ISD::SRL, DL, NVT, InL,
1691  DAG.getConstant(Amt, DL, ShTy)),
1692  DAG.getNode(ISD::SHL, DL, NVT, InH,
1693  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1694  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1695  }
1696 }
1697 
1698 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
1699 /// this shift based on knowledge of the high bit of the shift amount. If we
1700 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
1701 /// shift amount.
1702 bool DAGTypeLegalizer::
1703 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1704  SDValue Amt = N->getOperand(1);
1705  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1706  EVT ShTy = Amt.getValueType();
1707  unsigned ShBits = ShTy.getScalarSizeInBits();
1708  unsigned NVTBits = NVT.getScalarSizeInBits();
1709  assert(isPowerOf2_32(NVTBits) &&
1710  "Expanded integer type size not a power of two!");
1711  SDLoc dl(N);
1712 
1713  APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
1714  KnownBits Known;
1715  DAG.computeKnownBits(N->getOperand(1), Known);
1716 
1717  // If we don't know anything about the high bits, exit.
1718  if (((Known.Zero|Known.One) & HighBitMask) == 0)
1719  return false;
1720 
1721  // Get the incoming operand to be shifted.
1722  SDValue InL, InH;
1723  GetExpandedInteger(N->getOperand(0), InL, InH);
1724 
1725  // If we know that any of the high bits of the shift amount are one, then we
1726  // can do this as a couple of simple shifts.
1727  if (Known.One.intersects(HighBitMask)) {
1728  // Mask out the high bit, which we know is set.
1729  Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
1730  DAG.getConstant(~HighBitMask, dl, ShTy));
1731 
1732  switch (N->getOpcode()) {
1733  default: llvm_unreachable("Unknown shift");
1734  case ISD::SHL:
1735  Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
1736  Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
1737  return true;
1738  case ISD::SRL:
1739  Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1740  Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
1741  return true;
1742  case ISD::SRA:
1743  Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
1744  DAG.getConstant(NVTBits - 1, dl, ShTy));
1745  Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
1746  return true;
1747  }
1748  }
1749 
1750  // If we know that all of the high bits of the shift amount are zero, then we
1751  // can do this as a couple of simple shifts.
1752  if (HighBitMask.isSubsetOf(Known.Zero)) {
1753  // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
1754  // shift if x is zero. We can use XOR here because x is known to be smaller
1755  // than 32.
1756  SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
1757  DAG.getConstant(NVTBits - 1, dl, ShTy));
1758 
1759  unsigned Op1, Op2;
1760  switch (N->getOpcode()) {
1761  default: llvm_unreachable("Unknown shift");
1762  case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
1763  case ISD::SRL:
1764  case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
1765  }
1766 
1767  // When shifting right the arithmetic for Lo and Hi is swapped.
1768  if (N->getOpcode() != ISD::SHL)
1769  std::swap(InL, InH);
1770 
1771  // Use a little trick to get the bits that move from Lo to Hi. First
1772  // shift by one bit.
1773  SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
1774  // Then compute the remaining shift with amount-1.
1775  SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
1776 
1777  Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
1778  Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
1779 
1780  if (N->getOpcode() != ISD::SHL)
1781  std::swap(Hi, Lo);
1782  return true;
1783  }
1784 
1785  return false;
1786 }
1787 
1788 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
1789 /// of any size.
1790 bool DAGTypeLegalizer::
1791 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1792  SDValue Amt = N->getOperand(1);
1793  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1794  EVT ShTy = Amt.getValueType();
1795  unsigned NVTBits = NVT.getSizeInBits();
1796  assert(isPowerOf2_32(NVTBits) &&
1797  "Expanded integer type size not a power of two!");
1798  SDLoc dl(N);
1799 
1800  // Get the incoming operand to be shifted.
1801  SDValue InL, InH;
1802  GetExpandedInteger(N->getOperand(0), InL, InH);
1803 
1804  SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
1805  SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
1806  SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
1807  SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1808  Amt, NVBitsNode, ISD::SETULT);
1809  SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1810  Amt, DAG.getConstant(0, dl, ShTy),
1811  ISD::SETEQ);
1812 
1813  SDValue LoS, HiS, LoL, HiL;
1814  switch (N->getOpcode()) {
1815  default: llvm_unreachable("Unknown shift");
1816  case ISD::SHL:
1817  // Short: ShAmt < NVTBits
1818  LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
1819  HiS = DAG.getNode(ISD::OR, dl, NVT,
1820  DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
1821  DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
1822 
1823  // Long: ShAmt >= NVTBits
1824  LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
1825  HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
1826 
1827  Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
1828  Hi = DAG.getSelect(dl, NVT, isZero, InH,
1829  DAG.getSelect(dl, NVT, isShort, HiS, HiL));
1830  return true;
1831  case ISD::SRL:
1832  // Short: ShAmt < NVTBits
1833  HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
1834  LoS = DAG.getNode(ISD::OR, dl, NVT,
1835  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1836  // FIXME: If Amt is zero, the following shift generates an undefined result
1837  // on some architectures.
1838  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1839 
1840  // Long: ShAmt >= NVTBits
1841  HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1842  LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1843 
1844  Lo = DAG.getSelect(dl, NVT, isZero, InL,
1845  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1846  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1847  return true;
1848  case ISD::SRA:
1849  // Short: ShAmt < NVTBits
1850  HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
1851  LoS = DAG.getNode(ISD::OR, dl, NVT,
1852  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1853  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1854 
1855  // Long: ShAmt >= NVTBits
1856  HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
1857  DAG.getConstant(NVTBits - 1, dl, ShTy));
1858  LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1859 
1860  Lo = DAG.getSelect(dl, NVT, isZero, InL,
1861  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1862  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1863  return true;
1864  }
1865 }
1866 
1867 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
1868 
1869  switch (Op) {
1870  default: llvm_unreachable("invalid min/max opcode");
1871  case ISD::SMAX:
1872  return std::make_pair(ISD::SETGT, ISD::UMAX);
1873  case ISD::UMAX:
1874  return std::make_pair(ISD::SETUGT, ISD::UMAX);
1875  case ISD::SMIN:
1876  return std::make_pair(ISD::SETLT, ISD::UMIN);
1877  case ISD::UMIN:
1878  return std::make_pair(ISD::SETULT, ISD::UMIN);
1879  }
1880 }
1881 
1882 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
1883  SDValue &Lo, SDValue &Hi) {
1884  SDLoc DL(N);
1885  ISD::NodeType LoOpc;
1886  ISD::CondCode CondC;
1887  std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
1888 
1889  // Expand the subcomponents.
1890  SDValue LHSL, LHSH, RHSL, RHSH;
1891  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1892  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1893 
1894  // Value types
1895  EVT NVT = LHSL.getValueType();
1896  EVT CCT = getSetCCResultType(NVT);
1897 
1898  // Hi part is always the same op
1899  Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
1900 
1901  // We need to know whether to select Lo part that corresponds to 'winning'
1902  // Hi part or if Hi parts are equal.
1903  SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
1904  SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
1905 
1906  // Lo part corresponding to the 'winning' Hi part
1907  SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
1908 
1909  // Recursed Lo part if Hi parts are equal, this uses unsigned version
1910  SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
1911 
1912  Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
1913 }
1914 
1915 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
1916  SDValue &Lo, SDValue &Hi) {
1917  SDLoc dl(N);
1918  // Expand the subcomponents.
1919  SDValue LHSL, LHSH, RHSL, RHSH;
1920  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1921  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1922 
1923  EVT NVT = LHSL.getValueType();
1924  SDValue LoOps[2] = { LHSL, RHSL };
1925  SDValue HiOps[3] = { LHSH, RHSH };
1926 
1927  bool HasOpCarry = TLI.isOperationLegalOrCustom(
1929  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1930  if (HasOpCarry) {
1931  SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
1932  if (N->getOpcode() == ISD::ADD) {
1933  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
1934  HiOps[2] = Lo.getValue(1);
1935  Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
1936  } else {
1937  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
1938  HiOps[2] = Lo.getValue(1);
1939  Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
1940  }
1941  return;
1942  }
1943 
1944  // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
1945  // them. TODO: Teach operation legalization how to expand unsupported
1946  // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
1947  // a carry of type MVT::Glue, but there doesn't seem to be any way to
1948  // generate a value of this type in the expanded code sequence.
1949  bool hasCarry =
1951  ISD::ADDC : ISD::SUBC,
1952  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1953 
1954  if (hasCarry) {
1955  SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
1956  if (N->getOpcode() == ISD::ADD) {
1957  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
1958  HiOps[2] = Lo.getValue(1);
1959  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
1960  } else {
1961  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
1962  HiOps[2] = Lo.getValue(1);
1963  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
1964  }
1965  return;
1966  }
1967 
1968  bool hasOVF =
1971  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1973 
1974  if (hasOVF) {
1975  EVT OvfVT = getSetCCResultType(NVT);
1976  SDVTList VTList = DAG.getVTList(NVT, OvfVT);
1977  int RevOpc;
1978  if (N->getOpcode() == ISD::ADD) {
1979  RevOpc = ISD::SUB;
1980  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
1981  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
1982  } else {
1983  RevOpc = ISD::ADD;
1984  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
1985  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
1986  }
1987  SDValue OVF = Lo.getValue(1);
1988 
1989  switch (BoolType) {
1991  OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
1994  OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
1995  Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
1996  break;
1998  OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
1999  Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
2000  }
2001  return;
2002  }
2003 
2004  if (N->getOpcode() == ISD::ADD) {
2005  Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
2006  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2007  SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
2008  ISD::SETULT);
2009 
2011  SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
2012  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
2013  return;
2014  }
2015 
2016  SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
2017  DAG.getConstant(1, dl, NVT),
2018  DAG.getConstant(0, dl, NVT));
2019  SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
2020  ISD::SETULT);
2021  SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
2022  DAG.getConstant(1, dl, NVT), Carry1);
2023  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
2024  } else {
2025  Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
2026  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2027  SDValue Cmp =
2028  DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
2029  LoOps[0], LoOps[1], ISD::SETULT);
2030 
2031  SDValue Borrow;
2033  Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
2034  else
2035  Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
2036  DAG.getConstant(0, dl, NVT));
2037 
2038  Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
2039  }
2040 }
2041 
2042 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
2043  SDValue &Lo, SDValue &Hi) {
2044  // Expand the subcomponents.
2045  SDValue LHSL, LHSH, RHSL, RHSH;
2046  SDLoc dl(N);
2047  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2048  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2049  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2050  SDValue LoOps[2] = { LHSL, RHSL };
2051  SDValue HiOps[3] = { LHSH, RHSH };
2052 
2053  if (N->getOpcode() == ISD::ADDC) {
2054  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2055  HiOps[2] = Lo.getValue(1);
2056  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2057  } else {
2058  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2059  HiOps[2] = Lo.getValue(1);
2060  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2061  }
2062 
2063  // Legalized the flag result - switch anything that used the old flag to
2064  // use the new one.
2065  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2066 }
2067 
2068 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
2069  SDValue &Lo, SDValue &Hi) {
2070  // Expand the subcomponents.
2071  SDValue LHSL, LHSH, RHSL, RHSH;
2072  SDLoc dl(N);
2073  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2074  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2075  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2076  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2077  SDValue HiOps[3] = { LHSH, RHSH };
2078 
2079  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2080  HiOps[2] = Lo.getValue(1);
2081  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2082 
2083  // Legalized the flag result - switch anything that used the old flag to
2084  // use the new one.
2085  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2086 }
2087 
2088 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
2089  SDValue &Lo, SDValue &Hi) {
2090  SDValue LHS = N->getOperand(0);
2091  SDValue RHS = N->getOperand(1);
2092  SDLoc dl(N);
2093 
2094  SDValue Ovf;
2095 
2096  bool HasOpCarry = TLI.isOperationLegalOrCustom(
2098  TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
2099 
2100  if (HasOpCarry) {
2101  // Expand the subcomponents.
2102  SDValue LHSL, LHSH, RHSL, RHSH;
2103  GetExpandedInteger(LHS, LHSL, LHSH);
2104  GetExpandedInteger(RHS, RHSL, RHSH);
2105  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2106  SDValue LoOps[2] = { LHSL, RHSL };
2107  SDValue HiOps[3] = { LHSH, RHSH };
2108 
2109  unsigned Opc = N->getOpcode() == ISD::UADDO ? ISD::ADDCARRY : ISD::SUBCARRY;
2110  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2111  HiOps[2] = Lo.getValue(1);
2112  Hi = DAG.getNode(Opc, dl, VTList, HiOps);
2113 
2114  Ovf = Hi.getValue(1);
2115  } else {
2116  // Expand the result by simply replacing it with the equivalent
2117  // non-overflow-checking operation.
2118  auto Opc = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
2119  SDValue Sum = DAG.getNode(Opc, dl, LHS.getValueType(), LHS, RHS);
2120  SplitInteger(Sum, Lo, Hi);
2121 
2122  // Calculate the overflow: addition overflows iff a + b < a, and subtraction
2123  // overflows iff a - b > a.
2124  auto Cond = N->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
2125  Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
2126  }
2127 
2128  // Legalized the flag result - switch anything that used the old flag to
2129  // use the new one.
2130  ReplaceValueWith(SDValue(N, 1), Ovf);
2131 }
2132 
2133 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
2134  SDValue &Lo, SDValue &Hi) {
2135  // Expand the subcomponents.
2136  SDValue LHSL, LHSH, RHSL, RHSH;
2137  SDLoc dl(N);
2138  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2139  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2140  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2141  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2142  SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
2143 
2144  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2145  HiOps[2] = Lo.getValue(1);
2146  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2147 
2148  // Legalized the flag result - switch anything that used the old flag to
2149  // use the new one.
2150  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2151 }
2152 
2153 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
2154  SDValue &Lo, SDValue &Hi) {
2155  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2156  SDLoc dl(N);
2157  SDValue Op = N->getOperand(0);
2158  if (Op.getValueType().bitsLE(NVT)) {
2159  // The low part is any extension of the input (which degenerates to a copy).
2160  Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
2161  Hi = DAG.getUNDEF(NVT); // The high part is undefined.
2162  } else {
2163  // For example, extension of an i48 to an i64. The operand type necessarily
2164  // promotes to the result type, so will end up being expanded too.
2165  assert(getTypeAction(Op.getValueType()) ==
2167  "Only know how to promote this result!");
2168  SDValue Res = GetPromotedInteger(Op);
2169  assert(Res.getValueType() == N->getValueType(0) &&
2170  "Operand over promoted?");
2171  // Split the promoted operand. This will simplify when it is expanded.
2172  SplitInteger(Res, Lo, Hi);
2173  }
2174 }
2175 
2176 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
2177  SDValue &Lo, SDValue &Hi) {
2178  SDLoc dl(N);
2179  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2180  EVT NVT = Lo.getValueType();
2181  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2182  unsigned NVTBits = NVT.getSizeInBits();
2183  unsigned EVTBits = EVT.getSizeInBits();
2184 
2185  if (NVTBits < EVTBits) {
2186  Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
2188  EVTBits - NVTBits)));
2189  } else {
2190  Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
2191  // The high part replicates the sign bit of Lo, make it explicit.
2192  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2193  DAG.getConstant(NVTBits - 1, dl,
2194  TLI.getPointerTy(DAG.getDataLayout())));
2195  }
2196 }
2197 
2198 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
2199  SDValue &Lo, SDValue &Hi) {
2200  SDLoc dl(N);
2201  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2202  EVT NVT = Lo.getValueType();
2203  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2204  unsigned NVTBits = NVT.getSizeInBits();
2205  unsigned EVTBits = EVT.getSizeInBits();
2206 
2207  if (NVTBits < EVTBits) {
2208  Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
2210  EVTBits - NVTBits)));
2211  } else {
2212  Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
2213  // The high part must be zero, make it explicit.
2214  Hi = DAG.getConstant(0, dl, NVT);
2215  }
2216 }
2217 
2218 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
2219  SDValue &Lo, SDValue &Hi) {
2220  SDLoc dl(N);
2221  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2222  Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
2223  Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
2224 }
2225 
2226 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
2227  SDValue &Lo, SDValue &Hi) {
2228  SDLoc dl(N);
2229  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2230  Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
2231  Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
2232 }
2233 
2234 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
2235  SDValue &Lo, SDValue &Hi) {
2236  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2237  unsigned NBitWidth = NVT.getSizeInBits();
2238  auto Constant = cast<ConstantSDNode>(N);
2239  const APInt &Cst = Constant->getAPIntValue();
2240  bool IsTarget = Constant->isTargetOpcode();
2241  bool IsOpaque = Constant->isOpaque();
2242  SDLoc dl(N);
2243  Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
2244  Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
2245  IsOpaque);
2246 }
2247 
2248 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
2249  SDValue &Lo, SDValue &Hi) {
2250  SDLoc dl(N);
2251  // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
2252  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2253  EVT NVT = Lo.getValueType();
2254 
2255  SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
2256  DAG.getConstant(0, dl, NVT), ISD::SETNE);
2257 
2258  SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
2259  SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
2260 
2261  Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
2262  DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
2263  DAG.getConstant(NVT.getSizeInBits(), dl,
2264  NVT)));
2265  Hi = DAG.getConstant(0, dl, NVT);
2266 }
2267 
2268 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
2269  SDValue &Lo, SDValue &Hi) {
2270  SDLoc dl(N);
2271  // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
2272  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2273  EVT NVT = Lo.getValueType();
2274  Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
2275  DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
2276  Hi = DAG.getConstant(0, dl, NVT);
2277 }
2278 
2279 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
2280  SDValue &Lo, SDValue &Hi) {
2281  SDLoc dl(N);
2282  // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
2283  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2284  EVT NVT = Lo.getValueType();
2285 
2286  SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
2287  DAG.getConstant(0, dl, NVT), ISD::SETNE);
2288 
2289  SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
2290  SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
2291 
2292  Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
2293  DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
2294  DAG.getConstant(NVT.getSizeInBits(), dl,
2295  NVT)));
2296  Hi = DAG.getConstant(0, dl, NVT);
2297 }
2298 
2299 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
2300  SDValue &Hi) {
2301  SDLoc dl(N);
2302  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2303  unsigned NBitWidth = NVT.getSizeInBits();
2304 
2305  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2306  Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT);
2307  // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
2308  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2309  DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
2310 }
2311 
2312 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
2313  SDValue &Hi) {
2314  SDLoc dl(N);
2315  EVT VT = N->getValueType(0);
2316 
2317  SDValue Op = N->getOperand(0);
2318  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2319  Op = GetPromotedFloat(Op);
2320 
2322  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
2323  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, true/*irrelevant*/, dl).first,
2324  Lo, Hi);
2325 }
2326 
2327 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
2328  SDValue &Hi) {
2329  SDLoc dl(N);
2330  EVT VT = N->getValueType(0);
2331 
2332  SDValue Op = N->getOperand(0);
2333  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2334  Op = GetPromotedFloat(Op);
2335 
2337  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
2338  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, false/*irrelevant*/, dl).first,
2339  Lo, Hi);
2340 }
2341 
2342 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
2343  SDValue &Lo, SDValue &Hi) {
2344  if (ISD::isNormalLoad(N)) {
2345  ExpandRes_NormalLoad(N, Lo, Hi);
2346  return;
2347  }
2348 
2349  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2350 
2351  EVT VT = N->getValueType(0);
2352  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2353  SDValue Ch = N->getChain();
2354  SDValue Ptr = N->getBasePtr();
2356  unsigned Alignment = N->getAlignment();
2357  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2358  AAMDNodes AAInfo = N->getAAInfo();
2359  SDLoc dl(N);
2360 
2361  assert(NVT.isByteSized() && "Expanded type not byte sized!");
2362 
2363  if (N->getMemoryVT().bitsLE(NVT)) {
2364  EVT MemVT = N->getMemoryVT();
2365 
2366  Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
2367  Alignment, MMOFlags, AAInfo);
2368 
2369  // Remember the chain.
2370  Ch = Lo.getValue(1);
2371 
2372  if (ExtType == ISD::SEXTLOAD) {
2373  // The high part is obtained by SRA'ing all but one of the bits of the
2374  // lo part.
2375  unsigned LoSize = Lo.getValueSizeInBits();
2376  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2377  DAG.getConstant(LoSize - 1, dl,
2378  TLI.getPointerTy(DAG.getDataLayout())));
2379  } else if (ExtType == ISD::ZEXTLOAD) {
2380  // The high part is just a zero.
2381  Hi = DAG.getConstant(0, dl, NVT);
2382  } else {
2383  assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
2384  // The high part is undefined.
2385  Hi = DAG.getUNDEF(NVT);
2386  }
2387  } else if (DAG.getDataLayout().isLittleEndian()) {
2388  // Little-endian - low bits are at low addresses.
2389  Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2390  AAInfo);
2391 
2392  unsigned ExcessBits =
2393  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
2394  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
2395 
2396  // Increment the pointer to the other half.
2397  unsigned IncrementSize = NVT.getSizeInBits()/8;
2398  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2399  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2400  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
2401  N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
2402  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2403 
2404  // Build a factor node to remember that this load is independent of the
2405  // other one.
2406  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2407  Hi.getValue(1));
2408  } else {
2409  // Big-endian - high bits are at low addresses. Favor aligned loads at
2410  // the cost of some bit-fiddling.
2411  EVT MemVT = N->getMemoryVT();
2412  unsigned EBytes = MemVT.getStoreSize();
2413  unsigned IncrementSize = NVT.getSizeInBits()/8;
2414  unsigned ExcessBits = (EBytes - IncrementSize)*8;
2415 
2416  // Load both the high bits and maybe some of the low bits.
2417  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
2419  MemVT.getSizeInBits() - ExcessBits),
2420  Alignment, MMOFlags, AAInfo);
2421 
2422  // Increment the pointer to the other half.
2423  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2424  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2425  // Load the rest of the low bits.
2426  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
2427  N->getPointerInfo().getWithOffset(IncrementSize),
2428  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
2429  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2430 
2431  // Build a factor node to remember that this load is independent of the
2432  // other one.
2433  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2434  Hi.getValue(1));
2435 
2436  if (ExcessBits < NVT.getSizeInBits()) {
2437  // Transfer low bits from the bottom of Hi to the top of Lo.
2438  Lo = DAG.getNode(
2439  ISD::OR, dl, NVT, Lo,
2440  DAG.getNode(ISD::SHL, dl, NVT, Hi,
2441  DAG.getConstant(ExcessBits, dl,
2442  TLI.getPointerTy(DAG.getDataLayout()))));
2443  // Move high bits to the right position in Hi.
2444  Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
2445  Hi,
2446  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
2447  TLI.getPointerTy(DAG.getDataLayout())));
2448  }
2449  }
2450 
2451  // Legalize the chain result - switch anything that used the old chain to
2452  // use the new one.
2453  ReplaceValueWith(SDValue(N, 1), Ch);
2454 }
2455 
2456 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
2457  SDValue &Lo, SDValue &Hi) {
2458  SDLoc dl(N);
2459  SDValue LL, LH, RL, RH;
2460  GetExpandedInteger(N->getOperand(0), LL, LH);
2461  GetExpandedInteger(N->getOperand(1), RL, RH);
2462  Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
2463  Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
2464 }
2465 
2466 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
2467  SDValue &Lo, SDValue &Hi) {
2468  EVT VT = N->getValueType(0);
2469  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2470  SDLoc dl(N);
2471 
2472  SDValue LL, LH, RL, RH;
2473  GetExpandedInteger(N->getOperand(0), LL, LH);
2474  GetExpandedInteger(N->getOperand(1), RL, RH);
2475 
2476  if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
2478  LL, LH, RL, RH))
2479  return;
2480 
2481  // If nothing else, we can make a libcall.
2482  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2483  if (VT == MVT::i16)
2484  LC = RTLIB::MUL_I16;
2485  else if (VT == MVT::i32)
2486  LC = RTLIB::MUL_I32;
2487  else if (VT == MVT::i64)
2488  LC = RTLIB::MUL_I64;
2489  else if (VT == MVT::i128)
2490  LC = RTLIB::MUL_I128;
2491 
2492  if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
2493  // We'll expand the multiplication by brute force because we have no other
2494  // options. This is a trivially-generalized version of the code from
2495  // Hacker's Delight (itself derived from Knuth's Algorithm M from section
2496  // 4.3.1).
2497  unsigned Bits = NVT.getSizeInBits();
2498  unsigned HalfBits = Bits >> 1;
2499  SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
2500  NVT);
2501  SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
2502  SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
2503 
2504  SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
2505  SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
2506 
2507  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2508  if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
2509  // The type from TLI is too small to fit the shift amount we want.
2510  // Override it with i32. The shift will have to be legalized.
2511  ShiftAmtTy = MVT::i32;
2512  }
2513  SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
2514  SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
2515  SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
2516  SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
2517 
2518  SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
2519  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
2520  SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
2521  SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
2522 
2523  SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
2524  DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
2525  SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
2526 
2527  SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
2528  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
2529  DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
2530  Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
2531  DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
2532 
2533  Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
2534  DAG.getNode(ISD::ADD, dl, NVT,
2535  DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
2536  DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
2537  return;
2538  }
2539 
2540  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2541  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true/*irrelevant*/, dl).first,
2542  Lo, Hi);
2543 }
2544 
2545 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
2546  SDValue &Hi) {
2547  SDLoc DL(N);
2548  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2549  SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
2550  SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
2551  Lo = R.getValue(0);
2552  Hi = R.getValue(1);
2553  ReplaceValueWith(SDValue(N, 1), R.getValue(2));
2554 }
2555 
2556 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
2557  SDValue &Hi) {
2559  SplitInteger(Result, Lo, Hi);
2560 }
2561 
2562 void DAGTypeLegalizer::ExpandIntRes_SMULFIX(SDNode *N, SDValue &Lo,
2563  SDValue &Hi) {
2564  SDLoc dl(N);
2565  EVT VT = N->getValueType(0);
2566  SDValue LHS = N->getOperand(0);
2567  SDValue RHS = N->getOperand(1);
2568  uint64_t Scale = N->getConstantOperandVal(2);
2569  if (!Scale) {
2570  SDValue Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
2571  SplitInteger(Result, Lo, Hi);
2572  return;
2573  }
2574 
2575  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2576  SDValue LL, LH, RL, RH;
2577  GetExpandedInteger(LHS, LL, LH);
2578  GetExpandedInteger(RHS, RL, RH);
2579  SmallVector<SDValue, 4> Result;
2580 
2581  if (!TLI.expandMUL_LOHI(ISD::SMUL_LOHI, VT, dl, LHS, RHS, Result, NVT, DAG,
2583  LL, LH, RL, RH)) {
2584  report_fatal_error("Unable to expand SMUL_FIX using SMUL_LOHI.");
2585  return;
2586  }
2587 
2588  unsigned VTSize = VT.getScalarSizeInBits();
2589  unsigned NVTSize = NVT.getScalarSizeInBits();
2590  EVT ShiftTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2591 
2592  // Shift whole amount by scale.
2593  SDValue ResultLL = Result[0];
2594  SDValue ResultLH = Result[1];
2595  SDValue ResultHL = Result[2];
2596  SDValue ResultHH = Result[3];
2597 
2598  // After getting the multplication result in 4 parts, we need to perform a
2599  // shift right by the amount of the scale to get the result in that scale.
2600  // Let's say we multiply 2 64 bit numbers. The resulting value can be held in
2601  // 128 bits that are cut into 4 32-bit parts:
2602  //
2603  // HH HL LH LL
2604  // |---32---|---32---|---32---|---32---|
2605  // 128 96 64 32 0
2606  //
2607  // |------VTSize-----|
2608  //
2609  // |NVTSize-|
2610  //
2611  // The resulting Lo and Hi will only need to be one of these 32-bit parts
2612  // after shifting.
2613  if (Scale < NVTSize) {
2614  // If the scale is less than the size of the VT we expand to, the Hi and
2615  // Lo of the result will be in the first 2 parts of the result after
2616  // shifting right. This only requires shifting by the scale as far as the
2617  // third part in the result (ResultHL).
2618  SDValue SRLAmnt = DAG.getConstant(Scale, dl, ShiftTy);
2619  SDValue SHLAmnt = DAG.getConstant(NVTSize - Scale, dl, ShiftTy);
2620  Lo = DAG.getNode(ISD::SRL, dl, NVT, ResultLL, SRLAmnt);
2621  Lo = DAG.getNode(ISD::OR, dl, NVT, Lo,
2622  DAG.getNode(ISD::SHL, dl, NVT, ResultLH, SHLAmnt));
2623  Hi = DAG.getNode(ISD::SRL, dl, NVT, ResultLH, SRLAmnt);
2624  Hi = DAG.getNode(ISD::OR, dl, NVT, Hi,
2625  DAG.getNode(ISD::SHL, dl, NVT, ResultHL, SHLAmnt));
2626  } else if (Scale == NVTSize) {
2627  // If the scales are equal, Lo and Hi are ResultLH and Result HL,
2628  // respectively. Avoid shifting to prevent undefined behavior.
2629  Lo = ResultLH;
2630  Hi = ResultHL;
2631  } else if (Scale < VTSize) {
2632  // If the scale is instead less than the old VT size, but greater than or
2633  // equal to the expanded VT size, the first part of the result (ResultLL) is
2634  // no longer a part of Lo because it would be scaled out anyway. Instead we
2635  // can start shifting right from the fourth part (ResultHH) to the second
2636  // part (ResultLH), and Result LH will be the new Lo.
2637  SDValue SRLAmnt = DAG.getConstant(Scale - NVTSize, dl, ShiftTy);
2638  SDValue SHLAmnt = DAG.getConstant(VTSize - Scale, dl, ShiftTy);
2639  Lo = DAG.getNode(ISD::SRL, dl, NVT, ResultLH, SRLAmnt);
2640  Lo = DAG.getNode(ISD::OR, dl, NVT, Lo,
2641  DAG.getNode(ISD::SHL, dl, NVT, ResultHL, SHLAmnt));
2642  Hi = DAG.getNode(ISD::SRL, dl, NVT, ResultHL, SRLAmnt);
2643  Hi = DAG.getNode(ISD::OR, dl, NVT, Hi,
2644  DAG.getNode(ISD::SHL, dl, NVT, ResultHH, SHLAmnt));
2645  } else {
2647  "Expected the scale to be less than the width of the operands");
2648  }
2649 }
2650 
2651 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
2652  SDValue &Lo, SDValue &Hi) {
2653  SDValue LHS = Node->getOperand(0);
2654  SDValue RHS = Node->getOperand(1);
2655  SDLoc dl(Node);
2656 
2657  // Expand the result by simply replacing it with the equivalent
2658  // non-overflow-checking operation.
2659  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
2660  ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
2661  LHS, RHS);
2662  SplitInteger(Sum, Lo, Hi);
2663 
2664  // Compute the overflow.
2665  //
2666  // LHSSign -> LHS >= 0
2667  // RHSSign -> RHS >= 0
2668  // SumSign -> Sum >= 0
2669  //
2670  // Add:
2671  // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
2672  // Sub:
2673  // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
2674  //
2675  EVT OType = Node->getValueType(1);
2676  SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
2677 
2678  SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
2679  SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
2680  SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
2681  Node->getOpcode() == ISD::SADDO ?
2683 
2684  SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
2685  SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
2686 
2687  SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
2688 
2689  // Use the calculated overflow everywhere.
2690  ReplaceValueWith(SDValue(Node, 1), Cmp);
2691 }
2692 
2693 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
2694  SDValue &Lo, SDValue &Hi) {
2695  EVT VT = N->getValueType(0);
2696  SDLoc dl(N);
2697  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2698 
2700  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2701  SplitInteger(Res.getValue(0), Lo, Hi);
2702  return;
2703  }
2704 
2705  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2706  if (VT == MVT::i16)
2707  LC = RTLIB::SDIV_I16;
2708  else if (VT == MVT::i32)
2709  LC = RTLIB::SDIV_I32;
2710  else if (VT == MVT::i64)
2711  LC = RTLIB::SDIV_I64;
2712  else if (VT == MVT::i128)
2713  LC = RTLIB::SDIV_I128;
2714  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
2715 
2716  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
2717 }
2718 
2719 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
2720  SDValue &Lo, SDValue &Hi) {
2721  EVT VT = N->getValueType(0);
2722  SDLoc dl(N);
2723 
2724  // If we can emit an efficient shift operation, do so now. Check to see if
2725  // the RHS is a constant.
2726  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
2727  return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
2728 
2729  // If we can determine that the high bit of the shift is zero or one, even if
2730  // the low bits are variable, emit this shift in an optimized form.
2731  if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
2732  return;
2733 
2734  // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
2735  unsigned PartsOpc;
2736  if (N->getOpcode() == ISD::SHL) {
2737  PartsOpc = ISD::SHL_PARTS;
2738  } else if (N->getOpcode() == ISD::SRL) {
2739  PartsOpc = ISD::SRL_PARTS;
2740  } else {
2741  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2742  PartsOpc = ISD::SRA_PARTS;
2743  }
2744 
2745  // Next check to see if the target supports this SHL_PARTS operation or if it
2746  // will custom expand it.
2747  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2748  TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
2749  if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
2750  Action == TargetLowering::Custom) {
2751  // Expand the subcomponents.
2752  SDValue LHSL, LHSH;
2753  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2754  EVT VT = LHSL.getValueType();
2755 
2756  // If the shift amount operand is coming from a vector legalization it may
2757  // have an illegal type. Fix that first by casting the operand, otherwise
2758  // the new SHL_PARTS operation would need further legalization.
2759  SDValue ShiftOp = N->getOperand(1);
2760  EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2761  assert(ShiftTy.getScalarSizeInBits() >=
2762  Log2_32_Ceil(VT.getScalarSizeInBits()) &&
2763  "ShiftAmountTy is too small to cover the range of this type!");
2764  if (ShiftOp.getValueType() != ShiftTy)
2765  ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
2766 
2767  SDValue Ops[] = { LHSL, LHSH, ShiftOp };
2768  Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
2769  Hi = Lo.getValue(1);
2770  return;
2771  }
2772 
2773  // Otherwise, emit a libcall.
2774  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2775  bool isSigned;
2776  if (N->getOpcode() == ISD::SHL) {
2777  isSigned = false; /*sign irrelevant*/
2778  if (VT == MVT::i16)
2779  LC = RTLIB::SHL_I16;
2780  else if (VT == MVT::i32)
2781  LC = RTLIB::SHL_I32;
2782  else if (VT == MVT::i64)
2783  LC = RTLIB::SHL_I64;
2784  else if (VT == MVT::i128)
2785  LC = RTLIB::SHL_I128;
2786  } else if (N->getOpcode() == ISD::SRL) {
2787  isSigned = false;
2788  if (VT == MVT::i16)
2789  LC = RTLIB::SRL_I16;
2790  else if (VT == MVT::i32)
2791  LC = RTLIB::SRL_I32;
2792  else if (VT == MVT::i64)
2793  LC = RTLIB::SRL_I64;
2794  else if (VT == MVT::i128)
2795  LC = RTLIB::SRL_I128;
2796  } else {
2797  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2798  isSigned = true;
2799  if (VT == MVT::i16)
2800  LC = RTLIB::SRA_I16;
2801  else if (VT == MVT::i32)
2802  LC = RTLIB::SRA_I32;
2803  else if (VT == MVT::i64)
2804  LC = RTLIB::SRA_I64;
2805  else if (VT == MVT::i128)
2806  LC = RTLIB::SRA_I128;
2807  }
2808 
2809  if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
2810  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2811  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, isSigned, dl).first, Lo, Hi);
2812  return;
2813  }
2814 
2815  if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
2816  llvm_unreachable("Unsupported shift!");
2817 }
2818 
2819 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
2820  SDValue &Lo, SDValue &Hi) {
2821  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2822  SDLoc dl(N);
2823  SDValue Op = N->getOperand(0);
2824  if (Op.getValueType().bitsLE(NVT)) {
2825  // The low part is sign extension of the input (degenerates to a copy).
2826  Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
2827  // The high part is obtained by SRA'ing all but one of the bits of low part.
2828  unsigned LoSize = NVT.getSizeInBits();
2829  Hi = DAG.getNode(
2830  ISD::SRA, dl, NVT, Lo,
2831  DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
2832  } else {
2833  // For example, extension of an i48 to an i64. The operand type necessarily
2834  // promotes to the result type, so will end up being expanded too.
2835  assert(getTypeAction(Op.getValueType()) ==
2837  "Only know how to promote this result!");
2838  SDValue Res = GetPromotedInteger(Op);
2839  assert(Res.getValueType() == N->getValueType(0) &&
2840  "Operand over promoted?");
2841  // Split the promoted operand. This will simplify when it is expanded.
2842  SplitInteger(Res, Lo, Hi);
2843  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
2844  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2846  ExcessBits)));
2847  }
2848 }
2849 
2850 void DAGTypeLegalizer::
2851 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2852  SDLoc dl(N);
2853  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2854  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2855 
2856  if (EVT.bitsLE(Lo.getValueType())) {
2857  // sext_inreg the low part if needed.
2858  Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
2859  N->getOperand(1));
2860 
2861  // The high part gets the sign extension from the lo-part. This handles
2862  // things like sextinreg V:i64 from i8.
2863  Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
2864  DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
2865  TLI.getPointerTy(DAG.getDataLayout())));
2866  } else {
2867  // For example, extension of an i48 to an i64. Leave the low part alone,
2868  // sext_inreg the high part.
2869  unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
2870  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2872  ExcessBits)));
2873  }
2874 }
2875 
2876 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
2877  SDValue &Lo, SDValue &Hi) {
2878  EVT VT = N->getValueType(0);
2879  SDLoc dl(N);
2880  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2881 
2883  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2884  SplitInteger(Res.getValue(1), Lo, Hi);
2885  return;
2886  }
2887 
2888  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2889  if (VT == MVT::i16)
2890  LC = RTLIB::SREM_I16;
2891  else if (VT == MVT::i32)
2892  LC = RTLIB::SREM_I32;
2893  else if (VT == MVT::i64)
2894  LC = RTLIB::SREM_I64;
2895  else if (VT == MVT::i128)
2896  LC = RTLIB::SREM_I128;
2897  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
2898 
2899  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
2900 }
2901 
2902 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
2903  SDValue &Lo, SDValue &Hi) {
2904  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2905  SDLoc dl(N);
2906  Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
2907  Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
2908  N->getOperand(0),
2909  DAG.getConstant(NVT.getSizeInBits(), dl,
2910  TLI.getPointerTy(DAG.getDataLayout())));
2911  Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
2912 }
2913 
2914 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
2915  SDValue &Lo, SDValue &Hi) {
2916  EVT VT = N->getValueType(0);
2917  SDLoc dl(N);
2918 
2919  if (N->getOpcode() == ISD::UMULO) {
2920  // This section expands the operation into the following sequence of
2921  // instructions. `iNh` here refers to a type which has half the bit width of
2922  // the type the original operation operated on.
2923  //
2924  // %0 = %LHS.HI != 0 && %RHS.HI != 0
2925  // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
2926  // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
2927  // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
2928  // %4 = add iN (%1.0 as iN) << Nh, (%2.0 as iN) << Nh
2929  // %5 = { iN, i1 } @uadd.with.overflow.iN( %4, %3 )
2930  //
2931  // %res = { %5.0, %0 || %1.1 || %2.1 || %5.1 }
2932  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
2933  SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
2934  SplitInteger(LHS, LHSLow, LHSHigh);
2935  SplitInteger(RHS, RHSLow, RHSHigh);
2936  EVT HalfVT = LHSLow.getValueType()
2937  , BitVT = N->getValueType(1);
2938  SDVTList VTHalfMulO = DAG.getVTList(HalfVT, BitVT);
2939  SDVTList VTFullAddO = DAG.getVTList(VT, BitVT);
2940 
2941  SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
2942  SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
2943  DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
2944  DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));
2945 
2946  SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, LHSHigh, RHSLow);
2947  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));
2948  SDValue OneInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
2949  One.getValue(0));
2950 
2951  SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, RHSHigh, LHSLow);
2952  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));
2953  SDValue TwoInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
2954  Two.getValue(0));
2955 
2956  // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
2957  // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
2958  // operation recursively legalized?).
2959  //
2960  // Many backends understand this pattern and will convert into LOHI
2961  // themselves, if applicable.
2962  SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
2963  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
2964  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow));
2965  SDValue Four = DAG.getNode(ISD::ADD, dl, VT, OneInHigh, TwoInHigh);
2966  SDValue Five = DAG.getNode(ISD::UADDO, dl, VTFullAddO, Three, Four);
2967  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Five.getValue(1));
2968  SplitInteger(Five, Lo, Hi);
2969  ReplaceValueWith(SDValue(N, 1), Overflow);
2970  return;
2971  }
2972 
2973  Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
2974  EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
2975  Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
2976 
2977  // Replace this with a libcall that will check overflow.
2978  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2979  if (VT == MVT::i32)
2980  LC = RTLIB::MULO_I32;
2981  else if (VT == MVT::i64)
2982  LC = RTLIB::MULO_I64;
2983  else if (VT == MVT::i128)
2984  LC = RTLIB::MULO_I128;
2985  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
2986 
2987  SDValue Temp = DAG.CreateStackTemporary(PtrVT);
2988  // Temporary for the overflow value, default it to zero.
2989  SDValue Chain =
2990  DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
2991  MachinePointerInfo());
2992 
2994  TargetLowering::ArgListEntry Entry;
2995  for (const SDValue &Op : N->op_values()) {
2996  EVT ArgVT = Op.getValueType();
2997  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2998  Entry.Node = Op;
2999  Entry.Ty = ArgTy;
3000  Entry.IsSExt = true;
3001  Entry.IsZExt = false;
3002  Args.push_back(Entry);
3003  }
3004 
3005  // Also pass the address of the overflow check.
3006  Entry.Node = Temp;
3007  Entry.Ty = PtrTy->getPointerTo();
3008  Entry.IsSExt = true;
3009  Entry.IsZExt = false;
3010  Args.push_back(Entry);
3011 
3012  SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
3013 
3015  CLI.setDebugLoc(dl)
3016  .setChain(Chain)
3017  .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
3018  .setSExtResult();
3019 
3020  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
3021 
3022  SplitInteger(CallInfo.first, Lo, Hi);
3023  SDValue Temp2 =
3024  DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
3025  SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
3026  DAG.getConstant(0, dl, PtrVT),
3027  ISD::SETNE);
3028  // Use the overflow from the libcall everywhere.
3029  ReplaceValueWith(SDValue(N, 1), Ofl);
3030 }
3031 
3032 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
3033  SDValue &Lo, SDValue &Hi) {
3034  EVT VT = N->getValueType(0);
3035  SDLoc dl(N);
3036  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3037 
3039  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3040  SplitInteger(Res.getValue(0), Lo, Hi);
3041  return;
3042  }
3043 
3044  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3045  if (VT == MVT::i16)
3046  LC = RTLIB::UDIV_I16;
3047  else if (VT == MVT::i32)
3048  LC = RTLIB::UDIV_I32;
3049  else if (VT == MVT::i64)
3050  LC = RTLIB::UDIV_I64;
3051  else if (VT == MVT::i128)
3052  LC = RTLIB::UDIV_I128;
3053  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
3054 
3055  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
3056 }
3057 
3058 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
3059  SDValue &Lo, SDValue &Hi) {
3060  EVT VT = N->getValueType(0);
3061  SDLoc dl(N);
3062  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3063 
3065  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3066  SplitInteger(Res.getValue(1), Lo, Hi);
3067  return;
3068  }
3069 
3070  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3071  if (VT == MVT::i16)
3072  LC = RTLIB::UREM_I16;
3073  else if (VT == MVT::i32)
3074  LC = RTLIB::UREM_I32;
3075  else if (VT == MVT::i64)
3076  LC = RTLIB::UREM_I64;
3077  else if (VT == MVT::i128)
3078  LC = RTLIB::UREM_I128;
3079  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
3080 
3081  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
3082 }
3083 
3084 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
3085  SDValue &Lo, SDValue &Hi) {
3086  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3087  SDLoc dl(N);
3088  SDValue Op = N->getOperand(0);
3089  if (Op.getValueType().bitsLE(NVT)) {
3090  // The low part is zero extension of the input (degenerates to a copy).
3091  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
3092  Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
3093  } else {
3094  // For example, extension of an i48 to an i64. The operand type necessarily
3095  // promotes to the result type, so will end up being expanded too.
3096  assert(getTypeAction(Op.getValueType()) ==
3098  "Only know how to promote this result!");
3099  SDValue Res = GetPromotedInteger(Op);
3100  assert(Res.getValueType() == N->getValueType(0) &&
3101  "Operand over promoted?");
3102  // Split the promoted operand. This will simplify when it is expanded.
3103  SplitInteger(Res, Lo, Hi);
3104  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
3105  Hi = DAG.getZeroExtendInReg(Hi, dl,
3107  ExcessBits));
3108  }
3109 }
3110 
3111 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
3112  SDValue &Lo, SDValue &Hi) {
3113  SDLoc dl(N);
3114  EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
3115  SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
3116  SDValue Zero = DAG.getConstant(0, dl, VT);
3117  SDValue Swap = DAG.getAtomicCmpSwap(
3119  cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
3120  N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
3121 
3122  ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
3123  ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
3124 }
3125 
3126 //===----------------------------------------------------------------------===//
3127 // Integer Operand Expansion
3128 //===----------------------------------------------------------------------===//
3129 
3130 /// ExpandIntegerOperand - This method is called when the specified operand of
3131 /// the specified node is found to need expansion. At this point, all of the
3132 /// result types of the node are known to be legal, but other operands of the
3133 /// node may need promotion or expansion as well as the specified one.
3134 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
3135  LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG);
3136  dbgs() << "\n");
3137  SDValue Res = SDValue();
3138 
3139  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
3140  return false;
3141 
3142  switch (N->getOpcode()) {
3143  default:
3144  #ifndef NDEBUG
3145  dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
3146  N->dump(&DAG); dbgs() << "\n";
3147  #endif
3148  llvm_unreachable("Do not know how to expand this operator's operand!");
3149 
3150  case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
3151  case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
3152  case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
3153  case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
3154  case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
3155  case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
3156  case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
3157  case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
3158  case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
3159  case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break;
3160  case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
3161  case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
3162  case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break;
3163 
3164  case ISD::SHL:
3165  case ISD::SRA:
3166  case ISD::SRL:
3167  case ISD::ROTL:
3168  case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
3169  case ISD::RETURNADDR:
3170  case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
3171 
3172  case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
3173  }
3174 
3175  // If the result is null, the sub-method took care of registering results etc.
3176  if (!Res.getNode()) return false;
3177 
3178  // If the result is N, the sub-method updated N in place. Tell the legalizer
3179  // core about this.
3180  if (Res.getNode() == N)
3181  return true;
3182 
3183  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3184  "Invalid operand expansion");
3185 
3186  ReplaceValueWith(SDValue(N, 0), Res);
3187  return false;
3188 }
3189 
3190 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
3191 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
3192 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
3193  SDValue &NewRHS,
3194  ISD::CondCode &CCCode,
3195  const SDLoc &dl) {
3196  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3197  GetExpandedInteger(NewLHS, LHSLo, LHSHi);
3198  GetExpandedInteger(NewRHS, RHSLo, RHSHi);
3199 
3200  if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
3201  if (RHSLo == RHSHi) {
3202  if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
3203  if (RHSCST->isAllOnesValue()) {
3204  // Equality comparison to -1.
3205  NewLHS = DAG.getNode(ISD::AND, dl,
3206  LHSLo.getValueType(), LHSLo, LHSHi);
3207  NewRHS = RHSLo;
3208  return;
3209  }
3210  }
3211  }
3212 
3213  NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
3214  NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
3215  NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
3216  NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
3217  return;
3218  }
3219 
3220  // If this is a comparison of the sign bit, just look at the top part.
3221  // X > -1, x < 0
3222  if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
3223  if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0
3224  (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1
3225  NewLHS = LHSHi;
3226  NewRHS = RHSHi;
3227  return;
3228  }
3229 
3230  // FIXME: This generated code sucks.
3231  ISD::CondCode LowCC;
3232  switch (CCCode) {
3233  default: llvm_unreachable("Unknown integer setcc!");
3234  case ISD::SETLT:
3235  case ISD::SETULT: LowCC = ISD::SETULT; break;
3236  case ISD::SETGT:
3237  case ISD::SETUGT: LowCC = ISD::SETUGT; break;
3238  case ISD::SETLE:
3239  case ISD::SETULE: LowCC = ISD::SETULE; break;
3240  case ISD::SETGE:
3241  case ISD::SETUGE: LowCC = ISD::SETUGE; break;
3242  }
3243 
3244  // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
3245  // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
3246  // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
3247 
3248  // NOTE: on targets without efficient SELECT of bools, we can always use
3249  // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
3250  TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
3251  nullptr);
3252  SDValue LoCmp, HiCmp;
3253  if (TLI.isTypeLegal(LHSLo.getValueType()) &&
3254  TLI.isTypeLegal(RHSLo.getValueType()))
3255  LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
3256  RHSLo, LowCC, false, DagCombineInfo, dl);
3257  if (!LoCmp.getNode())
3258  LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
3259  RHSLo, LowCC);
3260  if (TLI.isTypeLegal(LHSHi.getValueType()) &&
3261  TLI.isTypeLegal(RHSHi.getValueType()))
3262  HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
3263  RHSHi, CCCode, false, DagCombineInfo, dl);
3264  if (!HiCmp.getNode())
3265  HiCmp =
3266  DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
3267  LHSHi, RHSHi, DAG.getCondCode(CCCode));
3268 
3269  ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
3270  ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
3271 
3272  bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
3273  CCCode == ISD::SETUGE || CCCode == ISD::SETULE);
3274 
3275  if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) ||
3276  (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) ||
3277  (LoCmpC && LoCmpC->isNullValue())))) {
3278  // For LE / GE, if high part is known false, ignore the low part.
3279  // For LT / GT: if low part is known false, return the high part.
3280  // if high part is known true, ignore the low part.
3281  NewLHS = HiCmp;
3282  NewRHS = SDValue();
3283  return;
3284  }
3285 
3286  if (LHSHi == RHSHi) {
3287  // Comparing the low bits is enough.
3288  NewLHS = LoCmp;
3289  NewRHS = SDValue();
3290  return;
3291  }
3292 
3293  // Lower with SETCCCARRY if the target supports it.
3294  EVT HiVT = LHSHi.getValueType();
3295  EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
3296  bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
3297 
3298  // FIXME: Make all targets support this, then remove the other lowering.
3299  if (HasSETCCCARRY) {
3300  // SETCCCARRY can detect < and >= directly. For > and <=, flip
3301  // operands and condition code.
3302  bool FlipOperands = false;
3303  switch (CCCode) {
3304  case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
3305  case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
3306  case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
3307  case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
3308  default: break;
3309  }
3310  if (FlipOperands) {
3311  std::swap(LHSLo, RHSLo);
3312  std::swap(LHSHi, RHSHi);
3313  }
3314  // Perform a wide subtraction, feeding the carry from the low part into
3315  // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
3316  // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
3317  // zero or positive iff LHS >= RHS.
3318  EVT LoVT = LHSLo.getValueType();
3319  SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
3320  SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
3321  SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
3322  LHSHi, RHSHi, LowCmp.getValue(1),
3323  DAG.getCondCode(CCCode));
3324  NewLHS = Res;
3325  NewRHS = SDValue();
3326  return;
3327  }
3328 
3329  NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
3330  false, DagCombineInfo, dl);
3331  if (!NewLHS.getNode())
3332  NewLHS =
3333  DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
3334  NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
3335  NewRHS = SDValue();
3336 }
3337 
3338 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
3339  SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
3340  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
3341  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3342 
3343  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3344  // against zero to select between true and false values.
3345  if (!NewRHS.getNode()) {
3346  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3347  CCCode = ISD::SETNE;
3348  }
3349 
3350  // Update N to have the operands specified.
3351  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
3352  DAG.getCondCode(CCCode), NewLHS, NewRHS,
3353  N->getOperand(4)), 0);
3354 }
3355 
3356 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
3357  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3358  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
3359  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3360 
3361  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3362  // against zero to select between true and false values.
3363  if (!NewRHS.getNode()) {
3364  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3365  CCCode = ISD::SETNE;
3366  }
3367 
3368  // Update N to have the operands specified.
3369  return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
3370  N->getOperand(2), N->getOperand(3),
3371  DAG.getCondCode(CCCode)), 0);
3372 }
3373 
3374 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
3375  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3376  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3377  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3378 
3379  // If ExpandSetCCOperands returned a scalar, use it.
3380  if (!NewRHS.getNode()) {
3381  assert(NewLHS.getValueType() == N->getValueType(0) &&
3382  "Unexpected setcc expansion!");
3383  return NewLHS;
3384  }
3385 
3386  // Otherwise, update N to have the operands specified.
3387  return SDValue(
3388  DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
3389 }
3390 
3391 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
3392  SDValue LHS = N->getOperand(0);
3393  SDValue RHS = N->getOperand(1);
3394  SDValue Carry = N->getOperand(2);
3395  SDValue Cond = N->getOperand(3);
3396  SDLoc dl = SDLoc(N);
3397 
3398  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3399  GetExpandedInteger(LHS, LHSLo, LHSHi);
3400  GetExpandedInteger(RHS, RHSLo, RHSHi);
3401 
3402  // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
3403  SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
3404  SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
3405  return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
3406  LowCmp.getValue(1), Cond);
3407 }
3408 
3409 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
3410  // The value being shifted is legal, but the shift amount is too big.
3411  // It follows that either the result of the shift is undefined, or the
3412  // upper half of the shift amount is zero. Just use the lower half.
3413  SDValue Lo, Hi;
3414  GetExpandedInteger(N->getOperand(1), Lo, Hi);
3415  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
3416 }
3417 
3418 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
3419  // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
3420  // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
3421  // constant to valid type.
3422  SDValue Lo, Hi;
3423  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3424  return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
3425 }
3426 
3427 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
3428  SDValue Op = N->getOperand(0);
3429  EVT DstVT = N->getValueType(0);
3431  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3432  "Don't know how to expand this SINT_TO_FP!");
3433  return TLI.makeLibCall(DAG, LC, DstVT, Op, true, SDLoc(N)).first;
3434 }
3435 
3436 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
3437  if (ISD::isNormalStore(N))
3438  return ExpandOp_NormalStore(N, OpNo);
3439 
3440  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
3441  assert(OpNo == 1 && "Can only expand the stored value so far");
3442 
3443  EVT VT = N->getOperand(1).getValueType();
3444  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3445  SDValue Ch = N->getChain();
3446  SDValue Ptr = N->getBasePtr();
3447  unsigned Alignment = N->getAlignment();
3448  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3449  AAMDNodes AAInfo = N->getAAInfo();
3450  SDLoc dl(N);
3451  SDValue Lo, Hi;
3452 
3453  assert(NVT.isByteSized() && "Expanded type not byte sized!");
3454 
3455  if (N->getMemoryVT().bitsLE(NVT)) {
3456  GetExpandedInteger(N->getValue(), Lo, Hi);
3457  return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
3458  N->getMemoryVT(), Alignment, MMOFlags, AAInfo);
3459  }
3460 
3461  if (DAG.getDataLayout().isLittleEndian()) {
3462  // Little-endian - low bits are at low addresses.
3463  GetExpandedInteger(N->getValue(), Lo, Hi);
3464 
3465  Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3466  AAInfo);
3467 
3468  unsigned ExcessBits =
3469  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3470  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3471 
3472  // Increment the pointer to the other half.
3473  unsigned IncrementSize = NVT.getSizeInBits()/8;
3474  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3475  Hi = DAG.getTruncStore(
3476  Ch, dl, Hi, Ptr, N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3477  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3478  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3479  }
3480 
3481  // Big-endian - high bits are at low addresses. Favor aligned stores at
3482  // the cost of some bit-fiddling.
3483  GetExpandedInteger(N->getValue(), Lo, Hi);
3484 
3485  EVT ExtVT = N->getMemoryVT();
3486  unsigned EBytes = ExtVT.getStoreSize();
3487  unsigned IncrementSize = NVT.getSizeInBits()/8;
3488  unsigned ExcessBits = (EBytes - IncrementSize)*8;
3489  EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
3490  ExtVT.getSizeInBits() - ExcessBits);
3491 
3492  if (ExcessBits < NVT.getSizeInBits()) {
3493  // Transfer high bits from the top of Lo to the bottom of Hi.
3494  Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
3495  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3496  TLI.getPointerTy(DAG.getDataLayout())));
3497  Hi = DAG.getNode(
3498  ISD::OR, dl, NVT, Hi,
3499  DAG.getNode(ISD::SRL, dl, NVT, Lo,
3500  DAG.getConstant(ExcessBits, dl,
3501  TLI.getPointerTy(DAG.getDataLayout()))));
3502  }
3503 
3504  // Store both the high bits and maybe some of the low bits.
3505  Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment,
3506  MMOFlags, AAInfo);
3507 
3508  // Increment the pointer to the other half.
3509  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3510  // Store the lowest ExcessBits bits in the second half.
3511  Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
3512  N->getPointerInfo().getWithOffset(IncrementSize),
3513  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3514  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3515  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3516 }
3517 
3518 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
3519  SDValue InL, InH;
3520  GetExpandedInteger(N->getOperand(0), InL, InH);
3521  // Just truncate the low part of the source.
3522  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
3523 }
3524 
3525 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
3526  SDValue Op = N->getOperand(0);
3527  EVT SrcVT = Op.getValueType();
3528  EVT DstVT = N->getValueType(0);
3529  SDLoc dl(N);
3530 
3531  // The following optimization is valid only if every value in SrcVT (when
3532  // treated as signed) is representable in DstVT. Check that the mantissa
3533  // size of DstVT is >= than the number of bits in SrcVT -1.
3534  const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT);
3535  if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 &&
3537  // Do a signed conversion then adjust the result.
3538  SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op);
3539  SignedConv = TLI.LowerOperation(SignedConv, DAG);
3540 
3541  // The result of the signed conversion needs adjusting if the 'sign bit' of
3542  // the incoming integer was set. To handle this, we dynamically test to see
3543  // if it is set, and, if so, add a fudge factor.
3544 
3545  const uint64_t F32TwoE32 = 0x4F800000ULL;
3546  const uint64_t F32TwoE64 = 0x5F800000ULL;
3547  const uint64_t F32TwoE128 = 0x7F800000ULL;
3548 
3549  APInt FF(32, 0);
3550  if (SrcVT == MVT::i32)
3551  FF = APInt(32, F32TwoE32);
3552  else if (SrcVT == MVT::i64)
3553  FF = APInt(32, F32TwoE64);
3554  else if (SrcVT == MVT::i128)
3555  FF = APInt(32, F32TwoE128);
3556  else
3557  llvm_unreachable("Unsupported UINT_TO_FP!");
3558 
3559  // Check whether the sign bit is set.
3560  SDValue Lo, Hi;
3561  GetExpandedInteger(Op, Lo, Hi);
3562  SDValue SignSet = DAG.getSetCC(dl,
3563  getSetCCResultType(Hi.getValueType()),
3564  Hi,
3565  DAG.getConstant(0, dl, Hi.getValueType()),
3566  ISD::SETLT);
3567 
3568  // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
3569  SDValue FudgePtr =
3570  DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)),
3571  TLI.getPointerTy(DAG.getDataLayout()));
3572 
3573  // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
3574  SDValue Zero = DAG.getIntPtrConstant(0, dl);
3575  SDValue Four = DAG.getIntPtrConstant(4, dl);
3576  if (DAG.getDataLayout().isBigEndian())
3577  std::swap(Zero, Four);
3578  SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet,
3579  Zero, Four);
3580  unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment();
3581  FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(),
3582  FudgePtr, Offset);
3583  Alignment = std::min(Alignment, 4u);
3584 
3585  // Load the value out, extending it from f32 to the destination float type.
3586  // FIXME: Avoid the extend by constructing the right constant pool?
3587  SDValue Fudge = DAG.getExtLoad(
3588  ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), FudgePtr,
3590  Alignment);
3591  return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge);
3592  }
3593 
3594  // Otherwise, use a libcall.
3595  RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT);
3596  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3597  "Don't know how to expand this UINT_TO_FP!");
3598  return TLI.makeLibCall(DAG, LC, DstVT, Op, true, dl).first;
3599 }
3600 
3601 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
3602  SDLoc dl(N);
3603  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
3604  cast<AtomicSDNode>(N)->getMemoryVT(),
3605  N->getOperand(0),
3606  N->getOperand(1), N->getOperand(2),
3607  cast<AtomicSDNode>(N)->getMemOperand());
3608  return Swap.getValue(1);
3609 }
3610 
3611 
3612 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
3613  SDValue InOp0 = N->getOperand(0);
3614  EVT InVT = InOp0.getValueType();
3615 
3616  EVT OutVT = N->getValueType(0);
3617  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3618  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3619  unsigned OutNumElems = OutVT.getVectorNumElements();
3620  EVT NOutVTElem = NOutVT.getVectorElementType();
3621 
3622  SDLoc dl(N);
3623  SDValue BaseIdx = N->getOperand(1);
3624 
3626  Ops.reserve(OutNumElems);
3627  for (unsigned i = 0; i != OutNumElems; ++i) {
3628 
3629  // Extract the element from the original vector.
3630  SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
3631  BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
3633  InVT.getVectorElementType(), N->getOperand(0), Index);
3634 
3635  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext);
3636  // Insert the converted element to the new vector.
3637  Ops.push_back(Op);
3638  }
3639 
3640  return DAG.getBuildVector(NOutVT, dl, Ops);
3641 }
3642 
3643 
3644 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
3645  ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
3646  EVT VT = N->getValueType(0);
3647  SDLoc dl(N);
3648 
3649  ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
3650 
3651  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3652  SDValue V1 = GetPromotedInteger(N->getOperand(1));
3653  EVT OutVT = V0.getValueType();
3654 
3655  return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
3656 }
3657 
3658 
3659 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
3660  EVT OutVT = N->getValueType(0);
3661  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3662  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3663  unsigned NumElems = N->getNumOperands();
3664  EVT NOutVTElem = NOutVT.getVectorElementType();
3665 
3666  SDLoc dl(N);
3667 
3669  Ops.reserve(NumElems);
3670  for (unsigned i = 0; i != NumElems; ++i) {
3671  SDValue Op;
3672  // BUILD_VECTOR integer operand types are allowed to be larger than the
3673  // result's element type. This may still be true after the promotion. For
3674  // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
3675  // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
3676  if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
3677  Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
3678  else
3679  Op = N->getOperand(i);
3680  Ops.push_back(Op);
3681  }
3682 
3683  return DAG.getBuildVector(NOutVT, dl, Ops);
3684 }
3685 
3686 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
3687 
3688  SDLoc dl(N);
3689 
3690  assert(!N->getOperand(0).getValueType().isVector() &&
3691  "Input must be a scalar");
3692 
3693  EVT OutVT = N->getValueType(0);
3694  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3695  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3696  EVT NOutVTElem = NOutVT.getVectorElementType();
3697 
3698  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
3699 
3700  return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
3701 }
3702 
3703 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
3704  SDLoc dl(N);
3705 
3706  EVT OutVT = N->getValueType(0);
3707  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3708  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3709 
3710  EVT OutElemTy = NOutVT.getVectorElementType();
3711 
3712  unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
3713  unsigned NumOutElem = NOutVT.getVectorNumElements();
3714  unsigned NumOperands = N->getNumOperands();
3715  assert(NumElem * NumOperands == NumOutElem &&
3716  "Unexpected number of elements");
3717 
3718  // Take the elements from the first vector.
3719  SmallVector<SDValue, 8> Ops(NumOutElem);
3720  for (unsigned i = 0; i < NumOperands; ++i) {
3721  SDValue Op = N->getOperand(i);
3722  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
3723  Op = GetPromotedInteger(Op);
3724  EVT SclrTy = Op.getValueType().getVectorElementType();
3725  assert(NumElem == Op.getValueType().getVectorNumElements() &&
3726  "Unexpected number of elements");
3727 
3728  for (unsigned j = 0; j < NumElem; ++j) {
3729  SDValue Ext = DAG.getNode(
3730  ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
3731  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3732  Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
3733  }
3734  }
3735 
3736  return DAG.getBuildVector(NOutVT, dl, Ops);
3737 }
3738 
3739 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
3740  EVT VT = N->getValueType(0);
3741  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3742  assert(NVT.isVector() && "This type must be promoted to a vector type");
3743 
3744  SDLoc dl(N);
3745 
3746  // For operands whose TypeAction is to promote, extend the promoted node
3747  // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
3748  // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
3749  // type..
3750  if (getTypeAction(N->getOperand(0).getValueType())
3752  SDValue Promoted;
3753 
3754  switch(N->getOpcode()) {
3756  Promoted = SExtPromotedInteger(N->getOperand(0));
3757  break;
3759  Promoted = ZExtPromotedInteger(N->getOperand(0));
3760  break;
3762  Promoted = GetPromotedInteger(N->getOperand(0));
3763  break;
3764  default:
3765  llvm_unreachable("Node has unexpected Opcode");
3766  }
3767  return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
3768  }
3769 
3770  // Directly extend to the appropriate transform-to type.
3771  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
3772 }
3773 
3774 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
3775  EVT OutVT = N->getValueType(0);
3776  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3777  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3778 
3779  EVT NOutVTElem = NOutVT.getVectorElementType();
3780 
3781  SDLoc dl(N);
3782  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3783 
3784  SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
3785  NOutVTElem, N->getOperand(1));
3786  return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
3787  V0, ConvElem, N->getOperand(2));
3788 }
3789 
3790 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3791  SDLoc dl(N);
3792  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3793  SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
3794  TLI.getVectorIdxTy(DAG.getDataLayout()));
3796  V0->getValueType(0).getScalarType(), V0, V1);
3797 
3798  // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
3799  // element types. If this is the case then we need to expand the outgoing
3800  // value and not truncate it.
3801  return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
3802 }
3803 
3804 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
3805  SDLoc dl(N);
3806  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3807  MVT InVT = V0.getValueType().getSimpleVT();
3808  MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
3810  SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
3811  return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
3812 }
3813 
3814 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
3815  SDLoc dl(N);
3816  unsigned NumElems = N->getNumOperands();
3817 
3818  EVT RetSclrTy = N->getValueType(0).getVectorElementType();
3819 
3820  SmallVector<SDValue, 8> NewOps;
3821  NewOps.reserve(NumElems);
3822 
3823  // For each incoming vector
3824  for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
3825  SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
3826  EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
3827  unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
3828 
3829  for (unsigned i=0; i<NumElem; ++i) {
3830  // Extract element from incoming vector
3831  SDValue Ex = DAG.getNode(
3832  ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
3833  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3834  SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
3835  NewOps.push_back(Tr);
3836  }
3837  }
3838 
3839  return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
3840 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:571
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:552
static EVT getShiftAmountTyForConstant(unsigned Val, EVT VT, const TargetLowering &TLI, SelectionDAG &DAG)
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getValueType() const
Return the ValueType of the referenced return value.
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant, which is required to be operand #1) half of the integer or float value specified as operand #0.
Definition: ISDOpcodes.h:184
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC&#39;s if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:934
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:358
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:650
This class represents lattice values for constants.
Definition: AllocatorList.h:24
static MVT getVectorVT(MVT VT, unsigned NumElements)
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:367
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:260
SDValue getExpandedSaturationAdditionSubtraction(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
#define LLVM_FALLTHROUGH
Definition: Compiler.h:86
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:519
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none...
const SDValue & getBasePtr() const
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:223
void push_back(const T &Elt)
Definition: SmallVector.h:218
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:858
const SDValue & getValue() const
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
Definition: APInt.h:648
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:253
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:251
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1329
const SDValue & getBasePtr() const
const SDValue & getChain() const
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
unsigned getAlignment() const
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:802
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select&#39;s if you just have operands and don&#39;t want to check...
Definition: SelectionDAG.h:947
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:508
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:384
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Same for subtraction.
Definition: ISDOpcodes.h:254
void reserve(size_type N)
Definition: SmallVector.h:376
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:212
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align=0, int Offs=0, bool isT=false, unsigned char TargetFlags=0)
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:435
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:39
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
SDValue getExternalSymbol(const char *Sym, EVT VT)
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:210
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations...
Definition: ISDOpcodes.h:456
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
static uint32_t getAlignment(const MCSectionCOFF &Sec)
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:451
Shift and rotation operations.
Definition: ISDOpcodes.h:410
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:202
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:652
CallLoweringInfo & setChain(SDValue InChain)
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:191
RESULT = SMULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same wi...
Definition: ISDOpcodes.h:280
const SDValue & getPassThru() const
op_iterator op_end() const
unsigned getScalarValueSizeInBits() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
FLT_ROUNDS_ - Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest 2 Round to ...
Definition: ISDOpcodes.h:546
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:304
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt) For double-word atomic operations: ValLo, ValHi, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amtLo, amtHi) ValLo, ValHi, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amtLo, amtHi) These correspond to the atomicrmw instruction.
Definition: ISDOpcodes.h:810
const SDValue & getValue() const
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:457
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:398
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG...
Definition: ISDOpcodes.h:73
This is an SDNode representing atomic operations.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
This represents a list of ValueType&#39;s that has been intern&#39;d by a SelectionDAG.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
This class is used to represent an MSTORE node.
unsigned getSizeInBits() const
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:261
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:298
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:292
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:395
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:478
SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
const SDValue & getScale() const
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:429
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:201
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:221
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:849
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTruncating=false, bool IsCompressing=false)
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:959
op_iterator op_begin() const
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:747
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:497
const SDValue & getMask() const
UNDEF - An undefined node.
Definition: ISDOpcodes.h:178
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:524
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:327
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
Definition: APInt.h:636
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits...
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
Definition: MathExtras.h:610
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const SDValue & getBasePtr() const
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:429
Machine Value Type.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
Simple binary floating point operators.
Definition: ISDOpcodes.h:283
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:273
This is an important base class in LLVM.
Definition: Constant.h:42
iterator_range< value_op_iterator > op_values() const
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
Definition: SmallVector.h:129
const SDValue & getOperand(unsigned Num) const
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:934
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:332
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:247
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:232
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:588
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:57
const SDValue & getValue() const
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:719
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType, bool IsExpanding=false)
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:416
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo...
Definition: ISDOpcodes.h:796
std::vector< ArgListEntry > ArgListTy
const SDValue & getIndex() const
Extended Value Type.
Definition: ValueTypes.h:34
const SDValue & getBasePtr() const
This structure contains all information that is necessary for lowering calls.
This class contains a discriminated union of information about pointers in memory operands...
unsigned getNumOperands() const
Return the number of values used by this operation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, unsigned Alignment, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SyncScope::ID SSID)
Gets a node for an atomic cmpxchg op.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type...
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:265
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:50
void dump() const
Dump this node, for debugging.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:339
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:265
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
Definition: ISDOpcodes.h:451
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:222
const SDValue & getMask() const
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:847
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:731
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:644
const SDValue & getPassThru() const
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:644
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:413
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:581
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:622
Represents one node in the SelectionDAG.
static unsigned int semanticsPrecision(const fltSemantics &)
Definition: APFloat.cpp:155
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:133
MachinePointerInfo getWithOffset(int64_t O) const
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:539
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:941
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT...
Definition: ValueTypes.h:73
EVT getMemoryVT() const
Return the type of the in-memory value.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
Class for arbitrary precision integers.
Definition: APInt.h:70
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:420
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:241
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:468
const SDValue & getBasePtr() const
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:471
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition: APInt.h:530
Flags
Flags values. These may be or&#39;d together.
#define Success
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool ugt(const APInt &RHS) const
Unsigned greather than comparison.
Definition: APInt.h:1255
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:546
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Definition: SmallVector.h:133
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:151
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
Definition: APInt.h:1321
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:387
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, bool isSigned, const SDLoc &dl, bool doesNotReturn=false, bool isReturnValueUsed=true) const
Returns a pair of (return value, chain).
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:206
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:486
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:614
bool expandMUL_LOHI(unsigned Opcode, EVT VT, SDLoc dl, SDValue LHS, SDValue RHS, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL or [US]MUL_LOHI of n-bit values into two or four nodes, respectively, each computing an n/2-bit part of the result.
#define N
Flags getFlags() const
Return the raw flags of the source value,.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:323
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:273
Same for multiplication.
Definition: ISDOpcodes.h:257
const SDValue & getBasePtr() const
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
SDValue getValue(unsigned R) const
This class is used to represent an MSCATTER node.
unsigned getNumRegisters(LLVMContext &Context, EVT VT) const
Return the number of registers that this ValueType will eventually require.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getCondCode(ISD::CondCode Cond)
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, const Value *PtrVal, unsigned Alignment, AtomicOrdering Ordering, SyncScope::ID SSID)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands...
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input...
const MachinePointerInfo & getPointerInfo() const
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:345
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
This class is used to represent an MLOAD node.
ArrayRef< int > getMask() const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
This class is used to represent an MGATHER node.
SDValue getValueType(EVT)
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:81
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:776
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:443
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Definition: ValueTypes.h:223
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:198
Conversion operators.
Definition: ISDOpcodes.h:465
BooleanContent
Enum that describes how the target represents true/false values.
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:789
bool isBigEndian() const
Definition: DataLayout.h:222
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:474
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Op, int64_t Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object...
Definition: SelectionDAG.h:803
#define LLVM_DEBUG(X)
Definition: Debug.h:123
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:785
LLVMContext * getContext() const
Definition: SelectionDAG.h:404
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:375
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:242
const SDValue & getMask() const
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
This class is used to represent ISD::LOAD nodes.