LLVM  4.0.0
LegalizeIntegerTypes.cpp
Go to the documentation of this file.
1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements integer type expansion and promotion for LegalizeTypes.
11 // Promotion is the act of changing a computation in an illegal type into a
12 // computation in a larger type. For example, implementing i8 arithmetic in an
13 // i32 register (often needed on powerpc).
14 // Expansion is the act of changing a computation in an illegal type into a
15 // computation in two identical registers of a smaller type. For example,
16 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
17 // targets).
18 //
19 //===----------------------------------------------------------------------===//
20 
21 #include "LegalizeTypes.h"
22 #include "llvm/IR/DerivedTypes.h"
25 using namespace llvm;
26 
27 #define DEBUG_TYPE "legalize-types"
28 
29 //===----------------------------------------------------------------------===//
30 // Integer Result Promotion
31 //===----------------------------------------------------------------------===//
32 
33 /// PromoteIntegerResult - This method is called when a result of a node is
34 /// found to be in need of promotion to a larger type. At this point, the node
35 /// may also have invalid operands or may have other results that need
36 /// expansion, we just know that (at least) one result needs promotion.
37 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
38  DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); dbgs() << "\n");
39  SDValue Res = SDValue();
40 
41  // See if the target wants to custom expand this node.
42  if (CustomLowerNode(N, N->getValueType(ResNo), true))
43  return;
44 
45  switch (N->getOpcode()) {
46  default:
47 #ifndef NDEBUG
48  dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
49  N->dump(&DAG); dbgs() << "\n";
50 #endif
51  llvm_unreachable("Do not know how to promote this operator!");
52  case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
53  case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
54  case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
55  case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
56  case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
57  case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
58  case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
59  case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
61  case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
62  case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break;
64  case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
66  Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
67  case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
68  case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
69  break;
70  case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
71  break;
72  case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
73  case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
74  case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
75  case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
76  case ISD::SMIN:
77  case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
78  case ISD::UMIN:
79  case ISD::UMAX: Res = PromoteIntRes_ZExtIntBinOp(N); break;
80 
81  case ISD::SHL: Res = PromoteIntRes_SHL(N); break;
83  Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
84  case ISD::SRA: Res = PromoteIntRes_SRA(N); break;
85  case ISD::SRL: Res = PromoteIntRes_SRL(N); break;
86  case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
87  case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
88  case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
89 
91  Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
93  Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
95  Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
96  case ISD::BUILD_VECTOR:
97  Res = PromoteIntRes_BUILD_VECTOR(N); break;
99  Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
100  case ISD::CONCAT_VECTORS:
101  Res = PromoteIntRes_CONCAT_VECTORS(N); break;
102 
106  Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
107 
108  case ISD::SIGN_EXTEND:
109  case ISD::ZERO_EXTEND:
110  case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
111 
112  case ISD::FP_TO_SINT:
113  case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
114 
115  case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
116 
117  case ISD::AND:
118  case ISD::OR:
119  case ISD::XOR:
120  case ISD::ADD:
121  case ISD::SUB:
122  case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
123 
124  case ISD::SDIV:
125  case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
126 
127  case ISD::UDIV:
128  case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
129 
130  case ISD::SADDO:
131  case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
132  case ISD::UADDO:
133  case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
134  case ISD::SMULO:
135  case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
136 
137  case ISD::ATOMIC_LOAD:
138  Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
139 
143  case ISD::ATOMIC_LOAD_OR:
150  case ISD::ATOMIC_SWAP:
151  Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
152 
155  Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
156  break;
157  }
158 
159  // If the result is null then the sub-method took care of registering it.
160  if (Res.getNode())
161  SetPromotedInteger(SDValue(N, ResNo), Res);
162 }
163 
164 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
165  unsigned ResNo) {
166  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
167  return GetPromotedInteger(Op);
168 }
169 
170 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
171  // Sign-extend the new bits, and continue the assertion.
172  SDValue Op = SExtPromotedInteger(N->getOperand(0));
173  return DAG.getNode(ISD::AssertSext, SDLoc(N),
174  Op.getValueType(), Op, N->getOperand(1));
175 }
176 
177 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
178  // Zero the new bits, and continue the assertion.
179  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
180  return DAG.getNode(ISD::AssertZext, SDLoc(N),
181  Op.getValueType(), Op, N->getOperand(1));
182 }
183 
184 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
185  EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
186  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
187  N->getMemoryVT(), ResVT,
188  N->getChain(), N->getBasePtr(),
189  N->getMemOperand());
190  // Legalize the chain result - switch anything that used the old chain to
191  // use the new one.
192  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
193  return Res;
194 }
195 
196 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
197  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
198  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
199  N->getMemoryVT(),
200  N->getChain(), N->getBasePtr(),
201  Op2, N->getMemOperand());
202  // Legalize the chain result - switch anything that used the old chain to
203  // use the new one.
204  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
205  return Res;
206 }
207 
208 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
209  unsigned ResNo) {
210  if (ResNo == 1) {
212  EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
213  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
214 
215  // Only use the result of getSetCCResultType if it is legal,
216  // otherwise just use the promoted result type (NVT).
217  if (!TLI.isTypeLegal(SVT))
218  SVT = NVT;
219 
220  SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
221  SDValue Res = DAG.getAtomicCmpSwap(
223  N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
224  N->getMemOperand());
225  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
226  ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
227  return Res.getValue(1);
228  }
229 
230  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
231  SDValue Op3 = GetPromotedInteger(N->getOperand(3));
232  SDVTList VTs =
233  DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
234  SDValue Res = DAG.getAtomicCmpSwap(
235  N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
236  N->getBasePtr(), Op2, Op3, N->getMemOperand());
237  // Update the use to N with the newly created Res.
238  for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
239  ReplaceValueWith(SDValue(N, i), Res.getValue(i));
240  return Res;
241 }
242 
243 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
244  SDValue InOp = N->getOperand(0);
245  EVT InVT = InOp.getValueType();
246  EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
247  EVT OutVT = N->getValueType(0);
248  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
249  SDLoc dl(N);
250 
251  switch (getTypeAction(InVT)) {
253  break;
255  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
256  // The input promotes to the same size. Convert the promoted value.
257  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
258  break;
260  // Promote the integer operand by hand.
261  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
263  // Convert the promoted float by hand.
264  SDValue PromotedOp = GetPromotedFloat(InOp);
265  return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, PromotedOp);
266  break;
267  }
270  break;
272  // Convert the element to an integer and promote it by hand.
273  if (!NOutVT.isVector())
274  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
275  BitConvertToInteger(GetScalarizedVector(InOp)));
276  break;
278  // For example, i32 = BITCAST v2i16 on alpha. Convert the split
279  // pieces of the input into integers and reassemble in the final type.
280  SDValue Lo, Hi;
281  GetSplitVector(N->getOperand(0), Lo, Hi);
282  Lo = BitConvertToInteger(Lo);
283  Hi = BitConvertToInteger(Hi);
284 
285  if (DAG.getDataLayout().isBigEndian())
286  std::swap(Lo, Hi);
287 
288  InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
290  NOutVT.getSizeInBits()),
291  JoinIntegers(Lo, Hi));
292  return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
293  }
295  // The input is widened to the same size. Convert to the widened value.
296  // Make sure that the outgoing value is not a vector, because this would
297  // make us bitcast between two vectors which are legalized in different ways.
298  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector())
299  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
300  }
301 
302  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
303  CreateStackStoreLoad(InOp, OutVT));
304 }
305 
306 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
307  SDValue Op = GetPromotedInteger(N->getOperand(0));
308  EVT OVT = N->getValueType(0);
309  EVT NVT = Op.getValueType();
310  SDLoc dl(N);
311 
312  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
313  return DAG.getNode(
314  ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
315  DAG.getConstant(DiffBits, dl,
316  TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
317 }
318 
319 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
320  SDValue Op = GetPromotedInteger(N->getOperand(0));
321  EVT OVT = N->getValueType(0);
322  EVT NVT = Op.getValueType();
323  SDLoc dl(N);
324 
325  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
326  return DAG.getNode(
327  ISD::SRL, dl, NVT, DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
328  DAG.getConstant(DiffBits, dl,
329  TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
330 }
331 
332 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
333  // The pair element type may be legal, or may not promote to the same type as
334  // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
335  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
336  TLI.getTypeToTransformTo(*DAG.getContext(),
337  N->getValueType(0)), JoinIntegers(N->getOperand(0),
338  N->getOperand(1)));
339 }
340 
341 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
342  EVT VT = N->getValueType(0);
343  // FIXME there is no actual debug info here
344  SDLoc dl(N);
345  // Zero extend things like i1, sign extend everything else. It shouldn't
346  // matter in theory which one we pick, but this tends to give better code?
347  unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
348  SDValue Result = DAG.getNode(Opc, dl,
349  TLI.getTypeToTransformTo(*DAG.getContext(), VT),
350  SDValue(N, 0));
351  assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
352  return Result;
353 }
354 
355 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
356  // Zero extend to the promoted type and do the count there.
357  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
358  SDLoc dl(N);
359  EVT OVT = N->getValueType(0);
360  EVT NVT = Op.getValueType();
361  Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
362  // Subtract off the extra leading bits in the bigger type.
363  return DAG.getNode(
364  ISD::SUB, dl, NVT, Op,
365  DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
366  NVT));
367 }
368 
369 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
370  // Zero extend to the promoted type and do the count there.
371  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
372  return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op);
373 }
374 
375 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
376  SDValue Op = GetPromotedInteger(N->getOperand(0));
377  EVT OVT = N->getValueType(0);
378  EVT NVT = Op.getValueType();
379  SDLoc dl(N);
380  if (N->getOpcode() == ISD::CTTZ) {
381  // The count is the same in the promoted type except if the original
382  // value was zero. This can be handled by setting the bit just off
383  // the top of the original type.
384  auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
385  OVT.getScalarSizeInBits());
386  Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
387  }
388  return DAG.getNode(N->getOpcode(), dl, NVT, Op);
389 }
390 
391 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
392  SDLoc dl(N);
393  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
394  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0),
395  N->getOperand(1));
396 }
397 
398 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
399  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
400  unsigned NewOpc = N->getOpcode();
401  SDLoc dl(N);
402 
403  // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
404  // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
405  // and SINT conversions are Custom, there is no way to tell which is
406  // preferable. We choose SINT because that's the right thing on PPC.)
407  if (N->getOpcode() == ISD::FP_TO_UINT &&
408  !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
410  NewOpc = ISD::FP_TO_SINT;
411 
412  SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
413 
414  // Assert that the converted value fits in the original type. If it doesn't
415  // (eg: because the value being converted is too big), then the result of the
416  // original operation was undefined anyway, so the assert is still correct.
417  //
418  // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
419  // before legalization: fp-to-uint16, 65534. -> 0xfffe
420  // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
421  return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ?
422  ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
424 }
425 
426 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
427  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
428  SDLoc dl(N);
429 
430  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
431 }
432 
433 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
434  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
435  SDLoc dl(N);
436 
437  if (getTypeAction(N->getOperand(0).getValueType())
439  SDValue Res = GetPromotedInteger(N->getOperand(0));
440  assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
441 
442  // If the result and operand types are the same after promotion, simplify
443  // to an in-register extension.
444  if (NVT == Res.getValueType()) {
445  // The high bits are not guaranteed to be anything. Insert an extend.
446  if (N->getOpcode() == ISD::SIGN_EXTEND)
447  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
448  DAG.getValueType(N->getOperand(0).getValueType()));
449  if (N->getOpcode() == ISD::ZERO_EXTEND)
450  return DAG.getZeroExtendInReg(Res, dl,
452  assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
453  return Res;
454  }
455  }
456 
457  // Otherwise, just extend the original operand all the way to the larger type.
458  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
459 }
460 
461 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
462  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
463  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
464  ISD::LoadExtType ExtType =
466  SDLoc dl(N);
467  SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
468  N->getMemoryVT(), N->getMemOperand());
469 
470  // Legalize the chain result - switch anything that used the old chain to
471  // use the new one.
472  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
473  return Res;
474 }
475 
476 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
477  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
478  SDValue ExtSrc0 = GetPromotedInteger(N->getSrc0());
479 
480  SDLoc dl(N);
481  SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
482  N->getMask(), ExtSrc0, N->getMemoryVT(),
484  // Legalize the chain result - switch anything that used the old chain to
485  // use the new one.
486  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
487  return Res;
488 }
489 
490 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
491  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
492  SDValue ExtSrc0 = GetPromotedInteger(N->getValue());
493  assert(NVT == ExtSrc0.getValueType() &&
494  "Gather result type and the passThru agrument type should be the same");
495 
496  SDLoc dl(N);
497  SDValue Ops[] = {N->getChain(), ExtSrc0, N->getMask(), N->getBasePtr(),
498  N->getIndex()};
499  SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
500  N->getMemoryVT(), dl, Ops,
501  N->getMemOperand());
502  // Legalize the chain result - switch anything that used the old chain to
503  // use the new one.
504  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
505  return Res;
506 }
507 
508 /// Promote the overflow flag of an overflowing arithmetic node.
509 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
510  // Simply change the return type of the boolean result.
511  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
512  EVT ValueVTs[] = { N->getValueType(0), NVT };
513  SDValue Ops[] = { N->getOperand(0), N->getOperand(1) };
514  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
515  DAG.getVTList(ValueVTs), Ops);
516 
517  // Modified the sum result - switch anything that used the old sum to use
518  // the new one.
519  ReplaceValueWith(SDValue(N, 0), Res);
520 
521  return SDValue(Res.getNode(), 1);
522 }
523 
524 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
525  if (ResNo == 1)
526  return PromoteIntRes_Overflow(N);
527 
528  // The operation overflowed iff the result in the larger type is not the
529  // sign extension of its truncation to the original type.
530  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
531  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
532  EVT OVT = N->getOperand(0).getValueType();
533  EVT NVT = LHS.getValueType();
534  SDLoc dl(N);
535 
536  // Do the arithmetic in the larger type.
537  unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
538  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
539 
540  // Calculate the overflow flag: sign extend the arithmetic result from
541  // the original type.
542  SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
543  DAG.getValueType(OVT));
544  // Overflowed if and only if this is not equal to Res.
545  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
546 
547  // Use the calculated overflow everywhere.
548  ReplaceValueWith(SDValue(N, 1), Ofl);
549 
550  return Res;
551 }
552 
553 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
554  SDValue LHS = GetPromotedInteger(N->getOperand(1));
555  SDValue RHS = GetPromotedInteger(N->getOperand(2));
556  return DAG.getSelect(SDLoc(N),
557  LHS.getValueType(), N->getOperand(0), LHS, RHS);
558 }
559 
560 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
561  SDValue Mask = N->getOperand(0);
562  EVT OpTy = N->getOperand(1).getValueType();
563 
564  // Promote all the way up to the canonical SetCC type.
565  Mask = PromoteTargetBoolean(Mask, OpTy);
566  SDValue LHS = GetPromotedInteger(N->getOperand(1));
567  SDValue RHS = GetPromotedInteger(N->getOperand(2));
568  return DAG.getNode(ISD::VSELECT, SDLoc(N),
569  LHS.getValueType(), Mask, LHS, RHS);
570 }
571 
572 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
573  SDValue LHS = GetPromotedInteger(N->getOperand(2));
574  SDValue RHS = GetPromotedInteger(N->getOperand(3));
575  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
576  LHS.getValueType(), N->getOperand(0),
577  N->getOperand(1), LHS, RHS, N->getOperand(4));
578 }
579 
580 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
581  EVT SVT = getSetCCResultType(N->getOperand(0).getValueType());
582 
583  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
584 
585  // Only use the result of getSetCCResultType if it is legal,
586  // otherwise just use the promoted result type (NVT).
587  if (!TLI.isTypeLegal(SVT))
588  SVT = NVT;
589 
590  SDLoc dl(N);
591  assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() &&
592  "Vector compare must return a vector result!");
593 
594  SDValue LHS = N->getOperand(0);
595  SDValue RHS = N->getOperand(1);
596  if (LHS.getValueType() != RHS.getValueType()) {
597  if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger &&
598  !LHS.getValueType().isVector())
599  LHS = GetPromotedInteger(LHS);
600  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger &&
601  !RHS.getValueType().isVector())
602  RHS = GetPromotedInteger(RHS);
603  }
604 
605  // Get the SETCC result using the canonical SETCC type.
606  SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, LHS, RHS,
607  N->getOperand(2));
608 
609  assert(NVT.bitsLE(SVT) && "Integer type overpromoted?");
610  // Convert to the expected type.
611  return DAG.getNode(ISD::TRUNCATE, dl, NVT, SetCC);
612 }
613 
614 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
615  SDValue LHS = N->getOperand(0);
616  SDValue RHS = N->getOperand(1);
617  if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
618  LHS = GetPromotedInteger(LHS);
619  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
620  RHS = ZExtPromotedInteger(RHS);
621  return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
622 }
623 
624 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
625  SDValue Op = GetPromotedInteger(N->getOperand(0));
626  return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
627  Op.getValueType(), Op, N->getOperand(1));
628 }
629 
630 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
631  // The input may have strange things in the top bits of the registers, but
632  // these operations don't care. They may have weird bits going out, but
633  // that too is okay if they are integer operations.
634  SDValue LHS = GetPromotedInteger(N->getOperand(0));
635  SDValue RHS = GetPromotedInteger(N->getOperand(1));
636  return DAG.getNode(N->getOpcode(), SDLoc(N),
637  LHS.getValueType(), LHS, RHS);
638 }
639 
640 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
641  // Sign extend the input.
642  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
643  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
644  return DAG.getNode(N->getOpcode(), SDLoc(N),
645  LHS.getValueType(), LHS, RHS);
646 }
647 
648 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
649  // Zero extend the input.
650  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
651  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
652  return DAG.getNode(N->getOpcode(), SDLoc(N),
653  LHS.getValueType(), LHS, RHS);
654 }
655 
656 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
657  SDValue LHS = N->getOperand(0);
658  SDValue RHS = N->getOperand(1);
659  // The input value must be properly sign extended.
660  if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
661  LHS = SExtPromotedInteger(LHS);
662  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
663  RHS = ZExtPromotedInteger(RHS);
664  return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
665 }
666 
667 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
668  SDValue LHS = N->getOperand(0);
669  SDValue RHS = N->getOperand(1);
670  // The input value must be properly zero extended.
671  if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
672  LHS = ZExtPromotedInteger(LHS);
673  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
674  RHS = ZExtPromotedInteger(RHS);
675  return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
676 }
677 
678 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
679  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
680  SDValue Res;
681  SDValue InOp = N->getOperand(0);
682  SDLoc dl(N);
683 
684  switch (getTypeAction(InOp.getValueType())) {
685  default: llvm_unreachable("Unknown type action!");
688  Res = InOp;
689  break;
691  Res = GetPromotedInteger(InOp);
692  break;
694  EVT InVT = InOp.getValueType();
695  assert(InVT.isVector() && "Cannot split scalar types");
696  unsigned NumElts = InVT.getVectorNumElements();
697  assert(NumElts == NVT.getVectorNumElements() &&
698  "Dst and Src must have the same number of elements");
699  assert(isPowerOf2_32(NumElts) &&
700  "Promoted vector type must be a power of two");
701 
702  SDValue EOp1, EOp2;
703  GetSplitVector(InOp, EOp1, EOp2);
704 
705  EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
706  NumElts/2);
707  EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
708  EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
709 
710  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
711  }
712 
713  // Truncate to NVT instead of VT
714  return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
715 }
716 
717 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
718  if (ResNo == 1)
719  return PromoteIntRes_Overflow(N);
720 
721  // The operation overflowed iff the result in the larger type is not the
722  // zero extension of its truncation to the original type.
723  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
724  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
725  EVT OVT = N->getOperand(0).getValueType();
726  EVT NVT = LHS.getValueType();
727  SDLoc dl(N);
728 
729  // Do the arithmetic in the larger type.
730  unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
731  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
732 
733  // Calculate the overflow flag: zero extend the arithmetic result from
734  // the original type.
735  SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
736  // Overflowed if and only if this is not equal to Res.
737  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
738 
739  // Use the calculated overflow everywhere.
740  ReplaceValueWith(SDValue(N, 1), Ofl);
741 
742  return Res;
743 }
744 
745 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
746  // Promote the overflow bit trivially.
747  if (ResNo == 1)
748  return PromoteIntRes_Overflow(N);
749 
750  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
751  SDLoc DL(N);
752  EVT SmallVT = LHS.getValueType();
753 
754  // To determine if the result overflowed in a larger type, we extend the
755  // input to the larger type, do the multiply (checking if it overflows),
756  // then also check the high bits of the result to see if overflow happened
757  // there.
758  if (N->getOpcode() == ISD::SMULO) {
759  LHS = SExtPromotedInteger(LHS);
760  RHS = SExtPromotedInteger(RHS);
761  } else {
762  LHS = ZExtPromotedInteger(LHS);
763  RHS = ZExtPromotedInteger(RHS);
764  }
765  SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
766  SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
767 
768  // Overflow occurred if it occurred in the larger type, or if the high part
769  // of the result does not zero/sign-extend the low part. Check this second
770  // possibility first.
771  SDValue Overflow;
772  if (N->getOpcode() == ISD::UMULO) {
773  // Unsigned overflow occurred if the high part is non-zero.
774  SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
775  DAG.getIntPtrConstant(SmallVT.getSizeInBits(),
776  DL));
777  Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
778  DAG.getConstant(0, DL, Hi.getValueType()),
779  ISD::SETNE);
780  } else {
781  // Signed overflow occurred if the high part does not sign extend the low.
782  SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
783  Mul, DAG.getValueType(SmallVT));
784  Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
785  }
786 
787  // The only other way for overflow to occur is if the multiplication in the
788  // larger type itself overflowed.
789  Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
790  SDValue(Mul.getNode(), 1));
791 
792  // Use the calculated overflow everywhere.
793  ReplaceValueWith(SDValue(N, 1), Overflow);
794  return Mul;
795 }
796 
797 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
798  return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
799  N->getValueType(0)));
800 }
801 
802 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
803  SDValue Chain = N->getOperand(0); // Get the chain.
804  SDValue Ptr = N->getOperand(1); // Get the pointer.
805  EVT VT = N->getValueType(0);
806  SDLoc dl(N);
807 
808  MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
809  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
810  // The argument is passed as NumRegs registers of type RegVT.
811 
812  SmallVector<SDValue, 8> Parts(NumRegs);
813  for (unsigned i = 0; i < NumRegs; ++i) {
814  Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
815  N->getConstantOperandVal(3));
816  Chain = Parts[i].getValue(1);
817  }
818 
819  // Handle endianness of the load.
820  if (DAG.getDataLayout().isBigEndian())
821  std::reverse(Parts.begin(), Parts.end());
822 
823  // Assemble the parts in the promoted type.
824  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
825  SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
826  for (unsigned i = 1; i < NumRegs; ++i) {
827  SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
828  // Shift it to the right position and "or" it in.
829  Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
830  DAG.getConstant(i * RegVT.getSizeInBits(), dl,
831  TLI.getPointerTy(DAG.getDataLayout())));
832  Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
833  }
834 
835  // Modified the chain result - switch anything that used the old chain to
836  // use the new one.
837  ReplaceValueWith(SDValue(N, 1), Chain);
838 
839  return Res;
840 }
841 
842 //===----------------------------------------------------------------------===//
843 // Integer Operand Promotion
844 //===----------------------------------------------------------------------===//
845 
846 /// PromoteIntegerOperand - This method is called when the specified operand of
847 /// the specified node is found to need promotion. At this point, all of the
848 /// result types of the node are known to be legal, but other operands of the
849 /// node may need promotion or expansion as well as the specified one.
850 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
851  DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); dbgs() << "\n");
852  SDValue Res = SDValue();
853 
854  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
855  return false;
856 
857  switch (N->getOpcode()) {
858  default:
859  #ifndef NDEBUG
860  dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
861  N->dump(&DAG); dbgs() << "\n";
862  #endif
863  llvm_unreachable("Do not know how to promote this operator's operand!");
864 
865  case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
866  case ISD::ATOMIC_STORE:
867  Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
868  break;
869  case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
870  case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
871  case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
872  case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
873  case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
874  case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
875  case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
877  Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
879  Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
880  case ISD::VSELECT:
881  case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
882  case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
883  case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
884  case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
885  case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
886  case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
887  OpNo); break;
888  case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
889  OpNo); break;
890  case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
891  OpNo); break;
892  case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
893  OpNo); break;
894  case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
895  OpNo); break;
896  case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
897  case ISD::FP16_TO_FP:
898  case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
899  case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
900  case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
901 
902  case ISD::SHL:
903  case ISD::SRA:
904  case ISD::SRL:
905  case ISD::ROTL:
906  case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
907  }
908 
909  // If the result is null, the sub-method took care of registering results etc.
910  if (!Res.getNode()) return false;
911 
912  // If the result is N, the sub-method updated N in place. Tell the legalizer
913  // core about this.
914  if (Res.getNode() == N)
915  return true;
916 
917  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
918  "Invalid operand expansion");
919 
920  ReplaceValueWith(SDValue(N, 0), Res);
921  return false;
922 }
923 
924 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
925 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
926 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
927  ISD::CondCode CCCode) {
928  // We have to insert explicit sign or zero extends. Note that we could
929  // insert sign extends for ALL conditions, but zero extend is cheaper on
930  // many machines (an AND instead of two shifts), so prefer it.
931  switch (CCCode) {
932  default: llvm_unreachable("Unknown integer comparison!");
933  case ISD::SETEQ:
934  case ISD::SETNE: {
935  SDValue OpL = GetPromotedInteger(NewLHS);
936  SDValue OpR = GetPromotedInteger(NewRHS);
937 
938  // We would prefer to promote the comparison operand with sign extension.
939  // If the width of OpL/OpR excluding the duplicated sign bits is no greater
940  // than the width of NewLHS/NewRH, we can avoid inserting real truncate
941  // instruction, which is redudant eventually.
942  unsigned OpLEffectiveBits =
943  OpL.getValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
944  unsigned OpREffectiveBits =
945  OpR.getValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
946  if (OpLEffectiveBits <= NewLHS.getValueSizeInBits() &&
947  OpREffectiveBits <= NewRHS.getValueSizeInBits()) {
948  NewLHS = OpL;
949  NewRHS = OpR;
950  } else {
951  NewLHS = ZExtPromotedInteger(NewLHS);
952  NewRHS = ZExtPromotedInteger(NewRHS);
953  }
954  break;
955  }
956  case ISD::SETUGE:
957  case ISD::SETUGT:
958  case ISD::SETULE:
959  case ISD::SETULT:
960  // ALL of these operations will work if we either sign or zero extend
961  // the operands (including the unsigned comparisons!). Zero extend is
962  // usually a simpler/cheaper operation, so prefer it.
963  NewLHS = ZExtPromotedInteger(NewLHS);
964  NewRHS = ZExtPromotedInteger(NewRHS);
965  break;
966  case ISD::SETGE:
967  case ISD::SETGT:
968  case ISD::SETLT:
969  case ISD::SETLE:
970  NewLHS = SExtPromotedInteger(NewLHS);
971  NewRHS = SExtPromotedInteger(NewRHS);
972  break;
973  }
974 }
975 
976 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
977  SDValue Op = GetPromotedInteger(N->getOperand(0));
978  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
979 }
980 
981 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
982  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
983  return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
984  N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
985 }
986 
987 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
988  // This should only occur in unusual situations like bitcasting to an
989  // x86_fp80, so just turn it into a store+load
990  return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
991 }
992 
993 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
994  assert(OpNo == 2 && "Don't know how to promote this operand!");
995 
996  SDValue LHS = N->getOperand(2);
997  SDValue RHS = N->getOperand(3);
998  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
999 
1000  // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1001  // legal types.
1002  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1003  N->getOperand(1), LHS, RHS, N->getOperand(4)),
1004  0);
1005 }
1006 
1007 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1008  assert(OpNo == 1 && "only know how to promote condition");
1009 
1010  // Promote all the way up to the canonical SetCC type.
1011  SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1012 
1013  // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1014  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1015  N->getOperand(2)), 0);
1016 }
1017 
1018 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1019  // Since the result type is legal, the operands must promote to it.
1020  EVT OVT = N->getOperand(0).getValueType();
1021  SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1022  SDValue Hi = GetPromotedInteger(N->getOperand(1));
1023  assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1024  SDLoc dl(N);
1025 
1026  Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1027  DAG.getConstant(OVT.getSizeInBits(), dl,
1028  TLI.getPointerTy(DAG.getDataLayout())));
1029  return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1030 }
1031 
1032 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1033  // The vector type is legal but the element type is not. This implies
1034  // that the vector is a power-of-two in length and that the element
1035  // type does not have a strange size (eg: it is not i1).
1036  EVT VecVT = N->getValueType(0);
1037  unsigned NumElts = VecVT.getVectorNumElements();
1038  assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1039  "Legal vector of one illegal element?");
1040 
1041  // Promote the inserted value. The type does not need to match the
1042  // vector element type. Check that any extra bits introduced will be
1043  // truncated away.
1046  "Type of inserted value narrower than vector element type!");
1047 
1048  SmallVector<SDValue, 16> NewOps;
1049  for (unsigned i = 0; i < NumElts; ++i)
1050  NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1051 
1052  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1053 }
1054 
1055 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1056  unsigned OpNo) {
1057  if (OpNo == 1) {
1058  // Promote the inserted value. This is valid because the type does not
1059  // have to match the vector element type.
1060 
1061  // Check that any extra bits introduced will be truncated away.
1064  "Type of inserted value narrower than vector element type!");
1065  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1066  GetPromotedInteger(N->getOperand(1)),
1067  N->getOperand(2)),
1068  0);
1069  }
1070 
1071  assert(OpNo == 2 && "Different operand and result vector types?");
1072 
1073  // Promote the index.
1074  SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1075  TLI.getVectorIdxTy(DAG.getDataLayout()));
1076  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1077  N->getOperand(1), Idx), 0);
1078 }
1079 
1080 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1081  // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1082  // the operand in place.
1083  return SDValue(DAG.UpdateNodeOperands(N,
1084  GetPromotedInteger(N->getOperand(0))), 0);
1085 }
1086 
1087 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1088  assert(OpNo == 0 && "Only know how to promote the condition!");
1089  SDValue Cond = N->getOperand(0);
1090  EVT OpTy = N->getOperand(1).getValueType();
1091 
1092  // Promote all the way up to the canonical SetCC type.
1093  EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1094  Cond = PromoteTargetBoolean(Cond, OpVT);
1095 
1096  return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1097  N->getOperand(2)), 0);
1098 }
1099 
1100 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1101  assert(OpNo == 0 && "Don't know how to promote this operand!");
1102 
1103  SDValue LHS = N->getOperand(0);
1104  SDValue RHS = N->getOperand(1);
1105  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1106 
1107  // The CC (#4) and the possible return values (#2 and #3) have legal types.
1108  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1109  N->getOperand(3), N->getOperand(4)), 0);
1110 }
1111 
1112 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1113  assert(OpNo == 0 && "Don't know how to promote this operand!");
1114 
1115  SDValue LHS = N->getOperand(0);
1116  SDValue RHS = N->getOperand(1);
1117  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1118 
1119  // The CC (#2) is always legal.
1120  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1121 }
1122 
1123 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1124  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1125  ZExtPromotedInteger(N->getOperand(1))), 0);
1126 }
1127 
1128 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1129  SDValue Op = GetPromotedInteger(N->getOperand(0));
1130  SDLoc dl(N);
1131  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1132  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1133  Op, DAG.getValueType(N->getOperand(0).getValueType()));
1134 }
1135 
1136 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1137  return SDValue(DAG.UpdateNodeOperands(N,
1138  SExtPromotedInteger(N->getOperand(0))), 0);
1139 }
1140 
1141 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1142  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1143  SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1144  SDLoc dl(N);
1145 
1146  SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1147 
1148  // Truncate the value and store the result.
1149  return DAG.getTruncStore(Ch, dl, Val, Ptr,
1150  N->getMemoryVT(), N->getMemOperand());
1151 }
1152 
1153 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1154  unsigned OpNo) {
1155 
1156  SDValue DataOp = N->getValue();
1157  EVT DataVT = DataOp.getValueType();
1158  SDValue Mask = N->getMask();
1159  SDLoc dl(N);
1160 
1161  bool TruncateStore = false;
1162  if (OpNo == 2) {
1163  // Mask comes before the data operand. If the data operand is legal, we just
1164  // promote the mask.
1165  // When the data operand has illegal type, we should legalize the data
1166  // operand first. The mask will be promoted/splitted/widened according to
1167  // the data operand type.
1168  if (TLI.isTypeLegal(DataVT))
1169  Mask = PromoteTargetBoolean(Mask, DataVT);
1170  else {
1171  if (getTypeAction(DataVT) == TargetLowering::TypePromoteInteger)
1172  return PromoteIntOp_MSTORE(N, 3);
1173 
1174  else if (getTypeAction(DataVT) == TargetLowering::TypeWidenVector)
1175  return WidenVecOp_MSTORE(N, 3);
1176 
1177  else {
1178  assert (getTypeAction(DataVT) == TargetLowering::TypeSplitVector);
1179  return SplitVecOp_MSTORE(N, 3);
1180  }
1181  }
1182  } else { // Data operand
1183  assert(OpNo == 3 && "Unexpected operand for promotion");
1184  DataOp = GetPromotedInteger(DataOp);
1185  Mask = PromoteTargetBoolean(Mask, DataOp.getValueType());
1186  TruncateStore = true;
1187  }
1188 
1189  return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask,
1190  N->getMemoryVT(), N->getMemOperand(),
1191  TruncateStore, N->isCompressingStore());
1192 }
1193 
1194 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1195  unsigned OpNo) {
1196  assert(OpNo == 2 && "Only know how to promote the mask!");
1197  EVT DataVT = N->getValueType(0);
1198  SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1199  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1200  NewOps[OpNo] = Mask;
1201  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1202 }
1203 
1204 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1205  unsigned OpNo) {
1206 
1207  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1208  if (OpNo == 2) {
1209  // The Mask
1210  EVT DataVT = N->getValueType(0);
1211  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1212  } else
1213  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1214 
1215  SDValue Res = SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1216  // updated in place.
1217  if (Res.getNode() == N)
1218  return Res;
1219 
1220  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
1221  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1222  return SDValue();
1223 }
1224 
1225 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1226  unsigned OpNo) {
1227  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1228  if (OpNo == 2) {
1229  // The Mask
1230  EVT DataVT = N->getValue().getValueType();
1231  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1232  } else
1233  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1234  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1235 }
1236 
1237 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
1238  SDValue Op = GetPromotedInteger(N->getOperand(0));
1239  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
1240 }
1241 
1242 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
1243  return SDValue(DAG.UpdateNodeOperands(N,
1244  ZExtPromotedInteger(N->getOperand(0))), 0);
1245 }
1246 
1247 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
1248  SDLoc dl(N);
1249  SDValue Op = GetPromotedInteger(N->getOperand(0));
1250  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1251  return DAG.getZeroExtendInReg(Op, dl,
1253 }
1254 
1255 
1256 //===----------------------------------------------------------------------===//
1257 // Integer Result Expansion
1258 //===----------------------------------------------------------------------===//
1259 
1260 /// ExpandIntegerResult - This method is called when the specified result of the
1261 /// specified node is found to need expansion. At this point, the node may also
1262 /// have invalid operands or may have other results that need promotion, we just
1263 /// know that (at least) one result needs expansion.
1264 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
1265  DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); dbgs() << "\n");
1266  SDValue Lo, Hi;
1267  Lo = Hi = SDValue();
1268 
1269  // See if the target wants to custom expand this node.
1270  if (CustomLowerNode(N, N->getValueType(ResNo), true))
1271  return;
1272 
1273  switch (N->getOpcode()) {
1274  default:
1275 #ifndef NDEBUG
1276  dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
1277  N->dump(&DAG); dbgs() << "\n";
1278 #endif
1279  llvm_unreachable("Do not know how to expand the result of this operator!");
1280 
1281  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1282  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
1283  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1284  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1285 
1286  case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1287  case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1288  case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1289  case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1290  case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1291 
1292  case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
1293  case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
1294  case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
1295  case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
1296  case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
1297  case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
1298  case ISD::CTLZ_ZERO_UNDEF:
1299  case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
1300  case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
1301  case ISD::CTTZ_ZERO_UNDEF:
1302  case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
1303  case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
1304  case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
1305  case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
1306  case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
1307  case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
1308  case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
1309  case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
1310  case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
1311  case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
1312  case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
1313  case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
1314  case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
1315  case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
1316  case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
1317  case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
1318 
1319  case ISD::ATOMIC_LOAD_ADD:
1320  case ISD::ATOMIC_LOAD_SUB:
1321  case ISD::ATOMIC_LOAD_AND:
1322  case ISD::ATOMIC_LOAD_OR:
1323  case ISD::ATOMIC_LOAD_XOR:
1324  case ISD::ATOMIC_LOAD_NAND:
1325  case ISD::ATOMIC_LOAD_MIN:
1326  case ISD::ATOMIC_LOAD_MAX:
1327  case ISD::ATOMIC_LOAD_UMIN:
1328  case ISD::ATOMIC_LOAD_UMAX:
1329  case ISD::ATOMIC_SWAP:
1330  case ISD::ATOMIC_CMP_SWAP: {
1331  std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
1332  SplitInteger(Tmp.first, Lo, Hi);
1333  ReplaceValueWith(SDValue(N, 1), Tmp.second);
1334  break;
1335  }
1337  AtomicSDNode *AN = cast<AtomicSDNode>(N);
1338  SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
1339  SDValue Tmp = DAG.getAtomicCmpSwap(
1340  ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
1341  N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
1342  AN->getMemOperand());
1343 
1344  // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
1345  // success simply by comparing the loaded value against the ingoing
1346  // comparison.
1347  SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
1348  N->getOperand(2), ISD::SETEQ);
1349 
1350  SplitInteger(Tmp, Lo, Hi);
1351  ReplaceValueWith(SDValue(N, 1), Success);
1352  ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
1353  break;
1354  }
1355 
1356  case ISD::AND:
1357  case ISD::OR:
1358  case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
1359 
1360  case ISD::UMAX:
1361  case ISD::SMAX:
1362  case ISD::UMIN:
1363  case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
1364 
1365  case ISD::ADD:
1366  case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
1367 
1368  case ISD::ADDC:
1369  case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
1370 
1371  case ISD::ADDE:
1372  case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
1373 
1374  case ISD::SHL:
1375  case ISD::SRA:
1376  case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
1377 
1378  case ISD::SADDO:
1379  case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
1380  case ISD::UADDO:
1381  case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
1382  case ISD::UMULO:
1383  case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
1384  }
1385 
1386  // If Lo/Hi is null, the sub-method took care of registering results etc.
1387  if (Lo.getNode())
1388  SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
1389 }
1390 
1391 /// Lower an atomic node to the appropriate builtin call.
1392 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
1393  unsigned Opc = Node->getOpcode();
1394  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
1395  RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
1396  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
1397 
1398  return ExpandChainLibCall(LC, Node, false);
1399 }
1400 
1401 /// N is a shift by a value that needs to be expanded,
1402 /// and the shift amount is a constant 'Amt'. Expand the operation.
1403 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
1404  SDValue &Lo, SDValue &Hi) {
1405  SDLoc DL(N);
1406  // Expand the incoming operand to be shifted, so that we have its parts
1407  SDValue InL, InH;
1408  GetExpandedInteger(N->getOperand(0), InL, InH);
1409 
1410  // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
1411  // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
1412  if (!Amt) {
1413  Lo = InL;
1414  Hi = InH;
1415  return;
1416  }
1417 
1418  EVT NVT = InL.getValueType();
1419  unsigned VTBits = N->getValueType(0).getSizeInBits();
1420  unsigned NVTBits = NVT.getSizeInBits();
1421  EVT ShTy = N->getOperand(1).getValueType();
1422 
1423  if (N->getOpcode() == ISD::SHL) {
1424  if (Amt.ugt(VTBits)) {
1425  Lo = Hi = DAG.getConstant(0, DL, NVT);
1426  } else if (Amt.ugt(NVTBits)) {
1427  Lo = DAG.getConstant(0, DL, NVT);
1428  Hi = DAG.getNode(ISD::SHL, DL,
1429  NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1430  } else if (Amt == NVTBits) {
1431  Lo = DAG.getConstant(0, DL, NVT);
1432  Hi = InL;
1433  } else {
1434  Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
1435  Hi = DAG.getNode(ISD::OR, DL, NVT,
1436  DAG.getNode(ISD::SHL, DL, NVT, InH,
1437  DAG.getConstant(Amt, DL, ShTy)),
1438  DAG.getNode(ISD::SRL, DL, NVT, InL,
1439  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1440  }
1441  return;
1442  }
1443 
1444  if (N->getOpcode() == ISD::SRL) {
1445  if (Amt.ugt(VTBits)) {
1446  Lo = Hi = DAG.getConstant(0, DL, NVT);
1447  } else if (Amt.ugt(NVTBits)) {
1448  Lo = DAG.getNode(ISD::SRL, DL,
1449  NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1450  Hi = DAG.getConstant(0, DL, NVT);
1451  } else if (Amt == NVTBits) {
1452  Lo = InH;
1453  Hi = DAG.getConstant(0, DL, NVT);
1454  } else {
1455  Lo = DAG.getNode(ISD::OR, DL, NVT,
1456  DAG.getNode(ISD::SRL, DL, NVT, InL,
1457  DAG.getConstant(Amt, DL, ShTy)),
1458  DAG.getNode(ISD::SHL, DL, NVT, InH,
1459  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1460  Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1461  }
1462  return;
1463  }
1464 
1465  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
1466  if (Amt.ugt(VTBits)) {
1467  Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1468  DAG.getConstant(NVTBits - 1, DL, ShTy));
1469  } else if (Amt.ugt(NVTBits)) {
1470  Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1471  DAG.getConstant(Amt - NVTBits, DL, ShTy));
1472  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1473  DAG.getConstant(NVTBits - 1, DL, ShTy));
1474  } else if (Amt == NVTBits) {
1475  Lo = InH;
1476  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1477  DAG.getConstant(NVTBits - 1, DL, ShTy));
1478  } else {
1479  Lo = DAG.getNode(ISD::OR, DL, NVT,
1480  DAG.getNode(ISD::SRL, DL, NVT, InL,
1481  DAG.getConstant(Amt, DL, ShTy)),
1482  DAG.getNode(ISD::SHL, DL, NVT, InH,
1483  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1484  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1485  }
1486 }
1487 
1488 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
1489 /// this shift based on knowledge of the high bit of the shift amount. If we
1490 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
1491 /// shift amount.
1492 bool DAGTypeLegalizer::
1493 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1494  SDValue Amt = N->getOperand(1);
1495  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1496  EVT ShTy = Amt.getValueType();
1497  unsigned ShBits = ShTy.getScalarSizeInBits();
1498  unsigned NVTBits = NVT.getScalarSizeInBits();
1499  assert(isPowerOf2_32(NVTBits) &&
1500  "Expanded integer type size not a power of two!");
1501  SDLoc dl(N);
1502 
1503  APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
1504  APInt KnownZero, KnownOne;
1505  DAG.computeKnownBits(N->getOperand(1), KnownZero, KnownOne);
1506 
1507  // If we don't know anything about the high bits, exit.
1508  if (((KnownZero|KnownOne) & HighBitMask) == 0)
1509  return false;
1510 
1511  // Get the incoming operand to be shifted.
1512  SDValue InL, InH;
1513  GetExpandedInteger(N->getOperand(0), InL, InH);
1514 
1515  // If we know that any of the high bits of the shift amount are one, then we
1516  // can do this as a couple of simple shifts.
1517  if (KnownOne.intersects(HighBitMask)) {
1518  // Mask out the high bit, which we know is set.
1519  Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
1520  DAG.getConstant(~HighBitMask, dl, ShTy));
1521 
1522  switch (N->getOpcode()) {
1523  default: llvm_unreachable("Unknown shift");
1524  case ISD::SHL:
1525  Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
1526  Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
1527  return true;
1528  case ISD::SRL:
1529  Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1530  Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
1531  return true;
1532  case ISD::SRA:
1533  Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
1534  DAG.getConstant(NVTBits - 1, dl, ShTy));
1535  Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
1536  return true;
1537  }
1538  }
1539 
1540  // If we know that all of the high bits of the shift amount are zero, then we
1541  // can do this as a couple of simple shifts.
1542  if ((KnownZero & HighBitMask) == HighBitMask) {
1543  // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
1544  // shift if x is zero. We can use XOR here because x is known to be smaller
1545  // than 32.
1546  SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
1547  DAG.getConstant(NVTBits - 1, dl, ShTy));
1548 
1549  unsigned Op1, Op2;
1550  switch (N->getOpcode()) {
1551  default: llvm_unreachable("Unknown shift");
1552  case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
1553  case ISD::SRL:
1554  case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
1555  }
1556 
1557  // When shifting right the arithmetic for Lo and Hi is swapped.
1558  if (N->getOpcode() != ISD::SHL)
1559  std::swap(InL, InH);
1560 
1561  // Use a little trick to get the bits that move from Lo to Hi. First
1562  // shift by one bit.
1563  SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
1564  // Then compute the remaining shift with amount-1.
1565  SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
1566 
1567  Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
1568  Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
1569 
1570  if (N->getOpcode() != ISD::SHL)
1571  std::swap(Hi, Lo);
1572  return true;
1573  }
1574 
1575  return false;
1576 }
1577 
1578 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
1579 /// of any size.
1580 bool DAGTypeLegalizer::
1581 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1582  SDValue Amt = N->getOperand(1);
1583  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1584  EVT ShTy = Amt.getValueType();
1585  unsigned NVTBits = NVT.getSizeInBits();
1586  assert(isPowerOf2_32(NVTBits) &&
1587  "Expanded integer type size not a power of two!");
1588  SDLoc dl(N);
1589 
1590  // Get the incoming operand to be shifted.
1591  SDValue InL, InH;
1592  GetExpandedInteger(N->getOperand(0), InL, InH);
1593 
1594  SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
1595  SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
1596  SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
1597  SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1598  Amt, NVBitsNode, ISD::SETULT);
1599  SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1600  Amt, DAG.getConstant(0, dl, ShTy),
1601  ISD::SETEQ);
1602 
1603  SDValue LoS, HiS, LoL, HiL;
1604  switch (N->getOpcode()) {
1605  default: llvm_unreachable("Unknown shift");
1606  case ISD::SHL:
1607  // Short: ShAmt < NVTBits
1608  LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
1609  HiS = DAG.getNode(ISD::OR, dl, NVT,
1610  DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
1611  DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
1612 
1613  // Long: ShAmt >= NVTBits
1614  LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
1615  HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
1616 
1617  Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
1618  Hi = DAG.getSelect(dl, NVT, isZero, InH,
1619  DAG.getSelect(dl, NVT, isShort, HiS, HiL));
1620  return true;
1621  case ISD::SRL:
1622  // Short: ShAmt < NVTBits
1623  HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
1624  LoS = DAG.getNode(ISD::OR, dl, NVT,
1625  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1626  // FIXME: If Amt is zero, the following shift generates an undefined result
1627  // on some architectures.
1628  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1629 
1630  // Long: ShAmt >= NVTBits
1631  HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1632  LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1633 
1634  Lo = DAG.getSelect(dl, NVT, isZero, InL,
1635  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1636  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1637  return true;
1638  case ISD::SRA:
1639  // Short: ShAmt < NVTBits
1640  HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
1641  LoS = DAG.getNode(ISD::OR, dl, NVT,
1642  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1643  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1644 
1645  // Long: ShAmt >= NVTBits
1646  HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
1647  DAG.getConstant(NVTBits - 1, dl, ShTy));
1648  LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1649 
1650  Lo = DAG.getSelect(dl, NVT, isZero, InL,
1651  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1652  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1653  return true;
1654  }
1655 }
1656 
1657 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
1658 
1659  switch (Op) {
1660  default: llvm_unreachable("invalid min/max opcode");
1661  case ISD::SMAX:
1662  return std::make_pair(ISD::SETGT, ISD::UMAX);
1663  case ISD::UMAX:
1664  return std::make_pair(ISD::SETUGT, ISD::UMAX);
1665  case ISD::SMIN:
1666  return std::make_pair(ISD::SETLT, ISD::UMIN);
1667  case ISD::UMIN:
1668  return std::make_pair(ISD::SETULT, ISD::UMIN);
1669  }
1670 }
1671 
1672 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
1673  SDValue &Lo, SDValue &Hi) {
1674  SDLoc DL(N);
1675  ISD::NodeType LoOpc;
1676  ISD::CondCode CondC;
1677  std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
1678 
1679  // Expand the subcomponents.
1680  SDValue LHSL, LHSH, RHSL, RHSH;
1681  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1682  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1683 
1684  // Value types
1685  EVT NVT = LHSL.getValueType();
1686  EVT CCT = getSetCCResultType(NVT);
1687 
1688  // Hi part is always the same op
1689  Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
1690 
1691  // We need to know whether to select Lo part that corresponds to 'winning'
1692  // Hi part or if Hi parts are equal.
1693  SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
1694  SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
1695 
1696  // Lo part corresponding to the 'winning' Hi part
1697  SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
1698 
1699  // Recursed Lo part if Hi parts are equal, this uses unsigned version
1700  SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
1701 
1702  Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
1703 }
1704 
1705 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
1706  SDValue &Lo, SDValue &Hi) {
1707  SDLoc dl(N);
1708  // Expand the subcomponents.
1709  SDValue LHSL, LHSH, RHSL, RHSH;
1710  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1711  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1712 
1713  EVT NVT = LHSL.getValueType();
1714  SDValue LoOps[2] = { LHSL, RHSL };
1715  SDValue HiOps[3] = { LHSH, RHSH };
1716 
1717  // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
1718  // them. TODO: Teach operation legalization how to expand unsupported
1719  // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
1720  // a carry of type MVT::Glue, but there doesn't seem to be any way to
1721  // generate a value of this type in the expanded code sequence.
1722  bool hasCarry =
1724  ISD::ADDC : ISD::SUBC,
1725  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1726 
1727  if (hasCarry) {
1728  SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
1729  if (N->getOpcode() == ISD::ADD) {
1730  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
1731  HiOps[2] = Lo.getValue(1);
1732  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
1733  } else {
1734  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
1735  HiOps[2] = Lo.getValue(1);
1736  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
1737  }
1738  return;
1739  }
1740 
1741  bool hasOVF =
1744  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1745  if (hasOVF) {
1746  SDVTList VTList = DAG.getVTList(NVT, NVT);
1748  int RevOpc;
1749  if (N->getOpcode() == ISD::ADD) {
1750  RevOpc = ISD::SUB;
1751  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
1752  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
1753  } else {
1754  RevOpc = ISD::ADD;
1755  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
1756  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
1757  }
1758  SDValue OVF = Lo.getValue(1);
1759 
1760  switch (BoolType) {
1762  OVF = DAG.getNode(ISD::AND, dl, NVT, DAG.getConstant(1, dl, NVT), OVF);
1765  Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
1766  break;
1768  Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
1769  }
1770  return;
1771  }
1772 
1773  if (N->getOpcode() == ISD::ADD) {
1774  Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
1775  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
1776  SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
1777  ISD::SETULT);
1778  SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
1779  DAG.getConstant(1, dl, NVT),
1780  DAG.getConstant(0, dl, NVT));
1781  SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
1782  ISD::SETULT);
1783  SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
1784  DAG.getConstant(1, dl, NVT), Carry1);
1785  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
1786  } else {
1787  Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
1788  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
1789  SDValue Cmp =
1790  DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
1791  LoOps[0], LoOps[1], ISD::SETULT);
1792  SDValue Borrow = DAG.getSelect(dl, NVT, Cmp,
1793  DAG.getConstant(1, dl, NVT),
1794  DAG.getConstant(0, dl, NVT));
1795  Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
1796  }
1797 }
1798 
1799 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
1800  SDValue &Lo, SDValue &Hi) {
1801  // Expand the subcomponents.
1802  SDValue LHSL, LHSH, RHSL, RHSH;
1803  SDLoc dl(N);
1804  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1805  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1806  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
1807  SDValue LoOps[2] = { LHSL, RHSL };
1808  SDValue HiOps[3] = { LHSH, RHSH };
1809 
1810  if (N->getOpcode() == ISD::ADDC) {
1811  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
1812  HiOps[2] = Lo.getValue(1);
1813  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
1814  } else {
1815  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
1816  HiOps[2] = Lo.getValue(1);
1817  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
1818  }
1819 
1820  // Legalized the flag result - switch anything that used the old flag to
1821  // use the new one.
1822  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
1823 }
1824 
1825 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
1826  SDValue &Lo, SDValue &Hi) {
1827  // Expand the subcomponents.
1828  SDValue LHSL, LHSH, RHSL, RHSH;
1829  SDLoc dl(N);
1830  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1831  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1832  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
1833  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
1834  SDValue HiOps[3] = { LHSH, RHSH };
1835 
1836  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
1837  HiOps[2] = Lo.getValue(1);
1838  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
1839 
1840  // Legalized the flag result - switch anything that used the old flag to
1841  // use the new one.
1842  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
1843 }
1844 
1845 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
1846  SDValue &Lo, SDValue &Hi) {
1847  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1848  SDLoc dl(N);
1849  SDValue Op = N->getOperand(0);
1850  if (Op.getValueType().bitsLE(NVT)) {
1851  // The low part is any extension of the input (which degenerates to a copy).
1852  Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
1853  Hi = DAG.getUNDEF(NVT); // The high part is undefined.
1854  } else {
1855  // For example, extension of an i48 to an i64. The operand type necessarily
1856  // promotes to the result type, so will end up being expanded too.
1857  assert(getTypeAction(Op.getValueType()) ==
1859  "Only know how to promote this result!");
1860  SDValue Res = GetPromotedInteger(Op);
1861  assert(Res.getValueType() == N->getValueType(0) &&
1862  "Operand over promoted?");
1863  // Split the promoted operand. This will simplify when it is expanded.
1864  SplitInteger(Res, Lo, Hi);
1865  }
1866 }
1867 
1868 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
1869  SDValue &Lo, SDValue &Hi) {
1870  SDLoc dl(N);
1871  GetExpandedInteger(N->getOperand(0), Lo, Hi);
1872  EVT NVT = Lo.getValueType();
1873  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
1874  unsigned NVTBits = NVT.getSizeInBits();
1875  unsigned EVTBits = EVT.getSizeInBits();
1876 
1877  if (NVTBits < EVTBits) {
1878  Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
1880  EVTBits - NVTBits)));
1881  } else {
1882  Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
1883  // The high part replicates the sign bit of Lo, make it explicit.
1884  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
1885  DAG.getConstant(NVTBits - 1, dl,
1886  TLI.getPointerTy(DAG.getDataLayout())));
1887  }
1888 }
1889 
1890 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
1891  SDValue &Lo, SDValue &Hi) {
1892  SDLoc dl(N);
1893  GetExpandedInteger(N->getOperand(0), Lo, Hi);
1894  EVT NVT = Lo.getValueType();
1895  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
1896  unsigned NVTBits = NVT.getSizeInBits();
1897  unsigned EVTBits = EVT.getSizeInBits();
1898 
1899  if (NVTBits < EVTBits) {
1900  Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
1902  EVTBits - NVTBits)));
1903  } else {
1904  Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
1905  // The high part must be zero, make it explicit.
1906  Hi = DAG.getConstant(0, dl, NVT);
1907  }
1908 }
1909 
1910 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
1911  SDValue &Lo, SDValue &Hi) {
1912  SDLoc dl(N);
1913  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
1914  Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
1915  Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
1916 }
1917 
1918 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
1919  SDValue &Lo, SDValue &Hi) {
1920  SDLoc dl(N);
1921  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
1922  Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
1923  Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
1924 }
1925 
1926 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
1927  SDValue &Lo, SDValue &Hi) {
1928  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1929  unsigned NBitWidth = NVT.getSizeInBits();
1930  auto Constant = cast<ConstantSDNode>(N);
1931  const APInt &Cst = Constant->getAPIntValue();
1932  bool IsTarget = Constant->isTargetOpcode();
1933  bool IsOpaque = Constant->isOpaque();
1934  SDLoc dl(N);
1935  Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
1936  Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
1937  IsOpaque);
1938 }
1939 
1940 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
1941  SDValue &Lo, SDValue &Hi) {
1942  SDLoc dl(N);
1943  // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
1944  GetExpandedInteger(N->getOperand(0), Lo, Hi);
1945  EVT NVT = Lo.getValueType();
1946 
1947  SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
1948  DAG.getConstant(0, dl, NVT), ISD::SETNE);
1949 
1950  SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
1951  SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
1952 
1953  Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
1954  DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
1955  DAG.getConstant(NVT.getSizeInBits(), dl,
1956  NVT)));
1957  Hi = DAG.getConstant(0, dl, NVT);
1958 }
1959 
1960 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
1961  SDValue &Lo, SDValue &Hi) {
1962  SDLoc dl(N);
1963  // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
1964  GetExpandedInteger(N->getOperand(0), Lo, Hi);
1965  EVT NVT = Lo.getValueType();
1966  Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
1967  DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
1968  Hi = DAG.getConstant(0, dl, NVT);
1969 }
1970 
1971 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
1972  SDValue &Lo, SDValue &Hi) {
1973  SDLoc dl(N);
1974  // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
1975  GetExpandedInteger(N->getOperand(0), Lo, Hi);
1976  EVT NVT = Lo.getValueType();
1977 
1978  SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
1979  DAG.getConstant(0, dl, NVT), ISD::SETNE);
1980 
1981  SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
1982  SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
1983 
1984  Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
1985  DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
1986  DAG.getConstant(NVT.getSizeInBits(), dl,
1987  NVT)));
1988  Hi = DAG.getConstant(0, dl, NVT);
1989 }
1990 
1991 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
1992  SDValue &Hi) {
1993  SDLoc dl(N);
1994  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1995  unsigned NBitWidth = NVT.getSizeInBits();
1996 
1997  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
1998  Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT);
1999  // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
2000  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2001  DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
2002 }
2003 
2004 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
2005  SDValue &Hi) {
2006  SDLoc dl(N);
2007  EVT VT = N->getValueType(0);
2008 
2009  SDValue Op = N->getOperand(0);
2010  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2011  Op = GetPromotedFloat(Op);
2012 
2014  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
2015  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, true/*irrelevant*/, dl).first,
2016  Lo, Hi);
2017 }
2018 
2019 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
2020  SDValue &Hi) {
2021  SDLoc dl(N);
2022  EVT VT = N->getValueType(0);
2023 
2024  SDValue Op = N->getOperand(0);
2025  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2026  Op = GetPromotedFloat(Op);
2027 
2029  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
2030  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, false/*irrelevant*/, dl).first,
2031  Lo, Hi);
2032 }
2033 
2034 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
2035  SDValue &Lo, SDValue &Hi) {
2036  if (ISD::isNormalLoad(N)) {
2037  ExpandRes_NormalLoad(N, Lo, Hi);
2038  return;
2039  }
2040 
2041  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2042 
2043  EVT VT = N->getValueType(0);
2044  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2045  SDValue Ch = N->getChain();
2046  SDValue Ptr = N->getBasePtr();
2047  ISD::LoadExtType ExtType = N->getExtensionType();
2048  unsigned Alignment = N->getAlignment();
2049  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2050  AAMDNodes AAInfo = N->getAAInfo();
2051  SDLoc dl(N);
2052 
2053  assert(NVT.isByteSized() && "Expanded type not byte sized!");
2054 
2055  if (N->getMemoryVT().bitsLE(NVT)) {
2056  EVT MemVT = N->getMemoryVT();
2057 
2058  Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
2059  Alignment, MMOFlags, AAInfo);
2060 
2061  // Remember the chain.
2062  Ch = Lo.getValue(1);
2063 
2064  if (ExtType == ISD::SEXTLOAD) {
2065  // The high part is obtained by SRA'ing all but one of the bits of the
2066  // lo part.
2067  unsigned LoSize = Lo.getValueSizeInBits();
2068  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2069  DAG.getConstant(LoSize - 1, dl,
2070  TLI.getPointerTy(DAG.getDataLayout())));
2071  } else if (ExtType == ISD::ZEXTLOAD) {
2072  // The high part is just a zero.
2073  Hi = DAG.getConstant(0, dl, NVT);
2074  } else {
2075  assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
2076  // The high part is undefined.
2077  Hi = DAG.getUNDEF(NVT);
2078  }
2079  } else if (DAG.getDataLayout().isLittleEndian()) {
2080  // Little-endian - low bits are at low addresses.
2081  Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2082  AAInfo);
2083 
2084  unsigned ExcessBits =
2085  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
2086  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
2087 
2088  // Increment the pointer to the other half.
2089  unsigned IncrementSize = NVT.getSizeInBits()/8;
2090  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2091  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2092  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
2093  N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
2094  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2095 
2096  // Build a factor node to remember that this load is independent of the
2097  // other one.
2098  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2099  Hi.getValue(1));
2100  } else {
2101  // Big-endian - high bits are at low addresses. Favor aligned loads at
2102  // the cost of some bit-fiddling.
2103  EVT MemVT = N->getMemoryVT();
2104  unsigned EBytes = MemVT.getStoreSize();
2105  unsigned IncrementSize = NVT.getSizeInBits()/8;
2106  unsigned ExcessBits = (EBytes - IncrementSize)*8;
2107 
2108  // Load both the high bits and maybe some of the low bits.
2109  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
2111  MemVT.getSizeInBits() - ExcessBits),
2112  Alignment, MMOFlags, AAInfo);
2113 
2114  // Increment the pointer to the other half.
2115  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2116  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2117  // Load the rest of the low bits.
2118  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
2119  N->getPointerInfo().getWithOffset(IncrementSize),
2120  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
2121  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2122 
2123  // Build a factor node to remember that this load is independent of the
2124  // other one.
2125  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2126  Hi.getValue(1));
2127 
2128  if (ExcessBits < NVT.getSizeInBits()) {
2129  // Transfer low bits from the bottom of Hi to the top of Lo.
2130  Lo = DAG.getNode(
2131  ISD::OR, dl, NVT, Lo,
2132  DAG.getNode(ISD::SHL, dl, NVT, Hi,
2133  DAG.getConstant(ExcessBits, dl,
2134  TLI.getPointerTy(DAG.getDataLayout()))));
2135  // Move high bits to the right position in Hi.
2136  Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
2137  Hi,
2138  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
2139  TLI.getPointerTy(DAG.getDataLayout())));
2140  }
2141  }
2142 
2143  // Legalize the chain result - switch anything that used the old chain to
2144  // use the new one.
2145  ReplaceValueWith(SDValue(N, 1), Ch);
2146 }
2147 
2148 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
2149  SDValue &Lo, SDValue &Hi) {
2150  SDLoc dl(N);
2151  SDValue LL, LH, RL, RH;
2152  GetExpandedInteger(N->getOperand(0), LL, LH);
2153  GetExpandedInteger(N->getOperand(1), RL, RH);
2154  Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
2155  Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
2156 }
2157 
2158 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
2159  SDValue &Lo, SDValue &Hi) {
2160  EVT VT = N->getValueType(0);
2161  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2162  SDLoc dl(N);
2163 
2164  SDValue LL, LH, RL, RH;
2165  GetExpandedInteger(N->getOperand(0), LL, LH);
2166  GetExpandedInteger(N->getOperand(1), RL, RH);
2167 
2168  if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
2170  LL, LH, RL, RH))
2171  return;
2172 
2173  // If nothing else, we can make a libcall.
2175  if (VT == MVT::i16)
2176  LC = RTLIB::MUL_I16;
2177  else if (VT == MVT::i32)
2178  LC = RTLIB::MUL_I32;
2179  else if (VT == MVT::i64)
2180  LC = RTLIB::MUL_I64;
2181  else if (VT == MVT::i128)
2182  LC = RTLIB::MUL_I128;
2183 
2184  if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
2185  // We'll expand the multiplication by brute force because we have no other
2186  // options. This is a trivially-generalized version of the code from
2187  // Hacker's Delight (itself derived from Knuth's Algorithm M from section
2188  // 4.3.1).
2189  unsigned Bits = NVT.getSizeInBits();
2190  unsigned HalfBits = Bits >> 1;
2191  SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
2192  NVT);
2193  SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
2194  SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
2195 
2196  SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
2197  SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
2198 
2199  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2200  if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
2201  // The type from TLI is too small to fit the shift amount we want.
2202  // Override it with i32. The shift will have to be legalized.
2203  ShiftAmtTy = MVT::i32;
2204  }
2205  SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
2206  SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
2207  SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
2208  SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
2209 
2210  SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
2211  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
2212  SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
2213  SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
2214 
2215  SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
2216  DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
2217  SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
2218 
2219  SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
2220  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
2221  DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
2222  Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
2223  DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
2224 
2225  Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
2226  DAG.getNode(ISD::ADD, dl, NVT,
2227  DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
2228  DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
2229  return;
2230  }
2231 
2232  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2233  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true/*irrelevant*/, dl).first,
2234  Lo, Hi);
2235 }
2236 
2237 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
2238  SDValue &Hi) {
2239  SDLoc DL(N);
2240  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2241  SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
2242  SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
2243  Lo = R.getValue(0);
2244  Hi = R.getValue(1);
2245  ReplaceValueWith(SDValue(N, 1), R.getValue(2));
2246 }
2247 
2248 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
2249  SDValue &Lo, SDValue &Hi) {
2250  SDValue LHS = Node->getOperand(0);
2251  SDValue RHS = Node->getOperand(1);
2252  SDLoc dl(Node);
2253 
2254  // Expand the result by simply replacing it with the equivalent
2255  // non-overflow-checking operation.
2256  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
2257  ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
2258  LHS, RHS);
2259  SplitInteger(Sum, Lo, Hi);
2260 
2261  // Compute the overflow.
2262  //
2263  // LHSSign -> LHS >= 0
2264  // RHSSign -> RHS >= 0
2265  // SumSign -> Sum >= 0
2266  //
2267  // Add:
2268  // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
2269  // Sub:
2270  // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
2271  //
2272  EVT OType = Node->getValueType(1);
2273  SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
2274 
2275  SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
2276  SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
2277  SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
2278  Node->getOpcode() == ISD::SADDO ?
2280 
2281  SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
2282  SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
2283 
2284  SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
2285 
2286  // Use the calculated overflow everywhere.
2287  ReplaceValueWith(SDValue(Node, 1), Cmp);
2288 }
2289 
2290 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
2291  SDValue &Lo, SDValue &Hi) {
2292  EVT VT = N->getValueType(0);
2293  SDLoc dl(N);
2294  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2295 
2297  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2298  SplitInteger(Res.getValue(0), Lo, Hi);
2299  return;
2300  }
2301 
2303  if (VT == MVT::i16)
2304  LC = RTLIB::SDIV_I16;
2305  else if (VT == MVT::i32)
2306  LC = RTLIB::SDIV_I32;
2307  else if (VT == MVT::i64)
2308  LC = RTLIB::SDIV_I64;
2309  else if (VT == MVT::i128)
2310  LC = RTLIB::SDIV_I128;
2311  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
2312 
2313  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
2314 }
2315 
2316 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
2317  SDValue &Lo, SDValue &Hi) {
2318  EVT VT = N->getValueType(0);
2319  SDLoc dl(N);
2320 
2321  // If we can emit an efficient shift operation, do so now. Check to see if
2322  // the RHS is a constant.
2323  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
2324  return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
2325 
2326  // If we can determine that the high bit of the shift is zero or one, even if
2327  // the low bits are variable, emit this shift in an optimized form.
2328  if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
2329  return;
2330 
2331  // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
2332  unsigned PartsOpc;
2333  if (N->getOpcode() == ISD::SHL) {
2334  PartsOpc = ISD::SHL_PARTS;
2335  } else if (N->getOpcode() == ISD::SRL) {
2336  PartsOpc = ISD::SRL_PARTS;
2337  } else {
2338  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2339  PartsOpc = ISD::SRA_PARTS;
2340  }
2341 
2342  // Next check to see if the target supports this SHL_PARTS operation or if it
2343  // will custom expand it.
2344  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2345  TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
2346  if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
2347  Action == TargetLowering::Custom) {
2348  // Expand the subcomponents.
2349  SDValue LHSL, LHSH;
2350  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2351  EVT VT = LHSL.getValueType();
2352 
2353  // If the shift amount operand is coming from a vector legalization it may
2354  // have an illegal type. Fix that first by casting the operand, otherwise
2355  // the new SHL_PARTS operation would need further legalization.
2356  SDValue ShiftOp = N->getOperand(1);
2357  EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2358  assert(ShiftTy.getScalarSizeInBits() >=
2359  Log2_32_Ceil(VT.getScalarSizeInBits()) &&
2360  "ShiftAmountTy is too small to cover the range of this type!");
2361  if (ShiftOp.getValueType() != ShiftTy)
2362  ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
2363 
2364  SDValue Ops[] = { LHSL, LHSH, ShiftOp };
2365  Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
2366  Hi = Lo.getValue(1);
2367  return;
2368  }
2369 
2370  // Otherwise, emit a libcall.
2372  bool isSigned;
2373  if (N->getOpcode() == ISD::SHL) {
2374  isSigned = false; /*sign irrelevant*/
2375  if (VT == MVT::i16)
2376  LC = RTLIB::SHL_I16;
2377  else if (VT == MVT::i32)
2378  LC = RTLIB::SHL_I32;
2379  else if (VT == MVT::i64)
2380  LC = RTLIB::SHL_I64;
2381  else if (VT == MVT::i128)
2382  LC = RTLIB::SHL_I128;
2383  } else if (N->getOpcode() == ISD::SRL) {
2384  isSigned = false;
2385  if (VT == MVT::i16)
2386  LC = RTLIB::SRL_I16;
2387  else if (VT == MVT::i32)
2388  LC = RTLIB::SRL_I32;
2389  else if (VT == MVT::i64)
2390  LC = RTLIB::SRL_I64;
2391  else if (VT == MVT::i128)
2392  LC = RTLIB::SRL_I128;
2393  } else {
2394  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2395  isSigned = true;
2396  if (VT == MVT::i16)
2397  LC = RTLIB::SRA_I16;
2398  else if (VT == MVT::i32)
2399  LC = RTLIB::SRA_I32;
2400  else if (VT == MVT::i64)
2401  LC = RTLIB::SRA_I64;
2402  else if (VT == MVT::i128)
2403  LC = RTLIB::SRA_I128;
2404  }
2405 
2406  if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
2407  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2408  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, isSigned, dl).first, Lo, Hi);
2409  return;
2410  }
2411 
2412  if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
2413  llvm_unreachable("Unsupported shift!");
2414 }
2415 
2416 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
2417  SDValue &Lo, SDValue &Hi) {
2418  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2419  SDLoc dl(N);
2420  SDValue Op = N->getOperand(0);
2421  if (Op.getValueType().bitsLE(NVT)) {
2422  // The low part is sign extension of the input (degenerates to a copy).
2423  Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
2424  // The high part is obtained by SRA'ing all but one of the bits of low part.
2425  unsigned LoSize = NVT.getSizeInBits();
2426  Hi = DAG.getNode(
2427  ISD::SRA, dl, NVT, Lo,
2428  DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
2429  } else {
2430  // For example, extension of an i48 to an i64. The operand type necessarily
2431  // promotes to the result type, so will end up being expanded too.
2432  assert(getTypeAction(Op.getValueType()) ==
2434  "Only know how to promote this result!");
2435  SDValue Res = GetPromotedInteger(Op);
2436  assert(Res.getValueType() == N->getValueType(0) &&
2437  "Operand over promoted?");
2438  // Split the promoted operand. This will simplify when it is expanded.
2439  SplitInteger(Res, Lo, Hi);
2440  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
2441  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2443  ExcessBits)));
2444  }
2445 }
2446 
2447 void DAGTypeLegalizer::
2448 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2449  SDLoc dl(N);
2450  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2451  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2452 
2453  if (EVT.bitsLE(Lo.getValueType())) {
2454  // sext_inreg the low part if needed.
2455  Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
2456  N->getOperand(1));
2457 
2458  // The high part gets the sign extension from the lo-part. This handles
2459  // things like sextinreg V:i64 from i8.
2460  Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
2461  DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
2462  TLI.getPointerTy(DAG.getDataLayout())));
2463  } else {
2464  // For example, extension of an i48 to an i64. Leave the low part alone,
2465  // sext_inreg the high part.
2466  unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
2467  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2469  ExcessBits)));
2470  }
2471 }
2472 
2473 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
2474  SDValue &Lo, SDValue &Hi) {
2475  EVT VT = N->getValueType(0);
2476  SDLoc dl(N);
2477  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2478 
2480  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2481  SplitInteger(Res.getValue(1), Lo, Hi);
2482  return;
2483  }
2484 
2486  if (VT == MVT::i16)
2487  LC = RTLIB::SREM_I16;
2488  else if (VT == MVT::i32)
2489  LC = RTLIB::SREM_I32;
2490  else if (VT == MVT::i64)
2491  LC = RTLIB::SREM_I64;
2492  else if (VT == MVT::i128)
2493  LC = RTLIB::SREM_I128;
2494  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
2495 
2496  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
2497 }
2498 
2499 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
2500  SDValue &Lo, SDValue &Hi) {
2501  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2502  SDLoc dl(N);
2503  Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
2504  Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
2505  N->getOperand(0),
2506  DAG.getConstant(NVT.getSizeInBits(), dl,
2507  TLI.getPointerTy(DAG.getDataLayout())));
2508  Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
2509 }
2510 
2511 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
2512  SDValue &Lo, SDValue &Hi) {
2513  SDValue LHS = N->getOperand(0);
2514  SDValue RHS = N->getOperand(1);
2515  SDLoc dl(N);
2516 
2517  // Expand the result by simply replacing it with the equivalent
2518  // non-overflow-checking operation.
2519  SDValue Sum = DAG.getNode(N->getOpcode() == ISD::UADDO ?
2520  ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
2521  LHS, RHS);
2522  SplitInteger(Sum, Lo, Hi);
2523 
2524  // Calculate the overflow: addition overflows iff a + b < a, and subtraction
2525  // overflows iff a - b > a.
2526  SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS,
2527  N->getOpcode () == ISD::UADDO ?
2529 
2530  // Use the calculated overflow everywhere.
2531  ReplaceValueWith(SDValue(N, 1), Ofl);
2532 }
2533 
2534 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
2535  SDValue &Lo, SDValue &Hi) {
2536  EVT VT = N->getValueType(0);
2537  SDLoc dl(N);
2538 
2539  // A divide for UMULO should be faster than a function call.
2540  if (N->getOpcode() == ISD::UMULO) {
2541  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
2542 
2543  SDValue MUL = DAG.getNode(ISD::MUL, dl, LHS.getValueType(), LHS, RHS);
2544  SplitInteger(MUL, Lo, Hi);
2545 
2546  // A divide for UMULO will be faster than a function call. Select to
2547  // make sure we aren't using 0.
2548  SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(VT),
2549  RHS, DAG.getConstant(0, dl, VT), ISD::SETEQ);
2550  SDValue NotZero = DAG.getSelect(dl, VT, isZero,
2551  DAG.getConstant(1, dl, VT), RHS);
2552  SDValue DIV = DAG.getNode(ISD::UDIV, dl, VT, MUL, NotZero);
2553  SDValue Overflow = DAG.getSetCC(dl, N->getValueType(1), DIV, LHS,
2554  ISD::SETNE);
2555  Overflow = DAG.getSelect(dl, N->getValueType(1), isZero,
2556  DAG.getConstant(0, dl, N->getValueType(1)),
2557  Overflow);
2558  ReplaceValueWith(SDValue(N, 1), Overflow);
2559  return;
2560  }
2561 
2562  Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
2563  EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
2564  Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
2565 
2566  // Replace this with a libcall that will check overflow.
2568  if (VT == MVT::i32)
2569  LC = RTLIB::MULO_I32;
2570  else if (VT == MVT::i64)
2571  LC = RTLIB::MULO_I64;
2572  else if (VT == MVT::i128)
2573  LC = RTLIB::MULO_I128;
2574  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
2575 
2576  SDValue Temp = DAG.CreateStackTemporary(PtrVT);
2577  // Temporary for the overflow value, default it to zero.
2578  SDValue Chain =
2579  DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
2580  MachinePointerInfo());
2581 
2584  for (const SDValue &Op : N->op_values()) {
2585  EVT ArgVT = Op.getValueType();
2586  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2587  Entry.Node = Op;
2588  Entry.Ty = ArgTy;
2589  Entry.isSExt = true;
2590  Entry.isZExt = false;
2591  Args.push_back(Entry);
2592  }
2593 
2594  // Also pass the address of the overflow check.
2595  Entry.Node = Temp;
2596  Entry.Ty = PtrTy->getPointerTo();
2597  Entry.isSExt = true;
2598  Entry.isZExt = false;
2599  Args.push_back(Entry);
2600 
2601  SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
2602 
2604  CLI.setDebugLoc(dl).setChain(Chain)
2605  .setCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
2606  .setSExtResult();
2607 
2608  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2609 
2610  SplitInteger(CallInfo.first, Lo, Hi);
2611  SDValue Temp2 =
2612  DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
2613  SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
2614  DAG.getConstant(0, dl, PtrVT),
2615  ISD::SETNE);
2616  // Use the overflow from the libcall everywhere.
2617  ReplaceValueWith(SDValue(N, 1), Ofl);
2618 }
2619 
2620 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
2621  SDValue &Lo, SDValue &Hi) {
2622  EVT VT = N->getValueType(0);
2623  SDLoc dl(N);
2624  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2625 
2627  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2628  SplitInteger(Res.getValue(0), Lo, Hi);
2629  return;
2630  }
2631 
2633  if (VT == MVT::i16)
2634  LC = RTLIB::UDIV_I16;
2635  else if (VT == MVT::i32)
2636  LC = RTLIB::UDIV_I32;
2637  else if (VT == MVT::i64)
2638  LC = RTLIB::UDIV_I64;
2639  else if (VT == MVT::i128)
2640  LC = RTLIB::UDIV_I128;
2641  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
2642 
2643  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
2644 }
2645 
2646 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
2647  SDValue &Lo, SDValue &Hi) {
2648  EVT VT = N->getValueType(0);
2649  SDLoc dl(N);
2650  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2651 
2653  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2654  SplitInteger(Res.getValue(1), Lo, Hi);
2655  return;
2656  }
2657 
2659  if (VT == MVT::i16)
2660  LC = RTLIB::UREM_I16;
2661  else if (VT == MVT::i32)
2662  LC = RTLIB::UREM_I32;
2663  else if (VT == MVT::i64)
2664  LC = RTLIB::UREM_I64;
2665  else if (VT == MVT::i128)
2666  LC = RTLIB::UREM_I128;
2667  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
2668 
2669  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
2670 }
2671 
2672 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
2673  SDValue &Lo, SDValue &Hi) {
2674  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2675  SDLoc dl(N);
2676  SDValue Op = N->getOperand(0);
2677  if (Op.getValueType().bitsLE(NVT)) {
2678  // The low part is zero extension of the input (degenerates to a copy).
2679  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
2680  Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
2681  } else {
2682  // For example, extension of an i48 to an i64. The operand type necessarily
2683  // promotes to the result type, so will end up being expanded too.
2684  assert(getTypeAction(Op.getValueType()) ==
2686  "Only know how to promote this result!");
2687  SDValue Res = GetPromotedInteger(Op);
2688  assert(Res.getValueType() == N->getValueType(0) &&
2689  "Operand over promoted?");
2690  // Split the promoted operand. This will simplify when it is expanded.
2691  SplitInteger(Res, Lo, Hi);
2692  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
2693  Hi = DAG.getZeroExtendInReg(Hi, dl,
2695  ExcessBits));
2696  }
2697 }
2698 
2699 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
2700  SDValue &Lo, SDValue &Hi) {
2701  SDLoc dl(N);
2702  EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
2703  SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
2704  SDValue Zero = DAG.getConstant(0, dl, VT);
2705  SDValue Swap = DAG.getAtomicCmpSwap(
2707  cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
2708  N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
2709 
2710  ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
2711  ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
2712 }
2713 
2714 //===----------------------------------------------------------------------===//
2715 // Integer Operand Expansion
2716 //===----------------------------------------------------------------------===//
2717 
2718 /// ExpandIntegerOperand - This method is called when the specified operand of
2719 /// the specified node is found to need expansion. At this point, all of the
2720 /// result types of the node are known to be legal, but other operands of the
2721 /// node may need promotion or expansion as well as the specified one.
2722 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
2723  DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); dbgs() << "\n");
2724  SDValue Res = SDValue();
2725 
2726  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2727  return false;
2728 
2729  switch (N->getOpcode()) {
2730  default:
2731  #ifndef NDEBUG
2732  dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
2733  N->dump(&DAG); dbgs() << "\n";
2734  #endif
2735  llvm_unreachable("Do not know how to expand this operator's operand!");
2736 
2737  case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
2738  case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
2739  case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
2740  case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
2741  case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
2742  case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
2743  case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
2744  case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
2745  case ISD::SETCCE: Res = ExpandIntOp_SETCCE(N); break;
2746  case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break;
2747  case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
2748  case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
2749  case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break;
2750 
2751  case ISD::SHL:
2752  case ISD::SRA:
2753  case ISD::SRL:
2754  case ISD::ROTL:
2755  case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
2756  case ISD::RETURNADDR:
2757  case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
2758 
2759  case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
2760  }
2761 
2762  // If the result is null, the sub-method took care of registering results etc.
2763  if (!Res.getNode()) return false;
2764 
2765  // If the result is N, the sub-method updated N in place. Tell the legalizer
2766  // core about this.
2767  if (Res.getNode() == N)
2768  return true;
2769 
2770  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2771  "Invalid operand expansion");
2772 
2773  ReplaceValueWith(SDValue(N, 0), Res);
2774  return false;
2775 }
2776 
2777 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
2778 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
2779 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
2780  SDValue &NewRHS,
2781  ISD::CondCode &CCCode,
2782  const SDLoc &dl) {
2783  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2784  GetExpandedInteger(NewLHS, LHSLo, LHSHi);
2785  GetExpandedInteger(NewRHS, RHSLo, RHSHi);
2786 
2787  if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
2788  if (RHSLo == RHSHi) {
2789  if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
2790  if (RHSCST->isAllOnesValue()) {
2791  // Equality comparison to -1.
2792  NewLHS = DAG.getNode(ISD::AND, dl,
2793  LHSLo.getValueType(), LHSLo, LHSHi);
2794  NewRHS = RHSLo;
2795  return;
2796  }
2797  }
2798  }
2799 
2800  NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
2801  NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
2802  NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
2803  NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
2804  return;
2805  }
2806 
2807  // If this is a comparison of the sign bit, just look at the top part.
2808  // X > -1, x < 0
2809  if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
2810  if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0
2811  (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1
2812  NewLHS = LHSHi;
2813  NewRHS = RHSHi;
2814  return;
2815  }
2816 
2817  // FIXME: This generated code sucks.
2818  ISD::CondCode LowCC;
2819  switch (CCCode) {
2820  default: llvm_unreachable("Unknown integer setcc!");
2821  case ISD::SETLT:
2822  case ISD::SETULT: LowCC = ISD::SETULT; break;
2823  case ISD::SETGT:
2824  case ISD::SETUGT: LowCC = ISD::SETUGT; break;
2825  case ISD::SETLE:
2826  case ISD::SETULE: LowCC = ISD::SETULE; break;
2827  case ISD::SETGE:
2828  case ISD::SETUGE: LowCC = ISD::SETUGE; break;
2829  }
2830 
2831  // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
2832  // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
2833  // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
2834 
2835  // NOTE: on targets without efficient SELECT of bools, we can always use
2836  // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
2837  TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
2838  nullptr);
2839  SDValue LoCmp, HiCmp;
2840  if (TLI.isTypeLegal(LHSLo.getValueType()) &&
2841  TLI.isTypeLegal(RHSLo.getValueType()))
2842  LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
2843  RHSLo, LowCC, false, DagCombineInfo, dl);
2844  if (!LoCmp.getNode())
2845  LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
2846  RHSLo, LowCC);
2847  if (TLI.isTypeLegal(LHSHi.getValueType()) &&
2848  TLI.isTypeLegal(RHSHi.getValueType()))
2849  HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
2850  RHSHi, CCCode, false, DagCombineInfo, dl);
2851  if (!HiCmp.getNode())
2852  HiCmp =
2853  DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
2854  LHSHi, RHSHi, DAG.getCondCode(CCCode));
2855 
2856  ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
2857  ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
2858 
2859  bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
2860  CCCode == ISD::SETUGE || CCCode == ISD::SETULE);
2861 
2862  if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) ||
2863  (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) ||
2864  (LoCmpC && LoCmpC->isNullValue())))) {
2865  // For LE / GE, if high part is known false, ignore the low part.
2866  // For LT / GT: if low part is known false, return the high part.
2867  // if high part is known true, ignore the low part.
2868  NewLHS = HiCmp;
2869  NewRHS = SDValue();
2870  return;
2871  }
2872 
2873  if (LHSHi == RHSHi) {
2874  // Comparing the low bits is enough.
2875  NewLHS = LoCmp;
2876  NewRHS = SDValue();
2877  return;
2878  }
2879 
2880  // Lower with SETCCE if the target supports it.
2881  // FIXME: Make all targets support this, then remove the other lowering.
2882  if (TLI.getOperationAction(
2883  ISD::SETCCE,
2884  TLI.getTypeToExpandTo(*DAG.getContext(), LHSLo.getValueType())) ==
2886  // SETCCE can detect < and >= directly. For > and <=, flip operands and
2887  // condition code.
2888  bool FlipOperands = false;
2889  switch (CCCode) {
2890  case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
2891  case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
2892  case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
2893  case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
2894  default: break;
2895  }
2896  if (FlipOperands) {
2897  std::swap(LHSLo, RHSLo);
2898  std::swap(LHSHi, RHSHi);
2899  }
2900  // Perform a wide subtraction, feeding the carry from the low part into
2901  // SETCCE. The SETCCE operation is essentially looking at the high part of
2902  // the result of LHS - RHS. It is negative iff LHS < RHS. It is zero or
2903  // positive iff LHS >= RHS.
2904  SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), MVT::Glue);
2905  SDValue LowCmp = DAG.getNode(ISD::SUBC, dl, VTList, LHSLo, RHSLo);
2906  SDValue Res =
2907  DAG.getNode(ISD::SETCCE, dl, getSetCCResultType(LHSLo.getValueType()),
2908  LHSHi, RHSHi, LowCmp.getValue(1), DAG.getCondCode(CCCode));
2909  NewLHS = Res;
2910  NewRHS = SDValue();
2911  return;
2912  }
2913 
2914  NewLHS = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()),
2915  LHSHi, RHSHi, ISD::SETEQ, false,
2916  DagCombineInfo, dl);
2917  if (!NewLHS.getNode())
2918  NewLHS = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()),
2919  LHSHi, RHSHi, ISD::SETEQ);
2920  NewLHS = DAG.getSelect(dl, LoCmp.getValueType(),
2921  NewLHS, LoCmp, HiCmp);
2922  NewRHS = SDValue();
2923 }
2924 
2925 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
2926  SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
2927  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
2928  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
2929 
2930  // If ExpandSetCCOperands returned a scalar, we need to compare the result
2931  // against zero to select between true and false values.
2932  if (!NewRHS.getNode()) {
2933  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2934  CCCode = ISD::SETNE;
2935  }
2936 
2937  // Update N to have the operands specified.
2938  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2939  DAG.getCondCode(CCCode), NewLHS, NewRHS,
2940  N->getOperand(4)), 0);
2941 }
2942 
2943 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
2944  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
2945  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
2946  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
2947 
2948  // If ExpandSetCCOperands returned a scalar, we need to compare the result
2949  // against zero to select between true and false values.
2950  if (!NewRHS.getNode()) {
2951  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2952  CCCode = ISD::SETNE;
2953  }
2954 
2955  // Update N to have the operands specified.
2956  return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
2957  N->getOperand(2), N->getOperand(3),
2958  DAG.getCondCode(CCCode)), 0);
2959 }
2960 
2961 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
2962  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
2963  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2964  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
2965 
2966  // If ExpandSetCCOperands returned a scalar, use it.
2967  if (!NewRHS.getNode()) {
2968  assert(NewLHS.getValueType() == N->getValueType(0) &&
2969  "Unexpected setcc expansion!");
2970  return NewLHS;
2971  }
2972 
2973  // Otherwise, update N to have the operands specified.
2974  return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
2975  DAG.getCondCode(CCCode)), 0);
2976 }
2977 
2978 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCE(SDNode *N) {
2979  SDValue LHS = N->getOperand(0);
2980  SDValue RHS = N->getOperand(1);
2981  SDValue Carry = N->getOperand(2);
2982  SDValue Cond = N->getOperand(3);
2983  SDLoc dl = SDLoc(N);
2984 
2985  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2986  GetExpandedInteger(LHS, LHSLo, LHSHi);
2987  GetExpandedInteger(RHS, RHSLo, RHSHi);
2988 
2989  // Expand to a SUBE for the low part and a smaller SETCCE for the high.
2990  SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), MVT::Glue);
2991  SDValue LowCmp = DAG.getNode(ISD::SUBE, dl, VTList, LHSLo, RHSLo, Carry);
2992  return DAG.getNode(ISD::SETCCE, dl, N->getValueType(0), LHSHi, RHSHi,
2993  LowCmp.getValue(1), Cond);
2994 }
2995 
2996 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
2997  // The value being shifted is legal, but the shift amount is too big.
2998  // It follows that either the result of the shift is undefined, or the
2999  // upper half of the shift amount is zero. Just use the lower half.
3000  SDValue Lo, Hi;
3001  GetExpandedInteger(N->getOperand(1), Lo, Hi);
3002  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
3003 }
3004 
3005 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
3006  // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
3007  // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
3008  // constant to valid type.
3009  SDValue Lo, Hi;
3010  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3011  return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
3012 }
3013 
3014 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
3015  SDValue Op = N->getOperand(0);
3016  EVT DstVT = N->getValueType(0);
3019  "Don't know how to expand this SINT_TO_FP!");
3020  return TLI.makeLibCall(DAG, LC, DstVT, Op, true, SDLoc(N)).first;
3021 }
3022 
3023 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
3024  if (ISD::isNormalStore(N))
3025  return ExpandOp_NormalStore(N, OpNo);
3026 
3027  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
3028  assert(OpNo == 1 && "Can only expand the stored value so far");
3029 
3030  EVT VT = N->getOperand(1).getValueType();
3031  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3032  SDValue Ch = N->getChain();
3033  SDValue Ptr = N->getBasePtr();
3034  unsigned Alignment = N->getAlignment();
3035  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3036  AAMDNodes AAInfo = N->getAAInfo();
3037  SDLoc dl(N);
3038  SDValue Lo, Hi;
3039 
3040  assert(NVT.isByteSized() && "Expanded type not byte sized!");
3041 
3042  if (N->getMemoryVT().bitsLE(NVT)) {
3043  GetExpandedInteger(N->getValue(), Lo, Hi);
3044  return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
3045  N->getMemoryVT(), Alignment, MMOFlags, AAInfo);
3046  }
3047 
3048  if (DAG.getDataLayout().isLittleEndian()) {
3049  // Little-endian - low bits are at low addresses.
3050  GetExpandedInteger(N->getValue(), Lo, Hi);
3051 
3052  Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3053  AAInfo);
3054 
3055  unsigned ExcessBits =
3056  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3057  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3058 
3059  // Increment the pointer to the other half.
3060  unsigned IncrementSize = NVT.getSizeInBits()/8;
3061  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
3062  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
3063  Hi = DAG.getTruncStore(
3064  Ch, dl, Hi, Ptr, N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3065  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3066  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3067  }
3068 
3069  // Big-endian - high bits are at low addresses. Favor aligned stores at
3070  // the cost of some bit-fiddling.
3071  GetExpandedInteger(N->getValue(), Lo, Hi);
3072 
3073  EVT ExtVT = N->getMemoryVT();
3074  unsigned EBytes = ExtVT.getStoreSize();
3075  unsigned IncrementSize = NVT.getSizeInBits()/8;
3076  unsigned ExcessBits = (EBytes - IncrementSize)*8;
3077  EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
3078  ExtVT.getSizeInBits() - ExcessBits);
3079 
3080  if (ExcessBits < NVT.getSizeInBits()) {
3081  // Transfer high bits from the top of Lo to the bottom of Hi.
3082  Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
3083  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3084  TLI.getPointerTy(DAG.getDataLayout())));
3085  Hi = DAG.getNode(
3086  ISD::OR, dl, NVT, Hi,
3087  DAG.getNode(ISD::SRL, dl, NVT, Lo,
3088  DAG.getConstant(ExcessBits, dl,
3089  TLI.getPointerTy(DAG.getDataLayout()))));
3090  }
3091 
3092  // Store both the high bits and maybe some of the low bits.
3093  Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment,
3094  MMOFlags, AAInfo);
3095 
3096  // Increment the pointer to the other half.
3097  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
3098  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
3099  // Store the lowest ExcessBits bits in the second half.
3100  Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
3101  N->getPointerInfo().getWithOffset(IncrementSize),
3102  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3103  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3104  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3105 }
3106 
3107 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
3108  SDValue InL, InH;
3109  GetExpandedInteger(N->getOperand(0), InL, InH);
3110  // Just truncate the low part of the source.
3111  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
3112 }
3113 
3114 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
3115  SDValue Op = N->getOperand(0);
3116  EVT SrcVT = Op.getValueType();
3117  EVT DstVT = N->getValueType(0);
3118  SDLoc dl(N);
3119 
3120  // The following optimization is valid only if every value in SrcVT (when
3121  // treated as signed) is representable in DstVT. Check that the mantissa
3122  // size of DstVT is >= than the number of bits in SrcVT -1.
3123  const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT);
3124  if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 &&
3126  // Do a signed conversion then adjust the result.
3127  SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op);
3128  SignedConv = TLI.LowerOperation(SignedConv, DAG);
3129 
3130  // The result of the signed conversion needs adjusting if the 'sign bit' of
3131  // the incoming integer was set. To handle this, we dynamically test to see
3132  // if it is set, and, if so, add a fudge factor.
3133 
3134  const uint64_t F32TwoE32 = 0x4F800000ULL;
3135  const uint64_t F32TwoE64 = 0x5F800000ULL;
3136  const uint64_t F32TwoE128 = 0x7F800000ULL;
3137 
3138  APInt FF(32, 0);
3139  if (SrcVT == MVT::i32)
3140  FF = APInt(32, F32TwoE32);
3141  else if (SrcVT == MVT::i64)
3142  FF = APInt(32, F32TwoE64);
3143  else if (SrcVT == MVT::i128)
3144  FF = APInt(32, F32TwoE128);
3145  else
3146  llvm_unreachable("Unsupported UINT_TO_FP!");
3147 
3148  // Check whether the sign bit is set.
3149  SDValue Lo, Hi;
3150  GetExpandedInteger(Op, Lo, Hi);
3151  SDValue SignSet = DAG.getSetCC(dl,
3152  getSetCCResultType(Hi.getValueType()),
3153  Hi,
3154  DAG.getConstant(0, dl, Hi.getValueType()),
3155  ISD::SETLT);
3156 
3157  // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
3158  SDValue FudgePtr =
3159  DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)),
3160  TLI.getPointerTy(DAG.getDataLayout()));
3161 
3162  // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
3163  SDValue Zero = DAG.getIntPtrConstant(0, dl);
3164  SDValue Four = DAG.getIntPtrConstant(4, dl);
3165  if (DAG.getDataLayout().isBigEndian())
3166  std::swap(Zero, Four);
3167  SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet,
3168  Zero, Four);
3169  unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment();
3170  FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(),
3171  FudgePtr, Offset);
3172  Alignment = std::min(Alignment, 4u);
3173 
3174  // Load the value out, extending it from f32 to the destination float type.
3175  // FIXME: Avoid the extend by constructing the right constant pool?
3176  SDValue Fudge = DAG.getExtLoad(
3177  ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), FudgePtr,
3179  Alignment);
3180  return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge);
3181  }
3182 
3183  // Otherwise, use a libcall.
3184  RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT);
3186  "Don't know how to expand this UINT_TO_FP!");
3187  return TLI.makeLibCall(DAG, LC, DstVT, Op, true, dl).first;
3188 }
3189 
3190 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
3191  SDLoc dl(N);
3192  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
3193  cast<AtomicSDNode>(N)->getMemoryVT(),
3194  N->getOperand(0),
3195  N->getOperand(1), N->getOperand(2),
3196  cast<AtomicSDNode>(N)->getMemOperand());
3197  return Swap.getValue(1);
3198 }
3199 
3200 
3201 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
3202  SDValue InOp0 = N->getOperand(0);
3203  EVT InVT = InOp0.getValueType();
3204 
3205  EVT OutVT = N->getValueType(0);
3206  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3207  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3208  unsigned OutNumElems = OutVT.getVectorNumElements();
3209  EVT NOutVTElem = NOutVT.getVectorElementType();
3210 
3211  SDLoc dl(N);
3212  SDValue BaseIdx = N->getOperand(1);
3213 
3215  Ops.reserve(OutNumElems);
3216  for (unsigned i = 0; i != OutNumElems; ++i) {
3217 
3218  // Extract the element from the original vector.
3219  SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
3220  BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
3222  InVT.getVectorElementType(), N->getOperand(0), Index);
3223 
3224  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext);
3225  // Insert the converted element to the new vector.
3226  Ops.push_back(Op);
3227  }
3228 
3229  return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, Ops);
3230 }
3231 
3232 
3233 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
3234  ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
3235  EVT VT = N->getValueType(0);
3236  SDLoc dl(N);
3237 
3238  ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
3239 
3240  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3241  SDValue V1 = GetPromotedInteger(N->getOperand(1));
3242  EVT OutVT = V0.getValueType();
3243 
3244  return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
3245 }
3246 
3247 
3248 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
3249  EVT OutVT = N->getValueType(0);
3250  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3251  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3252  unsigned NumElems = N->getNumOperands();
3253  EVT NOutVTElem = NOutVT.getVectorElementType();
3254 
3255  SDLoc dl(N);
3256 
3258  Ops.reserve(NumElems);
3259  for (unsigned i = 0; i != NumElems; ++i) {
3260  SDValue Op;
3261  // BUILD_VECTOR integer operand types are allowed to be larger than the
3262  // result's element type. This may still be true after the promotion. For
3263  // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
3264  // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
3265  if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
3266  Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
3267  else
3268  Op = N->getOperand(i);
3269  Ops.push_back(Op);
3270  }
3271 
3272  return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, Ops);
3273 }
3274 
3275 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
3276 
3277  SDLoc dl(N);
3278 
3279  assert(!N->getOperand(0).getValueType().isVector() &&
3280  "Input must be a scalar");
3281 
3282  EVT OutVT = N->getValueType(0);
3283  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3284  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3285  EVT NOutVTElem = NOutVT.getVectorElementType();
3286 
3287  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
3288 
3289  return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
3290 }
3291 
3292 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
3293  SDLoc dl(N);
3294 
3295  EVT OutVT = N->getValueType(0);
3296  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3297  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3298 
3299  EVT InElemTy = OutVT.getVectorElementType();
3300  EVT OutElemTy = NOutVT.getVectorElementType();
3301 
3302  unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
3303  unsigned NumOutElem = NOutVT.getVectorNumElements();
3304  unsigned NumOperands = N->getNumOperands();
3305  assert(NumElem * NumOperands == NumOutElem &&
3306  "Unexpected number of elements");
3307 
3308  // Take the elements from the first vector.
3309  SmallVector<SDValue, 8> Ops(NumOutElem);
3310  for (unsigned i = 0; i < NumOperands; ++i) {
3311  SDValue Op = N->getOperand(i);
3312  for (unsigned j = 0; j < NumElem; ++j) {
3313  SDValue Ext = DAG.getNode(
3314  ISD::EXTRACT_VECTOR_ELT, dl, InElemTy, Op,
3315  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3316  Ops[i * NumElem + j] = DAG.getNode(ISD::ANY_EXTEND, dl, OutElemTy, Ext);
3317  }
3318  }
3319 
3320  return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, Ops);
3321 }
3322 
3323 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
3324  EVT VT = N->getValueType(0);
3325  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3326  assert(NVT.isVector() && "This type must be promoted to a vector type");
3327 
3328  SDLoc dl(N);
3329 
3330  // For operands whose TypeAction is to promote, extend the promoted node
3331  // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
3332  // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
3333  // type..
3334  if (getTypeAction(N->getOperand(0).getValueType())
3336  SDValue Promoted;
3337 
3338  switch(N->getOpcode()) {
3340  Promoted = SExtPromotedInteger(N->getOperand(0));
3341  break;
3343  Promoted = ZExtPromotedInteger(N->getOperand(0));
3344  break;
3346  Promoted = GetPromotedInteger(N->getOperand(0));
3347  break;
3348  default:
3349  llvm_unreachable("Node has unexpected Opcode");
3350  }
3351  return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
3352  }
3353 
3354  // Directly extend to the appropriate transform-to type.
3355  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
3356 }
3357 
3358 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
3359  EVT OutVT = N->getValueType(0);
3360  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3361  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3362 
3363  EVT NOutVTElem = NOutVT.getVectorElementType();
3364 
3365  SDLoc dl(N);
3366  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3367 
3368  SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
3369  NOutVTElem, N->getOperand(1));
3370  return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
3371  V0, ConvElem, N->getOperand(2));
3372 }
3373 
3374 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3375  SDLoc dl(N);
3376  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3377  SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
3378  TLI.getVectorIdxTy(DAG.getDataLayout()));
3379  SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
3380  V0->getValueType(0).getScalarType(), V0, V1);
3381 
3382  // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
3383  // element types. If this is the case then we need to expand the outgoing
3384  // value and not truncate it.
3385  return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
3386 }
3387 
3388 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
3389  SDLoc dl(N);
3390  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3391  MVT InVT = V0.getValueType().getSimpleVT();
3392  MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
3394  SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
3395  return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
3396 }
3397 
3398 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
3399  SDLoc dl(N);
3400  unsigned NumElems = N->getNumOperands();
3401 
3402  EVT RetSclrTy = N->getValueType(0).getVectorElementType();
3403 
3404  SmallVector<SDValue, 8> NewOps;
3405  NewOps.reserve(NumElems);
3406 
3407  // For each incoming vector
3408  for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
3409  SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
3410  EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
3411  unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
3412 
3413  for (unsigned i=0; i<NumElem; ++i) {
3414  // Extract element from incoming vector
3415  SDValue Ex = DAG.getNode(
3416  ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
3417  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3418  SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
3419  NewOps.push_back(Tr);
3420  }
3421  }
3422 
3423  return DAG.getNode(ISD::BUILD_VECTOR, dl, N->getValueType(0), NewOps);
3424 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:500
void push_back(const T &Elt)
Definition: SmallVector.h:211
unsigned Log2_32_Ceil(uint32_t Value)
Log2_32_Ceil - This function returns the ceil log base 2 of the specified value, 32 if the value is z...
Definition: MathExtras.h:526
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
SDValue getValue(unsigned R) const
const SDValue & getValue() const
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant, which is required to be operand #1) half of the integer or float value specified as operand #0.
Definition: ISDOpcodes.h:184
Flags getFlags() const
Return the raw flags of the source value,.
LLVMContext * getContext() const
Definition: SelectionDAG.h:333
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:804
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:304
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:572
size_t i
unsigned getNumRegisters(LLVMContext &Context, EVT VT) const
Return the number of registers that this ValueType will eventually require.
static MVT getVectorVT(MVT VT, unsigned NumElements)
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:313
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:449
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none...
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:219
void computeKnownBits(SDValue Op, APInt &KnownZero, APInt &KnownOne, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in the KnownZero/KnownO...
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
Definition: APInt.h:536
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:237
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
const SDValue & getSrc0() const
Type * getTypeForEVT(LLVMContext &Context) const
getTypeForEVT - This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:204
unsigned getSizeInBits() const
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:711
unsigned getNumOperands() const
Return the number of values used by this operation.
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned Num) const
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
Definition: SelectionDAG.h:817
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:440
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT TVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:330
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Same for subtraction.
Definition: ISDOpcodes.h:240
void reserve(size_type N)
Definition: SmallVector.h:377
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
const SDValue & getBasePtr() const
SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align=0, int Offs=0, bool isT=false, unsigned char TargetFlags=0)
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:369
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:39
bool bitsLT(EVT VT) const
bitsLT - Return true if this has less bits than VT.
Definition: ValueTypes.h:212
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
SDValue getExternalSymbol(const char *Sym, EVT VT)
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:209
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations...
Definition: ISDOpcodes.h:388
bool isVector() const
isVector - Return true if this is a vector value type.
Definition: ValueTypes.h:133
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:440
Shift and rotation operations.
Definition: ISDOpcodes.h:344
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:327
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:190
FLT_ROUNDS_ - Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest 2 Round to ...
Definition: ISDOpcodes.h:475
MachinePointerInfo getWithOffset(int64_t O) const
EVT getScalarType() const
getScalarType - If this is a vector type, return the element type, otherwise return this...
Definition: ValueTypes.h:233
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt) For double-word atomic operations: ValLo, ValHi, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amtLo, amtHi) ValLo, ValHi, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amtLo, amtHi) These correspond to the atomicrmw instruction.
Definition: ISDOpcodes.h:719
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG...
Definition: ISDOpcodes.h:73
This is an SDNode representing atomic operations.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
This class is used to represent an MSTORE node.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getVectorElementType() const
getVectorElementType - Given a vector type, return the type of each element.
Definition: ValueTypes.h:239
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:241
static unsigned getAlignment(GlobalVariable *GV)
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:410
SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:363
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:220
const SDValue & getBasePtr() const
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:737
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTruncating=false, bool IsCompressing=false)
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:842
EVT getMemoryVT() const
Return the type of the in-memory value.
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:656
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:427
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, const Value *PtrVal, unsigned Alignment, AtomicOrdering Ordering, SynchronizationScope SynchScope)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands...
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:328
bool bitsLE(EVT VT) const
bitsLE - Return true if this has no more bits than VT.
Definition: ValueTypes.h:218
Maximum length of the test input libFuzzer tries to guess a good value based on the corpus and reports it always prefer smaller inputs during the corpus shuffle When libFuzzer itself reports a bug this exit code will be used If indicates the maximal total time in seconds to run the fuzzer minimizes the provided crash input Use with etc Experimental Use value profile to guide fuzzing Number of simultaneous worker processes to run the jobs If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
UNDEF - An undefined node.
Definition: ISDOpcodes.h:178
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:453
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:274
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
Definition: APInt.h:518
SDNode * getNode() const
get the SDNode which holds the desired result
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
MinAlign - A and B are either alignments or offsets.
Definition: MathExtras.h:589
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:262
unsigned getStoreSize() const
getStoreSize - Return the number of bytes overwritten by a store of the specified value type...
Definition: ValueTypes.h:268
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
Definition: APInt.h:1147
constexpr bool isPowerOf2_32(uint32_t Value)
isPowerOf2_32 - This function returns true if the argument is a power of two > 0. ...
Definition: MathExtras.h:399
MVT - Machine Value Type.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
Simple binary floating point operators.
Definition: ISDOpcodes.h:246
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
This is an important base class in LLVM.
Definition: Constant.h:42
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:818
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:279
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:228
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:484
uint32_t Offset
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
const SDValue & getBasePtr() const
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:57
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:628
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType, bool IsExpanding=false)
const SDValue & getValue() const
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:350
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo...
Definition: ISDOpcodes.h:705
EVT - Extended Value Type.
Definition: ValueTypes.h:31
std::vector< ArgListEntry > ArgListTy
const SDValue & getMask() const
bool bitsEq(EVT VT) const
bitsEq - Return true if this has the same number of bits as VT.
Definition: ValueTypes.h:194
This structure contains all information that is necessary for lowering calls.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements)
getVectorVT - Returns the EVT that represents a vector NumElements in length, where each element is o...
Definition: ValueTypes.h:70
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, bool isSigned, const SDLoc &dl, bool doesNotReturn=false, bool isReturnValueUsed=true) const
Returns a pair of (return value, chain).
This class contains a discriminated union of information about pointers in memory operands...
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
const MachinePointerInfo & getPointerInfo() const
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
ArrayRef< int > getMask() const
bool ugt(const APInt &RHS) const
Unsigned greather than comparison.
Definition: APInt.h:1083
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:50
void dump() const
Dump this node, for debugging.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:285
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:843
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:566
const SDValue & getChain() const
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:625
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:347
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:510
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:558
Represents one node in the SelectionDAG.
static unsigned int semanticsPrecision(const fltSemantics &)
Definition: APFloat.cpp:136
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
unsigned Log2_32(uint32_t Value)
Log2_32 - This function returns the floor log base 2 of the specified value, -1 if the value is zero...
Definition: MathExtras.h:513
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:586
const SDValue & getValue() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Class for arbitrary precision integers.
Definition: APInt.h:77
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:354
op_iterator op_begin() const
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:400
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:403
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition: APInt.h:426
iterator_range< value_op_iterator > op_values() const
Flags
Flags values. These may be or'd together.
const SDValue & getMask() const
#define Success
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:528
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:333
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:418
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:536
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
Definition: ValueTypes.h:256
#define N
op_iterator op_end() const
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:287
Same for multiplication.
Definition: ISDOpcodes.h:243
static volatile int Zero
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
This class is used to represent an MSCATTER node.
const SDValue & getIndex() const
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
EVT getValueType() const
Return the ValueType of the referenced return value.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getCondCode(ISD::CondCode Cond)
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a carry value...
Definition: ISDOpcodes.h:383
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, unsigned Alignment, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SynchronizationScope SynchScope)
Gets a node for an atomic cmpxchg op.
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input...
bool isByteSized() const
isByteSized - Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:183
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:291
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This class is used to represent an MLOAD node.
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const SDValue & getBasePtr() const
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:239
This class is used to represent an MGATHER node.
SDValue getValueType(EVT)
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:81
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits...
#define DEBUG(X)
Definition: Debug.h:100
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:678
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:377
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:197
Conversion operators.
Definition: ISDOpcodes.h:397
BooleanContent
Enum that describes how the target represents true/false values.
int * Ptr
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:381
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:698
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT SrcTy)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value...
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:406
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
unsigned getAlignment() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
bool isBigEndian() const
Definition: DataLayout.h:221
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
getIntegerVT - Returns the EVT that represents an integer with the given number of bits...
Definition: ValueTypes.h:61
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:694
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:321
MVT getSimpleVT() const
getSimpleVT - Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:226
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:248
This class is used to represent ISD::LOAD nodes.