LLVM  9.0.0svn
LegalizeIntegerTypes.cpp
Go to the documentation of this file.
1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements integer type expansion and promotion for LegalizeTypes.
10 // Promotion is the act of changing a computation in an illegal type into a
11 // computation in a larger type. For example, implementing i8 arithmetic in an
12 // i32 register (often needed on powerpc).
13 // Expansion is the act of changing a computation in an illegal type into a
14 // computation in two identical registers of a smaller type. For example,
15 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
16 // targets).
17 //
18 //===----------------------------------------------------------------------===//
19 
20 #include "LegalizeTypes.h"
21 #include "llvm/IR/DerivedTypes.h"
23 #include "llvm/Support/KnownBits.h"
25 using namespace llvm;
26 
27 #define DEBUG_TYPE "legalize-types"
28 
29 //===----------------------------------------------------------------------===//
30 // Integer Result Promotion
31 //===----------------------------------------------------------------------===//
32 
33 /// PromoteIntegerResult - This method is called when a result of a node is
34 /// found to be in need of promotion to a larger type. At this point, the node
35 /// may also have invalid operands or may have other results that need
36 /// expansion, we just know that (at least) one result needs promotion.
37 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
38  LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG);
39  dbgs() << "\n");
40  SDValue Res = SDValue();
41 
42  // See if the target wants to custom expand this node.
43  if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
44  LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
45  return;
46  }
47 
48  switch (N->getOpcode()) {
49  default:
50 #ifndef NDEBUG
51  dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
52  N->dump(&DAG); dbgs() << "\n";
53 #endif
54  llvm_unreachable("Do not know how to promote this operator!");
55  case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
56  case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
57  case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
58  case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
59  case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
60  case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
61  case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
62  case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
64  case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
65  case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break;
67  case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
69  Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
70  case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
71  case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
72  break;
73  case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
74  break;
75  case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
76  case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
77  case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
78  case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
79  case ISD::SMIN:
80  case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
81  case ISD::UMIN:
82  case ISD::UMAX: Res = PromoteIntRes_ZExtIntBinOp(N); break;
83 
84  case ISD::SHL: Res = PromoteIntRes_SHL(N); break;
86  Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
87  case ISD::SRA: Res = PromoteIntRes_SRA(N); break;
88  case ISD::SRL: Res = PromoteIntRes_SRL(N); break;
89  case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
90  case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
91  case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
92 
94  Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
96  Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
98  Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
99  case ISD::BUILD_VECTOR:
100  Res = PromoteIntRes_BUILD_VECTOR(N); break;
102  Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
103  case ISD::CONCAT_VECTORS:
104  Res = PromoteIntRes_CONCAT_VECTORS(N); break;
105 
109  Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
110 
111  case ISD::SIGN_EXTEND:
112  case ISD::ZERO_EXTEND:
113  case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
114 
115  case ISD::FP_TO_SINT:
116  case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
117 
118  case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
119 
120  case ISD::FLT_ROUNDS_: Res = PromoteIntRes_FLT_ROUNDS(N); break;
121 
122  case ISD::AND:
123  case ISD::OR:
124  case ISD::XOR:
125  case ISD::ADD:
126  case ISD::SUB:
127  case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
128 
129  case ISD::SDIV:
130  case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
131 
132  case ISD::UDIV:
133  case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
134 
135  case ISD::SADDO:
136  case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
137  case ISD::UADDO:
138  case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
139  case ISD::SMULO:
140  case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
141 
142  case ISD::ADDE:
143  case ISD::SUBE:
144  case ISD::ADDCARRY:
145  case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
146 
147  case ISD::SADDSAT:
148  case ISD::UADDSAT:
149  case ISD::SSUBSAT:
150  case ISD::USUBSAT: Res = PromoteIntRes_ADDSUBSAT(N); break;
151  case ISD::SMULFIX:
152  case ISD::UMULFIX: Res = PromoteIntRes_MULFIX(N); break;
153  case ISD::ABS: Res = PromoteIntRes_ABS(N); break;
154 
155  case ISD::ATOMIC_LOAD:
156  Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
157 
162  case ISD::ATOMIC_LOAD_OR:
169  case ISD::ATOMIC_SWAP:
170  Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
171 
174  Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
175  break;
176 
177  case ISD::VECREDUCE_ADD:
178  case ISD::VECREDUCE_MUL:
179  case ISD::VECREDUCE_AND:
180  case ISD::VECREDUCE_OR:
181  case ISD::VECREDUCE_XOR:
182  case ISD::VECREDUCE_SMAX:
183  case ISD::VECREDUCE_SMIN:
184  case ISD::VECREDUCE_UMAX:
185  case ISD::VECREDUCE_UMIN:
186  Res = PromoteIntRes_VECREDUCE(N);
187  break;
188  }
189 
190  // If the result is null then the sub-method took care of registering it.
191  if (Res.getNode())
192  SetPromotedInteger(SDValue(N, ResNo), Res);
193 }
194 
195 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
196  unsigned ResNo) {
197  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
198  return GetPromotedInteger(Op);
199 }
200 
201 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
202  // Sign-extend the new bits, and continue the assertion.
203  SDValue Op = SExtPromotedInteger(N->getOperand(0));
204  return DAG.getNode(ISD::AssertSext, SDLoc(N),
205  Op.getValueType(), Op, N->getOperand(1));
206 }
207 
208 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
209  // Zero the new bits, and continue the assertion.
210  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
211  return DAG.getNode(ISD::AssertZext, SDLoc(N),
212  Op.getValueType(), Op, N->getOperand(1));
213 }
214 
215 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
216  EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
217  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
218  N->getMemoryVT(), ResVT,
219  N->getChain(), N->getBasePtr(),
220  N->getMemOperand());
221  // Legalize the chain result - switch anything that used the old chain to
222  // use the new one.
223  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
224  return Res;
225 }
226 
227 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
228  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
229  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
230  N->getMemoryVT(),
231  N->getChain(), N->getBasePtr(),
232  Op2, N->getMemOperand());
233  // Legalize the chain result - switch anything that used the old chain to
234  // use the new one.
235  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
236  return Res;
237 }
238 
239 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
240  unsigned ResNo) {
241  if (ResNo == 1) {
243  EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
244  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
245 
246  // Only use the result of getSetCCResultType if it is legal,
247  // otherwise just use the promoted result type (NVT).
248  if (!TLI.isTypeLegal(SVT))
249  SVT = NVT;
250 
251  SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
252  SDValue Res = DAG.getAtomicCmpSwap(
254  N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
255  N->getMemOperand());
256  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
257  ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
258  return Res.getValue(1);
259  }
260 
261  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
262  SDValue Op3 = GetPromotedInteger(N->getOperand(3));
263  SDVTList VTs =
264  DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
265  SDValue Res = DAG.getAtomicCmpSwap(
266  N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
267  N->getBasePtr(), Op2, Op3, N->getMemOperand());
268  // Update the use to N with the newly created Res.
269  for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
270  ReplaceValueWith(SDValue(N, i), Res.getValue(i));
271  return Res;
272 }
273 
274 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
275  SDValue InOp = N->getOperand(0);
276  EVT InVT = InOp.getValueType();
277  EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
278  EVT OutVT = N->getValueType(0);
279  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
280  SDLoc dl(N);
281 
282  switch (getTypeAction(InVT)) {
284  break;
286  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
287  // The input promotes to the same size. Convert the promoted value.
288  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
289  break;
291  // Promote the integer operand by hand.
292  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
294  // Convert the promoted float by hand.
295  if (!NOutVT.isVector())
296  return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
297  break;
298  }
301  break;
303  // Convert the element to an integer and promote it by hand.
304  if (!NOutVT.isVector())
305  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
306  BitConvertToInteger(GetScalarizedVector(InOp)));
307  break;
309  // For example, i32 = BITCAST v2i16 on alpha. Convert the split
310  // pieces of the input into integers and reassemble in the final type.
311  SDValue Lo, Hi;
312  GetSplitVector(N->getOperand(0), Lo, Hi);
313  Lo = BitConvertToInteger(Lo);
314  Hi = BitConvertToInteger(Hi);
315 
316  if (DAG.getDataLayout().isBigEndian())
317  std::swap(Lo, Hi);
318 
319  InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
321  NOutVT.getSizeInBits()),
322  JoinIntegers(Lo, Hi));
323  return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
324  }
326  // The input is widened to the same size. Convert to the widened value.
327  // Make sure that the outgoing value is not a vector, because this would
328  // make us bitcast between two vectors which are legalized in different ways.
329  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector())
330  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
331  // If the output type is also a vector and widening it to the same size
332  // as the widened input type would be a legal type, we can widen the bitcast
333  // and handle the promotion after.
334  if (NOutVT.isVector()) {
335  unsigned WidenInSize = NInVT.getSizeInBits();
336  unsigned OutSize = OutVT.getSizeInBits();
337  if (WidenInSize % OutSize == 0) {
338  unsigned Scale = WidenInSize / OutSize;
339  EVT WideOutVT = EVT::getVectorVT(*DAG.getContext(),
340  OutVT.getVectorElementType(),
341  OutVT.getVectorNumElements() * Scale);
342  if (isTypeLegal(WideOutVT)) {
343  InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
344  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
345  InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
346  DAG.getConstant(0, dl, IdxTy));
347  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
348  }
349  }
350  }
351  }
352 
353  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
354  CreateStackStoreLoad(InOp, OutVT));
355 }
356 
357 // Helper for BSWAP/BITREVERSE promotion to ensure we can fit the shift amount
358 // in the VT returned by getShiftAmountTy and to return a safe VT if we can't.
359 static EVT getShiftAmountTyForConstant(unsigned Val, EVT VT,
360  const TargetLowering &TLI,
361  SelectionDAG &DAG) {
362  EVT ShiftVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
363  // If the value won't fit in the prefered type, just use something safe. It
364  // will be legalized when the shift is expanded.
365  if ((Log2_32(Val) + 1) > ShiftVT.getScalarSizeInBits())
366  ShiftVT = MVT::i32;
367  return ShiftVT;
368 }
369 
370 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
371  SDValue Op = GetPromotedInteger(N->getOperand(0));
372  EVT OVT = N->getValueType(0);
373  EVT NVT = Op.getValueType();
374  SDLoc dl(N);
375 
376  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
377  EVT ShiftVT = getShiftAmountTyForConstant(DiffBits, NVT, TLI, DAG);
378  return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
379  DAG.getConstant(DiffBits, dl, ShiftVT));
380 }
381 
382 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
383  SDValue Op = GetPromotedInteger(N->getOperand(0));
384  EVT OVT = N->getValueType(0);
385  EVT NVT = Op.getValueType();
386  SDLoc dl(N);
387 
388  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
389  EVT ShiftVT = getShiftAmountTyForConstant(DiffBits, NVT, TLI, DAG);
390  return DAG.getNode(ISD::SRL, dl, NVT,
391  DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
392  DAG.getConstant(DiffBits, dl, ShiftVT));
393 }
394 
395 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
396  // The pair element type may be legal, or may not promote to the same type as
397  // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
398  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
399  TLI.getTypeToTransformTo(*DAG.getContext(),
400  N->getValueType(0)), JoinIntegers(N->getOperand(0),
401  N->getOperand(1)));
402 }
403 
404 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
405  EVT VT = N->getValueType(0);
406  // FIXME there is no actual debug info here
407  SDLoc dl(N);
408  // Zero extend things like i1, sign extend everything else. It shouldn't
409  // matter in theory which one we pick, but this tends to give better code?
410  unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
411  SDValue Result = DAG.getNode(Opc, dl,
412  TLI.getTypeToTransformTo(*DAG.getContext(), VT),
413  SDValue(N, 0));
414  assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
415  return Result;
416 }
417 
418 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
419  // Zero extend to the promoted type and do the count there.
420  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
421  SDLoc dl(N);
422  EVT OVT = N->getValueType(0);
423  EVT NVT = Op.getValueType();
424  Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
425  // Subtract off the extra leading bits in the bigger type.
426  return DAG.getNode(
427  ISD::SUB, dl, NVT, Op,
428  DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
429  NVT));
430 }
431 
432 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
433  // Zero extend to the promoted type and do the count there.
434  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
435  return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op);
436 }
437 
438 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
439  SDValue Op = GetPromotedInteger(N->getOperand(0));
440  EVT OVT = N->getValueType(0);
441  EVT NVT = Op.getValueType();
442  SDLoc dl(N);
443  if (N->getOpcode() == ISD::CTTZ) {
444  // The count is the same in the promoted type except if the original
445  // value was zero. This can be handled by setting the bit just off
446  // the top of the original type.
447  auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
448  OVT.getScalarSizeInBits());
449  Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
450  }
451  return DAG.getNode(N->getOpcode(), dl, NVT, Op);
452 }
453 
454 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
455  SDLoc dl(N);
456  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
457 
458  SDValue Op0 = N->getOperand(0);
459  SDValue Op1 = N->getOperand(1);
460 
461  // If the input also needs to be promoted, do that first so we can get a
462  // get a good idea for the output type.
463  if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
465  SDValue In = GetPromotedInteger(Op0);
466 
467  // If the new type is larger than NVT, use it. We probably won't need to
468  // promote it again.
469  EVT SVT = In.getValueType().getScalarType();
470  if (SVT.bitsGE(NVT)) {
471  SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
472  return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
473  }
474  }
475 
476  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
477 }
478 
479 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
480  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
481  unsigned NewOpc = N->getOpcode();
482  SDLoc dl(N);
483 
484  // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
485  // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
486  // and SINT conversions are Custom, there is no way to tell which is
487  // preferable. We choose SINT because that's the right thing on PPC.)
488  if (N->getOpcode() == ISD::FP_TO_UINT &&
489  !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
491  NewOpc = ISD::FP_TO_SINT;
492 
493  SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
494 
495  // Assert that the converted value fits in the original type. If it doesn't
496  // (eg: because the value being converted is too big), then the result of the
497  // original operation was undefined anyway, so the assert is still correct.
498  //
499  // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
500  // before legalization: fp-to-uint16, 65534. -> 0xfffe
501  // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
502  return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ?
503  ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
505 }
506 
507 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
508  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
509  SDLoc dl(N);
510 
511  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
512 }
513 
514 SDValue DAGTypeLegalizer::PromoteIntRes_FLT_ROUNDS(SDNode *N) {
515  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
516  SDLoc dl(N);
517 
518  return DAG.getNode(N->getOpcode(), dl, NVT);
519 }
520 
521 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
522  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
523  SDLoc dl(N);
524 
525  if (getTypeAction(N->getOperand(0).getValueType())
527  SDValue Res = GetPromotedInteger(N->getOperand(0));
528  assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
529 
530  // If the result and operand types are the same after promotion, simplify
531  // to an in-register extension.
532  if (NVT == Res.getValueType()) {
533  // The high bits are not guaranteed to be anything. Insert an extend.
534  if (N->getOpcode() == ISD::SIGN_EXTEND)
535  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
536  DAG.getValueType(N->getOperand(0).getValueType()));
537  if (N->getOpcode() == ISD::ZERO_EXTEND)
538  return DAG.getZeroExtendInReg(Res, dl,
540  assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
541  return Res;
542  }
543  }
544 
545  // Otherwise, just extend the original operand all the way to the larger type.
546  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
547 }
548 
549 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
550  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
551  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
554  SDLoc dl(N);
555  SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
556  N->getMemoryVT(), N->getMemOperand());
557 
558  // Legalize the chain result - switch anything that used the old chain to
559  // use the new one.
560  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
561  return Res;
562 }
563 
564 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
565  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
566  SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
567 
568  SDLoc dl(N);
569  SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
570  N->getMask(), ExtPassThru, N->getMemoryVT(),
572  // Legalize the chain result - switch anything that used the old chain to
573  // use the new one.
574  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
575  return Res;
576 }
577 
578 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
579  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
580  SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
581  assert(NVT == ExtPassThru.getValueType() &&
582  "Gather result type and the passThru agrument type should be the same");
583 
584  SDLoc dl(N);
585  SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
586  N->getIndex(), N->getScale() };
587  SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
588  N->getMemoryVT(), dl, Ops,
589  N->getMemOperand());
590  // Legalize the chain result - switch anything that used the old chain to
591  // use the new one.
592  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
593  return Res;
594 }
595 
596 /// Promote the overflow flag of an overflowing arithmetic node.
597 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
598  // Change the return type of the boolean result while obeying
599  // getSetCCResultType.
600  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
601  EVT VT = N->getValueType(0);
602  EVT SVT = getSetCCResultType(VT);
603  SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
604  unsigned NumOps = N->getNumOperands();
605  assert(NumOps <= 3 && "Too many operands");
606  if (NumOps == 3)
607  Ops[2] = N->getOperand(2);
608 
609  SDLoc dl(N);
610  SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
611  makeArrayRef(Ops, NumOps));
612 
613  // Modified the sum result - switch anything that used the old sum to use
614  // the new one.
615  ReplaceValueWith(SDValue(N, 0), Res);
616 
617  // Convert to the expected type.
618  return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT);
619 }
620 
621 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSAT(SDNode *N) {
622  // For promoting iN -> iM, this can be expanded by
623  // 1. ANY_EXTEND iN to iM
624  // 2. SHL by M-N
625  // 3. [US][ADD|SUB]SAT
626  // 4. L/ASHR by M-N
627  SDLoc dl(N);
628  SDValue Op1 = N->getOperand(0);
629  SDValue Op2 = N->getOperand(1);
630  unsigned OldBits = Op1.getScalarValueSizeInBits();
631 
632  unsigned Opcode = N->getOpcode();
633  unsigned ShiftOp;
634  switch (Opcode) {
635  case ISD::SADDSAT:
636  case ISD::SSUBSAT:
637  ShiftOp = ISD::SRA;
638  break;
639  case ISD::UADDSAT:
640  case ISD::USUBSAT:
641  ShiftOp = ISD::SRL;
642  break;
643  default:
644  llvm_unreachable("Expected opcode to be signed or unsigned saturation "
645  "addition or subtraction");
646  }
647 
648  SDValue Op1Promoted = GetPromotedInteger(Op1);
649  SDValue Op2Promoted = GetPromotedInteger(Op2);
650 
651  EVT PromotedType = Op1Promoted.getValueType();
652  unsigned NewBits = PromotedType.getScalarSizeInBits();
653  unsigned SHLAmount = NewBits - OldBits;
654  EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
655  SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT);
656  Op1Promoted =
657  DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
658  Op2Promoted =
659  DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
660 
661  SDValue Result =
662  DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
663  return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
664 }
665 
666 SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
667  // Can just promote the operands then continue with operation.
668  SDLoc dl(N);
669  SDValue Op1Promoted, Op2Promoted;
670  if (N->getOpcode() == ISD::SMULFIX) {
671  Op1Promoted = SExtPromotedInteger(N->getOperand(0));
672  Op2Promoted = SExtPromotedInteger(N->getOperand(1));
673  } else {
674  Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
675  Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
676  }
677  EVT PromotedType = Op1Promoted.getValueType();
678  return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
679  N->getOperand(2));
680 }
681 
682 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
683  if (ResNo == 1)
684  return PromoteIntRes_Overflow(N);
685 
686  // The operation overflowed iff the result in the larger type is not the
687  // sign extension of its truncation to the original type.
688  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
689  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
690  EVT OVT = N->getOperand(0).getValueType();
691  EVT NVT = LHS.getValueType();
692  SDLoc dl(N);
693 
694  // Do the arithmetic in the larger type.
695  unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
696  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
697 
698  // Calculate the overflow flag: sign extend the arithmetic result from
699  // the original type.
700  SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
701  DAG.getValueType(OVT));
702  // Overflowed if and only if this is not equal to Res.
703  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
704 
705  // Use the calculated overflow everywhere.
706  ReplaceValueWith(SDValue(N, 1), Ofl);
707 
708  return Res;
709 }
710 
711 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
712  SDValue LHS = GetPromotedInteger(N->getOperand(1));
713  SDValue RHS = GetPromotedInteger(N->getOperand(2));
714  return DAG.getSelect(SDLoc(N),
715  LHS.getValueType(), N->getOperand(0), LHS, RHS);
716 }
717 
718 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
719  SDValue Mask = N->getOperand(0);
720 
721  SDValue LHS = GetPromotedInteger(N->getOperand(1));
722  SDValue RHS = GetPromotedInteger(N->getOperand(2));
723  return DAG.getNode(ISD::VSELECT, SDLoc(N),
724  LHS.getValueType(), Mask, LHS, RHS);
725 }
726 
727 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
728  SDValue LHS = GetPromotedInteger(N->getOperand(2));
729  SDValue RHS = GetPromotedInteger(N->getOperand(3));
730  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
731  LHS.getValueType(), N->getOperand(0),
732  N->getOperand(1), LHS, RHS, N->getOperand(4));
733 }
734 
735 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
736  EVT InVT = N->getOperand(0).getValueType();
737  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
738 
739  EVT SVT = getSetCCResultType(InVT);
740 
741  // If we got back a type that needs to be promoted, this likely means the
742  // the input type also needs to be promoted. So get the promoted type for
743  // the input and try the query again.
744  if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
745  if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
746  InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
747  SVT = getSetCCResultType(InVT);
748  } else {
749  // Input type isn't promoted, just use the default promoted type.
750  SVT = NVT;
751  }
752  }
753 
754  SDLoc dl(N);
755  assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() &&
756  "Vector compare must return a vector result!");
757 
758  // Get the SETCC result using the canonical SETCC type.
759  SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
760  N->getOperand(1), N->getOperand(2));
761 
762  // Convert to the expected type.
763  return DAG.getSExtOrTrunc(SetCC, dl, NVT);
764 }
765 
766 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
767  SDValue LHS = GetPromotedInteger(N->getOperand(0));
768  SDValue RHS = N->getOperand(1);
769  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
770  RHS = ZExtPromotedInteger(RHS);
771  return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
772 }
773 
774 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
775  SDValue Op = GetPromotedInteger(N->getOperand(0));
776  return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
777  Op.getValueType(), Op, N->getOperand(1));
778 }
779 
780 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
781  // The input may have strange things in the top bits of the registers, but
782  // these operations don't care. They may have weird bits going out, but
783  // that too is okay if they are integer operations.
784  SDValue LHS = GetPromotedInteger(N->getOperand(0));
785  SDValue RHS = GetPromotedInteger(N->getOperand(1));
786  return DAG.getNode(N->getOpcode(), SDLoc(N),
787  LHS.getValueType(), LHS, RHS);
788 }
789 
790 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
791  // Sign extend the input.
792  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
793  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
794  return DAG.getNode(N->getOpcode(), SDLoc(N),
795  LHS.getValueType(), LHS, RHS);
796 }
797 
798 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
799  // Zero extend the input.
800  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
801  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
802  return DAG.getNode(N->getOpcode(), SDLoc(N),
803  LHS.getValueType(), LHS, RHS);
804 }
805 
806 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
807  // The input value must be properly sign extended.
808  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
809  SDValue RHS = N->getOperand(1);
810  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
811  RHS = ZExtPromotedInteger(RHS);
812  return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
813 }
814 
815 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
816  // The input value must be properly zero extended.
817  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
818  SDValue RHS = N->getOperand(1);
819  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
820  RHS = ZExtPromotedInteger(RHS);
821  return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
822 }
823 
824 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
825  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
826  SDValue Res;
827  SDValue InOp = N->getOperand(0);
828  SDLoc dl(N);
829 
830  switch (getTypeAction(InOp.getValueType())) {
831  default: llvm_unreachable("Unknown type action!");
834  Res = InOp;
835  break;
837  Res = GetPromotedInteger(InOp);
838  break;
840  EVT InVT = InOp.getValueType();
841  assert(InVT.isVector() && "Cannot split scalar types");
842  unsigned NumElts = InVT.getVectorNumElements();
843  assert(NumElts == NVT.getVectorNumElements() &&
844  "Dst and Src must have the same number of elements");
845  assert(isPowerOf2_32(NumElts) &&
846  "Promoted vector type must be a power of two");
847 
848  SDValue EOp1, EOp2;
849  GetSplitVector(InOp, EOp1, EOp2);
850 
851  EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
852  NumElts/2);
853  EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
854  EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
855 
856  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
857  }
859  SDValue WideInOp = GetWidenedVector(InOp);
860 
861  // Truncate widened InOp.
862  unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
863  EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
864  N->getValueType(0).getScalarType(), NumElem);
865  SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
866 
867  // Zero extend so that the elements are of same type as those of NVT
868  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
869  NumElem);
870  SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
871 
872  // Extract the low NVT subvector.
873  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
874  SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy);
875  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
876  }
877  }
878 
879  // Truncate to NVT instead of VT
880  return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
881 }
882 
883 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
884  if (ResNo == 1)
885  return PromoteIntRes_Overflow(N);
886 
887  // The operation overflowed iff the result in the larger type is not the
888  // zero extension of its truncation to the original type.
889  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
890  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
891  EVT OVT = N->getOperand(0).getValueType();
892  EVT NVT = LHS.getValueType();
893  SDLoc dl(N);
894 
895  // Do the arithmetic in the larger type.
896  unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
897  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
898 
899  // Calculate the overflow flag: zero extend the arithmetic result from
900  // the original type.
901  SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT.getScalarType());
902  // Overflowed if and only if this is not equal to Res.
903  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
904 
905  // Use the calculated overflow everywhere.
906  ReplaceValueWith(SDValue(N, 1), Ofl);
907 
908  return Res;
909 }
910 
911 // Handle promotion for the ADDE/SUBE/ADDCARRY/SUBCARRY nodes. Notice that
912 // the third operand of ADDE/SUBE nodes is carry flag, which differs from
913 // the ADDCARRY/SUBCARRY nodes in that the third operand is carry Boolean.
914 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
915  if (ResNo == 1)
916  return PromoteIntRes_Overflow(N);
917 
918  // We need to sign-extend the operands so the carry value computed by the
919  // wide operation will be equivalent to the carry value computed by the
920  // narrow operation.
921  // An ADDCARRY can generate carry only if any of the operands has its
922  // most significant bit set. Sign extension propagates the most significant
923  // bit into the higher bits which means the extra bit that the narrow
924  // addition would need (i.e. the carry) will be propagated through the higher
925  // bits of the wide addition.
926  // A SUBCARRY can generate borrow only if LHS < RHS and this property will be
927  // preserved by sign extension.
928  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
929  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
930 
931  EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
932 
933  // Do the arithmetic in the wide type.
934  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
935  LHS, RHS, N->getOperand(2));
936 
937  // Update the users of the original carry/borrow value.
938  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
939 
940  return SDValue(Res.getNode(), 0);
941 }
942 
943 SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
944  SDValue Op0 = SExtPromotedInteger(N->getOperand(0));
945  return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0);
946 }
947 
948 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
949  // Promote the overflow bit trivially.
950  if (ResNo == 1)
951  return PromoteIntRes_Overflow(N);
952 
953  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
954  SDLoc DL(N);
955  EVT SmallVT = LHS.getValueType();
956 
957  // To determine if the result overflowed in a larger type, we extend the
958  // input to the larger type, do the multiply (checking if it overflows),
959  // then also check the high bits of the result to see if overflow happened
960  // there.
961  if (N->getOpcode() == ISD::SMULO) {
962  LHS = SExtPromotedInteger(LHS);
963  RHS = SExtPromotedInteger(RHS);
964  } else {
965  LHS = ZExtPromotedInteger(LHS);
966  RHS = ZExtPromotedInteger(RHS);
967  }
968  SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
969  SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
970 
971  // Overflow occurred if it occurred in the larger type, or if the high part
972  // of the result does not zero/sign-extend the low part. Check this second
973  // possibility first.
974  SDValue Overflow;
975  if (N->getOpcode() == ISD::UMULO) {
976  // Unsigned overflow occurred if the high part is non-zero.
977  unsigned Shift = SmallVT.getScalarSizeInBits();
978  EVT ShiftTy = getShiftAmountTyForConstant(Shift, Mul.getValueType(),
979  TLI, DAG);
980  SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
981  DAG.getConstant(Shift, DL, ShiftTy));
982  Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
983  DAG.getConstant(0, DL, Hi.getValueType()),
984  ISD::SETNE);
985  } else {
986  // Signed overflow occurred if the high part does not sign extend the low.
987  SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
988  Mul, DAG.getValueType(SmallVT));
989  Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
990  }
991 
992  // The only other way for overflow to occur is if the multiplication in the
993  // larger type itself overflowed.
994  Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
995  SDValue(Mul.getNode(), 1));
996 
997  // Use the calculated overflow everywhere.
998  ReplaceValueWith(SDValue(N, 1), Overflow);
999  return Mul;
1000 }
1001 
1002 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
1003  return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1004  N->getValueType(0)));
1005 }
1006 
1007 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
1008  SDValue Chain = N->getOperand(0); // Get the chain.
1009  SDValue Ptr = N->getOperand(1); // Get the pointer.
1010  EVT VT = N->getValueType(0);
1011  SDLoc dl(N);
1012 
1013  MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
1014  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
1015  // The argument is passed as NumRegs registers of type RegVT.
1016 
1017  SmallVector<SDValue, 8> Parts(NumRegs);
1018  for (unsigned i = 0; i < NumRegs; ++i) {
1019  Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
1020  N->getConstantOperandVal(3));
1021  Chain = Parts[i].getValue(1);
1022  }
1023 
1024  // Handle endianness of the load.
1025  if (DAG.getDataLayout().isBigEndian())
1026  std::reverse(Parts.begin(), Parts.end());
1027 
1028  // Assemble the parts in the promoted type.
1029  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1030  SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
1031  for (unsigned i = 1; i < NumRegs; ++i) {
1032  SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
1033  // Shift it to the right position and "or" it in.
1034  Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
1035  DAG.getConstant(i * RegVT.getSizeInBits(), dl,
1036  TLI.getPointerTy(DAG.getDataLayout())));
1037  Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
1038  }
1039 
1040  // Modified the chain result - switch anything that used the old chain to
1041  // use the new one.
1042  ReplaceValueWith(SDValue(N, 1), Chain);
1043 
1044  return Res;
1045 }
1046 
1047 //===----------------------------------------------------------------------===//
1048 // Integer Operand Promotion
1049 //===----------------------------------------------------------------------===//
1050 
1051 /// PromoteIntegerOperand - This method is called when the specified operand of
1052 /// the specified node is found to need promotion. At this point, all of the
1053 /// result types of the node are known to be legal, but other operands of the
1054 /// node may need promotion or expansion as well as the specified one.
1055 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
1056  LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG);
1057  dbgs() << "\n");
1058  SDValue Res = SDValue();
1059 
1060  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
1061  LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
1062  return false;
1063  }
1064 
1065  switch (N->getOpcode()) {
1066  default:
1067  #ifndef NDEBUG
1068  dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
1069  N->dump(&DAG); dbgs() << "\n";
1070  #endif
1071  llvm_unreachable("Do not know how to promote this operator's operand!");
1072 
1073  case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
1074  case ISD::ATOMIC_STORE:
1075  Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
1076  break;
1077  case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
1078  case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
1079  case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
1080  case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
1081  case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
1082  case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
1083  case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
1085  Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
1086  case ISD::SCALAR_TO_VECTOR:
1087  Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
1088  case ISD::VSELECT:
1089  case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
1090  case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
1091  case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
1092  case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
1093  case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
1094  case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
1095  OpNo); break;
1096  case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
1097  OpNo); break;
1098  case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
1099  OpNo); break;
1100  case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
1101  OpNo); break;
1102  case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
1103  OpNo); break;
1104  case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
1105  case ISD::FP16_TO_FP:
1106  case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
1107  case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
1108  case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
1109 
1110  case ISD::SHL:
1111  case ISD::SRA:
1112  case ISD::SRL:
1113  case ISD::ROTL:
1114  case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
1115 
1116  case ISD::ADDCARRY:
1117  case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
1118 
1119  case ISD::FRAMEADDR:
1120  case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
1121 
1122  case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break;
1123 
1124  case ISD::SMULFIX:
1125  case ISD::UMULFIX: Res = PromoteIntOp_MULFIX(N); break;
1126 
1127  case ISD::FPOWI: Res = PromoteIntOp_FPOWI(N); break;
1128 
1129  case ISD::VECREDUCE_ADD:
1130  case ISD::VECREDUCE_MUL:
1131  case ISD::VECREDUCE_AND:
1132  case ISD::VECREDUCE_OR:
1133  case ISD::VECREDUCE_XOR:
1134  case ISD::VECREDUCE_SMAX:
1135  case ISD::VECREDUCE_SMIN:
1136  case ISD::VECREDUCE_UMAX:
1137  case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break;
1138  }
1139 
1140  // If the result is null, the sub-method took care of registering results etc.
1141  if (!Res.getNode()) return false;
1142 
1143  // If the result is N, the sub-method updated N in place. Tell the legalizer
1144  // core about this.
1145  if (Res.getNode() == N)
1146  return true;
1147 
1148  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1149  "Invalid operand expansion");
1150 
1151  ReplaceValueWith(SDValue(N, 0), Res);
1152  return false;
1153 }
1154 
1155 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
1156 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
1157 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
1158  ISD::CondCode CCCode) {
1159  // We have to insert explicit sign or zero extends. Note that we could
1160  // insert sign extends for ALL conditions. For those operations where either
1161  // zero or sign extension would be valid, use SExtOrZExtPromotedInteger
1162  // which will choose the cheapest for the target.
1163  switch (CCCode) {
1164  default: llvm_unreachable("Unknown integer comparison!");
1165  case ISD::SETEQ:
1166  case ISD::SETNE: {
1167  SDValue OpL = GetPromotedInteger(NewLHS);
1168  SDValue OpR = GetPromotedInteger(NewRHS);
1169 
1170  // We would prefer to promote the comparison operand with sign extension.
1171  // If the width of OpL/OpR excluding the duplicated sign bits is no greater
1172  // than the width of NewLHS/NewRH, we can avoid inserting real truncate
1173  // instruction, which is redundant eventually.
1174  unsigned OpLEffectiveBits =
1175  OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
1176  unsigned OpREffectiveBits =
1177  OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
1178  if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() &&
1179  OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()) {
1180  NewLHS = OpL;
1181  NewRHS = OpR;
1182  } else {
1183  NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1184  NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1185  }
1186  break;
1187  }
1188  case ISD::SETUGE:
1189  case ISD::SETUGT:
1190  case ISD::SETULE:
1191  case ISD::SETULT:
1192  NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1193  NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1194  break;
1195  case ISD::SETGE:
1196  case ISD::SETGT:
1197  case ISD::SETLT:
1198  case ISD::SETLE:
1199  NewLHS = SExtPromotedInteger(NewLHS);
1200  NewRHS = SExtPromotedInteger(NewRHS);
1201  break;
1202  }
1203 }
1204 
1205 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1206  SDValue Op = GetPromotedInteger(N->getOperand(0));
1207  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1208 }
1209 
1210 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1211  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1212  return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1213  N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1214 }
1215 
1216 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1217  // This should only occur in unusual situations like bitcasting to an
1218  // x86_fp80, so just turn it into a store+load
1219  return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1220 }
1221 
1222 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1223  assert(OpNo == 2 && "Don't know how to promote this operand!");
1224 
1225  SDValue LHS = N->getOperand(2);
1226  SDValue RHS = N->getOperand(3);
1227  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1228 
1229  // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1230  // legal types.
1231  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1232  N->getOperand(1), LHS, RHS, N->getOperand(4)),
1233  0);
1234 }
1235 
1236 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1237  assert(OpNo == 1 && "only know how to promote condition");
1238 
1239  // Promote all the way up to the canonical SetCC type.
1240  SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1241 
1242  // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1243  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1244  N->getOperand(2)), 0);
1245 }
1246 
1247 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1248  // Since the result type is legal, the operands must promote to it.
1249  EVT OVT = N->getOperand(0).getValueType();
1250  SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1251  SDValue Hi = GetPromotedInteger(N->getOperand(1));
1252  assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1253  SDLoc dl(N);
1254 
1255  Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1256  DAG.getConstant(OVT.getSizeInBits(), dl,
1257  TLI.getPointerTy(DAG.getDataLayout())));
1258  return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1259 }
1260 
1261 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1262  // The vector type is legal but the element type is not. This implies
1263  // that the vector is a power-of-two in length and that the element
1264  // type does not have a strange size (eg: it is not i1).
1265  EVT VecVT = N->getValueType(0);
1266  unsigned NumElts = VecVT.getVectorNumElements();
1267  assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1268  "Legal vector of one illegal element?");
1269 
1270  // Promote the inserted value. The type does not need to match the
1271  // vector element type. Check that any extra bits introduced will be
1272  // truncated away.
1275  "Type of inserted value narrower than vector element type!");
1276 
1277  SmallVector<SDValue, 16> NewOps;
1278  for (unsigned i = 0; i < NumElts; ++i)
1279  NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1280 
1281  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1282 }
1283 
1284 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1285  unsigned OpNo) {
1286  if (OpNo == 1) {
1287  // Promote the inserted value. This is valid because the type does not
1288  // have to match the vector element type.
1289 
1290  // Check that any extra bits introduced will be truncated away.
1293  "Type of inserted value narrower than vector element type!");
1294  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1295  GetPromotedInteger(N->getOperand(1)),
1296  N->getOperand(2)),
1297  0);
1298  }
1299 
1300  assert(OpNo == 2 && "Different operand and result vector types?");
1301 
1302  // Promote the index.
1303  SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1304  TLI.getVectorIdxTy(DAG.getDataLayout()));
1305  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1306  N->getOperand(1), Idx), 0);
1307 }
1308 
1309 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1310  // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1311  // the operand in place.
1312  return SDValue(DAG.UpdateNodeOperands(N,
1313  GetPromotedInteger(N->getOperand(0))), 0);
1314 }
1315 
1316 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1317  assert(OpNo == 0 && "Only know how to promote the condition!");
1318  SDValue Cond = N->getOperand(0);
1319  EVT OpTy = N->getOperand(1).getValueType();
1320 
1321  if (N->getOpcode() == ISD::VSELECT)
1322  if (SDValue Res = WidenVSELECTAndMask(N))
1323  return Res;
1324 
1325  // Promote all the way up to the canonical SetCC type.
1326  EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1327  Cond = PromoteTargetBoolean(Cond, OpVT);
1328 
1329  return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1330  N->getOperand(2)), 0);
1331 }
1332 
1333 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1334  assert(OpNo == 0 && "Don't know how to promote this operand!");
1335 
1336  SDValue LHS = N->getOperand(0);
1337  SDValue RHS = N->getOperand(1);
1338  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1339 
1340  // The CC (#4) and the possible return values (#2 and #3) have legal types.
1341  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1342  N->getOperand(3), N->getOperand(4)), 0);
1343 }
1344 
1345 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1346  assert(OpNo == 0 && "Don't know how to promote this operand!");
1347 
1348  SDValue LHS = N->getOperand(0);
1349  SDValue RHS = N->getOperand(1);
1350  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1351 
1352  // The CC (#2) is always legal.
1353  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1354 }
1355 
1356 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1357  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1358  ZExtPromotedInteger(N->getOperand(1))), 0);
1359 }
1360 
1361 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1362  SDValue Op = GetPromotedInteger(N->getOperand(0));
1363  SDLoc dl(N);
1364  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1365  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1366  Op, DAG.getValueType(N->getOperand(0).getValueType()));
1367 }
1368 
1369 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1370  return SDValue(DAG.UpdateNodeOperands(N,
1371  SExtPromotedInteger(N->getOperand(0))), 0);
1372 }
1373 
1374 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1375  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1376  SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1377  SDLoc dl(N);
1378 
1379  SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1380 
1381  // Truncate the value and store the result.
1382  return DAG.getTruncStore(Ch, dl, Val, Ptr,
1383  N->getMemoryVT(), N->getMemOperand());
1384 }
1385 
1386 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1387  unsigned OpNo) {
1388 
1389  SDValue DataOp = N->getValue();
1390  EVT DataVT = DataOp.getValueType();
1391  SDValue Mask = N->getMask();
1392  SDLoc dl(N);
1393 
1394  bool TruncateStore = false;
1395  if (OpNo == 3) {
1396  Mask = PromoteTargetBoolean(Mask, DataVT);
1397  // Update in place.
1398  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1399  NewOps[3] = Mask;
1400  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1401  } else { // Data operand
1402  assert(OpNo == 1 && "Unexpected operand for promotion");
1403  DataOp = GetPromotedInteger(DataOp);
1404  TruncateStore = true;
1405  }
1406 
1407  return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask,
1408  N->getMemoryVT(), N->getMemOperand(),
1409  TruncateStore, N->isCompressingStore());
1410 }
1411 
1412 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1413  unsigned OpNo) {
1414  assert(OpNo == 2 && "Only know how to promote the mask!");
1415  EVT DataVT = N->getValueType(0);
1416  SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1417  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1418  NewOps[OpNo] = Mask;
1419  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1420 }
1421 
1422 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1423  unsigned OpNo) {
1424 
1425  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1426  if (OpNo == 2) {
1427  // The Mask
1428  EVT DataVT = N->getValueType(0);
1429  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1430  } else if (OpNo == 4) {
1431  // Need to sign extend the index since the bits will likely be used.
1432  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1433  } else
1434  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1435 
1436  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1437 }
1438 
1439 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1440  unsigned OpNo) {
1441  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1442  if (OpNo == 2) {
1443  // The Mask
1444  EVT DataVT = N->getValue().getValueType();
1445  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1446  } else if (OpNo == 4) {
1447  // Need to sign extend the index since the bits will likely be used.
1448  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1449  } else
1450  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1451  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1452 }
1453 
1454 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
1455  SDValue Op = GetPromotedInteger(N->getOperand(0));
1456  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
1457 }
1458 
1459 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
1460  return SDValue(DAG.UpdateNodeOperands(N,
1461  ZExtPromotedInteger(N->getOperand(0))), 0);
1462 }
1463 
1464 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
1465  SDLoc dl(N);
1466  SDValue Op = GetPromotedInteger(N->getOperand(0));
1467  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1468  return DAG.getZeroExtendInReg(Op, dl,
1470 }
1471 
1472 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
1473  assert(OpNo == 2 && "Don't know how to promote this operand!");
1474 
1475  SDValue LHS = N->getOperand(0);
1476  SDValue RHS = N->getOperand(1);
1477  SDValue Carry = N->getOperand(2);
1478  SDLoc DL(N);
1479 
1480  Carry = PromoteTargetBoolean(Carry, LHS.getValueType());
1481 
1482  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
1483 }
1484 
1485 SDValue DAGTypeLegalizer::PromoteIntOp_MULFIX(SDNode *N) {
1486  SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1487  return SDValue(
1488  DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
1489 }
1490 
1491 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
1492  // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
1493  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
1494  return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
1495 }
1496 
1497 SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) {
1498  assert(OpNo > 1 && "Don't know how to promote this operand!");
1499  // Promote the rw, locality, and cache type arguments to a supported integer
1500  // width.
1501  SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1502  SDValue Op3 = ZExtPromotedInteger(N->getOperand(3));
1503  SDValue Op4 = ZExtPromotedInteger(N->getOperand(4));
1504  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
1505  Op2, Op3, Op4),
1506  0);
1507 }
1508 
1509 SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) {
1510  SDValue Op = SExtPromotedInteger(N->getOperand(1));
1511  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
1512 }
1513 
1514 SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) {
1515  SDLoc dl(N);
1516  SDValue Op;
1517  switch (N->getOpcode()) {
1518  default: llvm_unreachable("Expected integer vector reduction");
1519  case ISD::VECREDUCE_ADD:
1520  case ISD::VECREDUCE_MUL:
1521  case ISD::VECREDUCE_AND:
1522  case ISD::VECREDUCE_OR:
1523  case ISD::VECREDUCE_XOR:
1524  Op = GetPromotedInteger(N->getOperand(0));
1525  break;
1526  case ISD::VECREDUCE_SMAX:
1527  case ISD::VECREDUCE_SMIN:
1528  Op = SExtPromotedInteger(N->getOperand(0));
1529  break;
1530  case ISD::VECREDUCE_UMAX:
1531  case ISD::VECREDUCE_UMIN:
1532  Op = ZExtPromotedInteger(N->getOperand(0));
1533  break;
1534  }
1535 
1536  EVT EltVT = Op.getValueType().getVectorElementType();
1537  EVT VT = N->getValueType(0);
1538  if (VT.bitsGE(EltVT))
1539  return DAG.getNode(N->getOpcode(), SDLoc(N), VT, Op);
1540 
1541  // Result size must be >= element size. If this is not the case after
1542  // promotion, also promote the result type and then truncate.
1543  SDValue Reduce = DAG.getNode(N->getOpcode(), dl, EltVT, Op);
1544  return DAG.getNode(ISD::TRUNCATE, dl, VT, Reduce);
1545 }
1546 
1547 //===----------------------------------------------------------------------===//
1548 // Integer Result Expansion
1549 //===----------------------------------------------------------------------===//
1550 
1551 /// ExpandIntegerResult - This method is called when the specified result of the
1552 /// specified node is found to need expansion. At this point, the node may also
1553 /// have invalid operands or may have other results that need promotion, we just
1554 /// know that (at least) one result needs expansion.
1555 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
1556  LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG);
1557  dbgs() << "\n");
1558  SDValue Lo, Hi;
1559  Lo = Hi = SDValue();
1560 
1561  // See if the target wants to custom expand this node.
1562  if (CustomLowerNode(N, N->getValueType(ResNo), true))
1563  return;
1564 
1565  switch (N->getOpcode()) {
1566  default:
1567 #ifndef NDEBUG
1568  dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
1569  N->dump(&DAG); dbgs() << "\n";
1570 #endif
1571  report_fatal_error("Do not know how to expand the result of this "
1572  "operator!");
1573 
1574  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1575  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
1576  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1577  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1578 
1579  case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1580  case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1581  case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1582  case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1583  case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1584 
1585  case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
1586  case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
1587  case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
1588  case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
1589  case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
1590  case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
1591  case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break;
1592  case ISD::CTLZ_ZERO_UNDEF:
1593  case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
1594  case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
1595  case ISD::CTTZ_ZERO_UNDEF:
1596  case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
1597  case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
1598  case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
1599  case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
1600  case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
1601  case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
1602  case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
1603  case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
1604  case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
1605  case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
1606  case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
1607  case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
1608  case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
1609  case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
1610  case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
1611  case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
1612 
1613  case ISD::ATOMIC_LOAD_ADD:
1614  case ISD::ATOMIC_LOAD_SUB:
1615  case ISD::ATOMIC_LOAD_AND:
1616  case ISD::ATOMIC_LOAD_CLR:
1617  case ISD::ATOMIC_LOAD_OR:
1618  case ISD::ATOMIC_LOAD_XOR:
1619  case ISD::ATOMIC_LOAD_NAND:
1620  case ISD::ATOMIC_LOAD_MIN:
1621  case ISD::ATOMIC_LOAD_MAX:
1622  case ISD::ATOMIC_LOAD_UMIN:
1623  case ISD::ATOMIC_LOAD_UMAX:
1624  case ISD::ATOMIC_SWAP:
1625  case ISD::ATOMIC_CMP_SWAP: {
1626  std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
1627  SplitInteger(Tmp.first, Lo, Hi);
1628  ReplaceValueWith(SDValue(N, 1), Tmp.second);
1629  break;
1630  }
1632  AtomicSDNode *AN = cast<AtomicSDNode>(N);
1633  SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
1634  SDValue Tmp = DAG.getAtomicCmpSwap(
1635  ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
1636  N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
1637  AN->getMemOperand());
1638 
1639  // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
1640  // success simply by comparing the loaded value against the ingoing
1641  // comparison.
1642  SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
1643  N->getOperand(2), ISD::SETEQ);
1644 
1645  SplitInteger(Tmp, Lo, Hi);
1646  ReplaceValueWith(SDValue(N, 1), Success);
1647  ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
1648  break;
1649  }
1650 
1651  case ISD::AND:
1652  case ISD::OR:
1653  case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
1654 
1655  case ISD::UMAX:
1656  case ISD::SMAX:
1657  case ISD::UMIN:
1658  case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
1659 
1660  case ISD::ADD:
1661  case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
1662 
1663  case ISD::ADDC:
1664  case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
1665 
1666  case ISD::ADDE:
1667  case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
1668 
1669  case ISD::ADDCARRY:
1670  case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
1671 
1672  case ISD::SHL:
1673  case ISD::SRA:
1674  case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
1675 
1676  case ISD::SADDO:
1677  case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
1678  case ISD::UADDO:
1679  case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
1680  case ISD::UMULO:
1681  case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
1682 
1683  case ISD::SADDSAT:
1684  case ISD::UADDSAT:
1685  case ISD::SSUBSAT:
1686  case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
1687  case ISD::SMULFIX:
1688  case ISD::UMULFIX: ExpandIntRes_MULFIX(N, Lo, Hi); break;
1689 
1690  case ISD::VECREDUCE_ADD:
1691  case ISD::VECREDUCE_MUL:
1692  case ISD::VECREDUCE_AND:
1693  case ISD::VECREDUCE_OR:
1694  case ISD::VECREDUCE_XOR:
1695  case ISD::VECREDUCE_SMAX:
1696  case ISD::VECREDUCE_SMIN:
1697  case ISD::VECREDUCE_UMAX:
1698  case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break;
1699  }
1700 
1701  // If Lo/Hi is null, the sub-method took care of registering results etc.
1702  if (Lo.getNode())
1703  SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
1704 }
1705 
1706 /// Lower an atomic node to the appropriate builtin call.
1707 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
1708  unsigned Opc = Node->getOpcode();
1709  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
1710  RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
1711  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
1712 
1713  return ExpandChainLibCall(LC, Node, false);
1714 }
1715 
1716 /// N is a shift by a value that needs to be expanded,
1717 /// and the shift amount is a constant 'Amt'. Expand the operation.
1718 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
1719  SDValue &Lo, SDValue &Hi) {
1720  SDLoc DL(N);
1721  // Expand the incoming operand to be shifted, so that we have its parts
1722  SDValue InL, InH;
1723  GetExpandedInteger(N->getOperand(0), InL, InH);
1724 
1725  // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
1726  // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
1727  if (!Amt) {
1728  Lo = InL;
1729  Hi = InH;
1730  return;
1731  }
1732 
1733  EVT NVT = InL.getValueType();
1734  unsigned VTBits = N->getValueType(0).getSizeInBits();
1735  unsigned NVTBits = NVT.getSizeInBits();
1736  EVT ShTy = N->getOperand(1).getValueType();
1737 
1738  if (N->getOpcode() == ISD::SHL) {
1739  if (Amt.ugt(VTBits)) {
1740  Lo = Hi = DAG.getConstant(0, DL, NVT);
1741  } else if (Amt.ugt(NVTBits)) {
1742  Lo = DAG.getConstant(0, DL, NVT);
1743  Hi = DAG.getNode(ISD::SHL, DL,
1744  NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1745  } else if (Amt == NVTBits) {
1746  Lo = DAG.getConstant(0, DL, NVT);
1747  Hi = InL;
1748  } else {
1749  Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
1750  Hi = DAG.getNode(ISD::OR, DL, NVT,
1751  DAG.getNode(ISD::SHL, DL, NVT, InH,
1752  DAG.getConstant(Amt, DL, ShTy)),
1753  DAG.getNode(ISD::SRL, DL, NVT, InL,
1754  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1755  }
1756  return;
1757  }
1758 
1759  if (N->getOpcode() == ISD::SRL) {
1760  if (Amt.ugt(VTBits)) {
1761  Lo = Hi = DAG.getConstant(0, DL, NVT);
1762  } else if (Amt.ugt(NVTBits)) {
1763  Lo = DAG.getNode(ISD::SRL, DL,
1764  NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1765  Hi = DAG.getConstant(0, DL, NVT);
1766  } else if (Amt == NVTBits) {
1767  Lo = InH;
1768  Hi = DAG.getConstant(0, DL, NVT);
1769  } else {
1770  Lo = DAG.getNode(ISD::OR, DL, NVT,
1771  DAG.getNode(ISD::SRL, DL, NVT, InL,
1772  DAG.getConstant(Amt, DL, ShTy)),
1773  DAG.getNode(ISD::SHL, DL, NVT, InH,
1774  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1775  Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1776  }
1777  return;
1778  }
1779 
1780  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
1781  if (Amt.ugt(VTBits)) {
1782  Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1783  DAG.getConstant(NVTBits - 1, DL, ShTy));
1784  } else if (Amt.ugt(NVTBits)) {
1785  Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1786  DAG.getConstant(Amt - NVTBits, DL, ShTy));
1787  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1788  DAG.getConstant(NVTBits - 1, DL, ShTy));
1789  } else if (Amt == NVTBits) {
1790  Lo = InH;
1791  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1792  DAG.getConstant(NVTBits - 1, DL, ShTy));
1793  } else {
1794  Lo = DAG.getNode(ISD::OR, DL, NVT,
1795  DAG.getNode(ISD::SRL, DL, NVT, InL,
1796  DAG.getConstant(Amt, DL, ShTy)),
1797  DAG.getNode(ISD::SHL, DL, NVT, InH,
1798  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1799  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1800  }
1801 }
1802 
1803 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
1804 /// this shift based on knowledge of the high bit of the shift amount. If we
1805 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
1806 /// shift amount.
1807 bool DAGTypeLegalizer::
1808 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1809  SDValue Amt = N->getOperand(1);
1810  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1811  EVT ShTy = Amt.getValueType();
1812  unsigned ShBits = ShTy.getScalarSizeInBits();
1813  unsigned NVTBits = NVT.getScalarSizeInBits();
1814  assert(isPowerOf2_32(NVTBits) &&
1815  "Expanded integer type size not a power of two!");
1816  SDLoc dl(N);
1817 
1818  APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
1819  KnownBits Known = DAG.computeKnownBits(N->getOperand(1));
1820 
1821  // If we don't know anything about the high bits, exit.
1822  if (((Known.Zero|Known.One) & HighBitMask) == 0)
1823  return false;
1824 
1825  // Get the incoming operand to be shifted.
1826  SDValue InL, InH;
1827  GetExpandedInteger(N->getOperand(0), InL, InH);
1828 
1829  // If we know that any of the high bits of the shift amount are one, then we
1830  // can do this as a couple of simple shifts.
1831  if (Known.One.intersects(HighBitMask)) {
1832  // Mask out the high bit, which we know is set.
1833  Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
1834  DAG.getConstant(~HighBitMask, dl, ShTy));
1835 
1836  switch (N->getOpcode()) {
1837  default: llvm_unreachable("Unknown shift");
1838  case ISD::SHL:
1839  Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
1840  Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
1841  return true;
1842  case ISD::SRL:
1843  Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1844  Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
1845  return true;
1846  case ISD::SRA:
1847  Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
1848  DAG.getConstant(NVTBits - 1, dl, ShTy));
1849  Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
1850  return true;
1851  }
1852  }
1853 
1854  // If we know that all of the high bits of the shift amount are zero, then we
1855  // can do this as a couple of simple shifts.
1856  if (HighBitMask.isSubsetOf(Known.Zero)) {
1857  // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
1858  // shift if x is zero. We can use XOR here because x is known to be smaller
1859  // than 32.
1860  SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
1861  DAG.getConstant(NVTBits - 1, dl, ShTy));
1862 
1863  unsigned Op1, Op2;
1864  switch (N->getOpcode()) {
1865  default: llvm_unreachable("Unknown shift");
1866  case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
1867  case ISD::SRL:
1868  case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
1869  }
1870 
1871  // When shifting right the arithmetic for Lo and Hi is swapped.
1872  if (N->getOpcode() != ISD::SHL)
1873  std::swap(InL, InH);
1874 
1875  // Use a little trick to get the bits that move from Lo to Hi. First
1876  // shift by one bit.
1877  SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
1878  // Then compute the remaining shift with amount-1.
1879  SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
1880 
1881  Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
1882  Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
1883 
1884  if (N->getOpcode() != ISD::SHL)
1885  std::swap(Hi, Lo);
1886  return true;
1887  }
1888 
1889  return false;
1890 }
1891 
1892 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
1893 /// of any size.
1894 bool DAGTypeLegalizer::
1895 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1896  SDValue Amt = N->getOperand(1);
1897  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1898  EVT ShTy = Amt.getValueType();
1899  unsigned NVTBits = NVT.getSizeInBits();
1900  assert(isPowerOf2_32(NVTBits) &&
1901  "Expanded integer type size not a power of two!");
1902  SDLoc dl(N);
1903 
1904  // Get the incoming operand to be shifted.
1905  SDValue InL, InH;
1906  GetExpandedInteger(N->getOperand(0), InL, InH);
1907 
1908  SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
1909  SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
1910  SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
1911  SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1912  Amt, NVBitsNode, ISD::SETULT);
1913  SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1914  Amt, DAG.getConstant(0, dl, ShTy),
1915  ISD::SETEQ);
1916 
1917  SDValue LoS, HiS, LoL, HiL;
1918  switch (N->getOpcode()) {
1919  default: llvm_unreachable("Unknown shift");
1920  case ISD::SHL:
1921  // Short: ShAmt < NVTBits
1922  LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
1923  HiS = DAG.getNode(ISD::OR, dl, NVT,
1924  DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
1925  DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
1926 
1927  // Long: ShAmt >= NVTBits
1928  LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
1929  HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
1930 
1931  Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
1932  Hi = DAG.getSelect(dl, NVT, isZero, InH,
1933  DAG.getSelect(dl, NVT, isShort, HiS, HiL));
1934  return true;
1935  case ISD::SRL:
1936  // Short: ShAmt < NVTBits
1937  HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
1938  LoS = DAG.getNode(ISD::OR, dl, NVT,
1939  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1940  // FIXME: If Amt is zero, the following shift generates an undefined result
1941  // on some architectures.
1942  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1943 
1944  // Long: ShAmt >= NVTBits
1945  HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1946  LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1947 
1948  Lo = DAG.getSelect(dl, NVT, isZero, InL,
1949  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1950  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1951  return true;
1952  case ISD::SRA:
1953  // Short: ShAmt < NVTBits
1954  HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
1955  LoS = DAG.getNode(ISD::OR, dl, NVT,
1956  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1957  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1958 
1959  // Long: ShAmt >= NVTBits
1960  HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
1961  DAG.getConstant(NVTBits - 1, dl, ShTy));
1962  LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1963 
1964  Lo = DAG.getSelect(dl, NVT, isZero, InL,
1965  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1966  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1967  return true;
1968  }
1969 }
1970 
1971 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
1972 
1973  switch (Op) {
1974  default: llvm_unreachable("invalid min/max opcode");
1975  case ISD::SMAX:
1976  return std::make_pair(ISD::SETGT, ISD::UMAX);
1977  case ISD::UMAX:
1978  return std::make_pair(ISD::SETUGT, ISD::UMAX);
1979  case ISD::SMIN:
1980  return std::make_pair(ISD::SETLT, ISD::UMIN);
1981  case ISD::UMIN:
1982  return std::make_pair(ISD::SETULT, ISD::UMIN);
1983  }
1984 }
1985 
1986 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
1987  SDValue &Lo, SDValue &Hi) {
1988  SDLoc DL(N);
1989  ISD::NodeType LoOpc;
1990  ISD::CondCode CondC;
1991  std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
1992 
1993  // Expand the subcomponents.
1994  SDValue LHSL, LHSH, RHSL, RHSH;
1995  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1996  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1997 
1998  // Value types
1999  EVT NVT = LHSL.getValueType();
2000  EVT CCT = getSetCCResultType(NVT);
2001 
2002  // Hi part is always the same op
2003  Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
2004 
2005  // We need to know whether to select Lo part that corresponds to 'winning'
2006  // Hi part or if Hi parts are equal.
2007  SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
2008  SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
2009 
2010  // Lo part corresponding to the 'winning' Hi part
2011  SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
2012 
2013  // Recursed Lo part if Hi parts are equal, this uses unsigned version
2014  SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
2015 
2016  Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
2017 }
2018 
2019 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
2020  SDValue &Lo, SDValue &Hi) {
2021  SDLoc dl(N);
2022  // Expand the subcomponents.
2023  SDValue LHSL, LHSH, RHSL, RHSH;
2024  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2025  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2026 
2027  EVT NVT = LHSL.getValueType();
2028  SDValue LoOps[2] = { LHSL, RHSL };
2029  SDValue HiOps[3] = { LHSH, RHSH };
2030 
2031  bool HasOpCarry = TLI.isOperationLegalOrCustom(
2033  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2034  if (HasOpCarry) {
2035  SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
2036  if (N->getOpcode() == ISD::ADD) {
2037  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2038  HiOps[2] = Lo.getValue(1);
2039  Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
2040  } else {
2041  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2042  HiOps[2] = Lo.getValue(1);
2043  Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
2044  }
2045  return;
2046  }
2047 
2048  // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
2049  // them. TODO: Teach operation legalization how to expand unsupported
2050  // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
2051  // a carry of type MVT::Glue, but there doesn't seem to be any way to
2052  // generate a value of this type in the expanded code sequence.
2053  bool hasCarry =
2055  ISD::ADDC : ISD::SUBC,
2056  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2057 
2058  if (hasCarry) {
2059  SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
2060  if (N->getOpcode() == ISD::ADD) {
2061  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2062  HiOps[2] = Lo.getValue(1);
2063  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2064  } else {
2065  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2066  HiOps[2] = Lo.getValue(1);
2067  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2068  }
2069  return;
2070  }
2071 
2072  bool hasOVF =
2075  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2077 
2078  if (hasOVF) {
2079  EVT OvfVT = getSetCCResultType(NVT);
2080  SDVTList VTList = DAG.getVTList(NVT, OvfVT);
2081  int RevOpc;
2082  if (N->getOpcode() == ISD::ADD) {
2083  RevOpc = ISD::SUB;
2084  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2085  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2086  } else {
2087  RevOpc = ISD::ADD;
2088  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2089  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2090  }
2091  SDValue OVF = Lo.getValue(1);
2092 
2093  switch (BoolType) {
2095  OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
2098  OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
2099  Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
2100  break;
2102  OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
2103  Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
2104  }
2105  return;
2106  }
2107 
2108  if (N->getOpcode() == ISD::ADD) {
2109  Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
2110  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2111  SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
2112  ISD::SETULT);
2113 
2115  SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
2116  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
2117  return;
2118  }
2119 
2120  SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
2121  DAG.getConstant(1, dl, NVT),
2122  DAG.getConstant(0, dl, NVT));
2123  SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
2124  ISD::SETULT);
2125  SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
2126  DAG.getConstant(1, dl, NVT), Carry1);
2127  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
2128  } else {
2129  Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
2130  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2131  SDValue Cmp =
2132  DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
2133  LoOps[0], LoOps[1], ISD::SETULT);
2134 
2135  SDValue Borrow;
2137  Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
2138  else
2139  Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
2140  DAG.getConstant(0, dl, NVT));
2141 
2142  Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
2143  }
2144 }
2145 
2146 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
2147  SDValue &Lo, SDValue &Hi) {
2148  // Expand the subcomponents.
2149  SDValue LHSL, LHSH, RHSL, RHSH;
2150  SDLoc dl(N);
2151  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2152  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2153  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2154  SDValue LoOps[2] = { LHSL, RHSL };
2155  SDValue HiOps[3] = { LHSH, RHSH };
2156 
2157  if (N->getOpcode() == ISD::ADDC) {
2158  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2159  HiOps[2] = Lo.getValue(1);
2160  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2161  } else {
2162  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2163  HiOps[2] = Lo.getValue(1);
2164  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2165  }
2166 
2167  // Legalized the flag result - switch anything that used the old flag to
2168  // use the new one.
2169  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2170 }
2171 
2172 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
2173  SDValue &Lo, SDValue &Hi) {
2174  // Expand the subcomponents.
2175  SDValue LHSL, LHSH, RHSL, RHSH;
2176  SDLoc dl(N);
2177  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2178  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2179  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2180  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2181  SDValue HiOps[3] = { LHSH, RHSH };
2182 
2183  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2184  HiOps[2] = Lo.getValue(1);
2185  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2186 
2187  // Legalized the flag result - switch anything that used the old flag to
2188  // use the new one.
2189  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2190 }
2191 
2192 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
2193  SDValue &Lo, SDValue &Hi) {
2194  SDValue LHS = N->getOperand(0);
2195  SDValue RHS = N->getOperand(1);
2196  SDLoc dl(N);
2197 
2198  SDValue Ovf;
2199 
2200  bool HasOpCarry = TLI.isOperationLegalOrCustom(
2202  TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
2203 
2204  if (HasOpCarry) {
2205  // Expand the subcomponents.
2206  SDValue LHSL, LHSH, RHSL, RHSH;
2207  GetExpandedInteger(LHS, LHSL, LHSH);
2208  GetExpandedInteger(RHS, RHSL, RHSH);
2209  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2210  SDValue LoOps[2] = { LHSL, RHSL };
2211  SDValue HiOps[3] = { LHSH, RHSH };
2212 
2213  unsigned Opc = N->getOpcode() == ISD::UADDO ? ISD::ADDCARRY : ISD::SUBCARRY;
2214  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2215  HiOps[2] = Lo.getValue(1);
2216  Hi = DAG.getNode(Opc, dl, VTList, HiOps);
2217 
2218  Ovf = Hi.getValue(1);
2219  } else {
2220  // Expand the result by simply replacing it with the equivalent
2221  // non-overflow-checking operation.
2222  auto Opc = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
2223  SDValue Sum = DAG.getNode(Opc, dl, LHS.getValueType(), LHS, RHS);
2224  SplitInteger(Sum, Lo, Hi);
2225 
2226  // Calculate the overflow: addition overflows iff a + b < a, and subtraction
2227  // overflows iff a - b > a.
2228  auto Cond = N->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
2229  Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
2230  }
2231 
2232  // Legalized the flag result - switch anything that used the old flag to
2233  // use the new one.
2234  ReplaceValueWith(SDValue(N, 1), Ovf);
2235 }
2236 
2237 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
2238  SDValue &Lo, SDValue &Hi) {
2239  // Expand the subcomponents.
2240  SDValue LHSL, LHSH, RHSL, RHSH;
2241  SDLoc dl(N);
2242  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2243  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2244  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2245  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2246  SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
2247 
2248  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2249  HiOps[2] = Lo.getValue(1);
2250  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2251 
2252  // Legalized the flag result - switch anything that used the old flag to
2253  // use the new one.
2254  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2255 }
2256 
2257 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
2258  SDValue &Lo, SDValue &Hi) {
2259  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2260  SDLoc dl(N);
2261  SDValue Op = N->getOperand(0);
2262  if (Op.getValueType().bitsLE(NVT)) {
2263  // The low part is any extension of the input (which degenerates to a copy).
2264  Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
2265  Hi = DAG.getUNDEF(NVT); // The high part is undefined.
2266  } else {
2267  // For example, extension of an i48 to an i64. The operand type necessarily
2268  // promotes to the result type, so will end up being expanded too.
2269  assert(getTypeAction(Op.getValueType()) ==
2271  "Only know how to promote this result!");
2272  SDValue Res = GetPromotedInteger(Op);
2273  assert(Res.getValueType() == N->getValueType(0) &&
2274  "Operand over promoted?");
2275  // Split the promoted operand. This will simplify when it is expanded.
2276  SplitInteger(Res, Lo, Hi);
2277  }
2278 }
2279 
2280 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
2281  SDValue &Lo, SDValue &Hi) {
2282  SDLoc dl(N);
2283  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2284  EVT NVT = Lo.getValueType();
2285  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2286  unsigned NVTBits = NVT.getSizeInBits();
2287  unsigned EVTBits = EVT.getSizeInBits();
2288 
2289  if (NVTBits < EVTBits) {
2290  Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
2292  EVTBits - NVTBits)));
2293  } else {
2294  Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
2295  // The high part replicates the sign bit of Lo, make it explicit.
2296  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2297  DAG.getConstant(NVTBits - 1, dl,
2298  TLI.getPointerTy(DAG.getDataLayout())));
2299  }
2300 }
2301 
2302 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
2303  SDValue &Lo, SDValue &Hi) {
2304  SDLoc dl(N);
2305  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2306  EVT NVT = Lo.getValueType();
2307  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2308  unsigned NVTBits = NVT.getSizeInBits();
2309  unsigned EVTBits = EVT.getSizeInBits();
2310 
2311  if (NVTBits < EVTBits) {
2312  Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
2314  EVTBits - NVTBits)));
2315  } else {
2316  Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
2317  // The high part must be zero, make it explicit.
2318  Hi = DAG.getConstant(0, dl, NVT);
2319  }
2320 }
2321 
2322 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
2323  SDValue &Lo, SDValue &Hi) {
2324  SDLoc dl(N);
2325  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2326  Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
2327  Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
2328 }
2329 
2330 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
2331  SDValue &Lo, SDValue &Hi) {
2332  SDLoc dl(N);
2333  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2334  Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
2335  Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
2336 }
2337 
2338 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
2339  SDValue &Lo, SDValue &Hi) {
2340  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2341  unsigned NBitWidth = NVT.getSizeInBits();
2342  auto Constant = cast<ConstantSDNode>(N);
2343  const APInt &Cst = Constant->getAPIntValue();
2344  bool IsTarget = Constant->isTargetOpcode();
2345  bool IsOpaque = Constant->isOpaque();
2346  SDLoc dl(N);
2347  Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
2348  Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
2349  IsOpaque);
2350 }
2351 
2352 void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
2353  SDLoc dl(N);
2354 
2355  // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo)
2356  EVT VT = N->getValueType(0);
2357  SDValue N0 = N->getOperand(0);
2358  SDValue Neg = DAG.getNode(ISD::SUB, dl, VT,
2359  DAG.getConstant(0, dl, VT), N0);
2360  SDValue NegLo, NegHi;
2361  SplitInteger(Neg, NegLo, NegHi);
2362 
2363  GetExpandedInteger(N0, Lo, Hi);
2364  EVT NVT = Lo.getValueType();
2365  SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT),
2366  DAG.getConstant(0, dl, NVT), Hi, ISD::SETGT);
2367  Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo);
2368  Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi);
2369 }
2370 
2371 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
2372  SDValue &Lo, SDValue &Hi) {
2373  SDLoc dl(N);
2374  // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
2375  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2376  EVT NVT = Lo.getValueType();
2377 
2378  SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
2379  DAG.getConstant(0, dl, NVT), ISD::SETNE);
2380 
2381  SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
2382  SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
2383 
2384  Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
2385  DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
2386  DAG.getConstant(NVT.getSizeInBits(), dl,
2387  NVT)));
2388  Hi = DAG.getConstant(0, dl, NVT);
2389 }
2390 
2391 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
2392  SDValue &Lo, SDValue &Hi) {
2393  SDLoc dl(N);
2394  // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
2395  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2396  EVT NVT = Lo.getValueType();
2397  Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
2398  DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
2399  Hi = DAG.getConstant(0, dl, NVT);
2400 }
2401 
2402 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
2403  SDValue &Lo, SDValue &Hi) {
2404  SDLoc dl(N);
2405  // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
2406  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2407  EVT NVT = Lo.getValueType();
2408 
2409  SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
2410  DAG.getConstant(0, dl, NVT), ISD::SETNE);
2411 
2412  SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
2413  SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
2414 
2415  Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
2416  DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
2417  DAG.getConstant(NVT.getSizeInBits(), dl,
2418  NVT)));
2419  Hi = DAG.getConstant(0, dl, NVT);
2420 }
2421 
2422 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
2423  SDValue &Hi) {
2424  SDLoc dl(N);
2425  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2426  unsigned NBitWidth = NVT.getSizeInBits();
2427 
2428  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2429  Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT);
2430  // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
2431  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2432  DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
2433 }
2434 
2435 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
2436  SDValue &Hi) {
2437  SDLoc dl(N);
2438  EVT VT = N->getValueType(0);
2439 
2440  SDValue Op = N->getOperand(0);
2441  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2442  Op = GetPromotedFloat(Op);
2443 
2445  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
2446  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, true/*irrelevant*/, dl).first,
2447  Lo, Hi);
2448 }
2449 
2450 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
2451  SDValue &Hi) {
2452  SDLoc dl(N);
2453  EVT VT = N->getValueType(0);
2454 
2455  SDValue Op = N->getOperand(0);
2456  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2457  Op = GetPromotedFloat(Op);
2458 
2460  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
2461  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, false/*irrelevant*/, dl).first,
2462  Lo, Hi);
2463 }
2464 
2465 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
2466  SDValue &Lo, SDValue &Hi) {
2467  if (ISD::isNormalLoad(N)) {
2468  ExpandRes_NormalLoad(N, Lo, Hi);
2469  return;
2470  }
2471 
2472  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2473 
2474  EVT VT = N->getValueType(0);
2475  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2476  SDValue Ch = N->getChain();
2477  SDValue Ptr = N->getBasePtr();
2479  unsigned Alignment = N->getAlignment();
2480  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2481  AAMDNodes AAInfo = N->getAAInfo();
2482  SDLoc dl(N);
2483 
2484  assert(NVT.isByteSized() && "Expanded type not byte sized!");
2485 
2486  if (N->getMemoryVT().bitsLE(NVT)) {
2487  EVT MemVT = N->getMemoryVT();
2488 
2489  Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
2490  Alignment, MMOFlags, AAInfo);
2491 
2492  // Remember the chain.
2493  Ch = Lo.getValue(1);
2494 
2495  if (ExtType == ISD::SEXTLOAD) {
2496  // The high part is obtained by SRA'ing all but one of the bits of the
2497  // lo part.
2498  unsigned LoSize = Lo.getValueSizeInBits();
2499  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2500  DAG.getConstant(LoSize - 1, dl,
2501  TLI.getPointerTy(DAG.getDataLayout())));
2502  } else if (ExtType == ISD::ZEXTLOAD) {
2503  // The high part is just a zero.
2504  Hi = DAG.getConstant(0, dl, NVT);
2505  } else {
2506  assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
2507  // The high part is undefined.
2508  Hi = DAG.getUNDEF(NVT);
2509  }
2510  } else if (DAG.getDataLayout().isLittleEndian()) {
2511  // Little-endian - low bits are at low addresses.
2512  Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2513  AAInfo);
2514 
2515  unsigned ExcessBits =
2516  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
2517  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
2518 
2519  // Increment the pointer to the other half.
2520  unsigned IncrementSize = NVT.getSizeInBits()/8;
2521  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2522  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2523  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
2524  N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
2525  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2526 
2527  // Build a factor node to remember that this load is independent of the
2528  // other one.
2529  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2530  Hi.getValue(1));
2531  } else {
2532  // Big-endian - high bits are at low addresses. Favor aligned loads at
2533  // the cost of some bit-fiddling.
2534  EVT MemVT = N->getMemoryVT();
2535  unsigned EBytes = MemVT.getStoreSize();
2536  unsigned IncrementSize = NVT.getSizeInBits()/8;
2537  unsigned ExcessBits = (EBytes - IncrementSize)*8;
2538 
2539  // Load both the high bits and maybe some of the low bits.
2540  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
2542  MemVT.getSizeInBits() - ExcessBits),
2543  Alignment, MMOFlags, AAInfo);
2544 
2545  // Increment the pointer to the other half.
2546  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2547  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2548  // Load the rest of the low bits.
2549  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
2550  N->getPointerInfo().getWithOffset(IncrementSize),
2551  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
2552  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2553 
2554  // Build a factor node to remember that this load is independent of the
2555  // other one.
2556  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2557  Hi.getValue(1));
2558 
2559  if (ExcessBits < NVT.getSizeInBits()) {
2560  // Transfer low bits from the bottom of Hi to the top of Lo.
2561  Lo = DAG.getNode(
2562  ISD::OR, dl, NVT, Lo,
2563  DAG.getNode(ISD::SHL, dl, NVT, Hi,
2564  DAG.getConstant(ExcessBits, dl,
2565  TLI.getPointerTy(DAG.getDataLayout()))));
2566  // Move high bits to the right position in Hi.
2567  Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
2568  Hi,
2569  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
2570  TLI.getPointerTy(DAG.getDataLayout())));
2571  }
2572  }
2573 
2574  // Legalize the chain result - switch anything that used the old chain to
2575  // use the new one.
2576  ReplaceValueWith(SDValue(N, 1), Ch);
2577 }
2578 
2579 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
2580  SDValue &Lo, SDValue &Hi) {
2581  SDLoc dl(N);
2582  SDValue LL, LH, RL, RH;
2583  GetExpandedInteger(N->getOperand(0), LL, LH);
2584  GetExpandedInteger(N->getOperand(1), RL, RH);
2585  Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
2586  Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
2587 }
2588 
2589 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
2590  SDValue &Lo, SDValue &Hi) {
2591  EVT VT = N->getValueType(0);
2592  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2593  SDLoc dl(N);
2594 
2595  SDValue LL, LH, RL, RH;
2596  GetExpandedInteger(N->getOperand(0), LL, LH);
2597  GetExpandedInteger(N->getOperand(1), RL, RH);
2598 
2599  if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
2601  LL, LH, RL, RH))
2602  return;
2603 
2604  // If nothing else, we can make a libcall.
2605  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2606  if (VT == MVT::i16)
2607  LC = RTLIB::MUL_I16;
2608  else if (VT == MVT::i32)
2609  LC = RTLIB::MUL_I32;
2610  else if (VT == MVT::i64)
2611  LC = RTLIB::MUL_I64;
2612  else if (VT == MVT::i128)
2613  LC = RTLIB::MUL_I128;
2614 
2615  if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
2616  // We'll expand the multiplication by brute force because we have no other
2617  // options. This is a trivially-generalized version of the code from
2618  // Hacker's Delight (itself derived from Knuth's Algorithm M from section
2619  // 4.3.1).
2620  unsigned Bits = NVT.getSizeInBits();
2621  unsigned HalfBits = Bits >> 1;
2622  SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
2623  NVT);
2624  SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
2625  SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
2626 
2627  SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
2628  SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
2629 
2630  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2631  if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
2632  // The type from TLI is too small to fit the shift amount we want.
2633  // Override it with i32. The shift will have to be legalized.
2634  ShiftAmtTy = MVT::i32;
2635  }
2636  SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
2637  SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
2638  SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
2639  SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
2640 
2641  SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
2642  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
2643  SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
2644  SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
2645 
2646  SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
2647  DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
2648  SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
2649 
2650  SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
2651  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
2652  DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
2653  Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
2654  DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
2655 
2656  Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
2657  DAG.getNode(ISD::ADD, dl, NVT,
2658  DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
2659  DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
2660  return;
2661  }
2662 
2663  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2664  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true/*irrelevant*/, dl).first,
2665  Lo, Hi);
2666 }
2667 
2668 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
2669  SDValue &Hi) {
2670  SDLoc DL(N);
2671  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2672  SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
2673  SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
2674  Lo = R.getValue(0);
2675  Hi = R.getValue(1);
2676  ReplaceValueWith(SDValue(N, 1), R.getValue(2));
2677 }
2678 
2679 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
2680  SDValue &Hi) {
2681  SDValue Result = TLI.expandAddSubSat(N, DAG);
2682  SplitInteger(Result, Lo, Hi);
2683 }
2684 
2685 void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
2686  SDValue &Hi) {
2687  assert(
2688  (N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::UMULFIX) &&
2689  "Expected operand to be signed or unsigned fixed point multiplication");
2690 
2691  SDLoc dl(N);
2692  EVT VT = N->getValueType(0);
2693  SDValue LHS = N->getOperand(0);
2694  SDValue RHS = N->getOperand(1);
2695  uint64_t Scale = N->getConstantOperandVal(2);
2696  if (!Scale) {
2697  SDValue Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
2698  SplitInteger(Result, Lo, Hi);
2699  return;
2700  }
2701 
2702  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2703  SDValue LL, LH, RL, RH;
2704  GetExpandedInteger(LHS, LL, LH);
2705  GetExpandedInteger(RHS, RL, RH);
2706  SmallVector<SDValue, 4> Result;
2707 
2708  bool Signed = N->getOpcode() == ISD::SMULFIX;
2709  unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
2710  if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
2712  LL, LH, RL, RH)) {
2713  report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI.");
2714  return;
2715  }
2716 
2717  unsigned VTSize = VT.getScalarSizeInBits();
2718  unsigned NVTSize = NVT.getScalarSizeInBits();
2719  EVT ShiftTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2720 
2721  // Shift whole amount by scale.
2722  SDValue ResultLL = Result[0];
2723  SDValue ResultLH = Result[1];
2724  SDValue ResultHL = Result[2];
2725  SDValue ResultHH = Result[3];
2726 
2727  // After getting the multplication result in 4 parts, we need to perform a
2728  // shift right by the amount of the scale to get the result in that scale.
2729  // Let's say we multiply 2 64 bit numbers. The resulting value can be held in
2730  // 128 bits that are cut into 4 32-bit parts:
2731  //
2732  // HH HL LH LL
2733  // |---32---|---32---|---32---|---32---|
2734  // 128 96 64 32 0
2735  //
2736  // |------VTSize-----|
2737  //
2738  // |NVTSize-|
2739  //
2740  // The resulting Lo and Hi will only need to be one of these 32-bit parts
2741  // after shifting.
2742  if (Scale < NVTSize) {
2743  // If the scale is less than the size of the VT we expand to, the Hi and
2744  // Lo of the result will be in the first 2 parts of the result after
2745  // shifting right. This only requires shifting by the scale as far as the
2746  // third part in the result (ResultHL).
2747  SDValue SRLAmnt = DAG.getConstant(Scale, dl, ShiftTy);
2748  SDValue SHLAmnt = DAG.getConstant(NVTSize - Scale, dl, ShiftTy);
2749  Lo = DAG.getNode(ISD::SRL, dl, NVT, ResultLL, SRLAmnt);
2750  Lo = DAG.getNode(ISD::OR, dl, NVT, Lo,
2751  DAG.getNode(ISD::SHL, dl, NVT, ResultLH, SHLAmnt));
2752  Hi = DAG.getNode(ISD::SRL, dl, NVT, ResultLH, SRLAmnt);
2753  Hi = DAG.getNode(ISD::OR, dl, NVT, Hi,
2754  DAG.getNode(ISD::SHL, dl, NVT, ResultHL, SHLAmnt));
2755  } else if (Scale == NVTSize) {
2756  // If the scales are equal, Lo and Hi are ResultLH and Result HL,
2757  // respectively. Avoid shifting to prevent undefined behavior.
2758  Lo = ResultLH;
2759  Hi = ResultHL;
2760  } else if (Scale < VTSize) {
2761  // If the scale is instead less than the old VT size, but greater than or
2762  // equal to the expanded VT size, the first part of the result (ResultLL) is
2763  // no longer a part of Lo because it would be scaled out anyway. Instead we
2764  // can start shifting right from the fourth part (ResultHH) to the second
2765  // part (ResultLH), and Result LH will be the new Lo.
2766  SDValue SRLAmnt = DAG.getConstant(Scale - NVTSize, dl, ShiftTy);
2767  SDValue SHLAmnt = DAG.getConstant(VTSize - Scale, dl, ShiftTy);
2768  Lo = DAG.getNode(ISD::SRL, dl, NVT, ResultLH, SRLAmnt);
2769  Lo = DAG.getNode(ISD::OR, dl, NVT, Lo,
2770  DAG.getNode(ISD::SHL, dl, NVT, ResultHL, SHLAmnt));
2771  Hi = DAG.getNode(ISD::SRL, dl, NVT, ResultHL, SRLAmnt);
2772  Hi = DAG.getNode(ISD::OR, dl, NVT, Hi,
2773  DAG.getNode(ISD::SHL, dl, NVT, ResultHH, SHLAmnt));
2774  } else if (Scale == VTSize) {
2775  assert(
2776  !Signed &&
2777  "Only unsigned types can have a scale equal to the operand bit width");
2778 
2779  Lo = ResultHL;
2780  Hi = ResultHH;
2781  } else {
2782  llvm_unreachable("Expected the scale to be less than or equal to the width "
2783  "of the operands");
2784  }
2785 }
2786 
2787 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
2788  SDValue &Lo, SDValue &Hi) {
2789  SDValue LHS = Node->getOperand(0);
2790  SDValue RHS = Node->getOperand(1);
2791  SDLoc dl(Node);
2792 
2793  // Expand the result by simply replacing it with the equivalent
2794  // non-overflow-checking operation.
2795  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
2796  ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
2797  LHS, RHS);
2798  SplitInteger(Sum, Lo, Hi);
2799 
2800  // Compute the overflow.
2801  //
2802  // LHSSign -> LHS >= 0
2803  // RHSSign -> RHS >= 0
2804  // SumSign -> Sum >= 0
2805  //
2806  // Add:
2807  // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
2808  // Sub:
2809  // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
2810  //
2811  EVT OType = Node->getValueType(1);
2812  SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
2813 
2814  SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
2815  SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
2816  SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
2817  Node->getOpcode() == ISD::SADDO ?
2819 
2820  SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
2821  SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
2822 
2823  SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
2824 
2825  // Use the calculated overflow everywhere.
2826  ReplaceValueWith(SDValue(Node, 1), Cmp);
2827 }
2828 
2829 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
2830  SDValue &Lo, SDValue &Hi) {
2831  EVT VT = N->getValueType(0);
2832  SDLoc dl(N);
2833  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2834 
2836  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2837  SplitInteger(Res.getValue(0), Lo, Hi);
2838  return;
2839  }
2840 
2841  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2842  if (VT == MVT::i16)
2843  LC = RTLIB::SDIV_I16;
2844  else if (VT == MVT::i32)
2845  LC = RTLIB::SDIV_I32;
2846  else if (VT == MVT::i64)
2847  LC = RTLIB::SDIV_I64;
2848  else if (VT == MVT::i128)
2849  LC = RTLIB::SDIV_I128;
2850  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
2851 
2852  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
2853 }
2854 
2855 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
2856  SDValue &Lo, SDValue &Hi) {
2857  EVT VT = N->getValueType(0);
2858  SDLoc dl(N);
2859 
2860  // If we can emit an efficient shift operation, do so now. Check to see if
2861  // the RHS is a constant.
2862  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
2863  return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
2864 
2865  // If we can determine that the high bit of the shift is zero or one, even if
2866  // the low bits are variable, emit this shift in an optimized form.
2867  if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
2868  return;
2869 
2870  // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
2871  unsigned PartsOpc;
2872  if (N->getOpcode() == ISD::SHL) {
2873  PartsOpc = ISD::SHL_PARTS;
2874  } else if (N->getOpcode() == ISD::SRL) {
2875  PartsOpc = ISD::SRL_PARTS;
2876  } else {
2877  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2878  PartsOpc = ISD::SRA_PARTS;
2879  }
2880 
2881  // Next check to see if the target supports this SHL_PARTS operation or if it
2882  // will custom expand it. Don't lower this to SHL_PARTS when we optimise for
2883  // size, but create a libcall instead.
2884  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2885  TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
2886  const bool LegalOrCustom =
2887  (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
2888  Action == TargetLowering::Custom;
2889 
2890  if (LegalOrCustom && TLI.shouldExpandShift(DAG, N)) {
2891  // Expand the subcomponents.
2892  SDValue LHSL, LHSH;
2893  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2894  EVT VT = LHSL.getValueType();
2895 
2896  // If the shift amount operand is coming from a vector legalization it may
2897  // have an illegal type. Fix that first by casting the operand, otherwise
2898  // the new SHL_PARTS operation would need further legalization.
2899  SDValue ShiftOp = N->getOperand(1);
2900  EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2901  assert(ShiftTy.getScalarSizeInBits() >=
2902  Log2_32_Ceil(VT.getScalarSizeInBits()) &&
2903  "ShiftAmountTy is too small to cover the range of this type!");
2904  if (ShiftOp.getValueType() != ShiftTy)
2905  ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
2906 
2907  SDValue Ops[] = { LHSL, LHSH, ShiftOp };
2908  Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
2909  Hi = Lo.getValue(1);
2910  return;
2911  }
2912 
2913  // Otherwise, emit a libcall.
2914  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2915  bool isSigned;
2916  if (N->getOpcode() == ISD::SHL) {
2917  isSigned = false; /*sign irrelevant*/
2918  if (VT == MVT::i16)
2919  LC = RTLIB::SHL_I16;
2920  else if (VT == MVT::i32)
2921  LC = RTLIB::SHL_I32;
2922  else if (VT == MVT::i64)
2923  LC = RTLIB::SHL_I64;
2924  else if (VT == MVT::i128)
2925  LC = RTLIB::SHL_I128;
2926  } else if (N->getOpcode() == ISD::SRL) {
2927  isSigned = false;
2928  if (VT == MVT::i16)
2929  LC = RTLIB::SRL_I16;
2930  else if (VT == MVT::i32)
2931  LC = RTLIB::SRL_I32;
2932  else if (VT == MVT::i64)
2933  LC = RTLIB::SRL_I64;
2934  else if (VT == MVT::i128)
2935  LC = RTLIB::SRL_I128;
2936  } else {
2937  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2938  isSigned = true;
2939  if (VT == MVT::i16)
2940  LC = RTLIB::SRA_I16;
2941  else if (VT == MVT::i32)
2942  LC = RTLIB::SRA_I32;
2943  else if (VT == MVT::i64)
2944  LC = RTLIB::SRA_I64;
2945  else if (VT == MVT::i128)
2946  LC = RTLIB::SRA_I128;
2947  }
2948 
2949  if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
2950  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2951  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, isSigned, dl).first, Lo, Hi);
2952  return;
2953  }
2954 
2955  if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
2956  llvm_unreachable("Unsupported shift!");
2957 }
2958 
2959 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
2960  SDValue &Lo, SDValue &Hi) {
2961  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2962  SDLoc dl(N);
2963  SDValue Op = N->getOperand(0);
2964  if (Op.getValueType().bitsLE(NVT)) {
2965  // The low part is sign extension of the input (degenerates to a copy).
2966  Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
2967  // The high part is obtained by SRA'ing all but one of the bits of low part.
2968  unsigned LoSize = NVT.getSizeInBits();
2969  Hi = DAG.getNode(
2970  ISD::SRA, dl, NVT, Lo,
2971  DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
2972  } else {
2973  // For example, extension of an i48 to an i64. The operand type necessarily
2974  // promotes to the result type, so will end up being expanded too.
2975  assert(getTypeAction(Op.getValueType()) ==
2977  "Only know how to promote this result!");
2978  SDValue Res = GetPromotedInteger(Op);
2979  assert(Res.getValueType() == N->getValueType(0) &&
2980  "Operand over promoted?");
2981  // Split the promoted operand. This will simplify when it is expanded.
2982  SplitInteger(Res, Lo, Hi);
2983  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
2984  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2986  ExcessBits)));
2987  }
2988 }
2989 
2990 void DAGTypeLegalizer::
2991 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2992  SDLoc dl(N);
2993  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2994  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2995 
2996  if (EVT.bitsLE(Lo.getValueType())) {
2997  // sext_inreg the low part if needed.
2998  Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
2999  N->getOperand(1));
3000 
3001  // The high part gets the sign extension from the lo-part. This handles
3002  // things like sextinreg V:i64 from i8.
3003  Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
3004  DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
3005  TLI.getPointerTy(DAG.getDataLayout())));
3006  } else {
3007  // For example, extension of an i48 to an i64. Leave the low part alone,
3008  // sext_inreg the high part.
3009  unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
3010  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
3012  ExcessBits)));
3013  }
3014 }
3015 
3016 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
3017  SDValue &Lo, SDValue &Hi) {
3018  EVT VT = N->getValueType(0);
3019  SDLoc dl(N);
3020  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3021 
3023  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3024  SplitInteger(Res.getValue(1), Lo, Hi);
3025  return;
3026  }
3027 
3028  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3029  if (VT == MVT::i16)
3030  LC = RTLIB::SREM_I16;
3031  else if (VT == MVT::i32)
3032  LC = RTLIB::SREM_I32;
3033  else if (VT == MVT::i64)
3034  LC = RTLIB::SREM_I64;
3035  else if (VT == MVT::i128)
3036  LC = RTLIB::SREM_I128;
3037  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
3038 
3039  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
3040 }
3041 
3042 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
3043  SDValue &Lo, SDValue &Hi) {
3044  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3045  SDLoc dl(N);
3046  Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
3047  Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
3048  N->getOperand(0),
3049  DAG.getConstant(NVT.getSizeInBits(), dl,
3050  TLI.getPointerTy(DAG.getDataLayout())));
3051  Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
3052 }
3053 
3054 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
3055  SDValue &Lo, SDValue &Hi) {
3056  EVT VT = N->getValueType(0);
3057  SDLoc dl(N);
3058 
3059  if (N->getOpcode() == ISD::UMULO) {
3060  // This section expands the operation into the following sequence of
3061  // instructions. `iNh` here refers to a type which has half the bit width of
3062  // the type the original operation operated on.
3063  //
3064  // %0 = %LHS.HI != 0 && %RHS.HI != 0
3065  // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
3066  // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
3067  // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
3068  // %4 = add iN (%1.0 as iN) << Nh, (%2.0 as iN) << Nh
3069  // %5 = { iN, i1 } @uadd.with.overflow.iN( %4, %3 )
3070  //
3071  // %res = { %5.0, %0 || %1.1 || %2.1 || %5.1 }
3072  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
3073  SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
3074  SplitInteger(LHS, LHSLow, LHSHigh);
3075  SplitInteger(RHS, RHSLow, RHSHigh);
3076  EVT HalfVT = LHSLow.getValueType()
3077  , BitVT = N->getValueType(1);
3078  SDVTList VTHalfMulO = DAG.getVTList(HalfVT, BitVT);
3079  SDVTList VTFullAddO = DAG.getVTList(VT, BitVT);
3080 
3081  SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
3082  SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
3083  DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
3084  DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));
3085 
3086  SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, LHSHigh, RHSLow);
3087  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));
3088  SDValue OneInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3089  One.getValue(0));
3090 
3091  SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, RHSHigh, LHSLow);
3092  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));
3093  SDValue TwoInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3094  Two.getValue(0));
3095 
3096  // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
3097  // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
3098  // operation recursively legalized?).
3099  //
3100  // Many backends understand this pattern and will convert into LOHI
3101  // themselves, if applicable.
3102  SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
3103  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
3104  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow));
3105  SDValue Four = DAG.getNode(ISD::ADD, dl, VT, OneInHigh, TwoInHigh);
3106  SDValue Five = DAG.getNode(ISD::UADDO, dl, VTFullAddO, Three, Four);
3107  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Five.getValue(1));
3108  SplitInteger(Five, Lo, Hi);
3109  ReplaceValueWith(SDValue(N, 1), Overflow);
3110  return;
3111  }
3112 
3113  Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
3114  EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
3115  Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
3116 
3117  // Replace this with a libcall that will check overflow.
3118  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3119  if (VT == MVT::i32)
3120  LC = RTLIB::MULO_I32;
3121  else if (VT == MVT::i64)
3122  LC = RTLIB::MULO_I64;
3123  else if (VT == MVT::i128)
3124  LC = RTLIB::MULO_I128;
3125  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
3126 
3127  SDValue Temp = DAG.CreateStackTemporary(PtrVT);
3128  // Temporary for the overflow value, default it to zero.
3129  SDValue Chain =
3130  DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
3131  MachinePointerInfo());
3132 
3134  TargetLowering::ArgListEntry Entry;
3135  for (const SDValue &Op : N->op_values()) {
3136  EVT ArgVT = Op.getValueType();
3137  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
3138  Entry.Node = Op;
3139  Entry.Ty = ArgTy;
3140  Entry.IsSExt = true;
3141  Entry.IsZExt = false;
3142  Args.push_back(Entry);
3143  }
3144 
3145  // Also pass the address of the overflow check.
3146  Entry.Node = Temp;
3147  Entry.Ty = PtrTy->getPointerTo();
3148  Entry.IsSExt = true;
3149  Entry.IsZExt = false;
3150  Args.push_back(Entry);
3151 
3152  SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
3153 
3155  CLI.setDebugLoc(dl)
3156  .setChain(Chain)
3157  .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
3158  .setSExtResult();
3159 
3160  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
3161 
3162  SplitInteger(CallInfo.first, Lo, Hi);
3163  SDValue Temp2 =
3164  DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
3165  SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
3166  DAG.getConstant(0, dl, PtrVT),
3167  ISD::SETNE);
3168  // Use the overflow from the libcall everywhere.
3169  ReplaceValueWith(SDValue(N, 1), Ofl);
3170 }
3171 
3172 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
3173  SDValue &Lo, SDValue &Hi) {
3174  EVT VT = N->getValueType(0);
3175  SDLoc dl(N);
3176  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3177 
3179  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3180  SplitInteger(Res.getValue(0), Lo, Hi);
3181  return;
3182  }
3183 
3184  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3185  if (VT == MVT::i16)
3186  LC = RTLIB::UDIV_I16;
3187  else if (VT == MVT::i32)
3188  LC = RTLIB::UDIV_I32;
3189  else if (VT == MVT::i64)
3190  LC = RTLIB::UDIV_I64;
3191  else if (VT == MVT::i128)
3192  LC = RTLIB::UDIV_I128;
3193  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
3194 
3195  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
3196 }
3197 
3198 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
3199  SDValue &Lo, SDValue &Hi) {
3200  EVT VT = N->getValueType(0);
3201  SDLoc dl(N);
3202  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3203 
3205  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3206  SplitInteger(Res.getValue(1), Lo, Hi);
3207  return;
3208  }
3209 
3210  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3211  if (VT == MVT::i16)
3212  LC = RTLIB::UREM_I16;
3213  else if (VT == MVT::i32)
3214  LC = RTLIB::UREM_I32;
3215  else if (VT == MVT::i64)
3216  LC = RTLIB::UREM_I64;
3217  else if (VT == MVT::i128)
3218  LC = RTLIB::UREM_I128;
3219  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
3220 
3221  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
3222 }
3223 
3224 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
3225  SDValue &Lo, SDValue &Hi) {
3226  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3227  SDLoc dl(N);
3228  SDValue Op = N->getOperand(0);
3229  if (Op.getValueType().bitsLE(NVT)) {
3230  // The low part is zero extension of the input (degenerates to a copy).
3231  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
3232  Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
3233  } else {
3234  // For example, extension of an i48 to an i64. The operand type necessarily
3235  // promotes to the result type, so will end up being expanded too.
3236  assert(getTypeAction(Op.getValueType()) ==
3238  "Only know how to promote this result!");
3239  SDValue Res = GetPromotedInteger(Op);
3240  assert(Res.getValueType() == N->getValueType(0) &&
3241  "Operand over promoted?");
3242  // Split the promoted operand. This will simplify when it is expanded.
3243  SplitInteger(Res, Lo, Hi);
3244  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
3245  Hi = DAG.getZeroExtendInReg(Hi, dl,
3247  ExcessBits));
3248  }
3249 }
3250 
3251 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
3252  SDValue &Lo, SDValue &Hi) {
3253  SDLoc dl(N);
3254  EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
3255  SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
3256  SDValue Zero = DAG.getConstant(0, dl, VT);
3257  SDValue Swap = DAG.getAtomicCmpSwap(
3259  cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
3260  N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
3261 
3262  ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
3263  ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
3264 }
3265 
3266 void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N,
3267  SDValue &Lo, SDValue &Hi) {
3268  // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate
3269  // both halves independently.
3270  SDValue Res = TLI.expandVecReduce(N, DAG);
3271  SplitInteger(Res, Lo, Hi);
3272 }
3273 
3274 //===----------------------------------------------------------------------===//
3275 // Integer Operand Expansion
3276 //===----------------------------------------------------------------------===//
3277 
3278 /// ExpandIntegerOperand - This method is called when the specified operand of
3279 /// the specified node is found to need expansion. At this point, all of the
3280 /// result types of the node are known to be legal, but other operands of the
3281 /// node may need promotion or expansion as well as the specified one.
3282 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
3283  LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG);
3284  dbgs() << "\n");
3285  SDValue Res = SDValue();
3286 
3287  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
3288  return false;
3289 
3290  switch (N->getOpcode()) {
3291  default:
3292  #ifndef NDEBUG
3293  dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
3294  N->dump(&DAG); dbgs() << "\n";
3295  #endif
3296  report_fatal_error("Do not know how to expand this operator's operand!");
3297 
3298  case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
3299  case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
3300  case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
3301  case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
3302  case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
3303  case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
3304  case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
3305  case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
3306  case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
3307  case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break;
3308  case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
3309  case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
3310  case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break;
3311 
3312  case ISD::SHL:
3313  case ISD::SRA:
3314  case ISD::SRL:
3315  case ISD::ROTL:
3316  case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
3317  case ISD::RETURNADDR:
3318  case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
3319 
3320  case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
3321  }
3322 
3323  // If the result is null, the sub-method took care of registering results etc.
3324  if (!Res.getNode()) return false;
3325 
3326  // If the result is N, the sub-method updated N in place. Tell the legalizer
3327  // core about this.
3328  if (Res.getNode() == N)
3329  return true;
3330 
3331  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3332  "Invalid operand expansion");
3333 
3334  ReplaceValueWith(SDValue(N, 0), Res);
3335  return false;
3336 }
3337 
3338 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
3339 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
3340 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
3341  SDValue &NewRHS,
3342  ISD::CondCode &CCCode,
3343  const SDLoc &dl) {
3344  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3345  GetExpandedInteger(NewLHS, LHSLo, LHSHi);
3346  GetExpandedInteger(NewRHS, RHSLo, RHSHi);
3347 
3348  if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
3349  if (RHSLo == RHSHi) {
3350  if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
3351  if (RHSCST->isAllOnesValue()) {
3352  // Equality comparison to -1.
3353  NewLHS = DAG.getNode(ISD::AND, dl,
3354  LHSLo.getValueType(), LHSLo, LHSHi);
3355  NewRHS = RHSLo;
3356  return;
3357  }
3358  }
3359  }
3360 
3361  NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
3362  NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
3363  NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
3364  NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
3365  return;
3366  }
3367 
3368  // If this is a comparison of the sign bit, just look at the top part.
3369  // X > -1, x < 0
3370  if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
3371  if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0
3372  (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1
3373  NewLHS = LHSHi;
3374  NewRHS = RHSHi;
3375  return;
3376  }
3377 
3378  // FIXME: This generated code sucks.
3379  ISD::CondCode LowCC;
3380  switch (CCCode) {
3381  default: llvm_unreachable("Unknown integer setcc!");
3382  case ISD::SETLT:
3383  case ISD::SETULT: LowCC = ISD::SETULT; break;
3384  case ISD::SETGT:
3385  case ISD::SETUGT: LowCC = ISD::SETUGT; break;
3386  case ISD::SETLE:
3387  case ISD::SETULE: LowCC = ISD::SETULE; break;
3388  case ISD::SETGE:
3389  case ISD::SETUGE: LowCC = ISD::SETUGE; break;
3390  }
3391 
3392  // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
3393  // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
3394  // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
3395 
3396  // NOTE: on targets without efficient SELECT of bools, we can always use
3397  // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
3398  TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
3399  nullptr);
3400  SDValue LoCmp, HiCmp;
3401  if (TLI.isTypeLegal(LHSLo.getValueType()) &&
3402  TLI.isTypeLegal(RHSLo.getValueType()))
3403  LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
3404  RHSLo, LowCC, false, DagCombineInfo, dl);
3405  if (!LoCmp.getNode())
3406  LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
3407  RHSLo, LowCC);
3408  if (TLI.isTypeLegal(LHSHi.getValueType()) &&
3409  TLI.isTypeLegal(RHSHi.getValueType()))
3410  HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
3411  RHSHi, CCCode, false, DagCombineInfo, dl);
3412  if (!HiCmp.getNode())
3413  HiCmp =
3414  DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
3415  LHSHi, RHSHi, DAG.getCondCode(CCCode));
3416 
3417  ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
3418  ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
3419 
3420  bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
3421  CCCode == ISD::SETUGE || CCCode == ISD::SETULE);
3422 
3423  if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) ||
3424  (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) ||
3425  (LoCmpC && LoCmpC->isNullValue())))) {
3426  // For LE / GE, if high part is known false, ignore the low part.
3427  // For LT / GT: if low part is known false, return the high part.
3428  // if high part is known true, ignore the low part.
3429  NewLHS = HiCmp;
3430  NewRHS = SDValue();
3431  return;
3432  }
3433 
3434  if (LHSHi == RHSHi) {
3435  // Comparing the low bits is enough.
3436  NewLHS = LoCmp;
3437  NewRHS = SDValue();
3438  return;
3439  }
3440 
3441  // Lower with SETCCCARRY if the target supports it.
3442  EVT HiVT = LHSHi.getValueType();
3443  EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
3444  bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
3445 
3446  // FIXME: Make all targets support this, then remove the other lowering.
3447  if (HasSETCCCARRY) {
3448  // SETCCCARRY can detect < and >= directly. For > and <=, flip
3449  // operands and condition code.
3450  bool FlipOperands = false;
3451  switch (CCCode) {
3452  case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
3453  case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
3454  case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
3455  case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
3456  default: break;
3457  }
3458  if (FlipOperands) {
3459  std::swap(LHSLo, RHSLo);
3460  std::swap(LHSHi, RHSHi);
3461  }
3462  // Perform a wide subtraction, feeding the carry from the low part into
3463  // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
3464  // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
3465  // zero or positive iff LHS >= RHS.
3466  EVT LoVT = LHSLo.getValueType();
3467  SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
3468  SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
3469  SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
3470  LHSHi, RHSHi, LowCmp.getValue(1),
3471  DAG.getCondCode(CCCode));
3472  NewLHS = Res;
3473  NewRHS = SDValue();
3474  return;
3475  }
3476 
3477  NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
3478  false, DagCombineInfo, dl);
3479  if (!NewLHS.getNode())
3480  NewLHS =
3481  DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
3482  NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
3483  NewRHS = SDValue();
3484 }
3485 
3486 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
3487  SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
3488  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
3489  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3490 
3491  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3492  // against zero to select between true and false values.
3493  if (!NewRHS.getNode()) {
3494  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3495  CCCode = ISD::SETNE;
3496  }
3497 
3498  // Update N to have the operands specified.
3499  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
3500  DAG.getCondCode(CCCode), NewLHS, NewRHS,
3501  N->getOperand(4)), 0);
3502 }
3503 
3504 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
3505  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3506  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
3507  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3508 
3509  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3510  // against zero to select between true and false values.
3511  if (!NewRHS.getNode()) {
3512  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3513  CCCode = ISD::SETNE;
3514  }
3515 
3516  // Update N to have the operands specified.
3517  return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
3518  N->getOperand(2), N->getOperand(3),
3519  DAG.getCondCode(CCCode)), 0);
3520 }
3521 
3522 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
3523  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3524  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3525  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3526 
3527  // If ExpandSetCCOperands returned a scalar, use it.
3528  if (!NewRHS.getNode()) {
3529  assert(NewLHS.getValueType() == N->getValueType(0) &&
3530  "Unexpected setcc expansion!");
3531  return NewLHS;
3532  }
3533 
3534  // Otherwise, update N to have the operands specified.
3535  return SDValue(
3536  DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
3537 }
3538 
3539 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
3540  SDValue LHS = N->getOperand(0);
3541  SDValue RHS = N->getOperand(1);
3542  SDValue Carry = N->getOperand(2);
3543  SDValue Cond = N->getOperand(3);
3544  SDLoc dl = SDLoc(N);
3545 
3546  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3547  GetExpandedInteger(LHS, LHSLo, LHSHi);
3548  GetExpandedInteger(RHS, RHSLo, RHSHi);
3549 
3550  // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
3551  SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
3552  SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
3553  return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
3554  LowCmp.getValue(1), Cond);
3555 }
3556 
3557 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
3558  // The value being shifted is legal, but the shift amount is too big.
3559  // It follows that either the result of the shift is undefined, or the
3560  // upper half of the shift amount is zero. Just use the lower half.
3561  SDValue Lo, Hi;
3562  GetExpandedInteger(N->getOperand(1), Lo, Hi);
3563  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
3564 }
3565 
3566 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
3567  // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
3568  // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
3569  // constant to valid type.
3570  SDValue Lo, Hi;
3571  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3572  return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
3573 }
3574 
3575 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
3576  SDValue Op = N->getOperand(0);
3577  EVT DstVT = N->getValueType(0);
3579  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3580  "Don't know how to expand this SINT_TO_FP!");
3581  return TLI.makeLibCall(DAG, LC, DstVT, Op, true, SDLoc(N)).first;
3582 }
3583 
3584 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
3585  if (ISD::isNormalStore(N))
3586  return ExpandOp_NormalStore(N, OpNo);
3587 
3588  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
3589  assert(OpNo == 1 && "Can only expand the stored value so far");
3590 
3591  EVT VT = N->getOperand(1).getValueType();
3592  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3593  SDValue Ch = N->getChain();
3594  SDValue Ptr = N->getBasePtr();
3595  unsigned Alignment = N->getAlignment();
3596  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3597  AAMDNodes AAInfo = N->getAAInfo();
3598  SDLoc dl(N);
3599  SDValue Lo, Hi;
3600 
3601  assert(NVT.isByteSized() && "Expanded type not byte sized!");
3602 
3603  if (N->getMemoryVT().bitsLE(NVT)) {
3604  GetExpandedInteger(N->getValue(), Lo, Hi);
3605  return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
3606  N->getMemoryVT(), Alignment, MMOFlags, AAInfo);
3607  }
3608 
3609  if (DAG.getDataLayout().isLittleEndian()) {
3610  // Little-endian - low bits are at low addresses.
3611  GetExpandedInteger(N->getValue(), Lo, Hi);
3612 
3613  Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3614  AAInfo);
3615 
3616  unsigned ExcessBits =
3617  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3618  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3619 
3620  // Increment the pointer to the other half.
3621  unsigned IncrementSize = NVT.getSizeInBits()/8;
3622  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3623  Hi = DAG.getTruncStore(
3624  Ch, dl, Hi, Ptr, N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3625  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3626  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3627  }
3628 
3629  // Big-endian - high bits are at low addresses. Favor aligned stores at
3630  // the cost of some bit-fiddling.
3631  GetExpandedInteger(N->getValue(), Lo, Hi);
3632 
3633  EVT ExtVT = N->getMemoryVT();
3634  unsigned EBytes = ExtVT.getStoreSize();
3635  unsigned IncrementSize = NVT.getSizeInBits()/8;
3636  unsigned ExcessBits = (EBytes - IncrementSize)*8;
3637  EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
3638  ExtVT.getSizeInBits() - ExcessBits);
3639 
3640  if (ExcessBits < NVT.getSizeInBits()) {
3641  // Transfer high bits from the top of Lo to the bottom of Hi.
3642  Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
3643  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3644  TLI.getPointerTy(DAG.getDataLayout())));
3645  Hi = DAG.getNode(
3646  ISD::OR, dl, NVT, Hi,
3647  DAG.getNode(ISD::SRL, dl, NVT, Lo,
3648  DAG.getConstant(ExcessBits, dl,
3649  TLI.getPointerTy(DAG.getDataLayout()))));
3650  }
3651 
3652  // Store both the high bits and maybe some of the low bits.
3653  Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment,
3654  MMOFlags, AAInfo);
3655 
3656  // Increment the pointer to the other half.
3657  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3658  // Store the lowest ExcessBits bits in the second half.
3659  Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
3660  N->getPointerInfo().getWithOffset(IncrementSize),
3661  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3662  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3663  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3664 }
3665 
3666 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
3667  SDValue InL, InH;
3668  GetExpandedInteger(N->getOperand(0), InL, InH);
3669  // Just truncate the low part of the source.
3670  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
3671 }
3672 
3673 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
3674  SDValue Op = N->getOperand(0);
3675  EVT SrcVT = Op.getValueType();
3676  EVT DstVT = N->getValueType(0);
3677  SDLoc dl(N);
3678 
3679  // The following optimization is valid only if every value in SrcVT (when
3680  // treated as signed) is representable in DstVT. Check that the mantissa
3681  // size of DstVT is >= than the number of bits in SrcVT -1.
3682  const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT);
3683  if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 &&
3685  // Do a signed conversion then adjust the result.
3686  SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op);
3687  SignedConv = TLI.LowerOperation(SignedConv, DAG);
3688 
3689  // The result of the signed conversion needs adjusting if the 'sign bit' of
3690  // the incoming integer was set. To handle this, we dynamically test to see
3691  // if it is set, and, if so, add a fudge factor.
3692 
3693  const uint64_t F32TwoE32 = 0x4F800000ULL;
3694  const uint64_t F32TwoE64 = 0x5F800000ULL;
3695  const uint64_t F32TwoE128 = 0x7F800000ULL;
3696 
3697  APInt FF(32, 0);
3698  if (SrcVT == MVT::i32)
3699  FF = APInt(32, F32TwoE32);
3700  else if (SrcVT == MVT::i64)
3701  FF = APInt(32, F32TwoE64);
3702  else if (SrcVT == MVT::i128)
3703  FF = APInt(32, F32TwoE128);
3704  else
3705  llvm_unreachable("Unsupported UINT_TO_FP!");
3706 
3707  // Check whether the sign bit is set.
3708  SDValue Lo, Hi;
3709  GetExpandedInteger(Op, Lo, Hi);
3710  SDValue SignSet = DAG.getSetCC(dl,
3711  getSetCCResultType(Hi.getValueType()),
3712  Hi,
3713  DAG.getConstant(0, dl, Hi.getValueType()),
3714  ISD::SETLT);
3715 
3716  // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
3717  SDValue FudgePtr =
3718  DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)),
3719  TLI.getPointerTy(DAG.getDataLayout()));
3720 
3721  // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
3722  SDValue Zero = DAG.getIntPtrConstant(0, dl);
3723  SDValue Four = DAG.getIntPtrConstant(4, dl);
3724  if (DAG.getDataLayout().isBigEndian())
3725  std::swap(Zero, Four);
3726  SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet,
3727  Zero, Four);
3728  unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment();
3729  FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(),
3730  FudgePtr, Offset);
3731  Alignment = std::min(Alignment, 4u);
3732 
3733  // Load the value out, extending it from f32 to the destination float type.
3734  // FIXME: Avoid the extend by constructing the right constant pool?
3735  SDValue Fudge = DAG.getExtLoad(
3736  ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), FudgePtr,
3738  Alignment);
3739  return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge);
3740  }
3741 
3742  // Otherwise, use a libcall.
3743  RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT);
3744  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3745  "Don't know how to expand this UINT_TO_FP!");
3746  return TLI.makeLibCall(DAG, LC, DstVT, Op, true, dl).first;
3747 }
3748 
3749 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
3750  SDLoc dl(N);
3751  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
3752  cast<AtomicSDNode>(N)->getMemoryVT(),
3753  N->getOperand(0),
3754  N->getOperand(1), N->getOperand(2),
3755  cast<AtomicSDNode>(N)->getMemOperand());
3756  return Swap.getValue(1);
3757 }
3758 
3759 
3760 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
3761 
3762  EVT OutVT = N->getValueType(0);
3763  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3764  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3765  unsigned OutNumElems = OutVT.getVectorNumElements();
3766  EVT NOutVTElem = NOutVT.getVectorElementType();
3767 
3768  SDLoc dl(N);
3769  SDValue BaseIdx = N->getOperand(1);
3770 
3771  SDValue InOp0 = N->getOperand(0);
3772  if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger)
3773  InOp0 = GetPromotedInteger(N->getOperand(0));
3774 
3775  EVT InVT = InOp0.getValueType();
3776 
3778  Ops.reserve(OutNumElems);
3779  for (unsigned i = 0; i != OutNumElems; ++i) {
3780 
3781  // Extract the element from the original vector.
3782  SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
3783  BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
3785  InVT.getVectorElementType(), N->getOperand(0), Index);
3786 
3787  SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem);
3788  // Insert the converted element to the new vector.
3789  Ops.push_back(Op);
3790  }
3791 
3792  return DAG.getBuildVector(NOutVT, dl, Ops);
3793 }
3794 
3795 
3796 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
3797  ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
3798  EVT VT = N->getValueType(0);
3799  SDLoc dl(N);
3800 
3801  ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
3802 
3803  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3804  SDValue V1 = GetPromotedInteger(N->getOperand(1));
3805  EVT OutVT = V0.getValueType();
3806 
3807  return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
3808 }
3809 
3810 
3811 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
3812  EVT OutVT = N->getValueType(0);
3813  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3814  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3815  unsigned NumElems = N->getNumOperands();
3816  EVT NOutVTElem = NOutVT.getVectorElementType();
3817 
3818  SDLoc dl(N);
3819 
3821  Ops.reserve(NumElems);
3822  for (unsigned i = 0; i != NumElems; ++i) {
3823  SDValue Op;
3824  // BUILD_VECTOR integer operand types are allowed to be larger than the
3825  // result's element type. This may still be true after the promotion. For
3826  // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
3827  // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
3828  if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
3829  Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
3830  else
3831  Op = N->getOperand(i);
3832  Ops.push_back(Op);
3833  }
3834 
3835  return DAG.getBuildVector(NOutVT, dl, Ops);
3836 }
3837 
3838 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
3839 
3840  SDLoc dl(N);
3841 
3842  assert(!N->getOperand(0).getValueType().isVector() &&
3843  "Input must be a scalar");
3844 
3845  EVT OutVT = N->getValueType(0);
3846  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3847  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3848  EVT NOutVTElem = NOutVT.getVectorElementType();
3849 
3850  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
3851 
3852  return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
3853 }
3854 
3855 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
3856  SDLoc dl(N);
3857 
3858  EVT OutVT = N->getValueType(0);
3859  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3860  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3861 
3862  EVT OutElemTy = NOutVT.getVectorElementType();
3863 
3864  unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
3865  unsigned NumOutElem = NOutVT.getVectorNumElements();
3866  unsigned NumOperands = N->getNumOperands();
3867  assert(NumElem * NumOperands == NumOutElem &&
3868  "Unexpected number of elements");
3869 
3870  // Take the elements from the first vector.
3871  SmallVector<SDValue, 8> Ops(NumOutElem);
3872  for (unsigned i = 0; i < NumOperands; ++i) {
3873  SDValue Op = N->getOperand(i);
3874  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
3875  Op = GetPromotedInteger(Op);
3876  EVT SclrTy = Op.getValueType().getVectorElementType();
3877  assert(NumElem == Op.getValueType().getVectorNumElements() &&
3878  "Unexpected number of elements");
3879 
3880  for (unsigned j = 0; j < NumElem; ++j) {
3881  SDValue Ext = DAG.getNode(
3882  ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
3883  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3884  Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
3885  }
3886  }
3887 
3888  return DAG.getBuildVector(NOutVT, dl, Ops);
3889 }
3890 
3891 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
3892  EVT VT = N->getValueType(0);
3893  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3894  assert(NVT.isVector() && "This type must be promoted to a vector type");
3895 
3896  SDLoc dl(N);
3897 
3898  // For operands whose TypeAction is to promote, extend the promoted node
3899  // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
3900  // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
3901  // type..
3902  if (getTypeAction(N->getOperand(0).getValueType())
3904  SDValue Promoted;
3905 
3906  switch(N->getOpcode()) {
3908  Promoted = SExtPromotedInteger(N->getOperand(0));
3909  break;
3911  Promoted = ZExtPromotedInteger(N->getOperand(0));
3912  break;
3914  Promoted = GetPromotedInteger(N->getOperand(0));
3915  break;
3916  default:
3917  llvm_unreachable("Node has unexpected Opcode");
3918  }
3919  return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
3920  }
3921 
3922  // Directly extend to the appropriate transform-to type.
3923  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
3924 }
3925 
3926 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
3927  EVT OutVT = N->getValueType(0);
3928  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3929  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3930 
3931  EVT NOutVTElem = NOutVT.getVectorElementType();
3932 
3933  SDLoc dl(N);
3934  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3935 
3936  SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
3937  NOutVTElem, N->getOperand(1));
3938  return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
3939  V0, ConvElem, N->getOperand(2));
3940 }
3941 
3942 SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) {
3943  // The VECREDUCE result size may be larger than the element size, so
3944  // we can simply change the result type.
3945  SDLoc dl(N);
3946  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3947  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
3948 }
3949 
3950 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3951  SDLoc dl(N);
3952  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3953  SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
3954  TLI.getVectorIdxTy(DAG.getDataLayout()));
3956  V0->getValueType(0).getScalarType(), V0, V1);
3957 
3958  // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
3959  // element types. If this is the case then we need to expand the outgoing
3960  // value and not truncate it.
3961  return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
3962 }
3963 
3964 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
3965  SDLoc dl(N);
3966  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3967  MVT InVT = V0.getValueType().getSimpleVT();
3968  MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
3970  SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
3971  return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
3972 }
3973 
3974 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
3975  SDLoc dl(N);
3976  unsigned NumElems = N->getNumOperands();
3977 
3978  EVT RetSclrTy = N->getValueType(0).getVectorElementType();
3979 
3980  SmallVector<SDValue, 8> NewOps;
3981  NewOps.reserve(NumElems);
3982 
3983  // For each incoming vector
3984  for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
3985  SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
3986  EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
3987  unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
3988 
3989  for (unsigned i=0; i<NumElem; ++i) {
3990  // Extract element from incoming vector
3991  SDValue Ex = DAG.getNode(
3992  ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
3993  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3994  SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
3995  NewOps.push_back(Tr);
3996  }
3997  }
3998 
3999  return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
4000 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:570
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:551
static EVT getShiftAmountTyForConstant(unsigned Val, EVT VT, const TargetLowering &TLI, SelectionDAG &DAG)
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getValueType() const
Return the ValueType of the referenced return value.
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant, which is required to be operand #1) half of the integer or float value specified as operand #0.
Definition: ISDOpcodes.h:183
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC&#39;s if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:942
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:357
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:651
This class represents lattice values for constants.
Definition: AllocatorList.h:23
static MVT getVectorVT(MVT VT, unsigned NumElements)
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:366
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:259
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:518
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none...
const SDValue & getBasePtr() const
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:222
void push_back(const T &Elt)
Definition: SmallVector.h:211
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:857
const SDValue & getValue() const
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
Definition: APInt.h:647
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:252
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:250
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1328
const SDValue & getBasePtr() const
const SDValue & getChain() const
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
unsigned getAlignment() const
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:806
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select&#39;s if you just have operands and don&#39;t want to check...
Definition: SelectionDAG.h:955
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:507
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:383
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Same for subtraction.
Definition: ISDOpcodes.h:253
void reserve(size_type N)
Definition: SmallVector.h:369
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:211
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align=0, int Offs=0, bool isT=false, unsigned char TargetFlags=0)
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:434
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:38
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
SDValue getExternalSymbol(const char *Sym, EVT VT)
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
virtual bool shouldExpandShift(SelectionDAG &DAG, SDNode *N) const
Return true if SHIFT instructions should be expanded to SHIFT_PARTS instructions, and false if a libr...
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:209
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations...
Definition: ISDOpcodes.h:455
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
static uint32_t getAlignment(const MCSectionCOFF &Sec)
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:450
Shift and rotation operations.
Definition: ISDOpcodes.h:409
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:205
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth...
Definition: ISDOpcodes.h:392
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:651
CallLoweringInfo & setChain(SDValue InChain)
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:190
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:279
const SDValue & getPassThru() const
op_iterator op_end() const
unsigned getScalarValueSizeInBits() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
FLT_ROUNDS_ - Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest 2 Round to ...
Definition: ISDOpcodes.h:545
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:303
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt) For double-word atomic operations: ValLo, ValHi, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amtLo, amtHi) ValLo, ValHi, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amtLo, amtHi) These correspond to the atomicrmw instruction.
Definition: ISDOpcodes.h:814
const SDValue & getValue() const
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:462
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:403
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG...
Definition: ISDOpcodes.h:72
This is an SDNode representing atomic operations.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
This represents a list of ValueType&#39;s that has been intern&#39;d by a SelectionDAG.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
This class is used to represent an MSTORE node.
unsigned getSizeInBits() const
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:266
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:297
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:291
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:400
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:477
SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
const SDValue & getScale() const
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:428
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:232
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:857
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTruncating=false, bool IsCompressing=false)
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:968
op_iterator op_begin() const
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:751
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:496
const SDValue & getMask() const
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, bool isSigned, const SDLoc &dl, bool doesNotReturn=false, bool isReturnValueUsed=true, bool isPostTypeLegalization=false) const
Returns a pair of (return value, chain).
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands...
UNDEF - An undefined node.
Definition: ISDOpcodes.h:177
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:523
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:326
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
Definition: APInt.h:635
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits...
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
Definition: MathExtras.h:609
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const SDValue & getBasePtr() const
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:234
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:428
Machine Value Type.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
Simple binary floating point operators.
Definition: ISDOpcodes.h:282
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:272
This is an important base class in LLVM.
Definition: Constant.h:41
iterator_range< value_op_iterator > op_values() const
const SDValue & getOperand(unsigned Num) const
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:943
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:331
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:246
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:231
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:587
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:56
const SDValue & getValue() const
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:723
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType, bool IsExpanding=false)
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:415
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo...
Definition: ISDOpcodes.h:800
std::vector< ArgListEntry > ArgListTy
const SDValue & getIndex() const
Extended Value Type.
Definition: ValueTypes.h:33
const SDValue & getBasePtr() const
This structure contains all information that is necessary for lowering calls.
This class contains a discriminated union of information about pointers in memory operands...
unsigned getNumOperands() const
Return the number of values used by this operation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type...
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:264
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:49
void dump() const
Dump this node, for debugging.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:338
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:264
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
Definition: ISDOpcodes.h:450
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:221
const SDValue & getMask() const
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:841
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:739
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:645
const SDValue & getPassThru() const
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:643
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:412
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:580
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:631
Represents one node in the SelectionDAG.
static unsigned int semanticsPrecision(const fltSemantics &)
Definition: APFloat.cpp:154
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
MachinePointerInfo getWithOffset(int64_t O) const
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:538
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:940
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT...
Definition: ValueTypes.h:72
EVT getMemoryVT() const
Return the type of the in-memory value.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
Class for arbitrary precision integers.
Definition: APInt.h:69
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:419
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:240
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:467
const SDValue & getBasePtr() const
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:470
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition: APInt.h:529
Flags
Flags values. These may be or&#39;d together.
#define Success
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool ugt(const APInt &RHS) const
Unsigned greather than comparison.
Definition: APInt.h:1254
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:545
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:150
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
Definition: APInt.h:1320
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:386
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:205
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:485
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:615