LLVM  10.0.0svn
LegalizeIntegerTypes.cpp
Go to the documentation of this file.
1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements integer type expansion and promotion for LegalizeTypes.
10 // Promotion is the act of changing a computation in an illegal type into a
11 // computation in a larger type. For example, implementing i8 arithmetic in an
12 // i32 register (often needed on powerpc).
13 // Expansion is the act of changing a computation in an illegal type into a
14 // computation in two identical registers of a smaller type. For example,
15 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
16 // targets).
17 //
18 //===----------------------------------------------------------------------===//
19 
20 #include "LegalizeTypes.h"
21 #include "llvm/IR/DerivedTypes.h"
23 #include "llvm/Support/KnownBits.h"
25 using namespace llvm;
26 
27 #define DEBUG_TYPE "legalize-types"
28 
29 //===----------------------------------------------------------------------===//
30 // Integer Result Promotion
31 //===----------------------------------------------------------------------===//
32 
33 /// PromoteIntegerResult - This method is called when a result of a node is
34 /// found to be in need of promotion to a larger type. At this point, the node
35 /// may also have invalid operands or may have other results that need
36 /// expansion, we just know that (at least) one result needs promotion.
37 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
38  LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG);
39  dbgs() << "\n");
40  SDValue Res = SDValue();
41 
42  // See if the target wants to custom expand this node.
43  if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
44  LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
45  return;
46  }
47 
48  switch (N->getOpcode()) {
49  default:
50 #ifndef NDEBUG
51  dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
52  N->dump(&DAG); dbgs() << "\n";
53 #endif
54  llvm_unreachable("Do not know how to promote this operator!");
55  case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
56  case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
57  case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
58  case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
59  case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
60  case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
61  case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
62  case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
64  case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
65  case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break;
67  case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
69  Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
70  case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
71  case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
72  break;
73  case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
74  break;
75  case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
76  case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
77  case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
78  case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
79  case ISD::SMIN:
80  case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
81  case ISD::UMIN:
82  case ISD::UMAX: Res = PromoteIntRes_ZExtIntBinOp(N); break;
83 
84  case ISD::SHL: Res = PromoteIntRes_SHL(N); break;
86  Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
87  case ISD::SRA: Res = PromoteIntRes_SRA(N); break;
88  case ISD::SRL: Res = PromoteIntRes_SRL(N); break;
89  case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
90  case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
91  case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
92 
94  Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
96  Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
98  Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
99  case ISD::BUILD_VECTOR:
100  Res = PromoteIntRes_BUILD_VECTOR(N); break;
102  Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
103  case ISD::CONCAT_VECTORS:
104  Res = PromoteIntRes_CONCAT_VECTORS(N); break;
105 
109  Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
110 
111  case ISD::SIGN_EXTEND:
112  case ISD::ZERO_EXTEND:
113  case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
114 
115  case ISD::FP_TO_SINT:
116  case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
117 
118  case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
119 
120  case ISD::FLT_ROUNDS_: Res = PromoteIntRes_FLT_ROUNDS(N); break;
121 
122  case ISD::AND:
123  case ISD::OR:
124  case ISD::XOR:
125  case ISD::ADD:
126  case ISD::SUB:
127  case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
128 
129  case ISD::SDIV:
130  case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
131 
132  case ISD::UDIV:
133  case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
134 
135  case ISD::SADDO:
136  case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
137  case ISD::UADDO:
138  case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
139  case ISD::SMULO:
140  case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
141 
142  case ISD::ADDE:
143  case ISD::SUBE:
144  case ISD::ADDCARRY:
145  case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
146 
147  case ISD::SADDSAT:
148  case ISD::UADDSAT:
149  case ISD::SSUBSAT:
150  case ISD::USUBSAT: Res = PromoteIntRes_ADDSUBSAT(N); break;
151  case ISD::SMULFIX:
152  case ISD::SMULFIXSAT:
153  case ISD::UMULFIX: Res = PromoteIntRes_MULFIX(N); break;
154  case ISD::ABS: Res = PromoteIntRes_ABS(N); break;
155 
156  case ISD::ATOMIC_LOAD:
157  Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
158 
163  case ISD::ATOMIC_LOAD_OR:
170  case ISD::ATOMIC_SWAP:
171  Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
172 
175  Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
176  break;
177 
178  case ISD::VECREDUCE_ADD:
179  case ISD::VECREDUCE_MUL:
180  case ISD::VECREDUCE_AND:
181  case ISD::VECREDUCE_OR:
182  case ISD::VECREDUCE_XOR:
183  case ISD::VECREDUCE_SMAX:
184  case ISD::VECREDUCE_SMIN:
185  case ISD::VECREDUCE_UMAX:
186  case ISD::VECREDUCE_UMIN:
187  Res = PromoteIntRes_VECREDUCE(N);
188  break;
189  }
190 
191  // If the result is null then the sub-method took care of registering it.
192  if (Res.getNode())
193  SetPromotedInteger(SDValue(N, ResNo), Res);
194 }
195 
196 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
197  unsigned ResNo) {
198  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
199  return GetPromotedInteger(Op);
200 }
201 
202 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
203  // Sign-extend the new bits, and continue the assertion.
204  SDValue Op = SExtPromotedInteger(N->getOperand(0));
205  return DAG.getNode(ISD::AssertSext, SDLoc(N),
206  Op.getValueType(), Op, N->getOperand(1));
207 }
208 
209 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
210  // Zero the new bits, and continue the assertion.
211  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
212  return DAG.getNode(ISD::AssertZext, SDLoc(N),
213  Op.getValueType(), Op, N->getOperand(1));
214 }
215 
216 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
217  EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
218  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
219  N->getMemoryVT(), ResVT,
220  N->getChain(), N->getBasePtr(),
221  N->getMemOperand());
222  // Legalize the chain result - switch anything that used the old chain to
223  // use the new one.
224  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
225  return Res;
226 }
227 
228 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
229  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
230  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
231  N->getMemoryVT(),
232  N->getChain(), N->getBasePtr(),
233  Op2, N->getMemOperand());
234  // Legalize the chain result - switch anything that used the old chain to
235  // use the new one.
236  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
237  return Res;
238 }
239 
240 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
241  unsigned ResNo) {
242  if (ResNo == 1) {
244  EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
245  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
246 
247  // Only use the result of getSetCCResultType if it is legal,
248  // otherwise just use the promoted result type (NVT).
249  if (!TLI.isTypeLegal(SVT))
250  SVT = NVT;
251 
252  SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
253  SDValue Res = DAG.getAtomicCmpSwap(
255  N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
256  N->getMemOperand());
257  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
258  ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
259  return Res.getValue(1);
260  }
261 
262  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
263  SDValue Op3 = GetPromotedInteger(N->getOperand(3));
264  SDVTList VTs =
265  DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
266  SDValue Res = DAG.getAtomicCmpSwap(
267  N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
268  N->getBasePtr(), Op2, Op3, N->getMemOperand());
269  // Update the use to N with the newly created Res.
270  for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
271  ReplaceValueWith(SDValue(N, i), Res.getValue(i));
272  return Res;
273 }
274 
275 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
276  SDValue InOp = N->getOperand(0);
277  EVT InVT = InOp.getValueType();
278  EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
279  EVT OutVT = N->getValueType(0);
280  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
281  SDLoc dl(N);
282 
283  switch (getTypeAction(InVT)) {
285  break;
287  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
288  // The input promotes to the same size. Convert the promoted value.
289  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
290  break;
292  // Promote the integer operand by hand.
293  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
295  // Convert the promoted float by hand.
296  if (!NOutVT.isVector())
297  return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
298  break;
299  }
302  break;
304  // Convert the element to an integer and promote it by hand.
305  if (!NOutVT.isVector())
306  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
307  BitConvertToInteger(GetScalarizedVector(InOp)));
308  break;
310  if (!NOutVT.isVector()) {
311  // For example, i32 = BITCAST v2i16 on alpha. Convert the split
312  // pieces of the input into integers and reassemble in the final type.
313  SDValue Lo, Hi;
314  GetSplitVector(N->getOperand(0), Lo, Hi);
315  Lo = BitConvertToInteger(Lo);
316  Hi = BitConvertToInteger(Hi);
317 
318  if (DAG.getDataLayout().isBigEndian())
319  std::swap(Lo, Hi);
320 
321  InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
323  NOutVT.getSizeInBits()),
324  JoinIntegers(Lo, Hi));
325  return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
326  }
327  break;
328  }
330  // The input is widened to the same size. Convert to the widened value.
331  // Make sure that the outgoing value is not a vector, because this would
332  // make us bitcast between two vectors which are legalized in different ways.
333  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector())
334  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
335  // If the output type is also a vector and widening it to the same size
336  // as the widened input type would be a legal type, we can widen the bitcast
337  // and handle the promotion after.
338  if (NOutVT.isVector()) {
339  unsigned WidenInSize = NInVT.getSizeInBits();
340  unsigned OutSize = OutVT.getSizeInBits();
341  if (WidenInSize % OutSize == 0) {
342  unsigned Scale = WidenInSize / OutSize;
343  EVT WideOutVT = EVT::getVectorVT(*DAG.getContext(),
344  OutVT.getVectorElementType(),
345  OutVT.getVectorNumElements() * Scale);
346  if (isTypeLegal(WideOutVT)) {
347  InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
348  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
349  InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
350  DAG.getConstant(0, dl, IdxTy));
351  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
352  }
353  }
354  }
355  }
356 
357  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
358  CreateStackStoreLoad(InOp, OutVT));
359 }
360 
361 // Helper for BSWAP/BITREVERSE promotion to ensure we can fit the shift amount
362 // in the VT returned by getShiftAmountTy and to return a safe VT if we can't.
363 static EVT getShiftAmountTyForConstant(unsigned Val, EVT VT,
364  const TargetLowering &TLI,
365  SelectionDAG &DAG) {
366  EVT ShiftVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
367  // If the value won't fit in the prefered type, just use something safe. It
368  // will be legalized when the shift is expanded.
369  if ((Log2_32(Val) + 1) > ShiftVT.getScalarSizeInBits())
370  ShiftVT = MVT::i32;
371  return ShiftVT;
372 }
373 
374 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
375  SDValue Op = GetPromotedInteger(N->getOperand(0));
376  EVT OVT = N->getValueType(0);
377  EVT NVT = Op.getValueType();
378  SDLoc dl(N);
379 
380  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
381  EVT ShiftVT = getShiftAmountTyForConstant(DiffBits, NVT, TLI, DAG);
382  return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
383  DAG.getConstant(DiffBits, dl, ShiftVT));
384 }
385 
386 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
387  SDValue Op = GetPromotedInteger(N->getOperand(0));
388  EVT OVT = N->getValueType(0);
389  EVT NVT = Op.getValueType();
390  SDLoc dl(N);
391 
392  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
393  EVT ShiftVT = getShiftAmountTyForConstant(DiffBits, NVT, TLI, DAG);
394  return DAG.getNode(ISD::SRL, dl, NVT,
395  DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
396  DAG.getConstant(DiffBits, dl, ShiftVT));
397 }
398 
399 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
400  // The pair element type may be legal, or may not promote to the same type as
401  // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
402  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
403  TLI.getTypeToTransformTo(*DAG.getContext(),
404  N->getValueType(0)), JoinIntegers(N->getOperand(0),
405  N->getOperand(1)));
406 }
407 
408 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
409  EVT VT = N->getValueType(0);
410  // FIXME there is no actual debug info here
411  SDLoc dl(N);
412  // Zero extend things like i1, sign extend everything else. It shouldn't
413  // matter in theory which one we pick, but this tends to give better code?
414  unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
415  SDValue Result = DAG.getNode(Opc, dl,
416  TLI.getTypeToTransformTo(*DAG.getContext(), VT),
417  SDValue(N, 0));
418  assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
419  return Result;
420 }
421 
422 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
423  // Zero extend to the promoted type and do the count there.
424  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
425  SDLoc dl(N);
426  EVT OVT = N->getValueType(0);
427  EVT NVT = Op.getValueType();
428  Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
429  // Subtract off the extra leading bits in the bigger type.
430  return DAG.getNode(
431  ISD::SUB, dl, NVT, Op,
432  DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
433  NVT));
434 }
435 
436 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
437  // Zero extend to the promoted type and do the count there.
438  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
439  return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op);
440 }
441 
442 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
443  SDValue Op = GetPromotedInteger(N->getOperand(0));
444  EVT OVT = N->getValueType(0);
445  EVT NVT = Op.getValueType();
446  SDLoc dl(N);
447  if (N->getOpcode() == ISD::CTTZ) {
448  // The count is the same in the promoted type except if the original
449  // value was zero. This can be handled by setting the bit just off
450  // the top of the original type.
451  auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
452  OVT.getScalarSizeInBits());
453  Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
454  }
455  return DAG.getNode(N->getOpcode(), dl, NVT, Op);
456 }
457 
458 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
459  SDLoc dl(N);
460  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
461 
462  SDValue Op0 = N->getOperand(0);
463  SDValue Op1 = N->getOperand(1);
464 
465  // If the input also needs to be promoted, do that first so we can get a
466  // get a good idea for the output type.
467  if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
469  SDValue In = GetPromotedInteger(Op0);
470 
471  // If the new type is larger than NVT, use it. We probably won't need to
472  // promote it again.
473  EVT SVT = In.getValueType().getScalarType();
474  if (SVT.bitsGE(NVT)) {
475  SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
476  return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
477  }
478  }
479 
480  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
481 }
482 
483 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
484  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
485  unsigned NewOpc = N->getOpcode();
486  SDLoc dl(N);
487 
488  // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
489  // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
490  // and SINT conversions are Custom, there is no way to tell which is
491  // preferable. We choose SINT because that's the right thing on PPC.)
492  if (N->getOpcode() == ISD::FP_TO_UINT &&
493  !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
495  NewOpc = ISD::FP_TO_SINT;
496 
497  SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
498 
499  // Assert that the converted value fits in the original type. If it doesn't
500  // (eg: because the value being converted is too big), then the result of the
501  // original operation was undefined anyway, so the assert is still correct.
502  //
503  // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
504  // before legalization: fp-to-uint16, 65534. -> 0xfffe
505  // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
506  return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ?
507  ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
509 }
510 
511 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
512  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
513  SDLoc dl(N);
514 
515  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
516 }
517 
518 SDValue DAGTypeLegalizer::PromoteIntRes_FLT_ROUNDS(SDNode *N) {
519  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
520  SDLoc dl(N);
521 
522  return DAG.getNode(N->getOpcode(), dl, NVT);
523 }
524 
525 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
526  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
527  SDLoc dl(N);
528 
529  if (getTypeAction(N->getOperand(0).getValueType())
531  SDValue Res = GetPromotedInteger(N->getOperand(0));
532  assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
533 
534  // If the result and operand types are the same after promotion, simplify
535  // to an in-register extension.
536  if (NVT == Res.getValueType()) {
537  // The high bits are not guaranteed to be anything. Insert an extend.
538  if (N->getOpcode() == ISD::SIGN_EXTEND)
539  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
540  DAG.getValueType(N->getOperand(0).getValueType()));
541  if (N->getOpcode() == ISD::ZERO_EXTEND)
542  return DAG.getZeroExtendInReg(Res, dl,
544  assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
545  return Res;
546  }
547  }
548 
549  // Otherwise, just extend the original operand all the way to the larger type.
550  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
551 }
552 
553 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
554  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
555  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
558  SDLoc dl(N);
559  SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
560  N->getMemoryVT(), N->getMemOperand());
561 
562  // Legalize the chain result - switch anything that used the old chain to
563  // use the new one.
564  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
565  return Res;
566 }
567 
568 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
569  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
570  SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
571 
572  SDLoc dl(N);
573  SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
574  N->getMask(), ExtPassThru, N->getMemoryVT(),
576  // Legalize the chain result - switch anything that used the old chain to
577  // use the new one.
578  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
579  return Res;
580 }
581 
582 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
583  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
584  SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
585  assert(NVT == ExtPassThru.getValueType() &&
586  "Gather result type and the passThru agrument type should be the same");
587 
588  SDLoc dl(N);
589  SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
590  N->getIndex(), N->getScale() };
591  SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
592  N->getMemoryVT(), dl, Ops,
593  N->getMemOperand(), N->getIndexType());
594  // Legalize the chain result - switch anything that used the old chain to
595  // use the new one.
596  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
597  return Res;
598 }
599 
600 /// Promote the overflow flag of an overflowing arithmetic node.
601 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
602  // Change the return type of the boolean result while obeying
603  // getSetCCResultType.
604  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
605  EVT VT = N->getValueType(0);
606  EVT SVT = getSetCCResultType(VT);
607  SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
608  unsigned NumOps = N->getNumOperands();
609  assert(NumOps <= 3 && "Too many operands");
610  if (NumOps == 3)
611  Ops[2] = N->getOperand(2);
612 
613  SDLoc dl(N);
614  SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
615  makeArrayRef(Ops, NumOps));
616 
617  // Modified the sum result - switch anything that used the old sum to use
618  // the new one.
619  ReplaceValueWith(SDValue(N, 0), Res);
620 
621  // Convert to the expected type.
622  return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT);
623 }
624 
625 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSAT(SDNode *N) {
626  // For promoting iN -> iM, this can be expanded by
627  // 1. ANY_EXTEND iN to iM
628  // 2. SHL by M-N
629  // 3. [US][ADD|SUB]SAT
630  // 4. L/ASHR by M-N
631  SDLoc dl(N);
632  SDValue Op1 = N->getOperand(0);
633  SDValue Op2 = N->getOperand(1);
634  unsigned OldBits = Op1.getScalarValueSizeInBits();
635 
636  unsigned Opcode = N->getOpcode();
637  unsigned ShiftOp;
638  switch (Opcode) {
639  case ISD::SADDSAT:
640  case ISD::SSUBSAT:
641  ShiftOp = ISD::SRA;
642  break;
643  case ISD::UADDSAT:
644  case ISD::USUBSAT:
645  ShiftOp = ISD::SRL;
646  break;
647  default:
648  llvm_unreachable("Expected opcode to be signed or unsigned saturation "
649  "addition or subtraction");
650  }
651 
652  SDValue Op1Promoted = GetPromotedInteger(Op1);
653  SDValue Op2Promoted = GetPromotedInteger(Op2);
654 
655  EVT PromotedType = Op1Promoted.getValueType();
656  unsigned NewBits = PromotedType.getScalarSizeInBits();
657  unsigned SHLAmount = NewBits - OldBits;
658  EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
659  SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT);
660  Op1Promoted =
661  DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
662  Op2Promoted =
663  DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
664 
665  SDValue Result =
666  DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
667  return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
668 }
669 
670 SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
671  // Can just promote the operands then continue with operation.
672  SDLoc dl(N);
673  SDValue Op1Promoted, Op2Promoted;
674  bool Signed =
675  N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT;
676  if (Signed) {
677  Op1Promoted = SExtPromotedInteger(N->getOperand(0));
678  Op2Promoted = SExtPromotedInteger(N->getOperand(1));
679  } else {
680  Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
681  Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
682  }
683  EVT OldType = N->getOperand(0).getValueType();
684  EVT PromotedType = Op1Promoted.getValueType();
685  unsigned DiffSize =
686  PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits();
687 
688  bool Saturating = N->getOpcode() == ISD::SMULFIXSAT;
689  if (Saturating) {
690  // Promoting the operand and result values changes the saturation width,
691  // which is extends the values that we clamp to on saturation. This could be
692  // resolved by shifting one of the operands the same amount, which would
693  // also shift the result we compare against, then shifting back.
694  EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
695  Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
696  DAG.getConstant(DiffSize, dl, ShiftTy));
697  SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
698  Op2Promoted, N->getOperand(2));
699  unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL;
700  return DAG.getNode(ShiftOp, dl, PromotedType, Result,
701  DAG.getConstant(DiffSize, dl, ShiftTy));
702  }
703  return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
704  N->getOperand(2));
705 }
706 
707 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
708  if (ResNo == 1)
709  return PromoteIntRes_Overflow(N);
710 
711  // The operation overflowed iff the result in the larger type is not the
712  // sign extension of its truncation to the original type.
713  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
714  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
715  EVT OVT = N->getOperand(0).getValueType();
716  EVT NVT = LHS.getValueType();
717  SDLoc dl(N);
718 
719  // Do the arithmetic in the larger type.
720  unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
721  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
722 
723  // Calculate the overflow flag: sign extend the arithmetic result from
724  // the original type.
725  SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
726  DAG.getValueType(OVT));
727  // Overflowed if and only if this is not equal to Res.
728  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
729 
730  // Use the calculated overflow everywhere.
731  ReplaceValueWith(SDValue(N, 1), Ofl);
732 
733  return Res;
734 }
735 
736 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
737  SDValue LHS = GetPromotedInteger(N->getOperand(1));
738  SDValue RHS = GetPromotedInteger(N->getOperand(2));
739  return DAG.getSelect(SDLoc(N),
740  LHS.getValueType(), N->getOperand(0), LHS, RHS);
741 }
742 
743 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
744  SDValue Mask = N->getOperand(0);
745 
746  SDValue LHS = GetPromotedInteger(N->getOperand(1));
747  SDValue RHS = GetPromotedInteger(N->getOperand(2));
748  return DAG.getNode(ISD::VSELECT, SDLoc(N),
749  LHS.getValueType(), Mask, LHS, RHS);
750 }
751 
752 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
753  SDValue LHS = GetPromotedInteger(N->getOperand(2));
754  SDValue RHS = GetPromotedInteger(N->getOperand(3));
755  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
756  LHS.getValueType(), N->getOperand(0),
757  N->getOperand(1), LHS, RHS, N->getOperand(4));
758 }
759 
760 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
761  EVT InVT = N->getOperand(0).getValueType();
762  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
763 
764  EVT SVT = getSetCCResultType(InVT);
765 
766  // If we got back a type that needs to be promoted, this likely means the
767  // the input type also needs to be promoted. So get the promoted type for
768  // the input and try the query again.
769  if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
770  if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
771  InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
772  SVT = getSetCCResultType(InVT);
773  } else {
774  // Input type isn't promoted, just use the default promoted type.
775  SVT = NVT;
776  }
777  }
778 
779  SDLoc dl(N);
780  assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() &&
781  "Vector compare must return a vector result!");
782 
783  // Get the SETCC result using the canonical SETCC type.
784  SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
785  N->getOperand(1), N->getOperand(2));
786 
787  // Convert to the expected type.
788  return DAG.getSExtOrTrunc(SetCC, dl, NVT);
789 }
790 
791 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
792  SDValue LHS = GetPromotedInteger(N->getOperand(0));
793  SDValue RHS = N->getOperand(1);
794  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
795  RHS = ZExtPromotedInteger(RHS);
796  return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
797 }
798 
799 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
800  SDValue Op = GetPromotedInteger(N->getOperand(0));
801  return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
802  Op.getValueType(), Op, N->getOperand(1));
803 }
804 
805 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
806  // The input may have strange things in the top bits of the registers, but
807  // these operations don't care. They may have weird bits going out, but
808  // that too is okay if they are integer operations.
809  SDValue LHS = GetPromotedInteger(N->getOperand(0));
810  SDValue RHS = GetPromotedInteger(N->getOperand(1));
811  return DAG.getNode(N->getOpcode(), SDLoc(N),
812  LHS.getValueType(), LHS, RHS);
813 }
814 
815 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
816  // Sign extend the input.
817  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
818  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
819  return DAG.getNode(N->getOpcode(), SDLoc(N),
820  LHS.getValueType(), LHS, RHS);
821 }
822 
823 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
824  // Zero extend the input.
825  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
826  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
827  return DAG.getNode(N->getOpcode(), SDLoc(N),
828  LHS.getValueType(), LHS, RHS);
829 }
830 
831 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
832  // The input value must be properly sign extended.
833  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
834  SDValue RHS = N->getOperand(1);
835  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
836  RHS = ZExtPromotedInteger(RHS);
837  return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
838 }
839 
840 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
841  // The input value must be properly zero extended.
842  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
843  SDValue RHS = N->getOperand(1);
844  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
845  RHS = ZExtPromotedInteger(RHS);
846  return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
847 }
848 
849 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
850  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
851  SDValue Res;
852  SDValue InOp = N->getOperand(0);
853  SDLoc dl(N);
854 
855  switch (getTypeAction(InOp.getValueType())) {
856  default: llvm_unreachable("Unknown type action!");
859  Res = InOp;
860  break;
862  Res = GetPromotedInteger(InOp);
863  break;
865  EVT InVT = InOp.getValueType();
866  assert(InVT.isVector() && "Cannot split scalar types");
867  unsigned NumElts = InVT.getVectorNumElements();
868  assert(NumElts == NVT.getVectorNumElements() &&
869  "Dst and Src must have the same number of elements");
870  assert(isPowerOf2_32(NumElts) &&
871  "Promoted vector type must be a power of two");
872 
873  SDValue EOp1, EOp2;
874  GetSplitVector(InOp, EOp1, EOp2);
875 
876  EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
877  NumElts/2);
878  EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
879  EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
880 
881  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
882  }
884  SDValue WideInOp = GetWidenedVector(InOp);
885 
886  // Truncate widened InOp.
887  unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
888  EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
889  N->getValueType(0).getScalarType(), NumElem);
890  SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
891 
892  // Zero extend so that the elements are of same type as those of NVT
893  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
894  NumElem);
895  SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
896 
897  // Extract the low NVT subvector.
898  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
899  SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy);
900  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
901  }
902  }
903 
904  // Truncate to NVT instead of VT
905  return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
906 }
907 
908 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
909  if (ResNo == 1)
910  return PromoteIntRes_Overflow(N);
911 
912  // The operation overflowed iff the result in the larger type is not the
913  // zero extension of its truncation to the original type.
914  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
915  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
916  EVT OVT = N->getOperand(0).getValueType();
917  EVT NVT = LHS.getValueType();
918  SDLoc dl(N);
919 
920  // Do the arithmetic in the larger type.
921  unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
922  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
923 
924  // Calculate the overflow flag: zero extend the arithmetic result from
925  // the original type.
926  SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT.getScalarType());
927  // Overflowed if and only if this is not equal to Res.
928  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
929 
930  // Use the calculated overflow everywhere.
931  ReplaceValueWith(SDValue(N, 1), Ofl);
932 
933  return Res;
934 }
935 
936 // Handle promotion for the ADDE/SUBE/ADDCARRY/SUBCARRY nodes. Notice that
937 // the third operand of ADDE/SUBE nodes is carry flag, which differs from
938 // the ADDCARRY/SUBCARRY nodes in that the third operand is carry Boolean.
939 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
940  if (ResNo == 1)
941  return PromoteIntRes_Overflow(N);
942 
943  // We need to sign-extend the operands so the carry value computed by the
944  // wide operation will be equivalent to the carry value computed by the
945  // narrow operation.
946  // An ADDCARRY can generate carry only if any of the operands has its
947  // most significant bit set. Sign extension propagates the most significant
948  // bit into the higher bits which means the extra bit that the narrow
949  // addition would need (i.e. the carry) will be propagated through the higher
950  // bits of the wide addition.
951  // A SUBCARRY can generate borrow only if LHS < RHS and this property will be
952  // preserved by sign extension.
953  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
954  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
955 
956  EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
957 
958  // Do the arithmetic in the wide type.
959  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
960  LHS, RHS, N->getOperand(2));
961 
962  // Update the users of the original carry/borrow value.
963  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
964 
965  return SDValue(Res.getNode(), 0);
966 }
967 
968 SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
969  SDValue Op0 = SExtPromotedInteger(N->getOperand(0));
970  return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0);
971 }
972 
973 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
974  // Promote the overflow bit trivially.
975  if (ResNo == 1)
976  return PromoteIntRes_Overflow(N);
977 
978  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
979  SDLoc DL(N);
980  EVT SmallVT = LHS.getValueType();
981 
982  // To determine if the result overflowed in a larger type, we extend the
983  // input to the larger type, do the multiply (checking if it overflows),
984  // then also check the high bits of the result to see if overflow happened
985  // there.
986  if (N->getOpcode() == ISD::SMULO) {
987  LHS = SExtPromotedInteger(LHS);
988  RHS = SExtPromotedInteger(RHS);
989  } else {
990  LHS = ZExtPromotedInteger(LHS);
991  RHS = ZExtPromotedInteger(RHS);
992  }
993  SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
994  SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
995 
996  // Overflow occurred if it occurred in the larger type, or if the high part
997  // of the result does not zero/sign-extend the low part. Check this second
998  // possibility first.
999  SDValue Overflow;
1000  if (N->getOpcode() == ISD::UMULO) {
1001  // Unsigned overflow occurred if the high part is non-zero.
1002  unsigned Shift = SmallVT.getScalarSizeInBits();
1003  EVT ShiftTy = getShiftAmountTyForConstant(Shift, Mul.getValueType(),
1004  TLI, DAG);
1005  SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
1006  DAG.getConstant(Shift, DL, ShiftTy));
1007  Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
1008  DAG.getConstant(0, DL, Hi.getValueType()),
1009  ISD::SETNE);
1010  } else {
1011  // Signed overflow occurred if the high part does not sign extend the low.
1012  SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
1013  Mul, DAG.getValueType(SmallVT));
1014  Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
1015  }
1016 
1017  // The only other way for overflow to occur is if the multiplication in the
1018  // larger type itself overflowed.
1019  Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
1020  SDValue(Mul.getNode(), 1));
1021 
1022  // Use the calculated overflow everywhere.
1023  ReplaceValueWith(SDValue(N, 1), Overflow);
1024  return Mul;
1025 }
1026 
1027 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
1028  return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1029  N->getValueType(0)));
1030 }
1031 
1032 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
1033  SDValue Chain = N->getOperand(0); // Get the chain.
1034  SDValue Ptr = N->getOperand(1); // Get the pointer.
1035  EVT VT = N->getValueType(0);
1036  SDLoc dl(N);
1037 
1038  MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
1039  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
1040  // The argument is passed as NumRegs registers of type RegVT.
1041 
1042  SmallVector<SDValue, 8> Parts(NumRegs);
1043  for (unsigned i = 0; i < NumRegs; ++i) {
1044  Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
1045  N->getConstantOperandVal(3));
1046  Chain = Parts[i].getValue(1);
1047  }
1048 
1049  // Handle endianness of the load.
1050  if (DAG.getDataLayout().isBigEndian())
1051  std::reverse(Parts.begin(), Parts.end());
1052 
1053  // Assemble the parts in the promoted type.
1054  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1055  SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
1056  for (unsigned i = 1; i < NumRegs; ++i) {
1057  SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
1058  // Shift it to the right position and "or" it in.
1059  Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
1060  DAG.getConstant(i * RegVT.getSizeInBits(), dl,
1061  TLI.getPointerTy(DAG.getDataLayout())));
1062  Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
1063  }
1064 
1065  // Modified the chain result - switch anything that used the old chain to
1066  // use the new one.
1067  ReplaceValueWith(SDValue(N, 1), Chain);
1068 
1069  return Res;
1070 }
1071 
1072 //===----------------------------------------------------------------------===//
1073 // Integer Operand Promotion
1074 //===----------------------------------------------------------------------===//
1075 
1076 /// PromoteIntegerOperand - This method is called when the specified operand of
1077 /// the specified node is found to need promotion. At this point, all of the
1078 /// result types of the node are known to be legal, but other operands of the
1079 /// node may need promotion or expansion as well as the specified one.
1080 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
1081  LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG);
1082  dbgs() << "\n");
1083  SDValue Res = SDValue();
1084 
1085  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
1086  LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
1087  return false;
1088  }
1089 
1090  switch (N->getOpcode()) {
1091  default:
1092  #ifndef NDEBUG
1093  dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
1094  N->dump(&DAG); dbgs() << "\n";
1095  #endif
1096  llvm_unreachable("Do not know how to promote this operator's operand!");
1097 
1098  case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
1099  case ISD::ATOMIC_STORE:
1100  Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
1101  break;
1102  case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
1103  case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
1104  case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
1105  case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
1106  case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
1107  case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
1108  case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
1110  Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
1111  case ISD::SCALAR_TO_VECTOR:
1112  Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
1113  case ISD::VSELECT:
1114  case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
1115  case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
1116  case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
1117  case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
1118  case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
1119  case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
1120  OpNo); break;
1121  case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
1122  OpNo); break;
1123  case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
1124  OpNo); break;
1125  case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
1126  OpNo); break;
1127  case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
1128  OpNo); break;
1129  case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
1130  case ISD::FP16_TO_FP:
1131  case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
1132  case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
1133  case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
1134 
1135  case ISD::SHL:
1136  case ISD::SRA:
1137  case ISD::SRL:
1138  case ISD::ROTL:
1139  case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
1140 
1141  case ISD::ADDCARRY:
1142  case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
1143 
1144  case ISD::FRAMEADDR:
1145  case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
1146 
1147  case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break;
1148 
1149  case ISD::SMULFIX:
1150  case ISD::SMULFIXSAT:
1151  case ISD::UMULFIX: Res = PromoteIntOp_MULFIX(N); break;
1152 
1153  case ISD::FPOWI: Res = PromoteIntOp_FPOWI(N); break;
1154 
1155  case ISD::VECREDUCE_ADD:
1156  case ISD::VECREDUCE_MUL:
1157  case ISD::VECREDUCE_AND:
1158  case ISD::VECREDUCE_OR:
1159  case ISD::VECREDUCE_XOR:
1160  case ISD::VECREDUCE_SMAX:
1161  case ISD::VECREDUCE_SMIN:
1162  case ISD::VECREDUCE_UMAX:
1163  case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break;
1164  }
1165 
1166  // If the result is null, the sub-method took care of registering results etc.
1167  if (!Res.getNode()) return false;
1168 
1169  // If the result is N, the sub-method updated N in place. Tell the legalizer
1170  // core about this.
1171  if (Res.getNode() == N)
1172  return true;
1173 
1174  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1175  "Invalid operand expansion");
1176 
1177  ReplaceValueWith(SDValue(N, 0), Res);
1178  return false;
1179 }
1180 
1181 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
1182 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
1183 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
1184  ISD::CondCode CCCode) {
1185  // We have to insert explicit sign or zero extends. Note that we could
1186  // insert sign extends for ALL conditions. For those operations where either
1187  // zero or sign extension would be valid, use SExtOrZExtPromotedInteger
1188  // which will choose the cheapest for the target.
1189  switch (CCCode) {
1190  default: llvm_unreachable("Unknown integer comparison!");
1191  case ISD::SETEQ:
1192  case ISD::SETNE: {
1193  SDValue OpL = GetPromotedInteger(NewLHS);
1194  SDValue OpR = GetPromotedInteger(NewRHS);
1195 
1196  // We would prefer to promote the comparison operand with sign extension.
1197  // If the width of OpL/OpR excluding the duplicated sign bits is no greater
1198  // than the width of NewLHS/NewRH, we can avoid inserting real truncate
1199  // instruction, which is redundant eventually.
1200  unsigned OpLEffectiveBits =
1201  OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
1202  unsigned OpREffectiveBits =
1203  OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
1204  if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() &&
1205  OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()) {
1206  NewLHS = OpL;
1207  NewRHS = OpR;
1208  } else {
1209  NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1210  NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1211  }
1212  break;
1213  }
1214  case ISD::SETUGE:
1215  case ISD::SETUGT:
1216  case ISD::SETULE:
1217  case ISD::SETULT:
1218  NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1219  NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1220  break;
1221  case ISD::SETGE:
1222  case ISD::SETGT:
1223  case ISD::SETLT:
1224  case ISD::SETLE:
1225  NewLHS = SExtPromotedInteger(NewLHS);
1226  NewRHS = SExtPromotedInteger(NewRHS);
1227  break;
1228  }
1229 }
1230 
1231 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1232  SDValue Op = GetPromotedInteger(N->getOperand(0));
1233  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1234 }
1235 
1236 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1237  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1238  return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1239  N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1240 }
1241 
1242 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1243  // This should only occur in unusual situations like bitcasting to an
1244  // x86_fp80, so just turn it into a store+load
1245  return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1246 }
1247 
1248 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1249  assert(OpNo == 2 && "Don't know how to promote this operand!");
1250 
1251  SDValue LHS = N->getOperand(2);
1252  SDValue RHS = N->getOperand(3);
1253  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1254 
1255  // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1256  // legal types.
1257  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1258  N->getOperand(1), LHS, RHS, N->getOperand(4)),
1259  0);
1260 }
1261 
1262 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1263  assert(OpNo == 1 && "only know how to promote condition");
1264 
1265  // Promote all the way up to the canonical SetCC type.
1266  SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1267 
1268  // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1269  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1270  N->getOperand(2)), 0);
1271 }
1272 
1273 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1274  // Since the result type is legal, the operands must promote to it.
1275  EVT OVT = N->getOperand(0).getValueType();
1276  SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1277  SDValue Hi = GetPromotedInteger(N->getOperand(1));
1278  assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1279  SDLoc dl(N);
1280 
1281  Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1282  DAG.getConstant(OVT.getSizeInBits(), dl,
1283  TLI.getPointerTy(DAG.getDataLayout())));
1284  return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1285 }
1286 
1287 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1288  // The vector type is legal but the element type is not. This implies
1289  // that the vector is a power-of-two in length and that the element
1290  // type does not have a strange size (eg: it is not i1).
1291  EVT VecVT = N->getValueType(0);
1292  unsigned NumElts = VecVT.getVectorNumElements();
1293  assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1294  "Legal vector of one illegal element?");
1295 
1296  // Promote the inserted value. The type does not need to match the
1297  // vector element type. Check that any extra bits introduced will be
1298  // truncated away.
1301  "Type of inserted value narrower than vector element type!");
1302 
1303  SmallVector<SDValue, 16> NewOps;
1304  for (unsigned i = 0; i < NumElts; ++i)
1305  NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1306 
1307  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1308 }
1309 
1310 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1311  unsigned OpNo) {
1312  if (OpNo == 1) {
1313  // Promote the inserted value. This is valid because the type does not
1314  // have to match the vector element type.
1315 
1316  // Check that any extra bits introduced will be truncated away.
1319  "Type of inserted value narrower than vector element type!");
1320  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1321  GetPromotedInteger(N->getOperand(1)),
1322  N->getOperand(2)),
1323  0);
1324  }
1325 
1326  assert(OpNo == 2 && "Different operand and result vector types?");
1327 
1328  // Promote the index.
1329  SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1330  TLI.getVectorIdxTy(DAG.getDataLayout()));
1331  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1332  N->getOperand(1), Idx), 0);
1333 }
1334 
1335 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1336  // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1337  // the operand in place.
1338  return SDValue(DAG.UpdateNodeOperands(N,
1339  GetPromotedInteger(N->getOperand(0))), 0);
1340 }
1341 
1342 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1343  assert(OpNo == 0 && "Only know how to promote the condition!");
1344  SDValue Cond = N->getOperand(0);
1345  EVT OpTy = N->getOperand(1).getValueType();
1346 
1347  if (N->getOpcode() == ISD::VSELECT)
1348  if (SDValue Res = WidenVSELECTAndMask(N))
1349  return Res;
1350 
1351  // Promote all the way up to the canonical SetCC type.
1352  EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1353  Cond = PromoteTargetBoolean(Cond, OpVT);
1354 
1355  return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1356  N->getOperand(2)), 0);
1357 }
1358 
1359 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1360  assert(OpNo == 0 && "Don't know how to promote this operand!");
1361 
1362  SDValue LHS = N->getOperand(0);
1363  SDValue RHS = N->getOperand(1);
1364  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1365 
1366  // The CC (#4) and the possible return values (#2 and #3) have legal types.
1367  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1368  N->getOperand(3), N->getOperand(4)), 0);
1369 }
1370 
1371 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1372  assert(OpNo == 0 && "Don't know how to promote this operand!");
1373 
1374  SDValue LHS = N->getOperand(0);
1375  SDValue RHS = N->getOperand(1);
1376  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1377 
1378  // The CC (#2) is always legal.
1379  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1380 }
1381 
1382 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1383  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1384  ZExtPromotedInteger(N->getOperand(1))), 0);
1385 }
1386 
1387 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1388  SDValue Op = GetPromotedInteger(N->getOperand(0));
1389  SDLoc dl(N);
1390  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1391  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1392  Op, DAG.getValueType(N->getOperand(0).getValueType()));
1393 }
1394 
1395 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1396  return SDValue(DAG.UpdateNodeOperands(N,
1397  SExtPromotedInteger(N->getOperand(0))), 0);
1398 }
1399 
1400 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1401  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1402  SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1403  SDLoc dl(N);
1404 
1405  SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1406 
1407  // Truncate the value and store the result.
1408  return DAG.getTruncStore(Ch, dl, Val, Ptr,
1409  N->getMemoryVT(), N->getMemOperand());
1410 }
1411 
1412 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1413  unsigned OpNo) {
1414 
1415  SDValue DataOp = N->getValue();
1416  EVT DataVT = DataOp.getValueType();
1417  SDValue Mask = N->getMask();
1418  SDLoc dl(N);
1419 
1420  bool TruncateStore = false;
1421  if (OpNo == 3) {
1422  Mask = PromoteTargetBoolean(Mask, DataVT);
1423  // Update in place.
1424  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1425  NewOps[3] = Mask;
1426  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1427  } else { // Data operand
1428  assert(OpNo == 1 && "Unexpected operand for promotion");
1429  DataOp = GetPromotedInteger(DataOp);
1430  TruncateStore = true;
1431  }
1432 
1433  return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask,
1434  N->getMemoryVT(), N->getMemOperand(),
1435  TruncateStore, N->isCompressingStore());
1436 }
1437 
1438 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1439  unsigned OpNo) {
1440  assert(OpNo == 2 && "Only know how to promote the mask!");
1441  EVT DataVT = N->getValueType(0);
1442  SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1443  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1444  NewOps[OpNo] = Mask;
1445  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1446 }
1447 
1448 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1449  unsigned OpNo) {
1450 
1451  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1452  if (OpNo == 2) {
1453  // The Mask
1454  EVT DataVT = N->getValueType(0);
1455  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1456  } else if (OpNo == 4) {
1457  // The Index
1458  if (N->isIndexSigned())
1459  // Need to sign extend the index since the bits will likely be used.
1460  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1461  else
1462  NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
1463  } else
1464  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1465 
1466  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1467 }
1468 
1469 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1470  unsigned OpNo) {
1471  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1472  if (OpNo == 2) {
1473  // The Mask
1474  EVT DataVT = N->getValue().getValueType();
1475  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1476  } else if (OpNo == 4) {
1477  // The Index
1478  if (N->isIndexSigned())
1479  // Need to sign extend the index since the bits will likely be used.
1480  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1481  else
1482  NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
1483  } else
1484  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1485  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1486 }
1487 
1488 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
1489  SDValue Op = GetPromotedInteger(N->getOperand(0));
1490  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
1491 }
1492 
1493 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
1494  return SDValue(DAG.UpdateNodeOperands(N,
1495  ZExtPromotedInteger(N->getOperand(0))), 0);
1496 }
1497 
1498 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
1499  SDLoc dl(N);
1500  SDValue Op = GetPromotedInteger(N->getOperand(0));
1501  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1502  return DAG.getZeroExtendInReg(Op, dl,
1504 }
1505 
1506 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
1507  assert(OpNo == 2 && "Don't know how to promote this operand!");
1508 
1509  SDValue LHS = N->getOperand(0);
1510  SDValue RHS = N->getOperand(1);
1511  SDValue Carry = N->getOperand(2);
1512  SDLoc DL(N);
1513 
1514  Carry = PromoteTargetBoolean(Carry, LHS.getValueType());
1515 
1516  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
1517 }
1518 
1519 SDValue DAGTypeLegalizer::PromoteIntOp_MULFIX(SDNode *N) {
1520  SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1521  return SDValue(
1522  DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
1523 }
1524 
1525 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
1526  // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
1527  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
1528  return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
1529 }
1530 
1531 SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) {
1532  assert(OpNo > 1 && "Don't know how to promote this operand!");
1533  // Promote the rw, locality, and cache type arguments to a supported integer
1534  // width.
1535  SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1536  SDValue Op3 = ZExtPromotedInteger(N->getOperand(3));
1537  SDValue Op4 = ZExtPromotedInteger(N->getOperand(4));
1538  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
1539  Op2, Op3, Op4),
1540  0);
1541 }
1542 
1543 SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) {
1544  SDValue Op = SExtPromotedInteger(N->getOperand(1));
1545  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
1546 }
1547 
1548 SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) {
1549  SDLoc dl(N);
1550  SDValue Op;
1551  switch (N->getOpcode()) {
1552  default: llvm_unreachable("Expected integer vector reduction");
1553  case ISD::VECREDUCE_ADD:
1554  case ISD::VECREDUCE_MUL:
1555  case ISD::VECREDUCE_AND:
1556  case ISD::VECREDUCE_OR:
1557  case ISD::VECREDUCE_XOR:
1558  Op = GetPromotedInteger(N->getOperand(0));
1559  break;
1560  case ISD::VECREDUCE_SMAX:
1561  case ISD::VECREDUCE_SMIN:
1562  Op = SExtPromotedInteger(N->getOperand(0));
1563  break;
1564  case ISD::VECREDUCE_UMAX:
1565  case ISD::VECREDUCE_UMIN:
1566  Op = ZExtPromotedInteger(N->getOperand(0));
1567  break;
1568  }
1569 
1570  EVT EltVT = Op.getValueType().getVectorElementType();
1571  EVT VT = N->getValueType(0);
1572  if (VT.bitsGE(EltVT))
1573  return DAG.getNode(N->getOpcode(), SDLoc(N), VT, Op);
1574 
1575  // Result size must be >= element size. If this is not the case after
1576  // promotion, also promote the result type and then truncate.
1577  SDValue Reduce = DAG.getNode(N->getOpcode(), dl, EltVT, Op);
1578  return DAG.getNode(ISD::TRUNCATE, dl, VT, Reduce);
1579 }
1580 
1581 //===----------------------------------------------------------------------===//
1582 // Integer Result Expansion
1583 //===----------------------------------------------------------------------===//
1584 
1585 /// ExpandIntegerResult - This method is called when the specified result of the
1586 /// specified node is found to need expansion. At this point, the node may also
1587 /// have invalid operands or may have other results that need promotion, we just
1588 /// know that (at least) one result needs expansion.
1589 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
1590  LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG);
1591  dbgs() << "\n");
1592  SDValue Lo, Hi;
1593  Lo = Hi = SDValue();
1594 
1595  // See if the target wants to custom expand this node.
1596  if (CustomLowerNode(N, N->getValueType(ResNo), true))
1597  return;
1598 
1599  switch (N->getOpcode()) {
1600  default:
1601 #ifndef NDEBUG
1602  dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
1603  N->dump(&DAG); dbgs() << "\n";
1604 #endif
1605  report_fatal_error("Do not know how to expand the result of this "
1606  "operator!");
1607 
1608  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1609  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
1610  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1611  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1612 
1613  case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1614  case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1615  case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1616  case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1617  case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1618 
1619  case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
1620  case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
1621  case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
1622  case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
1623  case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
1624  case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
1625  case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break;
1626  case ISD::CTLZ_ZERO_UNDEF:
1627  case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
1628  case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
1629  case ISD::CTTZ_ZERO_UNDEF:
1630  case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
1631  case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
1632  case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
1633  case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
1634  case ISD::LLROUND: ExpandIntRes_LLROUND(N, Lo, Hi); break;
1635  case ISD::LLRINT: ExpandIntRes_LLRINT(N, Lo, Hi); break;
1636  case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
1637  case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
1638  case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
1639  case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
1640  case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
1641  case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
1642  case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
1643  case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
1644  case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
1645  case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
1646  case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
1647  case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
1648 
1649  case ISD::ATOMIC_LOAD_ADD:
1650  case ISD::ATOMIC_LOAD_SUB:
1651  case ISD::ATOMIC_LOAD_AND:
1652  case ISD::ATOMIC_LOAD_CLR:
1653  case ISD::ATOMIC_LOAD_OR:
1654  case ISD::ATOMIC_LOAD_XOR:
1655  case ISD::ATOMIC_LOAD_NAND:
1656  case ISD::ATOMIC_LOAD_MIN:
1657  case ISD::ATOMIC_LOAD_MAX:
1658  case ISD::ATOMIC_LOAD_UMIN:
1659  case ISD::ATOMIC_LOAD_UMAX:
1660  case ISD::ATOMIC_SWAP:
1661  case ISD::ATOMIC_CMP_SWAP: {
1662  std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
1663  SplitInteger(Tmp.first, Lo, Hi);
1664  ReplaceValueWith(SDValue(N, 1), Tmp.second);
1665  break;
1666  }
1668  AtomicSDNode *AN = cast<AtomicSDNode>(N);
1669  SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
1670  SDValue Tmp = DAG.getAtomicCmpSwap(
1671  ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
1672  N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
1673  AN->getMemOperand());
1674 
1675  // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
1676  // success simply by comparing the loaded value against the ingoing
1677  // comparison.
1678  SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
1679  N->getOperand(2), ISD::SETEQ);
1680 
1681  SplitInteger(Tmp, Lo, Hi);
1682  ReplaceValueWith(SDValue(N, 1), Success);
1683  ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
1684  break;
1685  }
1686 
1687  case ISD::AND:
1688  case ISD::OR:
1689  case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
1690 
1691  case ISD::UMAX:
1692  case ISD::SMAX:
1693  case ISD::UMIN:
1694  case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
1695 
1696  case ISD::ADD:
1697  case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
1698 
1699  case ISD::ADDC:
1700  case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
1701 
1702  case ISD::ADDE:
1703  case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
1704 
1705  case ISD::ADDCARRY:
1706  case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
1707 
1708  case ISD::SHL:
1709  case ISD::SRA:
1710  case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
1711 
1712  case ISD::SADDO:
1713  case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
1714  case ISD::UADDO:
1715  case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
1716  case ISD::UMULO:
1717  case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
1718 
1719  case ISD::SADDSAT:
1720  case ISD::UADDSAT:
1721  case ISD::SSUBSAT:
1722  case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
1723 
1724  case ISD::SMULFIX:
1725  case ISD::SMULFIXSAT:
1726  case ISD::UMULFIX: ExpandIntRes_MULFIX(N, Lo, Hi); break;
1727 
1728  case ISD::VECREDUCE_ADD:
1729  case ISD::VECREDUCE_MUL:
1730  case ISD::VECREDUCE_AND:
1731  case ISD::VECREDUCE_OR:
1732  case ISD::VECREDUCE_XOR:
1733  case ISD::VECREDUCE_SMAX:
1734  case ISD::VECREDUCE_SMIN:
1735  case ISD::VECREDUCE_UMAX:
1736  case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break;
1737  }
1738 
1739  // If Lo/Hi is null, the sub-method took care of registering results etc.
1740  if (Lo.getNode())
1741  SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
1742 }
1743 
1744 /// Lower an atomic node to the appropriate builtin call.
1745 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
1746  unsigned Opc = Node->getOpcode();
1747  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
1748  RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
1749  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
1750 
1751  return ExpandChainLibCall(LC, Node, false);
1752 }
1753 
1754 /// N is a shift by a value that needs to be expanded,
1755 /// and the shift amount is a constant 'Amt'. Expand the operation.
1756 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
1757  SDValue &Lo, SDValue &Hi) {
1758  SDLoc DL(N);
1759  // Expand the incoming operand to be shifted, so that we have its parts
1760  SDValue InL, InH;
1761  GetExpandedInteger(N->getOperand(0), InL, InH);
1762 
1763  // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
1764  // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
1765  if (!Amt) {
1766  Lo = InL;
1767  Hi = InH;
1768  return;
1769  }
1770 
1771  EVT NVT = InL.getValueType();
1772  unsigned VTBits = N->getValueType(0).getSizeInBits();
1773  unsigned NVTBits = NVT.getSizeInBits();
1774  EVT ShTy = N->getOperand(1).getValueType();
1775 
1776  if (N->getOpcode() == ISD::SHL) {
1777  if (Amt.ugt(VTBits)) {
1778  Lo = Hi = DAG.getConstant(0, DL, NVT);
1779  } else if (Amt.ugt(NVTBits)) {
1780  Lo = DAG.getConstant(0, DL, NVT);
1781  Hi = DAG.getNode(ISD::SHL, DL,
1782  NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1783  } else if (Amt == NVTBits) {
1784  Lo = DAG.getConstant(0, DL, NVT);
1785  Hi = InL;
1786  } else {
1787  Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
1788  Hi = DAG.getNode(ISD::OR, DL, NVT,
1789  DAG.getNode(ISD::SHL, DL, NVT, InH,
1790  DAG.getConstant(Amt, DL, ShTy)),
1791  DAG.getNode(ISD::SRL, DL, NVT, InL,
1792  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1793  }
1794  return;
1795  }
1796 
1797  if (N->getOpcode() == ISD::SRL) {
1798  if (Amt.ugt(VTBits)) {
1799  Lo = Hi = DAG.getConstant(0, DL, NVT);
1800  } else if (Amt.ugt(NVTBits)) {
1801  Lo = DAG.getNode(ISD::SRL, DL,
1802  NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1803  Hi = DAG.getConstant(0, DL, NVT);
1804  } else if (Amt == NVTBits) {
1805  Lo = InH;
1806  Hi = DAG.getConstant(0, DL, NVT);
1807  } else {
1808  Lo = DAG.getNode(ISD::OR, DL, NVT,
1809  DAG.getNode(ISD::SRL, DL, NVT, InL,
1810  DAG.getConstant(Amt, DL, ShTy)),
1811  DAG.getNode(ISD::SHL, DL, NVT, InH,
1812  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1813  Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1814  }
1815  return;
1816  }
1817 
1818  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
1819  if (Amt.ugt(VTBits)) {
1820  Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1821  DAG.getConstant(NVTBits - 1, DL, ShTy));
1822  } else if (Amt.ugt(NVTBits)) {
1823  Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1824  DAG.getConstant(Amt - NVTBits, DL, ShTy));
1825  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1826  DAG.getConstant(NVTBits - 1, DL, ShTy));
1827  } else if (Amt == NVTBits) {
1828  Lo = InH;
1829  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1830  DAG.getConstant(NVTBits - 1, DL, ShTy));
1831  } else {
1832  Lo = DAG.getNode(ISD::OR, DL, NVT,
1833  DAG.getNode(ISD::SRL, DL, NVT, InL,
1834  DAG.getConstant(Amt, DL, ShTy)),
1835  DAG.getNode(ISD::SHL, DL, NVT, InH,
1836  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1837  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1838  }
1839 }
1840 
1841 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
1842 /// this shift based on knowledge of the high bit of the shift amount. If we
1843 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
1844 /// shift amount.
1845 bool DAGTypeLegalizer::
1846 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1847  SDValue Amt = N->getOperand(1);
1848  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1849  EVT ShTy = Amt.getValueType();
1850  unsigned ShBits = ShTy.getScalarSizeInBits();
1851  unsigned NVTBits = NVT.getScalarSizeInBits();
1852  assert(isPowerOf2_32(NVTBits) &&
1853  "Expanded integer type size not a power of two!");
1854  SDLoc dl(N);
1855 
1856  APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
1857  KnownBits Known = DAG.computeKnownBits(N->getOperand(1));
1858 
1859  // If we don't know anything about the high bits, exit.
1860  if (((Known.Zero|Known.One) & HighBitMask) == 0)
1861  return false;
1862 
1863  // Get the incoming operand to be shifted.
1864  SDValue InL, InH;
1865  GetExpandedInteger(N->getOperand(0), InL, InH);
1866 
1867  // If we know that any of the high bits of the shift amount are one, then we
1868  // can do this as a couple of simple shifts.
1869  if (Known.One.intersects(HighBitMask)) {
1870  // Mask out the high bit, which we know is set.
1871  Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
1872  DAG.getConstant(~HighBitMask, dl, ShTy));
1873 
1874  switch (N->getOpcode()) {
1875  default: llvm_unreachable("Unknown shift");
1876  case ISD::SHL:
1877  Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
1878  Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
1879  return true;
1880  case ISD::SRL:
1881  Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1882  Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
1883  return true;
1884  case ISD::SRA:
1885  Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
1886  DAG.getConstant(NVTBits - 1, dl, ShTy));
1887  Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
1888  return true;
1889  }
1890  }
1891 
1892  // If we know that all of the high bits of the shift amount are zero, then we
1893  // can do this as a couple of simple shifts.
1894  if (HighBitMask.isSubsetOf(Known.Zero)) {
1895  // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
1896  // shift if x is zero. We can use XOR here because x is known to be smaller
1897  // than 32.
1898  SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
1899  DAG.getConstant(NVTBits - 1, dl, ShTy));
1900 
1901  unsigned Op1, Op2;
1902  switch (N->getOpcode()) {
1903  default: llvm_unreachable("Unknown shift");
1904  case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
1905  case ISD::SRL:
1906  case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
1907  }
1908 
1909  // When shifting right the arithmetic for Lo and Hi is swapped.
1910  if (N->getOpcode() != ISD::SHL)
1911  std::swap(InL, InH);
1912 
1913  // Use a little trick to get the bits that move from Lo to Hi. First
1914  // shift by one bit.
1915  SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
1916  // Then compute the remaining shift with amount-1.
1917  SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
1918 
1919  Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
1920  Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
1921 
1922  if (N->getOpcode() != ISD::SHL)
1923  std::swap(Hi, Lo);
1924  return true;
1925  }
1926 
1927  return false;
1928 }
1929 
1930 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
1931 /// of any size.
1932 bool DAGTypeLegalizer::
1933 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1934  SDValue Amt = N->getOperand(1);
1935  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1936  EVT ShTy = Amt.getValueType();
1937  unsigned NVTBits = NVT.getSizeInBits();
1938  assert(isPowerOf2_32(NVTBits) &&
1939  "Expanded integer type size not a power of two!");
1940  SDLoc dl(N);
1941 
1942  // Get the incoming operand to be shifted.
1943  SDValue InL, InH;
1944  GetExpandedInteger(N->getOperand(0), InL, InH);
1945 
1946  SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
1947  SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
1948  SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
1949  SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1950  Amt, NVBitsNode, ISD::SETULT);
1951  SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1952  Amt, DAG.getConstant(0, dl, ShTy),
1953  ISD::SETEQ);
1954 
1955  SDValue LoS, HiS, LoL, HiL;
1956  switch (N->getOpcode()) {
1957  default: llvm_unreachable("Unknown shift");
1958  case ISD::SHL:
1959  // Short: ShAmt < NVTBits
1960  LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
1961  HiS = DAG.getNode(ISD::OR, dl, NVT,
1962  DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
1963  DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
1964 
1965  // Long: ShAmt >= NVTBits
1966  LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
1967  HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
1968 
1969  Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
1970  Hi = DAG.getSelect(dl, NVT, isZero, InH,
1971  DAG.getSelect(dl, NVT, isShort, HiS, HiL));
1972  return true;
1973  case ISD::SRL:
1974  // Short: ShAmt < NVTBits
1975  HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
1976  LoS = DAG.getNode(ISD::OR, dl, NVT,
1977  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1978  // FIXME: If Amt is zero, the following shift generates an undefined result
1979  // on some architectures.
1980  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1981 
1982  // Long: ShAmt >= NVTBits
1983  HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1984  LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1985 
1986  Lo = DAG.getSelect(dl, NVT, isZero, InL,
1987  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1988  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1989  return true;
1990  case ISD::SRA:
1991  // Short: ShAmt < NVTBits
1992  HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
1993  LoS = DAG.getNode(ISD::OR, dl, NVT,
1994  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1995  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1996 
1997  // Long: ShAmt >= NVTBits
1998  HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
1999  DAG.getConstant(NVTBits - 1, dl, ShTy));
2000  LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
2001 
2002  Lo = DAG.getSelect(dl, NVT, isZero, InL,
2003  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
2004  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
2005  return true;
2006  }
2007 }
2008 
2009 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
2010 
2011  switch (Op) {
2012  default: llvm_unreachable("invalid min/max opcode");
2013  case ISD::SMAX:
2014  return std::make_pair(ISD::SETGT, ISD::UMAX);
2015  case ISD::UMAX:
2016  return std::make_pair(ISD::SETUGT, ISD::UMAX);
2017  case ISD::SMIN:
2018  return std::make_pair(ISD::SETLT, ISD::UMIN);
2019  case ISD::UMIN:
2020  return std::make_pair(ISD::SETULT, ISD::UMIN);
2021  }
2022 }
2023 
2024 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
2025  SDValue &Lo, SDValue &Hi) {
2026  SDLoc DL(N);
2027  ISD::NodeType LoOpc;
2028  ISD::CondCode CondC;
2029  std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
2030 
2031  // Expand the subcomponents.
2032  SDValue LHSL, LHSH, RHSL, RHSH;
2033  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2034  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2035 
2036  // Value types
2037  EVT NVT = LHSL.getValueType();
2038  EVT CCT = getSetCCResultType(NVT);
2039 
2040  // Hi part is always the same op
2041  Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
2042 
2043  // We need to know whether to select Lo part that corresponds to 'winning'
2044  // Hi part or if Hi parts are equal.
2045  SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
2046  SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
2047 
2048  // Lo part corresponding to the 'winning' Hi part
2049  SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
2050 
2051  // Recursed Lo part if Hi parts are equal, this uses unsigned version
2052  SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
2053 
2054  Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
2055 }
2056 
2057 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
2058  SDValue &Lo, SDValue &Hi) {
2059  SDLoc dl(N);
2060  // Expand the subcomponents.
2061  SDValue LHSL, LHSH, RHSL, RHSH;
2062  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2063  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2064 
2065  EVT NVT = LHSL.getValueType();
2066  SDValue LoOps[2] = { LHSL, RHSL };
2067  SDValue HiOps[3] = { LHSH, RHSH };
2068 
2069  bool HasOpCarry = TLI.isOperationLegalOrCustom(
2071  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2072  if (HasOpCarry) {
2073  SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
2074  if (N->getOpcode() == ISD::ADD) {
2075  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2076  HiOps[2] = Lo.getValue(1);
2077  Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
2078  } else {
2079  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2080  HiOps[2] = Lo.getValue(1);
2081  Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
2082  }
2083  return;
2084  }
2085 
2086  // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
2087  // them. TODO: Teach operation legalization how to expand unsupported
2088  // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
2089  // a carry of type MVT::Glue, but there doesn't seem to be any way to
2090  // generate a value of this type in the expanded code sequence.
2091  bool hasCarry =
2093  ISD::ADDC : ISD::SUBC,
2094  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2095 
2096  if (hasCarry) {
2097  SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
2098  if (N->getOpcode() == ISD::ADD) {
2099  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2100  HiOps[2] = Lo.getValue(1);
2101  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2102  } else {
2103  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2104  HiOps[2] = Lo.getValue(1);
2105  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2106  }
2107  return;
2108  }
2109 
2110  bool hasOVF =
2113  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2115 
2116  if (hasOVF) {
2117  EVT OvfVT = getSetCCResultType(NVT);
2118  SDVTList VTList = DAG.getVTList(NVT, OvfVT);
2119  int RevOpc;
2120  if (N->getOpcode() == ISD::ADD) {
2121  RevOpc = ISD::SUB;
2122  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2123  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2124  } else {
2125  RevOpc = ISD::ADD;
2126  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2127  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2128  }
2129  SDValue OVF = Lo.getValue(1);
2130 
2131  switch (BoolType) {
2133  OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
2136  OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
2137  Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
2138  break;
2140  OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
2141  Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
2142  }
2143  return;
2144  }
2145 
2146  if (N->getOpcode() == ISD::ADD) {
2147  Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
2148  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2149  SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
2150  ISD::SETULT);
2151 
2153  SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
2154  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
2155  return;
2156  }
2157 
2158  SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
2159  DAG.getConstant(1, dl, NVT),
2160  DAG.getConstant(0, dl, NVT));
2161  SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
2162  ISD::SETULT);
2163  SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
2164  DAG.getConstant(1, dl, NVT), Carry1);
2165  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
2166  } else {
2167  Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
2168  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2169  SDValue Cmp =
2170  DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
2171  LoOps[0], LoOps[1], ISD::SETULT);
2172 
2173  SDValue Borrow;
2175  Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
2176  else
2177  Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
2178  DAG.getConstant(0, dl, NVT));
2179 
2180  Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
2181  }
2182 }
2183 
2184 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
2185  SDValue &Lo, SDValue &Hi) {
2186  // Expand the subcomponents.
2187  SDValue LHSL, LHSH, RHSL, RHSH;
2188  SDLoc dl(N);
2189  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2190  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2191  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2192  SDValue LoOps[2] = { LHSL, RHSL };
2193  SDValue HiOps[3] = { LHSH, RHSH };
2194 
2195  if (N->getOpcode() == ISD::ADDC) {
2196  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2197  HiOps[2] = Lo.getValue(1);
2198  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2199  } else {
2200  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2201  HiOps[2] = Lo.getValue(1);
2202  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2203  }
2204 
2205  // Legalized the flag result - switch anything that used the old flag to
2206  // use the new one.
2207  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2208 }
2209 
2210 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
2211  SDValue &Lo, SDValue &Hi) {
2212  // Expand the subcomponents.
2213  SDValue LHSL, LHSH, RHSL, RHSH;
2214  SDLoc dl(N);
2215  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2216  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2217  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2218  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2219  SDValue HiOps[3] = { LHSH, RHSH };
2220 
2221  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2222  HiOps[2] = Lo.getValue(1);
2223  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2224 
2225  // Legalized the flag result - switch anything that used the old flag to
2226  // use the new one.
2227  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2228 }
2229 
2230 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
2231  SDValue &Lo, SDValue &Hi) {
2232  SDValue LHS = N->getOperand(0);
2233  SDValue RHS = N->getOperand(1);
2234  SDLoc dl(N);
2235 
2236  SDValue Ovf;
2237 
2238  bool HasOpCarry = TLI.isOperationLegalOrCustom(
2240  TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
2241 
2242  if (HasOpCarry) {
2243  // Expand the subcomponents.
2244  SDValue LHSL, LHSH, RHSL, RHSH;
2245  GetExpandedInteger(LHS, LHSL, LHSH);
2246  GetExpandedInteger(RHS, RHSL, RHSH);
2247  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2248  SDValue LoOps[2] = { LHSL, RHSL };
2249  SDValue HiOps[3] = { LHSH, RHSH };
2250 
2251  unsigned Opc = N->getOpcode() == ISD::UADDO ? ISD::ADDCARRY : ISD::SUBCARRY;
2252  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2253  HiOps[2] = Lo.getValue(1);
2254  Hi = DAG.getNode(Opc, dl, VTList, HiOps);
2255 
2256  Ovf = Hi.getValue(1);
2257  } else {
2258  // Expand the result by simply replacing it with the equivalent
2259  // non-overflow-checking operation.
2260  auto Opc = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
2261  SDValue Sum = DAG.getNode(Opc, dl, LHS.getValueType(), LHS, RHS);
2262  SplitInteger(Sum, Lo, Hi);
2263 
2264  // Calculate the overflow: addition overflows iff a + b < a, and subtraction
2265  // overflows iff a - b > a.
2266  auto Cond = N->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
2267  Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
2268  }
2269 
2270  // Legalized the flag result - switch anything that used the old flag to
2271  // use the new one.
2272  ReplaceValueWith(SDValue(N, 1), Ovf);
2273 }
2274 
2275 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
2276  SDValue &Lo, SDValue &Hi) {
2277  // Expand the subcomponents.
2278  SDValue LHSL, LHSH, RHSL, RHSH;
2279  SDLoc dl(N);
2280  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2281  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2282  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2283  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2284  SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
2285 
2286  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2287  HiOps[2] = Lo.getValue(1);
2288  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2289 
2290  // Legalized the flag result - switch anything that used the old flag to
2291  // use the new one.
2292  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2293 }
2294 
2295 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
2296  SDValue &Lo, SDValue &Hi) {
2297  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2298  SDLoc dl(N);
2299  SDValue Op = N->getOperand(0);
2300  if (Op.getValueType().bitsLE(NVT)) {
2301  // The low part is any extension of the input (which degenerates to a copy).
2302  Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
2303  Hi = DAG.getUNDEF(NVT); // The high part is undefined.
2304  } else {
2305  // For example, extension of an i48 to an i64. The operand type necessarily
2306  // promotes to the result type, so will end up being expanded too.
2307  assert(getTypeAction(Op.getValueType()) ==
2309  "Only know how to promote this result!");
2310  SDValue Res = GetPromotedInteger(Op);
2311  assert(Res.getValueType() == N->getValueType(0) &&
2312  "Operand over promoted?");
2313  // Split the promoted operand. This will simplify when it is expanded.
2314  SplitInteger(Res, Lo, Hi);
2315  }
2316 }
2317 
2318 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
2319  SDValue &Lo, SDValue &Hi) {
2320  SDLoc dl(N);
2321  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2322  EVT NVT = Lo.getValueType();
2323  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2324  unsigned NVTBits = NVT.getSizeInBits();
2325  unsigned EVTBits = EVT.getSizeInBits();
2326 
2327  if (NVTBits < EVTBits) {
2328  Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
2330  EVTBits - NVTBits)));
2331  } else {
2332  Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
2333  // The high part replicates the sign bit of Lo, make it explicit.
2334  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2335  DAG.getConstant(NVTBits - 1, dl,
2336  TLI.getPointerTy(DAG.getDataLayout())));
2337  }
2338 }
2339 
2340 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
2341  SDValue &Lo, SDValue &Hi) {
2342  SDLoc dl(N);
2343  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2344  EVT NVT = Lo.getValueType();
2345  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2346  unsigned NVTBits = NVT.getSizeInBits();
2347  unsigned EVTBits = EVT.getSizeInBits();
2348 
2349  if (NVTBits < EVTBits) {
2350  Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
2352  EVTBits - NVTBits)));
2353  } else {
2354  Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
2355  // The high part must be zero, make it explicit.
2356  Hi = DAG.getConstant(0, dl, NVT);
2357  }
2358 }
2359 
2360 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
2361  SDValue &Lo, SDValue &Hi) {
2362  SDLoc dl(N);
2363  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2364  Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
2365  Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
2366 }
2367 
2368 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
2369  SDValue &Lo, SDValue &Hi) {
2370  SDLoc dl(N);
2371  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2372  Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
2373  Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
2374 }
2375 
2376 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
2377  SDValue &Lo, SDValue &Hi) {
2378  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2379  unsigned NBitWidth = NVT.getSizeInBits();
2380  auto Constant = cast<ConstantSDNode>(N);
2381  const APInt &Cst = Constant->getAPIntValue();
2382  bool IsTarget = Constant->isTargetOpcode();
2383  bool IsOpaque = Constant->isOpaque();
2384  SDLoc dl(N);
2385  Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
2386  Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
2387  IsOpaque);
2388 }
2389 
2390 void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
2391  SDLoc dl(N);
2392 
2393  // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo)
2394  EVT VT = N->getValueType(0);
2395  SDValue N0 = N->getOperand(0);
2396  SDValue Neg = DAG.getNode(ISD::SUB, dl, VT,
2397  DAG.getConstant(0, dl, VT), N0);
2398  SDValue NegLo, NegHi;
2399  SplitInteger(Neg, NegLo, NegHi);
2400 
2401  GetExpandedInteger(N0, Lo, Hi);
2402  EVT NVT = Lo.getValueType();
2403  SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT),
2404  DAG.getConstant(0, dl, NVT), Hi, ISD::SETGT);
2405  Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo);
2406  Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi);
2407 }
2408 
2409 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
2410  SDValue &Lo, SDValue &Hi) {
2411  SDLoc dl(N);
2412  // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
2413  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2414  EVT NVT = Lo.getValueType();
2415 
2416  SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
2417  DAG.getConstant(0, dl, NVT), ISD::SETNE);
2418 
2419  SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
2420  SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
2421 
2422  Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
2423  DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
2424  DAG.getConstant(NVT.getSizeInBits(), dl,
2425  NVT)));
2426  Hi = DAG.getConstant(0, dl, NVT);
2427 }
2428 
2429 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
2430  SDValue &Lo, SDValue &Hi) {
2431  SDLoc dl(N);
2432  // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
2433  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2434  EVT NVT = Lo.getValueType();
2435  Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
2436  DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
2437  Hi = DAG.getConstant(0, dl, NVT);
2438 }
2439 
2440 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
2441  SDValue &Lo, SDValue &Hi) {
2442  SDLoc dl(N);
2443  // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
2444  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2445  EVT NVT = Lo.getValueType();
2446 
2447  SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
2448  DAG.getConstant(0, dl, NVT), ISD::SETNE);
2449 
2450  SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
2451  SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
2452 
2453  Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
2454  DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
2455  DAG.getConstant(NVT.getSizeInBits(), dl,
2456  NVT)));
2457  Hi = DAG.getConstant(0, dl, NVT);
2458 }
2459 
2460 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
2461  SDValue &Hi) {
2462  SDLoc dl(N);
2463  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2464  unsigned NBitWidth = NVT.getSizeInBits();
2465 
2466  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2467  Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT);
2468  // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
2469  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2470  DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
2471 }
2472 
2473 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
2474  SDValue &Hi) {
2475  SDLoc dl(N);
2476  EVT VT = N->getValueType(0);
2477 
2478  SDValue Op = N->getOperand(0);
2479  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2480  Op = GetPromotedFloat(Op);
2481 
2483  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
2485  CallOptions.setSExt(true);
2486  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, dl).first,
2487  Lo, Hi);
2488 }
2489 
2490 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
2491  SDValue &Hi) {
2492  SDLoc dl(N);
2493  EVT VT = N->getValueType(0);
2494 
2495  SDValue Op = N->getOperand(0);
2496  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2497  Op = GetPromotedFloat(Op);
2498 
2500  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
2502  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, dl).first,
2503  Lo, Hi);
2504 }
2505 
2506 void DAGTypeLegalizer::ExpandIntRes_LLROUND(SDNode *N, SDValue &Lo,
2507  SDValue &Hi) {
2508  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2510  if (VT == MVT::f32)
2511  LC = RTLIB::LLROUND_F32;
2512  else if (VT == MVT::f64)
2513  LC = RTLIB::LLROUND_F64;
2514  else if (VT == MVT::f80)
2515  LC = RTLIB::LLROUND_F80;
2516  else if (VT == MVT::f128)
2517  LC = RTLIB::LLROUND_F128;
2518  else if (VT == MVT::ppcf128)
2519  LC = RTLIB::LLROUND_PPCF128;
2520  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!");
2521 
2522  SDValue Op = N->getOperand(0);
2523  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2524  Op = GetPromotedFloat(Op);
2525 
2526  SDLoc dl(N);
2527  EVT RetVT = N->getValueType(0);
2529  CallOptions.setSExt(true);
2530  SplitInteger(TLI.makeLibCall(DAG, LC, RetVT, Op, CallOptions, dl).first,
2531  Lo, Hi);
2532 }
2533 
2534 void DAGTypeLegalizer::ExpandIntRes_LLRINT(SDNode *N, SDValue &Lo,
2535  SDValue &Hi) {
2536  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2538  if (VT == MVT::f32)
2539  LC = RTLIB::LLRINT_F32;
2540  else if (VT == MVT::f64)
2541  LC = RTLIB::LLRINT_F64;
2542  else if (VT == MVT::f80)
2543  LC = RTLIB::LLRINT_F80;
2544  else if (VT == MVT::f128)
2545  LC = RTLIB::LLRINT_F128;
2546  else if (VT == MVT::ppcf128)
2547  LC = RTLIB::LLRINT_PPCF128;
2548  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!");
2549 
2550  SDValue Op = N->getOperand(0);
2551  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2552  Op = GetPromotedFloat(Op);
2553 
2554  SDLoc dl(N);
2555  EVT RetVT = N->getValueType(0);
2557  CallOptions.setSExt(true);
2558  SplitInteger(TLI.makeLibCall(DAG, LC, RetVT, Op, CallOptions, dl).first,
2559  Lo, Hi);
2560 }
2561 
2562 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
2563  SDValue &Lo, SDValue &Hi) {
2564  if (ISD::isNormalLoad(N)) {
2565  ExpandRes_NormalLoad(N, Lo, Hi);
2566  return;
2567  }
2568 
2569  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2570 
2571  EVT VT = N->getValueType(0);
2572  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2573  SDValue Ch = N->getChain();
2574  SDValue Ptr = N->getBasePtr();
2576  unsigned Alignment = N->getAlignment();
2577  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2578  AAMDNodes AAInfo = N->getAAInfo();
2579  SDLoc dl(N);
2580 
2581  assert(NVT.isByteSized() && "Expanded type not byte sized!");
2582 
2583  if (N->getMemoryVT().bitsLE(NVT)) {
2584  EVT MemVT = N->getMemoryVT();
2585 
2586  Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
2587  Alignment, MMOFlags, AAInfo);
2588 
2589  // Remember the chain.
2590  Ch = Lo.getValue(1);
2591 
2592  if (ExtType == ISD::SEXTLOAD) {
2593  // The high part is obtained by SRA'ing all but one of the bits of the
2594  // lo part.
2595  unsigned LoSize = Lo.getValueSizeInBits();
2596  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2597  DAG.getConstant(LoSize - 1, dl,
2598  TLI.getPointerTy(DAG.getDataLayout())));
2599  } else if (ExtType == ISD::ZEXTLOAD) {
2600  // The high part is just a zero.
2601  Hi = DAG.getConstant(0, dl, NVT);
2602  } else {
2603  assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
2604  // The high part is undefined.
2605  Hi = DAG.getUNDEF(NVT);
2606  }
2607  } else if (DAG.getDataLayout().isLittleEndian()) {
2608  // Little-endian - low bits are at low addresses.
2609  Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2610  AAInfo);
2611 
2612  unsigned ExcessBits =
2613  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
2614  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
2615 
2616  // Increment the pointer to the other half.
2617  unsigned IncrementSize = NVT.getSizeInBits()/8;
2618  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2619  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2620  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
2621  N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
2622  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2623 
2624  // Build a factor node to remember that this load is independent of the
2625  // other one.
2626  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2627  Hi.getValue(1));
2628  } else {
2629  // Big-endian - high bits are at low addresses. Favor aligned loads at
2630  // the cost of some bit-fiddling.
2631  EVT MemVT = N->getMemoryVT();
2632  unsigned EBytes = MemVT.getStoreSize();
2633  unsigned IncrementSize = NVT.getSizeInBits()/8;
2634  unsigned ExcessBits = (EBytes - IncrementSize)*8;
2635 
2636  // Load both the high bits and maybe some of the low bits.
2637  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
2639  MemVT.getSizeInBits() - ExcessBits),
2640  Alignment, MMOFlags, AAInfo);
2641 
2642  // Increment the pointer to the other half.
2643  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2644  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2645  // Load the rest of the low bits.
2646  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
2647  N->getPointerInfo().getWithOffset(IncrementSize),
2648  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
2649  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2650 
2651  // Build a factor node to remember that this load is independent of the
2652  // other one.
2653  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2654  Hi.getValue(1));
2655 
2656  if (ExcessBits < NVT.getSizeInBits()) {
2657  // Transfer low bits from the bottom of Hi to the top of Lo.
2658  Lo = DAG.getNode(
2659  ISD::OR, dl, NVT, Lo,
2660  DAG.getNode(ISD::SHL, dl, NVT, Hi,
2661  DAG.getConstant(ExcessBits, dl,
2662  TLI.getPointerTy(DAG.getDataLayout()))));
2663  // Move high bits to the right position in Hi.
2664  Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
2665  Hi,
2666  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
2667  TLI.getPointerTy(DAG.getDataLayout())));
2668  }
2669  }
2670 
2671  // Legalize the chain result - switch anything that used the old chain to
2672  // use the new one.
2673  ReplaceValueWith(SDValue(N, 1), Ch);
2674 }
2675 
2676 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
2677  SDValue &Lo, SDValue &Hi) {
2678  SDLoc dl(N);
2679  SDValue LL, LH, RL, RH;
2680  GetExpandedInteger(N->getOperand(0), LL, LH);
2681  GetExpandedInteger(N->getOperand(1), RL, RH);
2682  Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
2683  Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
2684 }
2685 
2686 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
2687  SDValue &Lo, SDValue &Hi) {
2688  EVT VT = N->getValueType(0);
2689  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2690  SDLoc dl(N);
2691 
2692  SDValue LL, LH, RL, RH;
2693  GetExpandedInteger(N->getOperand(0), LL, LH);
2694  GetExpandedInteger(N->getOperand(1), RL, RH);
2695 
2696  if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
2698  LL, LH, RL, RH))
2699  return;
2700 
2701  // If nothing else, we can make a libcall.
2702  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2703  if (VT == MVT::i16)
2704  LC = RTLIB::MUL_I16;
2705  else if (VT == MVT::i32)
2706  LC = RTLIB::MUL_I32;
2707  else if (VT == MVT::i64)
2708  LC = RTLIB::MUL_I64;
2709  else if (VT == MVT::i128)
2710  LC = RTLIB::MUL_I128;
2711 
2712  if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
2713  // We'll expand the multiplication by brute force because we have no other
2714  // options. This is a trivially-generalized version of the code from
2715  // Hacker's Delight (itself derived from Knuth's Algorithm M from section
2716  // 4.3.1).
2717  unsigned Bits = NVT.getSizeInBits();
2718  unsigned HalfBits = Bits >> 1;
2719  SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
2720  NVT);
2721  SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
2722  SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
2723 
2724  SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
2725  SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
2726 
2727  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2728  if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
2729  // The type from TLI is too small to fit the shift amount we want.
2730  // Override it with i32. The shift will have to be legalized.
2731  ShiftAmtTy = MVT::i32;
2732  }
2733  SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
2734  SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
2735  SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
2736  SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
2737 
2738  SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
2739  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
2740  SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
2741  SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
2742 
2743  SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
2744  DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
2745  SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
2746 
2747  SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
2748  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
2749  DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
2750  Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
2751  DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
2752 
2753  Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
2754  DAG.getNode(ISD::ADD, dl, NVT,
2755  DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
2756  DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
2757  return;
2758  }
2759 
2760  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2762  CallOptions.setSExt(true);
2763  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
2764  Lo, Hi);
2765 }
2766 
2767 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
2768  SDValue &Hi) {
2769  SDLoc DL(N);
2770  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2771  SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
2772  SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
2773  Lo = R.getValue(0);
2774  Hi = R.getValue(1);
2775  ReplaceValueWith(SDValue(N, 1), R.getValue(2));
2776 }
2777 
2778 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
2779  SDValue &Hi) {
2780  SDValue Result = TLI.expandAddSubSat(N, DAG);
2781  SplitInteger(Result, Lo, Hi);
2782 }
2783 
2784 /// This performs an expansion of the integer result for a fixed point
2785 /// multiplication. The default expansion performs rounding down towards
2786 /// negative infinity, though targets that do care about rounding should specify
2787 /// a target hook for rounding and provide their own expansion or lowering of
2788 /// fixed point multiplication to be consistent with rounding.
2789 void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
2790  SDValue &Hi) {
2791  SDLoc dl(N);
2792  EVT VT = N->getValueType(0);
2793  unsigned VTSize = VT.getScalarSizeInBits();
2794  SDValue LHS = N->getOperand(0);
2795  SDValue RHS = N->getOperand(1);
2796  uint64_t Scale = N->getConstantOperandVal(2);
2797  bool Saturating = N->getOpcode() == ISD::SMULFIXSAT;
2798  EVT BoolVT = getSetCCResultType(VT);
2799  SDValue Zero = DAG.getConstant(0, dl, VT);
2800  if (!Scale) {
2801  SDValue Result;
2802  if (!Saturating) {
2803  Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
2804  } else {
2805  Result = DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
2806  SDValue Product = Result.getValue(0);
2807  SDValue Overflow = Result.getValue(1);
2808 
2809  APInt MinVal = APInt::getSignedMinValue(VTSize);
2810  APInt MaxVal = APInt::getSignedMaxValue(VTSize);
2811  SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
2812  SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
2813  SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Product, Zero, ISD::SETLT);
2814  Result = DAG.getSelect(dl, VT, ProdNeg, SatMax, SatMin);
2815  Result = DAG.getSelect(dl, VT, Overflow, Result, Product);
2816  }
2817  SplitInteger(Result, Lo, Hi);
2818  return;
2819  }
2820 
2821  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2822  SDValue LL, LH, RL, RH;
2823  GetExpandedInteger(LHS, LL, LH);
2824  GetExpandedInteger(RHS, RL, RH);
2825  SmallVector<SDValue, 4> Result;
2826 
2827  bool Signed = (N->getOpcode() == ISD::SMULFIX ||
2828  N->getOpcode() == ISD::SMULFIXSAT);
2829  unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
2830  if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
2832  LL, LH, RL, RH)) {
2833  report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI.");
2834  return;
2835  }
2836 
2837  unsigned NVTSize = NVT.getScalarSizeInBits();
2838  assert((VTSize == NVTSize * 2) && "Expected the new value type to be half "
2839  "the size of the current value type");
2840  EVT ShiftTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2841 
2842  // Shift whole amount by scale.
2843  SDValue ResultLL = Result[0];
2844  SDValue ResultLH = Result[1];
2845  SDValue ResultHL = Result[2];
2846  SDValue ResultHH = Result[3];
2847 
2848  SDValue SatMax, SatMin;
2849  SDValue NVTZero = DAG.getConstant(0, dl, NVT);
2850  SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT);
2851  EVT BoolNVT = getSetCCResultType(NVT);
2852 
2853  // After getting the multplication result in 4 parts, we need to perform a
2854  // shift right by the amount of the scale to get the result in that scale.
2855  // Let's say we multiply 2 64 bit numbers. The resulting value can be held in
2856  // 128 bits that are cut into 4 32-bit parts:
2857  //
2858  // HH HL LH LL
2859  // |---32---|---32---|---32---|---32---|
2860  // 128 96 64 32 0
2861  //
2862  // |------VTSize-----|
2863  //
2864  // |NVTSize-|
2865  //
2866  // The resulting Lo and Hi will only need to be one of these 32-bit parts
2867  // after shifting.
2868  if (Scale < NVTSize) {
2869  // If the scale is less than the size of the VT we expand to, the Hi and
2870  // Lo of the result will be in the first 2 parts of the result after
2871  // shifting right. This only requires shifting by the scale as far as the
2872  // third part in the result (ResultHL).
2873  SDValue SRLAmnt = DAG.getConstant(Scale, dl, ShiftTy);
2874  SDValue SHLAmnt = DAG.getConstant(NVTSize - Scale, dl, ShiftTy);
2875  Lo = DAG.getNode(ISD::SRL, dl, NVT, ResultLL, SRLAmnt);
2876  Lo = DAG.getNode(ISD::OR, dl, NVT, Lo,
2877  DAG.getNode(ISD::SHL, dl, NVT, ResultLH, SHLAmnt));
2878  Hi = DAG.getNode(ISD::SRL, dl, NVT, ResultLH, SRLAmnt);
2879  Hi = DAG.getNode(ISD::OR, dl, NVT, Hi,
2880  DAG.getNode(ISD::SHL, dl, NVT, ResultHL, SHLAmnt));
2881 
2882  // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the
2883  // highest bit of HH determines saturation direction in the event of
2884  // saturation.
2885  // The number of overflow bits we can check are VTSize - Scale + 1 (we
2886  // include the sign bit). If these top bits are > 0, then we overflowed past
2887  // the max value. If these top bits are < -1, then we overflowed past the
2888  // min value. Otherwise, we did not overflow.
2889  if (Saturating) {
2890  unsigned OverflowBits = VTSize - Scale + 1;
2891  assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
2892  "Extent of overflow bits must start within HL");
2893  SDValue HLHiMask = DAG.getConstant(
2894  APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT);
2895  SDValue HLLoMask = DAG.getConstant(
2896  APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT);
2897 
2898  // HH > 0 or HH == 0 && HL > HLLoMask
2899  SDValue HHPos = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
2900  SDValue HHZero = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
2901  SDValue HLPos =
2902  DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT);
2903  SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHPos,
2904  DAG.getNode(ISD::AND, dl, BoolNVT, HHZero, HLPos));
2905 
2906  // HH < -1 or HH == -1 && HL < HLHiMask
2907  SDValue HHNeg = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
2908  SDValue HHNeg1 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
2909  SDValue HLNeg =
2910  DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT);
2911  SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHNeg,
2912  DAG.getNode(ISD::AND, dl, BoolNVT, HHNeg1, HLNeg));
2913  }
2914  } else if (Scale == NVTSize) {
2915  // If the scales are equal, Lo and Hi are ResultLH and Result HL,
2916  // respectively. Avoid shifting to prevent undefined behavior.
2917  Lo = ResultLH;
2918  Hi = ResultHL;
2919 
2920  // We overflow max if HH > 0 or HH == 0 && HL sign bit is 1.
2921  // We overflow min if HH < -1 or HH == -1 && HL sign bit is 0.
2922  if (Saturating) {
2923  SDValue HHPos = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
2924  SDValue HHZero = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
2925  SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT);
2926  SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHPos,
2927  DAG.getNode(ISD::AND, dl, BoolNVT, HHZero, HLNeg));
2928 
2929  SDValue HHNeg = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
2930  SDValue HHNeg1 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
2931  SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE);
2932  SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHNeg,
2933  DAG.getNode(ISD::AND, dl, BoolNVT, HHNeg1, HLPos));
2934  }
2935  } else if (Scale < VTSize) {
2936  // If the scale is instead less than the old VT size, but greater than or
2937  // equal to the expanded VT size, the first part of the result (ResultLL) is
2938  // no longer a part of Lo because it would be scaled out anyway. Instead we
2939  // can start shifting right from the fourth part (ResultHH) to the second
2940  // part (ResultLH), and Result LH will be the new Lo.
2941  SDValue SRLAmnt = DAG.getConstant(Scale - NVTSize, dl, ShiftTy);
2942  SDValue SHLAmnt = DAG.getConstant(VTSize - Scale, dl, ShiftTy);
2943  Lo = DAG.getNode(ISD::SRL, dl, NVT, ResultLH, SRLAmnt);
2944  Lo = DAG.getNode(ISD::OR, dl, NVT, Lo,
2945  DAG.getNode(ISD::SHL, dl, NVT, ResultHL, SHLAmnt));
2946  Hi = DAG.getNode(ISD::SRL, dl, NVT, ResultHL, SRLAmnt);
2947  Hi = DAG.getNode(ISD::OR, dl, NVT, Hi,
2948  DAG.getNode(ISD::SHL, dl, NVT, ResultHH, SHLAmnt));
2949 
2950  // This is similar to the case when we saturate if Scale < NVTSize, but we
2951  // only need to chech HH.
2952  if (Saturating) {
2953  unsigned OverflowBits = VTSize - Scale + 1;
2954  SDValue HHHiMask = DAG.getConstant(
2955  APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT);
2956  SDValue HHLoMask = DAG.getConstant(
2957  APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT);
2958 
2959  SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT);
2960  SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT);
2961  }
2962  } else if (Scale == VTSize) {
2963  assert(
2964  !Signed &&
2965  "Only unsigned types can have a scale equal to the operand bit width");
2966 
2967  Lo = ResultHL;
2968  Hi = ResultHH;
2969  } else {
2970  llvm_unreachable("Expected the scale to be less than or equal to the width "
2971  "of the operands");
2972  }
2973 
2974  if (Saturating) {
2975  APInt LHMax = APInt::getSignedMaxValue(NVTSize);
2976  APInt LLMax = APInt::getAllOnesValue(NVTSize);
2977  APInt LHMin = APInt::getSignedMinValue(NVTSize);
2978  Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(LHMax, dl, NVT), Hi);
2979  Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(LHMin, dl, NVT), Hi);
2980  Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(LLMax, dl, NVT), Lo);
2981  Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo);
2982  }
2983 }
2984 
2985 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
2986  SDValue &Lo, SDValue &Hi) {
2987  SDValue LHS = Node->getOperand(0);
2988  SDValue RHS = Node->getOperand(1);
2989  SDLoc dl(Node);
2990 
2991  // Expand the result by simply replacing it with the equivalent
2992  // non-overflow-checking operation.
2993  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
2994  ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
2995  LHS, RHS);
2996  SplitInteger(Sum, Lo, Hi);
2997 
2998  // Compute the overflow.
2999  //
3000  // LHSSign -> LHS >= 0
3001  // RHSSign -> RHS >= 0
3002  // SumSign -> Sum >= 0
3003  //
3004  // Add:
3005  // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3006  // Sub:
3007  // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3008  //
3009  EVT OType = Node->getValueType(1);
3010  SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
3011 
3012  SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3013  SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3014  SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
3015  Node->getOpcode() == ISD::SADDO ?
3017 
3018  SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3019  SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3020 
3021  SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3022 
3023  // Use the calculated overflow everywhere.
3024  ReplaceValueWith(SDValue(Node, 1), Cmp);
3025 }
3026 
3027 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
3028  SDValue &Lo, SDValue &Hi) {
3029  EVT VT = N->getValueType(0);
3030  SDLoc dl(N);
3031  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3032 
3034  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3035  SplitInteger(Res.getValue(0), Lo, Hi);
3036  return;
3037  }
3038 
3039  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3040  if (VT == MVT::i16)
3041  LC = RTLIB::SDIV_I16;
3042  else if (VT == MVT::i32)
3043  LC = RTLIB::SDIV_I32;
3044  else if (VT == MVT::i64)
3045  LC = RTLIB::SDIV_I64;
3046  else if (VT == MVT::i128)
3047  LC = RTLIB::SDIV_I128;
3048  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
3049 
3051  CallOptions.setSExt(true);
3052  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3053 }
3054 
3055 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
3056  SDValue &Lo, SDValue &Hi) {
3057  EVT VT = N->getValueType(0);
3058  SDLoc dl(N);
3059 
3060  // If we can emit an efficient shift operation, do so now. Check to see if
3061  // the RHS is a constant.
3062  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
3063  return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
3064 
3065  // If we can determine that the high bit of the shift is zero or one, even if
3066  // the low bits are variable, emit this shift in an optimized form.
3067  if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
3068  return;
3069 
3070  // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
3071  unsigned PartsOpc;
3072  if (N->getOpcode() == ISD::SHL) {
3073  PartsOpc = ISD::SHL_PARTS;
3074  } else if (N->getOpcode() == ISD::SRL) {
3075  PartsOpc = ISD::SRL_PARTS;
3076  } else {
3077  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3078  PartsOpc = ISD::SRA_PARTS;
3079  }
3080 
3081  // Next check to see if the target supports this SHL_PARTS operation or if it
3082  // will custom expand it. Don't lower this to SHL_PARTS when we optimise for
3083  // size, but create a libcall instead.
3084  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3085  TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
3086  const bool LegalOrCustom =
3087  (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
3088  Action == TargetLowering::Custom;
3089 
3090  if (LegalOrCustom && TLI.shouldExpandShift(DAG, N)) {
3091  // Expand the subcomponents.
3092  SDValue LHSL, LHSH;
3093  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3094  EVT VT = LHSL.getValueType();
3095 
3096  // If the shift amount operand is coming from a vector legalization it may
3097  // have an illegal type. Fix that first by casting the operand, otherwise
3098  // the new SHL_PARTS operation would need further legalization.
3099  SDValue ShiftOp = N->getOperand(1);
3100  EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3101  assert(ShiftTy.getScalarSizeInBits() >=
3102  Log2_32_Ceil(VT.getScalarSizeInBits()) &&
3103  "ShiftAmountTy is too small to cover the range of this type!");
3104  if (ShiftOp.getValueType() != ShiftTy)
3105  ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
3106 
3107  SDValue Ops[] = { LHSL, LHSH, ShiftOp };
3108  Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
3109  Hi = Lo.getValue(1);
3110  return;
3111  }
3112 
3113  // Otherwise, emit a libcall.
3114  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3115  bool isSigned;
3116  if (N->getOpcode() == ISD::SHL) {
3117  isSigned = false; /*sign irrelevant*/
3118  if (VT == MVT::i16)
3119  LC = RTLIB::SHL_I16;
3120  else if (VT == MVT::i32)
3121  LC = RTLIB::SHL_I32;
3122  else if (VT == MVT::i64)
3123  LC = RTLIB::SHL_I64;
3124  else if (VT == MVT::i128)
3125  LC = RTLIB::SHL_I128;
3126  } else if (N->getOpcode() == ISD::SRL) {
3127  isSigned = false;
3128  if (VT == MVT::i16)
3129  LC = RTLIB::SRL_I16;
3130  else if (VT == MVT::i32)
3131  LC = RTLIB::SRL_I32;
3132  else if (VT == MVT::i64)
3133  LC = RTLIB::SRL_I64;
3134  else if (VT == MVT::i128)
3135  LC = RTLIB::SRL_I128;
3136  } else {
3137  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3138  isSigned = true;
3139  if (VT == MVT::i16)
3140  LC = RTLIB::SRA_I16;
3141  else if (VT == MVT::i32)
3142  LC = RTLIB::SRA_I32;
3143  else if (VT == MVT::i64)
3144  LC = RTLIB::SRA_I64;
3145  else if (VT == MVT::i128)
3146  LC = RTLIB::SRA_I128;
3147  }
3148 
3149  if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
3150  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3152  CallOptions.setSExt(isSigned);
3153  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3154  return;
3155  }
3156 
3157  if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
3158  llvm_unreachable("Unsupported shift!");
3159 }
3160 
3161 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
3162  SDValue &Lo, SDValue &Hi) {
3163  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3164  SDLoc dl(N);
3165  SDValue Op = N->getOperand(0);
3166  if (Op.getValueType().bitsLE(NVT)) {
3167  // The low part is sign extension of the input (degenerates to a copy).
3168  Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
3169  // The high part is obtained by SRA'ing all but one of the bits of low part.
3170  unsigned LoSize = NVT.getSizeInBits();
3171  Hi = DAG.getNode(
3172  ISD::SRA, dl, NVT, Lo,
3173  DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
3174  } else {
3175  // For example, extension of an i48 to an i64. The operand type necessarily
3176  // promotes to the result type, so will end up being expanded too.
3177  assert(getTypeAction(Op.getValueType()) ==
3179  "Only know how to promote this result!");
3180  SDValue Res = GetPromotedInteger(Op);
3181  assert(Res.getValueType() == N->getValueType(0) &&
3182  "Operand over promoted?");
3183  // Split the promoted operand. This will simplify when it is expanded.
3184  SplitInteger(Res, Lo, Hi);
3185  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
3186  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
3188  ExcessBits)));
3189  }
3190 }
3191 
3192 void DAGTypeLegalizer::
3193 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
3194  SDLoc dl(N);
3195  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3196  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3197 
3198  if (EVT.bitsLE(Lo.getValueType())) {
3199  // sext_inreg the low part if needed.
3200  Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
3201  N->getOperand(1));
3202 
3203  // The high part gets the sign extension from the lo-part. This handles
3204  // things like sextinreg V:i64 from i8.
3205  Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
3206  DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
3207  TLI.getPointerTy(DAG.getDataLayout())));
3208  } else {
3209  // For example, extension of an i48 to an i64. Leave the low part alone,
3210  // sext_inreg the high part.
3211  unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
3212  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
3214  ExcessBits)));
3215  }
3216 }
3217 
3218 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
3219  SDValue &Lo, SDValue &Hi) {
3220  EVT VT = N->getValueType(0);
3221  SDLoc dl(N);
3222  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3223 
3225  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3226  SplitInteger(Res.getValue(1), Lo, Hi);
3227  return;
3228  }
3229 
3230  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3231  if (VT == MVT::i16)
3232  LC = RTLIB::SREM_I16;
3233  else if (VT == MVT::i32)
3234  LC = RTLIB::SREM_I32;
3235  else if (VT == MVT::i64)
3236  LC = RTLIB::SREM_I64;
3237  else if (VT == MVT::i128)
3238  LC = RTLIB::SREM_I128;
3239  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
3240 
3242  CallOptions.setSExt(true);
3243  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3244 }
3245 
3246 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
3247  SDValue &Lo, SDValue &Hi) {
3248  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3249  SDLoc dl(N);
3250  Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
3251  Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
3252  N->getOperand(0),
3253  DAG.getConstant(NVT.getSizeInBits(), dl,
3254  TLI.getPointerTy(DAG.getDataLayout())));
3255  Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
3256 }
3257 
3258 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
3259  SDValue &Lo, SDValue &Hi) {
3260  EVT VT = N->getValueType(0);
3261  SDLoc dl(N);
3262 
3263  if (N->getOpcode() == ISD::UMULO) {
3264  // This section expands the operation into the following sequence of
3265  // instructions. `iNh` here refers to a type which has half the bit width of
3266  // the type the original operation operated on.
3267  //
3268  // %0 = %LHS.HI != 0 && %RHS.HI != 0
3269  // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
3270  // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
3271  // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
3272  // %4 = add iN (%1.0 as iN) << Nh, (%2.0 as iN) << Nh
3273  // %5 = { iN, i1 } @uadd.with.overflow.iN( %4, %3 )
3274  //
3275  // %res = { %5.0, %0 || %1.1 || %2.1 || %5.1 }
3276  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
3277  SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
3278  SplitInteger(LHS, LHSLow, LHSHigh);
3279  SplitInteger(RHS, RHSLow, RHSHigh);
3280  EVT HalfVT = LHSLow.getValueType()
3281  , BitVT = N->getValueType(1);
3282  SDVTList VTHalfMulO = DAG.getVTList(HalfVT, BitVT);
3283  SDVTList VTFullAddO = DAG.getVTList(VT, BitVT);
3284 
3285  SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
3286  SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
3287  DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
3288  DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));
3289 
3290  SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, LHSHigh, RHSLow);
3291  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));
3292  SDValue OneInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3293  One.getValue(0));
3294 
3295  SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, RHSHigh, LHSLow);
3296  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));
3297  SDValue TwoInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3298  Two.getValue(0));
3299 
3300  // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
3301  // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
3302  // operation recursively legalized?).
3303  //
3304  // Many backends understand this pattern and will convert into LOHI
3305  // themselves, if applicable.
3306  SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
3307  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
3308  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow));
3309  SDValue Four = DAG.getNode(ISD::ADD, dl, VT, OneInHigh, TwoInHigh);
3310  SDValue Five = DAG.getNode(ISD::UADDO, dl, VTFullAddO, Three, Four);
3311  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Five.getValue(1));
3312  SplitInteger(Five, Lo, Hi);
3313  ReplaceValueWith(SDValue(N, 1), Overflow);
3314  return;
3315  }
3316 
3317  Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
3318  EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
3319  Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
3320 
3321  // Replace this with a libcall that will check overflow.
3322  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3323  if (VT == MVT::i32)
3324  LC = RTLIB::MULO_I32;
3325  else if (VT == MVT::i64)
3326  LC = RTLIB::MULO_I64;
3327  else if (VT == MVT::i128)
3328  LC = RTLIB::MULO_I128;
3329  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
3330 
3331  SDValue Temp = DAG.CreateStackTemporary(PtrVT);
3332  // Temporary for the overflow value, default it to zero.
3333  SDValue Chain =
3334  DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
3335  MachinePointerInfo());
3336 
3338  TargetLowering::ArgListEntry Entry;
3339  for (const SDValue &Op : N->op_values()) {
3340  EVT ArgVT = Op.getValueType();
3341  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
3342  Entry.Node = Op;
3343  Entry.Ty = ArgTy;
3344  Entry.IsSExt = true;
3345  Entry.IsZExt = false;
3346  Args.push_back(Entry);
3347  }
3348 
3349  // Also pass the address of the overflow check.
3350  Entry.Node = Temp;
3351  Entry.Ty = PtrTy->getPointerTo();
3352  Entry.IsSExt = true;
3353  Entry.IsZExt = false;
3354  Args.push_back(Entry);
3355 
3356  SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
3357 
3359  CLI.setDebugLoc(dl)
3360  .setChain(Chain)
3361  .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
3362  .setSExtResult();
3363 
3364  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
3365 
3366  SplitInteger(CallInfo.first, Lo, Hi);
3367  SDValue Temp2 =
3368  DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
3369  SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
3370  DAG.getConstant(0, dl, PtrVT),
3371  ISD::SETNE);
3372  // Use the overflow from the libcall everywhere.
3373  ReplaceValueWith(SDValue(N, 1), Ofl);
3374 }
3375 
3376 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
3377  SDValue &Lo, SDValue &Hi) {
3378  EVT VT = N->getValueType(0);
3379  SDLoc dl(N);
3380  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3381 
3383  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3384  SplitInteger(Res.getValue(0), Lo, Hi);
3385  return;
3386  }
3387 
3388  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3389  if (VT == MVT::i16)
3390  LC = RTLIB::UDIV_I16;
3391  else if (VT == MVT::i32)
3392  LC = RTLIB::UDIV_I32;
3393  else if (VT == MVT::i64)
3394  LC = RTLIB::UDIV_I64;
3395  else if (VT == MVT::i128)
3396  LC = RTLIB::UDIV_I128;
3397  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
3398 
3400  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3401 }
3402 
3403 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
3404  SDValue &Lo, SDValue &Hi) {
3405  EVT VT = N->getValueType(0);
3406  SDLoc dl(N);
3407  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3408 
3410  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3411  SplitInteger(Res.getValue(1), Lo, Hi);
3412  return;
3413  }
3414 
3415  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3416  if (VT == MVT::i16)
3417  LC = RTLIB::UREM_I16;
3418  else if (VT == MVT::i32)
3419  LC = RTLIB::UREM_I32;
3420  else if (VT == MVT::i64)
3421  LC = RTLIB::UREM_I64;
3422  else if (VT == MVT::i128)
3423  LC = RTLIB::UREM_I128;
3424  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
3425 
3427  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3428 }
3429 
3430 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
3431  SDValue &Lo, SDValue &Hi) {
3432  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3433  SDLoc dl(N);
3434  SDValue Op = N->getOperand(0);
3435  if (Op.getValueType().bitsLE(NVT)) {
3436  // The low part is zero extension of the input (degenerates to a copy).
3437  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
3438  Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
3439  } else {
3440  // For example, extension of an i48 to an i64. The operand type necessarily
3441  // promotes to the result type, so will end up being expanded too.
3442  assert(getTypeAction(Op.getValueType()) ==
3444  "Only know how to promote this result!");
3445  SDValue Res = GetPromotedInteger(Op);
3446  assert(Res.getValueType() == N->getValueType(0) &&
3447  "Operand over promoted?");
3448  // Split the promoted operand. This will simplify when it is expanded.
3449  SplitInteger(Res, Lo, Hi);
3450  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
3451  Hi = DAG.getZeroExtendInReg(Hi, dl,
3453  ExcessBits));
3454  }
3455 }
3456 
3457 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
3458  SDValue &Lo, SDValue &Hi) {
3459  SDLoc dl(N);
3460  EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
3461  SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
3462  SDValue Zero = DAG.getConstant(0, dl, VT);
3463  SDValue Swap = DAG.getAtomicCmpSwap(
3465  cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
3466  N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
3467 
3468  ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
3469  ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
3470 }
3471 
3472 void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N,
3473  SDValue &Lo, SDValue &Hi) {
3474  // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate
3475  // both halves independently.
3476  SDValue Res = TLI.expandVecReduce(N, DAG);
3477  SplitInteger(Res, Lo, Hi);
3478 }
3479 
3480 //===----------------------------------------------------------------------===//
3481 // Integer Operand Expansion
3482 //===----------------------------------------------------------------------===//
3483 
3484 /// ExpandIntegerOperand - This method is called when the specified operand of
3485 /// the specified node is found to need expansion. At this point, all of the
3486 /// result types of the node are known to be legal, but other operands of the
3487 /// node may need promotion or expansion as well as the specified one.
3488 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
3489  LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG);
3490  dbgs() << "\n");
3491  SDValue Res = SDValue();
3492 
3493  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
3494  return false;
3495 
3496  switch (N->getOpcode()) {
3497  default:
3498  #ifndef NDEBUG
3499  dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
3500  N->dump(&DAG); dbgs() << "\n";
3501  #endif
3502  report_fatal_error("Do not know how to expand this operator's operand!");
3503 
3504  case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
3505  case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
3506  case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
3507  case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
3508  case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
3509  case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
3510  case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
3511  case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
3512  case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
3513  case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break;
3514  case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
3515  case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
3516  case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break;
3517 
3518  case ISD::SHL:
3519  case ISD::SRA:
3520  case ISD::SRL:
3521  case ISD::ROTL:
3522  case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
3523  case ISD::RETURNADDR:
3524  case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
3525 
3526  case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
3527  }
3528 
3529  // If the result is null, the sub-method took care of registering results etc.
3530  if (!Res.getNode()) return false;
3531 
3532  // If the result is N, the sub-method updated N in place. Tell the legalizer
3533  // core about this.
3534  if (Res.getNode() == N)
3535  return true;
3536 
3537  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3538  "Invalid operand expansion");
3539 
3540  ReplaceValueWith(SDValue(N, 0), Res);
3541  return false;
3542 }
3543 
3544 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
3545 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
3546 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
3547  SDValue &NewRHS,
3548  ISD::CondCode &CCCode,
3549  const SDLoc &dl) {
3550  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3551  GetExpandedInteger(NewLHS, LHSLo, LHSHi);
3552  GetExpandedInteger(NewRHS, RHSLo, RHSHi);
3553 
3554  if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
3555  if (RHSLo == RHSHi) {
3556  if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
3557  if (RHSCST->isAllOnesValue()) {
3558  // Equality comparison to -1.
3559  NewLHS = DAG.getNode(ISD::AND, dl,
3560  LHSLo.getValueType(), LHSLo, LHSHi);
3561  NewRHS = RHSLo;
3562  return;
3563  }
3564  }
3565  }
3566 
3567  NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
3568  NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
3569  NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
3570  NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
3571  return;
3572  }
3573 
3574  // If this is a comparison of the sign bit, just look at the top part.
3575  // X > -1, x < 0
3576  if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
3577  if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0
3578  (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1
3579  NewLHS = LHSHi;
3580  NewRHS = RHSHi;
3581  return;
3582  }
3583 
3584  // FIXME: This generated code sucks.
3585  ISD::CondCode LowCC;
3586  switch (CCCode) {
3587  default: llvm_unreachable("Unknown integer setcc!");
3588  case ISD::SETLT:
3589  case ISD::SETULT: LowCC = ISD::SETULT; break;
3590  case ISD::SETGT:
3591  case ISD::SETUGT: LowCC = ISD::SETUGT; break;
3592  case ISD::SETLE:
3593  case ISD::SETULE: LowCC = ISD::SETULE; break;
3594  case ISD::SETGE:
3595  case ISD::SETUGE: LowCC = ISD::SETUGE; break;
3596  }
3597 
3598  // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
3599  // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
3600  // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
3601 
3602  // NOTE: on targets without efficient SELECT of bools, we can always use
3603  // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
3604  TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
3605  nullptr);
3606  SDValue LoCmp, HiCmp;
3607  if (TLI.isTypeLegal(LHSLo.getValueType()) &&
3608  TLI.isTypeLegal(RHSLo.getValueType()))
3609  LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
3610  RHSLo, LowCC, false, DagCombineInfo, dl);
3611  if (!LoCmp.getNode())
3612  LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
3613  RHSLo, LowCC);
3614  if (TLI.isTypeLegal(LHSHi.getValueType()) &&
3615  TLI.isTypeLegal(RHSHi.getValueType()))
3616  HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
3617  RHSHi, CCCode, false, DagCombineInfo, dl);
3618  if (!HiCmp.getNode())
3619  HiCmp =
3620  DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
3621  LHSHi, RHSHi, DAG.getCondCode(CCCode));
3622 
3623  ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
3624  ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
3625 
3626  bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
3627  CCCode == ISD::SETUGE || CCCode == ISD::SETULE);
3628 
3629  if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) ||
3630  (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) ||
3631  (LoCmpC && LoCmpC->isNullValue())))) {
3632  // For LE / GE, if high part is known false, ignore the low part.
3633  // For LT / GT: if low part is known false, return the high part.
3634  // if high part is known true, ignore the low part.
3635  NewLHS = HiCmp;
3636  NewRHS = SDValue();
3637  return;
3638  }
3639 
3640  if (LHSHi == RHSHi) {
3641  // Comparing the low bits is enough.
3642  NewLHS = LoCmp;
3643  NewRHS = SDValue();
3644  return;
3645  }
3646 
3647  // Lower with SETCCCARRY if the target supports it.
3648  EVT HiVT = LHSHi.getValueType();
3649  EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
3650  bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
3651 
3652  // FIXME: Make all targets support this, then remove the other lowering.
3653  if (HasSETCCCARRY) {
3654  // SETCCCARRY can detect < and >= directly. For > and <=, flip
3655  // operands and condition code.
3656  bool FlipOperands = false;
3657  switch (CCCode) {
3658  case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
3659  case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
3660  case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
3661  case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
3662  default: break;
3663  }
3664  if (FlipOperands) {
3665  std::swap(LHSLo, RHSLo);
3666  std::swap(LHSHi, RHSHi);
3667  }
3668  // Perform a wide subtraction, feeding the carry from the low part into
3669  // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
3670  // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
3671  // zero or positive iff LHS >= RHS.
3672  EVT LoVT = LHSLo.getValueType();
3673  SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
3674  SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
3675  SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
3676  LHSHi, RHSHi, LowCmp.getValue(1),
3677  DAG.getCondCode(CCCode));
3678  NewLHS = Res;
3679  NewRHS = SDValue();
3680  return;
3681  }
3682 
3683  NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
3684  false, DagCombineInfo, dl);
3685  if (!NewLHS.getNode())
3686  NewLHS =
3687  DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
3688  NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
3689  NewRHS = SDValue();
3690 }
3691 
3692 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
3693  SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
3694  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
3695  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3696 
3697  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3698  // against zero to select between true and false values.
3699  if (!NewRHS.getNode()) {
3700  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3701  CCCode = ISD::SETNE;
3702  }
3703 
3704  // Update N to have the operands specified.
3705  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
3706  DAG.getCondCode(CCCode), NewLHS, NewRHS,
3707  N->getOperand(4)), 0);
3708 }
3709 
3710 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
3711  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3712  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
3713  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3714 
3715  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3716  // against zero to select between true and false values.
3717  if (!NewRHS.getNode()) {
3718  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3719  CCCode = ISD::SETNE;
3720  }
3721 
3722  // Update N to have the operands specified.
3723  return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
3724  N->getOperand(2), N->getOperand(3),
3725  DAG.getCondCode(CCCode)), 0);
3726 }
3727 
3728 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
3729  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3730  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3731  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3732 
3733  // If ExpandSetCCOperands returned a scalar, use it.
3734  if (!NewRHS.getNode()) {
3735  assert(NewLHS.getValueType() == N->getValueType(0) &&
3736  "Unexpected setcc expansion!");
3737  return NewLHS;
3738  }
3739 
3740  // Otherwise, update N to have the operands specified.
3741  return SDValue(
3742  DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
3743 }
3744 
3745 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
3746  SDValue LHS = N->getOperand(0);
3747  SDValue RHS = N->getOperand(1);
3748  SDValue Carry = N->getOperand(2);
3749  SDValue Cond = N->getOperand(3);
3750  SDLoc dl = SDLoc(N);
3751 
3752  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3753  GetExpandedInteger(LHS, LHSLo, LHSHi);
3754  GetExpandedInteger(RHS, RHSLo, RHSHi);
3755 
3756  // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
3757  SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
3758  SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
3759  return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
3760  LowCmp.getValue(1), Cond);
3761 }
3762 
3763 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
3764  // The value being shifted is legal, but the shift amount is too big.
3765  // It follows that either the result of the shift is undefined, or the
3766  // upper half of the shift amount is zero. Just use the lower half.
3767  SDValue Lo, Hi;
3768  GetExpandedInteger(N->getOperand(1), Lo, Hi);
3769  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
3770 }
3771 
3772 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
3773  // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
3774  // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
3775  // constant to valid type.
3776  SDValue Lo, Hi;
3777  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3778  return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
3779 }
3780 
3781 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
3782  SDValue Op = N->getOperand(0);
3783  EVT DstVT = N->getValueType(0);
3785  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3786  "Don't know how to expand this SINT_TO_FP!");
3788  CallOptions.setSExt(true);
3789  return TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N)).first;
3790 }
3791 
3792 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
3793  if (ISD::isNormalStore(N))
3794  return ExpandOp_NormalStore(N, OpNo);
3795 
3796  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
3797  assert(OpNo == 1 && "Can only expand the stored value so far");
3798 
3799  EVT VT = N->getOperand(1).getValueType();
3800  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3801  SDValue Ch = N->getChain();
3802  SDValue Ptr = N->getBasePtr();
3803  unsigned Alignment = N->getAlignment();
3804  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3805  AAMDNodes AAInfo = N->getAAInfo();
3806  SDLoc dl(N);
3807  SDValue Lo, Hi;
3808 
3809  assert(NVT.isByteSized() && "Expanded type not byte sized!");
3810 
3811  if (N->getMemoryVT().bitsLE(NVT)) {
3812  GetExpandedInteger(N->getValue(), Lo, Hi);
3813  return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
3814  N->getMemoryVT(), Alignment, MMOFlags, AAInfo);
3815  }
3816 
3817  if (DAG.getDataLayout().isLittleEndian()) {
3818  // Little-endian - low bits are at low addresses.
3819  GetExpandedInteger(N->getValue(), Lo, Hi);
3820 
3821  Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3822  AAInfo);
3823 
3824  unsigned ExcessBits =
3825  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3826  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3827 
3828  // Increment the pointer to the other half.
3829  unsigned IncrementSize = NVT.getSizeInBits()/8;
3830  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3831  Hi = DAG.getTruncStore(
3832  Ch, dl, Hi, Ptr, N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3833  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3834  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3835  }
3836 
3837  // Big-endian - high bits are at low addresses. Favor aligned stores at
3838  // the cost of some bit-fiddling.
3839  GetExpandedInteger(N->getValue(), Lo, Hi);
3840 
3841  EVT ExtVT = N->getMemoryVT();
3842  unsigned EBytes = ExtVT.getStoreSize();
3843  unsigned IncrementSize = NVT.getSizeInBits()/8;
3844  unsigned ExcessBits = (EBytes - IncrementSize)*8;
3845  EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
3846  ExtVT.getSizeInBits() - ExcessBits);
3847 
3848  if (ExcessBits < NVT.getSizeInBits()) {
3849  // Transfer high bits from the top of Lo to the bottom of Hi.
3850  Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
3851  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3852  TLI.getPointerTy(DAG.getDataLayout())));
3853  Hi = DAG.getNode(
3854  ISD::OR, dl, NVT, Hi,
3855  DAG.getNode(ISD::SRL, dl, NVT, Lo,
3856  DAG.getConstant(ExcessBits, dl,
3857  TLI.getPointerTy(DAG.getDataLayout()))));
3858  }
3859 
3860  // Store both the high bits and maybe some of the low bits.
3861  Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment,
3862  MMOFlags, AAInfo);
3863 
3864  // Increment the pointer to the other half.
3865  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3866  // Store the lowest ExcessBits bits in the second half.
3867  Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
3868  N->getPointerInfo().getWithOffset(IncrementSize),
3869  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3870  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3871  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3872 }
3873 
3874 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
3875  SDValue InL, InH;
3876  GetExpandedInteger(N->getOperand(0), InL, InH);
3877  // Just truncate the low part of the source.
3878  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
3879 }
3880 
3881 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
3882  SDValue Op = N->getOperand(0);
3883  EVT SrcVT = Op.getValueType();
3884  EVT DstVT = N->getValueType(0);
3885  SDLoc dl(N);
3886 
3887  // The following optimization is valid only if every value in SrcVT (when
3888  // treated as signed) is representable in DstVT. Check that the mantissa
3889  // size of DstVT is >= than the number of bits in SrcVT -1.
3890  const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT);
3891  if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 &&
3893  // Do a signed conversion then adjust the result.
3894  SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op);
3895  SignedConv = TLI.LowerOperation(SignedConv, DAG);
3896 
3897  // The result of the signed conversion needs adjusting if the 'sign bit' of
3898  // the incoming integer was set. To handle this, we dynamically test to see
3899  // if it is set, and, if so, add a fudge factor.
3900 
3901  const uint64_t F32TwoE32 = 0x4F800000ULL;
3902  const uint64_t F32TwoE64 = 0x5F800000ULL;
3903  const uint64_t F32TwoE128 = 0x7F800000ULL;
3904 
3905  APInt FF(32, 0);
3906  if (SrcVT == MVT::i32)
3907  FF = APInt(32, F32TwoE32);
3908  else if (SrcVT == MVT::i64)
3909  FF = APInt(32, F32TwoE64);
3910  else if (SrcVT == MVT::i128)
3911  FF = APInt(32, F32TwoE128);
3912  else
3913  llvm_unreachable("Unsupported UINT_TO_FP!");
3914 
3915  // Check whether the sign bit is set.
3916  SDValue Lo, Hi;
3917  GetExpandedInteger(Op, Lo, Hi);
3918  SDValue SignSet = DAG.getSetCC(dl,
3919  getSetCCResultType(Hi.getValueType()),
3920  Hi,
3921  DAG.getConstant(0, dl, Hi.getValueType()),
3922  ISD::SETLT);
3923 
3924  // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
3925  SDValue FudgePtr =
3926  DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)),
3927  TLI.getPointerTy(DAG.getDataLayout()));
3928 
3929  // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
3930  SDValue Zero = DAG.getIntPtrConstant(0, dl);
3931  SDValue Four = DAG.getIntPtrConstant(4, dl);
3932  if (DAG.getDataLayout().isBigEndian())
3933  std::swap(Zero, Four);
3934  SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet,
3935  Zero, Four);
3936  unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment();
3937  FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(),
3938  FudgePtr, Offset);
3939  Alignment = std::min(Alignment, 4u);
3940 
3941  // Load the value out, extending it from f32 to the destination float type.
3942  // FIXME: Avoid the extend by constructing the right constant pool?
3943  SDValue Fudge = DAG.getExtLoad(
3944  ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), FudgePtr,
3946  Alignment);
3947  return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge);
3948  }
3949 
3950  // Otherwise, use a libcall.
3951  RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT);
3952  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3953  "Don't know how to expand this UINT_TO_FP!");
3955  CallOptions.setSExt(true);
3956  return TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, dl).first;
3957 }
3958 
3959 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
3960  SDLoc dl(N);
3961  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
3962  cast<AtomicSDNode>(N)->getMemoryVT(),
3963  N->getOperand(0),
3964  N->getOperand(1), N->getOperand(2),
3965  cast<AtomicSDNode>(N)->getMemOperand());
3966  return Swap.getValue(1);
3967 }
3968 
3969 
3970 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
3971 
3972  EVT OutVT = N->getValueType(0);
3973  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3974  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3975  unsigned OutNumElems = OutVT.getVectorNumElements();
3976  EVT NOutVTElem = NOutVT.getVectorElementType();
3977 
3978  SDLoc dl(N);
3979  SDValue BaseIdx = N->getOperand(1);
3980 
3981  SDValue InOp0 = N->getOperand(0);
3982  if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger)
3983  InOp0 = GetPromotedInteger(N->getOperand(0));
3984 
3985  EVT InVT = InOp0.getValueType();
3986 
3988  Ops.reserve(OutNumElems);
3989  for (unsigned i = 0; i != OutNumElems; ++i) {
3990 
3991  // Extract the element from the original vector.
3992  SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
3993  BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
3995  InVT.getVectorElementType(), N->getOperand(0), Index);
3996 
3997  SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem);
3998  // Insert the converted element to the new vector.
3999  Ops.push_back(Op);
4000  }
4001 
4002  return DAG.getBuildVector(NOutVT, dl, Ops);
4003 }
4004 
4005 
4006 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
4007  ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
4008  EVT VT = N->getValueType(0);
4009  SDLoc dl(N);
4010 
4011  ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
4012 
4013  SDValue V0 = GetPromotedInteger(N->getOperand(0));
4014  SDValue V1 = GetPromotedInteger(N->getOperand(1));
4015  EVT OutVT = V0.getValueType();
4016 
4017  return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
4018 }
4019 
4020 
4021 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
4022  EVT OutVT = N->getValueType(0);
4023  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4024  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4025  unsigned NumElems = N->getNumOperands();
4026  EVT NOutVTElem = NOutVT.getVectorElementType();
4027 
4028  SDLoc dl(N);
4029 
4031  Ops.reserve(NumElems);
4032  for (unsigned i = 0; i != NumElems; ++i) {
4033  SDValue Op;
4034  // BUILD_VECTOR integer operand types are allowed to be larger than the
4035  // result's element type. This may still be true after the promotion. For
4036  // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
4037  // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
4038  if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
4039  Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
4040  else
4041  Op = N->getOperand(i);
4042  Ops.push_back(Op);
4043  }
4044 
4045  return DAG.getBuildVector(NOutVT, dl, Ops);
4046 }
4047 
4048 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
4049 
4050  SDLoc dl(N);
4051 
4052  assert(!N->getOperand(0).getValueType().isVector() &&
4053  "Input must be a scalar");
4054 
4055  EVT OutVT = N->getValueType(0);
4056  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4057  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4058  EVT NOutVTElem = NOutVT.getVectorElementType();
4059 
4060  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
4061 
4062  return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
4063 }
4064 
4065 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
4066  SDLoc dl(N);
4067 
4068  EVT OutVT = N->getValueType(0);
4069  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4070  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4071 
4072  EVT OutElemTy = NOutVT.getVectorElementType();
4073 
4074  unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
4075  unsigned NumOutElem = NOutVT.getVectorNumElements();
4076  unsigned NumOperands = N->getNumOperands();
4077  assert(NumElem * NumOperands == NumOutElem &&
4078  "Unexpected number of elements");
4079 
4080  // Take the elements from the first vector.
4081  SmallVector<SDValue, 8> Ops(NumOutElem);
4082  for (unsigned i = 0; i < NumOperands; ++i) {
4083  SDValue Op = N->getOperand(i);
4084  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
4085  Op = GetPromotedInteger(Op);
4086  EVT SclrTy = Op.getValueType().getVectorElementType();
4087  assert(NumElem == Op.getValueType().getVectorNumElements() &&
4088  "Unexpected number of elements");
4089 
4090  for (unsigned j = 0; j < NumElem; ++j) {
4091  SDValue Ext = DAG.getNode(
4092  ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
4093  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4094  Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
4095  }
4096  }
4097 
4098  return DAG.getBuildVector(NOutVT, dl, Ops);
4099 }
4100 
4101 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
4102  EVT VT = N->getValueType(0);
4103  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4104  assert(NVT.isVector() && "This type must be promoted to a vector type");
4105 
4106  SDLoc dl(N);
4107 
4108  // For operands whose TypeAction is to promote, extend the promoted node
4109  // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
4110  // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
4111  // type..
4112  if (getTypeAction(N->getOperand(0).getValueType())
4114  SDValue Promoted;
4115 
4116  switch(N->getOpcode()) {
4118  Promoted = SExtPromotedInteger(N->getOperand(0));
4119  break;
4121  Promoted = ZExtPromotedInteger(N->getOperand(0));
4122  break;
4124  Promoted = GetPromotedInteger(N->getOperand(0));
4125  break;
4126  default:
4127  llvm_unreachable("Node has unexpected Opcode");
4128  }
4129  return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
4130  }
4131 
4132  // Directly extend to the appropriate transform-to type.
4133  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
4134 }
4135 
4136 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
4137  EVT OutVT = N->getValueType(0);
4138  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4139  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4140 
4141  EVT NOutVTElem = NOutVT.getVectorElementType();
4142 
4143  SDLoc dl(N);
4144  SDValue V0 = GetPromotedInteger(N->getOperand(0));
4145 
4146  SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
4147  NOutVTElem, N->getOperand(1));
4148  return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
4149  V0, ConvElem, N->getOperand(2));
4150 }
4151 
4152 SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) {
4153  // The VECREDUCE result size may be larger than the element size, so
4154  // we can simply change the result type.
4155  SDLoc dl(N);
4156  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4157  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
4158 }
4159 
4160 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4161  SDLoc dl(N);
4162  SDValue V0 = GetPromotedInteger(N->getOperand(0));
4163  SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
4164  TLI.getVectorIdxTy(DAG.getDataLayout()));
4166  V0->getValueType(0).getScalarType(), V0, V1);
4167 
4168  // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
4169  // element types. If this is the case then we need to expand the outgoing
4170  // value and not truncate it.
4171  return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
4172 }
4173 
4174 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
4175  SDLoc dl(N);
4176  SDValue V0 = GetPromotedInteger(N->getOperand(0));
4177  MVT InVT = V0.getValueType().getSimpleVT();
4178  MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
4180  SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
4181  return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
4182 }
4183 
4184 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
4185  SDLoc dl(N);
4186  unsigned NumElems = N->getNumOperands();
4187 
4188  EVT RetSclrTy = N->getValueType(0).getVectorElementType();
4189 
4190  SmallVector<SDValue, 8> NewOps;
4191  NewOps.reserve(NumElems);
4192 
4193  // For each incoming vector
4194  for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
4195  SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
4196  EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
4197  unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
4198 
4199  for (unsigned i=0; i<NumElem; ++i) {
4200  // Extract element from incoming vector
4201  SDValue Ex = DAG.getNode(
4202  ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
4203  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4204  SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
4205  NewOps.push_back(Tr);
4206  }
4207  }
4208 
4209  return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
4210 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:595
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:551
static EVT getShiftAmountTyForConstant(unsigned Val, EVT VT, const TargetLowering &TLI, SelectionDAG &DAG)
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getValueType() const
Return the ValueType of the referenced return value.
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant, which is required to be operand #1) half of the integer or float value specified as operand #0.
Definition: ISDOpcodes.h:183
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
Definition: APInt.h:561
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC&#39;s if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:959
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:382
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:678
This class represents lattice values for constants.
Definition: AllocatorList.h:23
static MVT getVectorVT(MVT VT, unsigned NumElements)
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:391
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:259
This structure is used to pass arguments to makeLibCall function.
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:284
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:543
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none...
const SDValue & getBasePtr() const
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:222
void push_back(const T &Elt)
Definition: SmallVector.h:211
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:860
const SDValue & getValue() const
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
Definition: APInt.h:647
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:252
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:250
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1328
const SDValue & getBasePtr() const
const SDValue & getChain() const
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
unsigned getAlignment() const
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:833
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select&#39;s if you just have operands and don&#39;t want to check...
Definition: SelectionDAG.h:972
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:532
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:408
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align=0, int Offs=0, bool isT=false, unsigned TargetFlags=0)
Same for subtraction.
Definition: ISDOpcodes.h:253
void reserve(size_type N)
Definition: SmallVector.h:369
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:211
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:459
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:38
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition: APInt.h:534
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
SDValue getExternalSymbol(const char *Sym, EVT VT)
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
virtual bool shouldExpandShift(SelectionDAG &DAG, SDNode *N) const
Return true if SHIFT instructions should be expanded to SHIFT_PARTS instructions, and false if a libr...
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:209
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations...
Definition: ISDOpcodes.h:480
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
static uint32_t getAlignment(const MCSectionCOFF &Sec)
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:450
Shift and rotation operations.
Definition: ISDOpcodes.h:434
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:261
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth...
Definition: ISDOpcodes.h:417
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:654
CallLoweringInfo & setChain(SDValue InChain)
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:190
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:279
const SDValue & getPassThru() const
op_iterator op_end() const
unsigned getScalarValueSizeInBits() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
FLT_ROUNDS_ - Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest 2 Round to ...
Definition: ISDOpcodes.h:570
SimpleValueType SimpleTy
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:303
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt) For double-word atomic operations: ValLo, ValHi, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amtLo, amtHi) ValLo, ValHi, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amtLo, amtHi) These correspond to the atomicrmw instruction.
Definition: ISDOpcodes.h:841
const SDValue & getValue() const
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:473
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:413
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG...
Definition: ISDOpcodes.h:72
This is an SDNode representing atomic operations.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
This represents a list of ValueType&#39;s that has been intern&#39;d by a SelectionDAG.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
This class is used to represent an MSTORE node.
unsigned getSizeInBits() const
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:273
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:297
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:291
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:410
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:502
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
const SDValue & getScale() const
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:453
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:233
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:874
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTruncating=false, bool IsCompressing=false)
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:1012
op_iterator op_begin() const
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:778
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:521
const SDValue & getMask() const
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl) const
Returns a pair of (return value, chain).
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands...
UNDEF - An undefined node.
Definition: ISDOpcodes.h:177
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:548
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:351
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
Definition: APInt.h:635
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits...
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
Definition: MathExtras.h:614
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const SDValue & getBasePtr() const
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:234
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:428
Machine Value Type.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
Simple binary floating point operators.
Definition: ISDOpcodes.h:287
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:272
This is an important base class in LLVM.
Definition: Constant.h:41
iterator_range< value_op_iterator > op_values() const
const SDValue & getOperand(unsigned Num) const
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:987
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:356
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:246
Carry-using nodes for multiple precision addition and subtraction.
Definition: