LLVM  9.0.0svn
LegalizeIntegerTypes.cpp
Go to the documentation of this file.
1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements integer type expansion and promotion for LegalizeTypes.
10 // Promotion is the act of changing a computation in an illegal type into a
11 // computation in a larger type. For example, implementing i8 arithmetic in an
12 // i32 register (often needed on powerpc).
13 // Expansion is the act of changing a computation in an illegal type into a
14 // computation in two identical registers of a smaller type. For example,
15 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
16 // targets).
17 //
18 //===----------------------------------------------------------------------===//
19 
20 #include "LegalizeTypes.h"
21 #include "llvm/IR/DerivedTypes.h"
23 #include "llvm/Support/KnownBits.h"
25 using namespace llvm;
26 
27 #define DEBUG_TYPE "legalize-types"
28 
29 //===----------------------------------------------------------------------===//
30 // Integer Result Promotion
31 //===----------------------------------------------------------------------===//
32 
33 /// PromoteIntegerResult - This method is called when a result of a node is
34 /// found to be in need of promotion to a larger type. At this point, the node
35 /// may also have invalid operands or may have other results that need
36 /// expansion, we just know that (at least) one result needs promotion.
37 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
38  LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG);
39  dbgs() << "\n");
40  SDValue Res = SDValue();
41 
42  // See if the target wants to custom expand this node.
43  if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
44  LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
45  return;
46  }
47 
48  switch (N->getOpcode()) {
49  default:
50 #ifndef NDEBUG
51  dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
52  N->dump(&DAG); dbgs() << "\n";
53 #endif
54  llvm_unreachable("Do not know how to promote this operator!");
55  case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
56  case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
57  case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
58  case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
59  case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
60  case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
61  case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
62  case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
64  case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
65  case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break;
67  case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
69  Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
70  case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
71  case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
72  break;
73  case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
74  break;
75  case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
76  case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
77  case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
78  case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
79  case ISD::SMIN:
80  case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
81  case ISD::UMIN:
82  case ISD::UMAX: Res = PromoteIntRes_ZExtIntBinOp(N); break;
83 
84  case ISD::SHL: Res = PromoteIntRes_SHL(N); break;
86  Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
87  case ISD::SRA: Res = PromoteIntRes_SRA(N); break;
88  case ISD::SRL: Res = PromoteIntRes_SRL(N); break;
89  case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
90  case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
91  case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
92 
94  Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
96  Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
98  Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
99  case ISD::BUILD_VECTOR:
100  Res = PromoteIntRes_BUILD_VECTOR(N); break;
102  Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
103  case ISD::CONCAT_VECTORS:
104  Res = PromoteIntRes_CONCAT_VECTORS(N); break;
105 
109  Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
110 
111  case ISD::SIGN_EXTEND:
112  case ISD::ZERO_EXTEND:
113  case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
114 
115  case ISD::FP_TO_SINT:
116  case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
117 
118  case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
119 
120  case ISD::FLT_ROUNDS_: Res = PromoteIntRes_FLT_ROUNDS(N); break;
121 
122  case ISD::AND:
123  case ISD::OR:
124  case ISD::XOR:
125  case ISD::ADD:
126  case ISD::SUB:
127  case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
128 
129  case ISD::SDIV:
130  case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
131 
132  case ISD::UDIV:
133  case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
134 
135  case ISD::SADDO:
136  case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
137  case ISD::UADDO:
138  case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
139  case ISD::SMULO:
140  case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
141 
142  case ISD::ADDE:
143  case ISD::SUBE:
144  case ISD::ADDCARRY:
145  case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
146 
147  case ISD::SADDSAT:
148  case ISD::UADDSAT:
149  case ISD::SSUBSAT:
150  case ISD::USUBSAT: Res = PromoteIntRes_ADDSUBSAT(N); break;
151  case ISD::SMULFIX:
152  case ISD::SMULFIXSAT:
153  case ISD::UMULFIX: Res = PromoteIntRes_MULFIX(N); break;
154  case ISD::ABS: Res = PromoteIntRes_ABS(N); break;
155 
156  case ISD::ATOMIC_LOAD:
157  Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
158 
163  case ISD::ATOMIC_LOAD_OR:
170  case ISD::ATOMIC_SWAP:
171  Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
172 
175  Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
176  break;
177 
178  case ISD::VECREDUCE_ADD:
179  case ISD::VECREDUCE_MUL:
180  case ISD::VECREDUCE_AND:
181  case ISD::VECREDUCE_OR:
182  case ISD::VECREDUCE_XOR:
183  case ISD::VECREDUCE_SMAX:
184  case ISD::VECREDUCE_SMIN:
185  case ISD::VECREDUCE_UMAX:
186  case ISD::VECREDUCE_UMIN:
187  Res = PromoteIntRes_VECREDUCE(N);
188  break;
189  }
190 
191  // If the result is null then the sub-method took care of registering it.
192  if (Res.getNode())
193  SetPromotedInteger(SDValue(N, ResNo), Res);
194 }
195 
196 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
197  unsigned ResNo) {
198  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
199  return GetPromotedInteger(Op);
200 }
201 
202 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
203  // Sign-extend the new bits, and continue the assertion.
204  SDValue Op = SExtPromotedInteger(N->getOperand(0));
205  return DAG.getNode(ISD::AssertSext, SDLoc(N),
206  Op.getValueType(), Op, N->getOperand(1));
207 }
208 
209 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
210  // Zero the new bits, and continue the assertion.
211  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
212  return DAG.getNode(ISD::AssertZext, SDLoc(N),
213  Op.getValueType(), Op, N->getOperand(1));
214 }
215 
216 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
217  EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
218  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
219  N->getMemoryVT(), ResVT,
220  N->getChain(), N->getBasePtr(),
221  N->getMemOperand());
222  // Legalize the chain result - switch anything that used the old chain to
223  // use the new one.
224  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
225  return Res;
226 }
227 
228 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
229  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
230  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
231  N->getMemoryVT(),
232  N->getChain(), N->getBasePtr(),
233  Op2, N->getMemOperand());
234  // Legalize the chain result - switch anything that used the old chain to
235  // use the new one.
236  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
237  return Res;
238 }
239 
240 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
241  unsigned ResNo) {
242  if (ResNo == 1) {
244  EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
245  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
246 
247  // Only use the result of getSetCCResultType if it is legal,
248  // otherwise just use the promoted result type (NVT).
249  if (!TLI.isTypeLegal(SVT))
250  SVT = NVT;
251 
252  SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
253  SDValue Res = DAG.getAtomicCmpSwap(
255  N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
256  N->getMemOperand());
257  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
258  ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
259  return Res.getValue(1);
260  }
261 
262  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
263  SDValue Op3 = GetPromotedInteger(N->getOperand(3));
264  SDVTList VTs =
265  DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
266  SDValue Res = DAG.getAtomicCmpSwap(
267  N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
268  N->getBasePtr(), Op2, Op3, N->getMemOperand());
269  // Update the use to N with the newly created Res.
270  for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
271  ReplaceValueWith(SDValue(N, i), Res.getValue(i));
272  return Res;
273 }
274 
275 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
276  SDValue InOp = N->getOperand(0);
277  EVT InVT = InOp.getValueType();
278  EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
279  EVT OutVT = N->getValueType(0);
280  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
281  SDLoc dl(N);
282 
283  switch (getTypeAction(InVT)) {
285  break;
287  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
288  // The input promotes to the same size. Convert the promoted value.
289  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
290  break;
292  // Promote the integer operand by hand.
293  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
295  // Convert the promoted float by hand.
296  if (!NOutVT.isVector())
297  return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
298  break;
299  }
302  break;
304  // Convert the element to an integer and promote it by hand.
305  if (!NOutVT.isVector())
306  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
307  BitConvertToInteger(GetScalarizedVector(InOp)));
308  break;
310  if (!NOutVT.isVector()) {
311  // For example, i32 = BITCAST v2i16 on alpha. Convert the split
312  // pieces of the input into integers and reassemble in the final type.
313  SDValue Lo, Hi;
314  GetSplitVector(N->getOperand(0), Lo, Hi);
315  Lo = BitConvertToInteger(Lo);
316  Hi = BitConvertToInteger(Hi);
317 
318  if (DAG.getDataLayout().isBigEndian())
319  std::swap(Lo, Hi);
320 
321  InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
323  NOutVT.getSizeInBits()),
324  JoinIntegers(Lo, Hi));
325  return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
326  }
327  break;
328  }
330  // The input is widened to the same size. Convert to the widened value.
331  // Make sure that the outgoing value is not a vector, because this would
332  // make us bitcast between two vectors which are legalized in different ways.
333  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector())
334  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
335  // If the output type is also a vector and widening it to the same size
336  // as the widened input type would be a legal type, we can widen the bitcast
337  // and handle the promotion after.
338  if (NOutVT.isVector()) {
339  unsigned WidenInSize = NInVT.getSizeInBits();
340  unsigned OutSize = OutVT.getSizeInBits();
341  if (WidenInSize % OutSize == 0) {
342  unsigned Scale = WidenInSize / OutSize;
343  EVT WideOutVT = EVT::getVectorVT(*DAG.getContext(),
344  OutVT.getVectorElementType(),
345  OutVT.getVectorNumElements() * Scale);
346  if (isTypeLegal(WideOutVT)) {
347  InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
348  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
349  InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
350  DAG.getConstant(0, dl, IdxTy));
351  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
352  }
353  }
354  }
355  }
356 
357  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
358  CreateStackStoreLoad(InOp, OutVT));
359 }
360 
361 // Helper for BSWAP/BITREVERSE promotion to ensure we can fit the shift amount
362 // in the VT returned by getShiftAmountTy and to return a safe VT if we can't.
363 static EVT getShiftAmountTyForConstant(unsigned Val, EVT VT,
364  const TargetLowering &TLI,
365  SelectionDAG &DAG) {
366  EVT ShiftVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
367  // If the value won't fit in the prefered type, just use something safe. It
368  // will be legalized when the shift is expanded.
369  if ((Log2_32(Val) + 1) > ShiftVT.getScalarSizeInBits())
370  ShiftVT = MVT::i32;
371  return ShiftVT;
372 }
373 
374 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
375  SDValue Op = GetPromotedInteger(N->getOperand(0));
376  EVT OVT = N->getValueType(0);
377  EVT NVT = Op.getValueType();
378  SDLoc dl(N);
379 
380  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
381  EVT ShiftVT = getShiftAmountTyForConstant(DiffBits, NVT, TLI, DAG);
382  return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
383  DAG.getConstant(DiffBits, dl, ShiftVT));
384 }
385 
386 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
387  SDValue Op = GetPromotedInteger(N->getOperand(0));
388  EVT OVT = N->getValueType(0);
389  EVT NVT = Op.getValueType();
390  SDLoc dl(N);
391 
392  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
393  EVT ShiftVT = getShiftAmountTyForConstant(DiffBits, NVT, TLI, DAG);
394  return DAG.getNode(ISD::SRL, dl, NVT,
395  DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
396  DAG.getConstant(DiffBits, dl, ShiftVT));
397 }
398 
399 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
400  // The pair element type may be legal, or may not promote to the same type as
401  // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
402  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
403  TLI.getTypeToTransformTo(*DAG.getContext(),
404  N->getValueType(0)), JoinIntegers(N->getOperand(0),
405  N->getOperand(1)));
406 }
407 
408 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
409  EVT VT = N->getValueType(0);
410  // FIXME there is no actual debug info here
411  SDLoc dl(N);
412  // Zero extend things like i1, sign extend everything else. It shouldn't
413  // matter in theory which one we pick, but this tends to give better code?
414  unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
415  SDValue Result = DAG.getNode(Opc, dl,
416  TLI.getTypeToTransformTo(*DAG.getContext(), VT),
417  SDValue(N, 0));
418  assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
419  return Result;
420 }
421 
422 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
423  // Zero extend to the promoted type and do the count there.
424  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
425  SDLoc dl(N);
426  EVT OVT = N->getValueType(0);
427  EVT NVT = Op.getValueType();
428  Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
429  // Subtract off the extra leading bits in the bigger type.
430  return DAG.getNode(
431  ISD::SUB, dl, NVT, Op,
432  DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
433  NVT));
434 }
435 
436 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
437  // Zero extend to the promoted type and do the count there.
438  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
439  return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op);
440 }
441 
442 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
443  SDValue Op = GetPromotedInteger(N->getOperand(0));
444  EVT OVT = N->getValueType(0);
445  EVT NVT = Op.getValueType();
446  SDLoc dl(N);
447  if (N->getOpcode() == ISD::CTTZ) {
448  // The count is the same in the promoted type except if the original
449  // value was zero. This can be handled by setting the bit just off
450  // the top of the original type.
451  auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
452  OVT.getScalarSizeInBits());
453  Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
454  }
455  return DAG.getNode(N->getOpcode(), dl, NVT, Op);
456 }
457 
458 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
459  SDLoc dl(N);
460  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
461 
462  SDValue Op0 = N->getOperand(0);
463  SDValue Op1 = N->getOperand(1);
464 
465  // If the input also needs to be promoted, do that first so we can get a
466  // get a good idea for the output type.
467  if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
469  SDValue In = GetPromotedInteger(Op0);
470 
471  // If the new type is larger than NVT, use it. We probably won't need to
472  // promote it again.
473  EVT SVT = In.getValueType().getScalarType();
474  if (SVT.bitsGE(NVT)) {
475  SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
476  return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
477  }
478  }
479 
480  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
481 }
482 
483 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
484  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
485  unsigned NewOpc = N->getOpcode();
486  SDLoc dl(N);
487 
488  // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
489  // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
490  // and SINT conversions are Custom, there is no way to tell which is
491  // preferable. We choose SINT because that's the right thing on PPC.)
492  if (N->getOpcode() == ISD::FP_TO_UINT &&
493  !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
495  NewOpc = ISD::FP_TO_SINT;
496 
497  SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
498 
499  // Assert that the converted value fits in the original type. If it doesn't
500  // (eg: because the value being converted is too big), then the result of the
501  // original operation was undefined anyway, so the assert is still correct.
502  //
503  // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
504  // before legalization: fp-to-uint16, 65534. -> 0xfffe
505  // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
506  return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ?
507  ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
509 }
510 
511 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
512  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
513  SDLoc dl(N);
514 
515  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
516 }
517 
518 SDValue DAGTypeLegalizer::PromoteIntRes_FLT_ROUNDS(SDNode *N) {
519  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
520  SDLoc dl(N);
521 
522  return DAG.getNode(N->getOpcode(), dl, NVT);
523 }
524 
525 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
526  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
527  SDLoc dl(N);
528 
529  if (getTypeAction(N->getOperand(0).getValueType())
531  SDValue Res = GetPromotedInteger(N->getOperand(0));
532  assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
533 
534  // If the result and operand types are the same after promotion, simplify
535  // to an in-register extension.
536  if (NVT == Res.getValueType()) {
537  // The high bits are not guaranteed to be anything. Insert an extend.
538  if (N->getOpcode() == ISD::SIGN_EXTEND)
539  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
540  DAG.getValueType(N->getOperand(0).getValueType()));
541  if (N->getOpcode() == ISD::ZERO_EXTEND)
542  return DAG.getZeroExtendInReg(Res, dl,
544  assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
545  return Res;
546  }
547  }
548 
549  // Otherwise, just extend the original operand all the way to the larger type.
550  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
551 }
552 
553 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
554  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
555  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
558  SDLoc dl(N);
559  SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
560  N->getMemoryVT(), N->getMemOperand());
561 
562  // Legalize the chain result - switch anything that used the old chain to
563  // use the new one.
564  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
565  return Res;
566 }
567 
568 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
569  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
570  SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
571 
572  SDLoc dl(N);
573  SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
574  N->getMask(), ExtPassThru, N->getMemoryVT(),
576  // Legalize the chain result - switch anything that used the old chain to
577  // use the new one.
578  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
579  return Res;
580 }
581 
582 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
583  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
584  SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
585  assert(NVT == ExtPassThru.getValueType() &&
586  "Gather result type and the passThru agrument type should be the same");
587 
588  SDLoc dl(N);
589  SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
590  N->getIndex(), N->getScale() };
591  SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
592  N->getMemoryVT(), dl, Ops,
593  N->getMemOperand());
594  // Legalize the chain result - switch anything that used the old chain to
595  // use the new one.
596  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
597  return Res;
598 }
599 
600 /// Promote the overflow flag of an overflowing arithmetic node.
601 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
602  // Change the return type of the boolean result while obeying
603  // getSetCCResultType.
604  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
605  EVT VT = N->getValueType(0);
606  EVT SVT = getSetCCResultType(VT);
607  SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
608  unsigned NumOps = N->getNumOperands();
609  assert(NumOps <= 3 && "Too many operands");
610  if (NumOps == 3)
611  Ops[2] = N->getOperand(2);
612 
613  SDLoc dl(N);
614  SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
615  makeArrayRef(Ops, NumOps));
616 
617  // Modified the sum result - switch anything that used the old sum to use
618  // the new one.
619  ReplaceValueWith(SDValue(N, 0), Res);
620 
621  // Convert to the expected type.
622  return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT);
623 }
624 
625 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSAT(SDNode *N) {
626  // For promoting iN -> iM, this can be expanded by
627  // 1. ANY_EXTEND iN to iM
628  // 2. SHL by M-N
629  // 3. [US][ADD|SUB]SAT
630  // 4. L/ASHR by M-N
631  SDLoc dl(N);
632  SDValue Op1 = N->getOperand(0);
633  SDValue Op2 = N->getOperand(1);
634  unsigned OldBits = Op1.getScalarValueSizeInBits();
635 
636  unsigned Opcode = N->getOpcode();
637  unsigned ShiftOp;
638  switch (Opcode) {
639  case ISD::SADDSAT:
640  case ISD::SSUBSAT:
641  ShiftOp = ISD::SRA;
642  break;
643  case ISD::UADDSAT:
644  case ISD::USUBSAT:
645  ShiftOp = ISD::SRL;
646  break;
647  default:
648  llvm_unreachable("Expected opcode to be signed or unsigned saturation "
649  "addition or subtraction");
650  }
651 
652  SDValue Op1Promoted = GetPromotedInteger(Op1);
653  SDValue Op2Promoted = GetPromotedInteger(Op2);
654 
655  EVT PromotedType = Op1Promoted.getValueType();
656  unsigned NewBits = PromotedType.getScalarSizeInBits();
657  unsigned SHLAmount = NewBits - OldBits;
658  EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
659  SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT);
660  Op1Promoted =
661  DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
662  Op2Promoted =
663  DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
664 
665  SDValue Result =
666  DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
667  return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
668 }
669 
670 SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
671  // Can just promote the operands then continue with operation.
672  SDLoc dl(N);
673  SDValue Op1Promoted, Op2Promoted;
674  bool Signed =
675  N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT;
676  if (Signed) {
677  Op1Promoted = SExtPromotedInteger(N->getOperand(0));
678  Op2Promoted = SExtPromotedInteger(N->getOperand(1));
679  } else {
680  Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
681  Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
682  }
683  EVT OldType = N->getOperand(0).getValueType();
684  EVT PromotedType = Op1Promoted.getValueType();
685  unsigned DiffSize =
686  PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits();
687 
688  bool Saturating = N->getOpcode() == ISD::SMULFIXSAT;
689  if (Saturating) {
690  // Promoting the operand and result values changes the saturation width,
691  // which is extends the values that we clamp to on saturation. This could be
692  // resolved by shifting one of the operands the same amount, which would
693  // also shift the result we compare against, then shifting back.
694  EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
695  Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
696  DAG.getConstant(DiffSize, dl, ShiftTy));
697  SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
698  Op2Promoted, N->getOperand(2));
699  unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL;
700  return DAG.getNode(ShiftOp, dl, PromotedType, Result,
701  DAG.getConstant(DiffSize, dl, ShiftTy));
702  }
703  return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
704  N->getOperand(2));
705 }
706 
707 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
708  if (ResNo == 1)
709  return PromoteIntRes_Overflow(N);
710 
711  // The operation overflowed iff the result in the larger type is not the
712  // sign extension of its truncation to the original type.
713  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
714  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
715  EVT OVT = N->getOperand(0).getValueType();
716  EVT NVT = LHS.getValueType();
717  SDLoc dl(N);
718 
719  // Do the arithmetic in the larger type.
720  unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
721  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
722 
723  // Calculate the overflow flag: sign extend the arithmetic result from
724  // the original type.
725  SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
726  DAG.getValueType(OVT));
727  // Overflowed if and only if this is not equal to Res.
728  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
729 
730  // Use the calculated overflow everywhere.
731  ReplaceValueWith(SDValue(N, 1), Ofl);
732 
733  return Res;
734 }
735 
736 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
737  SDValue LHS = GetPromotedInteger(N->getOperand(1));
738  SDValue RHS = GetPromotedInteger(N->getOperand(2));
739  return DAG.getSelect(SDLoc(N),
740  LHS.getValueType(), N->getOperand(0), LHS, RHS);
741 }
742 
743 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
744  SDValue Mask = N->getOperand(0);
745 
746  SDValue LHS = GetPromotedInteger(N->getOperand(1));
747  SDValue RHS = GetPromotedInteger(N->getOperand(2));
748  return DAG.getNode(ISD::VSELECT, SDLoc(N),
749  LHS.getValueType(), Mask, LHS, RHS);
750 }
751 
752 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
753  SDValue LHS = GetPromotedInteger(N->getOperand(2));
754  SDValue RHS = GetPromotedInteger(N->getOperand(3));
755  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
756  LHS.getValueType(), N->getOperand(0),
757  N->getOperand(1), LHS, RHS, N->getOperand(4));
758 }
759 
760 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
761  EVT InVT = N->getOperand(0).getValueType();
762  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
763 
764  EVT SVT = getSetCCResultType(InVT);
765 
766  // If we got back a type that needs to be promoted, this likely means the
767  // the input type also needs to be promoted. So get the promoted type for
768  // the input and try the query again.
769  if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
770  if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
771  InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
772  SVT = getSetCCResultType(InVT);
773  } else {
774  // Input type isn't promoted, just use the default promoted type.
775  SVT = NVT;
776  }
777  }
778 
779  SDLoc dl(N);
780  assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() &&
781  "Vector compare must return a vector result!");
782 
783  // Get the SETCC result using the canonical SETCC type.
784  SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
785  N->getOperand(1), N->getOperand(2));
786 
787  // Convert to the expected type.
788  return DAG.getSExtOrTrunc(SetCC, dl, NVT);
789 }
790 
791 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
792  SDValue LHS = GetPromotedInteger(N->getOperand(0));
793  SDValue RHS = N->getOperand(1);
794  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
795  RHS = ZExtPromotedInteger(RHS);
796  return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
797 }
798 
799 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
800  SDValue Op = GetPromotedInteger(N->getOperand(0));
801  return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
802  Op.getValueType(), Op, N->getOperand(1));
803 }
804 
805 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
806  // The input may have strange things in the top bits of the registers, but
807  // these operations don't care. They may have weird bits going out, but
808  // that too is okay if they are integer operations.
809  SDValue LHS = GetPromotedInteger(N->getOperand(0));
810  SDValue RHS = GetPromotedInteger(N->getOperand(1));
811  return DAG.getNode(N->getOpcode(), SDLoc(N),
812  LHS.getValueType(), LHS, RHS);
813 }
814 
815 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
816  // Sign extend the input.
817  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
818  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
819  return DAG.getNode(N->getOpcode(), SDLoc(N),
820  LHS.getValueType(), LHS, RHS);
821 }
822 
823 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
824  // Zero extend the input.
825  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
826  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
827  return DAG.getNode(N->getOpcode(), SDLoc(N),
828  LHS.getValueType(), LHS, RHS);
829 }
830 
831 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
832  // The input value must be properly sign extended.
833  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
834  SDValue RHS = N->getOperand(1);
835  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
836  RHS = ZExtPromotedInteger(RHS);
837  return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
838 }
839 
840 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
841  // The input value must be properly zero extended.
842  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
843  SDValue RHS = N->getOperand(1);
844  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
845  RHS = ZExtPromotedInteger(RHS);
846  return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
847 }
848 
849 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
850  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
851  SDValue Res;
852  SDValue InOp = N->getOperand(0);
853  SDLoc dl(N);
854 
855  switch (getTypeAction(InOp.getValueType())) {
856  default: llvm_unreachable("Unknown type action!");
859  Res = InOp;
860  break;
862  Res = GetPromotedInteger(InOp);
863  break;
865  EVT InVT = InOp.getValueType();
866  assert(InVT.isVector() && "Cannot split scalar types");
867  unsigned NumElts = InVT.getVectorNumElements();
868  assert(NumElts == NVT.getVectorNumElements() &&
869  "Dst and Src must have the same number of elements");
870  assert(isPowerOf2_32(NumElts) &&
871  "Promoted vector type must be a power of two");
872 
873  SDValue EOp1, EOp2;
874  GetSplitVector(InOp, EOp1, EOp2);
875 
876  EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
877  NumElts/2);
878  EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
879  EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
880 
881  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
882  }
884  SDValue WideInOp = GetWidenedVector(InOp);
885 
886  // Truncate widened InOp.
887  unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
888  EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
889  N->getValueType(0).getScalarType(), NumElem);
890  SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
891 
892  // Zero extend so that the elements are of same type as those of NVT
893  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
894  NumElem);
895  SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
896 
897  // Extract the low NVT subvector.
898  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
899  SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy);
900  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
901  }
902  }
903 
904  // Truncate to NVT instead of VT
905  return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
906 }
907 
908 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
909  if (ResNo == 1)
910  return PromoteIntRes_Overflow(N);
911 
912  // The operation overflowed iff the result in the larger type is not the
913  // zero extension of its truncation to the original type.
914  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
915  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
916  EVT OVT = N->getOperand(0).getValueType();
917  EVT NVT = LHS.getValueType();
918  SDLoc dl(N);
919 
920  // Do the arithmetic in the larger type.
921  unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
922  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
923 
924  // Calculate the overflow flag: zero extend the arithmetic result from
925  // the original type.
926  SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT.getScalarType());
927  // Overflowed if and only if this is not equal to Res.
928  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
929 
930  // Use the calculated overflow everywhere.
931  ReplaceValueWith(SDValue(N, 1), Ofl);
932 
933  return Res;
934 }
935 
936 // Handle promotion for the ADDE/SUBE/ADDCARRY/SUBCARRY nodes. Notice that
937 // the third operand of ADDE/SUBE nodes is carry flag, which differs from
938 // the ADDCARRY/SUBCARRY nodes in that the third operand is carry Boolean.
939 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
940  if (ResNo == 1)
941  return PromoteIntRes_Overflow(N);
942 
943  // We need to sign-extend the operands so the carry value computed by the
944  // wide operation will be equivalent to the carry value computed by the
945  // narrow operation.
946  // An ADDCARRY can generate carry only if any of the operands has its
947  // most significant bit set. Sign extension propagates the most significant
948  // bit into the higher bits which means the extra bit that the narrow
949  // addition would need (i.e. the carry) will be propagated through the higher
950  // bits of the wide addition.
951  // A SUBCARRY can generate borrow only if LHS < RHS and this property will be
952  // preserved by sign extension.
953  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
954  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
955 
956  EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
957 
958  // Do the arithmetic in the wide type.
959  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
960  LHS, RHS, N->getOperand(2));
961 
962  // Update the users of the original carry/borrow value.
963  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
964 
965  return SDValue(Res.getNode(), 0);
966 }
967 
968 SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
969  SDValue Op0 = SExtPromotedInteger(N->getOperand(0));
970  return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0);
971 }
972 
973 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
974  // Promote the overflow bit trivially.
975  if (ResNo == 1)
976  return PromoteIntRes_Overflow(N);
977 
978  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
979  SDLoc DL(N);
980  EVT SmallVT = LHS.getValueType();
981 
982  // To determine if the result overflowed in a larger type, we extend the
983  // input to the larger type, do the multiply (checking if it overflows),
984  // then also check the high bits of the result to see if overflow happened
985  // there.
986  if (N->getOpcode() == ISD::SMULO) {
987  LHS = SExtPromotedInteger(LHS);
988  RHS = SExtPromotedInteger(RHS);
989  } else {
990  LHS = ZExtPromotedInteger(LHS);
991  RHS = ZExtPromotedInteger(RHS);
992  }
993  SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
994  SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
995 
996  // Overflow occurred if it occurred in the larger type, or if the high part
997  // of the result does not zero/sign-extend the low part. Check this second
998  // possibility first.
999  SDValue Overflow;
1000  if (N->getOpcode() == ISD::UMULO) {
1001  // Unsigned overflow occurred if the high part is non-zero.
1002  unsigned Shift = SmallVT.getScalarSizeInBits();
1003  EVT ShiftTy = getShiftAmountTyForConstant(Shift, Mul.getValueType(),
1004  TLI, DAG);
1005  SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
1006  DAG.getConstant(Shift, DL, ShiftTy));
1007  Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
1008  DAG.getConstant(0, DL, Hi.getValueType()),
1009  ISD::SETNE);
1010  } else {
1011  // Signed overflow occurred if the high part does not sign extend the low.
1012  SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
1013  Mul, DAG.getValueType(SmallVT));
1014  Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
1015  }
1016 
1017  // The only other way for overflow to occur is if the multiplication in the
1018  // larger type itself overflowed.
1019  Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
1020  SDValue(Mul.getNode(), 1));
1021 
1022  // Use the calculated overflow everywhere.
1023  ReplaceValueWith(SDValue(N, 1), Overflow);
1024  return Mul;
1025 }
1026 
1027 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
1028  return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1029  N->getValueType(0)));
1030 }
1031 
1032 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
1033  SDValue Chain = N->getOperand(0); // Get the chain.
1034  SDValue Ptr = N->getOperand(1); // Get the pointer.
1035  EVT VT = N->getValueType(0);
1036  SDLoc dl(N);
1037 
1038  MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
1039  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
1040  // The argument is passed as NumRegs registers of type RegVT.
1041 
1042  SmallVector<SDValue, 8> Parts(NumRegs);
1043  for (unsigned i = 0; i < NumRegs; ++i) {
1044  Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
1045  N->getConstantOperandVal(3));
1046  Chain = Parts[i].getValue(1);
1047  }
1048 
1049  // Handle endianness of the load.
1050  if (DAG.getDataLayout().isBigEndian())
1051  std::reverse(Parts.begin(), Parts.end());
1052 
1053  // Assemble the parts in the promoted type.
1054  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1055  SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
1056  for (unsigned i = 1; i < NumRegs; ++i) {
1057  SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
1058  // Shift it to the right position and "or" it in.
1059  Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
1060  DAG.getConstant(i * RegVT.getSizeInBits(), dl,
1061  TLI.getPointerTy(DAG.getDataLayout())));
1062  Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
1063  }
1064 
1065  // Modified the chain result - switch anything that used the old chain to
1066  // use the new one.
1067  ReplaceValueWith(SDValue(N, 1), Chain);
1068 
1069  return Res;
1070 }
1071 
1072 //===----------------------------------------------------------------------===//
1073 // Integer Operand Promotion
1074 //===----------------------------------------------------------------------===//
1075 
1076 /// PromoteIntegerOperand - This method is called when the specified operand of
1077 /// the specified node is found to need promotion. At this point, all of the
1078 /// result types of the node are known to be legal, but other operands of the
1079 /// node may need promotion or expansion as well as the specified one.
1080 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
1081  LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG);
1082  dbgs() << "\n");
1083  SDValue Res = SDValue();
1084 
1085  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
1086  LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
1087  return false;
1088  }
1089 
1090  switch (N->getOpcode()) {
1091  default:
1092  #ifndef NDEBUG
1093  dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
1094  N->dump(&DAG); dbgs() << "\n";
1095  #endif
1096  llvm_unreachable("Do not know how to promote this operator's operand!");
1097 
1098  case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
1099  case ISD::ATOMIC_STORE:
1100  Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
1101  break;
1102  case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
1103  case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
1104  case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
1105  case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
1106  case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
1107  case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
1108  case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
1110  Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
1111  case ISD::SCALAR_TO_VECTOR:
1112  Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
1113  case ISD::VSELECT:
1114  case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
1115  case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
1116  case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
1117  case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
1118  case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
1119  case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
1120  OpNo); break;
1121  case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
1122  OpNo); break;
1123  case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
1124  OpNo); break;
1125  case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
1126  OpNo); break;
1127  case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
1128  OpNo); break;
1129  case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
1130  case ISD::FP16_TO_FP:
1131  case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
1132  case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
1133  case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
1134 
1135  case ISD::SHL:
1136  case ISD::SRA:
1137  case ISD::SRL:
1138  case ISD::ROTL:
1139  case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
1140 
1141  case ISD::ADDCARRY:
1142  case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
1143 
1144  case ISD::FRAMEADDR:
1145  case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
1146 
1147  case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break;
1148 
1149  case ISD::SMULFIX:
1150  case ISD::SMULFIXSAT:
1151  case ISD::UMULFIX: Res = PromoteIntOp_MULFIX(N); break;
1152 
1153  case ISD::FPOWI: Res = PromoteIntOp_FPOWI(N); break;
1154 
1155  case ISD::VECREDUCE_ADD:
1156  case ISD::VECREDUCE_MUL:
1157  case ISD::VECREDUCE_AND:
1158  case ISD::VECREDUCE_OR:
1159  case ISD::VECREDUCE_XOR:
1160  case ISD::VECREDUCE_SMAX:
1161  case ISD::VECREDUCE_SMIN:
1162  case ISD::VECREDUCE_UMAX:
1163  case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break;
1164  }
1165 
1166  // If the result is null, the sub-method took care of registering results etc.
1167  if (!Res.getNode()) return false;
1168 
1169  // If the result is N, the sub-method updated N in place. Tell the legalizer
1170  // core about this.
1171  if (Res.getNode() == N)
1172  return true;
1173 
1174  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1175  "Invalid operand expansion");
1176 
1177  ReplaceValueWith(SDValue(N, 0), Res);
1178  return false;
1179 }
1180 
1181 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
1182 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
1183 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
1184  ISD::CondCode CCCode) {
1185  // We have to insert explicit sign or zero extends. Note that we could
1186  // insert sign extends for ALL conditions. For those operations where either
1187  // zero or sign extension would be valid, use SExtOrZExtPromotedInteger
1188  // which will choose the cheapest for the target.
1189  switch (CCCode) {
1190  default: llvm_unreachable("Unknown integer comparison!");
1191  case ISD::SETEQ:
1192  case ISD::SETNE: {
1193  SDValue OpL = GetPromotedInteger(NewLHS);
1194  SDValue OpR = GetPromotedInteger(NewRHS);
1195 
1196  // We would prefer to promote the comparison operand with sign extension.
1197  // If the width of OpL/OpR excluding the duplicated sign bits is no greater
1198  // than the width of NewLHS/NewRH, we can avoid inserting real truncate
1199  // instruction, which is redundant eventually.
1200  unsigned OpLEffectiveBits =
1201  OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
1202  unsigned OpREffectiveBits =
1203  OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
1204  if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() &&
1205  OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()) {
1206  NewLHS = OpL;
1207  NewRHS = OpR;
1208  } else {
1209  NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1210  NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1211  }
1212  break;
1213  }
1214  case ISD::SETUGE:
1215  case ISD::SETUGT:
1216  case ISD::SETULE:
1217  case ISD::SETULT:
1218  NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1219  NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1220  break;
1221  case ISD::SETGE:
1222  case ISD::SETGT:
1223  case ISD::SETLT:
1224  case ISD::SETLE:
1225  NewLHS = SExtPromotedInteger(NewLHS);
1226  NewRHS = SExtPromotedInteger(NewRHS);
1227  break;
1228  }
1229 }
1230 
1231 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1232  SDValue Op = GetPromotedInteger(N->getOperand(0));
1233  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1234 }
1235 
1236 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1237  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1238  return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1239  N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1240 }
1241 
1242 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1243  // This should only occur in unusual situations like bitcasting to an
1244  // x86_fp80, so just turn it into a store+load
1245  return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1246 }
1247 
1248 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1249  assert(OpNo == 2 && "Don't know how to promote this operand!");
1250 
1251  SDValue LHS = N->getOperand(2);
1252  SDValue RHS = N->getOperand(3);
1253  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1254 
1255  // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1256  // legal types.
1257  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1258  N->getOperand(1), LHS, RHS, N->getOperand(4)),
1259  0);
1260 }
1261 
1262 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1263  assert(OpNo == 1 && "only know how to promote condition");
1264 
1265  // Promote all the way up to the canonical SetCC type.
1266  SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1267 
1268  // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1269  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1270  N->getOperand(2)), 0);
1271 }
1272 
1273 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1274  // Since the result type is legal, the operands must promote to it.
1275  EVT OVT = N->getOperand(0).getValueType();
1276  SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1277  SDValue Hi = GetPromotedInteger(N->getOperand(1));
1278  assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1279  SDLoc dl(N);
1280 
1281  Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1282  DAG.getConstant(OVT.getSizeInBits(), dl,
1283  TLI.getPointerTy(DAG.getDataLayout())));
1284  return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1285 }
1286 
1287 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1288  // The vector type is legal but the element type is not. This implies
1289  // that the vector is a power-of-two in length and that the element
1290  // type does not have a strange size (eg: it is not i1).
1291  EVT VecVT = N->getValueType(0);
1292  unsigned NumElts = VecVT.getVectorNumElements();
1293  assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1294  "Legal vector of one illegal element?");
1295 
1296  // Promote the inserted value. The type does not need to match the
1297  // vector element type. Check that any extra bits introduced will be
1298  // truncated away.
1301  "Type of inserted value narrower than vector element type!");
1302 
1303  SmallVector<SDValue, 16> NewOps;
1304  for (unsigned i = 0; i < NumElts; ++i)
1305  NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1306 
1307  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1308 }
1309 
1310 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1311  unsigned OpNo) {
1312  if (OpNo == 1) {
1313  // Promote the inserted value. This is valid because the type does not
1314  // have to match the vector element type.
1315 
1316  // Check that any extra bits introduced will be truncated away.
1319  "Type of inserted value narrower than vector element type!");
1320  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1321  GetPromotedInteger(N->getOperand(1)),
1322  N->getOperand(2)),
1323  0);
1324  }
1325 
1326  assert(OpNo == 2 && "Different operand and result vector types?");
1327 
1328  // Promote the index.
1329  SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1330  TLI.getVectorIdxTy(DAG.getDataLayout()));
1331  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1332  N->getOperand(1), Idx), 0);
1333 }
1334 
1335 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1336  // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1337  // the operand in place.
1338  return SDValue(DAG.UpdateNodeOperands(N,
1339  GetPromotedInteger(N->getOperand(0))), 0);
1340 }
1341 
1342 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1343  assert(OpNo == 0 && "Only know how to promote the condition!");
1344  SDValue Cond = N->getOperand(0);
1345  EVT OpTy = N->getOperand(1).getValueType();
1346 
1347  if (N->getOpcode() == ISD::VSELECT)
1348  if (SDValue Res = WidenVSELECTAndMask(N))
1349  return Res;
1350 
1351  // Promote all the way up to the canonical SetCC type.
1352  EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1353  Cond = PromoteTargetBoolean(Cond, OpVT);
1354 
1355  return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1356  N->getOperand(2)), 0);
1357 }
1358 
1359 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1360  assert(OpNo == 0 && "Don't know how to promote this operand!");
1361 
1362  SDValue LHS = N->getOperand(0);
1363  SDValue RHS = N->getOperand(1);
1364  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1365 
1366  // The CC (#4) and the possible return values (#2 and #3) have legal types.
1367  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1368  N->getOperand(3), N->getOperand(4)), 0);
1369 }
1370 
1371 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1372  assert(OpNo == 0 && "Don't know how to promote this operand!");
1373 
1374  SDValue LHS = N->getOperand(0);
1375  SDValue RHS = N->getOperand(1);
1376  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1377 
1378  // The CC (#2) is always legal.
1379  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1380 }
1381 
1382 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1383  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1384  ZExtPromotedInteger(N->getOperand(1))), 0);
1385 }
1386 
1387 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1388  SDValue Op = GetPromotedInteger(N->getOperand(0));
1389  SDLoc dl(N);
1390  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1391  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1392  Op, DAG.getValueType(N->getOperand(0).getValueType()));
1393 }
1394 
1395 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1396  return SDValue(DAG.UpdateNodeOperands(N,
1397  SExtPromotedInteger(N->getOperand(0))), 0);
1398 }
1399 
1400 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1401  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1402  SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1403  SDLoc dl(N);
1404 
1405  SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1406 
1407  // Truncate the value and store the result.
1408  return DAG.getTruncStore(Ch, dl, Val, Ptr,
1409  N->getMemoryVT(), N->getMemOperand());
1410 }
1411 
1412 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1413  unsigned OpNo) {
1414 
1415  SDValue DataOp = N->getValue();
1416  EVT DataVT = DataOp.getValueType();
1417  SDValue Mask = N->getMask();
1418  SDLoc dl(N);
1419 
1420  bool TruncateStore = false;
1421  if (OpNo == 3) {
1422  Mask = PromoteTargetBoolean(Mask, DataVT);
1423  // Update in place.
1424  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1425  NewOps[3] = Mask;
1426  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1427  } else { // Data operand
1428  assert(OpNo == 1 && "Unexpected operand for promotion");
1429  DataOp = GetPromotedInteger(DataOp);
1430  TruncateStore = true;
1431  }
1432 
1433  return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask,
1434  N->getMemoryVT(), N->getMemOperand(),
1435  TruncateStore, N->isCompressingStore());
1436 }
1437 
1438 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1439  unsigned OpNo) {
1440  assert(OpNo == 2 && "Only know how to promote the mask!");
1441  EVT DataVT = N->getValueType(0);
1442  SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1443  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1444  NewOps[OpNo] = Mask;
1445  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1446 }
1447 
1448 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1449  unsigned OpNo) {
1450 
1451  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1452  if (OpNo == 2) {
1453  // The Mask
1454  EVT DataVT = N->getValueType(0);
1455  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1456  } else if (OpNo == 4) {
1457  // Need to sign extend the index since the bits will likely be used.
1458  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1459  } else
1460  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1461 
1462  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1463 }
1464 
1465 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1466  unsigned OpNo) {
1467  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1468  if (OpNo == 2) {
1469  // The Mask
1470  EVT DataVT = N->getValue().getValueType();
1471  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1472  } else if (OpNo == 4) {
1473  // Need to sign extend the index since the bits will likely be used.
1474  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1475  } else
1476  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1477  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1478 }
1479 
1480 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
1481  SDValue Op = GetPromotedInteger(N->getOperand(0));
1482  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
1483 }
1484 
1485 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
1486  return SDValue(DAG.UpdateNodeOperands(N,
1487  ZExtPromotedInteger(N->getOperand(0))), 0);
1488 }
1489 
1490 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
1491  SDLoc dl(N);
1492  SDValue Op = GetPromotedInteger(N->getOperand(0));
1493  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1494  return DAG.getZeroExtendInReg(Op, dl,
1496 }
1497 
1498 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
1499  assert(OpNo == 2 && "Don't know how to promote this operand!");
1500 
1501  SDValue LHS = N->getOperand(0);
1502  SDValue RHS = N->getOperand(1);
1503  SDValue Carry = N->getOperand(2);
1504  SDLoc DL(N);
1505 
1506  Carry = PromoteTargetBoolean(Carry, LHS.getValueType());
1507 
1508  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
1509 }
1510 
1511 SDValue DAGTypeLegalizer::PromoteIntOp_MULFIX(SDNode *N) {
1512  SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1513  return SDValue(
1514  DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
1515 }
1516 
1517 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
1518  // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
1519  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
1520  return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
1521 }
1522 
1523 SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) {
1524  assert(OpNo > 1 && "Don't know how to promote this operand!");
1525  // Promote the rw, locality, and cache type arguments to a supported integer
1526  // width.
1527  SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1528  SDValue Op3 = ZExtPromotedInteger(N->getOperand(3));
1529  SDValue Op4 = ZExtPromotedInteger(N->getOperand(4));
1530  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
1531  Op2, Op3, Op4),
1532  0);
1533 }
1534 
1535 SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) {
1536  SDValue Op = SExtPromotedInteger(N->getOperand(1));
1537  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
1538 }
1539 
1540 SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) {
1541  SDLoc dl(N);
1542  SDValue Op;
1543  switch (N->getOpcode()) {
1544  default: llvm_unreachable("Expected integer vector reduction");
1545  case ISD::VECREDUCE_ADD:
1546  case ISD::VECREDUCE_MUL:
1547  case ISD::VECREDUCE_AND:
1548  case ISD::VECREDUCE_OR:
1549  case ISD::VECREDUCE_XOR:
1550  Op = GetPromotedInteger(N->getOperand(0));
1551  break;
1552  case ISD::VECREDUCE_SMAX:
1553  case ISD::VECREDUCE_SMIN:
1554  Op = SExtPromotedInteger(N->getOperand(0));
1555  break;
1556  case ISD::VECREDUCE_UMAX:
1557  case ISD::VECREDUCE_UMIN:
1558  Op = ZExtPromotedInteger(N->getOperand(0));
1559  break;
1560  }
1561 
1562  EVT EltVT = Op.getValueType().getVectorElementType();
1563  EVT VT = N->getValueType(0);
1564  if (VT.bitsGE(EltVT))
1565  return DAG.getNode(N->getOpcode(), SDLoc(N), VT, Op);
1566 
1567  // Result size must be >= element size. If this is not the case after
1568  // promotion, also promote the result type and then truncate.
1569  SDValue Reduce = DAG.getNode(N->getOpcode(), dl, EltVT, Op);
1570  return DAG.getNode(ISD::TRUNCATE, dl, VT, Reduce);
1571 }
1572 
1573 //===----------------------------------------------------------------------===//
1574 // Integer Result Expansion
1575 //===----------------------------------------------------------------------===//
1576 
1577 /// ExpandIntegerResult - This method is called when the specified result of the
1578 /// specified node is found to need expansion. At this point, the node may also
1579 /// have invalid operands or may have other results that need promotion, we just
1580 /// know that (at least) one result needs expansion.
1581 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
1582  LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG);
1583  dbgs() << "\n");
1584  SDValue Lo, Hi;
1585  Lo = Hi = SDValue();
1586 
1587  // See if the target wants to custom expand this node.
1588  if (CustomLowerNode(N, N->getValueType(ResNo), true))
1589  return;
1590 
1591  switch (N->getOpcode()) {
1592  default:
1593 #ifndef NDEBUG
1594  dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
1595  N->dump(&DAG); dbgs() << "\n";
1596 #endif
1597  report_fatal_error("Do not know how to expand the result of this "
1598  "operator!");
1599 
1600  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1601  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
1602  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1603  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1604 
1605  case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1606  case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1607  case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1608  case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1609  case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1610 
1611  case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
1612  case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
1613  case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
1614  case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
1615  case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
1616  case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
1617  case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break;
1618  case ISD::CTLZ_ZERO_UNDEF:
1619  case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
1620  case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
1621  case ISD::CTTZ_ZERO_UNDEF:
1622  case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
1623  case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
1624  case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
1625  case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
1626  case ISD::LLROUND: ExpandIntRes_LLROUND(N, Lo, Hi); break;
1627  case ISD::LLRINT: ExpandIntRes_LLRINT(N, Lo, Hi); break;
1628  case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
1629  case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
1630  case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
1631  case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
1632  case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
1633  case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
1634  case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
1635  case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
1636  case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
1637  case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
1638  case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
1639  case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
1640 
1641  case ISD::ATOMIC_LOAD_ADD:
1642  case ISD::ATOMIC_LOAD_SUB:
1643  case ISD::ATOMIC_LOAD_AND:
1644  case ISD::ATOMIC_LOAD_CLR:
1645  case ISD::ATOMIC_LOAD_OR:
1646  case ISD::ATOMIC_LOAD_XOR:
1647  case ISD::ATOMIC_LOAD_NAND:
1648  case ISD::ATOMIC_LOAD_MIN:
1649  case ISD::ATOMIC_LOAD_MAX:
1650  case ISD::ATOMIC_LOAD_UMIN:
1651  case ISD::ATOMIC_LOAD_UMAX:
1652  case ISD::ATOMIC_SWAP:
1653  case ISD::ATOMIC_CMP_SWAP: {
1654  std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
1655  SplitInteger(Tmp.first, Lo, Hi);
1656  ReplaceValueWith(SDValue(N, 1), Tmp.second);
1657  break;
1658  }
1660  AtomicSDNode *AN = cast<AtomicSDNode>(N);
1661  SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
1662  SDValue Tmp = DAG.getAtomicCmpSwap(
1663  ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
1664  N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
1665  AN->getMemOperand());
1666 
1667  // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
1668  // success simply by comparing the loaded value against the ingoing
1669  // comparison.
1670  SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
1671  N->getOperand(2), ISD::SETEQ);
1672 
1673  SplitInteger(Tmp, Lo, Hi);
1674  ReplaceValueWith(SDValue(N, 1), Success);
1675  ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
1676  break;
1677  }
1678 
1679  case ISD::AND:
1680  case ISD::OR:
1681  case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
1682 
1683  case ISD::UMAX:
1684  case ISD::SMAX:
1685  case ISD::UMIN:
1686  case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
1687 
1688  case ISD::ADD:
1689  case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
1690 
1691  case ISD::ADDC:
1692  case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
1693 
1694  case ISD::ADDE:
1695  case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
1696 
1697  case ISD::ADDCARRY:
1698  case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
1699 
1700  case ISD::SHL:
1701  case ISD::SRA:
1702  case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
1703 
1704  case ISD::SADDO:
1705  case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
1706  case ISD::UADDO:
1707  case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
1708  case ISD::UMULO:
1709  case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
1710 
1711  case ISD::SADDSAT:
1712  case ISD::UADDSAT:
1713  case ISD::SSUBSAT:
1714  case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
1715 
1716  case ISD::SMULFIX:
1717  case ISD::SMULFIXSAT:
1718  case ISD::UMULFIX: ExpandIntRes_MULFIX(N, Lo, Hi); break;
1719 
1720  case ISD::VECREDUCE_ADD:
1721  case ISD::VECREDUCE_MUL:
1722  case ISD::VECREDUCE_AND:
1723  case ISD::VECREDUCE_OR:
1724  case ISD::VECREDUCE_XOR:
1725  case ISD::VECREDUCE_SMAX:
1726  case ISD::VECREDUCE_SMIN:
1727  case ISD::VECREDUCE_UMAX:
1728  case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break;
1729  }
1730 
1731  // If Lo/Hi is null, the sub-method took care of registering results etc.
1732  if (Lo.getNode())
1733  SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
1734 }
1735 
1736 /// Lower an atomic node to the appropriate builtin call.
1737 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
1738  unsigned Opc = Node->getOpcode();
1739  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
1740  RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
1741  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
1742 
1743  return ExpandChainLibCall(LC, Node, false);
1744 }
1745 
1746 /// N is a shift by a value that needs to be expanded,
1747 /// and the shift amount is a constant 'Amt'. Expand the operation.
1748 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
1749  SDValue &Lo, SDValue &Hi) {
1750  SDLoc DL(N);
1751  // Expand the incoming operand to be shifted, so that we have its parts
1752  SDValue InL, InH;
1753  GetExpandedInteger(N->getOperand(0), InL, InH);
1754 
1755  // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
1756  // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
1757  if (!Amt) {
1758  Lo = InL;
1759  Hi = InH;
1760  return;
1761  }
1762 
1763  EVT NVT = InL.getValueType();
1764  unsigned VTBits = N->getValueType(0).getSizeInBits();
1765  unsigned NVTBits = NVT.getSizeInBits();
1766  EVT ShTy = N->getOperand(1).getValueType();
1767 
1768  if (N->getOpcode() == ISD::SHL) {
1769  if (Amt.ugt(VTBits)) {
1770  Lo = Hi = DAG.getConstant(0, DL, NVT);
1771  } else if (Amt.ugt(NVTBits)) {
1772  Lo = DAG.getConstant(0, DL, NVT);
1773  Hi = DAG.getNode(ISD::SHL, DL,
1774  NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1775  } else if (Amt == NVTBits) {
1776  Lo = DAG.getConstant(0, DL, NVT);
1777  Hi = InL;
1778  } else {
1779  Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
1780  Hi = DAG.getNode(ISD::OR, DL, NVT,
1781  DAG.getNode(ISD::SHL, DL, NVT, InH,
1782  DAG.getConstant(Amt, DL, ShTy)),
1783  DAG.getNode(ISD::SRL, DL, NVT, InL,
1784  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1785  }
1786  return;
1787  }
1788 
1789  if (N->getOpcode() == ISD::SRL) {
1790  if (Amt.ugt(VTBits)) {
1791  Lo = Hi = DAG.getConstant(0, DL, NVT);
1792  } else if (Amt.ugt(NVTBits)) {
1793  Lo = DAG.getNode(ISD::SRL, DL,
1794  NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1795  Hi = DAG.getConstant(0, DL, NVT);
1796  } else if (Amt == NVTBits) {
1797  Lo = InH;
1798  Hi = DAG.getConstant(0, DL, NVT);
1799  } else {
1800  Lo = DAG.getNode(ISD::OR, DL, NVT,
1801  DAG.getNode(ISD::SRL, DL, NVT, InL,
1802  DAG.getConstant(Amt, DL, ShTy)),
1803  DAG.getNode(ISD::SHL, DL, NVT, InH,
1804  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1805  Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1806  }
1807  return;
1808  }
1809 
1810  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
1811  if (Amt.ugt(VTBits)) {
1812  Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1813  DAG.getConstant(NVTBits - 1, DL, ShTy));
1814  } else if (Amt.ugt(NVTBits)) {
1815  Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1816  DAG.getConstant(Amt - NVTBits, DL, ShTy));
1817  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1818  DAG.getConstant(NVTBits - 1, DL, ShTy));
1819  } else if (Amt == NVTBits) {
1820  Lo = InH;
1821  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1822  DAG.getConstant(NVTBits - 1, DL, ShTy));
1823  } else {
1824  Lo = DAG.getNode(ISD::OR, DL, NVT,
1825  DAG.getNode(ISD::SRL, DL, NVT, InL,
1826  DAG.getConstant(Amt, DL, ShTy)),
1827  DAG.getNode(ISD::SHL, DL, NVT, InH,
1828  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1829  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1830  }
1831 }
1832 
1833 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
1834 /// this shift based on knowledge of the high bit of the shift amount. If we
1835 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
1836 /// shift amount.
1837 bool DAGTypeLegalizer::
1838 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1839  SDValue Amt = N->getOperand(1);
1840  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1841  EVT ShTy = Amt.getValueType();
1842  unsigned ShBits = ShTy.getScalarSizeInBits();
1843  unsigned NVTBits = NVT.getScalarSizeInBits();
1844  assert(isPowerOf2_32(NVTBits) &&
1845  "Expanded integer type size not a power of two!");
1846  SDLoc dl(N);
1847 
1848  APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
1849  KnownBits Known = DAG.computeKnownBits(N->getOperand(1));
1850 
1851  // If we don't know anything about the high bits, exit.
1852  if (((Known.Zero|Known.One) & HighBitMask) == 0)
1853  return false;
1854 
1855  // Get the incoming operand to be shifted.
1856  SDValue InL, InH;
1857  GetExpandedInteger(N->getOperand(0), InL, InH);
1858 
1859  // If we know that any of the high bits of the shift amount are one, then we
1860  // can do this as a couple of simple shifts.
1861  if (Known.One.intersects(HighBitMask)) {
1862  // Mask out the high bit, which we know is set.
1863  Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
1864  DAG.getConstant(~HighBitMask, dl, ShTy));
1865 
1866  switch (N->getOpcode()) {
1867  default: llvm_unreachable("Unknown shift");
1868  case ISD::SHL:
1869  Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
1870  Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
1871  return true;
1872  case ISD::SRL:
1873  Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1874  Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
1875  return true;
1876  case ISD::SRA:
1877  Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
1878  DAG.getConstant(NVTBits - 1, dl, ShTy));
1879  Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
1880  return true;
1881  }
1882  }
1883 
1884  // If we know that all of the high bits of the shift amount are zero, then we
1885  // can do this as a couple of simple shifts.
1886  if (HighBitMask.isSubsetOf(Known.Zero)) {
1887  // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
1888  // shift if x is zero. We can use XOR here because x is known to be smaller
1889  // than 32.
1890  SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
1891  DAG.getConstant(NVTBits - 1, dl, ShTy));
1892 
1893  unsigned Op1, Op2;
1894  switch (N->getOpcode()) {
1895  default: llvm_unreachable("Unknown shift");
1896  case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
1897  case ISD::SRL:
1898  case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
1899  }
1900 
1901  // When shifting right the arithmetic for Lo and Hi is swapped.
1902  if (N->getOpcode() != ISD::SHL)
1903  std::swap(InL, InH);
1904 
1905  // Use a little trick to get the bits that move from Lo to Hi. First
1906  // shift by one bit.
1907  SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
1908  // Then compute the remaining shift with amount-1.
1909  SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
1910 
1911  Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
1912  Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
1913 
1914  if (N->getOpcode() != ISD::SHL)
1915  std::swap(Hi, Lo);
1916  return true;
1917  }
1918 
1919  return false;
1920 }
1921 
1922 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
1923 /// of any size.
1924 bool DAGTypeLegalizer::
1925 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1926  SDValue Amt = N->getOperand(1);
1927  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1928  EVT ShTy = Amt.getValueType();
1929  unsigned NVTBits = NVT.getSizeInBits();
1930  assert(isPowerOf2_32(NVTBits) &&
1931  "Expanded integer type size not a power of two!");
1932  SDLoc dl(N);
1933 
1934  // Get the incoming operand to be shifted.
1935  SDValue InL, InH;
1936  GetExpandedInteger(N->getOperand(0), InL, InH);
1937 
1938  SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
1939  SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
1940  SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
1941  SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1942  Amt, NVBitsNode, ISD::SETULT);
1943  SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1944  Amt, DAG.getConstant(0, dl, ShTy),
1945  ISD::SETEQ);
1946 
1947  SDValue LoS, HiS, LoL, HiL;
1948  switch (N->getOpcode()) {
1949  default: llvm_unreachable("Unknown shift");
1950  case ISD::SHL:
1951  // Short: ShAmt < NVTBits
1952  LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
1953  HiS = DAG.getNode(ISD::OR, dl, NVT,
1954  DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
1955  DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
1956 
1957  // Long: ShAmt >= NVTBits
1958  LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
1959  HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
1960 
1961  Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
1962  Hi = DAG.getSelect(dl, NVT, isZero, InH,
1963  DAG.getSelect(dl, NVT, isShort, HiS, HiL));
1964  return true;
1965  case ISD::SRL:
1966  // Short: ShAmt < NVTBits
1967  HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
1968  LoS = DAG.getNode(ISD::OR, dl, NVT,
1969  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1970  // FIXME: If Amt is zero, the following shift generates an undefined result
1971  // on some architectures.
1972  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1973 
1974  // Long: ShAmt >= NVTBits
1975  HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1976  LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1977 
1978  Lo = DAG.getSelect(dl, NVT, isZero, InL,
1979  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1980  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1981  return true;
1982  case ISD::SRA:
1983  // Short: ShAmt < NVTBits
1984  HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
1985  LoS = DAG.getNode(ISD::OR, dl, NVT,
1986  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1987  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1988 
1989  // Long: ShAmt >= NVTBits
1990  HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
1991  DAG.getConstant(NVTBits - 1, dl, ShTy));
1992  LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1993 
1994  Lo = DAG.getSelect(dl, NVT, isZero, InL,
1995  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1996  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1997  return true;
1998  }
1999 }
2000 
2001 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
2002 
2003  switch (Op) {
2004  default: llvm_unreachable("invalid min/max opcode");
2005  case ISD::SMAX:
2006  return std::make_pair(ISD::SETGT, ISD::UMAX);
2007  case ISD::UMAX:
2008  return std::make_pair(ISD::SETUGT, ISD::UMAX);
2009  case ISD::SMIN:
2010  return std::make_pair(ISD::SETLT, ISD::UMIN);
2011  case ISD::UMIN:
2012  return std::make_pair(ISD::SETULT, ISD::UMIN);
2013  }
2014 }
2015 
2016 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
2017  SDValue &Lo, SDValue &Hi) {
2018  SDLoc DL(N);
2019  ISD::NodeType LoOpc;
2020  ISD::CondCode CondC;
2021  std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
2022 
2023  // Expand the subcomponents.
2024  SDValue LHSL, LHSH, RHSL, RHSH;
2025  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2026  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2027 
2028  // Value types
2029  EVT NVT = LHSL.getValueType();
2030  EVT CCT = getSetCCResultType(NVT);
2031 
2032  // Hi part is always the same op
2033  Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
2034 
2035  // We need to know whether to select Lo part that corresponds to 'winning'
2036  // Hi part or if Hi parts are equal.
2037  SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
2038  SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
2039 
2040  // Lo part corresponding to the 'winning' Hi part
2041  SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
2042 
2043  // Recursed Lo part if Hi parts are equal, this uses unsigned version
2044  SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
2045 
2046  Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
2047 }
2048 
2049 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
2050  SDValue &Lo, SDValue &Hi) {
2051  SDLoc dl(N);
2052  // Expand the subcomponents.
2053  SDValue LHSL, LHSH, RHSL, RHSH;
2054  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2055  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2056 
2057  EVT NVT = LHSL.getValueType();
2058  SDValue LoOps[2] = { LHSL, RHSL };
2059  SDValue HiOps[3] = { LHSH, RHSH };
2060 
2061  bool HasOpCarry = TLI.isOperationLegalOrCustom(
2063  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2064  if (HasOpCarry) {
2065  SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
2066  if (N->getOpcode() == ISD::ADD) {
2067  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2068  HiOps[2] = Lo.getValue(1);
2069  Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
2070  } else {
2071  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2072  HiOps[2] = Lo.getValue(1);
2073  Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
2074  }
2075  return;
2076  }
2077 
2078  // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
2079  // them. TODO: Teach operation legalization how to expand unsupported
2080  // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
2081  // a carry of type MVT::Glue, but there doesn't seem to be any way to
2082  // generate a value of this type in the expanded code sequence.
2083  bool hasCarry =
2085  ISD::ADDC : ISD::SUBC,
2086  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2087 
2088  if (hasCarry) {
2089  SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
2090  if (N->getOpcode() == ISD::ADD) {
2091  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2092  HiOps[2] = Lo.getValue(1);
2093  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2094  } else {
2095  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2096  HiOps[2] = Lo.getValue(1);
2097  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2098  }
2099  return;
2100  }
2101 
2102  bool hasOVF =
2105  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2107 
2108  if (hasOVF) {
2109  EVT OvfVT = getSetCCResultType(NVT);
2110  SDVTList VTList = DAG.getVTList(NVT, OvfVT);
2111  int RevOpc;
2112  if (N->getOpcode() == ISD::ADD) {
2113  RevOpc = ISD::SUB;
2114  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2115  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2116  } else {
2117  RevOpc = ISD::ADD;
2118  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2119  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2120  }
2121  SDValue OVF = Lo.getValue(1);
2122 
2123  switch (BoolType) {
2125  OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
2128  OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
2129  Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
2130  break;
2132  OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
2133  Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
2134  }
2135  return;
2136  }
2137 
2138  if (N->getOpcode() == ISD::ADD) {
2139  Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
2140  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2141  SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
2142  ISD::SETULT);
2143 
2145  SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
2146  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
2147  return;
2148  }
2149 
2150  SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
2151  DAG.getConstant(1, dl, NVT),
2152  DAG.getConstant(0, dl, NVT));
2153  SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
2154  ISD::SETULT);
2155  SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
2156  DAG.getConstant(1, dl, NVT), Carry1);
2157  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
2158  } else {
2159  Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
2160  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2161  SDValue Cmp =
2162  DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
2163  LoOps[0], LoOps[1], ISD::SETULT);
2164 
2165  SDValue Borrow;
2167  Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
2168  else
2169  Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
2170  DAG.getConstant(0, dl, NVT));
2171 
2172  Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
2173  }
2174 }
2175 
2176 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
2177  SDValue &Lo, SDValue &Hi) {
2178  // Expand the subcomponents.
2179  SDValue LHSL, LHSH, RHSL, RHSH;
2180  SDLoc dl(N);
2181  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2182  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2183  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2184  SDValue LoOps[2] = { LHSL, RHSL };
2185  SDValue HiOps[3] = { LHSH, RHSH };
2186 
2187  if (N->getOpcode() == ISD::ADDC) {
2188  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2189  HiOps[2] = Lo.getValue(1);
2190  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2191  } else {
2192  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2193  HiOps[2] = Lo.getValue(1);
2194  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2195  }
2196 
2197  // Legalized the flag result - switch anything that used the old flag to
2198  // use the new one.
2199  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2200 }
2201 
2202 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
2203  SDValue &Lo, SDValue &Hi) {
2204  // Expand the subcomponents.
2205  SDValue LHSL, LHSH, RHSL, RHSH;
2206  SDLoc dl(N);
2207  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2208  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2209  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2210  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2211  SDValue HiOps[3] = { LHSH, RHSH };
2212 
2213  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2214  HiOps[2] = Lo.getValue(1);
2215  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2216 
2217  // Legalized the flag result - switch anything that used the old flag to
2218  // use the new one.
2219  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2220 }
2221 
2222 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
2223  SDValue &Lo, SDValue &Hi) {
2224  SDValue LHS = N->getOperand(0);
2225  SDValue RHS = N->getOperand(1);
2226  SDLoc dl(N);
2227 
2228  SDValue Ovf;
2229 
2230  bool HasOpCarry = TLI.isOperationLegalOrCustom(
2232  TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
2233 
2234  if (HasOpCarry) {
2235  // Expand the subcomponents.
2236  SDValue LHSL, LHSH, RHSL, RHSH;
2237  GetExpandedInteger(LHS, LHSL, LHSH);
2238  GetExpandedInteger(RHS, RHSL, RHSH);
2239  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2240  SDValue LoOps[2] = { LHSL, RHSL };
2241  SDValue HiOps[3] = { LHSH, RHSH };
2242 
2243  unsigned Opc = N->getOpcode() == ISD::UADDO ? ISD::ADDCARRY : ISD::SUBCARRY;
2244  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2245  HiOps[2] = Lo.getValue(1);
2246  Hi = DAG.getNode(Opc, dl, VTList, HiOps);
2247 
2248  Ovf = Hi.getValue(1);
2249  } else {
2250  // Expand the result by simply replacing it with the equivalent
2251  // non-overflow-checking operation.
2252  auto Opc = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
2253  SDValue Sum = DAG.getNode(Opc, dl, LHS.getValueType(), LHS, RHS);
2254  SplitInteger(Sum, Lo, Hi);
2255 
2256  // Calculate the overflow: addition overflows iff a + b < a, and subtraction
2257  // overflows iff a - b > a.
2258  auto Cond = N->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
2259  Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
2260  }
2261 
2262  // Legalized the flag result - switch anything that used the old flag to
2263  // use the new one.
2264  ReplaceValueWith(SDValue(N, 1), Ovf);
2265 }
2266 
2267 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
2268  SDValue &Lo, SDValue &Hi) {
2269  // Expand the subcomponents.
2270  SDValue LHSL, LHSH, RHSL, RHSH;
2271  SDLoc dl(N);
2272  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2273  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2274  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2275  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2276  SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
2277 
2278  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2279  HiOps[2] = Lo.getValue(1);
2280  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2281 
2282  // Legalized the flag result - switch anything that used the old flag to
2283  // use the new one.
2284  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2285 }
2286 
2287 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
2288  SDValue &Lo, SDValue &Hi) {
2289  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2290  SDLoc dl(N);
2291  SDValue Op = N->getOperand(0);
2292  if (Op.getValueType().bitsLE(NVT)) {
2293  // The low part is any extension of the input (which degenerates to a copy).
2294  Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
2295  Hi = DAG.getUNDEF(NVT); // The high part is undefined.
2296  } else {
2297  // For example, extension of an i48 to an i64. The operand type necessarily
2298  // promotes to the result type, so will end up being expanded too.
2299  assert(getTypeAction(Op.getValueType()) ==
2301  "Only know how to promote this result!");
2302  SDValue Res = GetPromotedInteger(Op);
2303  assert(Res.getValueType() == N->getValueType(0) &&
2304  "Operand over promoted?");
2305  // Split the promoted operand. This will simplify when it is expanded.
2306  SplitInteger(Res, Lo, Hi);
2307  }
2308 }
2309 
2310 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
2311  SDValue &Lo, SDValue &Hi) {
2312  SDLoc dl(N);
2313  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2314  EVT NVT = Lo.getValueType();
2315  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2316  unsigned NVTBits = NVT.getSizeInBits();
2317  unsigned EVTBits = EVT.getSizeInBits();
2318 
2319  if (NVTBits < EVTBits) {
2320  Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
2322  EVTBits - NVTBits)));
2323  } else {
2324  Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
2325  // The high part replicates the sign bit of Lo, make it explicit.
2326  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2327  DAG.getConstant(NVTBits - 1, dl,
2328  TLI.getPointerTy(DAG.getDataLayout())));
2329  }
2330 }
2331 
2332 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
2333  SDValue &Lo, SDValue &Hi) {
2334  SDLoc dl(N);
2335  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2336  EVT NVT = Lo.getValueType();
2337  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2338  unsigned NVTBits = NVT.getSizeInBits();
2339  unsigned EVTBits = EVT.getSizeInBits();
2340 
2341  if (NVTBits < EVTBits) {
2342  Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
2344  EVTBits - NVTBits)));
2345  } else {
2346  Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
2347  // The high part must be zero, make it explicit.
2348  Hi = DAG.getConstant(0, dl, NVT);
2349  }
2350 }
2351 
2352 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
2353  SDValue &Lo, SDValue &Hi) {
2354  SDLoc dl(N);
2355  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2356  Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
2357  Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
2358 }
2359 
2360 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
2361  SDValue &Lo, SDValue &Hi) {
2362  SDLoc dl(N);
2363  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2364  Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
2365  Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
2366 }
2367 
2368 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
2369  SDValue &Lo, SDValue &Hi) {
2370  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2371  unsigned NBitWidth = NVT.getSizeInBits();
2372  auto Constant = cast<ConstantSDNode>(N);
2373  const APInt &Cst = Constant->getAPIntValue();
2374  bool IsTarget = Constant->isTargetOpcode();
2375  bool IsOpaque = Constant->isOpaque();
2376  SDLoc dl(N);
2377  Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
2378  Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
2379  IsOpaque);
2380 }
2381 
2382 void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
2383  SDLoc dl(N);
2384 
2385  // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo)
2386  EVT VT = N->getValueType(0);
2387  SDValue N0 = N->getOperand(0);
2388  SDValue Neg = DAG.getNode(ISD::SUB, dl, VT,
2389  DAG.getConstant(0, dl, VT), N0);
2390  SDValue NegLo, NegHi;
2391  SplitInteger(Neg, NegLo, NegHi);
2392 
2393  GetExpandedInteger(N0, Lo, Hi);
2394  EVT NVT = Lo.getValueType();
2395  SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT),
2396  DAG.getConstant(0, dl, NVT), Hi, ISD::SETGT);
2397  Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo);
2398  Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi);
2399 }
2400 
2401 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
2402  SDValue &Lo, SDValue &Hi) {
2403  SDLoc dl(N);
2404  // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
2405  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2406  EVT NVT = Lo.getValueType();
2407 
2408  SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
2409  DAG.getConstant(0, dl, NVT), ISD::SETNE);
2410 
2411  SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
2412  SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
2413 
2414  Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
2415  DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
2416  DAG.getConstant(NVT.getSizeInBits(), dl,
2417  NVT)));
2418  Hi = DAG.getConstant(0, dl, NVT);
2419 }
2420 
2421 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
2422  SDValue &Lo, SDValue &Hi) {
2423  SDLoc dl(N);
2424  // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
2425  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2426  EVT NVT = Lo.getValueType();
2427  Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
2428  DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
2429  Hi = DAG.getConstant(0, dl, NVT);
2430 }
2431 
2432 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
2433  SDValue &Lo, SDValue &Hi) {
2434  SDLoc dl(N);
2435  // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
2436  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2437  EVT NVT = Lo.getValueType();
2438 
2439  SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
2440  DAG.getConstant(0, dl, NVT), ISD::SETNE);
2441 
2442  SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
2443  SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
2444 
2445  Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
2446  DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
2447  DAG.getConstant(NVT.getSizeInBits(), dl,
2448  NVT)));
2449  Hi = DAG.getConstant(0, dl, NVT);
2450 }
2451 
2452 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
2453  SDValue &Hi) {
2454  SDLoc dl(N);
2455  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2456  unsigned NBitWidth = NVT.getSizeInBits();
2457 
2458  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2459  Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT);
2460  // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
2461  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2462  DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
2463 }
2464 
2465 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
2466  SDValue &Hi) {
2467  SDLoc dl(N);
2468  EVT VT = N->getValueType(0);
2469 
2470  SDValue Op = N->getOperand(0);
2471  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2472  Op = GetPromotedFloat(Op);
2473 
2475  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
2476  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, true/*irrelevant*/, dl).first,
2477  Lo, Hi);
2478 }
2479 
2480 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
2481  SDValue &Hi) {
2482  SDLoc dl(N);
2483  EVT VT = N->getValueType(0);
2484 
2485  SDValue Op = N->getOperand(0);
2486  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2487  Op = GetPromotedFloat(Op);
2488 
2490  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
2491  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, false/*irrelevant*/, dl).first,
2492  Lo, Hi);
2493 }
2494 
2495 void DAGTypeLegalizer::ExpandIntRes_LLROUND(SDNode *N, SDValue &Lo,
2496  SDValue &Hi) {
2497  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2499  if (VT == MVT::f32)
2500  LC = RTLIB::LLROUND_F32;
2501  else if (VT == MVT::f64)
2502  LC = RTLIB::LLROUND_F64;
2503  else if (VT == MVT::f80)
2504  LC = RTLIB::LLROUND_F80;
2505  else if (VT == MVT::f128)
2506  LC = RTLIB::LLROUND_F128;
2507  else if (VT == MVT::ppcf128)
2508  LC = RTLIB::LLROUND_PPCF128;
2509  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!");
2510 
2511  SDValue Op = N->getOperand(0);
2512  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2513  Op = GetPromotedFloat(Op);
2514 
2515  SDLoc dl(N);
2516  EVT RetVT = N->getValueType(0);
2517  SplitInteger(TLI.makeLibCall(DAG, LC, RetVT, Op, true/*irrelevant*/, dl).first,
2518  Lo, Hi);
2519 }
2520 
2521 void DAGTypeLegalizer::ExpandIntRes_LLRINT(SDNode *N, SDValue &Lo,
2522  SDValue &Hi) {
2523  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2525  if (VT == MVT::f32)
2526  LC = RTLIB::LLRINT_F32;
2527  else if (VT == MVT::f64)
2528  LC = RTLIB::LLRINT_F64;
2529  else if (VT == MVT::f80)
2530  LC = RTLIB::LLRINT_F80;
2531  else if (VT == MVT::f128)
2532  LC = RTLIB::LLRINT_F128;
2533  else if (VT == MVT::ppcf128)
2534  LC = RTLIB::LLRINT_PPCF128;
2535  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!");
2536 
2537  SDValue Op = N->getOperand(0);
2538  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2539  Op = GetPromotedFloat(Op);
2540 
2541  SDLoc dl(N);
2542  EVT RetVT = N->getValueType(0);
2543  SplitInteger(TLI.makeLibCall(DAG, LC, RetVT, Op, true/*irrelevant*/, dl).first,
2544  Lo, Hi);
2545 }
2546 
2547 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
2548  SDValue &Lo, SDValue &Hi) {
2549  if (ISD::isNormalLoad(N)) {
2550  ExpandRes_NormalLoad(N, Lo, Hi);
2551  return;
2552  }
2553 
2554  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2555 
2556  EVT VT = N->getValueType(0);
2557  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2558  SDValue Ch = N->getChain();
2559  SDValue Ptr = N->getBasePtr();
2561  unsigned Alignment = N->getAlignment();
2562  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2563  AAMDNodes AAInfo = N->getAAInfo();
2564  SDLoc dl(N);
2565 
2566  assert(NVT.isByteSized() && "Expanded type not byte sized!");
2567 
2568  if (N->getMemoryVT().bitsLE(NVT)) {
2569  EVT MemVT = N->getMemoryVT();
2570 
2571  Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
2572  Alignment, MMOFlags, AAInfo);
2573 
2574  // Remember the chain.
2575  Ch = Lo.getValue(1);
2576 
2577  if (ExtType == ISD::SEXTLOAD) {
2578  // The high part is obtained by SRA'ing all but one of the bits of the
2579  // lo part.
2580  unsigned LoSize = Lo.getValueSizeInBits();
2581  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2582  DAG.getConstant(LoSize - 1, dl,
2583  TLI.getPointerTy(DAG.getDataLayout())));
2584  } else if (ExtType == ISD::ZEXTLOAD) {
2585  // The high part is just a zero.
2586  Hi = DAG.getConstant(0, dl, NVT);
2587  } else {
2588  assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
2589  // The high part is undefined.
2590  Hi = DAG.getUNDEF(NVT);
2591  }
2592  } else if (DAG.getDataLayout().isLittleEndian()) {
2593  // Little-endian - low bits are at low addresses.
2594  Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2595  AAInfo);
2596 
2597  unsigned ExcessBits =
2598  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
2599  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
2600 
2601  // Increment the pointer to the other half.
2602  unsigned IncrementSize = NVT.getSizeInBits()/8;
2603  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2604  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2605  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
2606  N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
2607  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2608 
2609  // Build a factor node to remember that this load is independent of the
2610  // other one.
2611  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2612  Hi.getValue(1));
2613  } else {
2614  // Big-endian - high bits are at low addresses. Favor aligned loads at
2615  // the cost of some bit-fiddling.
2616  EVT MemVT = N->getMemoryVT();
2617  unsigned EBytes = MemVT.getStoreSize();
2618  unsigned IncrementSize = NVT.getSizeInBits()/8;
2619  unsigned ExcessBits = (EBytes - IncrementSize)*8;
2620 
2621  // Load both the high bits and maybe some of the low bits.
2622  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
2624  MemVT.getSizeInBits() - ExcessBits),
2625  Alignment, MMOFlags, AAInfo);
2626 
2627  // Increment the pointer to the other half.
2628  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2629  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2630  // Load the rest of the low bits.
2631  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
2632  N->getPointerInfo().getWithOffset(IncrementSize),
2633  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
2634  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2635 
2636  // Build a factor node to remember that this load is independent of the
2637  // other one.
2638  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2639  Hi.getValue(1));
2640 
2641  if (ExcessBits < NVT.getSizeInBits()) {
2642  // Transfer low bits from the bottom of Hi to the top of Lo.
2643  Lo = DAG.getNode(
2644  ISD::OR, dl, NVT, Lo,
2645  DAG.getNode(ISD::SHL, dl, NVT, Hi,
2646  DAG.getConstant(ExcessBits, dl,
2647  TLI.getPointerTy(DAG.getDataLayout()))));
2648  // Move high bits to the right position in Hi.
2649  Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
2650  Hi,
2651  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
2652  TLI.getPointerTy(DAG.getDataLayout())));
2653  }
2654  }
2655 
2656  // Legalize the chain result - switch anything that used the old chain to
2657  // use the new one.
2658  ReplaceValueWith(SDValue(N, 1), Ch);
2659 }
2660 
2661 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
2662  SDValue &Lo, SDValue &Hi) {
2663  SDLoc dl(N);
2664  SDValue LL, LH, RL, RH;
2665  GetExpandedInteger(N->getOperand(0), LL, LH);
2666  GetExpandedInteger(N->getOperand(1), RL, RH);
2667  Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
2668  Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
2669 }
2670 
2671 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
2672  SDValue &Lo, SDValue &Hi) {
2673  EVT VT = N->getValueType(0);
2674  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2675  SDLoc dl(N);
2676 
2677  SDValue LL, LH, RL, RH;
2678  GetExpandedInteger(N->getOperand(0), LL, LH);
2679  GetExpandedInteger(N->getOperand(1), RL, RH);
2680 
2681  if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
2683  LL, LH, RL, RH))
2684  return;
2685 
2686  // If nothing else, we can make a libcall.
2687  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2688  if (VT == MVT::i16)
2689  LC = RTLIB::MUL_I16;
2690  else if (VT == MVT::i32)
2691  LC = RTLIB::MUL_I32;
2692  else if (VT == MVT::i64)
2693  LC = RTLIB::MUL_I64;
2694  else if (VT == MVT::i128)
2695  LC = RTLIB::MUL_I128;
2696 
2697  if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
2698  // We'll expand the multiplication by brute force because we have no other
2699  // options. This is a trivially-generalized version of the code from
2700  // Hacker's Delight (itself derived from Knuth's Algorithm M from section
2701  // 4.3.1).
2702  unsigned Bits = NVT.getSizeInBits();
2703  unsigned HalfBits = Bits >> 1;
2704  SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
2705  NVT);
2706  SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
2707  SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
2708 
2709  SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
2710  SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
2711 
2712  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2713  if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
2714  // The type from TLI is too small to fit the shift amount we want.
2715  // Override it with i32. The shift will have to be legalized.
2716  ShiftAmtTy = MVT::i32;
2717  }
2718  SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
2719  SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
2720  SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
2721  SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
2722 
2723  SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
2724  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
2725  SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
2726  SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
2727 
2728  SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
2729  DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
2730  SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
2731 
2732  SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
2733  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
2734  DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
2735  Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
2736  DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
2737 
2738  Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
2739  DAG.getNode(ISD::ADD, dl, NVT,
2740  DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
2741  DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
2742  return;
2743  }
2744 
2745  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2746  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true/*irrelevant*/, dl).first,
2747  Lo, Hi);
2748 }
2749 
2750 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
2751  SDValue &Hi) {
2752  SDLoc DL(N);
2753  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2754  SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
2755  SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
2756  Lo = R.getValue(0);
2757  Hi = R.getValue(1);
2758  ReplaceValueWith(SDValue(N, 1), R.getValue(2));
2759 }
2760 
2761 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
2762  SDValue &Hi) {
2763  SDValue Result = TLI.expandAddSubSat(N, DAG);
2764  SplitInteger(Result, Lo, Hi);
2765 }
2766 
2767 /// This performs an expansion of the integer result for a fixed point
2768 /// multiplication. The default expansion performs rounding down towards
2769 /// negative infinity, though targets that do care about rounding should specify
2770 /// a target hook for rounding and provide their own expansion or lowering of
2771 /// fixed point multiplication to be consistent with rounding.
2772 void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
2773  SDValue &Hi) {
2774  SDLoc dl(N);
2775  EVT VT = N->getValueType(0);
2776  unsigned VTSize = VT.getScalarSizeInBits();
2777  SDValue LHS = N->getOperand(0);
2778  SDValue RHS = N->getOperand(1);
2779  uint64_t Scale = N->getConstantOperandVal(2);
2780  bool Saturating = N->getOpcode() == ISD::SMULFIXSAT;
2781  EVT BoolVT =
2782  TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2783  SDValue Zero = DAG.getConstant(0, dl, VT);
2784  if (!Scale) {
2785  SDValue Result;
2786  if (!Saturating) {
2787  Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
2788  } else {
2789  Result = DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
2790  SDValue Product = Result.getValue(0);
2791  SDValue Overflow = Result.getValue(1);
2792 
2793  APInt MinVal = APInt::getSignedMinValue(VTSize);
2794  APInt MaxVal = APInt::getSignedMaxValue(VTSize);
2795  SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
2796  SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
2797  SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Product, Zero, ISD::SETLT);
2798  Result = DAG.getSelect(dl, VT, ProdNeg, SatMax, SatMin);
2799  Result = DAG.getSelect(dl, VT, Overflow, Result, Product);
2800  }
2801  SplitInteger(Result, Lo, Hi);
2802  return;
2803  }
2804 
2805  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2806  SDValue LL, LH, RL, RH;
2807  GetExpandedInteger(LHS, LL, LH);
2808  GetExpandedInteger(RHS, RL, RH);
2809  SmallVector<SDValue, 4> Result;
2810 
2811  bool Signed = (N->getOpcode() == ISD::SMULFIX ||
2812  N->getOpcode() == ISD::SMULFIXSAT);
2813  unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
2814  if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
2816  LL, LH, RL, RH)) {
2817  report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI.");
2818  return;
2819  }
2820 
2821  unsigned NVTSize = NVT.getScalarSizeInBits();
2822  assert((VTSize == NVTSize * 2) && "Expected the new value type to be half "
2823  "the size of the current value type");
2824  EVT ShiftTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2825 
2826  // Shift whole amount by scale.
2827  SDValue ResultLL = Result[0];
2828  SDValue ResultLH = Result[1];
2829  SDValue ResultHL = Result[2];
2830  SDValue ResultHH = Result[3];
2831 
2832  SDValue SatMax, SatMin;
2833  SDValue NVTZero = DAG.getConstant(0, dl, NVT);
2834  SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT);
2835  EVT BoolNVT =
2836  TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), NVT);
2837 
2838  // After getting the multplication result in 4 parts, we need to perform a
2839  // shift right by the amount of the scale to get the result in that scale.
2840  // Let's say we multiply 2 64 bit numbers. The resulting value can be held in
2841  // 128 bits that are cut into 4 32-bit parts:
2842  //
2843  // HH HL LH LL
2844  // |---32---|---32---|---32---|---32---|
2845  // 128 96 64 32 0
2846  //
2847  // |------VTSize-----|
2848  //
2849  // |NVTSize-|
2850  //
2851  // The resulting Lo and Hi will only need to be one of these 32-bit parts
2852  // after shifting.
2853  if (Scale < NVTSize) {
2854  // If the scale is less than the size of the VT we expand to, the Hi and
2855  // Lo of the result will be in the first 2 parts of the result after
2856  // shifting right. This only requires shifting by the scale as far as the
2857  // third part in the result (ResultHL).
2858  SDValue SRLAmnt = DAG.getConstant(Scale, dl, ShiftTy);
2859  SDValue SHLAmnt = DAG.getConstant(NVTSize - Scale, dl, ShiftTy);
2860  Lo = DAG.getNode(ISD::SRL, dl, NVT, ResultLL, SRLAmnt);
2861  Lo = DAG.getNode(ISD::OR, dl, NVT, Lo,
2862  DAG.getNode(ISD::SHL, dl, NVT, ResultLH, SHLAmnt));
2863  Hi = DAG.getNode(ISD::SRL, dl, NVT, ResultLH, SRLAmnt);
2864  Hi = DAG.getNode(ISD::OR, dl, NVT, Hi,
2865  DAG.getNode(ISD::SHL, dl, NVT, ResultHL, SHLAmnt));
2866 
2867  // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the
2868  // highest bit of HH determines saturation direction in the event of
2869  // saturation.
2870  // The number of overflow bits we can check are VTSize - Scale + 1 (we
2871  // include the sign bit). If these top bits are > 0, then we overflowed past
2872  // the max value. If these top bits are < -1, then we overflowed past the
2873  // min value. Otherwise, we did not overflow.
2874  if (Saturating) {
2875  unsigned OverflowBits = VTSize - Scale + 1;
2876  assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
2877  "Extent of overflow bits must start within HL");
2878  SDValue HLHiMask = DAG.getConstant(
2879  APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT);
2880  SDValue HLLoMask = DAG.getConstant(
2881  APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT);
2882 
2883  // HH > 0 or HH == 0 && HL > HLLoMask
2884  SDValue HHPos = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
2885  SDValue HHZero = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
2886  SDValue HLPos =
2887  DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT);
2888  SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHPos,
2889  DAG.getNode(ISD::AND, dl, BoolNVT, HHZero, HLPos));
2890 
2891  // HH < -1 or HH == -1 && HL < HLHiMask
2892  SDValue HHNeg = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
2893  SDValue HHNeg1 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
2894  SDValue HLNeg =
2895  DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT);
2896  SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHNeg,
2897  DAG.getNode(ISD::AND, dl, BoolNVT, HHNeg1, HLNeg));
2898  }
2899  } else if (Scale == NVTSize) {
2900  // If the scales are equal, Lo and Hi are ResultLH and Result HL,
2901  // respectively. Avoid shifting to prevent undefined behavior.
2902  Lo = ResultLH;
2903  Hi = ResultHL;
2904 
2905  // We overflow max if HH > 0 or HH == 0 && HL sign is negative.
2906  // We overflow min if HH < -1 or HH == -1 && HL sign is 0.
2907  if (Saturating) {
2908  SDValue HHPos = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
2909  SDValue HHZero = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
2910  SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT);
2911  SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHPos,
2912  DAG.getNode(ISD::AND, dl, BoolNVT, HHZero, HLNeg));
2913 
2914  SDValue HHNeg = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
2915  SDValue HHNeg1 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
2916  SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGT);
2917  SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHNeg,
2918  DAG.getNode(ISD::AND, dl, BoolNVT, HHNeg1, HLPos));
2919  }
2920  } else if (Scale < VTSize) {
2921  // If the scale is instead less than the old VT size, but greater than or
2922  // equal to the expanded VT size, the first part of the result (ResultLL) is
2923  // no longer a part of Lo because it would be scaled out anyway. Instead we
2924  // can start shifting right from the fourth part (ResultHH) to the second
2925  // part (ResultLH), and Result LH will be the new Lo.
2926  SDValue SRLAmnt = DAG.getConstant(Scale - NVTSize, dl, ShiftTy);
2927  SDValue SHLAmnt = DAG.getConstant(VTSize - Scale, dl, ShiftTy);
2928  Lo = DAG.getNode(ISD::SRL, dl, NVT, ResultLH, SRLAmnt);
2929  Lo = DAG.getNode(ISD::OR, dl, NVT, Lo,
2930  DAG.getNode(ISD::SHL, dl, NVT, ResultHL, SHLAmnt));
2931  Hi = DAG.getNode(ISD::SRL, dl, NVT, ResultHL, SRLAmnt);
2932  Hi = DAG.getNode(ISD::OR, dl, NVT, Hi,
2933  DAG.getNode(ISD::SHL, dl, NVT, ResultHH, SHLAmnt));
2934 
2935  // This is similar to the case when we saturate if Scale < NVTSize, but we
2936  // only need to chech HH.
2937  if (Saturating) {
2938  unsigned OverflowBits = VTSize - Scale + 1;
2939  SDValue HHHiMask = DAG.getConstant(
2940  APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT);
2941  SDValue HHLoMask = DAG.getConstant(
2942  APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT);
2943 
2944  SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT);
2945  SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT);
2946  }
2947  } else if (Scale == VTSize) {
2948  assert(
2949  !Signed &&
2950  "Only unsigned types can have a scale equal to the operand bit width");
2951 
2952  Lo = ResultHL;
2953  Hi = ResultHH;
2954  } else {
2955  llvm_unreachable("Expected the scale to be less than or equal to the width "
2956  "of the operands");
2957  }
2958 
2959  if (Saturating) {
2960  APInt LHMax = APInt::getSignedMaxValue(NVTSize);
2961  APInt LLMax = APInt::getAllOnesValue(NVTSize);
2962  APInt LHMin = APInt::getSignedMinValue(NVTSize);
2963  Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(LHMax, dl, NVT), Hi);
2964  Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(LHMin, dl, NVT), Hi);
2965  Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(LLMax, dl, NVT), Lo);
2966  Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo);
2967  }
2968 }
2969 
2970 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
2971  SDValue &Lo, SDValue &Hi) {
2972  SDValue LHS = Node->getOperand(0);
2973  SDValue RHS = Node->getOperand(1);
2974  SDLoc dl(Node);
2975 
2976  // Expand the result by simply replacing it with the equivalent
2977  // non-overflow-checking operation.
2978  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
2979  ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
2980  LHS, RHS);
2981  SplitInteger(Sum, Lo, Hi);
2982 
2983  // Compute the overflow.
2984  //
2985  // LHSSign -> LHS >= 0
2986  // RHSSign -> RHS >= 0
2987  // SumSign -> Sum >= 0
2988  //
2989  // Add:
2990  // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
2991  // Sub:
2992  // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
2993  //
2994  EVT OType = Node->getValueType(1);
2995  SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
2996 
2997  SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
2998  SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
2999  SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
3000  Node->getOpcode() == ISD::SADDO ?
3002 
3003  SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3004  SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3005 
3006  SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3007 
3008  // Use the calculated overflow everywhere.
3009  ReplaceValueWith(SDValue(Node, 1), Cmp);
3010 }
3011 
3012 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
3013  SDValue &Lo, SDValue &Hi) {
3014  EVT VT = N->getValueType(0);
3015  SDLoc dl(N);
3016  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3017 
3019  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3020  SplitInteger(Res.getValue(0), Lo, Hi);
3021  return;
3022  }
3023 
3024  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3025  if (VT == MVT::i16)
3026  LC = RTLIB::SDIV_I16;
3027  else if (VT == MVT::i32)
3028  LC = RTLIB::SDIV_I32;
3029  else if (VT == MVT::i64)
3030  LC = RTLIB::SDIV_I64;
3031  else if (VT == MVT::i128)
3032  LC = RTLIB::SDIV_I128;
3033  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
3034 
3035  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
3036 }
3037 
3038 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
3039  SDValue &Lo, SDValue &Hi) {
3040  EVT VT = N->getValueType(0);
3041  SDLoc dl(N);
3042 
3043  // If we can emit an efficient shift operation, do so now. Check to see if
3044  // the RHS is a constant.
3045  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
3046  return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
3047 
3048  // If we can determine that the high bit of the shift is zero or one, even if
3049  // the low bits are variable, emit this shift in an optimized form.
3050  if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
3051  return;
3052 
3053  // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
3054  unsigned PartsOpc;
3055  if (N->getOpcode() == ISD::SHL) {
3056  PartsOpc = ISD::SHL_PARTS;
3057  } else if (N->getOpcode() == ISD::SRL) {
3058  PartsOpc = ISD::SRL_PARTS;
3059  } else {
3060  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3061  PartsOpc = ISD::SRA_PARTS;
3062  }
3063 
3064  // Next check to see if the target supports this SHL_PARTS operation or if it
3065  // will custom expand it. Don't lower this to SHL_PARTS when we optimise for
3066  // size, but create a libcall instead.
3067  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3068  TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
3069  const bool LegalOrCustom =
3070  (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
3071  Action == TargetLowering::Custom;
3072 
3073  if (LegalOrCustom && TLI.shouldExpandShift(DAG, N)) {
3074  // Expand the subcomponents.
3075  SDValue LHSL, LHSH;
3076  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3077  EVT VT = LHSL.getValueType();
3078 
3079  // If the shift amount operand is coming from a vector legalization it may
3080  // have an illegal type. Fix that first by casting the operand, otherwise
3081  // the new SHL_PARTS operation would need further legalization.
3082  SDValue ShiftOp = N->getOperand(1);
3083  EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3084  assert(ShiftTy.getScalarSizeInBits() >=
3085  Log2_32_Ceil(VT.getScalarSizeInBits()) &&
3086  "ShiftAmountTy is too small to cover the range of this type!");
3087  if (ShiftOp.getValueType() != ShiftTy)
3088  ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
3089 
3090  SDValue Ops[] = { LHSL, LHSH, ShiftOp };
3091  Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
3092  Hi = Lo.getValue(1);
3093  return;
3094  }
3095 
3096  // Otherwise, emit a libcall.
3097  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3098  bool isSigned;
3099  if (N->getOpcode() == ISD::SHL) {
3100  isSigned = false; /*sign irrelevant*/
3101  if (VT == MVT::i16)
3102  LC = RTLIB::SHL_I16;
3103  else if (VT == MVT::i32)
3104  LC = RTLIB::SHL_I32;
3105  else if (VT == MVT::i64)
3106  LC = RTLIB::SHL_I64;
3107  else if (VT == MVT::i128)
3108  LC = RTLIB::SHL_I128;
3109  } else if (N->getOpcode() == ISD::SRL) {
3110  isSigned = false;
3111  if (VT == MVT::i16)
3112  LC = RTLIB::SRL_I16;
3113  else if (VT == MVT::i32)
3114  LC = RTLIB::SRL_I32;
3115  else if (VT == MVT::i64)
3116  LC = RTLIB::SRL_I64;
3117  else if (VT == MVT::i128)
3118  LC = RTLIB::SRL_I128;
3119  } else {
3120  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3121  isSigned = true;
3122  if (VT == MVT::i16)
3123  LC = RTLIB::SRA_I16;
3124  else if (VT == MVT::i32)
3125  LC = RTLIB::SRA_I32;
3126  else if (VT == MVT::i64)
3127  LC = RTLIB::SRA_I64;
3128  else if (VT == MVT::i128)
3129  LC = RTLIB::SRA_I128;
3130  }
3131 
3132  if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
3133  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3134  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, isSigned, dl).first, Lo, Hi);
3135  return;
3136  }
3137 
3138  if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
3139  llvm_unreachable("Unsupported shift!");
3140 }
3141 
3142 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
3143  SDValue &Lo, SDValue &Hi) {
3144  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3145  SDLoc dl(N);
3146  SDValue Op = N->getOperand(0);
3147  if (Op.getValueType().bitsLE(NVT)) {
3148  // The low part is sign extension of the input (degenerates to a copy).
3149  Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
3150  // The high part is obtained by SRA'ing all but one of the bits of low part.
3151  unsigned LoSize = NVT.getSizeInBits();
3152  Hi = DAG.getNode(
3153  ISD::SRA, dl, NVT, Lo,
3154  DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
3155  } else {
3156  // For example, extension of an i48 to an i64. The operand type necessarily
3157  // promotes to the result type, so will end up being expanded too.
3158  assert(getTypeAction(Op.getValueType()) ==
3160  "Only know how to promote this result!");
3161  SDValue Res = GetPromotedInteger(Op);
3162  assert(Res.getValueType() == N->getValueType(0) &&
3163  "Operand over promoted?");
3164  // Split the promoted operand. This will simplify when it is expanded.
3165  SplitInteger(Res, Lo, Hi);
3166  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
3167  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
3169  ExcessBits)));
3170  }
3171 }
3172 
3173 void DAGTypeLegalizer::
3174 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
3175  SDLoc dl(N);
3176  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3177  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3178 
3179  if (EVT.bitsLE(Lo.getValueType())) {
3180  // sext_inreg the low part if needed.
3181  Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
3182  N->getOperand(1));
3183 
3184  // The high part gets the sign extension from the lo-part. This handles
3185  // things like sextinreg V:i64 from i8.
3186  Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
3187  DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
3188  TLI.getPointerTy(DAG.getDataLayout())));
3189  } else {
3190  // For example, extension of an i48 to an i64. Leave the low part alone,
3191  // sext_inreg the high part.
3192  unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
3193  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
3195  ExcessBits)));
3196  }
3197 }
3198 
3199 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
3200  SDValue &Lo, SDValue &Hi) {
3201  EVT VT = N->getValueType(0);
3202  SDLoc dl(N);
3203  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3204 
3206  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3207  SplitInteger(Res.getValue(1), Lo, Hi);
3208  return;
3209  }
3210 
3211  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3212  if (VT == MVT::i16)
3213  LC = RTLIB::SREM_I16;
3214  else if (VT == MVT::i32)
3215  LC = RTLIB::SREM_I32;
3216  else if (VT == MVT::i64)
3217  LC = RTLIB::SREM_I64;
3218  else if (VT == MVT::i128)
3219  LC = RTLIB::SREM_I128;
3220  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
3221 
3222  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
3223 }
3224 
3225 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
3226  SDValue &Lo, SDValue &Hi) {
3227  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3228  SDLoc dl(N);
3229  Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
3230  Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
3231  N->getOperand(0),
3232  DAG.getConstant(NVT.getSizeInBits(), dl,
3233  TLI.getPointerTy(DAG.getDataLayout())));
3234  Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
3235 }
3236 
3237 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
3238  SDValue &Lo, SDValue &Hi) {
3239  EVT VT = N->getValueType(0);
3240  SDLoc dl(N);
3241 
3242  if (N->getOpcode() == ISD::UMULO) {
3243  // This section expands the operation into the following sequence of
3244  // instructions. `iNh` here refers to a type which has half the bit width of
3245  // the type the original operation operated on.
3246  //
3247  // %0 = %LHS.HI != 0 && %RHS.HI != 0
3248  // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
3249  // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
3250  // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
3251  // %4 = add iN (%1.0 as iN) << Nh, (%2.0 as iN) << Nh
3252  // %5 = { iN, i1 } @uadd.with.overflow.iN( %4, %3 )
3253  //
3254  // %res = { %5.0, %0 || %1.1 || %2.1 || %5.1 }
3255  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
3256  SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
3257  SplitInteger(LHS, LHSLow, LHSHigh);
3258  SplitInteger(RHS, RHSLow, RHSHigh);
3259  EVT HalfVT = LHSLow.getValueType()
3260  , BitVT = N->getValueType(1);
3261  SDVTList VTHalfMulO = DAG.getVTList(HalfVT, BitVT);
3262  SDVTList VTFullAddO = DAG.getVTList(VT, BitVT);
3263 
3264  SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
3265  SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
3266  DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
3267  DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));
3268 
3269  SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, LHSHigh, RHSLow);
3270  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));
3271  SDValue OneInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3272  One.getValue(0));
3273 
3274  SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, RHSHigh, LHSLow);
3275  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));
3276  SDValue TwoInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3277  Two.getValue(0));
3278 
3279  // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
3280  // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
3281  // operation recursively legalized?).
3282  //
3283  // Many backends understand this pattern and will convert into LOHI
3284  // themselves, if applicable.
3285  SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
3286  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
3287  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow));
3288  SDValue Four = DAG.getNode(ISD::ADD, dl, VT, OneInHigh, TwoInHigh);
3289  SDValue Five = DAG.getNode(ISD::UADDO, dl, VTFullAddO, Three, Four);
3290  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Five.getValue(1));
3291  SplitInteger(Five, Lo, Hi);
3292  ReplaceValueWith(SDValue(N, 1), Overflow);
3293  return;
3294  }
3295 
3296  Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
3297  EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
3298  Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
3299 
3300  // Replace this with a libcall that will check overflow.
3301  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3302  if (VT == MVT::i32)
3303  LC = RTLIB::MULO_I32;
3304  else if (VT == MVT::i64)
3305  LC = RTLIB::MULO_I64;
3306  else if (VT == MVT::i128)
3307  LC = RTLIB::MULO_I128;
3308  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
3309 
3310  SDValue Temp = DAG.CreateStackTemporary(PtrVT);
3311  // Temporary for the overflow value, default it to zero.
3312  SDValue Chain =
3313  DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
3314  MachinePointerInfo());
3315 
3317  TargetLowering::ArgListEntry Entry;
3318  for (const SDValue &Op : N->op_values()) {
3319  EVT ArgVT = Op.getValueType();
3320  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
3321  Entry.Node = Op;
3322  Entry.Ty = ArgTy;
3323  Entry.IsSExt = true;
3324  Entry.IsZExt = false;
3325  Args.push_back(Entry);
3326  }
3327 
3328  // Also pass the address of the overflow check.
3329  Entry.Node = Temp;
3330  Entry.Ty = PtrTy->getPointerTo();
3331  Entry.IsSExt = true;
3332  Entry.IsZExt = false;
3333  Args.push_back(Entry);
3334 
3335  SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
3336 
3338  CLI.setDebugLoc(dl)
3339  .setChain(Chain)
3340  .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
3341  .setSExtResult();
3342 
3343  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
3344 
3345  SplitInteger(CallInfo.first, Lo, Hi);
3346  SDValue Temp2 =
3347  DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
3348  SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
3349  DAG.getConstant(0, dl, PtrVT),
3350  ISD::SETNE);
3351  // Use the overflow from the libcall everywhere.
3352  ReplaceValueWith(SDValue(N, 1), Ofl);
3353 }
3354 
3355 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
3356  SDValue &Lo, SDValue &Hi) {
3357  EVT VT = N->getValueType(0);
3358  SDLoc dl(N);
3359  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3360 
3362  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3363  SplitInteger(Res.getValue(0), Lo, Hi);
3364  return;
3365  }
3366 
3367  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3368  if (VT == MVT::i16)
3369  LC = RTLIB::UDIV_I16;
3370  else if (VT == MVT::i32)
3371  LC = RTLIB::UDIV_I32;
3372  else if (VT == MVT::i64)
3373  LC = RTLIB::UDIV_I64;
3374  else if (VT == MVT::i128)
3375  LC = RTLIB::UDIV_I128;
3376  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
3377 
3378  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
3379 }
3380 
3381 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
3382  SDValue &Lo, SDValue &Hi) {
3383  EVT VT = N->getValueType(0);
3384  SDLoc dl(N);
3385  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3386 
3388  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3389  SplitInteger(Res.getValue(1), Lo, Hi);
3390  return;
3391  }
3392 
3393  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3394  if (VT == MVT::i16)
3395  LC = RTLIB::UREM_I16;
3396  else if (VT == MVT::i32)
3397  LC = RTLIB::UREM_I32;
3398  else if (VT == MVT::i64)
3399  LC = RTLIB::UREM_I64;
3400  else if (VT == MVT::i128)
3401  LC = RTLIB::UREM_I128;
3402  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
3403 
3404  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
3405 }
3406 
3407 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
3408  SDValue &Lo, SDValue &Hi) {
3409  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3410  SDLoc dl(N);
3411  SDValue Op = N->getOperand(0);
3412  if (Op.getValueType().bitsLE(NVT)) {
3413  // The low part is zero extension of the input (degenerates to a copy).
3414  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
3415  Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
3416  } else {
3417  // For example, extension of an i48 to an i64. The operand type necessarily
3418  // promotes to the result type, so will end up being expanded too.
3419  assert(getTypeAction(Op.getValueType()) ==
3421  "Only know how to promote this result!");
3422  SDValue Res = GetPromotedInteger(Op);
3423  assert(Res.getValueType() == N->getValueType(0) &&
3424  "Operand over promoted?");
3425  // Split the promoted operand. This will simplify when it is expanded.
3426  SplitInteger(Res, Lo, Hi);
3427  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
3428  Hi = DAG.getZeroExtendInReg(Hi, dl,
3430  ExcessBits));
3431  }
3432 }
3433 
3434 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
3435  SDValue &Lo, SDValue &Hi) {
3436  SDLoc dl(N);
3437  EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
3438  SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
3439  SDValue Zero = DAG.getConstant(0, dl, VT);
3440  SDValue Swap = DAG.getAtomicCmpSwap(
3442  cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
3443  N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
3444 
3445  ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
3446  ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
3447 }
3448 
3449 void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N,
3450  SDValue &Lo, SDValue &Hi) {
3451  // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate
3452  // both halves independently.
3453  SDValue Res = TLI.expandVecReduce(N, DAG);
3454  SplitInteger(Res, Lo, Hi);
3455 }
3456 
3457 //===----------------------------------------------------------------------===//
3458 // Integer Operand Expansion
3459 //===----------------------------------------------------------------------===//
3460 
3461 /// ExpandIntegerOperand - This method is called when the specified operand of
3462 /// the specified node is found to need expansion. At this point, all of the
3463 /// result types of the node are known to be legal, but other operands of the
3464 /// node may need promotion or expansion as well as the specified one.
3465 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
3466  LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG);
3467  dbgs() << "\n");
3468  SDValue Res = SDValue();
3469 
3470  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
3471  return false;
3472 
3473  switch (N->getOpcode()) {
3474  default:
3475  #ifndef NDEBUG
3476  dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
3477  N->dump(&DAG); dbgs() << "\n";
3478  #endif
3479  report_fatal_error("Do not know how to expand this operator's operand!");
3480 
3481  case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
3482  case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
3483  case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
3484  case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
3485  case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
3486  case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
3487  case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
3488  case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
3489  case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
3490  case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break;
3491  case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
3492  case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
3493  case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break;
3494 
3495  case ISD::SHL:
3496  case ISD::SRA:
3497  case ISD::SRL:
3498  case ISD::ROTL:
3499  case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
3500  case ISD::RETURNADDR:
3501  case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
3502 
3503  case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
3504  }
3505 
3506  // If the result is null, the sub-method took care of registering results etc.
3507  if (!Res.getNode()) return false;
3508 
3509  // If the result is N, the sub-method updated N in place. Tell the legalizer
3510  // core about this.
3511  if (Res.getNode() == N)
3512  return true;
3513 
3514  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3515  "Invalid operand expansion");
3516 
3517  ReplaceValueWith(SDValue(N, 0), Res);
3518  return false;
3519 }
3520 
3521 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
3522 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
3523 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
3524  SDValue &NewRHS,
3525  ISD::CondCode &CCCode,
3526  const SDLoc &dl) {
3527  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3528  GetExpandedInteger(NewLHS, LHSLo, LHSHi);
3529  GetExpandedInteger(NewRHS, RHSLo, RHSHi);
3530 
3531  if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
3532  if (RHSLo == RHSHi) {
3533  if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
3534  if (RHSCST->isAllOnesValue()) {
3535  // Equality comparison to -1.
3536  NewLHS = DAG.getNode(ISD::AND, dl,
3537  LHSLo.getValueType(), LHSLo, LHSHi);
3538  NewRHS = RHSLo;
3539  return;
3540  }
3541  }
3542  }
3543 
3544  NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
3545  NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
3546  NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
3547  NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
3548  return;
3549  }
3550 
3551  // If this is a comparison of the sign bit, just look at the top part.
3552  // X > -1, x < 0
3553  if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
3554  if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0
3555  (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1
3556  NewLHS = LHSHi;
3557  NewRHS = RHSHi;
3558  return;
3559  }
3560 
3561  // FIXME: This generated code sucks.
3562  ISD::CondCode LowCC;
3563  switch (CCCode) {
3564  default: llvm_unreachable("Unknown integer setcc!");
3565  case ISD::SETLT:
3566  case ISD::SETULT: LowCC = ISD::SETULT; break;
3567  case ISD::SETGT:
3568  case ISD::SETUGT: LowCC = ISD::SETUGT; break;
3569  case ISD::SETLE:
3570  case ISD::SETULE: LowCC = ISD::SETULE; break;
3571  case ISD::SETGE:
3572  case ISD::SETUGE: LowCC = ISD::SETUGE; break;
3573  }
3574 
3575  // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
3576  // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
3577  // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
3578 
3579  // NOTE: on targets without efficient SELECT of bools, we can always use
3580  // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
3581  TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
3582  nullptr);
3583  SDValue LoCmp, HiCmp;
3584  if (TLI.isTypeLegal(LHSLo.getValueType()) &&
3585  TLI.isTypeLegal(RHSLo.getValueType()))
3586  LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
3587  RHSLo, LowCC, false, DagCombineInfo, dl);
3588  if (!LoCmp.getNode())
3589  LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
3590  RHSLo, LowCC);
3591  if (TLI.isTypeLegal(LHSHi.getValueType()) &&
3592  TLI.isTypeLegal(RHSHi.getValueType()))
3593  HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
3594  RHSHi, CCCode, false, DagCombineInfo, dl);
3595  if (!HiCmp.getNode())
3596  HiCmp =
3597  DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
3598  LHSHi, RHSHi, DAG.getCondCode(CCCode));
3599 
3600  ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
3601  ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
3602 
3603  bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
3604  CCCode == ISD::SETUGE || CCCode == ISD::SETULE);
3605 
3606  if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) ||
3607  (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) ||
3608  (LoCmpC && LoCmpC->isNullValue())))) {
3609  // For LE / GE, if high part is known false, ignore the low part.
3610  // For LT / GT: if low part is known false, return the high part.
3611  // if high part is known true, ignore the low part.
3612  NewLHS = HiCmp;
3613  NewRHS = SDValue();
3614  return;
3615  }
3616 
3617  if (LHSHi == RHSHi) {
3618  // Comparing the low bits is enough.
3619  NewLHS = LoCmp;
3620  NewRHS = SDValue();
3621  return;
3622  }
3623 
3624  // Lower with SETCCCARRY if the target supports it.
3625  EVT HiVT = LHSHi.getValueType();
3626  EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
3627  bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
3628 
3629  // FIXME: Make all targets support this, then remove the other lowering.
3630  if (HasSETCCCARRY) {
3631  // SETCCCARRY can detect < and >= directly. For > and <=, flip
3632  // operands and condition code.
3633  bool FlipOperands = false;
3634  switch (CCCode) {
3635  case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
3636  case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
3637  case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
3638  case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
3639  default: break;
3640  }
3641  if (FlipOperands) {
3642  std::swap(LHSLo, RHSLo);
3643  std::swap(LHSHi, RHSHi);
3644  }
3645  // Perform a wide subtraction, feeding the carry from the low part into
3646  // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
3647  // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
3648  // zero or positive iff LHS >= RHS.
3649  EVT LoVT = LHSLo.getValueType();
3650  SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
3651  SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
3652  SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
3653  LHSHi, RHSHi, LowCmp.getValue(1),
3654  DAG.getCondCode(CCCode));
3655  NewLHS = Res;
3656  NewRHS = SDValue();
3657  return;
3658  }
3659 
3660  NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
3661  false, DagCombineInfo, dl);
3662  if (!NewLHS.getNode())
3663  NewLHS =
3664  DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
3665  NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
3666  NewRHS = SDValue();
3667 }
3668 
3669 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
3670  SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
3671  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
3672  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3673 
3674  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3675  // against zero to select between true and false values.
3676  if (!NewRHS.getNode()) {
3677  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3678  CCCode = ISD::SETNE;
3679  }
3680 
3681  // Update N to have the operands specified.
3682  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
3683  DAG.getCondCode(CCCode), NewLHS, NewRHS,
3684  N->getOperand(4)), 0);
3685 }
3686 
3687 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
3688  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3689  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
3690  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3691 
3692  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3693  // against zero to select between true and false values.
3694  if (!NewRHS.getNode()) {
3695  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3696  CCCode = ISD::SETNE;
3697  }
3698 
3699  // Update N to have the operands specified.
3700  return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
3701  N->getOperand(2), N->getOperand(3),
3702  DAG.getCondCode(CCCode)), 0);
3703 }
3704 
3705 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
3706  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3707  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3708  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3709 
3710  // If ExpandSetCCOperands returned a scalar, use it.
3711  if (!NewRHS.getNode()) {
3712  assert(NewLHS.getValueType() == N->getValueType(0) &&
3713  "Unexpected setcc expansion!");
3714  return NewLHS;
3715  }
3716 
3717  // Otherwise, update N to have the operands specified.
3718  return SDValue(
3719  DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
3720 }
3721 
3722 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
3723  SDValue LHS = N->getOperand(0);
3724  SDValue RHS = N->getOperand(1);
3725  SDValue Carry = N->getOperand(2);
3726  SDValue Cond = N->getOperand(3);
3727  SDLoc dl = SDLoc(N);
3728 
3729  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3730  GetExpandedInteger(LHS, LHSLo, LHSHi);
3731  GetExpandedInteger(RHS, RHSLo, RHSHi);
3732 
3733  // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
3734  SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
3735  SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
3736  return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
3737  LowCmp.getValue(1), Cond);
3738 }
3739 
3740 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
3741  // The value being shifted is legal, but the shift amount is too big.
3742  // It follows that either the result of the shift is undefined, or the
3743  // upper half of the shift amount is zero. Just use the lower half.
3744  SDValue Lo, Hi;
3745  GetExpandedInteger(N->getOperand(1), Lo, Hi);
3746  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
3747 }
3748 
3749 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
3750  // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
3751  // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
3752  // constant to valid type.
3753  SDValue Lo, Hi;
3754  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3755  return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
3756 }
3757 
3758 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
3759  SDValue Op = N->getOperand(0);
3760  EVT DstVT = N->getValueType(0);
3762  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3763  "Don't know how to expand this SINT_TO_FP!");
3764  return TLI.makeLibCall(DAG, LC, DstVT, Op, true, SDLoc(N)).first;
3765 }
3766 
3767 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
3768  if (ISD::isNormalStore(N))
3769  return ExpandOp_NormalStore(N, OpNo);
3770 
3771  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
3772  assert(OpNo == 1 && "Can only expand the stored value so far");
3773 
3774  EVT VT = N->getOperand(1).getValueType();
3775  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3776  SDValue Ch = N->getChain();
3777  SDValue Ptr = N->getBasePtr();
3778  unsigned Alignment = N->getAlignment();
3779  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3780  AAMDNodes AAInfo = N->getAAInfo();
3781  SDLoc dl(N);
3782  SDValue Lo, Hi;
3783 
3784  assert(NVT.isByteSized() && "Expanded type not byte sized!");
3785 
3786  if (N->getMemoryVT().bitsLE(NVT)) {
3787  GetExpandedInteger(N->getValue(), Lo, Hi);
3788  return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
3789  N->getMemoryVT(), Alignment, MMOFlags, AAInfo);
3790  }
3791 
3792  if (DAG.getDataLayout().isLittleEndian()) {
3793  // Little-endian - low bits are at low addresses.
3794  GetExpandedInteger(N->getValue(), Lo, Hi);
3795 
3796  Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3797  AAInfo);
3798 
3799  unsigned ExcessBits =
3800  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3801  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3802 
3803  // Increment the pointer to the other half.
3804  unsigned IncrementSize = NVT.getSizeInBits()/8;
3805  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3806  Hi = DAG.getTruncStore(
3807  Ch, dl, Hi, Ptr, N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3808  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3809  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3810  }
3811 
3812  // Big-endian - high bits are at low addresses. Favor aligned stores at
3813  // the cost of some bit-fiddling.
3814  GetExpandedInteger(N->getValue(), Lo, Hi);
3815 
3816  EVT ExtVT = N->getMemoryVT();
3817  unsigned EBytes = ExtVT.getStoreSize();
3818  unsigned IncrementSize = NVT.getSizeInBits()/8;
3819  unsigned ExcessBits = (EBytes - IncrementSize)*8;
3820  EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
3821  ExtVT.getSizeInBits() - ExcessBits);
3822 
3823  if (ExcessBits < NVT.getSizeInBits()) {
3824  // Transfer high bits from the top of Lo to the bottom of Hi.
3825  Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
3826  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3827  TLI.getPointerTy(DAG.getDataLayout())));
3828  Hi = DAG.getNode(
3829  ISD::OR, dl, NVT, Hi,
3830  DAG.getNode(ISD::SRL, dl, NVT, Lo,
3831  DAG.getConstant(ExcessBits, dl,
3832  TLI.getPointerTy(DAG.getDataLayout()))));
3833  }
3834 
3835  // Store both the high bits and maybe some of the low bits.
3836  Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment,
3837  MMOFlags, AAInfo);
3838 
3839  // Increment the pointer to the other half.
3840  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3841  // Store the lowest ExcessBits bits in the second half.
3842  Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
3843  N->getPointerInfo().getWithOffset(IncrementSize),
3844  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3845  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3846  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3847 }
3848 
3849 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
3850  SDValue InL, InH;
3851  GetExpandedInteger(N->getOperand(0), InL, InH);
3852  // Just truncate the low part of the source.
3853  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
3854 }
3855 
3856 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
3857  SDValue Op = N->getOperand(0);
3858  EVT SrcVT = Op.getValueType();
3859  EVT DstVT = N->getValueType(0);
3860  SDLoc dl(N);
3861 
3862  // The following optimization is valid only if every value in SrcVT (when
3863  // treated as signed) is representable in DstVT. Check that the mantissa
3864  // size of DstVT is >= than the number of bits in SrcVT -1.
3865  const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT);
3866  if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 &&
3868  // Do a signed conversion then adjust the result.
3869  SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op);
3870  SignedConv = TLI.LowerOperation(SignedConv, DAG);
3871 
3872  // The result of the signed conversion needs adjusting if the 'sign bit' of
3873  // the incoming integer was set. To handle this, we dynamically test to see
3874  // if it is set, and, if so, add a fudge factor.
3875 
3876  const uint64_t F32TwoE32 = 0x4F800000ULL;
3877  const uint64_t F32TwoE64 = 0x5F800000ULL;
3878  const uint64_t F32TwoE128 = 0x7F800000ULL;
3879 
3880  APInt FF(32, 0);
3881  if (SrcVT == MVT::i32)
3882  FF = APInt(32, F32TwoE32);
3883  else if (SrcVT == MVT::i64)
3884  FF = APInt(32, F32TwoE64);
3885  else if (SrcVT == MVT::i128)
3886  FF = APInt(32, F32TwoE128);
3887  else
3888  llvm_unreachable("Unsupported UINT_TO_FP!");
3889 
3890  // Check whether the sign bit is set.
3891  SDValue Lo, Hi;
3892  GetExpandedInteger(Op, Lo, Hi);
3893  SDValue SignSet = DAG.getSetCC(dl,
3894  getSetCCResultType(Hi.getValueType()),
3895  Hi,
3896  DAG.getConstant(0, dl, Hi.getValueType()),
3897  ISD::SETLT);
3898 
3899  // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
3900  SDValue FudgePtr =
3901  DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)),
3902  TLI.getPointerTy(DAG.getDataLayout()));
3903 
3904  // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
3905  SDValue Zero = DAG.getIntPtrConstant(0, dl);
3906  SDValue Four = DAG.getIntPtrConstant(4, dl);
3907  if (DAG.getDataLayout().isBigEndian())
3908  std::swap(Zero, Four);
3909  SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet,
3910  Zero, Four);
3911  unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment();
3912  FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(),
3913  FudgePtr, Offset);
3914  Alignment = std::min(Alignment, 4u);
3915 
3916  // Load the value out, extending it from f32 to the destination float type.
3917  // FIXME: Avoid the extend by constructing the right constant pool?
3918  SDValue Fudge = DAG.getExtLoad(
3919  ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), FudgePtr,
3921  Alignment);
3922  return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge);
3923  }
3924 
3925  // Otherwise, use a libcall.
3926  RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT);
3927  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3928  "Don't know how to expand this UINT_TO_FP!");
3929  return TLI.makeLibCall(DAG, LC, DstVT, Op, true, dl).first;
3930 }
3931 
3932 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
3933  SDLoc dl(N);
3934  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
3935  cast<AtomicSDNode>(N)->getMemoryVT(),
3936  N->getOperand(0),
3937  N->getOperand(1), N->getOperand(2),
3938  cast<AtomicSDNode>(N)->getMemOperand());
3939  return Swap.getValue(1);
3940 }
3941 
3942 
3943 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
3944 
3945  EVT OutVT = N->getValueType(0);
3946  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3947  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3948  unsigned OutNumElems = OutVT.getVectorNumElements();
3949  EVT NOutVTElem = NOutVT.getVectorElementType();
3950 
3951  SDLoc dl(N);
3952  SDValue BaseIdx = N->getOperand(1);
3953 
3954  SDValue InOp0 = N->getOperand(0);
3955  if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger)
3956  InOp0 = GetPromotedInteger(N->getOperand(0));
3957 
3958  EVT InVT = InOp0.getValueType();
3959 
3961  Ops.reserve(OutNumElems);
3962  for (unsigned i = 0; i != OutNumElems; ++i) {
3963 
3964  // Extract the element from the original vector.
3965  SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
3966  BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
3968  InVT.getVectorElementType(), N->getOperand(0), Index);
3969 
3970  SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem);
3971  // Insert the converted element to the new vector.
3972  Ops.push_back(Op);
3973  }
3974 
3975  return DAG.getBuildVector(NOutVT, dl, Ops);
3976 }
3977 
3978 
3979 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
3980  ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
3981  EVT VT = N->getValueType(0);
3982  SDLoc dl(N);
3983 
3984  ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
3985 
3986  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3987  SDValue V1 = GetPromotedInteger(N->getOperand(1));
3988  EVT OutVT = V0.getValueType();
3989 
3990  return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
3991 }
3992 
3993 
3994 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
3995  EVT OutVT = N->getValueType(0);
3996  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3997  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3998  unsigned NumElems = N->getNumOperands();
3999  EVT NOutVTElem = NOutVT.getVectorElementType();
4000 
4001  SDLoc dl(N);
4002 
4004  Ops.reserve(NumElems);
4005  for (unsigned i = 0; i != NumElems; ++i) {
4006  SDValue Op;
4007  // BUILD_VECTOR integer operand types are allowed to be larger than the
4008  // result's element type. This may still be true after the promotion. For
4009  // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
4010  // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
4011  if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
4012  Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
4013  else
4014  Op = N->getOperand(i);
4015  Ops.push_back(Op);
4016  }
4017 
4018  return DAG.getBuildVector(NOutVT, dl, Ops);
4019 }
4020 
4021 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
4022 
4023  SDLoc dl(N);
4024 
4025  assert(!N->getOperand(0).getValueType().isVector() &&
4026  "Input must be a scalar");
4027 
4028  EVT OutVT = N->getValueType(0);
4029  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4030  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4031  EVT NOutVTElem = NOutVT.getVectorElementType();
4032 
4033  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
4034 
4035  return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
4036 }
4037 
4038 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
4039  SDLoc dl(N);
4040 
4041  EVT OutVT = N->getValueType(0);
4042  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4043  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4044 
4045  EVT OutElemTy = NOutVT.getVectorElementType();
4046 
4047  unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
4048  unsigned NumOutElem = NOutVT.getVectorNumElements();
4049  unsigned NumOperands = N->getNumOperands();
4050  assert(NumElem * NumOperands == NumOutElem &&
4051  "Unexpected number of elements");
4052 
4053  // Take the elements from the first vector.
4054  SmallVector<SDValue, 8> Ops(NumOutElem);
4055  for (unsigned i = 0; i < NumOperands; ++i) {
4056  SDValue Op = N->getOperand(i);
4057  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
4058  Op = GetPromotedInteger(Op);
4059  EVT SclrTy = Op.getValueType().getVectorElementType();
4060  assert(NumElem == Op.getValueType().getVectorNumElements() &&
4061  "Unexpected number of elements");
4062 
4063  for (unsigned j = 0; j < NumElem; ++j) {
4064  SDValue Ext = DAG.getNode(
4065  ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
4066  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4067  Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
4068  }
4069  }
4070 
4071  return DAG.getBuildVector(NOutVT, dl, Ops);
4072 }
4073 
4074 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
4075  EVT VT = N->getValueType(0);
4076  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4077  assert(NVT.isVector() && "This type must be promoted to a vector type");
4078 
4079  SDLoc dl(N);
4080 
4081  // For operands whose TypeAction is to promote, extend the promoted node
4082  // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
4083  // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
4084  // type..
4085  if (getTypeAction(N->getOperand(0).getValueType())
4087  SDValue Promoted;
4088 
4089  switch(N->getOpcode()) {
4091  Promoted = SExtPromotedInteger(N->getOperand(0));
4092  break;
4094  Promoted = ZExtPromotedInteger(N->getOperand(0));
4095  break;
4097  Promoted = GetPromotedInteger(N->getOperand(0));
4098  break;
4099  default:
4100  llvm_unreachable("Node has unexpected Opcode");
4101  }
4102  return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
4103  }
4104 
4105  // Directly extend to the appropriate transform-to type.
4106  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
4107 }
4108 
4109 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
4110  EVT OutVT = N->getValueType(0);
4111  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4112  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4113 
4114  EVT NOutVTElem = NOutVT.getVectorElementType();
4115 
4116  SDLoc dl(N);
4117  SDValue V0 = GetPromotedInteger(N->getOperand(0));
4118 
4119  SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
4120  NOutVTElem, N->getOperand(1));
4121  return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
4122  V0, ConvElem, N->getOperand(2));
4123 }
4124 
4125 SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) {
4126  // The VECREDUCE result size may be larger than the element size, so
4127  // we can simply change the result type.
4128  SDLoc dl(N);
4129  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4130  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
4131 }
4132 
4133 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4134  SDLoc dl(N);
4135  SDValue V0 = GetPromotedInteger(N->getOperand(0));
4136  SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
4137  TLI.getVectorIdxTy(DAG.getDataLayout()));
4139  V0->getValueType(0).getScalarType(), V0, V1);
4140 
4141  // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
4142  // element types. If this is the case then we need to expand the outgoing
4143  // value and not truncate it.
4144  return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
4145 }
4146 
4147 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
4148  SDLoc dl(N);
4149  SDValue V0 = GetPromotedInteger(N->getOperand(0));
4150  MVT InVT = V0.getValueType().getSimpleVT();
4151  MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
4153  SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
4154  return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
4155 }
4156 
4157 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
4158  SDLoc dl(N);
4159  unsigned NumElems = N->getNumOperands();
4160 
4161  EVT RetSclrTy = N->getValueType(0).getVectorElementType();
4162 
4163  SmallVector<SDValue, 8> NewOps;
4164  NewOps.reserve(NumElems);
4165 
4166  // For each incoming vector
4167  for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
4168  SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
4169  EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
4170  unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
4171 
4172  for (unsigned i=0; i<NumElem; ++i) {
4173  // Extract element from incoming vector
4174  SDValue Ex = DAG.getNode(
4175  ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
4176  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4177  SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
4178  NewOps.push_back(Tr);
4179  }
4180  }
4181 
4182  return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
4183 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:595
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:551
static EVT getShiftAmountTyForConstant(unsigned Val, EVT VT, const TargetLowering &TLI, SelectionDAG &DAG)
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getValueType() const
Return the ValueType of the referenced return value.
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant, which is required to be operand #1) half of the integer or float value specified as operand #0.
Definition: ISDOpcodes.h:183
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
Definition: APInt.h:561
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC&#39;s if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:953
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:382
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:678
This class represents lattice values for constants.
Definition: AllocatorList.h:23
static MVT getVectorVT(MVT VT, unsigned NumElements)
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:391
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:259
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:284
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:543
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none...
const SDValue & getBasePtr() const
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:222
void push_back(const T &Elt)
Definition: SmallVector.h:211
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:857
const SDValue & getValue() const
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
Definition: APInt.h:647
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:252
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:250
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1328
const SDValue & getBasePtr() const
const SDValue & getChain() const
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
unsigned getAlignment() const
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:833
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select&#39;s if you just have operands and don&#39;t want to check...
Definition: SelectionDAG.h:966
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:532
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:408
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Same for subtraction.
Definition: ISDOpcodes.h:253
void reserve(size_type N)
Definition: SmallVector.h:369
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:211
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align=0, int Offs=0, bool isT=false, unsigned char TargetFlags=0)
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:459
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:38
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition: APInt.h:534
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
SDValue getExternalSymbol(const char *Sym, EVT VT)
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
virtual bool shouldExpandShift(SelectionDAG &DAG, SDNode *N) const
Return true if SHIFT instructions should be expanded to SHIFT_PARTS instructions, and false if a libr...
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:209
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations...
Definition: ISDOpcodes.h:480
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
static uint32_t getAlignment(const MCSectionCOFF &Sec)
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:450
Shift and rotation operations.
Definition: ISDOpcodes.h:434
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:205
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth...
Definition: ISDOpcodes.h:417
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:651
CallLoweringInfo & setChain(SDValue InChain)
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:190
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:279
const SDValue & getPassThru() const
op_iterator op_end() const
unsigned getScalarValueSizeInBits() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
FLT_ROUNDS_ - Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest 2 Round to ...
Definition: ISDOpcodes.h:570
SimpleValueType SimpleTy
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:303
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt) For double-word atomic operations: ValLo, ValHi, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amtLo, amtHi) ValLo, ValHi, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amtLo, amtHi) These correspond to the atomicrmw instruction.
Definition: ISDOpcodes.h:841
const SDValue & getValue() const
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:463
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:403
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG...
Definition: ISDOpcodes.h:72
This is an SDNode representing atomic operations.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
This represents a list of ValueType&#39;s that has been intern&#39;d by a SelectionDAG.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
This class is used to represent an MSTORE node.
unsigned getSizeInBits() const
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:273
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:297
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:291
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:400
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:502
SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
const SDValue & getScale() const
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:453
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:232
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:868
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTruncating=false, bool IsCompressing=false)
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:995
op_iterator op_begin() const
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:778
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:521
const SDValue & getMask() const
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, bool isSigned, const SDLoc &dl, bool doesNotReturn=false, bool isReturnValueUsed=true, bool isPostTypeLegalization=false) const
Returns a pair of (return value, chain).
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands...
UNDEF - An undefined node.
Definition: ISDOpcodes.h:177
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:548
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:351
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
Definition: APInt.h:635
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits...
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
Definition: MathExtras.h:609
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const SDValue & getBasePtr() const
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:234
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:428
Machine Value Type.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
Simple binary floating point operators.
Definition: ISDOpcodes.h:287
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:272
This is an important base class in LLVM.
Definition: Constant.h:41
iterator_range< value_op_iterator > op_values() const
const SDValue & getOperand(unsigned Num) const
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:970
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:356
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:246
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:231
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:587
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:56