LLVM  10.0.0svn
LegalizeIntegerTypes.cpp
Go to the documentation of this file.
1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements integer type expansion and promotion for LegalizeTypes.
10 // Promotion is the act of changing a computation in an illegal type into a
11 // computation in a larger type. For example, implementing i8 arithmetic in an
12 // i32 register (often needed on powerpc).
13 // Expansion is the act of changing a computation in an illegal type into a
14 // computation in two identical registers of a smaller type. For example,
15 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
16 // targets).
17 //
18 //===----------------------------------------------------------------------===//
19 
20 #include "LegalizeTypes.h"
21 #include "llvm/IR/DerivedTypes.h"
23 #include "llvm/Support/KnownBits.h"
25 using namespace llvm;
26 
27 #define DEBUG_TYPE "legalize-types"
28 
29 //===----------------------------------------------------------------------===//
30 // Integer Result Promotion
31 //===----------------------------------------------------------------------===//
32 
33 /// PromoteIntegerResult - This method is called when a result of a node is
34 /// found to be in need of promotion to a larger type. At this point, the node
35 /// may also have invalid operands or may have other results that need
36 /// expansion, we just know that (at least) one result needs promotion.
37 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
38  LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG);
39  dbgs() << "\n");
40  SDValue Res = SDValue();
41 
42  // See if the target wants to custom expand this node.
43  if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
44  LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
45  return;
46  }
47 
48  switch (N->getOpcode()) {
49  default:
50 #ifndef NDEBUG
51  dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
52  N->dump(&DAG); dbgs() << "\n";
53 #endif
54  llvm_unreachable("Do not know how to promote this operator!");
55  case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
56  case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
57  case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
58  case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
59  case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
60  case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
61  case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
62  case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
64  case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
65  case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break;
67  case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
69  Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
70  case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
71  case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
72  break;
73  case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
74  break;
75  case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
76  case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
77  case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
78  case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
79  case ISD::SMIN:
80  case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
81  case ISD::UMIN:
82  case ISD::UMAX: Res = PromoteIntRes_ZExtIntBinOp(N); break;
83 
84  case ISD::SHL: Res = PromoteIntRes_SHL(N); break;
86  Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
87  case ISD::SRA: Res = PromoteIntRes_SRA(N); break;
88  case ISD::SRL: Res = PromoteIntRes_SRL(N); break;
89  case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
90  case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
91  case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
92 
94  Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
96  Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
98  Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
99  case ISD::BUILD_VECTOR:
100  Res = PromoteIntRes_BUILD_VECTOR(N); break;
102  Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
103  case ISD::CONCAT_VECTORS:
104  Res = PromoteIntRes_CONCAT_VECTORS(N); break;
105 
109  Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
110 
111  case ISD::SIGN_EXTEND:
112  case ISD::ZERO_EXTEND:
113  case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
114 
117  case ISD::FP_TO_SINT:
118  case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
119 
120  case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
121 
122  case ISD::FLT_ROUNDS_: Res = PromoteIntRes_FLT_ROUNDS(N); break;
123 
124  case ISD::AND:
125  case ISD::OR:
126  case ISD::XOR:
127  case ISD::ADD:
128  case ISD::SUB:
129  case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
130 
131  case ISD::SDIV:
132  case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
133 
134  case ISD::UDIV:
135  case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
136 
137  case ISD::SADDO:
138  case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
139  case ISD::UADDO:
140  case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
141  case ISD::SMULO:
142  case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
143 
144  case ISD::ADDE:
145  case ISD::SUBE:
146  case ISD::ADDCARRY:
147  case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
148 
149  case ISD::SADDSAT:
150  case ISD::UADDSAT:
151  case ISD::SSUBSAT:
152  case ISD::USUBSAT: Res = PromoteIntRes_ADDSUBSAT(N); break;
153 
154  case ISD::SMULFIX:
155  case ISD::SMULFIXSAT:
156  case ISD::UMULFIX:
157  case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break;
158 
159  case ISD::ABS: Res = PromoteIntRes_ABS(N); break;
160 
161  case ISD::ATOMIC_LOAD:
162  Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
163 
168  case ISD::ATOMIC_LOAD_OR:
175  case ISD::ATOMIC_SWAP:
176  Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
177 
180  Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
181  break;
182 
183  case ISD::VECREDUCE_ADD:
184  case ISD::VECREDUCE_MUL:
185  case ISD::VECREDUCE_AND:
186  case ISD::VECREDUCE_OR:
187  case ISD::VECREDUCE_XOR:
188  case ISD::VECREDUCE_SMAX:
189  case ISD::VECREDUCE_SMIN:
190  case ISD::VECREDUCE_UMAX:
191  case ISD::VECREDUCE_UMIN:
192  Res = PromoteIntRes_VECREDUCE(N);
193  break;
194  }
195 
196  // If the result is null then the sub-method took care of registering it.
197  if (Res.getNode())
198  SetPromotedInteger(SDValue(N, ResNo), Res);
199 }
200 
201 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
202  unsigned ResNo) {
203  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
204  return GetPromotedInteger(Op);
205 }
206 
207 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
208  // Sign-extend the new bits, and continue the assertion.
209  SDValue Op = SExtPromotedInteger(N->getOperand(0));
210  return DAG.getNode(ISD::AssertSext, SDLoc(N),
211  Op.getValueType(), Op, N->getOperand(1));
212 }
213 
214 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
215  // Zero the new bits, and continue the assertion.
216  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
217  return DAG.getNode(ISD::AssertZext, SDLoc(N),
218  Op.getValueType(), Op, N->getOperand(1));
219 }
220 
221 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
222  EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
223  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
224  N->getMemoryVT(), ResVT,
225  N->getChain(), N->getBasePtr(),
226  N->getMemOperand());
227  // Legalize the chain result - switch anything that used the old chain to
228  // use the new one.
229  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
230  return Res;
231 }
232 
233 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
234  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
235  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
236  N->getMemoryVT(),
237  N->getChain(), N->getBasePtr(),
238  Op2, N->getMemOperand());
239  // Legalize the chain result - switch anything that used the old chain to
240  // use the new one.
241  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
242  return Res;
243 }
244 
245 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
246  unsigned ResNo) {
247  if (ResNo == 1) {
249  EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
250  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
251 
252  // Only use the result of getSetCCResultType if it is legal,
253  // otherwise just use the promoted result type (NVT).
254  if (!TLI.isTypeLegal(SVT))
255  SVT = NVT;
256 
257  SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
258  SDValue Res = DAG.getAtomicCmpSwap(
260  N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
261  N->getMemOperand());
262  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
263  ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
264  return Res.getValue(1);
265  }
266 
267  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
268  SDValue Op3 = GetPromotedInteger(N->getOperand(3));
269  SDVTList VTs =
270  DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
271  SDValue Res = DAG.getAtomicCmpSwap(
272  N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
273  N->getBasePtr(), Op2, Op3, N->getMemOperand());
274  // Update the use to N with the newly created Res.
275  for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
276  ReplaceValueWith(SDValue(N, i), Res.getValue(i));
277  return Res;
278 }
279 
280 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
281  SDValue InOp = N->getOperand(0);
282  EVT InVT = InOp.getValueType();
283  EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
284  EVT OutVT = N->getValueType(0);
285  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
286  SDLoc dl(N);
287 
288  switch (getTypeAction(InVT)) {
290  break;
292  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
293  // The input promotes to the same size. Convert the promoted value.
294  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
295  break;
297  // Promote the integer operand by hand.
298  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
300  // Convert the promoted float by hand.
301  if (!NOutVT.isVector())
302  return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
303  break;
304  }
307  break;
309  // Convert the element to an integer and promote it by hand.
310  if (!NOutVT.isVector())
311  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
312  BitConvertToInteger(GetScalarizedVector(InOp)));
313  break;
315  if (!NOutVT.isVector()) {
316  // For example, i32 = BITCAST v2i16 on alpha. Convert the split
317  // pieces of the input into integers and reassemble in the final type.
318  SDValue Lo, Hi;
319  GetSplitVector(N->getOperand(0), Lo, Hi);
320  Lo = BitConvertToInteger(Lo);
321  Hi = BitConvertToInteger(Hi);
322 
323  if (DAG.getDataLayout().isBigEndian())
324  std::swap(Lo, Hi);
325 
326  InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
328  NOutVT.getSizeInBits()),
329  JoinIntegers(Lo, Hi));
330  return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
331  }
332  break;
333  }
335  // The input is widened to the same size. Convert to the widened value.
336  // Make sure that the outgoing value is not a vector, because this would
337  // make us bitcast between two vectors which are legalized in different ways.
338  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector())
339  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
340  // If the output type is also a vector and widening it to the same size
341  // as the widened input type would be a legal type, we can widen the bitcast
342  // and handle the promotion after.
343  if (NOutVT.isVector()) {
344  unsigned WidenInSize = NInVT.getSizeInBits();
345  unsigned OutSize = OutVT.getSizeInBits();
346  if (WidenInSize % OutSize == 0) {
347  unsigned Scale = WidenInSize / OutSize;
348  EVT WideOutVT = EVT::getVectorVT(*DAG.getContext(),
349  OutVT.getVectorElementType(),
350  OutVT.getVectorNumElements() * Scale);
351  if (isTypeLegal(WideOutVT)) {
352  InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
353  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
354  InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
355  DAG.getConstant(0, dl, IdxTy));
356  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
357  }
358  }
359  }
360  }
361 
362  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
363  CreateStackStoreLoad(InOp, OutVT));
364 }
365 
366 // Helper for BSWAP/BITREVERSE promotion to ensure we can fit the shift amount
367 // in the VT returned by getShiftAmountTy and to return a safe VT if we can't.
368 static EVT getShiftAmountTyForConstant(unsigned Val, EVT VT,
369  const TargetLowering &TLI,
370  SelectionDAG &DAG) {
371  EVT ShiftVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
372  // If the value won't fit in the prefered type, just use something safe. It
373  // will be legalized when the shift is expanded.
374  if ((Log2_32(Val) + 1) > ShiftVT.getScalarSizeInBits())
375  ShiftVT = MVT::i32;
376  return ShiftVT;
377 }
378 
379 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
380  SDValue Op = GetPromotedInteger(N->getOperand(0));
381  EVT OVT = N->getValueType(0);
382  EVT NVT = Op.getValueType();
383  SDLoc dl(N);
384 
385  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
386  EVT ShiftVT = getShiftAmountTyForConstant(DiffBits, NVT, TLI, DAG);
387  return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
388  DAG.getConstant(DiffBits, dl, ShiftVT));
389 }
390 
391 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
392  SDValue Op = GetPromotedInteger(N->getOperand(0));
393  EVT OVT = N->getValueType(0);
394  EVT NVT = Op.getValueType();
395  SDLoc dl(N);
396 
397  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
398  EVT ShiftVT = getShiftAmountTyForConstant(DiffBits, NVT, TLI, DAG);
399  return DAG.getNode(ISD::SRL, dl, NVT,
400  DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
401  DAG.getConstant(DiffBits, dl, ShiftVT));
402 }
403 
404 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
405  // The pair element type may be legal, or may not promote to the same type as
406  // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
407  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
408  TLI.getTypeToTransformTo(*DAG.getContext(),
409  N->getValueType(0)), JoinIntegers(N->getOperand(0),
410  N->getOperand(1)));
411 }
412 
413 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
414  EVT VT = N->getValueType(0);
415  // FIXME there is no actual debug info here
416  SDLoc dl(N);
417  // Zero extend things like i1, sign extend everything else. It shouldn't
418  // matter in theory which one we pick, but this tends to give better code?
419  unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
420  SDValue Result = DAG.getNode(Opc, dl,
421  TLI.getTypeToTransformTo(*DAG.getContext(), VT),
422  SDValue(N, 0));
423  assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
424  return Result;
425 }
426 
427 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
428  // Zero extend to the promoted type and do the count there.
429  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
430  SDLoc dl(N);
431  EVT OVT = N->getValueType(0);
432  EVT NVT = Op.getValueType();
433  Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
434  // Subtract off the extra leading bits in the bigger type.
435  return DAG.getNode(
436  ISD::SUB, dl, NVT, Op,
437  DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
438  NVT));
439 }
440 
441 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
442  // Zero extend to the promoted type and do the count there.
443  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
444  return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op);
445 }
446 
447 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
448  SDValue Op = GetPromotedInteger(N->getOperand(0));
449  EVT OVT = N->getValueType(0);
450  EVT NVT = Op.getValueType();
451  SDLoc dl(N);
452  if (N->getOpcode() == ISD::CTTZ) {
453  // The count is the same in the promoted type except if the original
454  // value was zero. This can be handled by setting the bit just off
455  // the top of the original type.
456  auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
457  OVT.getScalarSizeInBits());
458  Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
459  }
460  return DAG.getNode(N->getOpcode(), dl, NVT, Op);
461 }
462 
463 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
464  SDLoc dl(N);
465  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
466 
467  SDValue Op0 = N->getOperand(0);
468  SDValue Op1 = N->getOperand(1);
469 
470  // If the input also needs to be promoted, do that first so we can get a
471  // get a good idea for the output type.
472  if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
474  SDValue In = GetPromotedInteger(Op0);
475 
476  // If the new type is larger than NVT, use it. We probably won't need to
477  // promote it again.
478  EVT SVT = In.getValueType().getScalarType();
479  if (SVT.bitsGE(NVT)) {
480  SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
481  return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
482  }
483  }
484 
485  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
486 }
487 
488 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
489  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
490  unsigned NewOpc = N->getOpcode();
491  SDLoc dl(N);
492 
493  // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
494  // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
495  // and SINT conversions are Custom, there is no way to tell which is
496  // preferable. We choose SINT because that's the right thing on PPC.)
497  if (N->getOpcode() == ISD::FP_TO_UINT &&
498  !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
500  NewOpc = ISD::FP_TO_SINT;
501 
502  if (N->getOpcode() == ISD::STRICT_FP_TO_UINT &&
505  NewOpc = ISD::STRICT_FP_TO_SINT;
506 
507  SDValue Res;
508  if (N->isStrictFPOpcode()) {
509  Res = DAG.getNode(NewOpc, dl, { NVT, MVT::Other },
510  { N->getOperand(0), N->getOperand(1) });
511  // Legalize the chain result - switch anything that used the old chain to
512  // use the new one.
513  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
514  } else
515  Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
516 
517  // Assert that the converted value fits in the original type. If it doesn't
518  // (eg: because the value being converted is too big), then the result of the
519  // original operation was undefined anyway, so the assert is still correct.
520  //
521  // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
522  // before legalization: fp-to-uint16, 65534. -> 0xfffe
523  // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
524  return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT ||
526  ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
528 }
529 
530 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
531  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
532  SDLoc dl(N);
533 
534  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
535 }
536 
537 SDValue DAGTypeLegalizer::PromoteIntRes_FLT_ROUNDS(SDNode *N) {
538  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
539  SDLoc dl(N);
540 
541  return DAG.getNode(N->getOpcode(), dl, NVT);
542 }
543 
544 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
545  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
546  SDLoc dl(N);
547 
548  if (getTypeAction(N->getOperand(0).getValueType())
550  SDValue Res = GetPromotedInteger(N->getOperand(0));
551  assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
552 
553  // If the result and operand types are the same after promotion, simplify
554  // to an in-register extension.
555  if (NVT == Res.getValueType()) {
556  // The high bits are not guaranteed to be anything. Insert an extend.
557  if (N->getOpcode() == ISD::SIGN_EXTEND)
558  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
559  DAG.getValueType(N->getOperand(0).getValueType()));
560  if (N->getOpcode() == ISD::ZERO_EXTEND)
561  return DAG.getZeroExtendInReg(Res, dl,
563  assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
564  return Res;
565  }
566  }
567 
568  // Otherwise, just extend the original operand all the way to the larger type.
569  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
570 }
571 
572 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
573  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
574  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
577  SDLoc dl(N);
578  SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
579  N->getMemoryVT(), N->getMemOperand());
580 
581  // Legalize the chain result - switch anything that used the old chain to
582  // use the new one.
583  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
584  return Res;
585 }
586 
587 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
588  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
589  SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
590 
591  SDLoc dl(N);
592  SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
593  N->getMask(), ExtPassThru, N->getMemoryVT(),
595  // Legalize the chain result - switch anything that used the old chain to
596  // use the new one.
597  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
598  return Res;
599 }
600 
601 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
602  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
603  SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
604  assert(NVT == ExtPassThru.getValueType() &&
605  "Gather result type and the passThru agrument type should be the same");
606 
607  SDLoc dl(N);
608  SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
609  N->getIndex(), N->getScale() };
610  SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
611  N->getMemoryVT(), dl, Ops,
612  N->getMemOperand(), N->getIndexType());
613  // Legalize the chain result - switch anything that used the old chain to
614  // use the new one.
615  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
616  return Res;
617 }
618 
619 /// Promote the overflow flag of an overflowing arithmetic node.
620 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
621  // Change the return type of the boolean result while obeying
622  // getSetCCResultType.
623  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
624  EVT VT = N->getValueType(0);
625  EVT SVT = getSetCCResultType(VT);
626  SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
627  unsigned NumOps = N->getNumOperands();
628  assert(NumOps <= 3 && "Too many operands");
629  if (NumOps == 3)
630  Ops[2] = N->getOperand(2);
631 
632  SDLoc dl(N);
633  SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
634  makeArrayRef(Ops, NumOps));
635 
636  // Modified the sum result - switch anything that used the old sum to use
637  // the new one.
638  ReplaceValueWith(SDValue(N, 0), Res);
639 
640  // Convert to the expected type.
641  return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT);
642 }
643 
644 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSAT(SDNode *N) {
645  // For promoting iN -> iM, this can be expanded by
646  // 1. ANY_EXTEND iN to iM
647  // 2. SHL by M-N
648  // 3. [US][ADD|SUB]SAT
649  // 4. L/ASHR by M-N
650  SDLoc dl(N);
651  SDValue Op1 = N->getOperand(0);
652  SDValue Op2 = N->getOperand(1);
653  unsigned OldBits = Op1.getScalarValueSizeInBits();
654 
655  unsigned Opcode = N->getOpcode();
656  unsigned ShiftOp;
657  switch (Opcode) {
658  case ISD::SADDSAT:
659  case ISD::SSUBSAT:
660  ShiftOp = ISD::SRA;
661  break;
662  case ISD::UADDSAT:
663  case ISD::USUBSAT:
664  ShiftOp = ISD::SRL;
665  break;
666  default:
667  llvm_unreachable("Expected opcode to be signed or unsigned saturation "
668  "addition or subtraction");
669  }
670 
671  SDValue Op1Promoted = GetPromotedInteger(Op1);
672  SDValue Op2Promoted = GetPromotedInteger(Op2);
673 
674  EVT PromotedType = Op1Promoted.getValueType();
675  unsigned NewBits = PromotedType.getScalarSizeInBits();
676  unsigned SHLAmount = NewBits - OldBits;
677  EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
678  SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT);
679  Op1Promoted =
680  DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
681  Op2Promoted =
682  DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
683 
684  SDValue Result =
685  DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
686  return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
687 }
688 
689 SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
690  // Can just promote the operands then continue with operation.
691  SDLoc dl(N);
692  SDValue Op1Promoted, Op2Promoted;
693  bool Signed =
694  N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT;
695  bool Saturating =
697  if (Signed) {
698  Op1Promoted = SExtPromotedInteger(N->getOperand(0));
699  Op2Promoted = SExtPromotedInteger(N->getOperand(1));
700  } else {
701  Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
702  Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
703  }
704  EVT OldType = N->getOperand(0).getValueType();
705  EVT PromotedType = Op1Promoted.getValueType();
706  unsigned DiffSize =
707  PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits();
708 
709  if (Saturating) {
710  // Promoting the operand and result values changes the saturation width,
711  // which is extends the values that we clamp to on saturation. This could be
712  // resolved by shifting one of the operands the same amount, which would
713  // also shift the result we compare against, then shifting back.
714  EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
715  Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
716  DAG.getConstant(DiffSize, dl, ShiftTy));
717  SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
718  Op2Promoted, N->getOperand(2));
719  unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL;
720  return DAG.getNode(ShiftOp, dl, PromotedType, Result,
721  DAG.getConstant(DiffSize, dl, ShiftTy));
722  }
723  return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
724  N->getOperand(2));
725 }
726 
727 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
728  if (ResNo == 1)
729  return PromoteIntRes_Overflow(N);
730 
731  // The operation overflowed iff the result in the larger type is not the
732  // sign extension of its truncation to the original type.
733  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
734  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
735  EVT OVT = N->getOperand(0).getValueType();
736  EVT NVT = LHS.getValueType();
737  SDLoc dl(N);
738 
739  // Do the arithmetic in the larger type.
740  unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
741  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
742 
743  // Calculate the overflow flag: sign extend the arithmetic result from
744  // the original type.
745  SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
746  DAG.getValueType(OVT));
747  // Overflowed if and only if this is not equal to Res.
748  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
749 
750  // Use the calculated overflow everywhere.
751  ReplaceValueWith(SDValue(N, 1), Ofl);
752 
753  return Res;
754 }
755 
756 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
757  SDValue LHS = GetPromotedInteger(N->getOperand(1));
758  SDValue RHS = GetPromotedInteger(N->getOperand(2));
759  return DAG.getSelect(SDLoc(N),
760  LHS.getValueType(), N->getOperand(0), LHS, RHS);
761 }
762 
763 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
764  SDValue Mask = N->getOperand(0);
765 
766  SDValue LHS = GetPromotedInteger(N->getOperand(1));
767  SDValue RHS = GetPromotedInteger(N->getOperand(2));
768  return DAG.getNode(ISD::VSELECT, SDLoc(N),
769  LHS.getValueType(), Mask, LHS, RHS);
770 }
771 
772 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
773  SDValue LHS = GetPromotedInteger(N->getOperand(2));
774  SDValue RHS = GetPromotedInteger(N->getOperand(3));
775  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
776  LHS.getValueType(), N->getOperand(0),
777  N->getOperand(1), LHS, RHS, N->getOperand(4));
778 }
779 
780 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
781  EVT InVT = N->getOperand(0).getValueType();
782  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
783 
784  EVT SVT = getSetCCResultType(InVT);
785 
786  // If we got back a type that needs to be promoted, this likely means the
787  // the input type also needs to be promoted. So get the promoted type for
788  // the input and try the query again.
789  if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
790  if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
791  InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
792  SVT = getSetCCResultType(InVT);
793  } else {
794  // Input type isn't promoted, just use the default promoted type.
795  SVT = NVT;
796  }
797  }
798 
799  SDLoc dl(N);
800  assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() &&
801  "Vector compare must return a vector result!");
802 
803  // Get the SETCC result using the canonical SETCC type.
804  SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
805  N->getOperand(1), N->getOperand(2));
806 
807  // Convert to the expected type.
808  return DAG.getSExtOrTrunc(SetCC, dl, NVT);
809 }
810 
811 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
812  SDValue LHS = GetPromotedInteger(N->getOperand(0));
813  SDValue RHS = N->getOperand(1);
814  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
815  RHS = ZExtPromotedInteger(RHS);
816  return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
817 }
818 
819 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
820  SDValue Op = GetPromotedInteger(N->getOperand(0));
821  return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
822  Op.getValueType(), Op, N->getOperand(1));
823 }
824 
825 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
826  // The input may have strange things in the top bits of the registers, but
827  // these operations don't care. They may have weird bits going out, but
828  // that too is okay if they are integer operations.
829  SDValue LHS = GetPromotedInteger(N->getOperand(0));
830  SDValue RHS = GetPromotedInteger(N->getOperand(1));
831  return DAG.getNode(N->getOpcode(), SDLoc(N),
832  LHS.getValueType(), LHS, RHS);
833 }
834 
835 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
836  // Sign extend the input.
837  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
838  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
839  return DAG.getNode(N->getOpcode(), SDLoc(N),
840  LHS.getValueType(), LHS, RHS);
841 }
842 
843 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
844  // Zero extend the input.
845  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
846  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
847  return DAG.getNode(N->getOpcode(), SDLoc(N),
848  LHS.getValueType(), LHS, RHS);
849 }
850 
851 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
852  // The input value must be properly sign extended.
853  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
854  SDValue RHS = N->getOperand(1);
855  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
856  RHS = ZExtPromotedInteger(RHS);
857  return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
858 }
859 
860 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
861  // The input value must be properly zero extended.
862  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
863  SDValue RHS = N->getOperand(1);
864  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
865  RHS = ZExtPromotedInteger(RHS);
866  return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
867 }
868 
869 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
870  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
871  SDValue Res;
872  SDValue InOp = N->getOperand(0);
873  SDLoc dl(N);
874 
875  switch (getTypeAction(InOp.getValueType())) {
876  default: llvm_unreachable("Unknown type action!");
879  Res = InOp;
880  break;
882  Res = GetPromotedInteger(InOp);
883  break;
885  EVT InVT = InOp.getValueType();
886  assert(InVT.isVector() && "Cannot split scalar types");
887  unsigned NumElts = InVT.getVectorNumElements();
888  assert(NumElts == NVT.getVectorNumElements() &&
889  "Dst and Src must have the same number of elements");
890  assert(isPowerOf2_32(NumElts) &&
891  "Promoted vector type must be a power of two");
892 
893  SDValue EOp1, EOp2;
894  GetSplitVector(InOp, EOp1, EOp2);
895 
896  EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
897  NumElts/2);
898  EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
899  EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
900 
901  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
902  }
904  SDValue WideInOp = GetWidenedVector(InOp);
905 
906  // Truncate widened InOp.
907  unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
908  EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
909  N->getValueType(0).getScalarType(), NumElem);
910  SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
911 
912  // Zero extend so that the elements are of same type as those of NVT
913  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
914  NumElem);
915  SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
916 
917  // Extract the low NVT subvector.
918  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
919  SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy);
920  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
921  }
922  }
923 
924  // Truncate to NVT instead of VT
925  return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
926 }
927 
928 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
929  if (ResNo == 1)
930  return PromoteIntRes_Overflow(N);
931 
932  // The operation overflowed iff the result in the larger type is not the
933  // zero extension of its truncation to the original type.
934  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
935  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
936  EVT OVT = N->getOperand(0).getValueType();
937  EVT NVT = LHS.getValueType();
938  SDLoc dl(N);
939 
940  // Do the arithmetic in the larger type.
941  unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
942  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
943 
944  // Calculate the overflow flag: zero extend the arithmetic result from
945  // the original type.
946  SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT.getScalarType());
947  // Overflowed if and only if this is not equal to Res.
948  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
949 
950  // Use the calculated overflow everywhere.
951  ReplaceValueWith(SDValue(N, 1), Ofl);
952 
953  return Res;
954 }
955 
956 // Handle promotion for the ADDE/SUBE/ADDCARRY/SUBCARRY nodes. Notice that
957 // the third operand of ADDE/SUBE nodes is carry flag, which differs from
958 // the ADDCARRY/SUBCARRY nodes in that the third operand is carry Boolean.
959 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
960  if (ResNo == 1)
961  return PromoteIntRes_Overflow(N);
962 
963  // We need to sign-extend the operands so the carry value computed by the
964  // wide operation will be equivalent to the carry value computed by the
965  // narrow operation.
966  // An ADDCARRY can generate carry only if any of the operands has its
967  // most significant bit set. Sign extension propagates the most significant
968  // bit into the higher bits which means the extra bit that the narrow
969  // addition would need (i.e. the carry) will be propagated through the higher
970  // bits of the wide addition.
971  // A SUBCARRY can generate borrow only if LHS < RHS and this property will be
972  // preserved by sign extension.
973  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
974  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
975 
976  EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
977 
978  // Do the arithmetic in the wide type.
979  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
980  LHS, RHS, N->getOperand(2));
981 
982  // Update the users of the original carry/borrow value.
983  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
984 
985  return SDValue(Res.getNode(), 0);
986 }
987 
988 SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
989  SDValue Op0 = SExtPromotedInteger(N->getOperand(0));
990  return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0);
991 }
992 
993 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
994  // Promote the overflow bit trivially.
995  if (ResNo == 1)
996  return PromoteIntRes_Overflow(N);
997 
998  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
999  SDLoc DL(N);
1000  EVT SmallVT = LHS.getValueType();
1001 
1002  // To determine if the result overflowed in a larger type, we extend the
1003  // input to the larger type, do the multiply (checking if it overflows),
1004  // then also check the high bits of the result to see if overflow happened
1005  // there.
1006  if (N->getOpcode() == ISD::SMULO) {
1007  LHS = SExtPromotedInteger(LHS);
1008  RHS = SExtPromotedInteger(RHS);
1009  } else {
1010  LHS = ZExtPromotedInteger(LHS);
1011  RHS = ZExtPromotedInteger(RHS);
1012  }
1013  SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
1014  SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
1015 
1016  // Overflow occurred if it occurred in the larger type, or if the high part
1017  // of the result does not zero/sign-extend the low part. Check this second
1018  // possibility first.
1019  SDValue Overflow;
1020  if (N->getOpcode() == ISD::UMULO) {
1021  // Unsigned overflow occurred if the high part is non-zero.
1022  unsigned Shift = SmallVT.getScalarSizeInBits();
1023  EVT ShiftTy = getShiftAmountTyForConstant(Shift, Mul.getValueType(),
1024  TLI, DAG);
1025  SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
1026  DAG.getConstant(Shift, DL, ShiftTy));
1027  Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
1028  DAG.getConstant(0, DL, Hi.getValueType()),
1029  ISD::SETNE);
1030  } else {
1031  // Signed overflow occurred if the high part does not sign extend the low.
1032  SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
1033  Mul, DAG.getValueType(SmallVT));
1034  Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
1035  }
1036 
1037  // The only other way for overflow to occur is if the multiplication in the
1038  // larger type itself overflowed.
1039  Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
1040  SDValue(Mul.getNode(), 1));
1041 
1042  // Use the calculated overflow everywhere.
1043  ReplaceValueWith(SDValue(N, 1), Overflow);
1044  return Mul;
1045 }
1046 
1047 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
1048  return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1049  N->getValueType(0)));
1050 }
1051 
1052 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
1053  SDValue Chain = N->getOperand(0); // Get the chain.
1054  SDValue Ptr = N->getOperand(1); // Get the pointer.
1055  EVT VT = N->getValueType(0);
1056  SDLoc dl(N);
1057 
1058  MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
1059  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
1060  // The argument is passed as NumRegs registers of type RegVT.
1061 
1062  SmallVector<SDValue, 8> Parts(NumRegs);
1063  for (unsigned i = 0; i < NumRegs; ++i) {
1064  Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
1065  N->getConstantOperandVal(3));
1066  Chain = Parts[i].getValue(1);
1067  }
1068 
1069  // Handle endianness of the load.
1070  if (DAG.getDataLayout().isBigEndian())
1071  std::reverse(Parts.begin(), Parts.end());
1072 
1073  // Assemble the parts in the promoted type.
1074  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1075  SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
1076  for (unsigned i = 1; i < NumRegs; ++i) {
1077  SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
1078  // Shift it to the right position and "or" it in.
1079  Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
1080  DAG.getConstant(i * RegVT.getSizeInBits(), dl,
1081  TLI.getPointerTy(DAG.getDataLayout())));
1082  Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
1083  }
1084 
1085  // Modified the chain result - switch anything that used the old chain to
1086  // use the new one.
1087  ReplaceValueWith(SDValue(N, 1), Chain);
1088 
1089  return Res;
1090 }
1091 
1092 //===----------------------------------------------------------------------===//
1093 // Integer Operand Promotion
1094 //===----------------------------------------------------------------------===//
1095 
1096 /// PromoteIntegerOperand - This method is called when the specified operand of
1097 /// the specified node is found to need promotion. At this point, all of the
1098 /// result types of the node are known to be legal, but other operands of the
1099 /// node may need promotion or expansion as well as the specified one.
1100 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
1101  LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG);
1102  dbgs() << "\n");
1103  SDValue Res = SDValue();
1104 
1105  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
1106  LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
1107  return false;
1108  }
1109 
1110  switch (N->getOpcode()) {
1111  default:
1112  #ifndef NDEBUG
1113  dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
1114  N->dump(&DAG); dbgs() << "\n";
1115  #endif
1116  llvm_unreachable("Do not know how to promote this operator's operand!");
1117 
1118  case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
1119  case ISD::ATOMIC_STORE:
1120  Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
1121  break;
1122  case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
1123  case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
1124  case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
1125  case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
1126  case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
1127  case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
1128  case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
1130  Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
1131  case ISD::SCALAR_TO_VECTOR:
1132  Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
1133  case ISD::VSELECT:
1134  case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
1135  case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
1136  case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
1137  case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
1138  case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
1139  case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
1140  OpNo); break;
1141  case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
1142  OpNo); break;
1143  case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
1144  OpNo); break;
1145  case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
1146  OpNo); break;
1147  case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
1148  OpNo); break;
1149  case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
1150  case ISD::FP16_TO_FP:
1151  case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
1152  case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
1153  case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
1154 
1155  case ISD::SHL:
1156  case ISD::SRA:
1157  case ISD::SRL:
1158  case ISD::ROTL:
1159  case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
1160 
1161  case ISD::ADDCARRY:
1162  case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
1163 
1164  case ISD::FRAMEADDR:
1165  case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
1166 
1167  case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break;
1168 
1169  case ISD::SMULFIX:
1170  case ISD::SMULFIXSAT:
1171  case ISD::UMULFIX:
1172  case ISD::UMULFIXSAT: Res = PromoteIntOp_MULFIX(N); break;
1173 
1174  case ISD::FPOWI: Res = PromoteIntOp_FPOWI(N); break;
1175 
1176  case ISD::VECREDUCE_ADD:
1177  case ISD::VECREDUCE_MUL:
1178  case ISD::VECREDUCE_AND:
1179  case ISD::VECREDUCE_OR:
1180  case ISD::VECREDUCE_XOR:
1181  case ISD::VECREDUCE_SMAX:
1182  case ISD::VECREDUCE_SMIN:
1183  case ISD::VECREDUCE_UMAX:
1184  case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break;
1185  }
1186 
1187  // If the result is null, the sub-method took care of registering results etc.
1188  if (!Res.getNode()) return false;
1189 
1190  // If the result is N, the sub-method updated N in place. Tell the legalizer
1191  // core about this.
1192  if (Res.getNode() == N)
1193  return true;
1194 
1195  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1196  "Invalid operand expansion");
1197 
1198  ReplaceValueWith(SDValue(N, 0), Res);
1199  return false;
1200 }
1201 
1202 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
1203 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
1204 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
1205  ISD::CondCode CCCode) {
1206  // We have to insert explicit sign or zero extends. Note that we could
1207  // insert sign extends for ALL conditions. For those operations where either
1208  // zero or sign extension would be valid, use SExtOrZExtPromotedInteger
1209  // which will choose the cheapest for the target.
1210  switch (CCCode) {
1211  default: llvm_unreachable("Unknown integer comparison!");
1212  case ISD::SETEQ:
1213  case ISD::SETNE: {
1214  SDValue OpL = GetPromotedInteger(NewLHS);
1215  SDValue OpR = GetPromotedInteger(NewRHS);
1216 
1217  // We would prefer to promote the comparison operand with sign extension.
1218  // If the width of OpL/OpR excluding the duplicated sign bits is no greater
1219  // than the width of NewLHS/NewRH, we can avoid inserting real truncate
1220  // instruction, which is redundant eventually.
1221  unsigned OpLEffectiveBits =
1222  OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
1223  unsigned OpREffectiveBits =
1224  OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
1225  if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() &&
1226  OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()) {
1227  NewLHS = OpL;
1228  NewRHS = OpR;
1229  } else {
1230  NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1231  NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1232  }
1233  break;
1234  }
1235  case ISD::SETUGE:
1236  case ISD::SETUGT:
1237  case ISD::SETULE:
1238  case ISD::SETULT:
1239  NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1240  NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1241  break;
1242  case ISD::SETGE:
1243  case ISD::SETGT:
1244  case ISD::SETLT:
1245  case ISD::SETLE:
1246  NewLHS = SExtPromotedInteger(NewLHS);
1247  NewRHS = SExtPromotedInteger(NewRHS);
1248  break;
1249  }
1250 }
1251 
1252 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1253  SDValue Op = GetPromotedInteger(N->getOperand(0));
1254  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1255 }
1256 
1257 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1258  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1259  return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1260  N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1261 }
1262 
1263 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1264  // This should only occur in unusual situations like bitcasting to an
1265  // x86_fp80, so just turn it into a store+load
1266  return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1267 }
1268 
1269 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1270  assert(OpNo == 2 && "Don't know how to promote this operand!");
1271 
1272  SDValue LHS = N->getOperand(2);
1273  SDValue RHS = N->getOperand(3);
1274  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1275 
1276  // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1277  // legal types.
1278  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1279  N->getOperand(1), LHS, RHS, N->getOperand(4)),
1280  0);
1281 }
1282 
1283 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1284  assert(OpNo == 1 && "only know how to promote condition");
1285 
1286  // Promote all the way up to the canonical SetCC type.
1287  SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1288 
1289  // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1290  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1291  N->getOperand(2)), 0);
1292 }
1293 
1294 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1295  // Since the result type is legal, the operands must promote to it.
1296  EVT OVT = N->getOperand(0).getValueType();
1297  SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1298  SDValue Hi = GetPromotedInteger(N->getOperand(1));
1299  assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1300  SDLoc dl(N);
1301 
1302  Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1303  DAG.getConstant(OVT.getSizeInBits(), dl,
1304  TLI.getPointerTy(DAG.getDataLayout())));
1305  return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1306 }
1307 
1308 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1309  // The vector type is legal but the element type is not. This implies
1310  // that the vector is a power-of-two in length and that the element
1311  // type does not have a strange size (eg: it is not i1).
1312  EVT VecVT = N->getValueType(0);
1313  unsigned NumElts = VecVT.getVectorNumElements();
1314  assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1315  "Legal vector of one illegal element?");
1316 
1317  // Promote the inserted value. The type does not need to match the
1318  // vector element type. Check that any extra bits introduced will be
1319  // truncated away.
1322  "Type of inserted value narrower than vector element type!");
1323 
1324  SmallVector<SDValue, 16> NewOps;
1325  for (unsigned i = 0; i < NumElts; ++i)
1326  NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1327 
1328  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1329 }
1330 
1331 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1332  unsigned OpNo) {
1333  if (OpNo == 1) {
1334  // Promote the inserted value. This is valid because the type does not
1335  // have to match the vector element type.
1336 
1337  // Check that any extra bits introduced will be truncated away.
1340  "Type of inserted value narrower than vector element type!");
1341  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1342  GetPromotedInteger(N->getOperand(1)),
1343  N->getOperand(2)),
1344  0);
1345  }
1346 
1347  assert(OpNo == 2 && "Different operand and result vector types?");
1348 
1349  // Promote the index.
1350  SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1351  TLI.getVectorIdxTy(DAG.getDataLayout()));
1352  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1353  N->getOperand(1), Idx), 0);
1354 }
1355 
1356 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1357  // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1358  // the operand in place.
1359  return SDValue(DAG.UpdateNodeOperands(N,
1360  GetPromotedInteger(N->getOperand(0))), 0);
1361 }
1362 
1363 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1364  assert(OpNo == 0 && "Only know how to promote the condition!");
1365  SDValue Cond = N->getOperand(0);
1366  EVT OpTy = N->getOperand(1).getValueType();
1367 
1368  if (N->getOpcode() == ISD::VSELECT)
1369  if (SDValue Res = WidenVSELECTAndMask(N))
1370  return Res;
1371 
1372  // Promote all the way up to the canonical SetCC type.
1373  EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1374  Cond = PromoteTargetBoolean(Cond, OpVT);
1375 
1376  return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1377  N->getOperand(2)), 0);
1378 }
1379 
1380 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1381  assert(OpNo == 0 && "Don't know how to promote this operand!");
1382 
1383  SDValue LHS = N->getOperand(0);
1384  SDValue RHS = N->getOperand(1);
1385  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1386 
1387  // The CC (#4) and the possible return values (#2 and #3) have legal types.
1388  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1389  N->getOperand(3), N->getOperand(4)), 0);
1390 }
1391 
1392 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1393  assert(OpNo == 0 && "Don't know how to promote this operand!");
1394 
1395  SDValue LHS = N->getOperand(0);
1396  SDValue RHS = N->getOperand(1);
1397  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1398 
1399  // The CC (#2) is always legal.
1400  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1401 }
1402 
1403 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1404  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1405  ZExtPromotedInteger(N->getOperand(1))), 0);
1406 }
1407 
1408 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1409  SDValue Op = GetPromotedInteger(N->getOperand(0));
1410  SDLoc dl(N);
1411  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1412  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1413  Op, DAG.getValueType(N->getOperand(0).getValueType()));
1414 }
1415 
1416 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1417  return SDValue(DAG.UpdateNodeOperands(N,
1418  SExtPromotedInteger(N->getOperand(0))), 0);
1419 }
1420 
1421 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1422  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1423  SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1424  SDLoc dl(N);
1425 
1426  SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1427 
1428  // Truncate the value and store the result.
1429  return DAG.getTruncStore(Ch, dl, Val, Ptr,
1430  N->getMemoryVT(), N->getMemOperand());
1431 }
1432 
1433 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1434  unsigned OpNo) {
1435 
1436  SDValue DataOp = N->getValue();
1437  EVT DataVT = DataOp.getValueType();
1438  SDValue Mask = N->getMask();
1439  SDLoc dl(N);
1440 
1441  bool TruncateStore = false;
1442  if (OpNo == 3) {
1443  Mask = PromoteTargetBoolean(Mask, DataVT);
1444  // Update in place.
1445  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1446  NewOps[3] = Mask;
1447  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1448  } else { // Data operand
1449  assert(OpNo == 1 && "Unexpected operand for promotion");
1450  DataOp = GetPromotedInteger(DataOp);
1451  TruncateStore = true;
1452  }
1453 
1454  return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask,
1455  N->getMemoryVT(), N->getMemOperand(),
1456  TruncateStore, N->isCompressingStore());
1457 }
1458 
1459 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1460  unsigned OpNo) {
1461  assert(OpNo == 2 && "Only know how to promote the mask!");
1462  EVT DataVT = N->getValueType(0);
1463  SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1464  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1465  NewOps[OpNo] = Mask;
1466  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1467 }
1468 
1469 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1470  unsigned OpNo) {
1471 
1472  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1473  if (OpNo == 2) {
1474  // The Mask
1475  EVT DataVT = N->getValueType(0);
1476  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1477  } else if (OpNo == 4) {
1478  // The Index
1479  if (N->isIndexSigned())
1480  // Need to sign extend the index since the bits will likely be used.
1481  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1482  else
1483  NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
1484  } else
1485  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1486 
1487  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1488 }
1489 
1490 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1491  unsigned OpNo) {
1492  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1493  if (OpNo == 2) {
1494  // The Mask
1495  EVT DataVT = N->getValue().getValueType();
1496  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1497  } else if (OpNo == 4) {
1498  // The Index
1499  if (N->isIndexSigned())
1500  // Need to sign extend the index since the bits will likely be used.
1501  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1502  else
1503  NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
1504  } else
1505  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1506  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1507 }
1508 
1509 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
1510  SDValue Op = GetPromotedInteger(N->getOperand(0));
1511  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
1512 }
1513 
1514 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
1515  return SDValue(DAG.UpdateNodeOperands(N,
1516  ZExtPromotedInteger(N->getOperand(0))), 0);
1517 }
1518 
1519 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
1520  SDLoc dl(N);
1521  SDValue Op = GetPromotedInteger(N->getOperand(0));
1522  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1523  return DAG.getZeroExtendInReg(Op, dl,
1525 }
1526 
1527 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
1528  assert(OpNo == 2 && "Don't know how to promote this operand!");
1529 
1530  SDValue LHS = N->getOperand(0);
1531  SDValue RHS = N->getOperand(1);
1532  SDValue Carry = N->getOperand(2);
1533  SDLoc DL(N);
1534 
1535  Carry = PromoteTargetBoolean(Carry, LHS.getValueType());
1536 
1537  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
1538 }
1539 
1540 SDValue DAGTypeLegalizer::PromoteIntOp_MULFIX(SDNode *N) {
1541  SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1542  return SDValue(
1543  DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
1544 }
1545 
1546 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
1547  // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
1548  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
1549  return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
1550 }
1551 
1552 SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) {
1553  assert(OpNo > 1 && "Don't know how to promote this operand!");
1554  // Promote the rw, locality, and cache type arguments to a supported integer
1555  // width.
1556  SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1557  SDValue Op3 = ZExtPromotedInteger(N->getOperand(3));
1558  SDValue Op4 = ZExtPromotedInteger(N->getOperand(4));
1559  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
1560  Op2, Op3, Op4),
1561  0);
1562 }
1563 
1564 SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) {
1565  SDValue Op = SExtPromotedInteger(N->getOperand(1));
1566  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
1567 }
1568 
1569 SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) {
1570  SDLoc dl(N);
1571  SDValue Op;
1572  switch (N->getOpcode()) {
1573  default: llvm_unreachable("Expected integer vector reduction");
1574  case ISD::VECREDUCE_ADD:
1575  case ISD::VECREDUCE_MUL:
1576  case ISD::VECREDUCE_AND:
1577  case ISD::VECREDUCE_OR:
1578  case ISD::VECREDUCE_XOR:
1579  Op = GetPromotedInteger(N->getOperand(0));
1580  break;
1581  case ISD::VECREDUCE_SMAX:
1582  case ISD::VECREDUCE_SMIN:
1583  Op = SExtPromotedInteger(N->getOperand(0));
1584  break;
1585  case ISD::VECREDUCE_UMAX:
1586  case ISD::VECREDUCE_UMIN:
1587  Op = ZExtPromotedInteger(N->getOperand(0));
1588  break;
1589  }
1590 
1591  EVT EltVT = Op.getValueType().getVectorElementType();
1592  EVT VT = N->getValueType(0);
1593  if (VT.bitsGE(EltVT))
1594  return DAG.getNode(N->getOpcode(), SDLoc(N), VT, Op);
1595 
1596  // Result size must be >= element size. If this is not the case after
1597  // promotion, also promote the result type and then truncate.
1598  SDValue Reduce = DAG.getNode(N->getOpcode(), dl, EltVT, Op);
1599  return DAG.getNode(ISD::TRUNCATE, dl, VT, Reduce);
1600 }
1601 
1602 //===----------------------------------------------------------------------===//
1603 // Integer Result Expansion
1604 //===----------------------------------------------------------------------===//
1605 
1606 /// ExpandIntegerResult - This method is called when the specified result of the
1607 /// specified node is found to need expansion. At this point, the node may also
1608 /// have invalid operands or may have other results that need promotion, we just
1609 /// know that (at least) one result needs expansion.
1610 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
1611  LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG);
1612  dbgs() << "\n");
1613  SDValue Lo, Hi;
1614  Lo = Hi = SDValue();
1615 
1616  // See if the target wants to custom expand this node.
1617  if (CustomLowerNode(N, N->getValueType(ResNo), true))
1618  return;
1619 
1620  switch (N->getOpcode()) {
1621  default:
1622 #ifndef NDEBUG
1623  dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
1624  N->dump(&DAG); dbgs() << "\n";
1625 #endif
1626  report_fatal_error("Do not know how to expand the result of this "
1627  "operator!");
1628 
1629  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1630  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
1631  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1632  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1633 
1634  case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1635  case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1636  case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1637  case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1638  case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1639 
1640  case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
1641  case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
1642  case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
1643  case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
1644  case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
1645  case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
1646  case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break;
1647  case ISD::CTLZ_ZERO_UNDEF:
1648  case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
1649  case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
1650  case ISD::CTTZ_ZERO_UNDEF:
1651  case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
1652  case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
1653  case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
1654  case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
1655  case ISD::LLROUND: ExpandIntRes_LLROUND(N, Lo, Hi); break;
1656  case ISD::LLRINT: ExpandIntRes_LLRINT(N, Lo, Hi); break;
1657  case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
1658  case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
1659  case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
1660  case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
1661  case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
1662  case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
1663  case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
1664  case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
1665  case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
1666  case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
1667  case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
1668  case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
1669 
1670  case ISD::ATOMIC_LOAD_ADD:
1671  case ISD::ATOMIC_LOAD_SUB:
1672  case ISD::ATOMIC_LOAD_AND:
1673  case ISD::ATOMIC_LOAD_CLR:
1674  case ISD::ATOMIC_LOAD_OR:
1675  case ISD::ATOMIC_LOAD_XOR:
1676  case ISD::ATOMIC_LOAD_NAND:
1677  case ISD::ATOMIC_LOAD_MIN:
1678  case ISD::ATOMIC_LOAD_MAX:
1679  case ISD::ATOMIC_LOAD_UMIN:
1680  case ISD::ATOMIC_LOAD_UMAX:
1681  case ISD::ATOMIC_SWAP:
1682  case ISD::ATOMIC_CMP_SWAP: {
1683  std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
1684  SplitInteger(Tmp.first, Lo, Hi);
1685  ReplaceValueWith(SDValue(N, 1), Tmp.second);
1686  break;
1687  }
1689  AtomicSDNode *AN = cast<AtomicSDNode>(N);
1690  SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
1691  SDValue Tmp = DAG.getAtomicCmpSwap(
1692  ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
1693  N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
1694  AN->getMemOperand());
1695 
1696  // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
1697  // success simply by comparing the loaded value against the ingoing
1698  // comparison.
1699  SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
1700  N->getOperand(2), ISD::SETEQ);
1701 
1702  SplitInteger(Tmp, Lo, Hi);
1703  ReplaceValueWith(SDValue(N, 1), Success);
1704  ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
1705  break;
1706  }
1707 
1708  case ISD::AND:
1709  case ISD::OR:
1710  case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
1711 
1712  case ISD::UMAX:
1713  case ISD::SMAX:
1714  case ISD::UMIN:
1715  case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
1716 
1717  case ISD::ADD:
1718  case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
1719 
1720  case ISD::ADDC:
1721  case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
1722 
1723  case ISD::ADDE:
1724  case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
1725 
1726  case ISD::ADDCARRY:
1727  case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
1728 
1729  case ISD::SHL:
1730  case ISD::SRA:
1731  case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
1732 
1733  case ISD::SADDO:
1734  case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
1735  case ISD::UADDO:
1736  case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
1737  case ISD::UMULO:
1738  case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
1739 
1740  case ISD::SADDSAT:
1741  case ISD::UADDSAT:
1742  case ISD::SSUBSAT:
1743  case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
1744 
1745  case ISD::SMULFIX:
1746  case ISD::SMULFIXSAT:
1747  case ISD::UMULFIX:
1748  case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break;
1749 
1750  case ISD::VECREDUCE_ADD:
1751  case ISD::VECREDUCE_MUL:
1752  case ISD::VECREDUCE_AND:
1753  case ISD::VECREDUCE_OR:
1754  case ISD::VECREDUCE_XOR:
1755  case ISD::VECREDUCE_SMAX:
1756  case ISD::VECREDUCE_SMIN:
1757  case ISD::VECREDUCE_UMAX:
1758  case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break;
1759  }
1760 
1761  // If Lo/Hi is null, the sub-method took care of registering results etc.
1762  if (Lo.getNode())
1763  SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
1764 }
1765 
1766 /// Lower an atomic node to the appropriate builtin call.
1767 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
1768  unsigned Opc = Node->getOpcode();
1769  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
1770  RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
1771  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
1772 
1773  return ExpandChainLibCall(LC, Node, false);
1774 }
1775 
1776 /// N is a shift by a value that needs to be expanded,
1777 /// and the shift amount is a constant 'Amt'. Expand the operation.
1778 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
1779  SDValue &Lo, SDValue &Hi) {
1780  SDLoc DL(N);
1781  // Expand the incoming operand to be shifted, so that we have its parts
1782  SDValue InL, InH;
1783  GetExpandedInteger(N->getOperand(0), InL, InH);
1784 
1785  // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
1786  // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
1787  if (!Amt) {
1788  Lo = InL;
1789  Hi = InH;
1790  return;
1791  }
1792 
1793  EVT NVT = InL.getValueType();
1794  unsigned VTBits = N->getValueType(0).getSizeInBits();
1795  unsigned NVTBits = NVT.getSizeInBits();
1796  EVT ShTy = N->getOperand(1).getValueType();
1797 
1798  if (N->getOpcode() == ISD::SHL) {
1799  if (Amt.ugt(VTBits)) {
1800  Lo = Hi = DAG.getConstant(0, DL, NVT);
1801  } else if (Amt.ugt(NVTBits)) {
1802  Lo = DAG.getConstant(0, DL, NVT);
1803  Hi = DAG.getNode(ISD::SHL, DL,
1804  NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1805  } else if (Amt == NVTBits) {
1806  Lo = DAG.getConstant(0, DL, NVT);
1807  Hi = InL;
1808  } else {
1809  Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
1810  Hi = DAG.getNode(ISD::OR, DL, NVT,
1811  DAG.getNode(ISD::SHL, DL, NVT, InH,
1812  DAG.getConstant(Amt, DL, ShTy)),
1813  DAG.getNode(ISD::SRL, DL, NVT, InL,
1814  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1815  }
1816  return;
1817  }
1818 
1819  if (N->getOpcode() == ISD::SRL) {
1820  if (Amt.ugt(VTBits)) {
1821  Lo = Hi = DAG.getConstant(0, DL, NVT);
1822  } else if (Amt.ugt(NVTBits)) {
1823  Lo = DAG.getNode(ISD::SRL, DL,
1824  NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1825  Hi = DAG.getConstant(0, DL, NVT);
1826  } else if (Amt == NVTBits) {
1827  Lo = InH;
1828  Hi = DAG.getConstant(0, DL, NVT);
1829  } else {
1830  Lo = DAG.getNode(ISD::OR, DL, NVT,
1831  DAG.getNode(ISD::SRL, DL, NVT, InL,
1832  DAG.getConstant(Amt, DL, ShTy)),
1833  DAG.getNode(ISD::SHL, DL, NVT, InH,
1834  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1835  Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1836  }
1837  return;
1838  }
1839 
1840  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
1841  if (Amt.ugt(VTBits)) {
1842  Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1843  DAG.getConstant(NVTBits - 1, DL, ShTy));
1844  } else if (Amt.ugt(NVTBits)) {
1845  Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1846  DAG.getConstant(Amt - NVTBits, DL, ShTy));
1847  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1848  DAG.getConstant(NVTBits - 1, DL, ShTy));
1849  } else if (Amt == NVTBits) {
1850  Lo = InH;
1851  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1852  DAG.getConstant(NVTBits - 1, DL, ShTy));
1853  } else {
1854  Lo = DAG.getNode(ISD::OR, DL, NVT,
1855  DAG.getNode(ISD::SRL, DL, NVT, InL,
1856  DAG.getConstant(Amt, DL, ShTy)),
1857  DAG.getNode(ISD::SHL, DL, NVT, InH,
1858  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1859  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1860  }
1861 }
1862 
1863 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
1864 /// this shift based on knowledge of the high bit of the shift amount. If we
1865 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
1866 /// shift amount.
1867 bool DAGTypeLegalizer::
1868 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1869  SDValue Amt = N->getOperand(1);
1870  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1871  EVT ShTy = Amt.getValueType();
1872  unsigned ShBits = ShTy.getScalarSizeInBits();
1873  unsigned NVTBits = NVT.getScalarSizeInBits();
1874  assert(isPowerOf2_32(NVTBits) &&
1875  "Expanded integer type size not a power of two!");
1876  SDLoc dl(N);
1877 
1878  APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
1879  KnownBits Known = DAG.computeKnownBits(N->getOperand(1));
1880 
1881  // If we don't know anything about the high bits, exit.
1882  if (((Known.Zero|Known.One) & HighBitMask) == 0)
1883  return false;
1884 
1885  // Get the incoming operand to be shifted.
1886  SDValue InL, InH;
1887  GetExpandedInteger(N->getOperand(0), InL, InH);
1888 
1889  // If we know that any of the high bits of the shift amount are one, then we
1890  // can do this as a couple of simple shifts.
1891  if (Known.One.intersects(HighBitMask)) {
1892  // Mask out the high bit, which we know is set.
1893  Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
1894  DAG.getConstant(~HighBitMask, dl, ShTy));
1895 
1896  switch (N->getOpcode()) {
1897  default: llvm_unreachable("Unknown shift");
1898  case ISD::SHL:
1899  Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
1900  Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
1901  return true;
1902  case ISD::SRL:
1903  Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1904  Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
1905  return true;
1906  case ISD::SRA:
1907  Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
1908  DAG.getConstant(NVTBits - 1, dl, ShTy));
1909  Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
1910  return true;
1911  }
1912  }
1913 
1914  // If we know that all of the high bits of the shift amount are zero, then we
1915  // can do this as a couple of simple shifts.
1916  if (HighBitMask.isSubsetOf(Known.Zero)) {
1917  // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
1918  // shift if x is zero. We can use XOR here because x is known to be smaller
1919  // than 32.
1920  SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
1921  DAG.getConstant(NVTBits - 1, dl, ShTy));
1922 
1923  unsigned Op1, Op2;
1924  switch (N->getOpcode()) {
1925  default: llvm_unreachable("Unknown shift");
1926  case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
1927  case ISD::SRL:
1928  case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
1929  }
1930 
1931  // When shifting right the arithmetic for Lo and Hi is swapped.
1932  if (N->getOpcode() != ISD::SHL)
1933  std::swap(InL, InH);
1934 
1935  // Use a little trick to get the bits that move from Lo to Hi. First
1936  // shift by one bit.
1937  SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
1938  // Then compute the remaining shift with amount-1.
1939  SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
1940 
1941  Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
1942  Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
1943 
1944  if (N->getOpcode() != ISD::SHL)
1945  std::swap(Hi, Lo);
1946  return true;
1947  }
1948 
1949  return false;
1950 }
1951 
1952 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
1953 /// of any size.
1954 bool DAGTypeLegalizer::
1955 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1956  SDValue Amt = N->getOperand(1);
1957  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1958  EVT ShTy = Amt.getValueType();
1959  unsigned NVTBits = NVT.getSizeInBits();
1960  assert(isPowerOf2_32(NVTBits) &&
1961  "Expanded integer type size not a power of two!");
1962  SDLoc dl(N);
1963 
1964  // Get the incoming operand to be shifted.
1965  SDValue InL, InH;
1966  GetExpandedInteger(N->getOperand(0), InL, InH);
1967 
1968  SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
1969  SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
1970  SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
1971  SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1972  Amt, NVBitsNode, ISD::SETULT);
1973  SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1974  Amt, DAG.getConstant(0, dl, ShTy),
1975  ISD::SETEQ);
1976 
1977  SDValue LoS, HiS, LoL, HiL;
1978  switch (N->getOpcode()) {
1979  default: llvm_unreachable("Unknown shift");
1980  case ISD::SHL:
1981  // Short: ShAmt < NVTBits
1982  LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
1983  HiS = DAG.getNode(ISD::OR, dl, NVT,
1984  DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
1985  DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
1986 
1987  // Long: ShAmt >= NVTBits
1988  LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
1989  HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
1990 
1991  Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
1992  Hi = DAG.getSelect(dl, NVT, isZero, InH,
1993  DAG.getSelect(dl, NVT, isShort, HiS, HiL));
1994  return true;
1995  case ISD::SRL:
1996  // Short: ShAmt < NVTBits
1997  HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
1998  LoS = DAG.getNode(ISD::OR, dl, NVT,
1999  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
2000  // FIXME: If Amt is zero, the following shift generates an undefined result
2001  // on some architectures.
2002  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
2003 
2004  // Long: ShAmt >= NVTBits
2005  HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
2006  LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
2007 
2008  Lo = DAG.getSelect(dl, NVT, isZero, InL,
2009  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
2010  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
2011  return true;
2012  case ISD::SRA:
2013  // Short: ShAmt < NVTBits
2014  HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
2015  LoS = DAG.getNode(ISD::OR, dl, NVT,
2016  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
2017  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
2018 
2019  // Long: ShAmt >= NVTBits
2020  HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
2021  DAG.getConstant(NVTBits - 1, dl, ShTy));
2022  LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
2023 
2024  Lo = DAG.getSelect(dl, NVT, isZero, InL,
2025  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
2026  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
2027  return true;
2028  }
2029 }
2030 
2031 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
2032 
2033  switch (Op) {
2034  default: llvm_unreachable("invalid min/max opcode");
2035  case ISD::SMAX:
2036  return std::make_pair(ISD::SETGT, ISD::UMAX);
2037  case ISD::UMAX:
2038  return std::make_pair(ISD::SETUGT, ISD::UMAX);
2039  case ISD::SMIN:
2040  return std::make_pair(ISD::SETLT, ISD::UMIN);
2041  case ISD::UMIN:
2042  return std::make_pair(ISD::SETULT, ISD::UMIN);
2043  }
2044 }
2045 
2046 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
2047  SDValue &Lo, SDValue &Hi) {
2048  SDLoc DL(N);
2049  ISD::NodeType LoOpc;
2050  ISD::CondCode CondC;
2051  std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
2052 
2053  // Expand the subcomponents.
2054  SDValue LHSL, LHSH, RHSL, RHSH;
2055  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2056  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2057 
2058  // Value types
2059  EVT NVT = LHSL.getValueType();
2060  EVT CCT = getSetCCResultType(NVT);
2061 
2062  // Hi part is always the same op
2063  Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
2064 
2065  // We need to know whether to select Lo part that corresponds to 'winning'
2066  // Hi part or if Hi parts are equal.
2067  SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
2068  SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
2069 
2070  // Lo part corresponding to the 'winning' Hi part
2071  SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
2072 
2073  // Recursed Lo part if Hi parts are equal, this uses unsigned version
2074  SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
2075 
2076  Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
2077 }
2078 
2079 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
2080  SDValue &Lo, SDValue &Hi) {
2081  SDLoc dl(N);
2082  // Expand the subcomponents.
2083  SDValue LHSL, LHSH, RHSL, RHSH;
2084  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2085  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2086 
2087  EVT NVT = LHSL.getValueType();
2088  SDValue LoOps[2] = { LHSL, RHSL };
2089  SDValue HiOps[3] = { LHSH, RHSH };
2090 
2091  bool HasOpCarry = TLI.isOperationLegalOrCustom(
2093  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2094  if (HasOpCarry) {
2095  SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
2096  if (N->getOpcode() == ISD::ADD) {
2097  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2098  HiOps[2] = Lo.getValue(1);
2099  Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
2100  } else {
2101  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2102  HiOps[2] = Lo.getValue(1);
2103  Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
2104  }
2105  return;
2106  }
2107 
2108  // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
2109  // them. TODO: Teach operation legalization how to expand unsupported
2110  // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
2111  // a carry of type MVT::Glue, but there doesn't seem to be any way to
2112  // generate a value of this type in the expanded code sequence.
2113  bool hasCarry =
2115  ISD::ADDC : ISD::SUBC,
2116  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2117 
2118  if (hasCarry) {
2119  SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
2120  if (N->getOpcode() == ISD::ADD) {
2121  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2122  HiOps[2] = Lo.getValue(1);
2123  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2124  } else {
2125  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2126  HiOps[2] = Lo.getValue(1);
2127  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2128  }
2129  return;
2130  }
2131 
2132  bool hasOVF =
2135  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2137 
2138  if (hasOVF) {
2139  EVT OvfVT = getSetCCResultType(NVT);
2140  SDVTList VTList = DAG.getVTList(NVT, OvfVT);
2141  int RevOpc;
2142  if (N->getOpcode() == ISD::ADD) {
2143  RevOpc = ISD::SUB;
2144  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2145  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2146  } else {
2147  RevOpc = ISD::ADD;
2148  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2149  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2150  }
2151  SDValue OVF = Lo.getValue(1);
2152 
2153  switch (BoolType) {
2155  OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
2158  OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
2159  Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
2160  break;
2162  OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
2163  Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
2164  }
2165  return;
2166  }
2167 
2168  if (N->getOpcode() == ISD::ADD) {
2169  Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
2170  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2171  SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
2172  ISD::SETULT);
2173 
2175  SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
2176  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
2177  return;
2178  }
2179 
2180  SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
2181  DAG.getConstant(1, dl, NVT),
2182  DAG.getConstant(0, dl, NVT));
2183  SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
2184  ISD::SETULT);
2185  SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
2186  DAG.getConstant(1, dl, NVT), Carry1);
2187  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
2188  } else {
2189  Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
2190  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2191  SDValue Cmp =
2192  DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
2193  LoOps[0], LoOps[1], ISD::SETULT);
2194 
2195  SDValue Borrow;
2197  Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
2198  else
2199  Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
2200  DAG.getConstant(0, dl, NVT));
2201 
2202  Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
2203  }
2204 }
2205 
2206 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
2207  SDValue &Lo, SDValue &Hi) {
2208  // Expand the subcomponents.
2209  SDValue LHSL, LHSH, RHSL, RHSH;
2210  SDLoc dl(N);
2211  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2212  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2213  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2214  SDValue LoOps[2] = { LHSL, RHSL };
2215  SDValue HiOps[3] = { LHSH, RHSH };
2216 
2217  if (N->getOpcode() == ISD::ADDC) {
2218  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2219  HiOps[2] = Lo.getValue(1);
2220  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2221  } else {
2222  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2223  HiOps[2] = Lo.getValue(1);
2224  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2225  }
2226 
2227  // Legalized the flag result - switch anything that used the old flag to
2228  // use the new one.
2229  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2230 }
2231 
2232 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
2233  SDValue &Lo, SDValue &Hi) {
2234  // Expand the subcomponents.
2235  SDValue LHSL, LHSH, RHSL, RHSH;
2236  SDLoc dl(N);
2237  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2238  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2239  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2240  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2241  SDValue HiOps[3] = { LHSH, RHSH };
2242 
2243  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2244  HiOps[2] = Lo.getValue(1);
2245  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2246 
2247  // Legalized the flag result - switch anything that used the old flag to
2248  // use the new one.
2249  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2250 }
2251 
2252 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
2253  SDValue &Lo, SDValue &Hi) {
2254  SDValue LHS = N->getOperand(0);
2255  SDValue RHS = N->getOperand(1);
2256  SDLoc dl(N);
2257 
2258  SDValue Ovf;
2259 
2260  bool HasOpCarry = TLI.isOperationLegalOrCustom(
2262  TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
2263 
2264  if (HasOpCarry) {
2265  // Expand the subcomponents.
2266  SDValue LHSL, LHSH, RHSL, RHSH;
2267  GetExpandedInteger(LHS, LHSL, LHSH);
2268  GetExpandedInteger(RHS, RHSL, RHSH);
2269  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2270  SDValue LoOps[2] = { LHSL, RHSL };
2271  SDValue HiOps[3] = { LHSH, RHSH };
2272 
2273  unsigned Opc = N->getOpcode() == ISD::UADDO ? ISD::ADDCARRY : ISD::SUBCARRY;
2274  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2275  HiOps[2] = Lo.getValue(1);
2276  Hi = DAG.getNode(Opc, dl, VTList, HiOps);
2277 
2278  Ovf = Hi.getValue(1);
2279  } else {
2280  // Expand the result by simply replacing it with the equivalent
2281  // non-overflow-checking operation.
2282  auto Opc = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
2283  SDValue Sum = DAG.getNode(Opc, dl, LHS.getValueType(), LHS, RHS);
2284  SplitInteger(Sum, Lo, Hi);
2285 
2286  // Calculate the overflow: addition overflows iff a + b < a, and subtraction
2287  // overflows iff a - b > a.
2288  auto Cond = N->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
2289  Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
2290  }
2291 
2292  // Legalized the flag result - switch anything that used the old flag to
2293  // use the new one.
2294  ReplaceValueWith(SDValue(N, 1), Ovf);
2295 }
2296 
2297 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
2298  SDValue &Lo, SDValue &Hi) {
2299  // Expand the subcomponents.
2300  SDValue LHSL, LHSH, RHSL, RHSH;
2301  SDLoc dl(N);
2302  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2303  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2304  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2305  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2306  SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
2307 
2308  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2309  HiOps[2] = Lo.getValue(1);
2310  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2311 
2312  // Legalized the flag result - switch anything that used the old flag to
2313  // use the new one.
2314  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2315 }
2316 
2317 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
2318  SDValue &Lo, SDValue &Hi) {
2319  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2320  SDLoc dl(N);
2321  SDValue Op = N->getOperand(0);
2322  if (Op.getValueType().bitsLE(NVT)) {
2323  // The low part is any extension of the input (which degenerates to a copy).
2324  Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
2325  Hi = DAG.getUNDEF(NVT); // The high part is undefined.
2326  } else {
2327  // For example, extension of an i48 to an i64. The operand type necessarily
2328  // promotes to the result type, so will end up being expanded too.
2329  assert(getTypeAction(Op.getValueType()) ==
2331  "Only know how to promote this result!");
2332  SDValue Res = GetPromotedInteger(Op);
2333  assert(Res.getValueType() == N->getValueType(0) &&
2334  "Operand over promoted?");
2335  // Split the promoted operand. This will simplify when it is expanded.
2336  SplitInteger(Res, Lo, Hi);
2337  }
2338 }
2339 
2340 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
2341  SDValue &Lo, SDValue &Hi) {
2342  SDLoc dl(N);
2343  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2344  EVT NVT = Lo.getValueType();
2345  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2346  unsigned NVTBits = NVT.getSizeInBits();
2347  unsigned EVTBits = EVT.getSizeInBits();
2348 
2349  if (NVTBits < EVTBits) {
2350  Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
2352  EVTBits - NVTBits)));
2353  } else {
2354  Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
2355  // The high part replicates the sign bit of Lo, make it explicit.
2356  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2357  DAG.getConstant(NVTBits - 1, dl,
2358  TLI.getPointerTy(DAG.getDataLayout())));
2359  }
2360 }
2361 
2362 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
2363  SDValue &Lo, SDValue &Hi) {
2364  SDLoc dl(N);
2365  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2366  EVT NVT = Lo.getValueType();
2367  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2368  unsigned NVTBits = NVT.getSizeInBits();
2369  unsigned EVTBits = EVT.getSizeInBits();
2370 
2371  if (NVTBits < EVTBits) {
2372  Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
2374  EVTBits - NVTBits)));
2375  } else {
2376  Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
2377  // The high part must be zero, make it explicit.
2378  Hi = DAG.getConstant(0, dl, NVT);
2379  }
2380 }
2381 
2382 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
2383  SDValue &Lo, SDValue &Hi) {
2384  SDLoc dl(N);
2385  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2386  Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
2387  Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
2388 }
2389 
2390 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
2391  SDValue &Lo, SDValue &Hi) {
2392  SDLoc dl(N);
2393  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2394  Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
2395  Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
2396 }
2397 
2398 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
2399  SDValue &Lo, SDValue &Hi) {
2400  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2401  unsigned NBitWidth = NVT.getSizeInBits();
2402  auto Constant = cast<ConstantSDNode>(N);
2403  const APInt &Cst = Constant->getAPIntValue();
2404  bool IsTarget = Constant->isTargetOpcode();
2405  bool IsOpaque = Constant->isOpaque();
2406  SDLoc dl(N);
2407  Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
2408  Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
2409  IsOpaque);
2410 }
2411 
2412 void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
2413  SDLoc dl(N);
2414 
2415  // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo)
2416  EVT VT = N->getValueType(0);
2417  SDValue N0 = N->getOperand(0);
2418  SDValue Neg = DAG.getNode(ISD::SUB, dl, VT,
2419  DAG.getConstant(0, dl, VT), N0);
2420  SDValue NegLo, NegHi;
2421  SplitInteger(Neg, NegLo, NegHi);
2422 
2423  GetExpandedInteger(N0, Lo, Hi);
2424  EVT NVT = Lo.getValueType();
2425  SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT),
2426  DAG.getConstant(0, dl, NVT), Hi, ISD::SETGT);
2427  Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo);
2428  Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi);
2429 }
2430 
2431 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
2432  SDValue &Lo, SDValue &Hi) {
2433  SDLoc dl(N);
2434  // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
2435  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2436  EVT NVT = Lo.getValueType();
2437 
2438  SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
2439  DAG.getConstant(0, dl, NVT), ISD::SETNE);
2440 
2441  SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
2442  SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
2443 
2444  Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
2445  DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
2446  DAG.getConstant(NVT.getSizeInBits(), dl,
2447  NVT)));
2448  Hi = DAG.getConstant(0, dl, NVT);
2449 }
2450 
2451 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
2452  SDValue &Lo, SDValue &Hi) {
2453  SDLoc dl(N);
2454  // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
2455  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2456  EVT NVT = Lo.getValueType();
2457  Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
2458  DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
2459  Hi = DAG.getConstant(0, dl, NVT);
2460 }
2461 
2462 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
2463  SDValue &Lo, SDValue &Hi) {
2464  SDLoc dl(N);
2465  // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
2466  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2467  EVT NVT = Lo.getValueType();
2468 
2469  SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
2470  DAG.getConstant(0, dl, NVT), ISD::SETNE);
2471 
2472  SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
2473  SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
2474 
2475  Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
2476  DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
2477  DAG.getConstant(NVT.getSizeInBits(), dl,
2478  NVT)));
2479  Hi = DAG.getConstant(0, dl, NVT);
2480 }
2481 
2482 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
2483  SDValue &Hi) {
2484  SDLoc dl(N);
2485  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2486  unsigned NBitWidth = NVT.getSizeInBits();
2487 
2488  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2489  Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT);
2490  // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
2491  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2492  DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
2493 }
2494 
2495 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
2496  SDValue &Hi) {
2497  SDLoc dl(N);
2498  EVT VT = N->getValueType(0);
2499 
2500  SDValue Op = N->getOperand(0);
2501  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2502  Op = GetPromotedFloat(Op);
2503 
2505  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
2507  CallOptions.setSExt(true);
2508  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, dl).first,
2509  Lo, Hi);
2510 }
2511 
2512 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
2513  SDValue &Hi) {
2514  SDLoc dl(N);
2515  EVT VT = N->getValueType(0);
2516 
2517  SDValue Op = N->getOperand(0);
2518  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2519  Op = GetPromotedFloat(Op);
2520 
2522  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
2524  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, dl).first,
2525  Lo, Hi);
2526 }
2527 
2528 void DAGTypeLegalizer::ExpandIntRes_LLROUND(SDNode *N, SDValue &Lo,
2529  SDValue &Hi) {
2530  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2532  if (VT == MVT::f32)
2533  LC = RTLIB::LLROUND_F32;
2534  else if (VT == MVT::f64)
2535  LC = RTLIB::LLROUND_F64;
2536  else if (VT == MVT::f80)
2537  LC = RTLIB::LLROUND_F80;
2538  else if (VT == MVT::f128)
2539  LC = RTLIB::LLROUND_F128;
2540  else if (VT == MVT::ppcf128)
2541  LC = RTLIB::LLROUND_PPCF128;
2542  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!");
2543 
2544  SDValue Op = N->getOperand(0);
2545  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2546  Op = GetPromotedFloat(Op);
2547 
2548  SDLoc dl(N);
2549  EVT RetVT = N->getValueType(0);
2551  CallOptions.setSExt(true);
2552  SplitInteger(TLI.makeLibCall(DAG, LC, RetVT, Op, CallOptions, dl).first,
2553  Lo, Hi);
2554 }
2555 
2556 void DAGTypeLegalizer::ExpandIntRes_LLRINT(SDNode *N, SDValue &Lo,
2557  SDValue &Hi) {
2558  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2560  if (VT == MVT::f32)
2561  LC = RTLIB::LLRINT_F32;
2562  else if (VT == MVT::f64)
2563  LC = RTLIB::LLRINT_F64;
2564  else if (VT == MVT::f80)
2565  LC = RTLIB::LLRINT_F80;
2566  else if (VT == MVT::f128)
2567  LC = RTLIB::LLRINT_F128;
2568  else if (VT == MVT::ppcf128)
2569  LC = RTLIB::LLRINT_PPCF128;
2570  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!");
2571 
2572  SDValue Op = N->getOperand(0);
2573  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2574  Op = GetPromotedFloat(Op);
2575 
2576  SDLoc dl(N);
2577  EVT RetVT = N->getValueType(0);
2579  CallOptions.setSExt(true);
2580  SplitInteger(TLI.makeLibCall(DAG, LC, RetVT, Op, CallOptions, dl).first,
2581  Lo, Hi);
2582 }
2583 
2584 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
2585  SDValue &Lo, SDValue &Hi) {
2586  if (ISD::isNormalLoad(N)) {
2587  ExpandRes_NormalLoad(N, Lo, Hi);
2588  return;
2589  }
2590 
2591  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2592 
2593  EVT VT = N->getValueType(0);
2594  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2595  SDValue Ch = N->getChain();
2596  SDValue Ptr = N->getBasePtr();
2598  unsigned Alignment = N->getAlignment();
2599  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2600  AAMDNodes AAInfo = N->getAAInfo();
2601  SDLoc dl(N);
2602 
2603  assert(NVT.isByteSized() && "Expanded type not byte sized!");
2604 
2605  if (N->getMemoryVT().bitsLE(NVT)) {
2606  EVT MemVT = N->getMemoryVT();
2607 
2608  Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
2609  Alignment, MMOFlags, AAInfo);
2610 
2611  // Remember the chain.
2612  Ch = Lo.getValue(1);
2613 
2614  if (ExtType == ISD::SEXTLOAD) {
2615  // The high part is obtained by SRA'ing all but one of the bits of the
2616  // lo part.
2617  unsigned LoSize = Lo.getValueSizeInBits();
2618  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2619  DAG.getConstant(LoSize - 1, dl,
2620  TLI.getPointerTy(DAG.getDataLayout())));
2621  } else if (ExtType == ISD::ZEXTLOAD) {
2622  // The high part is just a zero.
2623  Hi = DAG.getConstant(0, dl, NVT);
2624  } else {
2625  assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
2626  // The high part is undefined.
2627  Hi = DAG.getUNDEF(NVT);
2628  }
2629  } else if (DAG.getDataLayout().isLittleEndian()) {
2630  // Little-endian - low bits are at low addresses.
2631  Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2632  AAInfo);
2633 
2634  unsigned ExcessBits =
2635  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
2636  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
2637 
2638  // Increment the pointer to the other half.
2639  unsigned IncrementSize = NVT.getSizeInBits()/8;
2640  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2641  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2642  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
2643  N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
2644  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2645 
2646  // Build a factor node to remember that this load is independent of the
2647  // other one.
2648  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2649  Hi.getValue(1));
2650  } else {
2651  // Big-endian - high bits are at low addresses. Favor aligned loads at
2652  // the cost of some bit-fiddling.
2653  EVT MemVT = N->getMemoryVT();
2654  unsigned EBytes = MemVT.getStoreSize();
2655  unsigned IncrementSize = NVT.getSizeInBits()/8;
2656  unsigned ExcessBits = (EBytes - IncrementSize)*8;
2657 
2658  // Load both the high bits and maybe some of the low bits.
2659  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
2661  MemVT.getSizeInBits() - ExcessBits),
2662  Alignment, MMOFlags, AAInfo);
2663 
2664  // Increment the pointer to the other half.
2665  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2666  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2667  // Load the rest of the low bits.
2668  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
2669  N->getPointerInfo().getWithOffset(IncrementSize),
2670  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
2671  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2672 
2673  // Build a factor node to remember that this load is independent of the
2674  // other one.
2675  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2676  Hi.getValue(1));
2677 
2678  if (ExcessBits < NVT.getSizeInBits()) {
2679  // Transfer low bits from the bottom of Hi to the top of Lo.
2680  Lo = DAG.getNode(
2681  ISD::OR, dl, NVT, Lo,
2682  DAG.getNode(ISD::SHL, dl, NVT, Hi,
2683  DAG.getConstant(ExcessBits, dl,
2684  TLI.getPointerTy(DAG.getDataLayout()))));
2685  // Move high bits to the right position in Hi.
2686  Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
2687  Hi,
2688  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
2689  TLI.getPointerTy(DAG.getDataLayout())));
2690  }
2691  }
2692 
2693  // Legalize the chain result - switch anything that used the old chain to
2694  // use the new one.
2695  ReplaceValueWith(SDValue(N, 1), Ch);
2696 }
2697 
2698 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
2699  SDValue &Lo, SDValue &Hi) {
2700  SDLoc dl(N);
2701  SDValue LL, LH, RL, RH;
2702  GetExpandedInteger(N->getOperand(0), LL, LH);
2703  GetExpandedInteger(N->getOperand(1), RL, RH);
2704  Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
2705  Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
2706 }
2707 
2708 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
2709  SDValue &Lo, SDValue &Hi) {
2710  EVT VT = N->getValueType(0);
2711  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2712  SDLoc dl(N);
2713 
2714  SDValue LL, LH, RL, RH;
2715  GetExpandedInteger(N->getOperand(0), LL, LH);
2716  GetExpandedInteger(N->getOperand(1), RL, RH);
2717 
2718  if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
2720  LL, LH, RL, RH))
2721  return;
2722 
2723  // If nothing else, we can make a libcall.
2724  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2725  if (VT == MVT::i16)
2726  LC = RTLIB::MUL_I16;
2727  else if (VT == MVT::i32)
2728  LC = RTLIB::MUL_I32;
2729  else if (VT == MVT::i64)
2730  LC = RTLIB::MUL_I64;
2731  else if (VT == MVT::i128)
2732  LC = RTLIB::MUL_I128;
2733 
2734  if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
2735  // We'll expand the multiplication by brute force because we have no other
2736  // options. This is a trivially-generalized version of the code from
2737  // Hacker's Delight (itself derived from Knuth's Algorithm M from section
2738  // 4.3.1).
2739  unsigned Bits = NVT.getSizeInBits();
2740  unsigned HalfBits = Bits >> 1;
2741  SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
2742  NVT);
2743  SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
2744  SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
2745 
2746  SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
2747  SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
2748 
2749  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2750  if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
2751  // The type from TLI is too small to fit the shift amount we want.
2752  // Override it with i32. The shift will have to be legalized.
2753  ShiftAmtTy = MVT::i32;
2754  }
2755  SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
2756  SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
2757  SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
2758  SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
2759 
2760  SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
2761  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
2762  SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
2763  SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
2764 
2765  SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
2766  DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
2767  SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
2768 
2769  SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
2770  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
2771  DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
2772  Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
2773  DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
2774 
2775  Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
2776  DAG.getNode(ISD::ADD, dl, NVT,
2777  DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
2778  DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
2779  return;
2780  }
2781 
2782  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2784  CallOptions.setSExt(true);
2785  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
2786  Lo, Hi);
2787 }
2788 
2789 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
2790  SDValue &Hi) {
2791  SDLoc DL(N);
2792  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2793  SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
2794  SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
2795  Lo = R.getValue(0);
2796  Hi = R.getValue(1);
2797  ReplaceValueWith(SDValue(N, 1), R.getValue(2));
2798 }
2799 
2800 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
2801  SDValue &Hi) {
2802  SDValue Result = TLI.expandAddSubSat(N, DAG);
2803  SplitInteger(Result, Lo, Hi);
2804 }
2805 
2806 /// This performs an expansion of the integer result for a fixed point
2807 /// multiplication. The default expansion performs rounding down towards
2808 /// negative infinity, though targets that do care about rounding should specify
2809 /// a target hook for rounding and provide their own expansion or lowering of
2810 /// fixed point multiplication to be consistent with rounding.
2811 void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
2812  SDValue &Hi) {
2813  SDLoc dl(N);
2814  EVT VT = N->getValueType(0);
2815  unsigned VTSize = VT.getScalarSizeInBits();
2816  SDValue LHS = N->getOperand(0);
2817  SDValue RHS = N->getOperand(1);
2818  uint64_t Scale = N->getConstantOperandVal(2);
2819  bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT ||
2820  N->getOpcode() == ISD::UMULFIXSAT);
2821  bool Signed = (N->getOpcode() == ISD::SMULFIX ||
2822  N->getOpcode() == ISD::SMULFIXSAT);
2823 
2824  // Handle special case when scale is equal to zero.
2825  if (!Scale) {
2826  SDValue Result;
2827  if (!Saturating) {
2828  Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
2829  } else {
2830  EVT BoolVT = getSetCCResultType(VT);
2831  unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO;
2832  Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
2833  SDValue Product = Result.getValue(0);
2834  SDValue Overflow = Result.getValue(1);
2835  if (Signed) {
2836  APInt MinVal = APInt::getSignedMinValue(VTSize);
2837  APInt MaxVal = APInt::getSignedMaxValue(VTSize);
2838  SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
2839  SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
2840  SDValue Zero = DAG.getConstant(0, dl, VT);
2841  SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Product, Zero, ISD::SETLT);
2842  Result = DAG.getSelect(dl, VT, ProdNeg, SatMax, SatMin);
2843  Result = DAG.getSelect(dl, VT, Overflow, Result, Product);
2844  } else {
2845  // For unsigned multiplication, we only need to check the max since we
2846  // can't really overflow towards zero.
2847  APInt MaxVal = APInt::getMaxValue(VTSize);
2848  SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
2849  Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product);
2850  }
2851  }
2852  SplitInteger(Result, Lo, Hi);
2853  return;
2854  }
2855 
2856  // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will
2857  // cover for unhandled cases below, while still being valid for UMULFIX[SAT].
2858  assert(Scale <= VTSize && "Scale can't be larger than the value type size.");
2859 
2860  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2861  SDValue LL, LH, RL, RH;
2862  GetExpandedInteger(LHS, LL, LH);
2863  GetExpandedInteger(RHS, RL, RH);
2864  SmallVector<SDValue, 4> Result;
2865 
2866  unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
2867  if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
2869  LL, LH, RL, RH)) {
2870  report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI.");
2871  return;
2872  }
2873 
2874  unsigned NVTSize = NVT.getScalarSizeInBits();
2875  assert((VTSize == NVTSize * 2) && "Expected the new value type to be half "
2876  "the size of the current value type");
2877  EVT ShiftTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2878 
2879  // After getting the multiplication result in 4 parts, we need to perform a
2880  // shift right by the amount of the scale to get the result in that scale.
2881  //
2882  // Let's say we multiply 2 64 bit numbers. The resulting value can be held in
2883  // 128 bits that are cut into 4 32-bit parts:
2884  //
2885  // HH HL LH LL
2886  // |---32---|---32---|---32---|---32---|
2887  // 128 96 64 32 0
2888  //
2889  // |------VTSize-----|
2890  //
2891  // |NVTSize-|
2892  //
2893  // The resulting Lo and Hi would normally be in LL and LH after the shift. But
2894  // to avoid unneccessary shifting of all 4 parts, we can adjust the shift
2895  // amount and get Lo and Hi using two funnel shifts. Or for the special case
2896  // when Scale is a multiple of NVTSize we can just pick the result without
2897  // shifting.
2898  uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed.
2899  if (Scale % NVTSize) {
2900  SDValue ShiftAmount = DAG.getConstant(Scale % NVTSize, dl, ShiftTy);
2901  Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0],
2902  ShiftAmount);
2903  Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1],
2904  ShiftAmount);
2905  } else {
2906  Lo = Result[Part0];
2907  Hi = Result[Part0 + 1];
2908  }
2909 
2910  // Unless saturation is requested we are done. The result is in <Hi,Lo>.
2911  if (!Saturating)
2912  return;
2913 
2914  // Can not overflow when there is no integer part.
2915  if (Scale == VTSize)
2916  return;
2917 
2918  // To handle saturation we must check for overflow in the multiplication.
2919  //
2920  // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result)
2921  // aren't all zeroes.
2922  //
2923  // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result)
2924  // aren't all ones or all zeroes.
2925  //
2926  // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the
2927  // highest bit of HH determines saturation direction in the event of signed
2928  // saturation.
2929 
2930  SDValue ResultHL = Result[2];
2931  SDValue ResultHH = Result[3];
2932 
2933  SDValue SatMax, SatMin;
2934  SDValue NVTZero = DAG.getConstant(0, dl, NVT);
2935  SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT);
2936  EVT BoolNVT = getSetCCResultType(NVT);
2937 
2938  if (!Signed) {
2939  if (Scale < NVTSize) {
2940  // Overflow happened if ((HH | (HL >> Scale)) != 0).
2941  SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
2942  DAG.getConstant(Scale, dl, ShiftTy));
2943  SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH);
2944  SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE);
2945  } else if (Scale == NVTSize) {
2946  // Overflow happened if (HH != 0).
2947  SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE);
2948  } else if (Scale < VTSize) {
2949  // Overflow happened if ((HH >> (Scale - NVTSize)) != 0).
2950  SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
2951  DAG.getConstant(Scale - NVTSize, dl,
2952  ShiftTy));
2953  SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE);
2954  } else
2955  llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT"
2956  "(and saturation can't happen with Scale==VTSize).");
2957 
2958  Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi);
2959  Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo);
2960  return;
2961  }
2962 
2963  if (Scale < NVTSize) {
2964  // The number of overflow bits we can check are VTSize - Scale + 1 (we
2965  // include the sign bit). If these top bits are > 0, then we overflowed past
2966  // the max value. If these top bits are < -1, then we overflowed past the
2967  // min value. Otherwise, we did not overflow.
2968  unsigned OverflowBits = VTSize - Scale + 1;
2969  assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
2970  "Extent of overflow bits must start within HL");
2971  SDValue HLHiMask = DAG.getConstant(
2972  APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT);
2973  SDValue HLLoMask = DAG.getConstant(
2974  APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT);
2975  // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask).
2976  SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
2977  SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
2978  SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT);
2979  SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
2980  DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT));
2981  // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask).
2982  SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
2983  SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
2984  SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT);
2985  SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
2986  DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT));
2987  } else if (Scale == NVTSize) {
2988  // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1).
2989  SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
2990  SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
2991  SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT);
2992  SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
2993  DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg));
2994  // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0).
2995  SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
2996  SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
2997  SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE);
2998  SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
2999  DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos));
3000  } else if (Scale < VTSize) {
3001  // This is similar to the case when we saturate if Scale < NVTSize, but we
3002  // only need to check HH.
3003  unsigned OverflowBits = VTSize - Scale + 1;
3004  SDValue HHHiMask = DAG.getConstant(
3005  APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT);
3006  SDValue HHLoMask = DAG.getConstant(
3007  APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT);
3008  SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT);
3009  SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT);
3010  } else
3011  llvm_unreachable("Illegal scale for signed fixed point mul.");
3012 
3013  // Saturate to signed maximum.
3014  APInt MaxHi = APInt::getSignedMaxValue(NVTSize);
3015  APInt MaxLo = APInt::getAllOnesValue(NVTSize);
3016  Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi);
3017  Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo);
3018  // Saturate to signed minimum.
3019  APInt MinHi = APInt::getSignedMinValue(NVTSize);
3020  Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi);
3021  Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo);
3022 }
3023 
3024 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
3025  SDValue &Lo, SDValue &Hi) {
3026  SDValue LHS = Node->getOperand(0);
3027  SDValue RHS = Node->getOperand(1);
3028  SDLoc dl(Node);
3029 
3030  // Expand the result by simply replacing it with the equivalent
3031  // non-overflow-checking operation.
3032  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3033  ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3034  LHS, RHS);
3035  SplitInteger(Sum, Lo, Hi);
3036 
3037  // Compute the overflow.
3038  //
3039  // LHSSign -> LHS >= 0
3040  // RHSSign -> RHS >= 0
3041  // SumSign -> Sum >= 0
3042  //
3043  // Add:
3044  // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3045  // Sub:
3046  // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3047  //
3048  EVT OType = Node->getValueType(1);
3049  SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
3050 
3051  SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3052  SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3053  SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
3054  Node->getOpcode() == ISD::SADDO ?
3056 
3057  SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3058  SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3059 
3060  SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3061 
3062  // Use the calculated overflow everywhere.
3063  ReplaceValueWith(SDValue(Node, 1), Cmp);
3064 }
3065 
3066 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
3067  SDValue &Lo, SDValue &Hi) {
3068  EVT VT = N->getValueType(0);
3069  SDLoc dl(N);
3070  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3071 
3073  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3074  SplitInteger(Res.getValue(0), Lo, Hi);
3075  return;
3076  }
3077 
3078  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3079  if (VT == MVT::i16)
3080  LC = RTLIB::SDIV_I16;
3081  else if (VT == MVT::i32)
3082  LC = RTLIB::SDIV_I32;
3083  else if (VT == MVT::i64)
3084  LC = RTLIB::SDIV_I64;
3085  else if (VT == MVT::i128)
3086  LC = RTLIB::SDIV_I128;
3087  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
3088 
3090  CallOptions.setSExt(true);
3091  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3092 }
3093 
3094 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
3095  SDValue &Lo, SDValue &Hi) {
3096  EVT VT = N->getValueType(0);
3097  SDLoc dl(N);
3098 
3099  // If we can emit an efficient shift operation, do so now. Check to see if
3100  // the RHS is a constant.
3101  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
3102  return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
3103 
3104  // If we can determine that the high bit of the shift is zero or one, even if
3105  // the low bits are variable, emit this shift in an optimized form.
3106  if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
3107  return;
3108 
3109  // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
3110  unsigned PartsOpc;
3111  if (N->getOpcode() == ISD::SHL) {
3112  PartsOpc = ISD::SHL_PARTS;
3113  } else if (N->getOpcode() == ISD::SRL) {
3114  PartsOpc = ISD::SRL_PARTS;
3115  } else {
3116  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3117  PartsOpc = ISD::SRA_PARTS;
3118  }
3119 
3120  // Next check to see if the target supports this SHL_PARTS operation or if it
3121  // will custom expand it. Don't lower this to SHL_PARTS when we optimise for
3122  // size, but create a libcall instead.
3123  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3124  TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
3125  const bool LegalOrCustom =
3126  (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
3127  Action == TargetLowering::Custom;
3128 
3129  if (LegalOrCustom && TLI.shouldExpandShift(DAG, N)) {
3130  // Expand the subcomponents.
3131  SDValue LHSL, LHSH;
3132  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3133  EVT VT = LHSL.getValueType();
3134 
3135  // If the shift amount operand is coming from a vector legalization it may
3136  // have an illegal type. Fix that first by casting the operand, otherwise
3137  // the new SHL_PARTS operation would need further legalization.
3138  SDValue ShiftOp = N->getOperand(1);
3139  EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3140  assert(ShiftTy.getScalarSizeInBits() >=
3141  Log2_32_Ceil(VT.getScalarSizeInBits()) &&
3142  "ShiftAmountTy is too small to cover the range of this type!");
3143  if (ShiftOp.getValueType() != ShiftTy)
3144  ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
3145 
3146  SDValue Ops[] = { LHSL, LHSH, ShiftOp };
3147  Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
3148  Hi = Lo.getValue(1);
3149  return;
3150  }
3151 
3152  // Otherwise, emit a libcall.
3153  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3154  bool isSigned;
3155  if (N->getOpcode() == ISD::SHL) {
3156  isSigned = false; /*sign irrelevant*/
3157  if (VT == MVT::i16)
3158  LC = RTLIB::SHL_I16;
3159  else if (VT == MVT::i32)
3160  LC = RTLIB::SHL_I32;
3161  else if (VT == MVT::i64)
3162  LC = RTLIB::SHL_I64;
3163  else if (VT == MVT::i128)
3164  LC = RTLIB::SHL_I128;
3165  } else if (N->getOpcode() == ISD::SRL) {
3166  isSigned = false;
3167  if (VT == MVT::i16)
3168  LC = RTLIB::SRL_I16;
3169  else if (VT == MVT::i32)
3170  LC = RTLIB::SRL_I32;
3171  else if (VT == MVT::i64)
3172  LC = RTLIB::SRL_I64;
3173  else if (VT == MVT::i128)
3174  LC = RTLIB::SRL_I128;
3175  } else {
3176  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3177  isSigned = true;
3178  if (VT == MVT::i16)
3179  LC = RTLIB::SRA_I16;
3180  else if (VT == MVT::i32)
3181  LC = RTLIB::SRA_I32;
3182  else if (VT == MVT::i64)
3183  LC = RTLIB::SRA_I64;
3184  else if (VT == MVT::i128)
3185  LC = RTLIB::SRA_I128;
3186  }
3187 
3188  if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
3189  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3191  CallOptions.setSExt(isSigned);
3192  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3193  return;
3194  }
3195 
3196  if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
3197  llvm_unreachable("Unsupported shift!");
3198 }
3199 
3200 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
3201  SDValue &Lo, SDValue &Hi) {
3202  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3203  SDLoc dl(N);
3204  SDValue Op = N->getOperand(0);
3205  if (Op.getValueType().bitsLE(NVT)) {
3206  // The low part is sign extension of the input (degenerates to a copy).
3207  Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
3208  // The high part is obtained by SRA'ing all but one of the bits of low part.
3209  unsigned LoSize = NVT.getSizeInBits();
3210  Hi = DAG.getNode(
3211  ISD::SRA, dl, NVT, Lo,
3212  DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
3213  } else {
3214  // For example, extension of an i48 to an i64. The operand type necessarily
3215  // promotes to the result type, so will end up being expanded too.
3216  assert(getTypeAction(Op.getValueType()) ==
3218  "Only know how to promote this result!");
3219  SDValue Res = GetPromotedInteger(Op);
3220  assert(Res.getValueType() == N->getValueType(0) &&
3221  "Operand over promoted?");
3222  // Split the promoted operand. This will simplify when it is expanded.
3223  SplitInteger(Res, Lo, Hi);
3224  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
3225  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
3227  ExcessBits)));
3228  }
3229 }
3230 
3231 void DAGTypeLegalizer::
3232 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
3233  SDLoc dl(N);
3234  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3235  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3236 
3237  if (EVT.bitsLE(Lo.getValueType())) {
3238  // sext_inreg the low part if needed.
3239  Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
3240  N->getOperand(1));
3241 
3242  // The high part gets the sign extension from the lo-part. This handles
3243  // things like sextinreg V:i64 from i8.
3244  Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
3245  DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
3246  TLI.getPointerTy(DAG.getDataLayout())));
3247  } else {
3248  // For example, extension of an i48 to an i64. Leave the low part alone,
3249  // sext_inreg the high part.
3250  unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
3251  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
3253  ExcessBits)));
3254  }
3255 }
3256 
3257 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
3258  SDValue &Lo, SDValue &Hi) {
3259  EVT VT = N->getValueType(0);
3260  SDLoc dl(N);
3261  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3262 
3264  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3265  SplitInteger(Res.getValue(1), Lo, Hi);
3266  return;
3267  }
3268 
3269  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3270  if (VT == MVT::i16)
3271  LC = RTLIB::SREM_I16;
3272  else if (VT == MVT::i32)
3273  LC = RTLIB::SREM_I32;
3274  else if (VT == MVT::i64)
3275  LC = RTLIB::SREM_I64;
3276  else if (VT == MVT::i128)
3277  LC = RTLIB::SREM_I128;
3278  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
3279 
3281  CallOptions.setSExt(true);
3282  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3283 }
3284 
3285 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
3286  SDValue &Lo, SDValue &Hi) {
3287  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3288  SDLoc dl(N);
3289  Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
3290  Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
3291  N->getOperand(0),
3292  DAG.getConstant(NVT.getSizeInBits(), dl,
3293  TLI.getPointerTy(DAG.getDataLayout())));
3294  Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
3295 }
3296 
3297 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
3298  SDValue &Lo, SDValue &Hi) {
3299  EVT VT = N->getValueType(0);
3300  SDLoc dl(N);
3301 
3302  if (N->getOpcode() == ISD::UMULO) {
3303  // This section expands the operation into the following sequence of
3304  // instructions. `iNh` here refers to a type which has half the bit width of
3305  // the type the original operation operated on.
3306  //
3307  // %0 = %LHS.HI != 0 && %RHS.HI != 0
3308  // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
3309  // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
3310  // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
3311  // %4 = add iN (%1.0 as iN) << Nh, (%2.0 as iN) << Nh
3312  // %5 = { iN, i1 } @uadd.with.overflow.iN( %4, %3 )
3313  //
3314  // %res = { %5.0, %0 || %1.1 || %2.1 || %5.1 }
3315  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
3316  SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
3317  SplitInteger(LHS, LHSLow, LHSHigh);
3318  SplitInteger(RHS, RHSLow, RHSHigh);
3319  EVT HalfVT = LHSLow.getValueType()
3320  , BitVT = N->getValueType(1);
3321  SDVTList VTHalfMulO = DAG.getVTList(HalfVT, BitVT);
3322  SDVTList VTFullAddO = DAG.getVTList(VT, BitVT);
3323 
3324  SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
3325  SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
3326  DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
3327  DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));
3328 
3329  SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, LHSHigh, RHSLow);
3330  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));
3331  SDValue OneInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3332  One.getValue(0));
3333 
3334  SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, RHSHigh, LHSLow);
3335  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));
3336  SDValue TwoInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3337  Two.getValue(0));
3338 
3339  // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
3340  // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
3341  // operation recursively legalized?).
3342  //
3343  // Many backends understand this pattern and will convert into LOHI
3344  // themselves, if applicable.
3345  SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
3346  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
3347  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow));
3348  SDValue Four = DAG.getNode(ISD::ADD, dl, VT, OneInHigh, TwoInHigh);
3349  SDValue Five = DAG.getNode(ISD::UADDO, dl, VTFullAddO, Three, Four);
3350  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Five.getValue(1));
3351  SplitInteger(Five, Lo, Hi);
3352  ReplaceValueWith(SDValue(N, 1), Overflow);
3353  return;
3354  }
3355 
3356  Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
3357  EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
3358  Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
3359 
3360  // Replace this with a libcall that will check overflow.
3361  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3362  if (VT == MVT::i32)
3363  LC = RTLIB::MULO_I32;
3364  else if (VT == MVT::i64)
3365  LC = RTLIB::MULO_I64;
3366  else if (VT == MVT::i128)
3367  LC = RTLIB::MULO_I128;
3368  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
3369 
3370  SDValue Temp = DAG.CreateStackTemporary(PtrVT);
3371  // Temporary for the overflow value, default it to zero.
3372  SDValue Chain =
3373  DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
3374  MachinePointerInfo());
3375 
3377  TargetLowering::ArgListEntry Entry;
3378  for (const SDValue &Op : N->op_values()) {
3379  EVT ArgVT = Op.getValueType();
3380  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
3381  Entry.Node = Op;
3382  Entry.Ty = ArgTy;
3383  Entry.IsSExt = true;
3384  Entry.IsZExt = false;
3385  Args.push_back(Entry);
3386  }
3387 
3388  // Also pass the address of the overflow check.
3389  Entry.Node = Temp;
3390  Entry.Ty = PtrTy->getPointerTo();
3391  Entry.IsSExt = true;
3392  Entry.IsZExt = false;
3393  Args.push_back(Entry);
3394 
3395  SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
3396 
3398  CLI.setDebugLoc(dl)
3399  .setChain(Chain)
3400  .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
3401  .setSExtResult();
3402 
3403  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
3404 
3405  SplitInteger(CallInfo.first, Lo, Hi);
3406  SDValue Temp2 =
3407  DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
3408  SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
3409  DAG.getConstant(0, dl, PtrVT),
3410  ISD::SETNE);
3411  // Use the overflow from the libcall everywhere.
3412  ReplaceValueWith(SDValue(N, 1), Ofl);
3413 }
3414 
3415 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
3416  SDValue &Lo, SDValue &Hi) {
3417  EVT VT = N->getValueType(0);
3418  SDLoc dl(N);
3419  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3420 
3422  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3423  SplitInteger(Res.getValue(0), Lo, Hi);
3424  return;
3425  }
3426 
3427  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3428  if (VT == MVT::i16)
3429  LC = RTLIB::UDIV_I16;
3430  else if (VT == MVT::i32)
3431  LC = RTLIB::UDIV_I32;
3432  else if (VT == MVT::i64)
3433  LC = RTLIB::UDIV_I64;
3434  else if (VT == MVT::i128)
3435  LC = RTLIB::UDIV_I128;
3436  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
3437 
3439  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3440 }
3441 
3442 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
3443  SDValue &Lo, SDValue &Hi) {
3444  EVT VT = N->getValueType(0);
3445  SDLoc dl(N);
3446  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3447 
3449  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3450  SplitInteger(Res.getValue(1), Lo, Hi);
3451  return;
3452  }
3453 
3454  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3455  if (VT == MVT::i16)
3456  LC = RTLIB::UREM_I16;
3457  else if (VT == MVT::i32)
3458  LC = RTLIB::UREM_I32;
3459  else if (VT == MVT::i64)
3460  LC = RTLIB::UREM_I64;
3461  else if (VT == MVT::i128)
3462  LC = RTLIB::UREM_I128;
3463  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
3464 
3466  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3467 }
3468 
3469 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
3470  SDValue &Lo, SDValue &Hi) {
3471  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3472  SDLoc dl(N);
3473  SDValue Op = N->getOperand(0);
3474  if (Op.getValueType().bitsLE(NVT)) {
3475  // The low part is zero extension of the input (degenerates to a copy).
3476  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
3477  Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
3478  } else {
3479  // For example, extension of an i48 to an i64. The operand type necessarily
3480  // promotes to the result type, so will end up being expanded too.
3481  assert(getTypeAction(Op.getValueType()) ==
3483  "Only know how to promote this result!");
3484  SDValue Res = GetPromotedInteger(Op);
3485  assert(Res.getValueType() == N->getValueType(0) &&
3486  "Operand over promoted?");
3487  // Split the promoted operand. This will simplify when it is expanded.
3488  SplitInteger(Res, Lo, Hi);
3489  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
3490  Hi = DAG.getZeroExtendInReg(Hi, dl,
3492  ExcessBits));
3493  }
3494 }
3495 
3496 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
3497  SDValue &Lo, SDValue &Hi) {
3498  SDLoc dl(N);
3499  EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
3500  SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
3501  SDValue Zero = DAG.getConstant(0, dl, VT);
3502  SDValue Swap = DAG.getAtomicCmpSwap(
3504  cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
3505  N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
3506 
3507  ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
3508  ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
3509 }
3510 
3511 void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N,
3512  SDValue &Lo, SDValue &Hi) {
3513  // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate
3514  // both halves independently.
3515  SDValue Res = TLI.expandVecReduce(N, DAG);
3516  SplitInteger(Res, Lo, Hi);
3517 }
3518 
3519 //===----------------------------------------------------------------------===//
3520 // Integer Operand Expansion
3521 //===----------------------------------------------------------------------===//
3522 
3523 /// ExpandIntegerOperand - This method is called when the specified operand of
3524 /// the specified node is found to need expansion. At this point, all of the
3525 /// result types of the node are known to be legal, but other operands of the
3526 /// node may need promotion or expansion as well as the specified one.
3527 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
3528  LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG);
3529  dbgs() << "\n");
3530  SDValue Res = SDValue();
3531 
3532  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
3533  return false;
3534 
3535  switch (N->getOpcode()) {
3536  default:
3537  #ifndef NDEBUG
3538  dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
3539  N->dump(&DAG); dbgs() << "\n";
3540  #endif
3541  report_fatal_error("Do not know how to expand this operator's operand!");
3542 
3543  case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
3544  case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
3545  case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
3546  case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
3547  case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
3548  case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
3549  case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
3550  case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
3551  case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
3552  case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break;
3553  case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
3554  case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
3555  case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break;
3556 
3557  case ISD::SHL:
3558  case ISD::SRA:
3559  case ISD::SRL:
3560  case ISD::ROTL:
3561  case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
3562  case ISD::RETURNADDR:
3563  case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
3564 
3565  case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
3566  }
3567 
3568  // If the result is null, the sub-method took care of registering results etc.
3569  if (!Res.getNode()) return false;
3570 
3571  // If the result is N, the sub-method updated N in place. Tell the legalizer
3572  // core about this.
3573  if (Res.getNode() == N)
3574  return true;
3575 
3576  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3577  "Invalid operand expansion");
3578 
3579  ReplaceValueWith(SDValue(N, 0), Res);
3580  return false;
3581 }
3582 
3583 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
3584 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
3585 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
3586  SDValue &NewRHS,
3587  ISD::CondCode &CCCode,
3588  const SDLoc &dl) {
3589  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3590  GetExpandedInteger(NewLHS, LHSLo, LHSHi);
3591  GetExpandedInteger(NewRHS, RHSLo, RHSHi);
3592 
3593  if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
3594  if (RHSLo == RHSHi) {
3595  if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
3596  if (RHSCST->isAllOnesValue()) {
3597  // Equality comparison to -1.
3598  NewLHS = DAG.getNode(ISD::AND, dl,
3599  LHSLo.getValueType(), LHSLo, LHSHi);
3600  NewRHS = RHSLo;
3601  return;
3602  }
3603  }
3604  }
3605 
3606  NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
3607  NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
3608  NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
3609  NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
3610  return;
3611  }
3612 
3613  // If this is a comparison of the sign bit, just look at the top part.
3614  // X > -1, x < 0
3615  if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
3616  if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0
3617  (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1
3618  NewLHS = LHSHi;
3619  NewRHS = RHSHi;
3620  return;
3621  }
3622 
3623  // FIXME: This generated code sucks.
3624  ISD::CondCode LowCC;
3625  switch (CCCode) {
3626  default: llvm_unreachable("Unknown integer setcc!");
3627  case ISD::SETLT:
3628  case ISD::SETULT: LowCC = ISD::SETULT; break;
3629  case ISD::SETGT:
3630  case ISD::SETUGT: LowCC = ISD::SETUGT; break;
3631  case ISD::SETLE:
3632  case ISD::SETULE: LowCC = ISD::SETULE; break;
3633  case ISD::SETGE:
3634  case ISD::SETUGE: LowCC = ISD::SETUGE; break;
3635  }
3636 
3637  // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
3638  // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
3639  // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
3640 
3641  // NOTE: on targets without efficient SELECT of bools, we can always use
3642  // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
3643  TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
3644  nullptr);
3645  SDValue LoCmp, HiCmp;
3646  if (TLI.isTypeLegal(LHSLo.getValueType()) &&
3647  TLI.isTypeLegal(RHSLo.getValueType()))
3648  LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
3649  RHSLo, LowCC, false, DagCombineInfo, dl);
3650  if (!LoCmp.getNode())
3651  LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
3652  RHSLo, LowCC);
3653  if (TLI.isTypeLegal(LHSHi.getValueType()) &&
3654  TLI.isTypeLegal(RHSHi.getValueType()))
3655  HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
3656  RHSHi, CCCode, false, DagCombineInfo, dl);
3657  if (!HiCmp.getNode())
3658  HiCmp =
3659  DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
3660  LHSHi, RHSHi, DAG.getCondCode(CCCode));
3661 
3662  ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
3663  ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
3664 
3665  bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
3666  CCCode == ISD::SETUGE || CCCode == ISD::SETULE);
3667 
3668  if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) ||
3669  (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) ||
3670  (LoCmpC && LoCmpC->isNullValue())))) {
3671  // For LE / GE, if high part is known false, ignore the low part.
3672  // For LT / GT: if low part is known false, return the high part.
3673  // if high part is known true, ignore the low part.
3674  NewLHS = HiCmp;
3675  NewRHS = SDValue();
3676  return;
3677  }
3678 
3679  if (LHSHi == RHSHi) {
3680  // Comparing the low bits is enough.
3681  NewLHS = LoCmp;
3682  NewRHS = SDValue();
3683  return;
3684  }
3685 
3686  // Lower with SETCCCARRY if the target supports it.
3687  EVT HiVT = LHSHi.getValueType();
3688  EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
3689  bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
3690 
3691  // FIXME: Make all targets support this, then remove the other lowering.
3692  if (HasSETCCCARRY) {
3693  // SETCCCARRY can detect < and >= directly. For > and <=, flip
3694  // operands and condition code.
3695  bool FlipOperands = false;
3696  switch (CCCode) {
3697  case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
3698  case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
3699  case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
3700  case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
3701  default: break;
3702  }
3703  if (FlipOperands) {
3704  std::swap(LHSLo, RHSLo);
3705  std::swap(LHSHi, RHSHi);
3706  }
3707  // Perform a wide subtraction, feeding the carry from the low part into
3708  // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
3709  // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
3710  // zero or positive iff LHS >= RHS.
3711  EVT LoVT = LHSLo.getValueType();
3712  SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
3713  SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
3714  SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
3715  LHSHi, RHSHi, LowCmp.getValue(1),
3716  DAG.getCondCode(CCCode));
3717  NewLHS = Res;
3718  NewRHS = SDValue();
3719  return;
3720  }
3721 
3722  NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
3723  false, DagCombineInfo, dl);
3724  if (!NewLHS.getNode())
3725  NewLHS =
3726  DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
3727  NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
3728  NewRHS = SDValue();
3729 }
3730 
3731 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
3732  SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
3733  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
3734  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3735 
3736  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3737  // against zero to select between true and false values.
3738  if (!NewRHS.getNode()) {
3739  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3740  CCCode = ISD::SETNE;
3741  }
3742 
3743  // Update N to have the operands specified.
3744  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
3745  DAG.getCondCode(CCCode), NewLHS, NewRHS,
3746  N->getOperand(4)), 0);
3747 }
3748 
3749 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
3750  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3751  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
3752  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3753 
3754  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3755  // against zero to select between true and false values.
3756  if (!NewRHS.getNode()) {
3757  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3758  CCCode = ISD::SETNE;
3759  }
3760 
3761  // Update N to have the operands specified.
3762  return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
3763  N->getOperand(2), N->getOperand(3),
3764  DAG.getCondCode(CCCode)), 0);
3765 }
3766 
3767 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
3768  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3769  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3770  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3771 
3772  // If ExpandSetCCOperands returned a scalar, use it.
3773  if (!NewRHS.getNode()) {
3774  assert(NewLHS.getValueType() == N->getValueType(0) &&
3775  "Unexpected setcc expansion!");
3776  return NewLHS;
3777  }
3778 
3779  // Otherwise, update N to have the operands specified.
3780  return SDValue(
3781  DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
3782 }
3783 
3784 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
3785  SDValue LHS = N->getOperand(0);
3786  SDValue RHS = N->getOperand(1);
3787  SDValue Carry = N->getOperand(2);
3788  SDValue Cond = N->getOperand(3);
3789  SDLoc dl = SDLoc(N);
3790 
3791  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3792  GetExpandedInteger(LHS, LHSLo, LHSHi);
3793  GetExpandedInteger(RHS, RHSLo, RHSHi);
3794 
3795  // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
3796  SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
3797  SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
3798  return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
3799  LowCmp.getValue(1), Cond);
3800 }
3801 
3802 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
3803  // The value being shifted is legal, but the shift amount is too big.
3804  // It follows that either the result of the shift is undefined, or the
3805  // upper half of the shift amount is zero. Just use the lower half.
3806  SDValue Lo, Hi;
3807  GetExpandedInteger(N->getOperand(1), Lo, Hi);
3808  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
3809 }
3810 
3811 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
3812  // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
3813  // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
3814  // constant to valid type.
3815  SDValue Lo, Hi;
3816  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3817  return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
3818 }
3819 
3820 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
3821  SDValue Op = N->getOperand(0);
3822  EVT DstVT = N->getValueType(0);
3824  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3825  "Don't know how to expand this SINT_TO_FP!");
3827  CallOptions.setSExt(true);
3828  return TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N)).first;
3829 }
3830 
3831 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
3832  if (ISD::isNormalStore(N))
3833  return ExpandOp_NormalStore(N, OpNo);
3834 
3835  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
3836  assert(OpNo == 1 && "Can only expand the stored value so far");
3837 
3838  EVT VT = N->getOperand(1).getValueType();
3839  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3840  SDValue Ch = N->getChain();
3841  SDValue Ptr = N->getBasePtr();
3842  unsigned Alignment = N->getAlignment();
3843  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3844  AAMDNodes AAInfo = N->getAAInfo();
3845  SDLoc dl(N);
3846  SDValue Lo, Hi;
3847 
3848  assert(NVT.isByteSized() && "Expanded type not byte sized!");
3849 
3850  if (N->getMemoryVT().bitsLE(NVT)) {
3851  GetExpandedInteger(N->getValue(), Lo, Hi);
3852  return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
3853  N->getMemoryVT(), Alignment, MMOFlags, AAInfo);
3854  }
3855 
3856  if (DAG.getDataLayout().isLittleEndian()) {
3857  // Little-endian - low bits are at low addresses.
3858  GetExpandedInteger(N->getValue(), Lo, Hi);
3859 
3860  Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3861  AAInfo);
3862 
3863  unsigned ExcessBits =
3864  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3865  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3866 
3867  // Increment the pointer to the other half.
3868  unsigned IncrementSize = NVT.getSizeInBits()/8;
3869  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3870  Hi = DAG.getTruncStore(
3871  Ch, dl, Hi, Ptr, N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3872  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3873  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3874  }
3875 
3876  // Big-endian - high bits are at low addresses. Favor aligned stores at
3877  // the cost of some bit-fiddling.
3878  GetExpandedInteger(N->getValue(), Lo, Hi);
3879 
3880  EVT ExtVT = N->getMemoryVT();
3881  unsigned EBytes = ExtVT.getStoreSize();
3882  unsigned IncrementSize = NVT.getSizeInBits()/8;
3883  unsigned ExcessBits = (EBytes - IncrementSize)*8;
3884  EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
3885  ExtVT.getSizeInBits() - ExcessBits);
3886 
3887  if (ExcessBits < NVT.getSizeInBits()) {
3888  // Transfer high bits from the top of Lo to the bottom of Hi.
3889  Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
3890  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3891  TLI.getPointerTy(DAG.getDataLayout())));
3892  Hi = DAG.getNode(
3893  ISD::OR, dl, NVT, Hi,
3894  DAG.getNode(ISD::SRL, dl, NVT, Lo,
3895  DAG.getConstant(ExcessBits, dl,
3896  TLI.getPointerTy(DAG.getDataLayout()))));
3897  }
3898 
3899  // Store both the high bits and maybe some of the low bits.
3900  Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment,
3901  MMOFlags, AAInfo);
3902 
3903  // Increment the pointer to the other half.
3904  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3905  // Store the lowest ExcessBits bits in the second half.
3906  Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
3907  N->getPointerInfo().getWithOffset(IncrementSize),
3908  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3909  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3910  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3911 }
3912 
3913 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
3914  SDValue InL, InH;
3915  GetExpandedInteger(N->getOperand(0), InL, InH);
3916  // Just truncate the low part of the source.
3917  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
3918 }
3919 
3920 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
3921  SDValue Op = N->getOperand(0);
3922  EVT SrcVT = Op.getValueType();
3923  EVT DstVT = N->getValueType(0);
3924  SDLoc dl(N);
3925 
3926  // The following optimization is valid only if every value in SrcVT (when
3927  // treated as signed) is representable in DstVT. Check that the mantissa
3928  // size of DstVT is >= than the number of bits in SrcVT -1.
3929  const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT);
3930  if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 &&
3932  // Do a signed conversion then adjust the result.
3933  SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op);
3934  SignedConv = TLI.LowerOperation(SignedConv, DAG);
3935 
3936  // The result of the signed conversion needs adjusting if the 'sign bit' of
3937  // the incoming integer was set. To handle this, we dynamically test to see
3938  // if it is set, and, if so, add a fudge factor.
3939 
3940  const uint64_t F32TwoE32 = 0x4F800000ULL;
3941  const uint64_t F32TwoE64 = 0x5F800000ULL;
3942  const uint64_t F32TwoE128 = 0x7F800000ULL;
3943 
3944  APInt FF(32, 0);
3945  if (SrcVT == MVT::i32)
3946  FF = APInt(32, F32TwoE32);
3947  else if (SrcVT == MVT::i64)
3948  FF = APInt(32, F32TwoE64);
3949  else if (SrcVT == MVT::i128)
3950  FF = APInt(32, F32TwoE128);
3951  else
3952  llvm_unreachable("Unsupported UINT_TO_FP!");
3953 
3954  // Check whether the sign bit is set.
3955  SDValue Lo, Hi;
3956  GetExpandedInteger(Op, Lo, Hi);
3957  SDValue SignSet = DAG.getSetCC(dl,
3958  getSetCCResultType(Hi.getValueType()),
3959  Hi,
3960  DAG.getConstant(0, dl, Hi.getValueType()),
3961  ISD::SETLT);
3962 
3963  // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
3964  SDValue FudgePtr =
3965  DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)),
3966  TLI.getPointerTy(DAG.getDataLayout()));
3967 
3968  // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
3969  SDValue Zero = DAG.getIntPtrConstant(0, dl);
3970  SDValue Four = DAG.getIntPtrConstant(4, dl);
3971  if (DAG.getDataLayout().isBigEndian())
3972  std::swap(Zero, Four);
3973  SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet,
3974  Zero, Four);
3975  unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment();
3976  FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(),
3977  FudgePtr, Offset);
3978  Alignment = std::min(Alignment, 4u);
3979 
3980  // Load the value out, extending it from f32 to the destination float type.
3981  // FIXME: Avoid the extend by constructing the right constant pool?
3982  SDValue Fudge = DAG.getExtLoad(
3983  ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), FudgePtr,
3985  Alignment);
3986  return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge);
3987  }
3988 
3989  // Otherwise, use a libcall.
3990  RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT);
3991  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3992  "Don't know how to expand this UINT_TO_FP!");
3994  CallOptions.setSExt(true);
3995  return TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, dl).first;
3996 }
3997 
3998 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
3999  SDLoc dl(N);
4000  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
4001  cast<AtomicSDNode>(N)->getMemoryVT(),
4002  N->getOperand(0),
4003  N->getOperand(1), N->getOperand(2),
4004  cast<AtomicSDNode>(N)->getMemOperand());
4005  return Swap.getValue(1);
4006 }
4007 
4008 
4009 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
4010 
4011  EVT OutVT = N->getValueType(0);
4012  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4013  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4014  unsigned OutNumElems = OutVT.getVectorNumElements();
4015  EVT NOutVTElem = NOutVT.getVectorElementType();
4016 
4017  SDLoc dl(N);
4018  SDValue BaseIdx = N->getOperand(1);
4019 
4020  SDValue InOp0 = N->getOperand(0);
4021  if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger)
4022  InOp0 = GetPromotedInteger(N->getOperand(0));
4023 
4024  EVT InVT = InOp0.getValueType();
4025 
4027  Ops.reserve(OutNumElems);
4028  for (unsigned i = 0; i != OutNumElems; ++i) {
4029 
4030  // Extract the element from the original vector.
4031  SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
4032  BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
4034  InVT.getVectorElementType(), N->getOperand(0), Index);
4035 
4036  SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem);
4037  // Insert the converted element to the new vector.
4038  Ops.push_back(Op);
4039  }
4040 
4041  return DAG.getBuildVector(NOutVT, dl, Ops);
4042 }
4043 
4044 
4045 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
4046  ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
4047  EVT VT = N->getValueType(0);
4048  SDLoc dl(N);
4049 
4050  ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
4051 
4052  SDValue V0 = GetPromotedInteger(N->getOperand(0));
4053  SDValue V1 = GetPromotedInteger(N->getOperand(1));
4054  EVT OutVT = V0.getValueType();
4055 
4056  return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
4057 }
4058 
4059 
4060 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
4061  EVT OutVT = N->getValueType(0);
4062  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4063  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4064  unsigned NumElems = N->getNumOperands();
4065  EVT NOutVTElem = NOutVT.getVectorElementType();
4066 
4067  SDLoc dl(N);
4068 
4070  Ops.reserve(NumElems);
4071  for (unsigned i = 0; i != NumElems; ++i) {
4072  SDValue Op;
4073  // BUILD_VECTOR integer operand types are allowed to be larger than the
4074  // result's element type. This may still be true after the promotion. For
4075  // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
4076  // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
4077  if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
4078  Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
4079  else
4080  Op = N->getOperand(i);
4081  Ops.push_back(Op);
4082  }
4083 
4084  return DAG.getBuildVector(NOutVT, dl, Ops);
4085 }
4086 
4087 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
4088 
4089  SDLoc dl(N);
4090 
4091  assert(!N->getOperand(0).getValueType().isVector() &&
4092  "Input must be a scalar");
4093 
4094  EVT OutVT = N->getValueType(0);
4095  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4096  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4097  EVT NOutVTElem = NOutVT.getVectorElementType();
4098 
4099  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
4100 
4101  return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
4102 }
4103 
4104 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
4105  SDLoc dl(N);
4106 
4107  EVT OutVT = N->getValueType(0);
4108  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4109  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4110 
4111  EVT OutElemTy = NOutVT.getVectorElementType();
4112 
4113  unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
4114  unsigned NumOutElem = NOutVT.getVectorNumElements();
4115  unsigned NumOperands = N->getNumOperands();
4116  assert(NumElem * NumOperands == NumOutElem &&
4117  "Unexpected number of elements");
4118 
4119  // Take the elements from the first vector.
4120  SmallVector<SDValue, 8> Ops(NumOutElem);
4121  for (unsigned i = 0; i < NumOperands; ++i) {
4122  SDValue Op = N->getOperand(i);
4123  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
4124  Op = GetPromotedInteger(Op);
4125  EVT SclrTy = Op.getValueType().getVectorElementType();
4126  assert(NumElem == Op.getValueType().getVectorNumElements() &&
4127  "Unexpected number of elements");
4128 
4129  for (unsigned j = 0; j < NumElem; ++j) {
4130  SDValue Ext = DAG.getNode(
4131  ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
4132  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4133  Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
4134  }
4135  }
4136 
4137  return DAG.getBuildVector(NOutVT, dl, Ops);
4138 }
4139 
4140 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
4141  EVT VT = N->getValueType(0);
4142  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4143  assert(NVT.isVector() && "This type must be promoted to a vector type");
4144 
4145  SDLoc dl(N);
4146 
4147  // For operands whose TypeAction is to promote, extend the promoted node
4148  // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
4149  // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
4150  // type..
4151  if (getTypeAction(N->getOperand(0).getValueType())
4153  SDValue Promoted;
4154 
4155  switch(N->getOpcode()) {
4157  Promoted = SExtPromotedInteger(N->getOperand(0));
4158  break;
4160  Promoted = ZExtPromotedInteger(N->getOperand(0));
4161  break;
4163  Promoted = GetPromotedInteger(N->getOperand(0));
4164  break;
4165  default:
4166  llvm_unreachable("Node has unexpected Opcode");
4167  }
4168  return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
4169  }
4170 
4171  // Directly extend to the appropriate transform-to type.
4172  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
4173 }
4174 
4175 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
4176  EVT OutVT = N->getValueType(0);
4177  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4178  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4179 
4180  EVT NOutVTElem = NOutVT.getVectorElementType();
4181 
4182  SDLoc dl(N);
4183  SDValue V0 = GetPromotedInteger(N->getOperand(0));
4184 
4185  SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
4186  NOutVTElem, N->getOperand(1));
4187  return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
4188  V0, ConvElem, N->getOperand(2));
4189 }
4190 
4191 SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) {
4192  // The VECREDUCE result size may be larger than the element size, so
4193  // we can simply change the result type.
4194  SDLoc dl(N);
4195  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4196  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
4197 }
4198 
4199 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4200  SDLoc dl(N);
4201  SDValue V0 = GetPromotedInteger(N->getOperand(0));
4202  SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
4203  TLI.getVectorIdxTy(DAG.getDataLayout()));
4205  V0->getValueType(0).getScalarType(), V0, V1);
4206 
4207  // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
4208  // element types. If this is the case then we need to expand the outgoing
4209  // value and not truncate it.
4210  return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
4211 }
4212 
4213 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
4214  SDLoc dl(N);
4215  SDValue V0 = GetPromotedInteger(N->getOperand(0));
4216  MVT InVT = V0.getValueType().getSimpleVT();
4217  MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
4219  SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
4220  return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
4221 }
4222 
4223 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
4224  SDLoc dl(N);
4225  unsigned NumElems = N->getNumOperands();
4226 
4227  EVT RetSclrTy = N->getValueType(0).getVectorElementType();
4228 
4229  SmallVector<SDValue, 8> NewOps;
4230  NewOps.reserve(NumElems);
4231 
4232  // For each incoming vector
4233  for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
4234  SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
4235  EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
4236  unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
4237 
4238  for (unsigned i=0; i<NumElem; ++i) {
4239  // Extract element from incoming vector
4240  SDValue Ex = DAG.getNode(
4241  ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
4242  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4243  SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
4244  NewOps.push_back(Tr);
4245  }
4246  }
4247 
4248  return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
4249 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:595
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:551
static EVT getShiftAmountTyForConstant(unsigned Val, EVT VT, const TargetLowering &TLI, SelectionDAG &DAG)
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getValueType() const
Return the ValueType of the referenced return value.
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant, which is required to be operand #1) half of the integer or float value specified as operand #0.
Definition: ISDOpcodes.h:183
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
Definition: APInt.h:561
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC&#39;s if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:959
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:389
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:678
This class represents lattice values for constants.
Definition: AllocatorList.h:23
static MVT getVectorVT(MVT VT, unsigned NumElements)
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:398
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:259
This structure is used to pass arguments to makeLibCall function.
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:284
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:550
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none...
const SDValue & getBasePtr() const
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:222
void push_back(const T &Elt)
Definition: SmallVector.h:211
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:860
const SDValue & getValue() const
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
Definition: APInt.h:647
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:252
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:250
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1328
const SDValue & getBasePtr() const
const SDValue & getChain() const
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
unsigned getAlignment() const
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:833
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select&#39;s if you just have operands and don&#39;t want to check...
Definition: SelectionDAG.h:972
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:539
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:415
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align=0, int Offs=0, bool isT=false, unsigned TargetFlags=0)
Same for subtraction.
Definition: ISDOpcodes.h:253
void reserve(size_type N)
Definition: SmallVector.h:369
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:211
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:466
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:38
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition: APInt.h:534
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
SDValue getExternalSymbol(const char *Sym, EVT VT)
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
virtual bool shouldExpandShift(SelectionDAG &DAG, SDNode *N) const
Return true if SHIFT instructions should be expanded to SHIFT_PARTS instructions, and false if a libr...
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:209
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations...
Definition: ISDOpcodes.h:487
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
static uint32_t getAlignment(const MCSectionCOFF &Sec)
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:450
Shift and rotation operations.
Definition: ISDOpcodes.h:441
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:263
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth...
Definition: ISDOpcodes.h:424
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:654
CallLoweringInfo & setChain(SDValue InChain)
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:190
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:279
const SDValue & getPassThru() const
op_iterator op_end() const
unsigned getScalarValueSizeInBits() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
FLT_ROUNDS_ - Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest 2 Round to ...
Definition: ISDOpcodes.h:577
SimpleValueType SimpleTy
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:303
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt) For double-word atomic operations: ValLo, ValHi, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amtLo, amtHi) ValLo, ValHi, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amtLo, amtHi) These correspond to the atomicrmw instruction.
Definition: ISDOpcodes.h:841
const SDValue & getValue() const
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:473
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:413
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG...
Definition: ISDOpcodes.h:72
This is an SDNode representing atomic operations.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
This represents a list of ValueType&#39;s that has been intern&#39;d by a SelectionDAG.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
This class is used to represent an MSTORE node.
unsigned getSizeInBits() const
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:261
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:297
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:291
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:410
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:509
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
const SDValue & getScale() const
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:460
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:233
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:874
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTruncating=false, bool IsCompressing=false)
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:1012
op_iterator op_begin() const
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:778
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:528
const SDValue & getMask() const
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl) const
Returns a pair of (return value, chain).
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands...
UNDEF - An undefined node.
Definition: ISDOpcodes.h:177
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:555
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:358
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
Definition: APInt.h:635
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits...
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
Definition: MathExtras.h:614
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const SDValue & getBasePtr() const