LLVM  9.0.0svn
LegalizeIntegerTypes.cpp
Go to the documentation of this file.
1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements integer type expansion and promotion for LegalizeTypes.
10 // Promotion is the act of changing a computation in an illegal type into a
11 // computation in a larger type. For example, implementing i8 arithmetic in an
12 // i32 register (often needed on powerpc).
13 // Expansion is the act of changing a computation in an illegal type into a
14 // computation in two identical registers of a smaller type. For example,
15 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
16 // targets).
17 //
18 //===----------------------------------------------------------------------===//
19 
20 #include "LegalizeTypes.h"
21 #include "llvm/IR/DerivedTypes.h"
23 #include "llvm/Support/KnownBits.h"
25 using namespace llvm;
26 
27 #define DEBUG_TYPE "legalize-types"
28 
29 //===----------------------------------------------------------------------===//
30 // Integer Result Promotion
31 //===----------------------------------------------------------------------===//
32 
33 /// PromoteIntegerResult - This method is called when a result of a node is
34 /// found to be in need of promotion to a larger type. At this point, the node
35 /// may also have invalid operands or may have other results that need
36 /// expansion, we just know that (at least) one result needs promotion.
37 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
38  LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG);
39  dbgs() << "\n");
40  SDValue Res = SDValue();
41 
42  // See if the target wants to custom expand this node.
43  if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
44  LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
45  return;
46  }
47 
48  switch (N->getOpcode()) {
49  default:
50 #ifndef NDEBUG
51  dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
52  N->dump(&DAG); dbgs() << "\n";
53 #endif
54  llvm_unreachable("Do not know how to promote this operator!");
55  case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
56  case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
57  case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
58  case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
59  case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
60  case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
61  case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
62  case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
64  case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
65  case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break;
67  case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
69  Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
70  case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
71  case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
72  break;
73  case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
74  break;
75  case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
76  case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
77  case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
78  case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
79  case ISD::SMIN:
80  case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
81  case ISD::UMIN:
82  case ISD::UMAX: Res = PromoteIntRes_ZExtIntBinOp(N); break;
83 
84  case ISD::SHL: Res = PromoteIntRes_SHL(N); break;
86  Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
87  case ISD::SRA: Res = PromoteIntRes_SRA(N); break;
88  case ISD::SRL: Res = PromoteIntRes_SRL(N); break;
89  case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
90  case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
91  case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
92 
94  Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
96  Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
98  Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
99  case ISD::BUILD_VECTOR:
100  Res = PromoteIntRes_BUILD_VECTOR(N); break;
102  Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
103  case ISD::CONCAT_VECTORS:
104  Res = PromoteIntRes_CONCAT_VECTORS(N); break;
105 
109  Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
110 
111  case ISD::SIGN_EXTEND:
112  case ISD::ZERO_EXTEND:
113  case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
114 
115  case ISD::FP_TO_SINT:
116  case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
117 
118  case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
119 
120  case ISD::FLT_ROUNDS_: Res = PromoteIntRes_FLT_ROUNDS(N); break;
121 
122  case ISD::AND:
123  case ISD::OR:
124  case ISD::XOR:
125  case ISD::ADD:
126  case ISD::SUB:
127  case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
128 
129  case ISD::SDIV:
130  case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
131 
132  case ISD::UDIV:
133  case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
134 
135  case ISD::SADDO:
136  case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
137  case ISD::UADDO:
138  case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
139  case ISD::SMULO:
140  case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
141 
142  case ISD::ADDE:
143  case ISD::SUBE:
144  case ISD::ADDCARRY:
145  case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
146 
147  case ISD::SADDSAT:
148  case ISD::UADDSAT:
149  case ISD::SSUBSAT:
150  case ISD::USUBSAT: Res = PromoteIntRes_ADDSUBSAT(N); break;
151  case ISD::SMULFIX:
152  case ISD::UMULFIX: Res = PromoteIntRes_MULFIX(N); break;
153 
154  case ISD::ATOMIC_LOAD:
155  Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
156 
161  case ISD::ATOMIC_LOAD_OR:
168  case ISD::ATOMIC_SWAP:
169  Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
170 
173  Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
174  break;
175  }
176 
177  // If the result is null then the sub-method took care of registering it.
178  if (Res.getNode())
179  SetPromotedInteger(SDValue(N, ResNo), Res);
180 }
181 
182 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
183  unsigned ResNo) {
184  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
185  return GetPromotedInteger(Op);
186 }
187 
188 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
189  // Sign-extend the new bits, and continue the assertion.
190  SDValue Op = SExtPromotedInteger(N->getOperand(0));
191  return DAG.getNode(ISD::AssertSext, SDLoc(N),
192  Op.getValueType(), Op, N->getOperand(1));
193 }
194 
195 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
196  // Zero the new bits, and continue the assertion.
197  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
198  return DAG.getNode(ISD::AssertZext, SDLoc(N),
199  Op.getValueType(), Op, N->getOperand(1));
200 }
201 
202 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
203  EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
204  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
205  N->getMemoryVT(), ResVT,
206  N->getChain(), N->getBasePtr(),
207  N->getMemOperand());
208  // Legalize the chain result - switch anything that used the old chain to
209  // use the new one.
210  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
211  return Res;
212 }
213 
214 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
215  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
216  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
217  N->getMemoryVT(),
218  N->getChain(), N->getBasePtr(),
219  Op2, N->getMemOperand());
220  // Legalize the chain result - switch anything that used the old chain to
221  // use the new one.
222  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
223  return Res;
224 }
225 
226 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
227  unsigned ResNo) {
228  if (ResNo == 1) {
230  EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
231  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
232 
233  // Only use the result of getSetCCResultType if it is legal,
234  // otherwise just use the promoted result type (NVT).
235  if (!TLI.isTypeLegal(SVT))
236  SVT = NVT;
237 
238  SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
239  SDValue Res = DAG.getAtomicCmpSwap(
241  N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
242  N->getMemOperand());
243  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
244  ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
245  return Res.getValue(1);
246  }
247 
248  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
249  SDValue Op3 = GetPromotedInteger(N->getOperand(3));
250  SDVTList VTs =
251  DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
252  SDValue Res = DAG.getAtomicCmpSwap(
253  N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
254  N->getBasePtr(), Op2, Op3, N->getMemOperand());
255  // Update the use to N with the newly created Res.
256  for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
257  ReplaceValueWith(SDValue(N, i), Res.getValue(i));
258  return Res;
259 }
260 
261 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
262  SDValue InOp = N->getOperand(0);
263  EVT InVT = InOp.getValueType();
264  EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
265  EVT OutVT = N->getValueType(0);
266  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
267  SDLoc dl(N);
268 
269  switch (getTypeAction(InVT)) {
271  break;
273  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
274  // The input promotes to the same size. Convert the promoted value.
275  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
276  break;
278  // Promote the integer operand by hand.
279  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
281  // Convert the promoted float by hand.
282  if (!NOutVT.isVector())
283  return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
284  break;
285  }
288  break;
290  // Convert the element to an integer and promote it by hand.
291  if (!NOutVT.isVector())
292  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
293  BitConvertToInteger(GetScalarizedVector(InOp)));
294  break;
296  // For example, i32 = BITCAST v2i16 on alpha. Convert the split
297  // pieces of the input into integers and reassemble in the final type.
298  SDValue Lo, Hi;
299  GetSplitVector(N->getOperand(0), Lo, Hi);
300  Lo = BitConvertToInteger(Lo);
301  Hi = BitConvertToInteger(Hi);
302 
303  if (DAG.getDataLayout().isBigEndian())
304  std::swap(Lo, Hi);
305 
306  InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
308  NOutVT.getSizeInBits()),
309  JoinIntegers(Lo, Hi));
310  return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
311  }
313  // The input is widened to the same size. Convert to the widened value.
314  // Make sure that the outgoing value is not a vector, because this would
315  // make us bitcast between two vectors which are legalized in different ways.
316  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector())
317  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
318  // If the output type is also a vector and widening it to the same size
319  // as the widened input type would be a legal type, we can widen the bitcast
320  // and handle the promotion after.
321  if (NOutVT.isVector()) {
322  unsigned WidenInSize = NInVT.getSizeInBits();
323  unsigned OutSize = OutVT.getSizeInBits();
324  if (WidenInSize % OutSize == 0) {
325  unsigned Scale = WidenInSize / OutSize;
326  EVT WideOutVT = EVT::getVectorVT(*DAG.getContext(),
327  OutVT.getVectorElementType(),
328  OutVT.getVectorNumElements() * Scale);
329  if (isTypeLegal(WideOutVT)) {
330  InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
331  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
332  InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
333  DAG.getConstant(0, dl, IdxTy));
334  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
335  }
336  }
337  }
338  }
339 
340  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
341  CreateStackStoreLoad(InOp, OutVT));
342 }
343 
344 // Helper for BSWAP/BITREVERSE promotion to ensure we can fit the shift amount
345 // in the VT returned by getShiftAmountTy and to return a safe VT if we can't.
346 static EVT getShiftAmountTyForConstant(unsigned Val, EVT VT,
347  const TargetLowering &TLI,
348  SelectionDAG &DAG) {
349  EVT ShiftVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
350  // If the value won't fit in the prefered type, just use something safe. It
351  // will be legalized when the shift is expanded.
352  if ((Log2_32(Val) + 1) > ShiftVT.getScalarSizeInBits())
353  ShiftVT = MVT::i32;
354  return ShiftVT;
355 }
356 
357 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
358  SDValue Op = GetPromotedInteger(N->getOperand(0));
359  EVT OVT = N->getValueType(0);
360  EVT NVT = Op.getValueType();
361  SDLoc dl(N);
362 
363  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
364  EVT ShiftVT = getShiftAmountTyForConstant(DiffBits, NVT, TLI, DAG);
365  return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
366  DAG.getConstant(DiffBits, dl, ShiftVT));
367 }
368 
369 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
370  SDValue Op = GetPromotedInteger(N->getOperand(0));
371  EVT OVT = N->getValueType(0);
372  EVT NVT = Op.getValueType();
373  SDLoc dl(N);
374 
375  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
376  EVT ShiftVT = getShiftAmountTyForConstant(DiffBits, NVT, TLI, DAG);
377  return DAG.getNode(ISD::SRL, dl, NVT,
378  DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
379  DAG.getConstant(DiffBits, dl, ShiftVT));
380 }
381 
382 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
383  // The pair element type may be legal, or may not promote to the same type as
384  // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
385  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
386  TLI.getTypeToTransformTo(*DAG.getContext(),
387  N->getValueType(0)), JoinIntegers(N->getOperand(0),
388  N->getOperand(1)));
389 }
390 
391 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
392  EVT VT = N->getValueType(0);
393  // FIXME there is no actual debug info here
394  SDLoc dl(N);
395  // Zero extend things like i1, sign extend everything else. It shouldn't
396  // matter in theory which one we pick, but this tends to give better code?
397  unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
398  SDValue Result = DAG.getNode(Opc, dl,
399  TLI.getTypeToTransformTo(*DAG.getContext(), VT),
400  SDValue(N, 0));
401  assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
402  return Result;
403 }
404 
405 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
406  // Zero extend to the promoted type and do the count there.
407  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
408  SDLoc dl(N);
409  EVT OVT = N->getValueType(0);
410  EVT NVT = Op.getValueType();
411  Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
412  // Subtract off the extra leading bits in the bigger type.
413  return DAG.getNode(
414  ISD::SUB, dl, NVT, Op,
415  DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
416  NVT));
417 }
418 
419 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
420  // Zero extend to the promoted type and do the count there.
421  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
422  return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op);
423 }
424 
425 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
426  SDValue Op = GetPromotedInteger(N->getOperand(0));
427  EVT OVT = N->getValueType(0);
428  EVT NVT = Op.getValueType();
429  SDLoc dl(N);
430  if (N->getOpcode() == ISD::CTTZ) {
431  // The count is the same in the promoted type except if the original
432  // value was zero. This can be handled by setting the bit just off
433  // the top of the original type.
434  auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
435  OVT.getScalarSizeInBits());
436  Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
437  }
438  return DAG.getNode(N->getOpcode(), dl, NVT, Op);
439 }
440 
441 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
442  SDLoc dl(N);
443  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
444 
445  SDValue Op0 = N->getOperand(0);
446  SDValue Op1 = N->getOperand(1);
447 
448  // If the input also needs to be promoted, do that first so we can get a
449  // get a good idea for the output type.
450  if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
452  SDValue In = GetPromotedInteger(Op0);
453 
454  // If the new type is larger than NVT, use it. We probably won't need to
455  // promote it again.
456  EVT SVT = In.getValueType().getScalarType();
457  if (SVT.bitsGE(NVT)) {
458  SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
459  return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
460  }
461  }
462 
463  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
464 }
465 
466 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
467  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
468  unsigned NewOpc = N->getOpcode();
469  SDLoc dl(N);
470 
471  // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
472  // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
473  // and SINT conversions are Custom, there is no way to tell which is
474  // preferable. We choose SINT because that's the right thing on PPC.)
475  if (N->getOpcode() == ISD::FP_TO_UINT &&
476  !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
478  NewOpc = ISD::FP_TO_SINT;
479 
480  SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
481 
482  // Assert that the converted value fits in the original type. If it doesn't
483  // (eg: because the value being converted is too big), then the result of the
484  // original operation was undefined anyway, so the assert is still correct.
485  //
486  // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
487  // before legalization: fp-to-uint16, 65534. -> 0xfffe
488  // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
489  return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ?
490  ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
492 }
493 
494 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
495  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
496  SDLoc dl(N);
497 
498  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
499 }
500 
501 SDValue DAGTypeLegalizer::PromoteIntRes_FLT_ROUNDS(SDNode *N) {
502  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
503  SDLoc dl(N);
504 
505  return DAG.getNode(N->getOpcode(), dl, NVT);
506 }
507 
508 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
509  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
510  SDLoc dl(N);
511 
512  if (getTypeAction(N->getOperand(0).getValueType())
514  SDValue Res = GetPromotedInteger(N->getOperand(0));
515  assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
516 
517  // If the result and operand types are the same after promotion, simplify
518  // to an in-register extension.
519  if (NVT == Res.getValueType()) {
520  // The high bits are not guaranteed to be anything. Insert an extend.
521  if (N->getOpcode() == ISD::SIGN_EXTEND)
522  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
523  DAG.getValueType(N->getOperand(0).getValueType()));
524  if (N->getOpcode() == ISD::ZERO_EXTEND)
525  return DAG.getZeroExtendInReg(Res, dl,
527  assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
528  return Res;
529  }
530  }
531 
532  // Otherwise, just extend the original operand all the way to the larger type.
533  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
534 }
535 
536 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
537  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
538  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
541  SDLoc dl(N);
542  SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
543  N->getMemoryVT(), N->getMemOperand());
544 
545  // Legalize the chain result - switch anything that used the old chain to
546  // use the new one.
547  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
548  return Res;
549 }
550 
551 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
552  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
553  SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
554 
555  SDLoc dl(N);
556  SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
557  N->getMask(), ExtPassThru, N->getMemoryVT(),
559  // Legalize the chain result - switch anything that used the old chain to
560  // use the new one.
561  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
562  return Res;
563 }
564 
565 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
566  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
567  SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
568  assert(NVT == ExtPassThru.getValueType() &&
569  "Gather result type and the passThru agrument type should be the same");
570 
571  SDLoc dl(N);
572  SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
573  N->getIndex(), N->getScale() };
574  SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
575  N->getMemoryVT(), dl, Ops,
576  N->getMemOperand());
577  // Legalize the chain result - switch anything that used the old chain to
578  // use the new one.
579  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
580  return Res;
581 }
582 
583 /// Promote the overflow flag of an overflowing arithmetic node.
584 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
585  // Simply change the return type of the boolean result.
586  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
587  EVT ValueVTs[] = { N->getValueType(0), NVT };
588  SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
589  unsigned NumOps = N->getNumOperands();
590  assert(NumOps <= 3 && "Too many operands");
591  if (NumOps == 3)
592  Ops[2] = N->getOperand(2);
593 
594  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
595  DAG.getVTList(ValueVTs), makeArrayRef(Ops, NumOps));
596 
597  // Modified the sum result - switch anything that used the old sum to use
598  // the new one.
599  ReplaceValueWith(SDValue(N, 0), Res);
600 
601  return SDValue(Res.getNode(), 1);
602 }
603 
604 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSAT(SDNode *N) {
605  // For promoting iN -> iM, this can be expanded by
606  // 1. ANY_EXTEND iN to iM
607  // 2. SHL by M-N
608  // 3. [US][ADD|SUB]SAT
609  // 4. L/ASHR by M-N
610  SDLoc dl(N);
611  SDValue Op1 = N->getOperand(0);
612  SDValue Op2 = N->getOperand(1);
613  unsigned OldBits = Op1.getScalarValueSizeInBits();
614 
615  unsigned Opcode = N->getOpcode();
616  unsigned ShiftOp;
617  switch (Opcode) {
618  case ISD::SADDSAT:
619  case ISD::SSUBSAT:
620  ShiftOp = ISD::SRA;
621  break;
622  case ISD::UADDSAT:
623  case ISD::USUBSAT:
624  ShiftOp = ISD::SRL;
625  break;
626  default:
627  llvm_unreachable("Expected opcode to be signed or unsigned saturation "
628  "addition or subtraction");
629  }
630 
631  SDValue Op1Promoted = GetPromotedInteger(Op1);
632  SDValue Op2Promoted = GetPromotedInteger(Op2);
633 
634  EVT PromotedType = Op1Promoted.getValueType();
635  unsigned NewBits = PromotedType.getScalarSizeInBits();
636  unsigned SHLAmount = NewBits - OldBits;
637  EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
638  SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT);
639  Op1Promoted =
640  DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
641  Op2Promoted =
642  DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
643 
644  SDValue Result =
645  DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
646  return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
647 }
648 
649 SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
650  // Can just promote the operands then continue with operation.
651  SDLoc dl(N);
652  SDValue Op1Promoted, Op2Promoted;
653  if (N->getOpcode() == ISD::SMULFIX) {
654  Op1Promoted = SExtPromotedInteger(N->getOperand(0));
655  Op2Promoted = SExtPromotedInteger(N->getOperand(1));
656  } else {
657  Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
658  Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
659  }
660  EVT PromotedType = Op1Promoted.getValueType();
661  return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
662  N->getOperand(2));
663 }
664 
665 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
666  if (ResNo == 1)
667  return PromoteIntRes_Overflow(N);
668 
669  // The operation overflowed iff the result in the larger type is not the
670  // sign extension of its truncation to the original type.
671  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
672  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
673  EVT OVT = N->getOperand(0).getValueType();
674  EVT NVT = LHS.getValueType();
675  SDLoc dl(N);
676 
677  // Do the arithmetic in the larger type.
678  unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
679  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
680 
681  // Calculate the overflow flag: sign extend the arithmetic result from
682  // the original type.
683  SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
684  DAG.getValueType(OVT));
685  // Overflowed if and only if this is not equal to Res.
686  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
687 
688  // Use the calculated overflow everywhere.
689  ReplaceValueWith(SDValue(N, 1), Ofl);
690 
691  return Res;
692 }
693 
694 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
695  SDValue LHS = GetPromotedInteger(N->getOperand(1));
696  SDValue RHS = GetPromotedInteger(N->getOperand(2));
697  return DAG.getSelect(SDLoc(N),
698  LHS.getValueType(), N->getOperand(0), LHS, RHS);
699 }
700 
701 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
702  SDValue Mask = N->getOperand(0);
703 
704  SDValue LHS = GetPromotedInteger(N->getOperand(1));
705  SDValue RHS = GetPromotedInteger(N->getOperand(2));
706  return DAG.getNode(ISD::VSELECT, SDLoc(N),
707  LHS.getValueType(), Mask, LHS, RHS);
708 }
709 
710 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
711  SDValue LHS = GetPromotedInteger(N->getOperand(2));
712  SDValue RHS = GetPromotedInteger(N->getOperand(3));
713  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
714  LHS.getValueType(), N->getOperand(0),
715  N->getOperand(1), LHS, RHS, N->getOperand(4));
716 }
717 
718 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
719  EVT InVT = N->getOperand(0).getValueType();
720  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
721 
722  EVT SVT = getSetCCResultType(InVT);
723 
724  // If we got back a type that needs to be promoted, this likely means the
725  // the input type also needs to be promoted. So get the promoted type for
726  // the input and try the query again.
727  if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
728  if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
729  InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
730  SVT = getSetCCResultType(InVT);
731  } else {
732  // Input type isn't promoted, just use the default promoted type.
733  SVT = NVT;
734  }
735  }
736 
737  SDLoc dl(N);
738  assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() &&
739  "Vector compare must return a vector result!");
740 
741  // Get the SETCC result using the canonical SETCC type.
742  SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
743  N->getOperand(1), N->getOperand(2));
744 
745  // Convert to the expected type.
746  return DAG.getSExtOrTrunc(SetCC, dl, NVT);
747 }
748 
749 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
750  SDValue LHS = GetPromotedInteger(N->getOperand(0));
751  SDValue RHS = N->getOperand(1);
752  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
753  RHS = ZExtPromotedInteger(RHS);
754  return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
755 }
756 
757 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
758  SDValue Op = GetPromotedInteger(N->getOperand(0));
759  return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
760  Op.getValueType(), Op, N->getOperand(1));
761 }
762 
763 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
764  // The input may have strange things in the top bits of the registers, but
765  // these operations don't care. They may have weird bits going out, but
766  // that too is okay if they are integer operations.
767  SDValue LHS = GetPromotedInteger(N->getOperand(0));
768  SDValue RHS = GetPromotedInteger(N->getOperand(1));
769  return DAG.getNode(N->getOpcode(), SDLoc(N),
770  LHS.getValueType(), LHS, RHS);
771 }
772 
773 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
774  // Sign extend the input.
775  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
776  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
777  return DAG.getNode(N->getOpcode(), SDLoc(N),
778  LHS.getValueType(), LHS, RHS);
779 }
780 
781 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
782  // Zero extend the input.
783  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
784  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
785  return DAG.getNode(N->getOpcode(), SDLoc(N),
786  LHS.getValueType(), LHS, RHS);
787 }
788 
789 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
790  // The input value must be properly sign extended.
791  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
792  SDValue RHS = N->getOperand(1);
793  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
794  RHS = ZExtPromotedInteger(RHS);
795  return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
796 }
797 
798 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
799  // The input value must be properly zero extended.
800  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
801  SDValue RHS = N->getOperand(1);
802  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
803  RHS = ZExtPromotedInteger(RHS);
804  return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
805 }
806 
807 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
808  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
809  SDValue Res;
810  SDValue InOp = N->getOperand(0);
811  SDLoc dl(N);
812 
813  switch (getTypeAction(InOp.getValueType())) {
814  default: llvm_unreachable("Unknown type action!");
817  Res = InOp;
818  break;
820  Res = GetPromotedInteger(InOp);
821  break;
823  EVT InVT = InOp.getValueType();
824  assert(InVT.isVector() && "Cannot split scalar types");
825  unsigned NumElts = InVT.getVectorNumElements();
826  assert(NumElts == NVT.getVectorNumElements() &&
827  "Dst and Src must have the same number of elements");
828  assert(isPowerOf2_32(NumElts) &&
829  "Promoted vector type must be a power of two");
830 
831  SDValue EOp1, EOp2;
832  GetSplitVector(InOp, EOp1, EOp2);
833 
834  EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
835  NumElts/2);
836  EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
837  EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
838 
839  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
840  }
842  SDValue WideInOp = GetWidenedVector(InOp);
843 
844  // Truncate widened InOp.
845  unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
846  EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
847  N->getValueType(0).getScalarType(), NumElem);
848  SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
849 
850  // Zero extend so that the elements are of same type as those of NVT
851  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
852  NumElem);
853  SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
854 
855  // Extract the low NVT subvector.
856  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
857  SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy);
858  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
859  }
860  }
861 
862  // Truncate to NVT instead of VT
863  return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
864 }
865 
866 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
867  if (ResNo == 1)
868  return PromoteIntRes_Overflow(N);
869 
870  // The operation overflowed iff the result in the larger type is not the
871  // zero extension of its truncation to the original type.
872  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
873  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
874  EVT OVT = N->getOperand(0).getValueType();
875  EVT NVT = LHS.getValueType();
876  SDLoc dl(N);
877 
878  // Do the arithmetic in the larger type.
879  unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
880  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
881 
882  // Calculate the overflow flag: zero extend the arithmetic result from
883  // the original type.
884  SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT.getScalarType());
885  // Overflowed if and only if this is not equal to Res.
886  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
887 
888  // Use the calculated overflow everywhere.
889  ReplaceValueWith(SDValue(N, 1), Ofl);
890 
891  return Res;
892 }
893 
894 // Handle promotion for the ADDE/SUBE/ADDCARRY/SUBCARRY nodes. Notice that
895 // the third operand of ADDE/SUBE nodes is carry flag, which differs from
896 // the ADDCARRY/SUBCARRY nodes in that the third operand is carry Boolean.
897 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
898  if (ResNo == 1)
899  return PromoteIntRes_Overflow(N);
900 
901  // We need to sign-extend the operands so the carry value computed by the
902  // wide operation will be equivalent to the carry value computed by the
903  // narrow operation.
904  // An ADDCARRY can generate carry only if any of the operands has its
905  // most significant bit set. Sign extension propagates the most significant
906  // bit into the higher bits which means the extra bit that the narrow
907  // addition would need (i.e. the carry) will be propagated through the higher
908  // bits of the wide addition.
909  // A SUBCARRY can generate borrow only if LHS < RHS and this property will be
910  // preserved by sign extension.
911  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
912  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
913 
914  EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
915 
916  // Do the arithmetic in the wide type.
917  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
918  LHS, RHS, N->getOperand(2));
919 
920  // Update the users of the original carry/borrow value.
921  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
922 
923  return SDValue(Res.getNode(), 0);
924 }
925 
926 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
927  // Promote the overflow bit trivially.
928  if (ResNo == 1)
929  return PromoteIntRes_Overflow(N);
930 
931  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
932  SDLoc DL(N);
933  EVT SmallVT = LHS.getValueType();
934 
935  // To determine if the result overflowed in a larger type, we extend the
936  // input to the larger type, do the multiply (checking if it overflows),
937  // then also check the high bits of the result to see if overflow happened
938  // there.
939  if (N->getOpcode() == ISD::SMULO) {
940  LHS = SExtPromotedInteger(LHS);
941  RHS = SExtPromotedInteger(RHS);
942  } else {
943  LHS = ZExtPromotedInteger(LHS);
944  RHS = ZExtPromotedInteger(RHS);
945  }
946  SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
947  SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
948 
949  // Overflow occurred if it occurred in the larger type, or if the high part
950  // of the result does not zero/sign-extend the low part. Check this second
951  // possibility first.
952  SDValue Overflow;
953  if (N->getOpcode() == ISD::UMULO) {
954  // Unsigned overflow occurred if the high part is non-zero.
955  unsigned Shift = SmallVT.getScalarSizeInBits();
956  EVT ShiftTy = getShiftAmountTyForConstant(Shift, Mul.getValueType(),
957  TLI, DAG);
958  SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
959  DAG.getConstant(Shift, DL, ShiftTy));
960  Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
961  DAG.getConstant(0, DL, Hi.getValueType()),
962  ISD::SETNE);
963  } else {
964  // Signed overflow occurred if the high part does not sign extend the low.
965  SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
966  Mul, DAG.getValueType(SmallVT));
967  Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
968  }
969 
970  // The only other way for overflow to occur is if the multiplication in the
971  // larger type itself overflowed.
972  Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
973  SDValue(Mul.getNode(), 1));
974 
975  // Use the calculated overflow everywhere.
976  ReplaceValueWith(SDValue(N, 1), Overflow);
977  return Mul;
978 }
979 
980 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
981  return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
982  N->getValueType(0)));
983 }
984 
985 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
986  SDValue Chain = N->getOperand(0); // Get the chain.
987  SDValue Ptr = N->getOperand(1); // Get the pointer.
988  EVT VT = N->getValueType(0);
989  SDLoc dl(N);
990 
991  MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
992  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
993  // The argument is passed as NumRegs registers of type RegVT.
994 
995  SmallVector<SDValue, 8> Parts(NumRegs);
996  for (unsigned i = 0; i < NumRegs; ++i) {
997  Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
998  N->getConstantOperandVal(3));
999  Chain = Parts[i].getValue(1);
1000  }
1001 
1002  // Handle endianness of the load.
1003  if (DAG.getDataLayout().isBigEndian())
1004  std::reverse(Parts.begin(), Parts.end());
1005 
1006  // Assemble the parts in the promoted type.
1007  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1008  SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
1009  for (unsigned i = 1; i < NumRegs; ++i) {
1010  SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
1011  // Shift it to the right position and "or" it in.
1012  Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
1013  DAG.getConstant(i * RegVT.getSizeInBits(), dl,
1014  TLI.getPointerTy(DAG.getDataLayout())));
1015  Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
1016  }
1017 
1018  // Modified the chain result - switch anything that used the old chain to
1019  // use the new one.
1020  ReplaceValueWith(SDValue(N, 1), Chain);
1021 
1022  return Res;
1023 }
1024 
1025 //===----------------------------------------------------------------------===//
1026 // Integer Operand Promotion
1027 //===----------------------------------------------------------------------===//
1028 
1029 /// PromoteIntegerOperand - This method is called when the specified operand of
1030 /// the specified node is found to need promotion. At this point, all of the
1031 /// result types of the node are known to be legal, but other operands of the
1032 /// node may need promotion or expansion as well as the specified one.
1033 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
1034  LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG);
1035  dbgs() << "\n");
1036  SDValue Res = SDValue();
1037 
1038  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
1039  LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
1040  return false;
1041  }
1042 
1043  switch (N->getOpcode()) {
1044  default:
1045  #ifndef NDEBUG
1046  dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
1047  N->dump(&DAG); dbgs() << "\n";
1048  #endif
1049  llvm_unreachable("Do not know how to promote this operator's operand!");
1050 
1051  case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
1052  case ISD::ATOMIC_STORE:
1053  Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
1054  break;
1055  case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
1056  case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
1057  case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
1058  case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
1059  case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
1060  case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
1061  case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
1063  Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
1064  case ISD::SCALAR_TO_VECTOR:
1065  Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
1066  case ISD::VSELECT:
1067  case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
1068  case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
1069  case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
1070  case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
1071  case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
1072  case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
1073  OpNo); break;
1074  case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
1075  OpNo); break;
1076  case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
1077  OpNo); break;
1078  case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
1079  OpNo); break;
1080  case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
1081  OpNo); break;
1082  case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
1083  case ISD::FP16_TO_FP:
1084  case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
1085  case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
1086  case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
1087 
1088  case ISD::SHL:
1089  case ISD::SRA:
1090  case ISD::SRL:
1091  case ISD::ROTL:
1092  case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
1093 
1094  case ISD::ADDCARRY:
1095  case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
1096 
1097  case ISD::FRAMEADDR:
1098  case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
1099 
1100  case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break;
1101 
1102  case ISD::SMULFIX:
1103  case ISD::UMULFIX: Res = PromoteIntOp_MULFIX(N); break;
1104 
1105  case ISD::FPOWI: Res = PromoteIntOp_FPOWI(N); break;
1106  }
1107 
1108  // If the result is null, the sub-method took care of registering results etc.
1109  if (!Res.getNode()) return false;
1110 
1111  // If the result is N, the sub-method updated N in place. Tell the legalizer
1112  // core about this.
1113  if (Res.getNode() == N)
1114  return true;
1115 
1116  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1117  "Invalid operand expansion");
1118 
1119  ReplaceValueWith(SDValue(N, 0), Res);
1120  return false;
1121 }
1122 
1123 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
1124 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
1125 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
1126  ISD::CondCode CCCode) {
1127  // We have to insert explicit sign or zero extends. Note that we could
1128  // insert sign extends for ALL conditions. For those operations where either
1129  // zero or sign extension would be valid, use SExtOrZExtPromotedInteger
1130  // which will choose the cheapest for the target.
1131  switch (CCCode) {
1132  default: llvm_unreachable("Unknown integer comparison!");
1133  case ISD::SETEQ:
1134  case ISD::SETNE: {
1135  SDValue OpL = GetPromotedInteger(NewLHS);
1136  SDValue OpR = GetPromotedInteger(NewRHS);
1137 
1138  // We would prefer to promote the comparison operand with sign extension.
1139  // If the width of OpL/OpR excluding the duplicated sign bits is no greater
1140  // than the width of NewLHS/NewRH, we can avoid inserting real truncate
1141  // instruction, which is redundant eventually.
1142  unsigned OpLEffectiveBits =
1143  OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
1144  unsigned OpREffectiveBits =
1145  OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
1146  if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() &&
1147  OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()) {
1148  NewLHS = OpL;
1149  NewRHS = OpR;
1150  } else {
1151  NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1152  NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1153  }
1154  break;
1155  }
1156  case ISD::SETUGE:
1157  case ISD::SETUGT:
1158  case ISD::SETULE:
1159  case ISD::SETULT:
1160  NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1161  NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1162  break;
1163  case ISD::SETGE:
1164  case ISD::SETGT:
1165  case ISD::SETLT:
1166  case ISD::SETLE:
1167  NewLHS = SExtPromotedInteger(NewLHS);
1168  NewRHS = SExtPromotedInteger(NewRHS);
1169  break;
1170  }
1171 }
1172 
1173 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1174  SDValue Op = GetPromotedInteger(N->getOperand(0));
1175  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1176 }
1177 
1178 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1179  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1180  return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1181  N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1182 }
1183 
1184 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1185  // This should only occur in unusual situations like bitcasting to an
1186  // x86_fp80, so just turn it into a store+load
1187  return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1188 }
1189 
1190 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1191  assert(OpNo == 2 && "Don't know how to promote this operand!");
1192 
1193  SDValue LHS = N->getOperand(2);
1194  SDValue RHS = N->getOperand(3);
1195  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1196 
1197  // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1198  // legal types.
1199  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1200  N->getOperand(1), LHS, RHS, N->getOperand(4)),
1201  0);
1202 }
1203 
1204 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1205  assert(OpNo == 1 && "only know how to promote condition");
1206 
1207  // Promote all the way up to the canonical SetCC type.
1208  SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1209 
1210  // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1211  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1212  N->getOperand(2)), 0);
1213 }
1214 
1215 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1216  // Since the result type is legal, the operands must promote to it.
1217  EVT OVT = N->getOperand(0).getValueType();
1218  SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1219  SDValue Hi = GetPromotedInteger(N->getOperand(1));
1220  assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1221  SDLoc dl(N);
1222 
1223  Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1224  DAG.getConstant(OVT.getSizeInBits(), dl,
1225  TLI.getPointerTy(DAG.getDataLayout())));
1226  return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1227 }
1228 
1229 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1230  // The vector type is legal but the element type is not. This implies
1231  // that the vector is a power-of-two in length and that the element
1232  // type does not have a strange size (eg: it is not i1).
1233  EVT VecVT = N->getValueType(0);
1234  unsigned NumElts = VecVT.getVectorNumElements();
1235  assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1236  "Legal vector of one illegal element?");
1237 
1238  // Promote the inserted value. The type does not need to match the
1239  // vector element type. Check that any extra bits introduced will be
1240  // truncated away.
1243  "Type of inserted value narrower than vector element type!");
1244 
1245  SmallVector<SDValue, 16> NewOps;
1246  for (unsigned i = 0; i < NumElts; ++i)
1247  NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1248 
1249  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1250 }
1251 
1252 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1253  unsigned OpNo) {
1254  if (OpNo == 1) {
1255  // Promote the inserted value. This is valid because the type does not
1256  // have to match the vector element type.
1257 
1258  // Check that any extra bits introduced will be truncated away.
1261  "Type of inserted value narrower than vector element type!");
1262  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1263  GetPromotedInteger(N->getOperand(1)),
1264  N->getOperand(2)),
1265  0);
1266  }
1267 
1268  assert(OpNo == 2 && "Different operand and result vector types?");
1269 
1270  // Promote the index.
1271  SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1272  TLI.getVectorIdxTy(DAG.getDataLayout()));
1273  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1274  N->getOperand(1), Idx), 0);
1275 }
1276 
1277 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1278  // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1279  // the operand in place.
1280  return SDValue(DAG.UpdateNodeOperands(N,
1281  GetPromotedInteger(N->getOperand(0))), 0);
1282 }
1283 
1284 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1285  assert(OpNo == 0 && "Only know how to promote the condition!");
1286  SDValue Cond = N->getOperand(0);
1287  EVT OpTy = N->getOperand(1).getValueType();
1288 
1289  if (N->getOpcode() == ISD::VSELECT)
1290  if (SDValue Res = WidenVSELECTAndMask(N))
1291  return Res;
1292 
1293  // Promote all the way up to the canonical SetCC type.
1294  EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1295  Cond = PromoteTargetBoolean(Cond, OpVT);
1296 
1297  return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1298  N->getOperand(2)), 0);
1299 }
1300 
1301 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1302  assert(OpNo == 0 && "Don't know how to promote this operand!");
1303 
1304  SDValue LHS = N->getOperand(0);
1305  SDValue RHS = N->getOperand(1);
1306  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1307 
1308  // The CC (#4) and the possible return values (#2 and #3) have legal types.
1309  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1310  N->getOperand(3), N->getOperand(4)), 0);
1311 }
1312 
1313 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1314  assert(OpNo == 0 && "Don't know how to promote this operand!");
1315 
1316  SDValue LHS = N->getOperand(0);
1317  SDValue RHS = N->getOperand(1);
1318  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1319 
1320  // The CC (#2) is always legal.
1321  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1322 }
1323 
1324 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1325  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1326  ZExtPromotedInteger(N->getOperand(1))), 0);
1327 }
1328 
1329 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1330  SDValue Op = GetPromotedInteger(N->getOperand(0));
1331  SDLoc dl(N);
1332  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1333  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1334  Op, DAG.getValueType(N->getOperand(0).getValueType()));
1335 }
1336 
1337 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1338  return SDValue(DAG.UpdateNodeOperands(N,
1339  SExtPromotedInteger(N->getOperand(0))), 0);
1340 }
1341 
1342 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1343  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1344  SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1345  SDLoc dl(N);
1346 
1347  SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1348 
1349  // Truncate the value and store the result.
1350  return DAG.getTruncStore(Ch, dl, Val, Ptr,
1351  N->getMemoryVT(), N->getMemOperand());
1352 }
1353 
1354 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1355  unsigned OpNo) {
1356 
1357  SDValue DataOp = N->getValue();
1358  EVT DataVT = DataOp.getValueType();
1359  SDValue Mask = N->getMask();
1360  SDLoc dl(N);
1361 
1362  bool TruncateStore = false;
1363  if (OpNo == 3) {
1364  Mask = PromoteTargetBoolean(Mask, DataVT);
1365  // Update in place.
1366  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1367  NewOps[3] = Mask;
1368  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1369  } else { // Data operand
1370  assert(OpNo == 1 && "Unexpected operand for promotion");
1371  DataOp = GetPromotedInteger(DataOp);
1372  TruncateStore = true;
1373  }
1374 
1375  return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask,
1376  N->getMemoryVT(), N->getMemOperand(),
1377  TruncateStore, N->isCompressingStore());
1378 }
1379 
1380 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1381  unsigned OpNo) {
1382  assert(OpNo == 2 && "Only know how to promote the mask!");
1383  EVT DataVT = N->getValueType(0);
1384  SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1385  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1386  NewOps[OpNo] = Mask;
1387  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1388 }
1389 
1390 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1391  unsigned OpNo) {
1392 
1393  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1394  if (OpNo == 2) {
1395  // The Mask
1396  EVT DataVT = N->getValueType(0);
1397  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1398  } else if (OpNo == 4) {
1399  // Need to sign extend the index since the bits will likely be used.
1400  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1401  } else
1402  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1403 
1404  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1405 }
1406 
1407 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1408  unsigned OpNo) {
1409  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1410  if (OpNo == 2) {
1411  // The Mask
1412  EVT DataVT = N->getValue().getValueType();
1413  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1414  } else if (OpNo == 4) {
1415  // Need to sign extend the index since the bits will likely be used.
1416  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1417  } else
1418  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1419  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1420 }
1421 
1422 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
1423  SDValue Op = GetPromotedInteger(N->getOperand(0));
1424  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
1425 }
1426 
1427 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
1428  return SDValue(DAG.UpdateNodeOperands(N,
1429  ZExtPromotedInteger(N->getOperand(0))), 0);
1430 }
1431 
1432 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
1433  SDLoc dl(N);
1434  SDValue Op = GetPromotedInteger(N->getOperand(0));
1435  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1436  return DAG.getZeroExtendInReg(Op, dl,
1438 }
1439 
1440 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
1441  assert(OpNo == 2 && "Don't know how to promote this operand!");
1442 
1443  SDValue LHS = N->getOperand(0);
1444  SDValue RHS = N->getOperand(1);
1445  SDValue Carry = N->getOperand(2);
1446  SDLoc DL(N);
1447 
1448  auto VT = getSetCCResultType(LHS.getValueType());
1450  switch (BoolType) {
1452  Carry = DAG.getAnyExtOrTrunc(Carry, DL, VT);
1453  break;
1455  Carry = DAG.getZExtOrTrunc(Carry, DL, VT);
1456  break;
1458  Carry = DAG.getSExtOrTrunc(Carry, DL, VT);
1459  break;
1460  }
1461 
1462  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
1463 }
1464 
1465 SDValue DAGTypeLegalizer::PromoteIntOp_MULFIX(SDNode *N) {
1466  SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1467  return SDValue(
1468  DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
1469 }
1470 
1471 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
1472  // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
1473  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
1474  return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
1475 }
1476 
1477 SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) {
1478  assert(OpNo > 1 && "Don't know how to promote this operand!");
1479  // Promote the rw, locality, and cache type arguments to a supported integer
1480  // width.
1481  SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1482  SDValue Op3 = ZExtPromotedInteger(N->getOperand(3));
1483  SDValue Op4 = ZExtPromotedInteger(N->getOperand(4));
1484  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
1485  Op2, Op3, Op4),
1486  0);
1487 }
1488 
1489 SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) {
1490  SDValue Op = SExtPromotedInteger(N->getOperand(1));
1491  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
1492 }
1493 
1494 //===----------------------------------------------------------------------===//
1495 // Integer Result Expansion
1496 //===----------------------------------------------------------------------===//
1497 
1498 /// ExpandIntegerResult - This method is called when the specified result of the
1499 /// specified node is found to need expansion. At this point, the node may also
1500 /// have invalid operands or may have other results that need promotion, we just
1501 /// know that (at least) one result needs expansion.
1502 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
1503  LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG);
1504  dbgs() << "\n");
1505  SDValue Lo, Hi;
1506  Lo = Hi = SDValue();
1507 
1508  // See if the target wants to custom expand this node.
1509  if (CustomLowerNode(N, N->getValueType(ResNo), true))
1510  return;
1511 
1512  switch (N->getOpcode()) {
1513  default:
1514 #ifndef NDEBUG
1515  dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
1516  N->dump(&DAG); dbgs() << "\n";
1517 #endif
1518  report_fatal_error("Do not know how to expand the result of this "
1519  "operator!");
1520 
1521  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1522  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
1523  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1524  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1525 
1526  case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1527  case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1528  case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1529  case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1530  case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1531 
1532  case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
1533  case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
1534  case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
1535  case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
1536  case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
1537  case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
1538  case ISD::CTLZ_ZERO_UNDEF:
1539  case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
1540  case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
1541  case ISD::CTTZ_ZERO_UNDEF:
1542  case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
1543  case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
1544  case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
1545  case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
1546  case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
1547  case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
1548  case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
1549  case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
1550  case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
1551  case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
1552  case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
1553  case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
1554  case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
1555  case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
1556  case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
1557  case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
1558 
1559  case ISD::ATOMIC_LOAD_ADD:
1560  case ISD::ATOMIC_LOAD_SUB:
1561  case ISD::ATOMIC_LOAD_AND:
1562  case ISD::ATOMIC_LOAD_CLR:
1563  case ISD::ATOMIC_LOAD_OR:
1564  case ISD::ATOMIC_LOAD_XOR:
1565  case ISD::ATOMIC_LOAD_NAND:
1566  case ISD::ATOMIC_LOAD_MIN:
1567  case ISD::ATOMIC_LOAD_MAX:
1568  case ISD::ATOMIC_LOAD_UMIN:
1569  case ISD::ATOMIC_LOAD_UMAX:
1570  case ISD::ATOMIC_SWAP:
1571  case ISD::ATOMIC_CMP_SWAP: {
1572  std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
1573  SplitInteger(Tmp.first, Lo, Hi);
1574  ReplaceValueWith(SDValue(N, 1), Tmp.second);
1575  break;
1576  }
1578  AtomicSDNode *AN = cast<AtomicSDNode>(N);
1579  SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
1580  SDValue Tmp = DAG.getAtomicCmpSwap(
1581  ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
1582  N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
1583  AN->getMemOperand());
1584 
1585  // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
1586  // success simply by comparing the loaded value against the ingoing
1587  // comparison.
1588  SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
1589  N->getOperand(2), ISD::SETEQ);
1590 
1591  SplitInteger(Tmp, Lo, Hi);
1592  ReplaceValueWith(SDValue(N, 1), Success);
1593  ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
1594  break;
1595  }
1596 
1597  case ISD::AND:
1598  case ISD::OR:
1599  case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
1600 
1601  case ISD::UMAX:
1602  case ISD::SMAX:
1603  case ISD::UMIN:
1604  case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
1605 
1606  case ISD::ADD:
1607  case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
1608 
1609  case ISD::ADDC:
1610  case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
1611 
1612  case ISD::ADDE:
1613  case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
1614 
1615  case ISD::ADDCARRY:
1616  case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
1617 
1618  case ISD::SHL:
1619  case ISD::SRA:
1620  case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
1621 
1622  case ISD::SADDO:
1623  case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
1624  case ISD::UADDO:
1625  case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
1626  case ISD::UMULO:
1627  case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
1628 
1629  case ISD::SADDSAT:
1630  case ISD::UADDSAT:
1631  case ISD::SSUBSAT:
1632  case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
1633  case ISD::SMULFIX:
1634  case ISD::UMULFIX: ExpandIntRes_MULFIX(N, Lo, Hi); break;
1635  }
1636 
1637  // If Lo/Hi is null, the sub-method took care of registering results etc.
1638  if (Lo.getNode())
1639  SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
1640 }
1641 
1642 /// Lower an atomic node to the appropriate builtin call.
1643 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
1644  unsigned Opc = Node->getOpcode();
1645  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
1646  RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
1647  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
1648 
1649  return ExpandChainLibCall(LC, Node, false);
1650 }
1651 
1652 /// N is a shift by a value that needs to be expanded,
1653 /// and the shift amount is a constant 'Amt'. Expand the operation.
1654 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
1655  SDValue &Lo, SDValue &Hi) {
1656  SDLoc DL(N);
1657  // Expand the incoming operand to be shifted, so that we have its parts
1658  SDValue InL, InH;
1659  GetExpandedInteger(N->getOperand(0), InL, InH);
1660 
1661  // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
1662  // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
1663  if (!Amt) {
1664  Lo = InL;
1665  Hi = InH;
1666  return;
1667  }
1668 
1669  EVT NVT = InL.getValueType();
1670  unsigned VTBits = N->getValueType(0).getSizeInBits();
1671  unsigned NVTBits = NVT.getSizeInBits();
1672  EVT ShTy = N->getOperand(1).getValueType();
1673 
1674  if (N->getOpcode() == ISD::SHL) {
1675  if (Amt.ugt(VTBits)) {
1676  Lo = Hi = DAG.getConstant(0, DL, NVT);
1677  } else if (Amt.ugt(NVTBits)) {
1678  Lo = DAG.getConstant(0, DL, NVT);
1679  Hi = DAG.getNode(ISD::SHL, DL,
1680  NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1681  } else if (Amt == NVTBits) {
1682  Lo = DAG.getConstant(0, DL, NVT);
1683  Hi = InL;
1684  } else {
1685  Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
1686  Hi = DAG.getNode(ISD::OR, DL, NVT,
1687  DAG.getNode(ISD::SHL, DL, NVT, InH,
1688  DAG.getConstant(Amt, DL, ShTy)),
1689  DAG.getNode(ISD::SRL, DL, NVT, InL,
1690  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1691  }
1692  return;
1693  }
1694 
1695  if (N->getOpcode() == ISD::SRL) {
1696  if (Amt.ugt(VTBits)) {
1697  Lo = Hi = DAG.getConstant(0, DL, NVT);
1698  } else if (Amt.ugt(NVTBits)) {
1699  Lo = DAG.getNode(ISD::SRL, DL,
1700  NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1701  Hi = DAG.getConstant(0, DL, NVT);
1702  } else if (Amt == NVTBits) {
1703  Lo = InH;
1704  Hi = DAG.getConstant(0, DL, NVT);
1705  } else {
1706  Lo = DAG.getNode(ISD::OR, DL, NVT,
1707  DAG.getNode(ISD::SRL, DL, NVT, InL,
1708  DAG.getConstant(Amt, DL, ShTy)),
1709  DAG.getNode(ISD::SHL, DL, NVT, InH,
1710  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1711  Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1712  }
1713  return;
1714  }
1715 
1716  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
1717  if (Amt.ugt(VTBits)) {
1718  Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1719  DAG.getConstant(NVTBits - 1, DL, ShTy));
1720  } else if (Amt.ugt(NVTBits)) {
1721  Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1722  DAG.getConstant(Amt - NVTBits, DL, ShTy));
1723  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1724  DAG.getConstant(NVTBits - 1, DL, ShTy));
1725  } else if (Amt == NVTBits) {
1726  Lo = InH;
1727  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1728  DAG.getConstant(NVTBits - 1, DL, ShTy));
1729  } else {
1730  Lo = DAG.getNode(ISD::OR, DL, NVT,
1731  DAG.getNode(ISD::SRL, DL, NVT, InL,
1732  DAG.getConstant(Amt, DL, ShTy)),
1733  DAG.getNode(ISD::SHL, DL, NVT, InH,
1734  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1735  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1736  }
1737 }
1738 
1739 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
1740 /// this shift based on knowledge of the high bit of the shift amount. If we
1741 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
1742 /// shift amount.
1743 bool DAGTypeLegalizer::
1744 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1745  SDValue Amt = N->getOperand(1);
1746  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1747  EVT ShTy = Amt.getValueType();
1748  unsigned ShBits = ShTy.getScalarSizeInBits();
1749  unsigned NVTBits = NVT.getScalarSizeInBits();
1750  assert(isPowerOf2_32(NVTBits) &&
1751  "Expanded integer type size not a power of two!");
1752  SDLoc dl(N);
1753 
1754  APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
1755  KnownBits Known = DAG.computeKnownBits(N->getOperand(1));
1756 
1757  // If we don't know anything about the high bits, exit.
1758  if (((Known.Zero|Known.One) & HighBitMask) == 0)
1759  return false;
1760 
1761  // Get the incoming operand to be shifted.
1762  SDValue InL, InH;
1763  GetExpandedInteger(N->getOperand(0), InL, InH);
1764 
1765  // If we know that any of the high bits of the shift amount are one, then we
1766  // can do this as a couple of simple shifts.
1767  if (Known.One.intersects(HighBitMask)) {
1768  // Mask out the high bit, which we know is set.
1769  Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
1770  DAG.getConstant(~HighBitMask, dl, ShTy));
1771 
1772  switch (N->getOpcode()) {
1773  default: llvm_unreachable("Unknown shift");
1774  case ISD::SHL:
1775  Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
1776  Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
1777  return true;
1778  case ISD::SRL:
1779  Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1780  Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
1781  return true;
1782  case ISD::SRA:
1783  Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
1784  DAG.getConstant(NVTBits - 1, dl, ShTy));
1785  Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
1786  return true;
1787  }
1788  }
1789 
1790  // If we know that all of the high bits of the shift amount are zero, then we
1791  // can do this as a couple of simple shifts.
1792  if (HighBitMask.isSubsetOf(Known.Zero)) {
1793  // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
1794  // shift if x is zero. We can use XOR here because x is known to be smaller
1795  // than 32.
1796  SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
1797  DAG.getConstant(NVTBits - 1, dl, ShTy));
1798 
1799  unsigned Op1, Op2;
1800  switch (N->getOpcode()) {
1801  default: llvm_unreachable("Unknown shift");
1802  case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
1803  case ISD::SRL:
1804  case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
1805  }
1806 
1807  // When shifting right the arithmetic for Lo and Hi is swapped.
1808  if (N->getOpcode() != ISD::SHL)
1809  std::swap(InL, InH);
1810 
1811  // Use a little trick to get the bits that move from Lo to Hi. First
1812  // shift by one bit.
1813  SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
1814  // Then compute the remaining shift with amount-1.
1815  SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
1816 
1817  Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
1818  Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
1819 
1820  if (N->getOpcode() != ISD::SHL)
1821  std::swap(Hi, Lo);
1822  return true;
1823  }
1824 
1825  return false;
1826 }
1827 
1828 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
1829 /// of any size.
1830 bool DAGTypeLegalizer::
1831 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1832  SDValue Amt = N->getOperand(1);
1833  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1834  EVT ShTy = Amt.getValueType();
1835  unsigned NVTBits = NVT.getSizeInBits();
1836  assert(isPowerOf2_32(NVTBits) &&
1837  "Expanded integer type size not a power of two!");
1838  SDLoc dl(N);
1839 
1840  // Get the incoming operand to be shifted.
1841  SDValue InL, InH;
1842  GetExpandedInteger(N->getOperand(0), InL, InH);
1843 
1844  SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
1845  SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
1846  SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
1847  SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1848  Amt, NVBitsNode, ISD::SETULT);
1849  SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1850  Amt, DAG.getConstant(0, dl, ShTy),
1851  ISD::SETEQ);
1852 
1853  SDValue LoS, HiS, LoL, HiL;
1854  switch (N->getOpcode()) {
1855  default: llvm_unreachable("Unknown shift");
1856  case ISD::SHL:
1857  // Short: ShAmt < NVTBits
1858  LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
1859  HiS = DAG.getNode(ISD::OR, dl, NVT,
1860  DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
1861  DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
1862 
1863  // Long: ShAmt >= NVTBits
1864  LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
1865  HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
1866 
1867  Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
1868  Hi = DAG.getSelect(dl, NVT, isZero, InH,
1869  DAG.getSelect(dl, NVT, isShort, HiS, HiL));
1870  return true;
1871  case ISD::SRL:
1872  // Short: ShAmt < NVTBits
1873  HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
1874  LoS = DAG.getNode(ISD::OR, dl, NVT,
1875  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1876  // FIXME: If Amt is zero, the following shift generates an undefined result
1877  // on some architectures.
1878  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1879 
1880  // Long: ShAmt >= NVTBits
1881  HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1882  LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1883 
1884  Lo = DAG.getSelect(dl, NVT, isZero, InL,
1885  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1886  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1887  return true;
1888  case ISD::SRA:
1889  // Short: ShAmt < NVTBits
1890  HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
1891  LoS = DAG.getNode(ISD::OR, dl, NVT,
1892  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1893  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1894 
1895  // Long: ShAmt >= NVTBits
1896  HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
1897  DAG.getConstant(NVTBits - 1, dl, ShTy));
1898  LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1899 
1900  Lo = DAG.getSelect(dl, NVT, isZero, InL,
1901  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1902  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1903  return true;
1904  }
1905 }
1906 
1907 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
1908 
1909  switch (Op) {
1910  default: llvm_unreachable("invalid min/max opcode");
1911  case ISD::SMAX:
1912  return std::make_pair(ISD::SETGT, ISD::UMAX);
1913  case ISD::UMAX:
1914  return std::make_pair(ISD::SETUGT, ISD::UMAX);
1915  case ISD::SMIN:
1916  return std::make_pair(ISD::SETLT, ISD::UMIN);
1917  case ISD::UMIN:
1918  return std::make_pair(ISD::SETULT, ISD::UMIN);
1919  }
1920 }
1921 
1922 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
1923  SDValue &Lo, SDValue &Hi) {
1924  SDLoc DL(N);
1925  ISD::NodeType LoOpc;
1926  ISD::CondCode CondC;
1927  std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
1928 
1929  // Expand the subcomponents.
1930  SDValue LHSL, LHSH, RHSL, RHSH;
1931  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1932  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1933 
1934  // Value types
1935  EVT NVT = LHSL.getValueType();
1936  EVT CCT = getSetCCResultType(NVT);
1937 
1938  // Hi part is always the same op
1939  Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
1940 
1941  // We need to know whether to select Lo part that corresponds to 'winning'
1942  // Hi part or if Hi parts are equal.
1943  SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
1944  SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
1945 
1946  // Lo part corresponding to the 'winning' Hi part
1947  SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
1948 
1949  // Recursed Lo part if Hi parts are equal, this uses unsigned version
1950  SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
1951 
1952  Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
1953 }
1954 
1955 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
1956  SDValue &Lo, SDValue &Hi) {
1957  SDLoc dl(N);
1958  // Expand the subcomponents.
1959  SDValue LHSL, LHSH, RHSL, RHSH;
1960  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1961  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1962 
1963  EVT NVT = LHSL.getValueType();
1964  SDValue LoOps[2] = { LHSL, RHSL };
1965  SDValue HiOps[3] = { LHSH, RHSH };
1966 
1967  bool HasOpCarry = TLI.isOperationLegalOrCustom(
1969  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1970  if (HasOpCarry) {
1971  SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
1972  if (N->getOpcode() == ISD::ADD) {
1973  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
1974  HiOps[2] = Lo.getValue(1);
1975  Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
1976  } else {
1977  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
1978  HiOps[2] = Lo.getValue(1);
1979  Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
1980  }
1981  return;
1982  }
1983 
1984  // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
1985  // them. TODO: Teach operation legalization how to expand unsupported
1986  // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
1987  // a carry of type MVT::Glue, but there doesn't seem to be any way to
1988  // generate a value of this type in the expanded code sequence.
1989  bool hasCarry =
1991  ISD::ADDC : ISD::SUBC,
1992  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1993 
1994  if (hasCarry) {
1995  SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
1996  if (N->getOpcode() == ISD::ADD) {
1997  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
1998  HiOps[2] = Lo.getValue(1);
1999  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2000  } else {
2001  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2002  HiOps[2] = Lo.getValue(1);
2003  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2004  }
2005  return;
2006  }
2007 
2008  bool hasOVF =
2011  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2013 
2014  if (hasOVF) {
2015  EVT OvfVT = getSetCCResultType(NVT);
2016  SDVTList VTList = DAG.getVTList(NVT, OvfVT);
2017  int RevOpc;
2018  if (N->getOpcode() == ISD::ADD) {
2019  RevOpc = ISD::SUB;
2020  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2021  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2022  } else {
2023  RevOpc = ISD::ADD;
2024  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2025  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2026  }
2027  SDValue OVF = Lo.getValue(1);
2028 
2029  switch (BoolType) {
2031  OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
2034  OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
2035  Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
2036  break;
2038  OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
2039  Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
2040  }
2041  return;
2042  }
2043 
2044  if (N->getOpcode() == ISD::ADD) {
2045  Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
2046  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2047  SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
2048  ISD::SETULT);
2049 
2051  SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
2052  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
2053  return;
2054  }
2055 
2056  SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
2057  DAG.getConstant(1, dl, NVT),
2058  DAG.getConstant(0, dl, NVT));
2059  SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
2060  ISD::SETULT);
2061  SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
2062  DAG.getConstant(1, dl, NVT), Carry1);
2063  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
2064  } else {
2065  Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
2066  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2067  SDValue Cmp =
2068  DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
2069  LoOps[0], LoOps[1], ISD::SETULT);
2070 
2071  SDValue Borrow;
2073  Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
2074  else
2075  Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
2076  DAG.getConstant(0, dl, NVT));
2077 
2078  Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
2079  }
2080 }
2081 
2082 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
2083  SDValue &Lo, SDValue &Hi) {
2084  // Expand the subcomponents.
2085  SDValue LHSL, LHSH, RHSL, RHSH;
2086  SDLoc dl(N);
2087  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2088  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2089  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2090  SDValue LoOps[2] = { LHSL, RHSL };
2091  SDValue HiOps[3] = { LHSH, RHSH };
2092 
2093  if (N->getOpcode() == ISD::ADDC) {
2094  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2095  HiOps[2] = Lo.getValue(1);
2096  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2097  } else {
2098  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2099  HiOps[2] = Lo.getValue(1);
2100  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2101  }
2102 
2103  // Legalized the flag result - switch anything that used the old flag to
2104  // use the new one.
2105  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2106 }
2107 
2108 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
2109  SDValue &Lo, SDValue &Hi) {
2110  // Expand the subcomponents.
2111  SDValue LHSL, LHSH, RHSL, RHSH;
2112  SDLoc dl(N);
2113  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2114  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2115  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2116  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2117  SDValue HiOps[3] = { LHSH, RHSH };
2118 
2119  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2120  HiOps[2] = Lo.getValue(1);
2121  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2122 
2123  // Legalized the flag result - switch anything that used the old flag to
2124  // use the new one.
2125  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2126 }
2127 
2128 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
2129  SDValue &Lo, SDValue &Hi) {
2130  SDValue LHS = N->getOperand(0);
2131  SDValue RHS = N->getOperand(1);
2132  SDLoc dl(N);
2133 
2134  SDValue Ovf;
2135 
2136  bool HasOpCarry = TLI.isOperationLegalOrCustom(
2138  TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
2139 
2140  if (HasOpCarry) {
2141  // Expand the subcomponents.
2142  SDValue LHSL, LHSH, RHSL, RHSH;
2143  GetExpandedInteger(LHS, LHSL, LHSH);
2144  GetExpandedInteger(RHS, RHSL, RHSH);
2145  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2146  SDValue LoOps[2] = { LHSL, RHSL };
2147  SDValue HiOps[3] = { LHSH, RHSH };
2148 
2149  unsigned Opc = N->getOpcode() == ISD::UADDO ? ISD::ADDCARRY : ISD::SUBCARRY;
2150  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2151  HiOps[2] = Lo.getValue(1);
2152  Hi = DAG.getNode(Opc, dl, VTList, HiOps);
2153 
2154  Ovf = Hi.getValue(1);
2155  } else {
2156  // Expand the result by simply replacing it with the equivalent
2157  // non-overflow-checking operation.
2158  auto Opc = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
2159  SDValue Sum = DAG.getNode(Opc, dl, LHS.getValueType(), LHS, RHS);
2160  SplitInteger(Sum, Lo, Hi);
2161 
2162  // Calculate the overflow: addition overflows iff a + b < a, and subtraction
2163  // overflows iff a - b > a.
2164  auto Cond = N->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
2165  Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
2166  }
2167 
2168  // Legalized the flag result - switch anything that used the old flag to
2169  // use the new one.
2170  ReplaceValueWith(SDValue(N, 1), Ovf);
2171 }
2172 
2173 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
2174  SDValue &Lo, SDValue &Hi) {
2175  // Expand the subcomponents.
2176  SDValue LHSL, LHSH, RHSL, RHSH;
2177  SDLoc dl(N);
2178  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2179  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2180  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2181  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2182  SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
2183 
2184  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2185  HiOps[2] = Lo.getValue(1);
2186  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2187 
2188  // Legalized the flag result - switch anything that used the old flag to
2189  // use the new one.
2190  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2191 }
2192 
2193 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
2194  SDValue &Lo, SDValue &Hi) {
2195  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2196  SDLoc dl(N);
2197  SDValue Op = N->getOperand(0);
2198  if (Op.getValueType().bitsLE(NVT)) {
2199  // The low part is any extension of the input (which degenerates to a copy).
2200  Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
2201  Hi = DAG.getUNDEF(NVT); // The high part is undefined.
2202  } else {
2203  // For example, extension of an i48 to an i64. The operand type necessarily
2204  // promotes to the result type, so will end up being expanded too.
2205  assert(getTypeAction(Op.getValueType()) ==
2207  "Only know how to promote this result!");
2208  SDValue Res = GetPromotedInteger(Op);
2209  assert(Res.getValueType() == N->getValueType(0) &&
2210  "Operand over promoted?");
2211  // Split the promoted operand. This will simplify when it is expanded.
2212  SplitInteger(Res, Lo, Hi);
2213  }
2214 }
2215 
2216 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
2217  SDValue &Lo, SDValue &Hi) {
2218  SDLoc dl(N);
2219  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2220  EVT NVT = Lo.getValueType();
2221  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2222  unsigned NVTBits = NVT.getSizeInBits();
2223  unsigned EVTBits = EVT.getSizeInBits();
2224 
2225  if (NVTBits < EVTBits) {
2226  Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
2228  EVTBits - NVTBits)));
2229  } else {
2230  Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
2231  // The high part replicates the sign bit of Lo, make it explicit.
2232  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2233  DAG.getConstant(NVTBits - 1, dl,
2234  TLI.getPointerTy(DAG.getDataLayout())));
2235  }
2236 }
2237 
2238 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
2239  SDValue &Lo, SDValue &Hi) {
2240  SDLoc dl(N);
2241  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2242  EVT NVT = Lo.getValueType();
2243  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2244  unsigned NVTBits = NVT.getSizeInBits();
2245  unsigned EVTBits = EVT.getSizeInBits();
2246 
2247  if (NVTBits < EVTBits) {
2248  Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
2250  EVTBits - NVTBits)));
2251  } else {
2252  Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
2253  // The high part must be zero, make it explicit.
2254  Hi = DAG.getConstant(0, dl, NVT);
2255  }
2256 }
2257 
2258 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
2259  SDValue &Lo, SDValue &Hi) {
2260  SDLoc dl(N);
2261  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2262  Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
2263  Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
2264 }
2265 
2266 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
2267  SDValue &Lo, SDValue &Hi) {
2268  SDLoc dl(N);
2269  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2270  Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
2271  Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
2272 }
2273 
2274 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
2275  SDValue &Lo, SDValue &Hi) {
2276  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2277  unsigned NBitWidth = NVT.getSizeInBits();
2278  auto Constant = cast<ConstantSDNode>(N);
2279  const APInt &Cst = Constant->getAPIntValue();
2280  bool IsTarget = Constant->isTargetOpcode();
2281  bool IsOpaque = Constant->isOpaque();
2282  SDLoc dl(N);
2283  Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
2284  Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
2285  IsOpaque);
2286 }
2287 
2288 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
2289  SDValue &Lo, SDValue &Hi) {
2290  SDLoc dl(N);
2291  // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
2292  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2293  EVT NVT = Lo.getValueType();
2294 
2295  SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
2296  DAG.getConstant(0, dl, NVT), ISD::SETNE);
2297 
2298  SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
2299  SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
2300 
2301  Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
2302  DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
2303  DAG.getConstant(NVT.getSizeInBits(), dl,
2304  NVT)));
2305  Hi = DAG.getConstant(0, dl, NVT);
2306 }
2307 
2308 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
2309  SDValue &Lo, SDValue &Hi) {
2310  SDLoc dl(N);
2311  // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
2312  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2313  EVT NVT = Lo.getValueType();
2314  Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
2315  DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
2316  Hi = DAG.getConstant(0, dl, NVT);
2317 }
2318 
2319 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
2320  SDValue &Lo, SDValue &Hi) {
2321  SDLoc dl(N);
2322  // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
2323  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2324  EVT NVT = Lo.getValueType();
2325 
2326  SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
2327  DAG.getConstant(0, dl, NVT), ISD::SETNE);
2328 
2329  SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
2330  SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
2331 
2332  Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
2333  DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
2334  DAG.getConstant(NVT.getSizeInBits(), dl,
2335  NVT)));
2336  Hi = DAG.getConstant(0, dl, NVT);
2337 }
2338 
2339 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
2340  SDValue &Hi) {
2341  SDLoc dl(N);
2342  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2343  unsigned NBitWidth = NVT.getSizeInBits();
2344 
2345  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2346  Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT);
2347  // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
2348  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2349  DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
2350 }
2351 
2352 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
2353  SDValue &Hi) {
2354  SDLoc dl(N);
2355  EVT VT = N->getValueType(0);
2356 
2357  SDValue Op = N->getOperand(0);
2358  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2359  Op = GetPromotedFloat(Op);
2360 
2362  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
2363  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, true/*irrelevant*/, dl).first,
2364  Lo, Hi);
2365 }
2366 
2367 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
2368  SDValue &Hi) {
2369  SDLoc dl(N);
2370  EVT VT = N->getValueType(0);
2371 
2372  SDValue Op = N->getOperand(0);
2373  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2374  Op = GetPromotedFloat(Op);
2375 
2377  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
2378  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, false/*irrelevant*/, dl).first,
2379  Lo, Hi);
2380 }
2381 
2382 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
2383  SDValue &Lo, SDValue &Hi) {
2384  if (ISD::isNormalLoad(N)) {
2385  ExpandRes_NormalLoad(N, Lo, Hi);
2386  return;
2387  }
2388 
2389  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2390 
2391  EVT VT = N->getValueType(0);
2392  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2393  SDValue Ch = N->getChain();
2394  SDValue Ptr = N->getBasePtr();
2396  unsigned Alignment = N->getAlignment();
2397  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2398  AAMDNodes AAInfo = N->getAAInfo();
2399  SDLoc dl(N);
2400 
2401  assert(NVT.isByteSized() && "Expanded type not byte sized!");
2402 
2403  if (N->getMemoryVT().bitsLE(NVT)) {
2404  EVT MemVT = N->getMemoryVT();
2405 
2406  Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
2407  Alignment, MMOFlags, AAInfo);
2408 
2409  // Remember the chain.
2410  Ch = Lo.getValue(1);
2411 
2412  if (ExtType == ISD::SEXTLOAD) {
2413  // The high part is obtained by SRA'ing all but one of the bits of the
2414  // lo part.
2415  unsigned LoSize = Lo.getValueSizeInBits();
2416  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2417  DAG.getConstant(LoSize - 1, dl,
2418  TLI.getPointerTy(DAG.getDataLayout())));
2419  } else if (ExtType == ISD::ZEXTLOAD) {
2420  // The high part is just a zero.
2421  Hi = DAG.getConstant(0, dl, NVT);
2422  } else {
2423  assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
2424  // The high part is undefined.
2425  Hi = DAG.getUNDEF(NVT);
2426  }
2427  } else if (DAG.getDataLayout().isLittleEndian()) {
2428  // Little-endian - low bits are at low addresses.
2429  Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2430  AAInfo);
2431 
2432  unsigned ExcessBits =
2433  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
2434  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
2435 
2436  // Increment the pointer to the other half.
2437  unsigned IncrementSize = NVT.getSizeInBits()/8;
2438  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2439  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2440  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
2441  N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
2442  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2443 
2444  // Build a factor node to remember that this load is independent of the
2445  // other one.
2446  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2447  Hi.getValue(1));
2448  } else {
2449  // Big-endian - high bits are at low addresses. Favor aligned loads at
2450  // the cost of some bit-fiddling.
2451  EVT MemVT = N->getMemoryVT();
2452  unsigned EBytes = MemVT.getStoreSize();
2453  unsigned IncrementSize = NVT.getSizeInBits()/8;
2454  unsigned ExcessBits = (EBytes - IncrementSize)*8;
2455 
2456  // Load both the high bits and maybe some of the low bits.
2457  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
2459  MemVT.getSizeInBits() - ExcessBits),
2460  Alignment, MMOFlags, AAInfo);
2461 
2462  // Increment the pointer to the other half.
2463  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2464  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2465  // Load the rest of the low bits.
2466  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
2467  N->getPointerInfo().getWithOffset(IncrementSize),
2468  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
2469  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2470 
2471  // Build a factor node to remember that this load is independent of the
2472  // other one.
2473  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2474  Hi.getValue(1));
2475 
2476  if (ExcessBits < NVT.getSizeInBits()) {
2477  // Transfer low bits from the bottom of Hi to the top of Lo.
2478  Lo = DAG.getNode(
2479  ISD::OR, dl, NVT, Lo,
2480  DAG.getNode(ISD::SHL, dl, NVT, Hi,
2481  DAG.getConstant(ExcessBits, dl,
2482  TLI.getPointerTy(DAG.getDataLayout()))));
2483  // Move high bits to the right position in Hi.
2484  Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
2485  Hi,
2486  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
2487  TLI.getPointerTy(DAG.getDataLayout())));
2488  }
2489  }
2490 
2491  // Legalize the chain result - switch anything that used the old chain to
2492  // use the new one.
2493  ReplaceValueWith(SDValue(N, 1), Ch);
2494 }
2495 
2496 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
2497  SDValue &Lo, SDValue &Hi) {
2498  SDLoc dl(N);
2499  SDValue LL, LH, RL, RH;
2500  GetExpandedInteger(N->getOperand(0), LL, LH);
2501  GetExpandedInteger(N->getOperand(1), RL, RH);
2502  Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
2503  Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
2504 }
2505 
2506 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
2507  SDValue &Lo, SDValue &Hi) {
2508  EVT VT = N->getValueType(0);
2509  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2510  SDLoc dl(N);
2511 
2512  SDValue LL, LH, RL, RH;
2513  GetExpandedInteger(N->getOperand(0), LL, LH);
2514  GetExpandedInteger(N->getOperand(1), RL, RH);
2515 
2516  if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
2518  LL, LH, RL, RH))
2519  return;
2520 
2521  // If nothing else, we can make a libcall.
2522  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2523  if (VT == MVT::i16)
2524  LC = RTLIB::MUL_I16;
2525  else if (VT == MVT::i32)
2526  LC = RTLIB::MUL_I32;
2527  else if (VT == MVT::i64)
2528  LC = RTLIB::MUL_I64;
2529  else if (VT == MVT::i128)
2530  LC = RTLIB::MUL_I128;
2531 
2532  if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
2533  // We'll expand the multiplication by brute force because we have no other
2534  // options. This is a trivially-generalized version of the code from
2535  // Hacker's Delight (itself derived from Knuth's Algorithm M from section
2536  // 4.3.1).
2537  unsigned Bits = NVT.getSizeInBits();
2538  unsigned HalfBits = Bits >> 1;
2539  SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
2540  NVT);
2541  SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
2542  SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
2543 
2544  SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
2545  SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
2546 
2547  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2548  if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
2549  // The type from TLI is too small to fit the shift amount we want.
2550  // Override it with i32. The shift will have to be legalized.
2551  ShiftAmtTy = MVT::i32;
2552  }
2553  SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
2554  SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
2555  SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
2556  SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
2557 
2558  SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
2559  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
2560  SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
2561  SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
2562 
2563  SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
2564  DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
2565  SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
2566 
2567  SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
2568  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
2569  DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
2570  Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
2571  DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
2572 
2573  Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
2574  DAG.getNode(ISD::ADD, dl, NVT,
2575  DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
2576  DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
2577  return;
2578  }
2579 
2580  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2581  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true/*irrelevant*/, dl).first,
2582  Lo, Hi);
2583 }
2584 
2585 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
2586  SDValue &Hi) {
2587  SDLoc DL(N);
2588  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2589  SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
2590  SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
2591  Lo = R.getValue(0);
2592  Hi = R.getValue(1);
2593  ReplaceValueWith(SDValue(N, 1), R.getValue(2));
2594 }
2595 
2596 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
2597  SDValue &Hi) {
2598  SDValue Result = TLI.expandAddSubSat(N, DAG);
2599  SplitInteger(Result, Lo, Hi);
2600 }
2601 
2602 void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
2603  SDValue &Hi) {
2604  assert(
2605  (N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::UMULFIX) &&
2606  "Expected operand to be signed or unsigned fixed point multiplication");
2607 
2608  SDLoc dl(N);
2609  EVT VT = N->getValueType(0);
2610  SDValue LHS = N->getOperand(0);
2611  SDValue RHS = N->getOperand(1);
2612  uint64_t Scale = N->getConstantOperandVal(2);
2613  if (!Scale) {
2614  SDValue Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
2615  SplitInteger(Result, Lo, Hi);
2616  return;
2617  }
2618 
2619  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2620  SDValue LL, LH, RL, RH;
2621  GetExpandedInteger(LHS, LL, LH);
2622  GetExpandedInteger(RHS, RL, RH);
2623  SmallVector<SDValue, 4> Result;
2624 
2625  bool Signed = N->getOpcode() == ISD::SMULFIX;
2626  unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
2627  if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
2629  LL, LH, RL, RH)) {
2630  report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI.");
2631  return;
2632  }
2633 
2634  unsigned VTSize = VT.getScalarSizeInBits();
2635  unsigned NVTSize = NVT.getScalarSizeInBits();
2636  EVT ShiftTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2637 
2638  // Shift whole amount by scale.
2639  SDValue ResultLL = Result[0];
2640  SDValue ResultLH = Result[1];
2641  SDValue ResultHL = Result[2];
2642  SDValue ResultHH = Result[3];
2643 
2644  // After getting the multplication result in 4 parts, we need to perform a
2645  // shift right by the amount of the scale to get the result in that scale.
2646  // Let's say we multiply 2 64 bit numbers. The resulting value can be held in
2647  // 128 bits that are cut into 4 32-bit parts:
2648  //
2649  // HH HL LH LL
2650  // |---32---|---32---|---32---|---32---|
2651  // 128 96 64 32 0
2652  //
2653  // |------VTSize-----|
2654  //
2655  // |NVTSize-|
2656  //
2657  // The resulting Lo and Hi will only need to be one of these 32-bit parts
2658  // after shifting.
2659  if (Scale < NVTSize) {
2660  // If the scale is less than the size of the VT we expand to, the Hi and
2661  // Lo of the result will be in the first 2 parts of the result after
2662  // shifting right. This only requires shifting by the scale as far as the
2663  // third part in the result (ResultHL).
2664  SDValue SRLAmnt = DAG.getConstant(Scale, dl, ShiftTy);
2665  SDValue SHLAmnt = DAG.getConstant(NVTSize - Scale, dl, ShiftTy);
2666  Lo = DAG.getNode(ISD::SRL, dl, NVT, ResultLL, SRLAmnt);
2667  Lo = DAG.getNode(ISD::OR, dl, NVT, Lo,
2668  DAG.getNode(ISD::SHL, dl, NVT, ResultLH, SHLAmnt));
2669  Hi = DAG.getNode(ISD::SRL, dl, NVT, ResultLH, SRLAmnt);
2670  Hi = DAG.getNode(ISD::OR, dl, NVT, Hi,
2671  DAG.getNode(ISD::SHL, dl, NVT, ResultHL, SHLAmnt));
2672  } else if (Scale == NVTSize) {
2673  // If the scales are equal, Lo and Hi are ResultLH and Result HL,
2674  // respectively. Avoid shifting to prevent undefined behavior.
2675  Lo = ResultLH;
2676  Hi = ResultHL;
2677  } else if (Scale < VTSize) {
2678  // If the scale is instead less than the old VT size, but greater than or
2679  // equal to the expanded VT size, the first part of the result (ResultLL) is
2680  // no longer a part of Lo because it would be scaled out anyway. Instead we
2681  // can start shifting right from the fourth part (ResultHH) to the second
2682  // part (ResultLH), and Result LH will be the new Lo.
2683  SDValue SRLAmnt = DAG.getConstant(Scale - NVTSize, dl, ShiftTy);
2684  SDValue SHLAmnt = DAG.getConstant(VTSize - Scale, dl, ShiftTy);
2685  Lo = DAG.getNode(ISD::SRL, dl, NVT, ResultLH, SRLAmnt);
2686  Lo = DAG.getNode(ISD::OR, dl, NVT, Lo,
2687  DAG.getNode(ISD::SHL, dl, NVT, ResultHL, SHLAmnt));
2688  Hi = DAG.getNode(ISD::SRL, dl, NVT, ResultHL, SRLAmnt);
2689  Hi = DAG.getNode(ISD::OR, dl, NVT, Hi,
2690  DAG.getNode(ISD::SHL, dl, NVT, ResultHH, SHLAmnt));
2691  } else if (Scale == VTSize) {
2692  assert(
2693  !Signed &&
2694  "Only unsigned types can have a scale equal to the operand bit width");
2695 
2696  Lo = ResultHL;
2697  Hi = ResultHH;
2698  } else {
2699  llvm_unreachable("Expected the scale to be less than or equal to the width "
2700  "of the operands");
2701  }
2702 }
2703 
2704 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
2705  SDValue &Lo, SDValue &Hi) {
2706  SDValue LHS = Node->getOperand(0);
2707  SDValue RHS = Node->getOperand(1);
2708  SDLoc dl(Node);
2709 
2710  // Expand the result by simply replacing it with the equivalent
2711  // non-overflow-checking operation.
2712  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
2713  ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
2714  LHS, RHS);
2715  SplitInteger(Sum, Lo, Hi);
2716 
2717  // Compute the overflow.
2718  //
2719  // LHSSign -> LHS >= 0
2720  // RHSSign -> RHS >= 0
2721  // SumSign -> Sum >= 0
2722  //
2723  // Add:
2724  // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
2725  // Sub:
2726  // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
2727  //
2728  EVT OType = Node->getValueType(1);
2729  SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
2730 
2731  SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
2732  SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
2733  SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
2734  Node->getOpcode() == ISD::SADDO ?
2736 
2737  SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
2738  SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
2739 
2740  SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
2741 
2742  // Use the calculated overflow everywhere.
2743  ReplaceValueWith(SDValue(Node, 1), Cmp);
2744 }
2745 
2746 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
2747  SDValue &Lo, SDValue &Hi) {
2748  EVT VT = N->getValueType(0);
2749  SDLoc dl(N);
2750  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2751 
2753  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2754  SplitInteger(Res.getValue(0), Lo, Hi);
2755  return;
2756  }
2757 
2758  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2759  if (VT == MVT::i16)
2760  LC = RTLIB::SDIV_I16;
2761  else if (VT == MVT::i32)
2762  LC = RTLIB::SDIV_I32;
2763  else if (VT == MVT::i64)
2764  LC = RTLIB::SDIV_I64;
2765  else if (VT == MVT::i128)
2766  LC = RTLIB::SDIV_I128;
2767  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
2768 
2769  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
2770 }
2771 
2772 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
2773  SDValue &Lo, SDValue &Hi) {
2774  EVT VT = N->getValueType(0);
2775  SDLoc dl(N);
2776 
2777  // If we can emit an efficient shift operation, do so now. Check to see if
2778  // the RHS is a constant.
2779  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
2780  return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
2781 
2782  // If we can determine that the high bit of the shift is zero or one, even if
2783  // the low bits are variable, emit this shift in an optimized form.
2784  if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
2785  return;
2786 
2787  // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
2788  unsigned PartsOpc;
2789  if (N->getOpcode() == ISD::SHL) {
2790  PartsOpc = ISD::SHL_PARTS;
2791  } else if (N->getOpcode() == ISD::SRL) {
2792  PartsOpc = ISD::SRL_PARTS;
2793  } else {
2794  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2795  PartsOpc = ISD::SRA_PARTS;
2796  }
2797 
2798  // Next check to see if the target supports this SHL_PARTS operation or if it
2799  // will custom expand it. Don't lower this to SHL_PARTS when we optimise for
2800  // size, but create a libcall instead.
2801  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2802  TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
2803  const bool LegalOrCustom =
2804  (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
2805  Action == TargetLowering::Custom;
2806 
2807  if (LegalOrCustom && TLI.shouldExpandShift(DAG, N)) {
2808  // Expand the subcomponents.
2809  SDValue LHSL, LHSH;
2810  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2811  EVT VT = LHSL.getValueType();
2812 
2813  // If the shift amount operand is coming from a vector legalization it may
2814  // have an illegal type. Fix that first by casting the operand, otherwise
2815  // the new SHL_PARTS operation would need further legalization.
2816  SDValue ShiftOp = N->getOperand(1);
2817  EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2818  assert(ShiftTy.getScalarSizeInBits() >=
2819  Log2_32_Ceil(VT.getScalarSizeInBits()) &&
2820  "ShiftAmountTy is too small to cover the range of this type!");
2821  if (ShiftOp.getValueType() != ShiftTy)
2822  ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
2823 
2824  SDValue Ops[] = { LHSL, LHSH, ShiftOp };
2825  Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
2826  Hi = Lo.getValue(1);
2827  return;
2828  }
2829 
2830  // Otherwise, emit a libcall.
2831  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2832  bool isSigned;
2833  if (N->getOpcode() == ISD::SHL) {
2834  isSigned = false; /*sign irrelevant*/
2835  if (VT == MVT::i16)
2836  LC = RTLIB::SHL_I16;
2837  else if (VT == MVT::i32)
2838  LC = RTLIB::SHL_I32;
2839  else if (VT == MVT::i64)
2840  LC = RTLIB::SHL_I64;
2841  else if (VT == MVT::i128)
2842  LC = RTLIB::SHL_I128;
2843  } else if (N->getOpcode() == ISD::SRL) {
2844  isSigned = false;
2845  if (VT == MVT::i16)
2846  LC = RTLIB::SRL_I16;
2847  else if (VT == MVT::i32)
2848  LC = RTLIB::SRL_I32;
2849  else if (VT == MVT::i64)
2850  LC = RTLIB::SRL_I64;
2851  else if (VT == MVT::i128)
2852  LC = RTLIB::SRL_I128;
2853  } else {
2854  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2855  isSigned = true;
2856  if (VT == MVT::i16)
2857  LC = RTLIB::SRA_I16;
2858  else if (VT == MVT::i32)
2859  LC = RTLIB::SRA_I32;
2860  else if (VT == MVT::i64)
2861  LC = RTLIB::SRA_I64;
2862  else if (VT == MVT::i128)
2863  LC = RTLIB::SRA_I128;
2864  }
2865 
2866  if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
2867  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2868  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, isSigned, dl).first, Lo, Hi);
2869  return;
2870  }
2871 
2872  if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
2873  llvm_unreachable("Unsupported shift!");
2874 }
2875 
2876 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
2877  SDValue &Lo, SDValue &Hi) {
2878  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2879  SDLoc dl(N);
2880  SDValue Op = N->getOperand(0);
2881  if (Op.getValueType().bitsLE(NVT)) {
2882  // The low part is sign extension of the input (degenerates to a copy).
2883  Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
2884  // The high part is obtained by SRA'ing all but one of the bits of low part.
2885  unsigned LoSize = NVT.getSizeInBits();
2886  Hi = DAG.getNode(
2887  ISD::SRA, dl, NVT, Lo,
2888  DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
2889  } else {
2890  // For example, extension of an i48 to an i64. The operand type necessarily
2891  // promotes to the result type, so will end up being expanded too.
2892  assert(getTypeAction(Op.getValueType()) ==
2894  "Only know how to promote this result!");
2895  SDValue Res = GetPromotedInteger(Op);
2896  assert(Res.getValueType() == N->getValueType(0) &&
2897  "Operand over promoted?");
2898  // Split the promoted operand. This will simplify when it is expanded.
2899  SplitInteger(Res, Lo, Hi);
2900  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
2901  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2903  ExcessBits)));
2904  }
2905 }
2906 
2907 void DAGTypeLegalizer::
2908 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2909  SDLoc dl(N);
2910  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2911  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2912 
2913  if (EVT.bitsLE(Lo.getValueType())) {
2914  // sext_inreg the low part if needed.
2915  Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
2916  N->getOperand(1));
2917 
2918  // The high part gets the sign extension from the lo-part. This handles
2919  // things like sextinreg V:i64 from i8.
2920  Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
2921  DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
2922  TLI.getPointerTy(DAG.getDataLayout())));
2923  } else {
2924  // For example, extension of an i48 to an i64. Leave the low part alone,
2925  // sext_inreg the high part.
2926  unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
2927  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2929  ExcessBits)));
2930  }
2931 }
2932 
2933 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
2934  SDValue &Lo, SDValue &Hi) {
2935  EVT VT = N->getValueType(0);
2936  SDLoc dl(N);
2937  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2938 
2940  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2941  SplitInteger(Res.getValue(1), Lo, Hi);
2942  return;
2943  }
2944 
2945  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2946  if (VT == MVT::i16)
2947  LC = RTLIB::SREM_I16;
2948  else if (VT == MVT::i32)
2949  LC = RTLIB::SREM_I32;
2950  else if (VT == MVT::i64)
2951  LC = RTLIB::SREM_I64;
2952  else if (VT == MVT::i128)
2953  LC = RTLIB::SREM_I128;
2954  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
2955 
2956  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
2957 }
2958 
2959 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
2960  SDValue &Lo, SDValue &Hi) {
2961  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2962  SDLoc dl(N);
2963  Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
2964  Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
2965  N->getOperand(0),
2966  DAG.getConstant(NVT.getSizeInBits(), dl,
2967  TLI.getPointerTy(DAG.getDataLayout())));
2968  Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
2969 }
2970 
2971 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
2972  SDValue &Lo, SDValue &Hi) {
2973  EVT VT = N->getValueType(0);
2974  SDLoc dl(N);
2975 
2976  if (N->getOpcode() == ISD::UMULO) {
2977  // This section expands the operation into the following sequence of
2978  // instructions. `iNh` here refers to a type which has half the bit width of
2979  // the type the original operation operated on.
2980  //
2981  // %0 = %LHS.HI != 0 && %RHS.HI != 0
2982  // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
2983  // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
2984  // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
2985  // %4 = add iN (%1.0 as iN) << Nh, (%2.0 as iN) << Nh
2986  // %5 = { iN, i1 } @uadd.with.overflow.iN( %4, %3 )
2987  //
2988  // %res = { %5.0, %0 || %1.1 || %2.1 || %5.1 }
2989  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
2990  SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
2991  SplitInteger(LHS, LHSLow, LHSHigh);
2992  SplitInteger(RHS, RHSLow, RHSHigh);
2993  EVT HalfVT = LHSLow.getValueType()
2994  , BitVT = N->getValueType(1);
2995  SDVTList VTHalfMulO = DAG.getVTList(HalfVT, BitVT);
2996  SDVTList VTFullAddO = DAG.getVTList(VT, BitVT);
2997 
2998  SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
2999  SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
3000  DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
3001  DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));
3002 
3003  SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, LHSHigh, RHSLow);
3004  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));
3005  SDValue OneInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3006  One.getValue(0));
3007 
3008  SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, RHSHigh, LHSLow);
3009  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));
3010  SDValue TwoInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3011  Two.getValue(0));
3012 
3013  // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
3014  // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
3015  // operation recursively legalized?).
3016  //
3017  // Many backends understand this pattern and will convert into LOHI
3018  // themselves, if applicable.
3019  SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
3020  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
3021  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow));
3022  SDValue Four = DAG.getNode(ISD::ADD, dl, VT, OneInHigh, TwoInHigh);
3023  SDValue Five = DAG.getNode(ISD::UADDO, dl, VTFullAddO, Three, Four);
3024  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Five.getValue(1));
3025  SplitInteger(Five, Lo, Hi);
3026  ReplaceValueWith(SDValue(N, 1), Overflow);
3027  return;
3028  }
3029 
3030  Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
3031  EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
3032  Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
3033 
3034  // Replace this with a libcall that will check overflow.
3035  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3036  if (VT == MVT::i32)
3037  LC = RTLIB::MULO_I32;
3038  else if (VT == MVT::i64)
3039  LC = RTLIB::MULO_I64;
3040  else if (VT == MVT::i128)
3041  LC = RTLIB::MULO_I128;
3042  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
3043 
3044  SDValue Temp = DAG.CreateStackTemporary(PtrVT);
3045  // Temporary for the overflow value, default it to zero.
3046  SDValue Chain =
3047  DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
3048  MachinePointerInfo());
3049 
3051  TargetLowering::ArgListEntry Entry;
3052  for (const SDValue &Op : N->op_values()) {
3053  EVT ArgVT = Op.getValueType();
3054  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
3055  Entry.Node = Op;
3056  Entry.Ty = ArgTy;
3057  Entry.IsSExt = true;
3058  Entry.IsZExt = false;
3059  Args.push_back(Entry);
3060  }
3061 
3062  // Also pass the address of the overflow check.
3063  Entry.Node = Temp;
3064  Entry.Ty = PtrTy->getPointerTo();
3065  Entry.IsSExt = true;
3066  Entry.IsZExt = false;
3067  Args.push_back(Entry);
3068 
3069  SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
3070 
3072  CLI.setDebugLoc(dl)
3073  .setChain(Chain)
3074  .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
3075  .setSExtResult();
3076 
3077  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
3078 
3079  SplitInteger(CallInfo.first, Lo, Hi);
3080  SDValue Temp2 =
3081  DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
3082  SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
3083  DAG.getConstant(0, dl, PtrVT),
3084  ISD::SETNE);
3085  // Use the overflow from the libcall everywhere.
3086  ReplaceValueWith(SDValue(N, 1), Ofl);
3087 }
3088 
3089 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
3090  SDValue &Lo, SDValue &Hi) {
3091  EVT VT = N->getValueType(0);
3092  SDLoc dl(N);
3093  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3094 
3096  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3097  SplitInteger(Res.getValue(0), Lo, Hi);
3098  return;
3099  }
3100 
3101  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3102  if (VT == MVT::i16)
3103  LC = RTLIB::UDIV_I16;
3104  else if (VT == MVT::i32)
3105  LC = RTLIB::UDIV_I32;
3106  else if (VT == MVT::i64)
3107  LC = RTLIB::UDIV_I64;
3108  else if (VT == MVT::i128)
3109  LC = RTLIB::UDIV_I128;
3110  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
3111 
3112  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
3113 }
3114 
3115 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
3116  SDValue &Lo, SDValue &Hi) {
3117  EVT VT = N->getValueType(0);
3118  SDLoc dl(N);
3119  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3120 
3122  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3123  SplitInteger(Res.getValue(1), Lo, Hi);
3124  return;
3125  }
3126 
3127  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3128  if (VT == MVT::i16)
3129  LC = RTLIB::UREM_I16;
3130  else if (VT == MVT::i32)
3131  LC = RTLIB::UREM_I32;
3132  else if (VT == MVT::i64)
3133  LC = RTLIB::UREM_I64;
3134  else if (VT == MVT::i128)
3135  LC = RTLIB::UREM_I128;
3136  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
3137 
3138  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
3139 }
3140 
3141 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
3142  SDValue &Lo, SDValue &Hi) {
3143  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3144  SDLoc dl(N);
3145  SDValue Op = N->getOperand(0);
3146  if (Op.getValueType().bitsLE(NVT)) {
3147  // The low part is zero extension of the input (degenerates to a copy).
3148  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
3149  Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
3150  } else {
3151  // For example, extension of an i48 to an i64. The operand type necessarily
3152  // promotes to the result type, so will end up being expanded too.
3153  assert(getTypeAction(Op.getValueType()) ==
3155  "Only know how to promote this result!");
3156  SDValue Res = GetPromotedInteger(Op);
3157  assert(Res.getValueType() == N->getValueType(0) &&
3158  "Operand over promoted?");
3159  // Split the promoted operand. This will simplify when it is expanded.
3160  SplitInteger(Res, Lo, Hi);
3161  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
3162  Hi = DAG.getZeroExtendInReg(Hi, dl,
3164  ExcessBits));
3165  }
3166 }
3167 
3168 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
3169  SDValue &Lo, SDValue &Hi) {
3170  SDLoc dl(N);
3171  EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
3172  SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
3173  SDValue Zero = DAG.getConstant(0, dl, VT);
3174  SDValue Swap = DAG.getAtomicCmpSwap(
3176  cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
3177  N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
3178 
3179  ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
3180  ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
3181 }
3182 
3183 //===----------------------------------------------------------------------===//
3184 // Integer Operand Expansion
3185 //===----------------------------------------------------------------------===//
3186 
3187 /// ExpandIntegerOperand - This method is called when the specified operand of
3188 /// the specified node is found to need expansion. At this point, all of the
3189 /// result types of the node are known to be legal, but other operands of the
3190 /// node may need promotion or expansion as well as the specified one.
3191 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
3192  LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG);
3193  dbgs() << "\n");
3194  SDValue Res = SDValue();
3195 
3196  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
3197  return false;
3198 
3199  switch (N->getOpcode()) {
3200  default:
3201  #ifndef NDEBUG
3202  dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
3203  N->dump(&DAG); dbgs() << "\n";
3204  #endif
3205  report_fatal_error("Do not know how to expand this operator's operand!");
3206 
3207  case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
3208  case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
3209  case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
3210  case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
3211  case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
3212  case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
3213  case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
3214  case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
3215  case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
3216  case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break;
3217  case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
3218  case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
3219  case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break;
3220 
3221  case ISD::SHL:
3222  case ISD::SRA:
3223  case ISD::SRL:
3224  case ISD::ROTL:
3225  case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
3226  case ISD::RETURNADDR:
3227  case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
3228 
3229  case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
3230  }
3231 
3232  // If the result is null, the sub-method took care of registering results etc.
3233  if (!Res.getNode()) return false;
3234 
3235  // If the result is N, the sub-method updated N in place. Tell the legalizer
3236  // core about this.
3237  if (Res.getNode() == N)
3238  return true;
3239 
3240  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3241  "Invalid operand expansion");
3242 
3243  ReplaceValueWith(SDValue(N, 0), Res);
3244  return false;
3245 }
3246 
3247 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
3248 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
3249 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
3250  SDValue &NewRHS,
3251  ISD::CondCode &CCCode,
3252  const SDLoc &dl) {
3253  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3254  GetExpandedInteger(NewLHS, LHSLo, LHSHi);
3255  GetExpandedInteger(NewRHS, RHSLo, RHSHi);
3256 
3257  if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
3258  if (RHSLo == RHSHi) {
3259  if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
3260  if (RHSCST->isAllOnesValue()) {
3261  // Equality comparison to -1.
3262  NewLHS = DAG.getNode(ISD::AND, dl,
3263  LHSLo.getValueType(), LHSLo, LHSHi);
3264  NewRHS = RHSLo;
3265  return;
3266  }
3267  }
3268  }
3269 
3270  NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
3271  NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
3272  NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
3273  NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
3274  return;
3275  }
3276 
3277  // If this is a comparison of the sign bit, just look at the top part.
3278  // X > -1, x < 0
3279  if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
3280  if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0
3281  (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1
3282  NewLHS = LHSHi;
3283  NewRHS = RHSHi;
3284  return;
3285  }
3286 
3287  // FIXME: This generated code sucks.
3288  ISD::CondCode LowCC;
3289  switch (CCCode) {
3290  default: llvm_unreachable("Unknown integer setcc!");
3291  case ISD::SETLT:
3292  case ISD::SETULT: LowCC = ISD::SETULT; break;
3293  case ISD::SETGT:
3294  case ISD::SETUGT: LowCC = ISD::SETUGT; break;
3295  case ISD::SETLE:
3296  case ISD::SETULE: LowCC = ISD::SETULE; break;
3297  case ISD::SETGE:
3298  case ISD::SETUGE: LowCC = ISD::SETUGE; break;
3299  }
3300 
3301  // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
3302  // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
3303  // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
3304 
3305  // NOTE: on targets without efficient SELECT of bools, we can always use
3306  // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
3307  TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
3308  nullptr);
3309  SDValue LoCmp, HiCmp;
3310  if (TLI.isTypeLegal(LHSLo.getValueType()) &&
3311  TLI.isTypeLegal(RHSLo.getValueType()))
3312  LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
3313  RHSLo, LowCC, false, DagCombineInfo, dl);
3314  if (!LoCmp.getNode())
3315  LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
3316  RHSLo, LowCC);
3317  if (TLI.isTypeLegal(LHSHi.getValueType()) &&
3318  TLI.isTypeLegal(RHSHi.getValueType()))
3319  HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
3320  RHSHi, CCCode, false, DagCombineInfo, dl);
3321  if (!HiCmp.getNode())
3322  HiCmp =
3323  DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
3324  LHSHi, RHSHi, DAG.getCondCode(CCCode));
3325 
3326  ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
3327  ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
3328 
3329  bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
3330  CCCode == ISD::SETUGE || CCCode == ISD::SETULE);
3331 
3332  if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) ||
3333  (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) ||
3334  (LoCmpC && LoCmpC->isNullValue())))) {
3335  // For LE / GE, if high part is known false, ignore the low part.
3336  // For LT / GT: if low part is known false, return the high part.
3337  // if high part is known true, ignore the low part.
3338  NewLHS = HiCmp;
3339  NewRHS = SDValue();
3340  return;
3341  }
3342 
3343  if (LHSHi == RHSHi) {
3344  // Comparing the low bits is enough.
3345  NewLHS = LoCmp;
3346  NewRHS = SDValue();
3347  return;
3348  }
3349 
3350  // Lower with SETCCCARRY if the target supports it.
3351  EVT HiVT = LHSHi.getValueType();
3352  EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
3353  bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
3354 
3355  // FIXME: Make all targets support this, then remove the other lowering.
3356  if (HasSETCCCARRY) {
3357  // SETCCCARRY can detect < and >= directly. For > and <=, flip
3358  // operands and condition code.
3359  bool FlipOperands = false;
3360  switch (CCCode) {
3361  case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
3362  case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
3363  case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
3364  case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
3365  default: break;
3366  }
3367  if (FlipOperands) {
3368  std::swap(LHSLo, RHSLo);
3369  std::swap(LHSHi, RHSHi);
3370  }
3371  // Perform a wide subtraction, feeding the carry from the low part into
3372  // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
3373  // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
3374  // zero or positive iff LHS >= RHS.
3375  EVT LoVT = LHSLo.getValueType();
3376  SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
3377  SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
3378  SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
3379  LHSHi, RHSHi, LowCmp.getValue(1),
3380  DAG.getCondCode(CCCode));
3381  NewLHS = Res;
3382  NewRHS = SDValue();
3383  return;
3384  }
3385 
3386  NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
3387  false, DagCombineInfo, dl);
3388  if (!NewLHS.getNode())
3389  NewLHS =
3390  DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
3391  NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
3392  NewRHS = SDValue();
3393 }
3394 
3395 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
3396  SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
3397  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
3398  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3399 
3400  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3401  // against zero to select between true and false values.
3402  if (!NewRHS.getNode()) {
3403  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3404  CCCode = ISD::SETNE;
3405  }
3406 
3407  // Update N to have the operands specified.
3408  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
3409  DAG.getCondCode(CCCode), NewLHS, NewRHS,
3410  N->getOperand(4)), 0);
3411 }
3412 
3413 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
3414  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3415  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
3416  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3417 
3418  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3419  // against zero to select between true and false values.
3420  if (!NewRHS.getNode()) {
3421  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3422  CCCode = ISD::SETNE;
3423  }
3424 
3425  // Update N to have the operands specified.
3426  return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
3427  N->getOperand(2), N->getOperand(3),
3428  DAG.getCondCode(CCCode)), 0);
3429 }
3430 
3431 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
3432  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3433  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3434  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3435 
3436  // If ExpandSetCCOperands returned a scalar, use it.
3437  if (!NewRHS.getNode()) {
3438  assert(NewLHS.getValueType() == N->getValueType(0) &&
3439  "Unexpected setcc expansion!");
3440  return NewLHS;
3441  }
3442 
3443  // Otherwise, update N to have the operands specified.
3444  return SDValue(
3445  DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
3446 }
3447 
3448 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
3449  SDValue LHS = N->getOperand(0);
3450  SDValue RHS = N->getOperand(1);
3451  SDValue Carry = N->getOperand(2);
3452  SDValue Cond = N->getOperand(3);
3453  SDLoc dl = SDLoc(N);
3454 
3455  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3456  GetExpandedInteger(LHS, LHSLo, LHSHi);
3457  GetExpandedInteger(RHS, RHSLo, RHSHi);
3458 
3459  // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
3460  SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
3461  SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
3462  return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
3463  LowCmp.getValue(1), Cond);
3464 }
3465 
3466 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
3467  // The value being shifted is legal, but the shift amount is too big.
3468  // It follows that either the result of the shift is undefined, or the
3469  // upper half of the shift amount is zero. Just use the lower half.
3470  SDValue Lo, Hi;
3471  GetExpandedInteger(N->getOperand(1), Lo, Hi);
3472  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
3473 }
3474 
3475 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
3476  // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
3477  // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
3478  // constant to valid type.
3479  SDValue Lo, Hi;
3480  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3481  return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
3482 }
3483 
3484 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
3485  SDValue Op = N->getOperand(0);
3486  EVT DstVT = N->getValueType(0);
3488  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3489  "Don't know how to expand this SINT_TO_FP!");
3490  return TLI.makeLibCall(DAG, LC, DstVT, Op, true, SDLoc(N)).first;
3491 }
3492 
3493 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
3494  if (ISD::isNormalStore(N))
3495  return ExpandOp_NormalStore(N, OpNo);
3496 
3497  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
3498  assert(OpNo == 1 && "Can only expand the stored value so far");
3499 
3500  EVT VT = N->getOperand(1).getValueType();
3501  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3502  SDValue Ch = N->getChain();
3503  SDValue Ptr = N->getBasePtr();
3504  unsigned Alignment = N->getAlignment();
3505  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3506  AAMDNodes AAInfo = N->getAAInfo();
3507  SDLoc dl(N);
3508  SDValue Lo, Hi;
3509 
3510  assert(NVT.isByteSized() && "Expanded type not byte sized!");
3511 
3512  if (N->getMemoryVT().bitsLE(NVT)) {
3513  GetExpandedInteger(N->getValue(), Lo, Hi);
3514  return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
3515  N->getMemoryVT(), Alignment, MMOFlags, AAInfo);
3516  }
3517 
3518  if (DAG.getDataLayout().isLittleEndian()) {
3519  // Little-endian - low bits are at low addresses.
3520  GetExpandedInteger(N->getValue(), Lo, Hi);
3521 
3522  Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3523  AAInfo);
3524 
3525  unsigned ExcessBits =
3526  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3527  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3528 
3529  // Increment the pointer to the other half.
3530  unsigned IncrementSize = NVT.getSizeInBits()/8;
3531  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3532  Hi = DAG.getTruncStore(
3533  Ch, dl, Hi, Ptr, N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3534  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3535  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3536  }
3537 
3538  // Big-endian - high bits are at low addresses. Favor aligned stores at
3539  // the cost of some bit-fiddling.
3540  GetExpandedInteger(N->getValue(), Lo, Hi);
3541 
3542  EVT ExtVT = N->getMemoryVT();
3543  unsigned EBytes = ExtVT.getStoreSize();
3544  unsigned IncrementSize = NVT.getSizeInBits()/8;
3545  unsigned ExcessBits = (EBytes - IncrementSize)*8;
3546  EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
3547  ExtVT.getSizeInBits() - ExcessBits);
3548 
3549  if (ExcessBits < NVT.getSizeInBits()) {
3550  // Transfer high bits from the top of Lo to the bottom of Hi.
3551  Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
3552  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3553  TLI.getPointerTy(DAG.getDataLayout())));
3554  Hi = DAG.getNode(
3555  ISD::OR, dl, NVT, Hi,
3556  DAG.getNode(ISD::SRL, dl, NVT, Lo,
3557  DAG.getConstant(ExcessBits, dl,
3558  TLI.getPointerTy(DAG.getDataLayout()))));
3559  }
3560 
3561  // Store both the high bits and maybe some of the low bits.
3562  Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment,
3563  MMOFlags, AAInfo);
3564 
3565  // Increment the pointer to the other half.
3566  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3567  // Store the lowest ExcessBits bits in the second half.
3568  Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
3569  N->getPointerInfo().getWithOffset(IncrementSize),
3570  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3571  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3572  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3573 }
3574 
3575 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
3576  SDValue InL, InH;
3577  GetExpandedInteger(N->getOperand(0), InL, InH);
3578  // Just truncate the low part of the source.
3579  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
3580 }
3581 
3582 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
3583  SDValue Op = N->getOperand(0);
3584  EVT SrcVT = Op.getValueType();
3585  EVT DstVT = N->getValueType(0);
3586  SDLoc dl(N);
3587 
3588  // The following optimization is valid only if every value in SrcVT (when
3589  // treated as signed) is representable in DstVT. Check that the mantissa
3590  // size of DstVT is >= than the number of bits in SrcVT -1.
3591  const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT);
3592  if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 &&
3594  // Do a signed conversion then adjust the result.
3595  SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op);
3596  SignedConv = TLI.LowerOperation(SignedConv, DAG);
3597 
3598  // The result of the signed conversion needs adjusting if the 'sign bit' of
3599  // the incoming integer was set. To handle this, we dynamically test to see
3600  // if it is set, and, if so, add a fudge factor.
3601 
3602  const uint64_t F32TwoE32 = 0x4F800000ULL;
3603  const uint64_t F32TwoE64 = 0x5F800000ULL;
3604  const uint64_t F32TwoE128 = 0x7F800000ULL;
3605 
3606  APInt FF(32, 0);
3607  if (SrcVT == MVT::i32)
3608  FF = APInt(32, F32TwoE32);
3609  else if (SrcVT == MVT::i64)
3610  FF = APInt(32, F32TwoE64);
3611  else if (SrcVT == MVT::i128)
3612  FF = APInt(32, F32TwoE128);
3613  else
3614  llvm_unreachable("Unsupported UINT_TO_FP!");
3615 
3616  // Check whether the sign bit is set.
3617  SDValue Lo, Hi;
3618  GetExpandedInteger(Op, Lo, Hi);
3619  SDValue SignSet = DAG.getSetCC(dl,
3620  getSetCCResultType(Hi.getValueType()),
3621  Hi,
3622  DAG.getConstant(0, dl, Hi.getValueType()),
3623  ISD::SETLT);
3624 
3625  // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
3626  SDValue FudgePtr =
3627  DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)),
3628  TLI.getPointerTy(DAG.getDataLayout()));
3629 
3630  // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
3631  SDValue Zero = DAG.getIntPtrConstant(0, dl);
3632  SDValue Four = DAG.getIntPtrConstant(4, dl);
3633  if (DAG.getDataLayout().isBigEndian())
3634  std::swap(Zero, Four);
3635  SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet,
3636  Zero, Four);
3637  unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment();
3638  FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(),
3639  FudgePtr, Offset);
3640  Alignment = std::min(Alignment, 4u);
3641 
3642  // Load the value out, extending it from f32 to the destination float type.
3643  // FIXME: Avoid the extend by constructing the right constant pool?
3644  SDValue Fudge = DAG.getExtLoad(
3645  ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), FudgePtr,
3647  Alignment);
3648  return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge);
3649  }
3650 
3651  // Otherwise, use a libcall.
3652  RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT);
3653  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3654  "Don't know how to expand this UINT_TO_FP!");
3655  return TLI.makeLibCall(DAG, LC, DstVT, Op, true, dl).first;
3656 }
3657 
3658 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
3659  SDLoc dl(N);
3660  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
3661  cast<AtomicSDNode>(N)->getMemoryVT(),
3662  N->getOperand(0),
3663  N->getOperand(1), N->getOperand(2),
3664  cast<AtomicSDNode>(N)->getMemOperand());
3665  return Swap.getValue(1);
3666 }
3667 
3668 
3669 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
3670  SDValue InOp0 = N->getOperand(0);
3671  EVT InVT = InOp0.getValueType();
3672 
3673  EVT OutVT = N->getValueType(0);
3674  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3675  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3676  unsigned OutNumElems = OutVT.getVectorNumElements();
3677  EVT NOutVTElem = NOutVT.getVectorElementType();
3678 
3679  SDLoc dl(N);
3680  SDValue BaseIdx = N->getOperand(1);
3681 
3683  Ops.reserve(OutNumElems);
3684  for (unsigned i = 0; i != OutNumElems; ++i) {
3685 
3686  // Extract the element from the original vector.
3687  SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
3688  BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
3690  InVT.getVectorElementType(), N->getOperand(0), Index);
3691 
3692  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext);
3693  // Insert the converted element to the new vector.
3694  Ops.push_back(Op);
3695  }
3696 
3697  return DAG.getBuildVector(NOutVT, dl, Ops);
3698 }
3699 
3700 
3701 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
3702  ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
3703  EVT VT = N->getValueType(0);
3704  SDLoc dl(N);
3705 
3706  ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
3707 
3708  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3709  SDValue V1 = GetPromotedInteger(N->getOperand(1));
3710  EVT OutVT = V0.getValueType();
3711 
3712  return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
3713 }
3714 
3715 
3716 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
3717  EVT OutVT = N->getValueType(0);
3718  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3719  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3720  unsigned NumElems = N->getNumOperands();
3721  EVT NOutVTElem = NOutVT.getVectorElementType();
3722 
3723  SDLoc dl(N);
3724 
3726  Ops.reserve(NumElems);
3727  for (unsigned i = 0; i != NumElems; ++i) {
3728  SDValue Op;
3729  // BUILD_VECTOR integer operand types are allowed to be larger than the
3730  // result's element type. This may still be true after the promotion. For
3731  // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
3732  // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
3733  if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
3734  Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
3735  else
3736  Op = N->getOperand(i);
3737  Ops.push_back(Op);
3738  }
3739 
3740  return DAG.getBuildVector(NOutVT, dl, Ops);
3741 }
3742 
3743 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
3744 
3745  SDLoc dl(N);
3746 
3747  assert(!N->getOperand(0).getValueType().isVector() &&
3748  "Input must be a scalar");
3749 
3750  EVT OutVT = N->getValueType(0);
3751  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3752  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3753  EVT NOutVTElem = NOutVT.getVectorElementType();
3754 
3755  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
3756 
3757  return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
3758 }
3759 
3760 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
3761  SDLoc dl(N);
3762 
3763  EVT OutVT = N->getValueType(0);
3764  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3765  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3766 
3767  EVT OutElemTy = NOutVT.getVectorElementType();
3768 
3769  unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
3770  unsigned NumOutElem = NOutVT.getVectorNumElements();
3771  unsigned NumOperands = N->getNumOperands();
3772  assert(NumElem * NumOperands == NumOutElem &&
3773  "Unexpected number of elements");
3774 
3775  // Take the elements from the first vector.
3776  SmallVector<SDValue, 8> Ops(NumOutElem);
3777  for (unsigned i = 0; i < NumOperands; ++i) {
3778  SDValue Op = N->getOperand(i);
3779  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
3780  Op = GetPromotedInteger(Op);
3781  EVT SclrTy = Op.getValueType().getVectorElementType();
3782  assert(NumElem == Op.getValueType().getVectorNumElements() &&
3783  "Unexpected number of elements");
3784 
3785  for (unsigned j = 0; j < NumElem; ++j) {
3786  SDValue Ext = DAG.getNode(
3787  ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
3788  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3789  Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
3790  }
3791  }
3792 
3793  return DAG.getBuildVector(NOutVT, dl, Ops);
3794 }
3795 
3796 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
3797  EVT VT = N->getValueType(0);
3798  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3799  assert(NVT.isVector() && "This type must be promoted to a vector type");
3800 
3801  SDLoc dl(N);
3802 
3803  // For operands whose TypeAction is to promote, extend the promoted node
3804  // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
3805  // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
3806  // type..
3807  if (getTypeAction(N->getOperand(0).getValueType())
3809  SDValue Promoted;
3810 
3811  switch(N->getOpcode()) {
3813  Promoted = SExtPromotedInteger(N->getOperand(0));
3814  break;
3816  Promoted = ZExtPromotedInteger(N->getOperand(0));
3817  break;
3819  Promoted = GetPromotedInteger(N->getOperand(0));
3820  break;
3821  default:
3822  llvm_unreachable("Node has unexpected Opcode");
3823  }
3824  return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
3825  }
3826 
3827  // Directly extend to the appropriate transform-to type.
3828  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
3829 }
3830 
3831 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
3832  EVT OutVT = N->getValueType(0);
3833  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3834  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3835 
3836  EVT NOutVTElem = NOutVT.getVectorElementType();
3837 
3838  SDLoc dl(N);
3839  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3840 
3841  SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
3842  NOutVTElem, N->getOperand(1));
3843  return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
3844  V0, ConvElem, N->getOperand(2));
3845 }
3846 
3847 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3848  SDLoc dl(N);
3849  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3850  SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
3851  TLI.getVectorIdxTy(DAG.getDataLayout()));
3853  V0->getValueType(0).getScalarType(), V0, V1);
3854 
3855  // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
3856  // element types. If this is the case then we need to expand the outgoing
3857  // value and not truncate it.
3858  return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
3859 }
3860 
3861 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
3862  SDLoc dl(N);
3863  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3864  MVT InVT = V0.getValueType().getSimpleVT();
3865  MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
3867  SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
3868  return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
3869 }
3870 
3871 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
3872  SDLoc dl(N);
3873  unsigned NumElems = N->getNumOperands();
3874 
3875  EVT RetSclrTy = N->getValueType(0).getVectorElementType();
3876 
3877  SmallVector<SDValue, 8> NewOps;
3878  NewOps.reserve(NumElems);
3879 
3880  // For each incoming vector
3881  for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
3882  SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
3883  EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
3884  unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
3885 
3886  for (unsigned i=0; i<NumElem; ++i) {
3887  // Extract element from incoming vector
3888  SDValue Ex = DAG.getNode(
3889  ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
3890  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3891  SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
3892  NewOps.push_back(Tr);
3893  }
3894  }
3895 
3896  return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
3897 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:570
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:551
static EVT getShiftAmountTyForConstant(unsigned Val, EVT VT, const TargetLowering &TLI, SelectionDAG &DAG)
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getValueType() const
Return the ValueType of the referenced return value.
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant, which is required to be operand #1) half of the integer or float value specified as operand #0.
Definition: ISDOpcodes.h:183
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC&#39;s if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:936
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:357
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:651
This class represents lattice values for constants.
Definition: AllocatorList.h:23
static MVT getVectorVT(MVT VT, unsigned NumElements)
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:366
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:259
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:518
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none...
const SDValue & getBasePtr() const
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:222
void push_back(const T &Elt)
Definition: SmallVector.h:211
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:857
const SDValue & getValue() const
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
Definition: APInt.h:647
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:252
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:250
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1328
const SDValue & getBasePtr() const
const SDValue & getChain() const
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
unsigned getAlignment() const
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:806
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select&#39;s if you just have operands and don&#39;t want to check...
Definition: SelectionDAG.h:949
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:507
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:383
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Same for subtraction.
Definition: ISDOpcodes.h:253
void reserve(size_type N)
Definition: SmallVector.h:368
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:211
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align=0, int Offs=0, bool isT=false, unsigned char TargetFlags=0)
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:434
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:38
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
SDValue getExternalSymbol(const char *Sym, EVT VT)
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
virtual bool shouldExpandShift(SelectionDAG &DAG, SDNode *N) const
Return true if SHIFT instructions should be expanded to SHIFT_PARTS instructions, and false if a libr...
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:209
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations...
Definition: ISDOpcodes.h:455
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
static uint32_t getAlignment(const MCSectionCOFF &Sec)
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:450
Shift and rotation operations.
Definition: ISDOpcodes.h:409
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:201
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:651
CallLoweringInfo & setChain(SDValue InChain)
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:190
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:279
const SDValue & getPassThru() const
op_iterator op_end() const
unsigned getScalarValueSizeInBits() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
FLT_ROUNDS_ - Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest 2 Round to ...
Definition: ISDOpcodes.h:545
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:303
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt) For double-word atomic operations: ValLo, ValHi, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amtLo, amtHi) ValLo, ValHi, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amtLo, amtHi) These correspond to the atomicrmw instruction.
Definition: ISDOpcodes.h:814
const SDValue & getValue() const
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:459
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:400
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG...
Definition: ISDOpcodes.h:72
This is an SDNode representing atomic operations.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
This represents a list of ValueType&#39;s that has been intern&#39;d by a SelectionDAG.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
This class is used to represent an MSTORE node.
unsigned getSizeInBits() const
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:266
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:297
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:291
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:397
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:477
SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
const SDValue & getScale() const
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:428
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:220
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:851
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTruncating=false, bool IsCompressing=false)
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:965
op_iterator op_begin() const
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:751
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:496
const SDValue & getMask() const
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, bool isSigned, const SDLoc &dl, bool doesNotReturn=false, bool isReturnValueUsed=true, bool isPostTypeLegalization=false) const
Returns a pair of (return value, chain).
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands...
UNDEF - An undefined node.
Definition: ISDOpcodes.h:177
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:523
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:326
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
Definition: APInt.h:635
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits...
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
Definition: MathExtras.h:609
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const SDValue & getBasePtr() const
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:234
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:428
Machine Value Type.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
Simple binary floating point operators.
Definition: ISDOpcodes.h:282
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:272
This is an important base class in LLVM.
Definition: Constant.h:41
iterator_range< value_op_iterator > op_values() const
const SDValue & getOperand(unsigned Num) const
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:940
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:331
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:246
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:231
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:587
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:56
const SDValue & getValue() const
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:723
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType, bool IsExpanding=false)
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:415
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo...
Definition: ISDOpcodes.h:800
std::vector< ArgListEntry > ArgListTy
const SDValue & getIndex() const
Extended Value Type.
Definition: ValueTypes.h:33
const SDValue & getBasePtr() const
This structure contains all information that is necessary for lowering calls.
This class contains a discriminated union of information about pointers in memory operands...
unsigned getNumOperands() const
Return the number of values used by this operation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type...
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:264
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:49
void dump() const
Dump this node, for debugging.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:338
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:264
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
Definition: ISDOpcodes.h:450
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:221
const SDValue & getMask() const
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:839
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:733
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:645
const SDValue & getPassThru() const
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:643
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:412
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:580
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:621
Represents one node in the SelectionDAG.
static unsigned int semanticsPrecision(const fltSemantics &)
Definition: APFloat.cpp:154
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
MachinePointerInfo getWithOffset(int64_t O) const
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:538
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:940
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT...
Definition: ValueTypes.h:72
EVT getMemoryVT() const
Return the type of the in-memory value.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
Class for arbitrary precision integers.
Definition: APInt.h:69
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:419
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:240
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:467
const SDValue & getBasePtr() const
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:470
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
Definition: APInt.h:529
Flags
Flags values. These may be or&#39;d together.
#define Success
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool ugt(const APInt &RHS) const
Unsigned greather than comparison.
Definition: APInt.h:1254
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:545
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:150
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
Definition: APInt.h:1320
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:386
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:205
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:485
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:615
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return &#39;Legal&#39;) or we ...
bool expandMUL_LOHI(unsigned Opcode, EVT VT, SDLoc dl, SDValue LHS, SDValue RHS, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL or [US]MUL_LOHI of n-bit values into two or four nodes, respectively, each computing an n/2-bit part of the result.
#define N
Flags getFlags() const
Return the raw flags of the source value,.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:322
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:272
Same for multiplication.
Definition: ISDOpcodes.h:256
const SDValue & getBasePtr() const
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
SDValue getValue(unsigned R) const
This class is used to represent an MSCATTER node.
unsigned getNumRegisters(LLVMContext &Context, EVT VT) const
Return the number of registers that this ValueType will eventually require.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getCondCode(ISD::CondCode Cond)
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input...
const MachinePointerInfo & getPointerInfo() const
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:344
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
This class is used to represent an MLOAD node.
ArrayRef< int > getMask() const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:250
This class is used to represent an MGATHER node.
SDValue getValueType(EVT)
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:80
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:780
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:442
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Definition: ValueTypes.h:222
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:197
Conversion operators.
Definition: ISDOpcodes.h:464
BooleanContent
Enum that describes how the target represents true/false values.
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:793
bool isBigEndian() const
Definition: DataLayout.h:221
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:473
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Op, int64_t Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object...
Definition: