LLVM  4.0.0
LegalizeDAG.cpp
Go to the documentation of this file.
1 //===-- LegalizeDAG.cpp - Implement SelectionDAG::Legalize ----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the SelectionDAG::Legalize method.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/ADT/SetVector.h"
15 #include "llvm/ADT/SmallPtrSet.h"
16 #include "llvm/ADT/SmallSet.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ADT/Triple.h"
23 #include "llvm/IR/CallingConv.h"
24 #include "llvm/IR/Constants.h"
25 #include "llvm/IR/DataLayout.h"
26 #include "llvm/IR/DebugInfo.h"
27 #include "llvm/IR/DerivedTypes.h"
28 #include "llvm/IR/Function.h"
29 #include "llvm/IR/LLVMContext.h"
30 #include "llvm/Support/Debug.h"
38 using namespace llvm;
39 
40 #define DEBUG_TYPE "legalizedag"
41 
42 namespace {
43 
44 struct FloatSignAsInt;
45 
46 //===----------------------------------------------------------------------===//
47 /// This takes an arbitrary SelectionDAG as input and
48 /// hacks on it until the target machine can handle it. This involves
49 /// eliminating value sizes the machine cannot handle (promoting small sizes to
50 /// large sizes or splitting up large values into small values) as well as
51 /// eliminating operations the machine cannot handle.
52 ///
53 /// This code also does a small amount of optimization and recognition of idioms
54 /// as part of its processing. For example, if a target does not support a
55 /// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
56 /// will attempt merge setcc and brc instructions into brcc's.
57 ///
58 class SelectionDAGLegalize {
59  const TargetMachine &TM;
60  const TargetLowering &TLI;
61  SelectionDAG &DAG;
62 
63  /// \brief The set of nodes which have already been legalized. We hold a
64  /// reference to it in order to update as necessary on node deletion.
65  SmallPtrSetImpl<SDNode *> &LegalizedNodes;
66 
67  /// \brief A set of all the nodes updated during legalization.
68  SmallSetVector<SDNode *, 16> *UpdatedNodes;
69 
70  EVT getSetCCResultType(EVT VT) const {
71  return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
72  }
73 
74  // Libcall insertion helpers.
75 
76 public:
77  SelectionDAGLegalize(SelectionDAG &DAG,
78  SmallPtrSetImpl<SDNode *> &LegalizedNodes,
79  SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
80  : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
81  LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
82 
83  /// \brief Legalizes the given operation.
84  void LegalizeOp(SDNode *Node);
85 
86 private:
87  SDValue OptimizeFloatStore(StoreSDNode *ST);
88 
89  void LegalizeLoadOps(SDNode *Node);
90  void LegalizeStoreOps(SDNode *Node);
91 
92  /// Some targets cannot handle a variable
93  /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
94  /// is necessary to spill the vector being inserted into to memory, perform
95  /// the insert there, and then read the result back.
96  SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
97  const SDLoc &dl);
98  SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
99  const SDLoc &dl);
100 
101  /// Return a vector shuffle operation which
102  /// performs the same shuffe in terms of order or result bytes, but on a type
103  /// whose vector element type is narrower than the original shuffle type.
104  /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
105  SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
106  SDValue N1, SDValue N2,
107  ArrayRef<int> Mask) const;
108 
109  bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC,
110  bool &NeedInvert, const SDLoc &dl);
111 
112  SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
113  SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops,
114  unsigned NumOps, bool isSigned, const SDLoc &dl);
115 
116  std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
117  SDNode *Node, bool isSigned);
118  SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
119  RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
120  RTLIB::Libcall Call_F128,
121  RTLIB::Libcall Call_PPCF128);
122  SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
123  RTLIB::Libcall Call_I8,
124  RTLIB::Libcall Call_I16,
125  RTLIB::Libcall Call_I32,
126  RTLIB::Libcall Call_I64,
127  RTLIB::Libcall Call_I128);
128  void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
129  void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
130 
131  SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
132  const SDLoc &dl);
133  SDValue ExpandBUILD_VECTOR(SDNode *Node);
134  SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
135  void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
136  SmallVectorImpl<SDValue> &Results);
137  void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
138  SDValue Value) const;
139  SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
140  SDValue NewIntValue) const;
141  SDValue ExpandFCOPYSIGN(SDNode *Node) const;
142  SDValue ExpandFABS(SDNode *Node) const;
143  SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue LegalOp, EVT DestVT,
144  const SDLoc &dl);
145  SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned,
146  const SDLoc &dl);
147  SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned,
148  const SDLoc &dl);
149 
150  SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl);
151  SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl);
152  SDValue ExpandBitCount(unsigned Opc, SDValue Op, const SDLoc &dl);
153 
154  SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
155  SDValue ExpandInsertToVectorThroughStack(SDValue Op);
156  SDValue ExpandVectorBuildThroughStack(SDNode* Node);
157 
158  SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
159  SDValue ExpandConstant(ConstantSDNode *CP);
160 
161  // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
162  bool ExpandNode(SDNode *Node);
163  void ConvertNodeToLibcall(SDNode *Node);
164  void PromoteNode(SDNode *Node);
165 
166 public:
167  // Node replacement helpers
168  void ReplacedNode(SDNode *N) {
169  LegalizedNodes.erase(N);
170  if (UpdatedNodes)
171  UpdatedNodes->insert(N);
172  }
173  void ReplaceNode(SDNode *Old, SDNode *New) {
174  DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
175  dbgs() << " with: "; New->dump(&DAG));
176 
177  assert(Old->getNumValues() == New->getNumValues() &&
178  "Replacing one node with another that produces a different number "
179  "of values!");
180  DAG.ReplaceAllUsesWith(Old, New);
181  if (UpdatedNodes)
182  UpdatedNodes->insert(New);
183  ReplacedNode(Old);
184  }
185  void ReplaceNode(SDValue Old, SDValue New) {
186  DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
187  dbgs() << " with: "; New->dump(&DAG));
188 
189  DAG.ReplaceAllUsesWith(Old, New);
190  if (UpdatedNodes)
191  UpdatedNodes->insert(New.getNode());
192  ReplacedNode(Old.getNode());
193  }
194  void ReplaceNode(SDNode *Old, const SDValue *New) {
195  DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
196 
197  DAG.ReplaceAllUsesWith(Old, New);
198  for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
199  DEBUG(dbgs() << (i == 0 ? " with: "
200  : " and: ");
201  New[i]->dump(&DAG));
202  if (UpdatedNodes)
203  UpdatedNodes->insert(New[i].getNode());
204  }
205  ReplacedNode(Old);
206  }
207 };
208 }
209 
210 /// Return a vector shuffle operation which
211 /// performs the same shuffe in terms of order or result bytes, but on a type
212 /// whose vector element type is narrower than the original shuffle type.
213 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
214 SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
215  EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
216  ArrayRef<int> Mask) const {
217  unsigned NumMaskElts = VT.getVectorNumElements();
218  unsigned NumDestElts = NVT.getVectorNumElements();
219  unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
220 
221  assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
222 
223  if (NumEltsGrowth == 1)
224  return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
225 
226  SmallVector<int, 8> NewMask;
227  for (unsigned i = 0; i != NumMaskElts; ++i) {
228  int Idx = Mask[i];
229  for (unsigned j = 0; j != NumEltsGrowth; ++j) {
230  if (Idx < 0)
231  NewMask.push_back(-1);
232  else
233  NewMask.push_back(Idx * NumEltsGrowth + j);
234  }
235  }
236  assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
237  assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
238  return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
239 }
240 
241 /// Expands the ConstantFP node to an integer constant or
242 /// a load from the constant pool.
243 SDValue
244 SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
245  bool Extend = false;
246  SDLoc dl(CFP);
247 
248  // If a FP immediate is precise when represented as a float and if the
249  // target can do an extending load from float to double, we put it into
250  // the constant pool as a float, even if it's is statically typed as a
251  // double. This shrinks FP constants and canonicalizes them for targets where
252  // an FP extending load is the same cost as a normal load (such as on the x87
253  // fp stack or PPC FP unit).
254  EVT VT = CFP->getValueType(0);
255  ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
256  if (!UseCP) {
257  assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
258  return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
259  (VT == MVT::f64) ? MVT::i64 : MVT::i32);
260  }
261 
262  APFloat APF = CFP->getValueAPF();
263  EVT OrigVT = VT;
264  EVT SVT = VT;
265 
266  // We don't want to shrink SNaNs. Converting the SNaN back to its real type
267  // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
268  if (!APF.isSignaling()) {
269  while (SVT != MVT::f32 && SVT != MVT::f16) {
270  SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
272  // Only do this if the target has a native EXTLOAD instruction from
273  // smaller type.
274  TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
275  TLI.ShouldShrinkFPConstant(OrigVT)) {
276  Type *SType = SVT.getTypeForEVT(*DAG.getContext());
277  LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
278  VT = SVT;
279  Extend = true;
280  }
281  }
282  }
283 
284  SDValue CPIdx =
285  DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
286  unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
287  if (Extend) {
288  SDValue Result = DAG.getExtLoad(
289  ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
290  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
291  Alignment);
292  return Result;
293  }
294  SDValue Result = DAG.getLoad(
295  OrigVT, dl, DAG.getEntryNode(), CPIdx,
296  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
297  return Result;
298 }
299 
300 /// Expands the Constant node to a load from the constant pool.
301 SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
302  SDLoc dl(CP);
303  EVT VT = CP->getValueType(0);
304  SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
305  TLI.getPointerTy(DAG.getDataLayout()));
306  unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
307  SDValue Result = DAG.getLoad(
308  VT, dl, DAG.getEntryNode(), CPIdx,
309  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
310  return Result;
311 }
312 
313 /// Some target cannot handle a variable insertion index for the
314 /// INSERT_VECTOR_ELT instruction. In this case, it
315 /// is necessary to spill the vector being inserted into to memory, perform
316 /// the insert there, and then read the result back.
317 SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
318  SDValue Val,
319  SDValue Idx,
320  const SDLoc &dl) {
321  SDValue Tmp1 = Vec;
322  SDValue Tmp2 = Val;
323  SDValue Tmp3 = Idx;
324 
325  // If the target doesn't support this, we have to spill the input vector
326  // to a temporary stack slot, update the element, then reload it. This is
327  // badness. We could also load the value into a vector register (either
328  // with a "move to register" or "extload into register" instruction, then
329  // permute it into place, if the idx is a constant and if the idx is
330  // supported by the target.
331  EVT VT = Tmp1.getValueType();
332  EVT EltVT = VT.getVectorElementType();
333  SDValue StackPtr = DAG.CreateStackTemporary(VT);
334 
335  int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
336 
337  // Store the vector.
338  SDValue Ch = DAG.getStore(
339  DAG.getEntryNode(), dl, Tmp1, StackPtr,
340  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
341 
342  SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
343 
344  // Store the scalar value.
345  Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT);
346  // Load the updated vector.
347  return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
348  DAG.getMachineFunction(), SPFI));
349 }
350 
351 SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
352  SDValue Idx,
353  const SDLoc &dl) {
354  if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
355  // SCALAR_TO_VECTOR requires that the type of the value being inserted
356  // match the element type of the vector being created, except for
357  // integers in which case the inserted value can be over width.
358  EVT EltVT = Vec.getValueType().getVectorElementType();
359  if (Val.getValueType() == EltVT ||
360  (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
361  SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
362  Vec.getValueType(), Val);
363 
364  unsigned NumElts = Vec.getValueType().getVectorNumElements();
365  // We generate a shuffle of InVec and ScVec, so the shuffle mask
366  // should be 0,1,2,3,4,5... with the appropriate element replaced with
367  // elt 0 of the RHS.
368  SmallVector<int, 8> ShufOps;
369  for (unsigned i = 0; i != NumElts; ++i)
370  ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
371 
372  return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
373  }
374  }
375  return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
376 }
377 
378 SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
379  // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
380  // FIXME: We shouldn't do this for TargetConstantFP's.
381  // FIXME: move this to the DAG Combiner! Note that we can't regress due
382  // to phase ordering between legalized code and the dag combiner. This
383  // probably means that we need to integrate dag combiner and legalizer
384  // together.
385  // We generally can't do this one for long doubles.
386  SDValue Chain = ST->getChain();
387  SDValue Ptr = ST->getBasePtr();
388  unsigned Alignment = ST->getAlignment();
389  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
390  AAMDNodes AAInfo = ST->getAAInfo();
391  SDLoc dl(ST);
392  if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) {
393  if (CFP->getValueType(0) == MVT::f32 &&
394  TLI.isTypeLegal(MVT::i32)) {
395  SDValue Con = DAG.getConstant(CFP->getValueAPF().
396  bitcastToAPInt().zextOrTrunc(32),
397  SDLoc(CFP), MVT::i32);
398  return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), Alignment,
399  MMOFlags, AAInfo);
400  }
401 
402  if (CFP->getValueType(0) == MVT::f64) {
403  // If this target supports 64-bit registers, do a single 64-bit store.
404  if (TLI.isTypeLegal(MVT::i64)) {
405  SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
406  zextOrTrunc(64), SDLoc(CFP), MVT::i64);
407  return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
408  Alignment, MMOFlags, AAInfo);
409  }
410 
411  if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
412  // Otherwise, if the target supports 32-bit registers, use 2 32-bit
413  // stores. If the target supports neither 32- nor 64-bits, this
414  // xform is certainly not worth it.
415  const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
416  SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
417  SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
418  if (DAG.getDataLayout().isBigEndian())
419  std::swap(Lo, Hi);
420 
421  Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), Alignment,
422  MMOFlags, AAInfo);
423  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
424  DAG.getConstant(4, dl, Ptr.getValueType()));
425  Hi = DAG.getStore(Chain, dl, Hi, Ptr,
426  ST->getPointerInfo().getWithOffset(4),
427  MinAlign(Alignment, 4U), MMOFlags, AAInfo);
428 
429  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
430  }
431  }
432  }
433  return SDValue(nullptr, 0);
434 }
435 
436 void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
437  StoreSDNode *ST = cast<StoreSDNode>(Node);
438  SDValue Chain = ST->getChain();
439  SDValue Ptr = ST->getBasePtr();
440  SDLoc dl(Node);
441 
442  unsigned Alignment = ST->getAlignment();
443  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
444  AAMDNodes AAInfo = ST->getAAInfo();
445 
446  if (!ST->isTruncatingStore()) {
447  if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
448  ReplaceNode(ST, OptStore);
449  return;
450  }
451 
452  {
453  SDValue Value = ST->getValue();
454  MVT VT = Value.getSimpleValueType();
455  switch (TLI.getOperationAction(ISD::STORE, VT)) {
456  default: llvm_unreachable("This action is not supported yet!");
457  case TargetLowering::Legal: {
458  // If this is an unaligned store and the target doesn't support it,
459  // expand it.
460  EVT MemVT = ST->getMemoryVT();
461  unsigned AS = ST->getAddressSpace();
462  unsigned Align = ST->getAlignment();
463  const DataLayout &DL = DAG.getDataLayout();
464  if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
465  SDValue Result = TLI.expandUnalignedStore(ST, DAG);
466  ReplaceNode(SDValue(ST, 0), Result);
467  }
468  break;
469  }
470  case TargetLowering::Custom: {
471  SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
472  if (Res && Res != SDValue(Node, 0))
473  ReplaceNode(SDValue(Node, 0), Res);
474  return;
475  }
477  MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
478  assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
479  "Can only promote stores to same size type");
480  Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
481  SDValue Result =
482  DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
483  Alignment, MMOFlags, AAInfo);
484  ReplaceNode(SDValue(Node, 0), Result);
485  break;
486  }
487  }
488  return;
489  }
490  } else {
491  SDValue Value = ST->getValue();
492 
493  EVT StVT = ST->getMemoryVT();
494  unsigned StWidth = StVT.getSizeInBits();
495  auto &DL = DAG.getDataLayout();
496 
497  if (StWidth != StVT.getStoreSizeInBits()) {
498  // Promote to a byte-sized store with upper bits zero if not
499  // storing an integral number of bytes. For example, promote
500  // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
501  EVT NVT = EVT::getIntegerVT(*DAG.getContext(),
502  StVT.getStoreSizeInBits());
503  Value = DAG.getZeroExtendInReg(Value, dl, StVT);
504  SDValue Result =
505  DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
506  Alignment, MMOFlags, AAInfo);
507  ReplaceNode(SDValue(Node, 0), Result);
508  } else if (StWidth & (StWidth - 1)) {
509  // If not storing a power-of-2 number of bits, expand as two stores.
510  assert(!StVT.isVector() && "Unsupported truncstore!");
511  unsigned RoundWidth = 1 << Log2_32(StWidth);
512  assert(RoundWidth < StWidth);
513  unsigned ExtraWidth = StWidth - RoundWidth;
514  assert(ExtraWidth < RoundWidth);
515  assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
516  "Store size not an integral number of bytes!");
517  EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
518  EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
519  SDValue Lo, Hi;
520  unsigned IncrementSize;
521 
522  if (DL.isLittleEndian()) {
523  // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
524  // Store the bottom RoundWidth bits.
525  Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
526  RoundVT, Alignment, MMOFlags, AAInfo);
527 
528  // Store the remaining ExtraWidth bits.
529  IncrementSize = RoundWidth / 8;
530  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
531  DAG.getConstant(IncrementSize, dl,
532  Ptr.getValueType()));
533  Hi = DAG.getNode(
534  ISD::SRL, dl, Value.getValueType(), Value,
535  DAG.getConstant(RoundWidth, dl,
536  TLI.getShiftAmountTy(Value.getValueType(), DL)));
537  Hi = DAG.getTruncStore(
538  Chain, dl, Hi, Ptr,
539  ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
540  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
541  } else {
542  // Big endian - avoid unaligned stores.
543  // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
544  // Store the top RoundWidth bits.
545  Hi = DAG.getNode(
546  ISD::SRL, dl, Value.getValueType(), Value,
547  DAG.getConstant(ExtraWidth, dl,
548  TLI.getShiftAmountTy(Value.getValueType(), DL)));
549  Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(),
550  RoundVT, Alignment, MMOFlags, AAInfo);
551 
552  // Store the remaining ExtraWidth bits.
553  IncrementSize = RoundWidth / 8;
554  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
555  DAG.getConstant(IncrementSize, dl,
556  Ptr.getValueType()));
557  Lo = DAG.getTruncStore(
558  Chain, dl, Value, Ptr,
559  ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
560  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
561  }
562 
563  // The order of the stores doesn't matter.
564  SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
565  ReplaceNode(SDValue(Node, 0), Result);
566  } else {
567  switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
568  default: llvm_unreachable("This action is not supported yet!");
569  case TargetLowering::Legal: {
570  EVT MemVT = ST->getMemoryVT();
571  unsigned AS = ST->getAddressSpace();
572  unsigned Align = ST->getAlignment();
573  // If this is an unaligned store and the target doesn't support it,
574  // expand it.
575  if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
576  SDValue Result = TLI.expandUnalignedStore(ST, DAG);
577  ReplaceNode(SDValue(ST, 0), Result);
578  }
579  break;
580  }
581  case TargetLowering::Custom: {
582  SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
583  if (Res && Res != SDValue(Node, 0))
584  ReplaceNode(SDValue(Node, 0), Res);
585  return;
586  }
588  assert(!StVT.isVector() &&
589  "Vector Stores are handled in LegalizeVectorOps");
590 
591  // TRUNCSTORE:i16 i32 -> STORE i16
592  assert(TLI.isTypeLegal(StVT) &&
593  "Do not know how to expand this store!");
594  Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
595  SDValue Result =
596  DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
597  Alignment, MMOFlags, AAInfo);
598  ReplaceNode(SDValue(Node, 0), Result);
599  break;
600  }
601  }
602  }
603 }
604 
605 void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
606  LoadSDNode *LD = cast<LoadSDNode>(Node);
607  SDValue Chain = LD->getChain(); // The chain.
608  SDValue Ptr = LD->getBasePtr(); // The base pointer.
609  SDValue Value; // The value returned by the load op.
610  SDLoc dl(Node);
611 
612  ISD::LoadExtType ExtType = LD->getExtensionType();
613  if (ExtType == ISD::NON_EXTLOAD) {
614  MVT VT = Node->getSimpleValueType(0);
615  SDValue RVal = SDValue(Node, 0);
616  SDValue RChain = SDValue(Node, 1);
617 
618  switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
619  default: llvm_unreachable("This action is not supported yet!");
620  case TargetLowering::Legal: {
621  EVT MemVT = LD->getMemoryVT();
622  unsigned AS = LD->getAddressSpace();
623  unsigned Align = LD->getAlignment();
624  const DataLayout &DL = DAG.getDataLayout();
625  // If this is an unaligned load and the target doesn't support it,
626  // expand it.
627  if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
628  std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
629  }
630  break;
631  }
632  case TargetLowering::Custom: {
633  if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
634  RVal = Res;
635  RChain = Res.getValue(1);
636  }
637  break;
638  }
640  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
641  assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
642  "Can only promote loads to same size type");
643 
644  SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
645  RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
646  RChain = Res.getValue(1);
647  break;
648  }
649  }
650  if (RChain.getNode() != Node) {
651  assert(RVal.getNode() != Node && "Load must be completely replaced");
652  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
653  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
654  if (UpdatedNodes) {
655  UpdatedNodes->insert(RVal.getNode());
656  UpdatedNodes->insert(RChain.getNode());
657  }
658  ReplacedNode(Node);
659  }
660  return;
661  }
662 
663  EVT SrcVT = LD->getMemoryVT();
664  unsigned SrcWidth = SrcVT.getSizeInBits();
665  unsigned Alignment = LD->getAlignment();
666  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
667  AAMDNodes AAInfo = LD->getAAInfo();
668 
669  if (SrcWidth != SrcVT.getStoreSizeInBits() &&
670  // Some targets pretend to have an i1 loading operation, and actually
671  // load an i8. This trick is correct for ZEXTLOAD because the top 7
672  // bits are guaranteed to be zero; it helps the optimizers understand
673  // that these bits are zero. It is also useful for EXTLOAD, since it
674  // tells the optimizers that those bits are undefined. It would be
675  // nice to have an effective generic way of getting these benefits...
676  // Until such a way is found, don't insist on promoting i1 here.
677  (SrcVT != MVT::i1 ||
678  TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
680  // Promote to a byte-sized load if not loading an integral number of
681  // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
682  unsigned NewWidth = SrcVT.getStoreSizeInBits();
683  EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
684  SDValue Ch;
685 
686  // The extra bits are guaranteed to be zero, since we stored them that
687  // way. A zext load from NVT thus automatically gives zext from SrcVT.
688 
689  ISD::LoadExtType NewExtType =
691 
692  SDValue Result =
693  DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), Chain, Ptr,
694  LD->getPointerInfo(), NVT, Alignment, MMOFlags, AAInfo);
695 
696  Ch = Result.getValue(1); // The chain.
697 
698  if (ExtType == ISD::SEXTLOAD)
699  // Having the top bits zero doesn't help when sign extending.
700  Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
701  Result.getValueType(),
702  Result, DAG.getValueType(SrcVT));
703  else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
704  // All the top bits are guaranteed to be zero - inform the optimizers.
705  Result = DAG.getNode(ISD::AssertZext, dl,
706  Result.getValueType(), Result,
707  DAG.getValueType(SrcVT));
708 
709  Value = Result;
710  Chain = Ch;
711  } else if (SrcWidth & (SrcWidth - 1)) {
712  // If not loading a power-of-2 number of bits, expand as two loads.
713  assert(!SrcVT.isVector() && "Unsupported extload!");
714  unsigned RoundWidth = 1 << Log2_32(SrcWidth);
715  assert(RoundWidth < SrcWidth);
716  unsigned ExtraWidth = SrcWidth - RoundWidth;
717  assert(ExtraWidth < RoundWidth);
718  assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
719  "Load size not an integral number of bytes!");
720  EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
721  EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
722  SDValue Lo, Hi, Ch;
723  unsigned IncrementSize;
724  auto &DL = DAG.getDataLayout();
725 
726  if (DL.isLittleEndian()) {
727  // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
728  // Load the bottom RoundWidth bits.
729  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
730  LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
731  AAInfo);
732 
733  // Load the remaining ExtraWidth bits.
734  IncrementSize = RoundWidth / 8;
735  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
736  DAG.getConstant(IncrementSize, dl,
737  Ptr.getValueType()));
738  Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
739  LD->getPointerInfo().getWithOffset(IncrementSize),
740  ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
741  AAInfo);
742 
743  // Build a factor node to remember that this load is independent of
744  // the other one.
745  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
746  Hi.getValue(1));
747 
748  // Move the top bits to the right place.
749  Hi = DAG.getNode(
750  ISD::SHL, dl, Hi.getValueType(), Hi,
751  DAG.getConstant(RoundWidth, dl,
752  TLI.getShiftAmountTy(Hi.getValueType(), DL)));
753 
754  // Join the hi and lo parts.
755  Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
756  } else {
757  // Big endian - avoid unaligned loads.
758  // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
759  // Load the top RoundWidth bits.
760  Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
761  LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
762  AAInfo);
763 
764  // Load the remaining ExtraWidth bits.
765  IncrementSize = RoundWidth / 8;
766  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
767  DAG.getConstant(IncrementSize, dl,
768  Ptr.getValueType()));
769  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
770  LD->getPointerInfo().getWithOffset(IncrementSize),
771  ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
772  AAInfo);
773 
774  // Build a factor node to remember that this load is independent of
775  // the other one.
776  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
777  Hi.getValue(1));
778 
779  // Move the top bits to the right place.
780  Hi = DAG.getNode(
781  ISD::SHL, dl, Hi.getValueType(), Hi,
782  DAG.getConstant(ExtraWidth, dl,
783  TLI.getShiftAmountTy(Hi.getValueType(), DL)));
784 
785  // Join the hi and lo parts.
786  Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
787  }
788 
789  Chain = Ch;
790  } else {
791  bool isCustom = false;
792  switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
793  SrcVT.getSimpleVT())) {
794  default: llvm_unreachable("This action is not supported yet!");
796  isCustom = true;
798  case TargetLowering::Legal: {
799  Value = SDValue(Node, 0);
800  Chain = SDValue(Node, 1);
801 
802  if (isCustom) {
803  if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
804  Value = Res;
805  Chain = Res.getValue(1);
806  }
807  } else {
808  // If this is an unaligned load and the target doesn't support it,
809  // expand it.
810  EVT MemVT = LD->getMemoryVT();
811  unsigned AS = LD->getAddressSpace();
812  unsigned Align = LD->getAlignment();
813  const DataLayout &DL = DAG.getDataLayout();
814  if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
815  std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
816  }
817  }
818  break;
819  }
821  EVT DestVT = Node->getValueType(0);
822  if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
823  // If the source type is not legal, see if there is a legal extload to
824  // an intermediate type that we can then extend further.
825  EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
826  if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
827  TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) {
828  // If we are loading a legal type, this is a non-extload followed by a
829  // full extend.
830  ISD::LoadExtType MidExtType =
831  (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
832 
833  SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
834  SrcVT, LD->getMemOperand());
835  unsigned ExtendOp =
836  ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
837  Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
838  Chain = Load.getValue(1);
839  break;
840  }
841 
842  // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
843  // normal undefined upper bits behavior to allow using an in-reg extend
844  // with the illegal FP type, so load as an integer and do the
845  // from-integer conversion.
846  if (SrcVT.getScalarType() == MVT::f16) {
847  EVT ISrcVT = SrcVT.changeTypeToInteger();
848  EVT IDestVT = DestVT.changeTypeToInteger();
849  EVT LoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
850 
851  SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, LoadVT,
852  Chain, Ptr, ISrcVT,
853  LD->getMemOperand());
854  Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result);
855  Chain = Result.getValue(1);
856  break;
857  }
858  }
859 
860  assert(!SrcVT.isVector() &&
861  "Vector Loads are handled in LegalizeVectorOps");
862 
863  // FIXME: This does not work for vectors on most targets. Sign-
864  // and zero-extend operations are currently folded into extending
865  // loads, whether they are legal or not, and then we end up here
866  // without any support for legalizing them.
867  assert(ExtType != ISD::EXTLOAD &&
868  "EXTLOAD should always be supported!");
869  // Turn the unsupported load into an EXTLOAD followed by an
870  // explicit zero/sign extend inreg.
871  SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
872  Node->getValueType(0),
873  Chain, Ptr, SrcVT,
874  LD->getMemOperand());
875  SDValue ValRes;
876  if (ExtType == ISD::SEXTLOAD)
877  ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
878  Result.getValueType(),
879  Result, DAG.getValueType(SrcVT));
880  else
881  ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType());
882  Value = ValRes;
883  Chain = Result.getValue(1);
884  break;
885  }
886  }
887 
888  // Since loads produce two values, make sure to remember that we legalized
889  // both of them.
890  if (Chain.getNode() != Node) {
891  assert(Value.getNode() != Node && "Load must be completely replaced");
892  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
893  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
894  if (UpdatedNodes) {
895  UpdatedNodes->insert(Value.getNode());
896  UpdatedNodes->insert(Chain.getNode());
897  }
898  ReplacedNode(Node);
899  }
900 }
901 
902 /// Return a legal replacement for the given operation, with all legal operands.
903 void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
904  DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
905 
906  if (Node->getOpcode() == ISD::TargetConstant) // Allow illegal target nodes.
907  return;
908 
909 #ifndef NDEBUG
910  for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
911  assert((TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
913  TLI.isTypeLegal(Node->getValueType(i))) &&
914  "Unexpected illegal type!");
915 
916  for (const SDValue &Op : Node->op_values())
917  assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
919  TLI.isTypeLegal(Op.getValueType()) ||
920  Op.getOpcode() == ISD::TargetConstant) &&
921  "Unexpected illegal type!");
922 #endif
923 
924  // Figure out the correct action; the way to query this varies by opcode
926  bool SimpleFinishLegalizing = true;
927  switch (Node->getOpcode()) {
930  case ISD::INTRINSIC_VOID:
931  case ISD::STACKSAVE:
932  Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
933  break;
935  Action = TLI.getOperationAction(Node->getOpcode(),
936  Node->getValueType(0));
937  break;
938  case ISD::VAARG:
939  Action = TLI.getOperationAction(Node->getOpcode(),
940  Node->getValueType(0));
941  if (Action != TargetLowering::Promote)
942  Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
943  break;
944  case ISD::FP_TO_FP16:
945  case ISD::SINT_TO_FP:
946  case ISD::UINT_TO_FP:
948  Action = TLI.getOperationAction(Node->getOpcode(),
949  Node->getOperand(0).getValueType());
950  break;
951  case ISD::FP_ROUND_INREG:
952  case ISD::SIGN_EXTEND_INREG: {
953  EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
954  Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
955  break;
956  }
957  case ISD::ATOMIC_STORE: {
958  Action = TLI.getOperationAction(Node->getOpcode(),
959  Node->getOperand(2).getValueType());
960  break;
961  }
962  case ISD::SELECT_CC:
963  case ISD::SETCC:
964  case ISD::BR_CC: {
965  unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
966  Node->getOpcode() == ISD::SETCC ? 2 :
967  Node->getOpcode() == ISD::SETCCE ? 3 : 1;
968  unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0;
969  MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
970  ISD::CondCode CCCode =
971  cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
972  Action = TLI.getCondCodeAction(CCCode, OpVT);
973  if (Action == TargetLowering::Legal) {
974  if (Node->getOpcode() == ISD::SELECT_CC)
975  Action = TLI.getOperationAction(Node->getOpcode(),
976  Node->getValueType(0));
977  else
978  Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
979  }
980  break;
981  }
982  case ISD::LOAD:
983  case ISD::STORE:
984  // FIXME: Model these properly. LOAD and STORE are complicated, and
985  // STORE expects the unlegalized operand in some cases.
986  SimpleFinishLegalizing = false;
987  break;
988  case ISD::CALLSEQ_START:
989  case ISD::CALLSEQ_END:
990  // FIXME: This shouldn't be necessary. These nodes have special properties
991  // dealing with the recursive nature of legalization. Removing this
992  // special case should be done as part of making LegalizeDAG non-recursive.
993  SimpleFinishLegalizing = false;
994  break;
996  case ISD::FLT_ROUNDS_:
997  case ISD::FPOWI:
998  case ISD::MERGE_VALUES:
999  case ISD::EH_RETURN:
1001  case ISD::EH_DWARF_CFA:
1002  case ISD::EH_SJLJ_SETJMP:
1003  case ISD::EH_SJLJ_LONGJMP:
1005  // These operations lie about being legal: when they claim to be legal,
1006  // they should actually be expanded.
1007  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1008  if (Action == TargetLowering::Legal)
1009  Action = TargetLowering::Expand;
1010  break;
1011  case ISD::INIT_TRAMPOLINE:
1013  case ISD::FRAMEADDR:
1014  case ISD::RETURNADDR:
1015  case ISD::ADDROFRETURNADDR:
1016  // These operations lie about being legal: when they claim to be legal,
1017  // they should actually be custom-lowered.
1018  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1019  if (Action == TargetLowering::Legal)
1020  Action = TargetLowering::Custom;
1021  break;
1022  case ISD::READCYCLECOUNTER:
1023  // READCYCLECOUNTER returns an i64, even if type legalization might have
1024  // expanded that to several smaller types.
1025  Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1026  break;
1027  case ISD::READ_REGISTER:
1028  case ISD::WRITE_REGISTER:
1029  // Named register is legal in the DAG, but blocked by register name
1030  // selection if not implemented by target (to chose the correct register)
1031  // They'll be converted to Copy(To/From)Reg.
1032  Action = TargetLowering::Legal;
1033  break;
1034  case ISD::DEBUGTRAP:
1035  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1036  if (Action == TargetLowering::Expand) {
1037  // replace ISD::DEBUGTRAP with ISD::TRAP
1038  SDValue NewVal;
1039  NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1040  Node->getOperand(0));
1041  ReplaceNode(Node, NewVal.getNode());
1042  LegalizeOp(NewVal.getNode());
1043  return;
1044  }
1045  break;
1046 
1047  default:
1048  if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1049  Action = TargetLowering::Legal;
1050  } else {
1051  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1052  }
1053  break;
1054  }
1055 
1056  if (SimpleFinishLegalizing) {
1057  SDNode *NewNode = Node;
1058  switch (Node->getOpcode()) {
1059  default: break;
1060  case ISD::SHL:
1061  case ISD::SRL:
1062  case ISD::SRA:
1063  case ISD::ROTL:
1064  case ISD::ROTR: {
1065  // Legalizing shifts/rotates requires adjusting the shift amount
1066  // to the appropriate width.
1067  SDValue Op0 = Node->getOperand(0);
1068  SDValue Op1 = Node->getOperand(1);
1069  if (!Op1.getValueType().isVector()) {
1070  SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1071  // The getShiftAmountOperand() may create a new operand node or
1072  // return the existing one. If new operand is created we need
1073  // to update the parent node.
1074  // Do not try to legalize SAO here! It will be automatically legalized
1075  // in the next round.
1076  if (SAO != Op1)
1077  NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1078  }
1079  }
1080  break;
1081  case ISD::SRL_PARTS:
1082  case ISD::SRA_PARTS:
1083  case ISD::SHL_PARTS: {
1084  // Legalizing shifts/rotates requires adjusting the shift amount
1085  // to the appropriate width.
1086  SDValue Op0 = Node->getOperand(0);
1087  SDValue Op1 = Node->getOperand(1);
1088  SDValue Op2 = Node->getOperand(2);
1089  if (!Op2.getValueType().isVector()) {
1090  SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1091  // The getShiftAmountOperand() may create a new operand node or
1092  // return the existing one. If new operand is created we need
1093  // to update the parent node.
1094  if (SAO != Op2)
1095  NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1096  }
1097  }
1098  break;
1099  }
1100 
1101  if (NewNode != Node) {
1102  ReplaceNode(Node, NewNode);
1103  Node = NewNode;
1104  }
1105  switch (Action) {
1106  case TargetLowering::Legal:
1107  return;
1108  case TargetLowering::Custom: {
1109  // FIXME: The handling for custom lowering with multiple results is
1110  // a complete mess.
1111  if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1112  if (!(Res.getNode() != Node || Res.getResNo() != 0))
1113  return;
1114 
1115  if (Node->getNumValues() == 1) {
1116  // We can just directly replace this node with the lowered value.
1117  ReplaceNode(SDValue(Node, 0), Res);
1118  return;
1119  }
1120 
1121  SmallVector<SDValue, 8> ResultVals;
1122  for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
1123  ResultVals.push_back(Res.getValue(i));
1124  ReplaceNode(Node, ResultVals.data());
1125  return;
1126  }
1128  }
1130  if (ExpandNode(Node))
1131  return;
1134  ConvertNodeToLibcall(Node);
1135  return;
1137  PromoteNode(Node);
1138  return;
1139  }
1140  }
1141 
1142  switch (Node->getOpcode()) {
1143  default:
1144 #ifndef NDEBUG
1145  dbgs() << "NODE: ";
1146  Node->dump( &DAG);
1147  dbgs() << "\n";
1148 #endif
1149  llvm_unreachable("Do not know how to legalize this operator!");
1150 
1151  case ISD::CALLSEQ_START:
1152  case ISD::CALLSEQ_END:
1153  break;
1154  case ISD::LOAD: {
1155  return LegalizeLoadOps(Node);
1156  }
1157  case ISD::STORE: {
1158  return LegalizeStoreOps(Node);
1159  }
1160  }
1161 }
1162 
1163 SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1164  SDValue Vec = Op.getOperand(0);
1165  SDValue Idx = Op.getOperand(1);
1166  SDLoc dl(Op);
1167 
1168  // Before we generate a new store to a temporary stack slot, see if there is
1169  // already one that we can use. There often is because when we scalarize
1170  // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1171  // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1172  // the vector. If all are expanded here, we don't want one store per vector
1173  // element.
1174 
1175  // Caches for hasPredecessorHelper
1178  Worklist.push_back(Idx.getNode());
1179  SDValue StackPtr, Ch;
1180  for (SDNode::use_iterator UI = Vec.getNode()->use_begin(),
1181  UE = Vec.getNode()->use_end(); UI != UE; ++UI) {
1182  SDNode *User = *UI;
1183  if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1184  if (ST->isIndexed() || ST->isTruncatingStore() ||
1185  ST->getValue() != Vec)
1186  continue;
1187 
1188  // Make sure that nothing else could have stored into the destination of
1189  // this store.
1190  if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1191  continue;
1192 
1193  // If the index is dependent on the store we will introduce a cycle when
1194  // creating the load (the load uses the index, and by replacing the chain
1195  // we will make the index dependent on the load).
1196  if (SDNode::hasPredecessorHelper(ST, Visited, Worklist))
1197  continue;
1198 
1199  StackPtr = ST->getBasePtr();
1200  Ch = SDValue(ST, 0);
1201  break;
1202  }
1203  }
1204 
1205  EVT VecVT = Vec.getValueType();
1206 
1207  if (!Ch.getNode()) {
1208  // Store the value to a temporary stack slot, then LOAD the returned part.
1209  StackPtr = DAG.CreateStackTemporary(VecVT);
1210  Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1211  MachinePointerInfo());
1212  }
1213 
1214  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1215 
1216  SDValue NewLoad;
1217 
1218  if (Op.getValueType().isVector())
1219  NewLoad =
1220  DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo());
1221  else
1222  NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1224  VecVT.getVectorElementType());
1225 
1226  // Replace the chain going out of the store, by the one out of the load.
1227  DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1228 
1229  // We introduced a cycle though, so update the loads operands, making sure
1230  // to use the original store's chain as an incoming chain.
1231  SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1232  NewLoad->op_end());
1233  NewLoadOperands[0] = Ch;
1234  NewLoad =
1235  SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1236  return NewLoad;
1237 }
1238 
1239 SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1240  assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1241 
1242  SDValue Vec = Op.getOperand(0);
1243  SDValue Part = Op.getOperand(1);
1244  SDValue Idx = Op.getOperand(2);
1245  SDLoc dl(Op);
1246 
1247  // Store the value to a temporary stack slot, then LOAD the returned part.
1248  EVT VecVT = Vec.getValueType();
1249  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1250  int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1251  MachinePointerInfo PtrInfo =
1252  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1253 
1254  // First store the whole vector.
1255  SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1256 
1257  // Then store the inserted part.
1258  SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1259 
1260  // Store the subvector.
1261  Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, MachinePointerInfo());
1262 
1263  // Finally, load the updated vector.
1264  return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1265 }
1266 
1267 SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1268  // We can't handle this case efficiently. Allocate a sufficiently
1269  // aligned object on the stack, store each element into it, then load
1270  // the result as a vector.
1271  // Create the stack frame object.
1272  EVT VT = Node->getValueType(0);
1273  EVT EltVT = VT.getVectorElementType();
1274  SDLoc dl(Node);
1275  SDValue FIPtr = DAG.CreateStackTemporary(VT);
1276  int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1277  MachinePointerInfo PtrInfo =
1278  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1279 
1280  // Emit a store of each element to the stack slot.
1281  SmallVector<SDValue, 8> Stores;
1282  unsigned TypeByteSize = EltVT.getSizeInBits() / 8;
1283  // Store (in the right endianness) the elements to memory.
1284  for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1285  // Ignore undef elements.
1286  if (Node->getOperand(i).isUndef()) continue;
1287 
1288  unsigned Offset = TypeByteSize*i;
1289 
1290  SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType());
1291  Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx);
1292 
1293  // If the destination vector element type is narrower than the source
1294  // element type, only store the bits necessary.
1295  if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) {
1296  Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1297  Node->getOperand(i), Idx,
1298  PtrInfo.getWithOffset(Offset), EltVT));
1299  } else
1300  Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1301  Idx, PtrInfo.getWithOffset(Offset)));
1302  }
1303 
1304  SDValue StoreChain;
1305  if (!Stores.empty()) // Not all undef elements?
1306  StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1307  else
1308  StoreChain = DAG.getEntryNode();
1309 
1310  // Result is a load from the stack slot.
1311  return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1312 }
1313 
1314 namespace {
1315 /// Keeps track of state when getting the sign of a floating-point value as an
1316 /// integer.
1317 struct FloatSignAsInt {
1318  EVT FloatVT;
1319  SDValue Chain;
1320  SDValue FloatPtr;
1321  SDValue IntPtr;
1322  MachinePointerInfo IntPointerInfo;
1323  MachinePointerInfo FloatPointerInfo;
1324  SDValue IntValue;
1325  APInt SignMask;
1326  uint8_t SignBit;
1327 };
1328 }
1329 
1330 /// Bitcast a floating-point value to an integer value. Only bitcast the part
1331 /// containing the sign bit if the target has no integer value capable of
1332 /// holding all bits of the floating-point value.
1333 void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1334  const SDLoc &DL,
1335  SDValue Value) const {
1336  EVT FloatVT = Value.getValueType();
1337  unsigned NumBits = FloatVT.getSizeInBits();
1338  State.FloatVT = FloatVT;
1339  EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1340  // Convert to an integer of the same size.
1341  if (TLI.isTypeLegal(IVT)) {
1342  State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1343  State.SignMask = APInt::getSignBit(NumBits);
1344  State.SignBit = NumBits - 1;
1345  return;
1346  }
1347 
1348  auto &DataLayout = DAG.getDataLayout();
1349  // Store the float to memory, then load the sign part out as an integer.
1350  MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8);
1351  // First create a temporary that is aligned for both the load and store.
1352  SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1353  int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1354  // Then store the float to it.
1355  State.FloatPtr = StackPtr;
1356  MachineFunction &MF = DAG.getMachineFunction();
1357  State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1358  State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1359  State.FloatPointerInfo);
1360 
1361  SDValue IntPtr;
1362  if (DataLayout.isBigEndian()) {
1363  assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1364  // Load out a legal integer with the same sign bit as the float.
1365  IntPtr = StackPtr;
1366  State.IntPointerInfo = State.FloatPointerInfo;
1367  } else {
1368  // Advance the pointer so that the loaded byte will contain the sign bit.
1369  unsigned ByteOffset = (FloatVT.getSizeInBits() / 8) - 1;
1370  IntPtr = DAG.getNode(ISD::ADD, DL, StackPtr.getValueType(), StackPtr,
1371  DAG.getConstant(ByteOffset, DL, StackPtr.getValueType()));
1372  State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1373  ByteOffset);
1374  }
1375 
1376  State.IntPtr = IntPtr;
1377  State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1378  State.IntPointerInfo, MVT::i8);
1379  State.SignMask = APInt::getOneBitSet(LoadTy.getSizeInBits(), 7);
1380  State.SignBit = 7;
1381 }
1382 
1383 /// Replace the integer value produced by getSignAsIntValue() with a new value
1384 /// and cast the result back to a floating-point type.
1385 SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1386  const SDLoc &DL,
1387  SDValue NewIntValue) const {
1388  if (!State.Chain)
1389  return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1390 
1391  // Override the part containing the sign bit in the value stored on the stack.
1392  SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1393  State.IntPointerInfo, MVT::i8);
1394  return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1395  State.FloatPointerInfo);
1396 }
1397 
1398 SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1399  SDLoc DL(Node);
1400  SDValue Mag = Node->getOperand(0);
1401  SDValue Sign = Node->getOperand(1);
1402 
1403  // Get sign bit into an integer value.
1404  FloatSignAsInt SignAsInt;
1405  getSignAsIntValue(SignAsInt, DL, Sign);
1406 
1407  EVT IntVT = SignAsInt.IntValue.getValueType();
1408  SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1409  SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1410  SignMask);
1411 
1412  // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1413  EVT FloatVT = Mag.getValueType();
1414  if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1415  TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1416  SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1417  SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1418  SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1419  DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1420  return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1421  }
1422 
1423  // Transform Mag value to integer, and clear the sign bit.
1424  FloatSignAsInt MagAsInt;
1425  getSignAsIntValue(MagAsInt, DL, Mag);
1426  EVT MagVT = MagAsInt.IntValue.getValueType();
1427  SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1428  SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1429  ClearSignMask);
1430 
1431  // Get the signbit at the right position for MagAsInt.
1432  int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1433  if (SignBit.getValueSizeInBits() > ClearedSign.getValueSizeInBits()) {
1434  if (ShiftAmount > 0) {
1435  SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, IntVT);
1436  SignBit = DAG.getNode(ISD::SRL, DL, IntVT, SignBit, ShiftCnst);
1437  } else if (ShiftAmount < 0) {
1438  SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, IntVT);
1439  SignBit = DAG.getNode(ISD::SHL, DL, IntVT, SignBit, ShiftCnst);
1440  }
1441  SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1442  } else if (SignBit.getValueSizeInBits() < ClearedSign.getValueSizeInBits()) {
1443  SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1444  if (ShiftAmount > 0) {
1445  SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, MagVT);
1446  SignBit = DAG.getNode(ISD::SRL, DL, MagVT, SignBit, ShiftCnst);
1447  } else if (ShiftAmount < 0) {
1448  SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, MagVT);
1449  SignBit = DAG.getNode(ISD::SHL, DL, MagVT, SignBit, ShiftCnst);
1450  }
1451  }
1452 
1453  // Store the part with the modified sign and convert back to float.
1454  SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1455  return modifySignAsInt(MagAsInt, DL, CopiedSign);
1456 }
1457 
1458 SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1459  SDLoc DL(Node);
1460  SDValue Value = Node->getOperand(0);
1461 
1462  // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1463  EVT FloatVT = Value.getValueType();
1464  if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1465  SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1466  return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1467  }
1468 
1469  // Transform value to integer, clear the sign bit and transform back.
1470  FloatSignAsInt ValueAsInt;
1471  getSignAsIntValue(ValueAsInt, DL, Value);
1472  EVT IntVT = ValueAsInt.IntValue.getValueType();
1473  SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1474  SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1475  ClearSignMask);
1476  return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1477 }
1478 
1479 void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1481  unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
1482  assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1483  " not tell us which reg is the stack pointer!");
1484  SDLoc dl(Node);
1485  EVT VT = Node->getValueType(0);
1486  SDValue Tmp1 = SDValue(Node, 0);
1487  SDValue Tmp2 = SDValue(Node, 1);
1488  SDValue Tmp3 = Node->getOperand(2);
1489  SDValue Chain = Tmp1.getOperand(0);
1490 
1491  // Chain the dynamic stack allocation so that it doesn't modify the stack
1492  // pointer when other instructions are using the stack.
1493  Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, dl, true), dl);
1494 
1495  SDValue Size = Tmp2.getOperand(1);
1496  SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1497  Chain = SP.getValue(1);
1498  unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue();
1499  unsigned StackAlign =
1500  DAG.getSubtarget().getFrameLowering()->getStackAlignment();
1501  Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
1502  if (Align > StackAlign)
1503  Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1504  DAG.getConstant(-(uint64_t)Align, dl, VT));
1505  Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1506 
1507  Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
1508  DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
1509 
1510  Results.push_back(Tmp1);
1511  Results.push_back(Tmp2);
1512 }
1513 
1514 /// Legalize a SETCC with given LHS and RHS and condition code CC on the current
1515 /// target.
1516 ///
1517 /// If the SETCC has been legalized using AND / OR, then the legalized node
1518 /// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert
1519 /// will be set to false.
1520 ///
1521 /// If the SETCC has been legalized by using getSetCCSwappedOperands(),
1522 /// then the values of LHS and RHS will be swapped, CC will be set to the
1523 /// new condition, and NeedInvert will be set to false.
1524 ///
1525 /// If the SETCC has been legalized using the inverse condcode, then LHS and
1526 /// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert
1527 /// will be set to true. The caller must invert the result of the SETCC with
1528 /// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect
1529 /// of a true/false result.
1530 ///
1531 /// \returns true if the SetCC has been legalized, false if it hasn't.
1532 bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS,
1533  SDValue &RHS, SDValue &CC,
1534  bool &NeedInvert,
1535  const SDLoc &dl) {
1536  MVT OpVT = LHS.getSimpleValueType();
1537  ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
1538  NeedInvert = false;
1539  switch (TLI.getCondCodeAction(CCCode, OpVT)) {
1540  default: llvm_unreachable("Unknown condition code action!");
1541  case TargetLowering::Legal:
1542  // Nothing to do.
1543  break;
1544  case TargetLowering::Expand: {
1546  if (TLI.isCondCodeLegal(InvCC, OpVT)) {
1547  std::swap(LHS, RHS);
1548  CC = DAG.getCondCode(InvCC);
1549  return true;
1550  }
1552  unsigned Opc = 0;
1553  switch (CCCode) {
1554  default: llvm_unreachable("Don't know how to expand this condition!");
1555  case ISD::SETO:
1556  assert(TLI.getCondCodeAction(ISD::SETOEQ, OpVT)
1558  && "If SETO is expanded, SETOEQ must be legal!");
1559  CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break;
1560  case ISD::SETUO:
1561  assert(TLI.getCondCodeAction(ISD::SETUNE, OpVT)
1563  && "If SETUO is expanded, SETUNE must be legal!");
1564  CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break;
1565  case ISD::SETOEQ:
1566  case ISD::SETOGT:
1567  case ISD::SETOGE:
1568  case ISD::SETOLT:
1569  case ISD::SETOLE:
1570  case ISD::SETONE:
1571  case ISD::SETUEQ:
1572  case ISD::SETUNE:
1573  case ISD::SETUGT:
1574  case ISD::SETUGE:
1575  case ISD::SETULT:
1576  case ISD::SETULE:
1577  // If we are floating point, assign and break, otherwise fall through.
1578  if (!OpVT.isInteger()) {
1579  // We can use the 4th bit to tell if we are the unordered
1580  // or ordered version of the opcode.
1581  CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
1582  Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
1583  CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
1584  break;
1585  }
1586  // Fallthrough if we are unsigned integer.
1588  case ISD::SETLE:
1589  case ISD::SETGT:
1590  case ISD::SETGE:
1591  case ISD::SETLT:
1592  // We only support using the inverted operation, which is computed above
1593  // and not a different manner of supporting expanding these cases.
1594  llvm_unreachable("Don't know how to expand this condition!");
1595  case ISD::SETNE:
1596  case ISD::SETEQ:
1597  // Try inverting the result of the inverse condition.
1598  InvCC = CCCode == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ;
1599  if (TLI.isCondCodeLegal(InvCC, OpVT)) {
1600  CC = DAG.getCondCode(InvCC);
1601  NeedInvert = true;
1602  return true;
1603  }
1604  // If inverting the condition didn't work then we have no means to expand
1605  // the condition.
1606  llvm_unreachable("Don't know how to expand this condition!");
1607  }
1608 
1609  SDValue SetCC1, SetCC2;
1610  if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
1611  // If we aren't the ordered or unorder operation,
1612  // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
1613  SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1);
1614  SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2);
1615  } else {
1616  // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
1617  SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1);
1618  SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2);
1619  }
1620  LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
1621  RHS = SDValue();
1622  CC = SDValue();
1623  return true;
1624  }
1625  }
1626  return false;
1627 }
1628 
1629 /// Emit a store/load combination to the stack. This stores
1630 /// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1631 /// a load from the stack slot to DestVT, extending it if needed.
1632 /// The resultant code need not be legal.
1633 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1634  EVT DestVT, const SDLoc &dl) {
1635  // Create the stack frame object.
1636  unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment(
1637  SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1638  SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign);
1639 
1640  FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1641  int SPFI = StackPtrFI->getIndex();
1642  MachinePointerInfo PtrInfo =
1643  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1644 
1645  unsigned SrcSize = SrcOp.getValueSizeInBits();
1646  unsigned SlotSize = SlotVT.getSizeInBits();
1647  unsigned DestSize = DestVT.getSizeInBits();
1648  Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1649  unsigned DestAlign = DAG.getDataLayout().getPrefTypeAlignment(DestType);
1650 
1651  // Emit a store to the stack slot. Use a truncstore if the input value is
1652  // later than DestVT.
1653  SDValue Store;
1654 
1655  if (SrcSize > SlotSize)
1656  Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo,
1657  SlotVT, SrcAlign);
1658  else {
1659  assert(SrcSize == SlotSize && "Invalid store");
1660  Store =
1661  DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1662  }
1663 
1664  // Result is a load from the stack slot.
1665  if (SlotSize == DestSize)
1666  return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1667 
1668  assert(SlotSize < DestSize && "Unknown extension!");
1669  return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1670  DestAlign);
1671 }
1672 
1673 SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1674  SDLoc dl(Node);
1675  // Create a vector sized/aligned stack slot, store the value to element #0,
1676  // then load the whole vector back out.
1677  SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1678 
1679  FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1680  int SPFI = StackPtrFI->getIndex();
1681 
1682  SDValue Ch = DAG.getTruncStore(
1683  DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1684  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1685  Node->getValueType(0).getVectorElementType());
1686  return DAG.getLoad(
1687  Node->getValueType(0), dl, Ch, StackPtr,
1688  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1689 }
1690 
1691 static bool
1693  const TargetLowering &TLI, SDValue &Res) {
1694  unsigned NumElems = Node->getNumOperands();
1695  SDLoc dl(Node);
1696  EVT VT = Node->getValueType(0);
1697 
1698  // Try to group the scalars into pairs, shuffle the pairs together, then
1699  // shuffle the pairs of pairs together, etc. until the vector has
1700  // been built. This will work only if all of the necessary shuffle masks
1701  // are legal.
1702 
1703  // We do this in two phases; first to check the legality of the shuffles,
1704  // and next, assuming that all shuffles are legal, to create the new nodes.
1705  for (int Phase = 0; Phase < 2; ++Phase) {
1707  NewIntermedVals;
1708  for (unsigned i = 0; i < NumElems; ++i) {
1709  SDValue V = Node->getOperand(i);
1710  if (V.isUndef())
1711  continue;
1712 
1713  SDValue Vec;
1714  if (Phase)
1715  Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1716  IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1717  }
1718 
1719  while (IntermedVals.size() > 2) {
1720  NewIntermedVals.clear();
1721  for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1722  // This vector and the next vector are shuffled together (simply to
1723  // append the one to the other).
1724  SmallVector<int, 16> ShuffleVec(NumElems, -1);
1725 
1726  SmallVector<int, 16> FinalIndices;
1727  FinalIndices.reserve(IntermedVals[i].second.size() +
1728  IntermedVals[i+1].second.size());
1729 
1730  int k = 0;
1731  for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1732  ++j, ++k) {
1733  ShuffleVec[k] = j;
1734  FinalIndices.push_back(IntermedVals[i].second[j]);
1735  }
1736  for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1737  ++j, ++k) {
1738  ShuffleVec[k] = NumElems + j;
1739  FinalIndices.push_back(IntermedVals[i+1].second[j]);
1740  }
1741 
1742  SDValue Shuffle;
1743  if (Phase)
1744  Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1745  IntermedVals[i+1].first,
1746  ShuffleVec);
1747  else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1748  return false;
1749  NewIntermedVals.push_back(
1750  std::make_pair(Shuffle, std::move(FinalIndices)));
1751  }
1752 
1753  // If we had an odd number of defined values, then append the last
1754  // element to the array of new vectors.
1755  if ((IntermedVals.size() & 1) != 0)
1756  NewIntermedVals.push_back(IntermedVals.back());
1757 
1758  IntermedVals.swap(NewIntermedVals);
1759  }
1760 
1761  assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1762  "Invalid number of intermediate vectors");
1763  SDValue Vec1 = IntermedVals[0].first;
1764  SDValue Vec2;
1765  if (IntermedVals.size() > 1)
1766  Vec2 = IntermedVals[1].first;
1767  else if (Phase)
1768  Vec2 = DAG.getUNDEF(VT);
1769 
1770  SmallVector<int, 16> ShuffleVec(NumElems, -1);
1771  for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1772  ShuffleVec[IntermedVals[0].second[i]] = i;
1773  for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1774  ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1775 
1776  if (Phase)
1777  Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1778  else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1779  return false;
1780  }
1781 
1782  return true;
1783 }
1784 
1785 /// Expand a BUILD_VECTOR node on targets that don't
1786 /// support the operation, but do support the resultant vector type.
1787 SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1788  unsigned NumElems = Node->getNumOperands();
1789  SDValue Value1, Value2;
1790  SDLoc dl(Node);
1791  EVT VT = Node->getValueType(0);
1792  EVT OpVT = Node->getOperand(0).getValueType();
1793  EVT EltVT = VT.getVectorElementType();
1794 
1795  // If the only non-undef value is the low element, turn this into a
1796  // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1797  bool isOnlyLowElement = true;
1798  bool MoreThanTwoValues = false;
1799  bool isConstant = true;
1800  for (unsigned i = 0; i < NumElems; ++i) {
1801  SDValue V = Node->getOperand(i);
1802  if (V.isUndef())
1803  continue;
1804  if (i > 0)
1805  isOnlyLowElement = false;
1806  if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1807  isConstant = false;
1808 
1809  if (!Value1.getNode()) {
1810  Value1 = V;
1811  } else if (!Value2.getNode()) {
1812  if (V != Value1)
1813  Value2 = V;
1814  } else if (V != Value1 && V != Value2) {
1815  MoreThanTwoValues = true;
1816  }
1817  }
1818 
1819  if (!Value1.getNode())
1820  return DAG.getUNDEF(VT);
1821 
1822  if (isOnlyLowElement)
1823  return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1824 
1825  // If all elements are constants, create a load from the constant pool.
1826  if (isConstant) {
1828  for (unsigned i = 0, e = NumElems; i != e; ++i) {
1829  if (ConstantFPSDNode *V =
1830  dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1831  CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1832  } else if (ConstantSDNode *V =
1833  dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1834  if (OpVT==EltVT)
1835  CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1836  else {
1837  // If OpVT and EltVT don't match, EltVT is not legal and the
1838  // element values have been promoted/truncated earlier. Undo this;
1839  // we don't want a v16i8 to become a v16i32 for example.
1840  const ConstantInt *CI = V->getConstantIntValue();
1841  CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1842  CI->getZExtValue()));
1843  }
1844  } else {
1845  assert(Node->getOperand(i).isUndef());
1846  Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1847  CV.push_back(UndefValue::get(OpNTy));
1848  }
1849  }
1850  Constant *CP = ConstantVector::get(CV);
1851  SDValue CPIdx =
1852  DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1853  unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
1854  return DAG.getLoad(
1855  VT, dl, DAG.getEntryNode(), CPIdx,
1856  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
1857  Alignment);
1858  }
1859 
1860  SmallSet<SDValue, 16> DefinedValues;
1861  for (unsigned i = 0; i < NumElems; ++i) {
1862  if (Node->getOperand(i).isUndef())
1863  continue;
1864  DefinedValues.insert(Node->getOperand(i));
1865  }
1866 
1867  if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
1868  if (!MoreThanTwoValues) {
1869  SmallVector<int, 8> ShuffleVec(NumElems, -1);
1870  for (unsigned i = 0; i < NumElems; ++i) {
1871  SDValue V = Node->getOperand(i);
1872  if (V.isUndef())
1873  continue;
1874  ShuffleVec[i] = V == Value1 ? 0 : NumElems;
1875  }
1876  if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
1877  // Get the splatted value into the low element of a vector register.
1878  SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
1879  SDValue Vec2;
1880  if (Value2.getNode())
1881  Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
1882  else
1883  Vec2 = DAG.getUNDEF(VT);
1884 
1885  // Return shuffle(LowValVec, undef, <0,0,0,0>)
1886  return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1887  }
1888  } else {
1889  SDValue Res;
1890  if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
1891  return Res;
1892  }
1893  }
1894 
1895  // Otherwise, we can't handle this case efficiently.
1896  return ExpandVectorBuildThroughStack(Node);
1897 }
1898 
1899 // Expand a node into a call to a libcall. If the result value
1900 // does not fit into a register, return the lo part and set the hi part to the
1901 // by-reg argument. If it does fit into a single register, return the result
1902 // and leave the Hi part unset.
1903 SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
1904  bool isSigned) {
1907  for (const SDValue &Op : Node->op_values()) {
1908  EVT ArgVT = Op.getValueType();
1909  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
1910  Entry.Node = Op;
1911  Entry.Ty = ArgTy;
1912  Entry.isSExt = isSigned;
1913  Entry.isZExt = !isSigned;
1914  Args.push_back(Entry);
1915  }
1916  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
1917  TLI.getPointerTy(DAG.getDataLayout()));
1918 
1919  Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
1920 
1921  // By default, the input chain to this libcall is the entry node of the
1922  // function. If the libcall is going to be emitted as a tail call then
1923  // TLI.isUsedByReturnOnly will change it to the right chain if the return
1924  // node which is being folded has a non-entry input chain.
1925  SDValue InChain = DAG.getEntryNode();
1926 
1927  // isTailCall may be true since the callee does not reference caller stack
1928  // frame. Check if it's in the right position and that the return types match.
1929  SDValue TCChain = InChain;
1930  const Function *F = DAG.getMachineFunction().getFunction();
1931  bool isTailCall =
1932  TLI.isInTailCallPosition(DAG, Node, TCChain) &&
1933  (RetTy == F->getReturnType() || F->getReturnType()->isVoidTy());
1934  if (isTailCall)
1935  InChain = TCChain;
1936 
1938  CLI.setDebugLoc(SDLoc(Node)).setChain(InChain)
1939  .setCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
1940  .setTailCall(isTailCall).setSExtResult(isSigned).setZExtResult(!isSigned);
1941 
1942  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
1943 
1944  if (!CallInfo.second.getNode())
1945  // It's a tailcall, return the chain (which is the DAG root).
1946  return DAG.getRoot();
1947 
1948  return CallInfo.first;
1949 }
1950 
1951 /// Generate a libcall taking the given operands as arguments
1952 /// and returning a result of type RetVT.
1953 SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT,
1954  const SDValue *Ops, unsigned NumOps,
1955  bool isSigned, const SDLoc &dl) {
1957  Args.reserve(NumOps);
1958 
1960  for (unsigned i = 0; i != NumOps; ++i) {
1961  Entry.Node = Ops[i];
1962  Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
1963  Entry.isSExt = isSigned;
1964  Entry.isZExt = !isSigned;
1965  Args.push_back(Entry);
1966  }
1967  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
1968  TLI.getPointerTy(DAG.getDataLayout()));
1969 
1970  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
1971 
1973  CLI.setDebugLoc(dl).setChain(DAG.getEntryNode())
1974  .setCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
1975  .setSExtResult(isSigned).setZExtResult(!isSigned);
1976 
1977  std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(CLI);
1978 
1979  return CallInfo.first;
1980 }
1981 
1982 // Expand a node into a call to a libcall. Similar to
1983 // ExpandLibCall except that the first operand is the in-chain.
1984 std::pair<SDValue, SDValue>
1985 SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC,
1986  SDNode *Node,
1987  bool isSigned) {
1988  SDValue InChain = Node->getOperand(0);
1989 
1992  for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) {
1993  EVT ArgVT = Node->getOperand(i).getValueType();
1994  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
1995  Entry.Node = Node->getOperand(i);
1996  Entry.Ty = ArgTy;
1997  Entry.isSExt = isSigned;
1998  Entry.isZExt = !isSigned;
1999  Args.push_back(Entry);
2000  }
2001  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2002  TLI.getPointerTy(DAG.getDataLayout()));
2003 
2004  Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
2005 
2007  CLI.setDebugLoc(SDLoc(Node)).setChain(InChain)
2008  .setCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
2009  .setSExtResult(isSigned).setZExtResult(!isSigned);
2010 
2011  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2012 
2013  return CallInfo;
2014 }
2015 
2016 SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2017  RTLIB::Libcall Call_F32,
2018  RTLIB::Libcall Call_F64,
2019  RTLIB::Libcall Call_F80,
2020  RTLIB::Libcall Call_F128,
2021  RTLIB::Libcall Call_PPCF128) {
2022  RTLIB::Libcall LC;
2023  switch (Node->getSimpleValueType(0).SimpleTy) {
2024  default: llvm_unreachable("Unexpected request for libcall!");
2025  case MVT::f32: LC = Call_F32; break;
2026  case MVT::f64: LC = Call_F64; break;
2027  case MVT::f80: LC = Call_F80; break;
2028  case MVT::f128: LC = Call_F128; break;
2029  case MVT::ppcf128: LC = Call_PPCF128; break;
2030  }
2031  return ExpandLibCall(LC, Node, false);
2032 }
2033 
2034 SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2035  RTLIB::Libcall Call_I8,
2036  RTLIB::Libcall Call_I16,
2037  RTLIB::Libcall Call_I32,
2038  RTLIB::Libcall Call_I64,
2039  RTLIB::Libcall Call_I128) {
2040  RTLIB::Libcall LC;
2041  switch (Node->getSimpleValueType(0).SimpleTy) {
2042  default: llvm_unreachable("Unexpected request for libcall!");
2043  case MVT::i8: LC = Call_I8; break;
2044  case MVT::i16: LC = Call_I16; break;
2045  case MVT::i32: LC = Call_I32; break;
2046  case MVT::i64: LC = Call_I64; break;
2047  case MVT::i128: LC = Call_I128; break;
2048  }
2049  return ExpandLibCall(LC, Node, isSigned);
2050 }
2051 
2052 /// Issue libcalls to __{u}divmod to compute div / rem pairs.
2053 void
2054 SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2055  SmallVectorImpl<SDValue> &Results) {
2056  unsigned Opcode = Node->getOpcode();
2057  bool isSigned = Opcode == ISD::SDIVREM;
2058 
2059  RTLIB::Libcall LC;
2060  switch (Node->getSimpleValueType(0).SimpleTy) {
2061  default: llvm_unreachable("Unexpected request for libcall!");
2062  case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2063  case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2064  case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2065  case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2066  case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2067  }
2068 
2069  // The input chain to this libcall is the entry node of the function.
2070  // Legalizing the call will automatically add the previous call to the
2071  // dependence.
2072  SDValue InChain = DAG.getEntryNode();
2073 
2074  EVT RetVT = Node->getValueType(0);
2075  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2076 
2079  for (const SDValue &Op : Node->op_values()) {
2080  EVT ArgVT = Op.getValueType();
2081  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2082  Entry.Node = Op;
2083  Entry.Ty = ArgTy;
2084  Entry.isSExt = isSigned;
2085  Entry.isZExt = !isSigned;
2086  Args.push_back(Entry);
2087  }
2088 
2089  // Also pass the return address of the remainder.
2090  SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2091  Entry.Node = FIPtr;
2092  Entry.Ty = RetTy->getPointerTo();
2093  Entry.isSExt = isSigned;
2094  Entry.isZExt = !isSigned;
2095  Args.push_back(Entry);
2096 
2097  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2098  TLI.getPointerTy(DAG.getDataLayout()));
2099 
2100  SDLoc dl(Node);
2102  CLI.setDebugLoc(dl).setChain(InChain)
2103  .setCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
2104  .setSExtResult(isSigned).setZExtResult(!isSigned);
2105 
2106  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2107 
2108  // Remainder is loaded back from the stack frame.
2109  SDValue Rem =
2110  DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2111  Results.push_back(CallInfo.first);
2112  Results.push_back(Rem);
2113 }
2114 
2115 /// Return true if sincos libcall is available.
2116 static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2117  RTLIB::Libcall LC;
2118  switch (Node->getSimpleValueType(0).SimpleTy) {
2119  default: llvm_unreachable("Unexpected request for libcall!");
2120  case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2121  case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2122  case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2123  case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2124  case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2125  }
2126  return TLI.getLibcallName(LC) != nullptr;
2127 }
2128 
2129 /// Return true if sincos libcall is available and can be used to combine sin
2130 /// and cos.
2131 static bool canCombineSinCosLibcall(SDNode *Node, const TargetLowering &TLI,
2132  const TargetMachine &TM) {
2133  if (!isSinCosLibcallAvailable(Node, TLI))
2134  return false;
2135  // GNU sin/cos functions set errno while sincos does not. Therefore
2136  // combining sin and cos is only safe if unsafe-fpmath is enabled.
2138  return false;
2139  return true;
2140 }
2141 
2142 /// Only issue sincos libcall if both sin and cos are needed.
2143 static bool useSinCos(SDNode *Node) {
2144  unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2145  ? ISD::FCOS : ISD::FSIN;
2146 
2147  SDValue Op0 = Node->getOperand(0);
2148  for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
2149  UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
2150  SDNode *User = *UI;
2151  if (User == Node)
2152  continue;
2153  // The other user might have been turned into sincos already.
2154  if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2155  return true;
2156  }
2157  return false;
2158 }
2159 
2160 /// Issue libcalls to sincos to compute sin / cos pairs.
2161 void
2162 SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2163  SmallVectorImpl<SDValue> &Results) {
2164  RTLIB::Libcall LC;
2165  switch (Node->getSimpleValueType(0).SimpleTy) {
2166  default: llvm_unreachable("Unexpected request for libcall!");
2167  case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2168  case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2169  case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2170  case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2171  case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2172  }
2173 
2174  // The input chain to this libcall is the entry node of the function.
2175  // Legalizing the call will automatically add the previous call to the
2176  // dependence.
2177  SDValue InChain = DAG.getEntryNode();
2178 
2179  EVT RetVT = Node->getValueType(0);
2180  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2181 
2184 
2185  // Pass the argument.
2186  Entry.Node = Node->getOperand(0);
2187  Entry.Ty = RetTy;
2188  Entry.isSExt = false;
2189  Entry.isZExt = false;
2190  Args.push_back(Entry);
2191 
2192  // Pass the return address of sin.
2193  SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2194  Entry.Node = SinPtr;
2195  Entry.Ty = RetTy->getPointerTo();
2196  Entry.isSExt = false;
2197  Entry.isZExt = false;
2198  Args.push_back(Entry);
2199 
2200  // Also pass the return address of the cos.
2201  SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2202  Entry.Node = CosPtr;
2203  Entry.Ty = RetTy->getPointerTo();
2204  Entry.isSExt = false;
2205  Entry.isZExt = false;
2206  Args.push_back(Entry);
2207 
2208  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2209  TLI.getPointerTy(DAG.getDataLayout()));
2210 
2211  SDLoc dl(Node);
2213  CLI.setDebugLoc(dl).setChain(InChain)
2214  .setCallee(TLI.getLibcallCallingConv(LC),
2215  Type::getVoidTy(*DAG.getContext()), Callee, std::move(Args));
2216 
2217  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2218 
2219  Results.push_back(
2220  DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2221  Results.push_back(
2222  DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2223 }
2224 
2225 /// This function is responsible for legalizing a
2226 /// INT_TO_FP operation of the specified operand when the target requests that
2227 /// we expand it. At this point, we know that the result and operand types are
2228 /// legal for the target.
2229 SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0,
2230  EVT DestVT,
2231  const SDLoc &dl) {
2232  // TODO: Should any fast-math-flags be set for the created nodes?
2233 
2234  if (Op0.getValueType() == MVT::i32 && TLI.isTypeLegal(MVT::f64)) {
2235  // simple 32-bit [signed|unsigned] integer to float/double expansion
2236 
2237  // Get the stack frame index of a 8 byte buffer.
2238  SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2239 
2240  // word offset constant for Hi/Lo address computation
2241  SDValue WordOff = DAG.getConstant(sizeof(int), dl,
2242  StackSlot.getValueType());
2243  // set up Hi and Lo (into buffer) address based on endian
2244  SDValue Hi = StackSlot;
2245  SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(),
2246  StackSlot, WordOff);
2247  if (DAG.getDataLayout().isLittleEndian())
2248  std::swap(Hi, Lo);
2249 
2250  // if signed map to unsigned space
2251  SDValue Op0Mapped;
2252  if (isSigned) {
2253  // constant used to invert sign bit (signed to unsigned mapping)
2254  SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32);
2255  Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit);
2256  } else {
2257  Op0Mapped = Op0;
2258  }
2259  // store the lo of the constructed double - based on integer input
2260  SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op0Mapped, Lo,
2261  MachinePointerInfo());
2262  // initial hi portion of constructed double
2263  SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2264  // store the hi of the constructed double - biased exponent
2265  SDValue Store2 =
2266  DAG.getStore(Store1, dl, InitialHi, Hi, MachinePointerInfo());
2267  // load the constructed double
2268  SDValue Load =
2269  DAG.getLoad(MVT::f64, dl, Store2, StackSlot, MachinePointerInfo());
2270  // FP constant to bias correct the final result
2271  SDValue Bias = DAG.getConstantFP(isSigned ?
2272  BitsToDouble(0x4330000080000000ULL) :
2273  BitsToDouble(0x4330000000000000ULL),
2274  dl, MVT::f64);
2275  // subtract the bias
2276  SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2277  // final result
2278  SDValue Result;
2279  // handle final rounding
2280  if (DestVT == MVT::f64) {
2281  // do nothing
2282  Result = Sub;
2283  } else if (DestVT.bitsLT(MVT::f64)) {
2284  Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub,
2285  DAG.getIntPtrConstant(0, dl));
2286  } else if (DestVT.bitsGT(MVT::f64)) {
2287  Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub);
2288  }
2289  return Result;
2290  }
2291  assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet");
2292  // Code below here assumes !isSigned without checking again.
2293 
2294  // Implementation of unsigned i64 to f64 following the algorithm in
2295  // __floatundidf in compiler_rt. This implementation has the advantage
2296  // of performing rounding correctly, both in the default rounding mode
2297  // and in all alternate rounding modes.
2298  // TODO: Generalize this for use with other types.
2299  if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f64) {
2300  SDValue TwoP52 =
2301  DAG.getConstant(UINT64_C(0x4330000000000000), dl, MVT::i64);
2302  SDValue TwoP84PlusTwoP52 =
2303  DAG.getConstantFP(BitsToDouble(UINT64_C(0x4530000000100000)), dl,
2304  MVT::f64);
2305  SDValue TwoP84 =
2306  DAG.getConstant(UINT64_C(0x4530000000000000), dl, MVT::i64);
2307 
2308  SDValue Lo = DAG.getZeroExtendInReg(Op0, dl, MVT::i32);
2309  SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0,
2310  DAG.getConstant(32, dl, MVT::i64));
2311  SDValue LoOr = DAG.getNode(ISD::OR, dl, MVT::i64, Lo, TwoP52);
2312  SDValue HiOr = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, TwoP84);
2313  SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, LoOr);
2314  SDValue HiFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, HiOr);
2315  SDValue HiSub = DAG.getNode(ISD::FSUB, dl, MVT::f64, HiFlt,
2316  TwoP84PlusTwoP52);
2317  return DAG.getNode(ISD::FADD, dl, MVT::f64, LoFlt, HiSub);
2318  }
2319 
2320  // Implementation of unsigned i64 to f32.
2321  // TODO: Generalize this for use with other types.
2322  if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f32) {
2323  // For unsigned conversions, convert them to signed conversions using the
2324  // algorithm from the x86_64 __floatundidf in compiler_rt.
2325  if (!isSigned) {
2326  SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0);
2327 
2328  SDValue ShiftConst = DAG.getConstant(
2329  1, dl, TLI.getShiftAmountTy(Op0.getValueType(), DAG.getDataLayout()));
2330  SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst);
2331  SDValue AndConst = DAG.getConstant(1, dl, MVT::i64);
2332  SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, AndConst);
2333  SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr);
2334 
2335  SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Or);
2336  SDValue Slow = DAG.getNode(ISD::FADD, dl, MVT::f32, SignCvt, SignCvt);
2337 
2338  // TODO: This really should be implemented using a branch rather than a
2339  // select. We happen to get lucky and machinesink does the right
2340  // thing most of the time. This would be a good candidate for a
2341  //pseudo-op, or, even better, for whole-function isel.
2342  SDValue SignBitTest = DAG.getSetCC(dl, getSetCCResultType(MVT::i64),
2343  Op0, DAG.getConstant(0, dl, MVT::i64), ISD::SETLT);
2344  return DAG.getSelect(dl, MVT::f32, SignBitTest, Slow, Fast);
2345  }
2346 
2347  // Otherwise, implement the fully general conversion.
2348 
2349  SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2350  DAG.getConstant(UINT64_C(0xfffffffffffff800), dl, MVT::i64));
2351  SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And,
2352  DAG.getConstant(UINT64_C(0x800), dl, MVT::i64));
2353  SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2354  DAG.getConstant(UINT64_C(0x7ff), dl, MVT::i64));
2355  SDValue Ne = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), And2,
2356  DAG.getConstant(UINT64_C(0), dl, MVT::i64),
2357  ISD::SETNE);
2358  SDValue Sel = DAG.getSelect(dl, MVT::i64, Ne, Or, Op0);
2359  SDValue Ge = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), Op0,
2360  DAG.getConstant(UINT64_C(0x0020000000000000), dl,
2361  MVT::i64),
2362  ISD::SETUGE);
2363  SDValue Sel2 = DAG.getSelect(dl, MVT::i64, Ge, Sel, Op0);
2364  EVT SHVT = TLI.getShiftAmountTy(Sel2.getValueType(), DAG.getDataLayout());
2365 
2366  SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2,
2367  DAG.getConstant(32, dl, SHVT));
2368  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh);
2369  SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc);
2370  SDValue TwoP32 =
2371  DAG.getConstantFP(BitsToDouble(UINT64_C(0x41f0000000000000)), dl,
2372  MVT::f64);
2373  SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt);
2374  SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2);
2375  SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo);
2376  SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2);
2377  return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd,
2378  DAG.getIntPtrConstant(0, dl));
2379  }
2380 
2381  SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2382 
2383  SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(Op0.getValueType()),
2384  Op0,
2385  DAG.getConstant(0, dl, Op0.getValueType()),
2386  ISD::SETLT);
2387  SDValue Zero = DAG.getIntPtrConstant(0, dl),
2388  Four = DAG.getIntPtrConstant(4, dl);
2389  SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2390  SignSet, Four, Zero);
2391 
2392  // If the sign bit of the integer is set, the large number will be treated
2393  // as a negative number. To counteract this, the dynamic code adds an
2394  // offset depending on the data type.
2395  uint64_t FF;
2396  switch (Op0.getSimpleValueType().SimpleTy) {
2397  default: llvm_unreachable("Unsupported integer type!");
2398  case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2399  case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2400  case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2401  case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2402  }
2403  if (DAG.getDataLayout().isLittleEndian())
2404  FF <<= 32;
2405  Constant *FudgeFactor = ConstantInt::get(
2406  Type::getInt64Ty(*DAG.getContext()), FF);
2407 
2408  SDValue CPIdx =
2409  DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2410  unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
2411  CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2412  Alignment = std::min(Alignment, 4u);
2413  SDValue FudgeInReg;
2414  if (DestVT == MVT::f32)
2415  FudgeInReg = DAG.getLoad(
2416  MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2417  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2418  Alignment);
2419  else {
2420  SDValue Load = DAG.getExtLoad(
2421  ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2422  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2423  Alignment);
2424  HandleSDNode Handle(Load);
2425  LegalizeOp(Load.getNode());
2426  FudgeInReg = Handle.getValue();
2427  }
2428 
2429  return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2430 }
2431 
2432 /// This function is responsible for legalizing a
2433 /// *INT_TO_FP operation of the specified operand when the target requests that
2434 /// we promote it. At this point, we know that the result and operand types are
2435 /// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2436 /// operation that takes a larger input.
2437 SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT,
2438  bool isSigned,
2439  const SDLoc &dl) {
2440  // First step, figure out the appropriate *INT_TO_FP operation to use.
2441  EVT NewInTy = LegalOp.getValueType();
2442 
2443  unsigned OpToUse = 0;
2444 
2445  // Scan for the appropriate larger type to use.
2446  while (1) {
2447  NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2448  assert(NewInTy.isInteger() && "Ran out of possibilities!");
2449 
2450  // If the target supports SINT_TO_FP of this type, use it.
2451  if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) {
2452  OpToUse = ISD::SINT_TO_FP;
2453  break;
2454  }
2455  if (isSigned) continue;
2456 
2457  // If the target supports UINT_TO_FP of this type, use it.
2458  if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) {
2459  OpToUse = ISD::UINT_TO_FP;
2460  break;
2461  }
2462 
2463  // Otherwise, try a larger type.
2464  }
2465 
2466  // Okay, we found the operation and type to use. Zero extend our input to the
2467  // desired type then run the operation on it.
2468  return DAG.getNode(OpToUse, dl, DestVT,
2469  DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2470  dl, NewInTy, LegalOp));
2471 }
2472 
2473 /// This function is responsible for legalizing a
2474 /// FP_TO_*INT operation of the specified operand when the target requests that
2475 /// we promote it. At this point, we know that the result and operand types are
2476 /// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2477 /// operation that returns a larger result.
2478 SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT,
2479  bool isSigned,
2480  const SDLoc &dl) {
2481  // First step, figure out the appropriate FP_TO*INT operation to use.
2482  EVT NewOutTy = DestVT;
2483 
2484  unsigned OpToUse = 0;
2485 
2486  // Scan for the appropriate larger type to use.
2487  while (1) {
2488  NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2489  assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2490 
2491  // A larger signed type can hold all unsigned values of the requested type,
2492  // so using FP_TO_SINT is valid
2493  if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) {
2494  OpToUse = ISD::FP_TO_SINT;
2495  break;
2496  }
2497 
2498  // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2499  if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) {
2500  OpToUse = ISD::FP_TO_UINT;
2501  break;
2502  }
2503 
2504  // Otherwise, try a larger type.
2505  }
2506 
2507 
2508  // Okay, we found the operation and type to use.
2509  SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2510 
2511  // Truncate the result of the extended FP_TO_*INT operation to the desired
2512  // size.
2513  return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2514 }
2515 
2516 /// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts.
2517 SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) {
2518  EVT VT = Op.getValueType();
2519  EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2520  unsigned Sz = VT.getScalarSizeInBits();
2521 
2522  SDValue Tmp, Tmp2, Tmp3;
2523 
2524  // If we can, perform BSWAP first and then the mask+swap the i4, then i2
2525  // and finally the i1 pairs.
2526  // TODO: We can easily support i4/i2 legal types if any target ever does.
2527  if (Sz >= 8 && isPowerOf2_32(Sz)) {
2528  // Create the masks - repeating the pattern every byte.
2529  APInt MaskHi4(Sz, 0), MaskHi2(Sz, 0), MaskHi1(Sz, 0);
2530  APInt MaskLo4(Sz, 0), MaskLo2(Sz, 0), MaskLo1(Sz, 0);
2531  for (unsigned J = 0; J != Sz; J += 8) {
2532  MaskHi4 = MaskHi4.Or(APInt(Sz, 0xF0ull << J));
2533  MaskLo4 = MaskLo4.Or(APInt(Sz, 0x0Full << J));
2534  MaskHi2 = MaskHi2.Or(APInt(Sz, 0xCCull << J));
2535  MaskLo2 = MaskLo2.Or(APInt(Sz, 0x33ull << J));
2536  MaskHi1 = MaskHi1.Or(APInt(Sz, 0xAAull << J));
2537  MaskLo1 = MaskLo1.Or(APInt(Sz, 0x55ull << J));
2538  }
2539 
2540  // BSWAP if the type is wider than a single byte.
2541  Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);
2542 
2543  // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4)
2544  Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT));
2545  Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT));
2546  Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, VT));
2547  Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, VT));
2548  Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2549 
2550  // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2)
2551  Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT));
2552  Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT));
2553  Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, VT));
2554  Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, VT));
2555  Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2556 
2557  // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1)
2558  Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT));
2559  Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT));
2560  Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, VT));
2561  Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, VT));
2562  Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2563  return Tmp;
2564  }
2565 
2566  Tmp = DAG.getConstant(0, dl, VT);
2567  for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
2568  if (I < J)
2569  Tmp2 =
2570  DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
2571  else
2572  Tmp2 =
2573  DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));
2574 
2575  APInt Shift(Sz, 1);
2576  Shift = Shift.shl(J);
2577  Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
2578  Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
2579  }
2580 
2581  return Tmp;
2582 }
2583 
2584 /// Open code the operations for BSWAP of the specified operation.
2585 SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) {
2586  EVT VT = Op.getValueType();
2587  EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2588  SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
2589  switch (VT.getSimpleVT().getScalarType().SimpleTy) {
2590  default: llvm_unreachable("Unhandled Expand type in BSWAP!");
2591  case MVT::i16:
2592  Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2593  Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2594  return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
2595  case MVT::i32:
2596  Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2597  Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2598  Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2599  Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2600  Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2601  DAG.getConstant(0xFF0000, dl, VT));
2602  Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
2603  Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2604  Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2605  return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2606  case MVT::i64:
2607  Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2608  Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2609  Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2610  Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2611  Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2612  Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2613  Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2614  Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2615  Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,
2616  DAG.getConstant(255ULL<<48, dl, VT));
2617  Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,
2618  DAG.getConstant(255ULL<<40, dl, VT));
2619  Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,
2620  DAG.getConstant(255ULL<<32, dl, VT));
2621  Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
2622  DAG.getConstant(255ULL<<24, dl, VT));
2623  Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2624  DAG.getConstant(255ULL<<16, dl, VT));
2625  Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
2626  DAG.getConstant(255ULL<<8 , dl, VT));
2627  Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
2628  Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
2629  Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2630  Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2631  Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
2632  Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2633  return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
2634  }
2635 }
2636 
2637 /// Expand the specified bitcount instruction into operations.
2638 SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op,
2639  const SDLoc &dl) {
2640  switch (Opc) {
2641  default: llvm_unreachable("Cannot expand this yet!");
2642  case ISD::CTPOP: {
2643  EVT VT = Op.getValueType();
2644  EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2645  unsigned Len = VT.getSizeInBits();
2646 
2647  assert(VT.isInteger() && Len <= 128 && Len % 8 == 0 &&
2648  "CTPOP not implemented for this type.");
2649 
2650  // This is the "best" algorithm from
2651  // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
2652 
2653  SDValue Mask55 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)),
2654  dl, VT);
2655  SDValue Mask33 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)),
2656  dl, VT);
2657  SDValue Mask0F = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)),
2658  dl, VT);
2659  SDValue Mask01 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)),
2660  dl, VT);
2661 
2662  // v = v - ((v >> 1) & 0x55555555...)
2663  Op = DAG.getNode(ISD::SUB, dl, VT, Op,
2664  DAG.getNode(ISD::AND, dl, VT,
2665  DAG.getNode(ISD::SRL, dl, VT, Op,
2666  DAG.getConstant(1, dl, ShVT)),
2667  Mask55));
2668  // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
2669  Op = DAG.getNode(ISD::ADD, dl, VT,
2670  DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
2671  DAG.getNode(ISD::AND, dl, VT,
2672  DAG.getNode(ISD::SRL, dl, VT, Op,
2673  DAG.getConstant(2, dl, ShVT)),
2674  Mask33));
2675  // v = (v + (v >> 4)) & 0x0F0F0F0F...
2676  Op = DAG.getNode(ISD::AND, dl, VT,
2677  DAG.getNode(ISD::ADD, dl, VT, Op,
2678  DAG.getNode(ISD::SRL, dl, VT, Op,
2679  DAG.getConstant(4, dl, ShVT))),
2680  Mask0F);
2681  // v = (v * 0x01010101...) >> (Len - 8)
2682  Op = DAG.getNode(ISD::SRL, dl, VT,
2683  DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
2684  DAG.getConstant(Len - 8, dl, ShVT));
2685 
2686  return Op;
2687  }
2688  case ISD::CTLZ_ZERO_UNDEF:
2689  // This trivially expands to CTLZ.
2690  return DAG.getNode(ISD::CTLZ, dl, Op.getValueType(), Op);
2691  case ISD::CTLZ: {
2692  EVT VT = Op.getValueType();
2693  unsigned len = VT.getSizeInBits();
2694 
2695  if (TLI.isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)) {
2696  EVT SetCCVT = getSetCCResultType(VT);
2697  SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op);
2698  SDValue Zero = DAG.getConstant(0, dl, VT);
2699  SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
2700  return DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
2701  DAG.getConstant(len, dl, VT), CTLZ);
2702  }
2703 
2704  // for now, we do this:
2705  // x = x | (x >> 1);
2706  // x = x | (x >> 2);
2707  // ...
2708  // x = x | (x >>16);
2709  // x = x | (x >>32); // for 64-bit input
2710  // return popcount(~x);
2711  //
2712  // Ref: "Hacker's Delight" by Henry Warren
2713  EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2714  for (unsigned i = 0; (1U << i) <= (len / 2); ++i) {
2715  SDValue Tmp3 = DAG.getConstant(1ULL << i, dl, ShVT);
2716  Op = DAG.getNode(ISD::OR, dl, VT, Op,
2717  DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3));
2718  }
2719  Op = DAG.getNOT(dl, Op, VT);
2720  return DAG.getNode(ISD::CTPOP, dl, VT, Op);
2721  }
2722  case ISD::CTTZ_ZERO_UNDEF:
2723  // This trivially expands to CTTZ.
2724  return DAG.getNode(ISD::CTTZ, dl, Op.getValueType(), Op);
2725  case ISD::CTTZ: {
2726  // for now, we use: { return popcount(~x & (x - 1)); }
2727  // unless the target has ctlz but not ctpop, in which case we use:
2728  // { return 32 - nlz(~x & (x-1)); }
2729  // Ref: "Hacker's Delight" by Henry Warren
2730  EVT VT = Op.getValueType();
2731  SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT,
2732  DAG.getNOT(dl, Op, VT),
2733  DAG.getNode(ISD::SUB, dl, VT, Op,
2734  DAG.getConstant(1, dl, VT)));
2735  // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
2736  if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) &&
2737  TLI.isOperationLegalOrCustom(ISD::CTLZ, VT))
2738  return DAG.getNode(ISD::SUB, dl, VT,
2739  DAG.getConstant(VT.getSizeInBits(), dl, VT),
2740  DAG.getNode(ISD::CTLZ, dl, VT, Tmp3));
2741  return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3);
2742  }
2743  }
2744 }
2745 
2746 bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2748  SDLoc dl(Node);
2749  SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2750  bool NeedInvert;
2751  switch (Node->getOpcode()) {
2752  case ISD::CTPOP:
2753  case ISD::CTLZ:
2754  case ISD::CTLZ_ZERO_UNDEF:
2755  case ISD::CTTZ:
2756  case ISD::CTTZ_ZERO_UNDEF:
2757  Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl);
2758  Results.push_back(Tmp1);
2759  break;
2760  case ISD::BITREVERSE:
2761  Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl));
2762  break;
2763  case ISD::BSWAP:
2764  Results.push_back(ExpandBSWAP(Node->getOperand(0), dl));
2765  break;
2766  case ISD::FRAMEADDR:
2767  case ISD::RETURNADDR:
2769  Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
2770  break;
2771  case ISD::EH_DWARF_CFA: {
2772  SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
2773  TLI.getPointerTy(DAG.getDataLayout()));
2774  SDValue Offset = DAG.getNode(ISD::ADD, dl,
2775  CfaArg.getValueType(),
2776  DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
2777  CfaArg.getValueType()),
2778  CfaArg);
2779  SDValue FA = DAG.getNode(
2780  ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
2781  DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
2782  Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
2783  FA, Offset));
2784  break;
2785  }
2786  case ISD::FLT_ROUNDS_:
2787  Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
2788  break;
2789  case ISD::EH_RETURN:
2790  case ISD::EH_LABEL:
2791  case ISD::PREFETCH:
2792  case ISD::VAEND:
2793  case ISD::EH_SJLJ_LONGJMP:
2794  // If the target didn't expand these, there's nothing to do, so just
2795  // preserve the chain and be done.
2796  Results.push_back(Node->getOperand(0));
2797  break;
2798  case ISD::READCYCLECOUNTER:
2799  // If the target didn't expand this, just return 'zero' and preserve the
2800  // chain.
2801  Results.append(Node->getNumValues() - 1,
2802  DAG.getConstant(0, dl, Node->getValueType(0)));
2803  Results.push_back(Node->getOperand(0));
2804  break;
2805  case ISD::EH_SJLJ_SETJMP:
2806  // If the target didn't expand this, just return 'zero' and preserve the
2807  // chain.
2808  Results.push_back(DAG.getConstant(0, dl, MVT::i32));
2809  Results.push_back(Node->getOperand(0));
2810  break;
2811  case ISD::ATOMIC_LOAD: {
2812  // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
2813  SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
2814  SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2815  SDValue Swap = DAG.getAtomicCmpSwap(
2816  ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2817  Node->getOperand(0), Node->getOperand(1), Zero, Zero,
2818  cast<AtomicSDNode>(Node)->getMemOperand());
2819  Results.push_back(Swap.getValue(0));
2820  Results.push_back(Swap.getValue(1));
2821  break;
2822  }
2823  case ISD::ATOMIC_STORE: {
2824  // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
2825  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2826  cast<AtomicSDNode>(Node)->getMemoryVT(),
2827  Node->getOperand(0),
2828  Node->getOperand(1), Node->getOperand(2),
2829  cast<AtomicSDNode>(Node)->getMemOperand());
2830  Results.push_back(Swap.getValue(1));
2831  break;
2832  }
2834  // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
2835  // splits out the success value as a comparison. Expanding the resulting
2836  // ATOMIC_CMP_SWAP will produce a libcall.
2837  SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2838  SDValue Res = DAG.getAtomicCmpSwap(
2839  ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2840  Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
2841  Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
2842 
2843  SDValue ExtRes = Res;
2844  SDValue LHS = Res;
2845  SDValue RHS = Node->getOperand(1);
2846 
2847  EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
2848  EVT OuterType = Node->getValueType(0);
2849  switch (TLI.getExtendForAtomicOps()) {
2850  case ISD::SIGN_EXTEND:
2851  LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
2852  DAG.getValueType(AtomicType));
2853  RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
2854  Node->getOperand(2), DAG.getValueType(AtomicType));
2855  ExtRes = LHS;
2856  break;
2857  case ISD::ZERO_EXTEND:
2858  LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
2859  DAG.getValueType(AtomicType));
2860  RHS = DAG.getNode(ISD::ZERO_EXTEND, dl, OuterType, Node->getOperand(2));
2861  ExtRes = LHS;
2862  break;
2863  case ISD::ANY_EXTEND:
2864  LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
2865  RHS = DAG.getNode(ISD::ZERO_EXTEND, dl, OuterType, Node->getOperand(2));
2866  break;
2867  default:
2868  llvm_unreachable("Invalid atomic op extension");
2869  }
2870 
2871  SDValue Success =
2872  DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
2873 
2874  Results.push_back(ExtRes.getValue(0));
2875  Results.push_back(Success);
2876  Results.push_back(Res.getValue(1));
2877  break;
2878  }
2880  ExpandDYNAMIC_STACKALLOC(Node, Results);
2881  break;
2882  case ISD::MERGE_VALUES:
2883  for (unsigned i = 0; i < Node->getNumValues(); i++)
2884  Results.push_back(Node->getOperand(i));
2885  break;
2886  case ISD::UNDEF: {
2887  EVT VT = Node->getValueType(0);
2888  if (VT.isInteger())
2889  Results.push_back(DAG.getConstant(0, dl, VT));
2890  else {
2891  assert(VT.isFloatingPoint() && "Unknown value type!");
2892  Results.push_back(DAG.getConstantFP(0, dl, VT));
2893  }
2894  break;
2895  }
2896  case ISD::FP_ROUND:
2897  case ISD::BITCAST:
2898  Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
2899  Node->getValueType(0), dl);
2900  Results.push_back(Tmp1);
2901  break;
2902  case ISD::FP_EXTEND:
2903  Tmp1 = EmitStackConvert(Node->getOperand(0),
2904  Node->getOperand(0).getValueType(),
2905  Node->getValueType(0), dl);
2906  Results.push_back(Tmp1);
2907  break;
2908  case ISD::SIGN_EXTEND_INREG: {
2909  EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2910  EVT VT = Node->getValueType(0);
2911 
2912  // An in-register sign-extend of a boolean is a negation:
2913  // 'true' (1) sign-extended is -1.
2914  // 'false' (0) sign-extended is 0.
2915  // However, we must mask the high bits of the source operand because the
2916  // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
2917 
2918  // TODO: Do this for vectors too?
2919  if (ExtraVT.getSizeInBits() == 1) {
2920  SDValue One = DAG.getConstant(1, dl, VT);
2921  SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
2922  SDValue Zero = DAG.getConstant(0, dl, VT);
2923  SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
2924  Results.push_back(Neg);
2925  break;
2926  }
2927 
2928  // NOTE: we could fall back on load/store here too for targets without
2929  // SRA. However, it is doubtful that any exist.
2930  EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2931  if (VT.isVector())
2932  ShiftAmountTy = VT;
2933  unsigned BitsDiff = VT.getScalarSizeInBits() -
2934  ExtraVT.getScalarSizeInBits();
2935  SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
2936  Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
2937  Node->getOperand(0), ShiftCst);
2938  Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
2939  Results.push_back(Tmp1);
2940  break;
2941  }
2942  case ISD::FP_ROUND_INREG: {
2943  // The only way we can lower this is to turn it into a TRUNCSTORE,
2944  // EXTLOAD pair, targeting a temporary location (a stack slot).
2945 
2946  // NOTE: there is a choice here between constantly creating new stack
2947  // slots and always reusing the same one. We currently always create
2948  // new ones, as reuse may inhibit scheduling.
2949  EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2950  Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT,
2951  Node->getValueType(0), dl);
2952  Results.push_back(Tmp1);
2953  break;
2954  }
2955  case ISD::SINT_TO_FP:
2956  case ISD::UINT_TO_FP:
2957  Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP,
2958  Node->getOperand(0), Node->getValueType(0), dl);
2959  Results.push_back(Tmp1);
2960  break;
2961  case ISD::FP_TO_SINT:
2962  if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
2963  Results.push_back(Tmp1);
2964  break;
2965  case ISD::FP_TO_UINT: {
2966  SDValue True, False;
2967  EVT VT = Node->getOperand(0).getValueType();
2968  EVT NVT = Node->getValueType(0);
2969  APFloat apf(DAG.EVTToAPFloatSemantics(VT),
2972  (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven);
2973  Tmp1 = DAG.getConstantFP(apf, dl, VT);
2974  Tmp2 = DAG.getSetCC(dl, getSetCCResultType(VT),
2975  Node->getOperand(0),
2976  Tmp1, ISD::SETLT);
2977  True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0));
2978  // TODO: Should any fast-math-flags be set for the FSUB?
2979  False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT,
2980  DAG.getNode(ISD::FSUB, dl, VT,
2981  Node->getOperand(0), Tmp1));
2982  False = DAG.getNode(ISD::XOR, dl, NVT, False,
2983  DAG.getConstant(x, dl, NVT));
2984  Tmp1 = DAG.getSelect(dl, NVT, Tmp2, True, False);
2985  Results.push_back(Tmp1);
2986  break;
2987  }
2988  case ISD::VAARG:
2989  Results.push_back(DAG.expandVAArg(Node));
2990  Results.push_back(Results[0].getValue(1));
2991  break;
2992  case ISD::VACOPY:
2993  Results.push_back(DAG.expandVACopy(Node));
2994  break;
2996  if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
2997  // This must be an access of the only element. Return it.
2998  Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
2999  Node->getOperand(0));
3000  else
3001  Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
3002  Results.push_back(Tmp1);
3003  break;
3005  Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
3006  break;
3007  case ISD::INSERT_SUBVECTOR:
3008  Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
3009  break;
3010  case ISD::CONCAT_VECTORS: {
3011  Results.push_back(ExpandVectorBuildThroughStack(Node));
3012  break;
3013  }
3014  case ISD::SCALAR_TO_VECTOR:
3015  Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
3016  break;
3018  Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
3019  Node->getOperand(1),
3020  Node->getOperand(2), dl));
3021  break;
3022  case ISD::VECTOR_SHUFFLE: {
3023  SmallVector<int, 32> NewMask;
3024  ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
3025 
3026  EVT VT = Node->getValueType(0);
3027  EVT EltVT = VT.getVectorElementType();
3028  SDValue Op0 = Node->getOperand(0);
3029  SDValue Op1 = Node->getOperand(1);
3030  if (!TLI.isTypeLegal(EltVT)) {
3031 
3032  EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3033 
3034  // BUILD_VECTOR operands are allowed to be wider than the element type.
3035  // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3036  // it.
3037  if (NewEltVT.bitsLT(EltVT)) {
3038 
3039  // Convert shuffle node.
3040  // If original node was v4i64 and the new EltVT is i32,
3041  // cast operands to v8i32 and re-build the mask.
3042 
3043  // Calculate new VT, the size of the new VT should be equal to original.
3044  EVT NewVT =
3045  EVT::getVectorVT(*DAG.getContext(), NewEltVT,
3046  VT.getSizeInBits() / NewEltVT.getSizeInBits());
3047  assert(NewVT.bitsEq(VT));
3048 
3049  // cast operands to new VT
3050  Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
3051  Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
3052 
3053  // Convert the shuffle mask
3054  unsigned int factor =
3056 
3057  // EltVT gets smaller
3058  assert(factor > 0);
3059 
3060  for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3061  if (Mask[i] < 0) {
3062  for (unsigned fi = 0; fi < factor; ++fi)
3063  NewMask.push_back(Mask[i]);
3064  }
3065  else {
3066  for (unsigned fi = 0; fi < factor; ++fi)
3067  NewMask.push_back(Mask[i]*factor+fi);
3068  }
3069  }
3070  Mask = NewMask;
3071  VT = NewVT;
3072  }
3073  EltVT = NewEltVT;
3074  }
3075  unsigned NumElems = VT.getVectorNumElements();
3077  for (unsigned i = 0; i != NumElems; ++i) {
3078  if (Mask[i] < 0) {
3079  Ops.push_back(DAG.getUNDEF(EltVT));
3080  continue;
3081  }
3082  unsigned Idx = Mask[i];
3083  if (Idx < NumElems)
3084  Ops.push_back(DAG.getNode(
3085  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
3086  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
3087  else
3088  Ops.push_back(DAG.getNode(
3089  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
3090  DAG.getConstant(Idx - NumElems, dl,
3091  TLI.getVectorIdxTy(DAG.getDataLayout()))));
3092  }
3093 
3094  Tmp1 = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
3095  // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3096  Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3097  Results.push_back(Tmp1);
3098  break;
3099  }
3100  case ISD::EXTRACT_ELEMENT: {
3101  EVT OpTy = Node->getOperand(0).getValueType();
3102  if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
3103  // 1 -> Hi
3104  Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3105  DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
3106  TLI.getShiftAmountTy(
3107  Node->getOperand(0).getValueType(),
3108  DAG.getDataLayout())));
3109  Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3110  } else {
3111  // 0 -> Lo
3112  Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3113  Node->getOperand(0));
3114  }
3115  Results.push_back(Tmp1);
3116  break;
3117  }
3118  case ISD::STACKSAVE:
3119  // Expand to CopyFromReg if the target set
3120  // StackPointerRegisterToSaveRestore.
3121  if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3122  Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3123  Node->getValueType(0)));
3124  Results.push_back(Results[0].getValue(1));
3125  } else {
3126  Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3127  Results.push_back(Node->getOperand(0));
3128  }
3129  break;
3130  case ISD::STACKRESTORE:
3131  // Expand to CopyToReg if the target set
3132  // StackPointerRegisterToSaveRestore.
3133  if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3134  Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3135  Node->getOperand(1)));
3136  } else {
3137  Results.push_back(Node->getOperand(0));
3138  }
3139  break;
3141  Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3142  Results.push_back(Results[0].getValue(0));
3143  break;
3144  case ISD::FCOPYSIGN:
3145  Results.push_back(ExpandFCOPYSIGN(Node));
3146  break;
3147  case ISD::FNEG:
3148  // Expand Y = FNEG(X) -> Y = SUB -0.0, X
3149  Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0));
3150  // TODO: If FNEG has fast-math-flags, propagate them to the FSUB.
3151  Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1,
3152  Node->getOperand(0));
3153  Results.push_back(Tmp1);
3154  break;
3155  case ISD::FABS:
3156  Results.push_back(ExpandFABS(Node));
3157  break;
3158  case ISD::SMIN:
3159  case ISD::SMAX:
3160  case ISD::UMIN:
3161  case ISD::UMAX: {
3162  // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3163  ISD::CondCode Pred;
3164  switch (Node->getOpcode()) {
3165  default: llvm_unreachable("How did we get here?");
3166  case ISD::SMAX: Pred = ISD::SETGT; break;
3167  case ISD::SMIN: Pred = ISD::SETLT; break;
3168  case ISD::UMAX: Pred = ISD::SETUGT; break;
3169  case ISD::UMIN: Pred = ISD::SETULT; break;
3170  }
3171  Tmp1 = Node->getOperand(0);
3172  Tmp2 = Node->getOperand(1);
3173  Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3174  Results.push_back(Tmp1);
3175  break;
3176  }
3177 
3178  case ISD::FSIN:
3179  case ISD::FCOS: {
3180  EVT VT = Node->getValueType(0);
3181  // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3182  // fcos which share the same operand and both are used.
3183  if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3184  canCombineSinCosLibcall(Node, TLI, TM))
3185  && useSinCos(Node)) {
3186  SDVTList VTs = DAG.getVTList(VT, VT);
3187  Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3188  if (Node->getOpcode() == ISD::FCOS)
3189  Tmp1 = Tmp1.getValue(1);
3190  Results.push_back(Tmp1);
3191  }
3192  break;
3193  }
3194  case ISD::FMAD:
3195  llvm_unreachable("Illegal fmad should never be formed");
3196 
3197  case ISD::FP16_TO_FP:
3198  if (Node->getValueType(0) != MVT::f32) {
3199  // We can extend to types bigger than f32 in two steps without changing
3200  // the result. Since "f16 -> f32" is much more commonly available, give
3201  // CodeGen the option of emitting that before resorting to a libcall.
3202  SDValue Res =
3203  DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3204  Results.push_back(
3205  DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3206  }
3207  break;
3208  case ISD::FP_TO_FP16:
3209  if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3210  SDValue Op = Node->getOperand(0);
3211  MVT SVT = Op.getSimpleValueType();
3212  if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3213  TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3214  // Under fastmath, we can expand this node into a fround followed by
3215  // a float-half conversion.
3216  SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3217  DAG.getIntPtrConstant(0, dl));
3218  Results.push_back(
3219  DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3220  }
3221  }
3222  break;
3223  case ISD::ConstantFP: {
3224  ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3225  // Check to see if this FP immediate is already legal.
3226  // If this is a legal constant, turn it into a TargetConstantFP node.
3227  if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0)))
3228  Results.push_back(ExpandConstantFP(CFP, true));
3229  break;
3230  }
3231  case ISD::Constant: {
3232  ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3233  Results.push_back(ExpandConstant(CP));
3234  break;
3235  }
3236  case ISD::FSUB: {
3237  EVT VT = Node->getValueType(0);
3238  if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3239  TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3240  const SDNodeFlags *Flags = &cast<BinaryWithFlagsSDNode>(Node)->Flags;
3241  Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3242  Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3243  Results.push_back(Tmp1);
3244  }
3245  break;
3246  }
3247  case ISD::SUB: {
3248  EVT VT = Node->getValueType(0);
3249  assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3250  TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3251  "Don't know how to expand this subtraction!");
3252  Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
3253  DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
3254  VT));
3255  Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3256  Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3257  break;
3258  }
3259  case ISD::UREM:
3260  case ISD::SREM: {
3261  EVT VT = Node->getValueType(0);
3262  bool isSigned = Node->getOpcode() == ISD::SREM;
3263  unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
3264  unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3265  Tmp2 = Node->getOperand(0);
3266  Tmp3 = Node->getOperand(1);
3267  if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3268  SDVTList VTs = DAG.getVTList(VT, VT);
3269  Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1);
3270  Results.push_back(Tmp1);
3271  } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) {
3272  // X % Y -> X-X/Y*Y
3273  Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3);
3274  Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3);
3275  Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1);
3276  Results.push_back(Tmp1);
3277  }
3278  break;
3279  }
3280  case ISD::UDIV:
3281  case ISD::SDIV: {
3282  bool isSigned = Node->getOpcode() == ISD::SDIV;
3283  unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3284  EVT VT = Node->getValueType(0);
3285  if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3286  SDVTList VTs = DAG.getVTList(VT, VT);
3287  Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3288  Node->getOperand(1));
3289  Results.push_back(Tmp1);
3290  }
3291  break;
3292  }
3293  case ISD::MULHU:
3294  case ISD::MULHS: {
3295  unsigned ExpandOpcode =
3297  EVT VT = Node->getValueType(0);
3298  SDVTList VTs = DAG.getVTList(VT, VT);
3299 
3300  Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3301  Node->getOperand(1));
3302  Results.push_back(Tmp1.getValue(1));
3303  break;
3304  }
3305  case ISD::UMUL_LOHI:
3306  case ISD::SMUL_LOHI: {
3307  SDValue LHS = Node->getOperand(0);
3308  SDValue RHS = Node->getOperand(1);
3309  MVT VT = LHS.getSimpleValueType();
3310  unsigned MULHOpcode =
3312 
3313  if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3314  Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3315  Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3316  break;
3317  }
3318 
3319  SmallVector<SDValue, 4> Halves;
3320  EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3321  assert(TLI.isTypeLegal(HalfType));
3322  if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves,
3323  HalfType, DAG,
3325  for (unsigned i = 0; i < 2; ++i) {
3326  SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3327  SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3328  SDValue Shift = DAG.getConstant(
3329  HalfType.getScalarSizeInBits(), dl,
3330  TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3331  Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3332  Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3333  }
3334  break;
3335  }
3336  break;
3337  }
3338  case ISD::MUL: {
3339  EVT VT = Node->getValueType(0);
3340  SDVTList VTs = DAG.getVTList(VT, VT);
3341  // See if multiply or divide can be lowered using two-result operations.
3342  // We just need the low half of the multiply; try both the signed
3343  // and unsigned forms. If the target supports both SMUL_LOHI and
3344  // UMUL_LOHI, form a preference by checking which forms of plain
3345  // MULH it supports.
3346  bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3347  bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3348  bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3349  bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3350  unsigned OpToUse = 0;
3351  if (HasSMUL_LOHI && !HasMULHS) {
3352  OpToUse = ISD::SMUL_LOHI;
3353  } else if (HasUMUL_LOHI && !HasMULHU) {
3354  OpToUse = ISD::UMUL_LOHI;
3355  } else if (HasSMUL_LOHI) {
3356  OpToUse = ISD::SMUL_LOHI;
3357  } else if (HasUMUL_LOHI) {
3358  OpToUse = ISD::UMUL_LOHI;
3359  }
3360  if (OpToUse) {
3361  Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3362  Node->getOperand(1)));
3363  break;
3364  }
3365 
3366  SDValue Lo, Hi;
3367  EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3368  if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3369  TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3370  TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3371  TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3372  TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3374  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3375  Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3376  SDValue Shift =
3377  DAG.getConstant(HalfType.getSizeInBits(), dl,
3378  TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3379  Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3380  Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3381  }
3382  break;
3383  }
3384  case ISD::SADDO:
3385  case ISD::SSUBO: {
3386  SDValue LHS = Node->getOperand(0);
3387  SDValue RHS = Node->getOperand(1);
3388  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3389  ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3390  LHS, RHS);
3391  Results.push_back(Sum);
3392  EVT ResultType = Node->getValueType(1);
3393  EVT OType = getSetCCResultType(Node->getValueType(0));
3394 
3395  SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
3396 
3397  // LHSSign -> LHS >= 0
3398  // RHSSign -> RHS >= 0
3399  // SumSign -> Sum >= 0
3400  //
3401  // Add:
3402  // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3403  // Sub:
3404  // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3405  //
3406  SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3407  SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3408  SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
3409  Node->getOpcode() == ISD::SADDO ?
3411 
3412  SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3413  SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3414 
3415  SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3416  Results.push_back(DAG.getBoolExtOrTrunc(Cmp, dl, ResultType, ResultType));
3417  break;
3418  }
3419  case ISD::UADDO:
3420  case ISD::USUBO: {
3421  SDValue LHS = Node->getOperand(0);
3422  SDValue RHS = Node->getOperand(1);
3423  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ?
3424  ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3425  LHS, RHS);
3426  Results.push_back(Sum);
3427 
3428  EVT ResultType = Node->getValueType(1);
3429  EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3430  ISD::CondCode CC
3431  = Node->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
3432  SDValue SetCC = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3433 
3434  Results.push_back(DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType));
3435  break;
3436  }
3437  case ISD::UMULO:
3438  case ISD::SMULO: {
3439  EVT VT = Node->getValueType(0);
3440  EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2);
3441  SDValue LHS = Node->getOperand(0);
3442  SDValue RHS = Node->getOperand(1);
3443  SDValue BottomHalf;
3444  SDValue TopHalf;
3445  static const unsigned Ops[2][3] =
3448  bool isSigned = Node->getOpcode() == ISD::SMULO;
3449  if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
3450  BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
3451  TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);
3452  } else if (TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
3453  BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,
3454  RHS);
3455  TopHalf = BottomHalf.getValue(1);
3456  } else if (TLI.isTypeLegal(WideVT)) {
3457  LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
3458  RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
3459  Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
3460  BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3461  DAG.getIntPtrConstant(0, dl));
3462  TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3463  DAG.getIntPtrConstant(1, dl));
3464  } else {
3465  // We can fall back to a libcall with an illegal type for the MUL if we
3466  // have a libcall big enough.
3467  // Also, we can fall back to a division in some cases, but that's a big
3468  // performance hit in the general case.
3470  if (WideVT == MVT::i16)
3471  LC = RTLIB::MUL_I16;
3472  else if (WideVT == MVT::i32)
3473  LC = RTLIB::MUL_I32;
3474  else if (WideVT == MVT::i64)
3475  LC = RTLIB::MUL_I64;
3476  else if (WideVT == MVT::i128)
3477  LC = RTLIB::MUL_I128;
3478  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!");
3479 
3480  // The high part is obtained by SRA'ing all but one of the bits of low
3481  // part.
3482  unsigned LoSize = VT.getSizeInBits();
3483  SDValue HiLHS =
3484  DAG.getNode(ISD::SRA, dl, VT, RHS,
3485  DAG.getConstant(LoSize - 1, dl,
3486  TLI.getPointerTy(DAG.getDataLayout())));
3487  SDValue HiRHS =
3488  DAG.getNode(ISD::SRA, dl, VT, LHS,
3489  DAG.getConstant(LoSize - 1, dl,
3490  TLI.getPointerTy(DAG.getDataLayout())));
3491 
3492  // Here we're passing the 2 arguments explicitly as 4 arguments that are
3493  // pre-lowered to the correct types. This all depends upon WideVT not
3494  // being a legal type for the architecture and thus has to be split to
3495  // two arguments.
3496  SDValue Ret;
3497  if(DAG.getDataLayout().isLittleEndian()) {
3498  // Halves of WideVT are packed into registers in different order
3499  // depending on platform endianness. This is usually handled by
3500  // the C calling convention, but we can't defer to it in
3501  // the legalizer.
3502  SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
3503  Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
3504  } else {
3505  SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };
3506  Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
3507  }
3508  BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret,
3509  DAG.getIntPtrConstant(0, dl));
3510  TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret,
3511  DAG.getIntPtrConstant(1, dl));
3512  // Ret is a node with an illegal type. Because such things are not
3513  // generally permitted during this phase of legalization, make sure the
3514  // node has no more uses. The above EXTRACT_ELEMENT nodes should have been
3515  // folded.
3516  assert(Ret->use_empty() &&
3517  "Unexpected uses of illegally type from expanded lib call.");
3518  }
3519 
3520  if (isSigned) {
3521  Tmp1 = DAG.getConstant(
3522  VT.getSizeInBits() - 1, dl,
3523  TLI.getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));
3524  Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1);
3525  TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, Tmp1,
3526  ISD::SETNE);
3527  } else {
3528  TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf,
3529  DAG.getConstant(0, dl, VT), ISD::SETNE);
3530  }
3531 
3532  // Truncate the result if SetCC returns a larger type than needed.
3533  EVT RType = Node->getValueType(1);
3534  if (RType.getSizeInBits() < TopHalf.getValueSizeInBits())
3535  TopHalf = DAG.getNode(ISD::TRUNCATE, dl, RType, TopHalf);
3536 
3537  assert(RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&
3538  "Unexpected result type for S/UMULO legalization");
3539 
3540  Results.push_back(BottomHalf);
3541  Results.push_back(TopHalf);
3542  break;
3543  }
3544  case ISD::BUILD_PAIR: {
3545  EVT PairTy = Node->getValueType(0);
3546  Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3547  Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3548  Tmp2 = DAG.getNode(
3549  ISD::SHL, dl, PairTy, Tmp2,
3550  DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3551  TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3552  Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3553  break;
3554  }
3555  case ISD::SELECT:
3556  Tmp1 = Node->getOperand(0);
3557  Tmp2 = Node->getOperand(1);
3558  Tmp3 = Node->getOperand(2);
3559  if (Tmp1.getOpcode() == ISD::SETCC) {
3560  Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3561  Tmp2, Tmp3,
3562  cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3563  } else {
3564  Tmp1 = DAG.getSelectCC(dl, Tmp1,
3565  DAG.getConstant(0, dl, Tmp1.getValueType()),
3566  Tmp2, Tmp3, ISD::SETNE);
3567  }
3568  Results.push_back(Tmp1);
3569  break;
3570  case ISD::BR_JT: {
3571  SDValue Chain = Node->getOperand(0);
3572  SDValue Table = Node->getOperand(1);
3573  SDValue Index = Node->getOperand(2);
3574 
3575  const DataLayout &TD = DAG.getDataLayout();
3576  EVT PTy = TLI.getPointerTy(TD);
3577 
3578  unsigned EntrySize =
3579  DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3580 
3581  Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3582  DAG.getConstant(EntrySize, dl, Index.getValueType()));
3583  SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3584  Index, Table);
3585 
3586  EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3587  SDValue LD = DAG.getExtLoad(
3588  ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3589  MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3590  Addr = LD;
3591  if (TLI.isJumpTableRelative()) {
3592  // For PIC, the sequence is:
3593  // BRIND(load(Jumptable + index) + RelocBase)
3594  // RelocBase can be JumpTable, GOT or some sort of global base.
3595  Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3596  TLI.getPICJumpTableRelocBase(Table, DAG));
3597  }
3598  Tmp1 = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr);
3599  Results.push_back(Tmp1);
3600  break;
3601  }
3602  case ISD::BRCOND:
3603  // Expand brcond's setcc into its constituent parts and create a BR_CC
3604  // Node.
3605  Tmp1 = Node->getOperand(0);
3606  Tmp2 = Node->getOperand(1);
3607  if (Tmp2.getOpcode() == ISD::SETCC) {
3608  Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
3609  Tmp1, Tmp2.getOperand(2),
3610  Tmp2.getOperand(0), Tmp2.getOperand(1),
3611  Node->getOperand(2));
3612  } else {
3613  // We test only the i1 bit. Skip the AND if UNDEF.
3614  Tmp3 = (Tmp2.isUndef()) ? Tmp2 :
3615  DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3616  DAG.getConstant(1, dl, Tmp2.getValueType()));
3617  Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3618  DAG.getCondCode(ISD::SETNE), Tmp3,
3619  DAG.getConstant(0, dl, Tmp3.getValueType()),
3620  Node->getOperand(2));
3621  }
3622  Results.push_back(Tmp1);
3623  break;
3624  case ISD::SETCC: {
3625  Tmp1 = Node->getOperand(0);
3626  Tmp2 = Node->getOperand(1);
3627  Tmp3 = Node->getOperand(2);
3628  bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2,
3629  Tmp3, NeedInvert, dl);
3630 
3631  if (Legalized) {
3632  // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3633  // condition code, create a new SETCC node.
3634  if (Tmp3.getNode())
3635  Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
3636  Tmp1, Tmp2, Tmp3);
3637 
3638  // If we expanded the SETCC by inverting the condition code, then wrap
3639  // the existing SETCC in a NOT to restore the intended condition.
3640  if (NeedInvert)
3641  Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
3642 
3643  Results.push_back(Tmp1);
3644  break;
3645  }
3646 
3647  // Otherwise, SETCC for the given comparison type must be completely
3648  // illegal; expand it into a SELECT_CC.
3649  EVT VT = Node->getValueType(0);
3650  int TrueValue;
3651  switch (TLI.getBooleanContents(Tmp1->getValueType(0))) {
3654  TrueValue = 1;
3655  break;
3657  TrueValue = -1;
3658  break;
3659  }
3660  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
3661  DAG.getConstant(TrueValue, dl, VT),
3662  DAG.getConstant(0, dl, VT),
3663  Tmp3);
3664  Results.push_back(Tmp1);
3665  break;
3666  }
3667  case ISD::SELECT_CC: {
3668  Tmp1 = Node->getOperand(0); // LHS
3669  Tmp2 = Node->getOperand(1); // RHS
3670  Tmp3 = Node->getOperand(2); // True
3671  Tmp4 = Node->getOperand(3); // False
3672  EVT VT = Node->getValueType(0);
3673  SDValue CC = Node->getOperand(4);
3674  ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
3675 
3676  if (TLI.isCondCodeLegal(CCOp, Tmp1.getSimpleValueType())) {
3677  // If the condition code is legal, then we need to expand this
3678  // node using SETCC and SELECT.
3679  EVT CmpVT = Tmp1.getValueType();
3680  assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
3681  "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
3682  "expanded.");
3683  EVT CCVT =
3684  TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT);
3685  SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC);
3686  Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
3687  break;
3688  }
3689 
3690  // SELECT_CC is legal, so the condition code must not be.
3691  bool Legalized = false;
3692  // Try to legalize by inverting the condition. This is for targets that
3693  // might support an ordered version of a condition, but not the unordered
3694  // version (or vice versa).
3695  ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp,
3696  Tmp1.getValueType().isInteger());
3697  if (TLI.isCondCodeLegal(InvCC, Tmp1.getSimpleValueType())) {
3698  // Use the new condition code and swap true and false
3699  Legalized = true;
3700  Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
3701  } else {
3702  // If The inverse is not legal, then try to swap the arguments using
3703  // the inverse condition code.
3704  ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
3705  if (TLI.isCondCodeLegal(SwapInvCC, Tmp1.getSimpleValueType())) {
3706  // The swapped inverse condition is legal, so swap true and false,
3707  // lhs and rhs.
3708  Legalized = true;
3709  Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
3710  }
3711  }
3712 
3713  if (!Legalized) {
3714  Legalized = LegalizeSetCCCondCode(
3715  getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert,
3716  dl);
3717 
3718  assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
3719 
3720  // If we expanded the SETCC by inverting the condition code, then swap
3721  // the True/False operands to match.
3722  if (NeedInvert)
3723  std::swap(Tmp3, Tmp4);
3724 
3725  // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3726  // condition code, create a new SELECT_CC node.
3727  if (CC.getNode()) {
3728  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
3729  Tmp1, Tmp2, Tmp3, Tmp4, CC);
3730  } else {
3731  Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
3732  CC = DAG.getCondCode(ISD::SETNE);
3733  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
3734  Tmp2, Tmp3, Tmp4, CC);
3735  }
3736  }
3737  Results.push_back(Tmp1);
3738  break;
3739  }
3740  case ISD::BR_CC: {
3741  Tmp1 = Node->getOperand(0); // Chain
3742  Tmp2 = Node->getOperand(2); // LHS
3743  Tmp3 = Node->getOperand(3); // RHS
3744  Tmp4 = Node->getOperand(1); // CC
3745 
3746  bool Legalized = LegalizeSetCCCondCode(getSetCCResultType(
3747  Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl);
3748  (void)Legalized;
3749  assert(Legalized && "Can't legalize BR_CC with legal condition!");
3750 
3751  // If we expanded the SETCC by inverting the condition code, then wrap
3752  // the existing SETCC in a NOT to restore the intended condition.
3753  if (NeedInvert)
3754  Tmp4 = DAG.getNOT(dl, Tmp4, Tmp4->getValueType(0));
3755 
3756  // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
3757  // node.
3758  if (Tmp4.getNode()) {
3759  Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
3760  Tmp4, Tmp2, Tmp3, Node->getOperand(4));
3761  } else {
3762  Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
3763  Tmp4 = DAG.getCondCode(ISD::SETNE);
3764  Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
3765  Tmp2, Tmp3, Node->getOperand(4));
3766  }
3767  Results.push_back(Tmp1);
3768  break;
3769  }
3770  case ISD::BUILD_VECTOR:
3771  Results.push_back(ExpandBUILD_VECTOR(Node));
3772  break;
3773  case ISD::SRA:
3774  case ISD::SRL:
3775  case ISD::SHL: {
3776  // Scalarize vector SRA/SRL/SHL.
3777  EVT VT = Node->getValueType(0);
3778  assert(VT.isVector() && "Unable to legalize non-vector shift");
3779  assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
3780  unsigned NumElem = VT.getVectorNumElements();
3781 
3782  SmallVector<SDValue, 8> Scalars;
3783  for (unsigned Idx = 0; Idx < NumElem; Idx++) {
3784  SDValue Ex = DAG.getNode(
3785  ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0),
3786  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3787  SDValue Sh = DAG.getNode(
3788  ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1),
3789  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3790  Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
3791  VT.getScalarType(), Ex, Sh));
3792  }
3793  SDValue Result =
3794  DAG.getNode(ISD::BUILD_VECTOR, dl, Node->getValueType(0), Scalars);
3795  ReplaceNode(SDValue(Node, 0), Result);
3796  break;
3797  }
3799  case ISD::GlobalAddress:
3800  case ISD::GlobalTLSAddress:
3801  case ISD::ExternalSymbol:
3802  case ISD::ConstantPool:
3803  case ISD::JumpTable:
3806  case ISD::INTRINSIC_VOID:
3807  // FIXME: Custom lowering for these operations shouldn't return null!
3808  break;
3809  }
3810 
3811  // Replace the original node with the legalized result.
3812  if (Results.empty())
3813  return false;
3814 
3815  ReplaceNode(Node, Results.data());
3816  return true;
3817 }
3818 
3819 void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
3821  SDLoc dl(Node);
3822  SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3823  unsigned Opc = Node->getOpcode();
3824  switch (Opc) {
3825  case ISD::ATOMIC_FENCE: {
3826  // If the target didn't lower this, lower it to '__sync_synchronize()' call
3827  // FIXME: handle "fence singlethread" more efficiently.
3829 
3831  CLI.setDebugLoc(dl)
3832  .setChain(Node->getOperand(0))
3833  .setCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3834  DAG.getExternalSymbol("__sync_synchronize",
3835  TLI.getPointerTy(DAG.getDataLayout())),
3836  std::move(Args));
3837 
3838  std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3839 
3840  Results.push_back(CallResult.second);
3841  break;
3842  }
3843  // By default, atomic intrinsics are marked Legal and lowered. Targets
3844  // which don't support them directly, however, may want libcalls, in which
3845  // case they mark them Expand, and we get here.
3846  case ISD::ATOMIC_SWAP:
3847  case ISD::ATOMIC_LOAD_ADD:
3848  case ISD::ATOMIC_LOAD_SUB:
3849  case ISD::ATOMIC_LOAD_AND:
3850  case ISD::ATOMIC_LOAD_OR:
3851  case ISD::ATOMIC_LOAD_XOR:
3852  case ISD::ATOMIC_LOAD_NAND:
3853  case ISD::ATOMIC_LOAD_MIN:
3854  case ISD::ATOMIC_LOAD_MAX:
3855  case ISD::ATOMIC_LOAD_UMIN:
3856  case ISD::ATOMIC_LOAD_UMAX:
3857  case ISD::ATOMIC_CMP_SWAP: {
3858  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
3859  RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
3860  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
3861 
3862  std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false);
3863  Results.push_back(Tmp.first);
3864  Results.push_back(Tmp.second);
3865  break;
3866  }
3867  case ISD::TRAP: {
3868  // If this operation is not supported, lower it to 'abort()' call
3871  CLI.setDebugLoc(dl)
3872  .setChain(Node->getOperand(0))
3873  .setCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3874  DAG.getExternalSymbol("abort",
3875  TLI.getPointerTy(DAG.getDataLayout())),
3876  std::move(Args));
3877  std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3878 
3879  Results.push_back(CallResult.second);
3880  break;
3881  }
3882  case ISD::FMINNUM:
3883  Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
3886  break;
3887  case ISD::FMAXNUM:
3888  Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
3891  break;
3892  case ISD::FSQRT:
3893  Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
3896  break;
3897  case ISD::FSIN:
3898  Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
3901  break;
3902  case ISD::FCOS:
3903  Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
3906  break;
3907  case ISD::FSINCOS:
3908  // Expand into sincos libcall.
3909  ExpandSinCosLibCall(Node, Results);
3910  break;
3911  case ISD::FLOG:
3912  Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64,
3915  break;
3916  case ISD::FLOG2:
3917  Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64,
3920  break;
3921  case ISD::FLOG10:
3922  Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64,
3925  break;
3926  case ISD::FEXP:
3927  Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64,
3930  break;
3931  case ISD::FEXP2:
3932  Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64,
3935  break;
3936  case ISD::FTRUNC:
3937  Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
3940  break;
3941  case ISD::FFLOOR:
3942  Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
3945  break;
3946  case ISD::FCEIL:
3947  Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
3950  break;
3951  case ISD::FRINT:
3952  Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
3955  break;
3956  case ISD::FNEARBYINT:
3957  Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
3962  break;
3963  case ISD::FROUND:
3964  Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32,
3969  break;
3970  case ISD::FPOWI:
3971  Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64,
3974  break;
3975  case ISD::FPOW:
3976  Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64,
3979  break;
3980  case ISD::FDIV:
3981  Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
3984  break;
3985  case ISD::FREM:
3986  Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
3989  break;
3990  case ISD::FMA:
3991  Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
3994  break;
3995  case ISD::FADD:
3996  Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
3999  break;
4000  case ISD::FMUL:
4001  Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4004  break;
4005  case ISD::FP16_TO_FP:
4006  if (Node->getValueType(0) == MVT::f32) {
4007  Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
4008  }
4009  break;
4010  case ISD::FP_TO_FP16: {
4011  RTLIB::Libcall LC =
4013  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
4014  Results.push_back(ExpandLibCall(LC, Node, false));
4015  break;
4016  }
4017  case ISD::FSUB:
4018  Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4021  break;
4022  case ISD::SREM:
4023  Results.push_back(ExpandIntLibCall(Node, true,
4027  break;
4028  case ISD::UREM:
4029  Results.push_back(ExpandIntLibCall(Node, false,
4033  break;
4034  case ISD::SDIV:
4035  Results.push_back(ExpandIntLibCall(Node, true,
4039  break;
4040  case ISD::UDIV:
4041  Results.push_back(ExpandIntLibCall(Node, false,
4045  break;
4046  case ISD::SDIVREM:
4047  case ISD::UDIVREM:
4048  // Expand into divrem libcall
4049  ExpandDivRemLibCall(Node, Results);
4050  break;
4051  case ISD::MUL:
4052  Results.push_back(ExpandIntLibCall(Node, false,
4053  RTLIB::MUL_I8,
4056  break;
4057  }
4058 
4059  // Replace the original node with the legalized result.
4060  if (!Results.empty())
4061  ReplaceNode(Node, Results.data());
4062 }
4063 
4064 // Determine the vector type to use in place of an original scalar element when
4065 // promoting equally sized vectors.
4067  MVT EltVT, MVT NewEltVT) {
4068  unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4069  MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
4070  assert(TLI.isTypeLegal(MidVT) && "unexpected");
4071  return MidVT;
4072 }
4073 
4074 void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4076  MVT OVT = Node->getSimpleValueType(0);
4077  if (Node->getOpcode() == ISD::UINT_TO_FP ||
4078  Node->getOpcode() == ISD::SINT_TO_FP ||
4079  Node->getOpcode() == ISD::SETCC ||
4080  Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4081  Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4082  OVT = Node->getOperand(0).getSimpleValueType();
4083  }
4084  if (Node->getOpcode() == ISD::BR_CC)
4085  OVT = Node->getOperand(2).getSimpleValueType();
4086  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4087  SDLoc dl(Node);
4088  SDValue Tmp1, Tmp2, Tmp3;
4089  switch (Node->getOpcode()) {
4090  case ISD::CTTZ:
4091  case ISD::CTTZ_ZERO_UNDEF:
4092  case ISD::CTLZ:
4093  case ISD::CTLZ_ZERO_UNDEF:
4094  case ISD::CTPOP:
4095  // Zero extend the argument.
4096  Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4097  if (Node->getOpcode() == ISD::CTTZ) {
4098  // The count is the same in the promoted type except if the original
4099  // value was zero. This can be handled by setting the bit just off
4100  // the top of the original type.
4101  auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
4102  OVT.getSizeInBits());
4103  Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
4104  DAG.getConstant(TopBit, dl, NVT));
4105  }
4106  // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4107  // already the correct result.
4108  Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4109  if (Node->getOpcode() == ISD::CTLZ ||
4110  Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
4111  // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4112  Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
4113  DAG.getConstant(NVT.getSizeInBits() -
4114  OVT.getSizeInBits(), dl, NVT));
4115  }
4116  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4117  break;
4118  case ISD::BITREVERSE:
4119  case ISD::BSWAP: {
4120  unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
4121  Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4122  Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4123  Tmp1 = DAG.getNode(
4124  ISD::SRL, dl, NVT, Tmp1,
4125  DAG.getConstant(DiffBits, dl,
4126  TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4127  Results.push_back(Tmp1);
4128  break;
4129  }
4130  case ISD::FP_TO_UINT:
4131  case ISD::FP_TO_SINT:
4132  Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0),
4133  Node->getOpcode() == ISD::FP_TO_SINT, dl);
4134  Results.push_back(Tmp1);
4135  break;
4136  case ISD::UINT_TO_FP:
4137  case ISD::SINT_TO_FP:
4138  Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0),
4139  Node->getOpcode() == ISD::SINT_TO_FP, dl);
4140  Results.push_back(Tmp1);
4141  break;
4142  case ISD::VAARG: {
4143  SDValue Chain = Node->getOperand(0); // Get the chain.
4144  SDValue Ptr = Node->getOperand(1); // Get the pointer.
4145 
4146  unsigned TruncOp;
4147  if (OVT.isVector()) {
4148  TruncOp = ISD::BITCAST;
4149  } else {
4150  assert(OVT.isInteger()
4151  && "VAARG promotion is supported only for vectors or integer types");
4152  TruncOp = ISD::TRUNCATE;
4153  }
4154 
4155  // Perform the larger operation, then convert back
4156  Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
4157  Node->getConstantOperandVal(3));
4158  Chain = Tmp1.getValue(1);
4159 
4160  Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
4161 
4162  // Modified the chain result - switch anything that used the old chain to
4163  // use the new one.
4164  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
4165  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
4166  if (UpdatedNodes) {
4167  UpdatedNodes->insert(Tmp2.getNode());
4168  UpdatedNodes->insert(Chain.getNode());
4169  }
4170  ReplacedNode(Node);
4171  break;
4172  }
4173  case ISD::SDIV:
4174  case ISD::SREM:
4175  case ISD::UDIV:
4176  case ISD::UREM:
4177  case ISD::AND:
4178  case ISD::OR:
4179  case ISD::XOR: {
4180  unsigned ExtOp, TruncOp;
4181  if (OVT.isVector()) {
4182  ExtOp = ISD::BITCAST;
4183  TruncOp = ISD::BITCAST;
4184  } else {
4185  assert(OVT.isInteger() && "Cannot promote logic operation");
4186 
4187  switch (Node->getOpcode()) {
4188  default:
4189  ExtOp = ISD::ANY_EXTEND;
4190  break;
4191  case ISD::SDIV:
4192  case ISD::SREM:
4193  ExtOp = ISD::SIGN_EXTEND;
4194  break;
4195  case ISD::UDIV:
4196  case ISD::UREM:
4197  ExtOp = ISD::ZERO_EXTEND;
4198  break;
4199  }
4200  TruncOp = ISD::TRUNCATE;
4201  }
4202  // Promote each of the values to the new type.
4203  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4204  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4205  // Perform the larger operation, then convert back
4206  Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4207  Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
4208  break;
4209  }
4210  case ISD::UMUL_LOHI:
4211  case ISD::SMUL_LOHI: {
4212  // Promote to a multiply in a wider integer type.
4213  unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
4214  : ISD::SIGN_EXTEND;
4215  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4216  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4217  Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
4218 
4219  auto &DL = DAG.getDataLayout();
4220  unsigned OriginalSize = OVT.getScalarSizeInBits();
4221  Tmp2 = DAG.getNode(
4222  ISD::SRL, dl, NVT, Tmp1,
4223  DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
4224  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4225  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
4226  break;
4227  }
4228  case ISD::SELECT: {
4229  unsigned ExtOp, TruncOp;
4230  if (Node->getValueType(0).isVector() ||
4231  Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
4232  ExtOp = ISD::BITCAST;
4233  TruncOp = ISD::BITCAST;
4234  } else if (Node->getValueType(0).isInteger()) {
4235  ExtOp = ISD::ANY_EXTEND;
4236  TruncOp = ISD::TRUNCATE;
4237  } else {
4238  ExtOp = ISD::FP_EXTEND;
4239  TruncOp = ISD::FP_ROUND;
4240  }
4241  Tmp1 = Node->getOperand(0);
4242  // Promote each of the values to the new type.
4243  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4244  Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4245  // Perform the larger operation, then round down.
4246  Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
4247  if (TruncOp != ISD::FP_ROUND)
4248  Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
4249  else
4250  Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
4251  DAG.getIntPtrConstant(0, dl));
4252  Results.push_back(Tmp1);
4253  break;
4254  }
4255  case ISD::VECTOR_SHUFFLE: {
4256  ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
4257 
4258  // Cast the two input vectors.
4259  Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
4260  Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
4261 
4262  // Convert the shuffle mask to the right # elements.
4263  Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
4264  Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
4265  Results.push_back(Tmp1);
4266  break;
4267  }
4268  case ISD::SETCC: {
4269  unsigned ExtOp = ISD::FP_EXTEND;
4270  if (NVT.isInteger()) {
4271  ISD::CondCode CCCode =
4272  cast<CondCodeSDNode>(Node->getOperand(2))->get();
4274  }
4275  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4276  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4277  Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
4278  Tmp1, Tmp2, Node->getOperand(2)));
4279  break;
4280  }
4281  case ISD::BR_CC: {
4282  unsigned ExtOp = ISD::FP_EXTEND;
4283  if (NVT.isInteger()) {
4284  ISD::CondCode CCCode =
4285  cast<CondCodeSDNode>(Node->getOperand(1))->get();
4287  }
4288  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4289  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4290  Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
4291  Node->getOperand(0), Node->getOperand(1),
4292  Tmp1, Tmp2, Node->getOperand(4)));
4293  break;
4294  }
4295  case ISD::FADD:
4296  case ISD::FSUB:
4297  case ISD::FMUL:
4298  case ISD::FDIV:
4299  case ISD::FREM:
4300  case ISD::FMINNUM:
4301  case ISD::FMAXNUM:
4302  case ISD::FPOW: {
4303  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4304  Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4305  Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
4306  Node->getFlags());
4307  Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4308  Tmp3, DAG.getIntPtrConstant(0, dl)));
4309  break;
4310  }
4311  case ISD::FMA: {
4312  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4313  Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4314  Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
4315  Results.push_back(
4316  DAG.getNode(ISD::FP_ROUND, dl, OVT,
4317  DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
4318  DAG.getIntPtrConstant(0, dl)));
4319  break;
4320  }
4321  case ISD::FCOPYSIGN:
4322  case ISD::FPOWI: {
4323  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4324  Tmp2 = Node->getOperand(1);
4325  Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4326 
4327  // fcopysign doesn't change anything but the sign bit, so
4328  // (fp_round (fcopysign (fpext a), b))
4329  // is as precise as
4330  // (fp_round (fpext a))
4331  // which is a no-op. Mark it as a TRUNCating FP_ROUND.
4332  const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
4333  Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4334  Tmp3, DAG.getIntPtrConstant(isTrunc, dl)));
4335  break;
4336  }
4337  case ISD::FFLOOR:
4338  case ISD::FCEIL:
4339  case ISD::FRINT:
4340  case ISD::FNEARBYINT:
4341  case ISD::FROUND:
4342  case ISD::FTRUNC:
4343  case ISD::FNEG:
4344  case ISD::FSQRT:
4345  case ISD::FSIN:
4346  case ISD::FCOS:
4347  case ISD::FLOG:
4348  case ISD::FLOG2:
4349  case ISD::FLOG10:
4350  case ISD::FABS:
4351  case ISD::FEXP:
4352  case ISD::FEXP2: {
4353  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4354  Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4355  Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4356  Tmp2, DAG.getIntPtrConstant(0, dl)));
4357  break;
4358  }
4359  case ISD::BUILD_VECTOR: {
4360  MVT EltVT = OVT.getVectorElementType();
4361  MVT NewEltVT = NVT.getVectorElementType();
4362 
4363  // Handle bitcasts to a different vector type with the same total bit size
4364  //
4365  // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
4366  // =>
4367  // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
4368 
4369  assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4370  "Invalid promote type for build_vector");
4371  assert(NewEltVT.bitsLT(EltVT) && "not handled");
4372 
4373  MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4374 
4375  SmallVector<SDValue, 8> NewOps;
4376  for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
4377  SDValue Op = Node->getOperand(I);
4378  NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
4379  }
4380 
4381  SDLoc SL(Node);
4382  SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps);
4383  SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4384  Results.push_back(CvtVec);
4385  break;
4386  }
4387  case ISD::EXTRACT_VECTOR_ELT: {
4388  MVT EltVT = OVT.getVectorElementType();
4389  MVT NewEltVT = NVT.getVectorElementType();
4390 
4391  // Handle bitcasts to a different vector type with the same total bit size.
4392  //
4393  // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
4394  // =>
4395  // v4i32:castx = bitcast x:v2i64
4396  //
4397  // i64 = bitcast
4398  // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
4399  // (i32 (extract_vector_elt castx, (2 * y + 1)))
4400  //
4401 
4402  assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4403  "Invalid promote type for extract_vector_elt");
4404  assert(NewEltVT.bitsLT(EltVT) && "not handled");
4405 
4406  MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4407  unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4408 
4409  SDValue Idx = Node->getOperand(1);
4410  EVT IdxVT = Idx.getValueType();
4411  SDLoc SL(Node);
4412  SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
4413  SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4414 
4415  SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4416 
4417  SmallVector<SDValue, 8> NewOps;
4418  for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4419  SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4420  SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4421 
4422  SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4423  CastVec, TmpIdx);
4424  NewOps.push_back(Elt);
4425  }
4426 
4427  SDValue NewVec = DAG.getNode(ISD::BUILD_VECTOR, SL, MidVT, NewOps);
4428 
4429  Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
4430  break;
4431  }
4432  case ISD::INSERT_VECTOR_ELT: {
4433  MVT EltVT = OVT.getVectorElementType();
4434  MVT NewEltVT = NVT.getVectorElementType();
4435 
4436  // Handle bitcasts to a different vector type with the same total bit size
4437  //
4438  // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
4439  // =>
4440  // v4i32:castx = bitcast x:v2i64
4441  // v2i32:casty = bitcast y:i64
4442  //
4443  // v2i64 = bitcast
4444  // (v4i32 insert_vector_elt
4445  // (v4i32 insert_vector_elt v4i32:castx,
4446  // (extract_vector_elt casty, 0), 2 * z),
4447  // (extract_vector_elt casty, 1), (2 * z + 1))
4448 
4449  assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4450  "Invalid promote type for insert_vector_elt");
4451  assert(NewEltVT.bitsLT(EltVT) && "not handled");
4452 
4453  MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4454  unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4455 
4456  SDValue Val = Node->getOperand(1);
4457  SDValue Idx = Node->getOperand(2);
4458  EVT IdxVT = Idx.getValueType();
4459  SDLoc SL(Node);
4460 
4461  SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
4462  SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4463 
4464  SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4465  SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4466 
4467  SDValue NewVec = CastVec;
4468  for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4469  SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4470  SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4471 
4472  SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4473  CastVal, IdxOffset);
4474 
4475  NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
4476  NewVec, Elt, InEltIdx);
4477  }
4478 
4479  Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
4480  break;
4481  }
4482  case ISD::SCALAR_TO_VECTOR: {
4483  MVT EltVT = OVT.getVectorElementType();
4484  MVT NewEltVT = NVT.getVectorElementType();
4485 
4486  // Handle bitcasts to different vector type with the same total bit size.
4487  //
4488  // e.g. v2i64 = scalar_to_vector x:i64
4489  // =>
4490  // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
4491  //
4492 
4493  MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4494  SDValue Val = Node->getOperand(0);
4495  SDLoc SL(Node);
4496 
4497  SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4498  SDValue Undef = DAG.getUNDEF(MidVT);
4499 
4500  SmallVector<SDValue, 8> NewElts;
4501  NewElts.push_back(CastVal);
4502  for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
4503  NewElts.push_back(Undef);
4504 
4505  SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
4506  SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4507  Results.push_back(CvtVec);
4508  break;
4509  }
4510  }
4511 
4512  // Replace the original node with the legalized result.
4513  if (!Results.empty())
4514  ReplaceNode(Node, Results.data());
4515 }
4516 
4517 /// This is the entry point for the file.
4520 
4521  SmallPtrSet<SDNode *, 16> LegalizedNodes;
4522  SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
4523 
4524  // Visit all the nodes. We start in topological order, so that we see
4525  // nodes with their original operands intact. Legalization can produce
4526  // new nodes which may themselves need to be legalized. Iterate until all
4527  // nodes have been legalized.
4528  for (;;) {
4529  bool AnyLegalized = false;
4530  for (auto NI = allnodes_end(); NI != allnodes_begin();) {
4531  --NI;
4532 
4533  SDNode *N = &*NI;
4534  if (N->use_empty() && N != getRoot().getNode()) {
4535  ++NI;
4536  DeleteNode(N);
4537  continue;
4538  }
4539 
4540  if (LegalizedNodes.insert(N).second) {
4541  AnyLegalized = true;
4542  Legalizer.LegalizeOp(N);
4543 
4544  if (N->use_empty() && N != getRoot().getNode()) {
4545  ++NI;
4546  DeleteNode(N);
4547  }
4548  }
4549  }
4550  if (!AnyLegalized)
4551  break;
4552 
4553  }
4554 
4555  // Remove dead nodes now.
4556  RemoveDeadNodes();
4557 }
4558 
4560  SmallSetVector<SDNode *, 16> &UpdatedNodes) {
4561  SmallPtrSet<SDNode *, 16> LegalizedNodes;
4562  SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
4563 
4564  // Directly insert the node in question, and legalize it. This will recurse
4565  // as needed through operands.
4566  LegalizedNodes.insert(N);
4567  Legalizer.LegalizeOp(N);
4568 
4569  return LegalizedNodes.count(N);
4570 }
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
bool LegalizeOp(SDNode *N, SmallSetVector< SDNode *, 16 > &UpdatedNodes)
Transforms a SelectionDAG node and any operands to it into a node that is compatible with the target ...
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
Definition: ISDOpcodes.h:673
static Constant * getFPTrunc(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:1601
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:500
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:467
bool use_empty() const
Return true if there are no uses of this node.
void push_back(const T &Elt)
Definition: SmallVector.h:211
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:762
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:102
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:524
SDValue getValue(unsigned R) const
static APInt getSignBit(unsigned BitWidth)
Get the SignBit for a specific bit width.
Definition: APInt.h:451
static MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant, which is required to be operand #1) half of the integer or float value specified as operand #0.
Definition: ISDOpcodes.h:184
Flags getFlags() const
Return the raw flags of the source value,.
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
Definition: APInt.h:458
NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:304
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:572
size_t i
static MVT getVectorVT(MVT VT, unsigned NumElements)
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:313
SDVTList getVTList() const
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none...
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:329
unsigned getScalarSizeInBits() const
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain...
Definition: ISDOpcodes.h:615
size_type count(PtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:380
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:237
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
Function Alias Analysis Results
Type * getTypeForEVT(LLVMContext &Context) const
getTypeForEVT - This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:204
unsigned getSizeInBits() const
bool isSignaling() const
Definition: APFloat.h:1037
static MVT getPromotedVectorElementType(const TargetLowering &TLI, MVT EltVT, MVT NewEltVT)
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:711
unsigned getNumOperands() const
Return the number of values used by this operation.
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.cpp:238
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned Num) const
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:170
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:330
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Same for subtraction.
Definition: ISDOpcodes.h:240
void reserve(size_type N)
Definition: SmallVector.h:377
void DeleteNode(SDNode *N)
Remove the specified node from the system.
const SDValue & getBasePtr() const
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1 at the ...
Definition: ISDOpcodes.h:299
The address of the GOT.
Definition: ISDOpcodes.h:66
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
Definition: ISDOpcodes.h:690
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:369
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
bool bitsLT(EVT VT) const
bitsLT - Return true if this has less bits than VT.
Definition: ValueTypes.h:212
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:159
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:345
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist)
Returns true if N is a predecessor of any node in Worklist.
const Triple & getTargetTriple() const
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic...
Definition: ISDOpcodes.h:114
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:209
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations...
Definition: ISDOpcodes.h:388
bool isVector() const
isVector - Return true if this is a vector value type.
Definition: ValueTypes.h:133
struct fuzzer::@269 Flags
Shift and rotation operations.
Definition: ISDOpcodes.h:344
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition: APInt.cpp:1122
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:190
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:32
static Constant * get(ArrayRef< Constant * > V)
Definition: Constants.cpp:994
FLT_ROUNDS_ - Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest 2 Round to ...
Definition: ISDOpcodes.h:475
MachinePointerInfo getWithOffset(int64_t O) const
SimpleValueType SimpleTy
APInt bitcastToAPInt() const
Definition: APFloat.h:1012
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence, and carry arbitrary information that target might want to know.
Definition: ISDOpcodes.h:622
MVT getScalarType() const
getScalarType - If this is a vector type, return the element type, otherwise return this...
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA)...
Definition: ISDOpcodes.h:96
EVT getScalarType() const
getScalarType - If this is a vector type, return the element type, otherwise return this...
Definition: ValueTypes.h:233
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt) For double-word atomic operations: ValLo, ValHi, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amtLo, amtHi) ValLo, ValHi, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amtLo, amtHi) These correspond to the atomicrmw instruction.
Definition: ISDOpcodes.h:719
size_type size() const
Definition: SmallSet.h:59
allnodes_const_iterator allnodes_end() const
Definition: SelectionDAG.h:362
bool bitsGE(EVT VT) const
bitsGE - Return true if this has no less bits than VT.
Definition: ValueTypes.h:206
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG...
Definition: ISDOpcodes.h:73
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
Definition: ValueTypes.h:123
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:60
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:611
FRAME_TO_ARGS_OFFSET - This node represents offset from frame pointer to first (possible) on-stack ar...
Definition: ISDOpcodes.h:91
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:154
EVT getVectorElementType() const
getVectorElementType - Given a vector type, return the type of each element.
Definition: ValueTypes.h:239
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
static unsigned getAlignment(GlobalVariable *GV)
#define F(x, y, z)
Definition: MD5.cpp:51
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:410
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here...
Definition: ISDOpcodes.h:118
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:220
const SDValue & getBasePtr() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
getHalfSizedIntegerVT - Finds the smallest simple value type that is greater than or equal to half th...
Definition: ValueTypes.h:293
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:737
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:842
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification, or lowering of the constant.
Definition: ISDOpcodes.h:125
EVT getMemoryVT() const
Return the type of the in-memory value.
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:656
static bool useSinCos(SDNode *Node)
Only issue sincos libcall if both sin and cos are needed.
const ConstantInt * getConstantIntValue() const
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
Definition: ISDOpcodes.h:875
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:151
Maximum length of the test input libFuzzer tries to guess a good value based on the corpus and reports it always prefer smaller inputs during the corpus shuffle When libFuzzer itself reports a bug this exit code will be used If indicates the maximal total time in seconds to run the fuzzer minimizes the provided crash input Use with etc Experimental Use value profile to guide fuzzing Number of simultaneous worker processes to run the jobs If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
UNDEF - An undefined node.
Definition: ISDOpcodes.h:178
This class is used to represent ISD::STORE nodes.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:453
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:274
SDNode * getNode() const
get the SDNode which holds the desired result
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
MinAlign - A and B are either alignments or offsets.
Definition: MathExtras.h:589
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:262
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:166
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
unsigned getStoreSizeInBits() const
getStoreSizeInBits - Return the number of bits overwritten by a store of the specified value type...
Definition: ValueTypes.h:274
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
virtual bool isShuffleMaskLegal(const SmallVectorImpl< int > &, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations, those with specific masks.
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition: ISDOpcodes.h:85
unsigned getVectorNumElements() const
APInt trunc(unsigned width) const
Truncate to new width.
Definition: APInt.cpp:916
void Legalize()
This transforms the SelectionDAG into a SelectionDAG that is compatible with the target instruction s...
constexpr bool isPowerOf2_32(uint32_t Value)
isPowerOf2_32 - This function returns true if the argument is a power of two > 0. ...
Definition: MathExtras.h:399
MVT - Machine Value Type.
const SDValue & getOperand(unsigned i) const
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
void swap(SmallVectorImpl &RHS)
Definition: SmallVector.h:675
Simple binary floating point operators.
Definition: ISDOpcodes.h:246
C - The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
This is an important base class in LLVM.
Definition: Constant.h:42
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE...
Definition: ISDOpcodes.h:637
bool isVector() const
isVector - Return true if this is a vector value type.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:818
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:279
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:36
This file contains the declarations for the subclasses of Constant, which represent the different fla...
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
Definition: APInt.h:1947
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:269
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:368
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:484
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:154
This class provides iterator support for SDUse operands that use a specific SDNode.
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
Definition: ISDOpcodes.h:667
uint32_t Offset
unsigned getOpcode() const
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:676
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:57
DEBUGTRAP - Trap intended to get the attention of a debugger.
Definition: ISDOpcodes.h:679
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y)...
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:80
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:628
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
bool isVolatile() const
const SDValue & getValue() const
static bool isValueValidForType(EVT VT, const APFloat &Val)
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:350
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo...
Definition: ISDOpcodes.h:705
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:485
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:392
EVT - Extended Value Type.
Definition: ValueTypes.h:31
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1337
std::vector< ArgListEntry > ArgListTy
const APFloat & getValueAPF() const
bool bitsEq(EVT VT) const
bitsEq - Return true if this has the same number of bits as VT.
Definition: ValueTypes.h:194
const ConstantFP * getConstantFPValue() const
This structure contains all information that is necessary for lowering calls.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements)
getVectorVT - Returns the EVT that represents a vector NumElements in length, where each element is o...
Definition: ValueTypes.h:70
This class contains a discriminated union of information about pointers in memory operands...
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:378
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:594
const MachinePointerInfo & getPointerInfo() const
bool bitsGT(EVT VT) const
bitsGT - Return true if this has more bits than VT.
Definition: ValueTypes.h:200
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:50
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:292
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:425
This is the shared class of boolean and integer constants.
Definition: Constants.h:88
void dump() const
Dump this node, for debugging.
allnodes_const_iterator allnodes_begin() const
Definition: SelectionDAG.h:361
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false...
Definition: SmallPtrSet.h:375
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:285
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:166
X = FP_ROUND_INREG(Y, VT) - This operator takes an FP register, and rounds it to a floating point val...
Definition: ISDOpcodes.h:482
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:566
const SDValue & getChain() const
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:625
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:347
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:510
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:558
Represents one node in the SelectionDAG.
CondCode getSetCCInverse(CondCode Operation, bool isInteger)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
double BitsToDouble(uint64_t Bits)
BitsToDouble - This function takes a 64-bit integer and returns the bit equivalent double...
Definition: MathExtras.h:549
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
unsigned Log2_32(uint32_t Value)
Log2_32 - This function returns the floor log base 2 of the specified value, -1 if the value is zero...
Definition: MathExtras.h:513
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:586
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
Definition: APInt.h:550
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Class for arbitrary precision integers.
Definition: APInt.h:77
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:354
op_iterator op_begin() const
static use_iterator use_end()
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:400
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:403
bool isGNUEnvironment() const
Definition: Triple.h:489
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:259
iterator_range< value_op_iterator > op_values() const
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
Definition: APInt.h:1942
Flags
Flags values. These may be or'd together.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
#define Success
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca...
Definition: ISDOpcodes.h:758
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:560
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer, a SRCVALUE for the destination, and a SRCVALUE for the source.
Definition: ISDOpcodes.h:633
These are IR-level optimization flags that may be propagated to SDNodes.
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:333
pointer data()
Return a pointer to the vector's buffer, even if empty().
Definition: SmallVector.h:142
bool isUndef() const
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:205
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:418
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:536
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
Definition: ValueTypes.h:256
void ReplaceAllUsesWith(SDValue From, SDValue Op)
Modify anything using 'From' to use 'To' instead.
#define I(x, y, z)
Definition: MD5.cpp:54
#define N
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:135
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
op_iterator op_end() const
Same for multiplication.
Definition: ISDOpcodes.h:243
static volatile int Zero
bool reachesChainWithoutSideEffects(SDValue Dest, unsigned Depth=2) const
Return true if this operand (which must be a chain) reaches the specified operand without crossing an...
FSINCOS - Compute both fsin and fcos as a single operation.
Definition: ISDOpcodes.h:530
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.setjmp intrinsic.
Definition: ISDOpcodes.h:108
EVT getValueType() const
Return the ValueType of the referenced return value.
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin...
Definition: ISDOpcodes.h:102
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a carry value...
Definition: ISDOpcodes.h:383
const APFloat & getValueAPF() const
Definition: Constants.h:300
bool isByteSized() const
isByteSized - Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:183
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:291
bool isFloatingPoint() const
isFloatingPoint - Return true if this is a FP, or a vector FP type.
Definition: ValueTypes.h:118
This class is used to form a handle around another node that is persistent and is updated across invo...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI)
Return true if sincos libcall is available.
LLVM Value Representation.
Definition: Value.h:71
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:249
static bool ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, const TargetLowering &TLI, SDValue &Res)
EVT changeTypeToInteger()
Return the type converted to an equivalently sized integer or vector with integer element type...
Definition: ValueTypes.h:95
bool isTruncatingStore() const
Return true if the op does a truncation before store.
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:239
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:81
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:685
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:331
#define DEBUG(X)
Definition: Debug.h:100
Primary interface to the complete machine description for the target machine.
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:678
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations...
Definition: ISDOpcodes.h:253
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:377
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:197
MVT getVectorElementType() const
Conversion operators.
Definition: ISDOpcodes.h:397
static APInt getNullValue(unsigned numBits)
Get the '0' value.
Definition: APInt.h:465
int * Ptr
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:698
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:406
unsigned getAlignment() const
static bool canCombineSinCosLibcall(SDNode *Node, const TargetLowering &TLI, const TargetMachine &TM)
Return true if sincos libcall is available and can be used to combine sin and cos.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
bool isBigEndian() const
Definition: DataLayout.h:221
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR - Perform various unary floating point operations.
Definition: ISDOpcodes.h:516
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
getIntegerVT - Returns the EVT that represents an integer with the given number of bits...
Definition: ValueTypes.h:61
Fast - This calling convention attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:42
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:694
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:321
MVT getSimpleVT() const
getSimpleVT - Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:226
const SDNodeFlags * getFlags() const
This could be defined as a virtual function and implemented more simply and directly, but it is not to avoid creating a vtable for this class.
This file describes how to lower LLVM code to machine code.
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
BRIND - Indirect branch.
Definition: ISDOpcodes.h:556
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:326
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:248
This class is used to represent ISD::LOAD nodes.
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary...
Definition: ISDOpcodes.h:545