LLVM  6.0.0svn
LegalizeDAG.cpp
Go to the documentation of this file.
1 //===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the SelectionDAG::Legalize method.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "llvm/ADT/APFloat.h"
15 #include "llvm/ADT/APInt.h"
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/SetVector.h"
18 #include "llvm/ADT/SmallPtrSet.h"
19 #include "llvm/ADT/SmallSet.h"
20 #include "llvm/ADT/SmallVector.h"
33 #include "llvm/IR/CallingConv.h"
34 #include "llvm/IR/Constants.h"
35 #include "llvm/IR/DataLayout.h"
36 #include "llvm/IR/DerivedTypes.h"
37 #include "llvm/IR/Function.h"
38 #include "llvm/IR/Metadata.h"
39 #include "llvm/IR/Type.h"
40 #include "llvm/Support/Casting.h"
41 #include "llvm/Support/Compiler.h"
42 #include "llvm/Support/Debug.h"
48 #include <algorithm>
49 #include <cassert>
50 #include <cstdint>
51 #include <tuple>
52 #include <utility>
53 
54 using namespace llvm;
55 
56 #define DEBUG_TYPE "legalizedag"
57 
58 namespace {
59 
60 /// Keeps track of state when getting the sign of a floating-point value as an
61 /// integer.
62 struct FloatSignAsInt {
63  EVT FloatVT;
64  SDValue Chain;
65  SDValue FloatPtr;
66  SDValue IntPtr;
67  MachinePointerInfo IntPointerInfo;
68  MachinePointerInfo FloatPointerInfo;
69  SDValue IntValue;
70  APInt SignMask;
71  uint8_t SignBit;
72 };
73 
74 //===----------------------------------------------------------------------===//
75 /// This takes an arbitrary SelectionDAG as input and
76 /// hacks on it until the target machine can handle it. This involves
77 /// eliminating value sizes the machine cannot handle (promoting small sizes to
78 /// large sizes or splitting up large values into small values) as well as
79 /// eliminating operations the machine cannot handle.
80 ///
81 /// This code also does a small amount of optimization and recognition of idioms
82 /// as part of its processing. For example, if a target does not support a
83 /// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
84 /// will attempt merge setcc and brc instructions into brcc's.
85 class SelectionDAGLegalize {
86  const TargetMachine &TM;
87  const TargetLowering &TLI;
88  SelectionDAG &DAG;
89 
90  /// \brief The set of nodes which have already been legalized. We hold a
91  /// reference to it in order to update as necessary on node deletion.
92  SmallPtrSetImpl<SDNode *> &LegalizedNodes;
93 
94  /// \brief A set of all the nodes updated during legalization.
95  SmallSetVector<SDNode *, 16> *UpdatedNodes;
96 
97  EVT getSetCCResultType(EVT VT) const {
98  return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
99  }
100 
101  // Libcall insertion helpers.
102 
103 public:
104  SelectionDAGLegalize(SelectionDAG &DAG,
105  SmallPtrSetImpl<SDNode *> &LegalizedNodes,
106  SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
107  : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
108  LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
109 
110  /// \brief Legalizes the given operation.
111  void LegalizeOp(SDNode *Node);
112 
113 private:
114  SDValue OptimizeFloatStore(StoreSDNode *ST);
115 
116  void LegalizeLoadOps(SDNode *Node);
117  void LegalizeStoreOps(SDNode *Node);
118 
119  /// Some targets cannot handle a variable
120  /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
121  /// is necessary to spill the vector being inserted into to memory, perform
122  /// the insert there, and then read the result back.
123  SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
124  const SDLoc &dl);
125  SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
126  const SDLoc &dl);
127 
128  /// Return a vector shuffle operation which
129  /// performs the same shuffe in terms of order or result bytes, but on a type
130  /// whose vector element type is narrower than the original shuffle type.
131  /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
132  SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
133  SDValue N1, SDValue N2,
134  ArrayRef<int> Mask) const;
135 
136  bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC,
137  bool &NeedInvert, const SDLoc &dl);
138 
139  SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
140  SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops,
141  unsigned NumOps, bool isSigned, const SDLoc &dl);
142 
143  std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
144  SDNode *Node, bool isSigned);
145  SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
146  RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
147  RTLIB::Libcall Call_F128,
148  RTLIB::Libcall Call_PPCF128);
149  SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
150  RTLIB::Libcall Call_I8,
151  RTLIB::Libcall Call_I16,
152  RTLIB::Libcall Call_I32,
153  RTLIB::Libcall Call_I64,
154  RTLIB::Libcall Call_I128);
155  void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
156  void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
157 
158  SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
159  const SDLoc &dl);
160  SDValue ExpandBUILD_VECTOR(SDNode *Node);
161  SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
162  void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
163  SmallVectorImpl<SDValue> &Results);
164  void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
165  SDValue Value) const;
166  SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
167  SDValue NewIntValue) const;
168  SDValue ExpandFCOPYSIGN(SDNode *Node) const;
169  SDValue ExpandFABS(SDNode *Node) const;
170  SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue LegalOp, EVT DestVT,
171  const SDLoc &dl);
172  SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned,
173  const SDLoc &dl);
174  SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned,
175  const SDLoc &dl);
176 
177  SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl);
178  SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl);
179  SDValue ExpandBitCount(unsigned Opc, SDValue Op, const SDLoc &dl);
180 
181  SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
182  SDValue ExpandInsertToVectorThroughStack(SDValue Op);
183  SDValue ExpandVectorBuildThroughStack(SDNode* Node);
184 
185  SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
186  SDValue ExpandConstant(ConstantSDNode *CP);
187 
188  // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
189  bool ExpandNode(SDNode *Node);
190  void ConvertNodeToLibcall(SDNode *Node);
191  void PromoteNode(SDNode *Node);
192 
193 public:
194  // Node replacement helpers
195 
196  void ReplacedNode(SDNode *N) {
197  LegalizedNodes.erase(N);
198  if (UpdatedNodes)
199  UpdatedNodes->insert(N);
200  }
201 
202  void ReplaceNode(SDNode *Old, SDNode *New) {
203  DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
204  dbgs() << " with: "; New->dump(&DAG));
205 
206  assert(Old->getNumValues() == New->getNumValues() &&
207  "Replacing one node with another that produces a different number "
208  "of values!");
209  DAG.ReplaceAllUsesWith(Old, New);
210  if (UpdatedNodes)
211  UpdatedNodes->insert(New);
212  ReplacedNode(Old);
213  }
214 
215  void ReplaceNode(SDValue Old, SDValue New) {
216  DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
217  dbgs() << " with: "; New->dump(&DAG));
218 
219  DAG.ReplaceAllUsesWith(Old, New);
220  if (UpdatedNodes)
221  UpdatedNodes->insert(New.getNode());
222  ReplacedNode(Old.getNode());
223  }
224 
225  void ReplaceNode(SDNode *Old, const SDValue *New) {
226  DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
227 
228  DAG.ReplaceAllUsesWith(Old, New);
229  for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
230  DEBUG(dbgs() << (i == 0 ? " with: "
231  : " and: ");
232  New[i]->dump(&DAG));
233  if (UpdatedNodes)
234  UpdatedNodes->insert(New[i].getNode());
235  }
236  ReplacedNode(Old);
237  }
238 };
239 
240 } // end anonymous namespace
241 
242 /// Return a vector shuffle operation which
243 /// performs the same shuffe in terms of order or result bytes, but on a type
244 /// whose vector element type is narrower than the original shuffle type.
245 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
246 SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
247  EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
248  ArrayRef<int> Mask) const {
249  unsigned NumMaskElts = VT.getVectorNumElements();
250  unsigned NumDestElts = NVT.getVectorNumElements();
251  unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
252 
253  assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
254 
255  if (NumEltsGrowth == 1)
256  return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
257 
258  SmallVector<int, 8> NewMask;
259  for (unsigned i = 0; i != NumMaskElts; ++i) {
260  int Idx = Mask[i];
261  for (unsigned j = 0; j != NumEltsGrowth; ++j) {
262  if (Idx < 0)
263  NewMask.push_back(-1);
264  else
265  NewMask.push_back(Idx * NumEltsGrowth + j);
266  }
267  }
268  assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
269  assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
270  return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
271 }
272 
273 /// Expands the ConstantFP node to an integer constant or
274 /// a load from the constant pool.
275 SDValue
276 SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
277  bool Extend = false;
278  SDLoc dl(CFP);
279 
280  // If a FP immediate is precise when represented as a float and if the
281  // target can do an extending load from float to double, we put it into
282  // the constant pool as a float, even if it's is statically typed as a
283  // double. This shrinks FP constants and canonicalizes them for targets where
284  // an FP extending load is the same cost as a normal load (such as on the x87
285  // fp stack or PPC FP unit).
286  EVT VT = CFP->getValueType(0);
287  ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
288  if (!UseCP) {
289  assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
290  return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
291  (VT == MVT::f64) ? MVT::i64 : MVT::i32);
292  }
293 
294  APFloat APF = CFP->getValueAPF();
295  EVT OrigVT = VT;
296  EVT SVT = VT;
297 
298  // We don't want to shrink SNaNs. Converting the SNaN back to its real type
299  // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
300  if (!APF.isSignaling()) {
301  while (SVT != MVT::f32 && SVT != MVT::f16) {
302  SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
304  // Only do this if the target has a native EXTLOAD instruction from
305  // smaller type.
306  TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
307  TLI.ShouldShrinkFPConstant(OrigVT)) {
308  Type *SType = SVT.getTypeForEVT(*DAG.getContext());
309  LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
310  VT = SVT;
311  Extend = true;
312  }
313  }
314  }
315 
316  SDValue CPIdx =
317  DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
318  unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
319  if (Extend) {
320  SDValue Result = DAG.getExtLoad(
321  ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
322  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
323  Alignment);
324  return Result;
325  }
326  SDValue Result = DAG.getLoad(
327  OrigVT, dl, DAG.getEntryNode(), CPIdx,
328  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
329  return Result;
330 }
331 
332 /// Expands the Constant node to a load from the constant pool.
333 SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
334  SDLoc dl(CP);
335  EVT VT = CP->getValueType(0);
336  SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
337  TLI.getPointerTy(DAG.getDataLayout()));
338  unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
339  SDValue Result = DAG.getLoad(
340  VT, dl, DAG.getEntryNode(), CPIdx,
341  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
342  return Result;
343 }
344 
345 /// Some target cannot handle a variable insertion index for the
346 /// INSERT_VECTOR_ELT instruction. In this case, it
347 /// is necessary to spill the vector being inserted into to memory, perform
348 /// the insert there, and then read the result back.
349 SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
350  SDValue Val,
351  SDValue Idx,
352  const SDLoc &dl) {
353  SDValue Tmp1 = Vec;
354  SDValue Tmp2 = Val;
355  SDValue Tmp3 = Idx;
356 
357  // If the target doesn't support this, we have to spill the input vector
358  // to a temporary stack slot, update the element, then reload it. This is
359  // badness. We could also load the value into a vector register (either
360  // with a "move to register" or "extload into register" instruction, then
361  // permute it into place, if the idx is a constant and if the idx is
362  // supported by the target.
363  EVT VT = Tmp1.getValueType();
364  EVT EltVT = VT.getVectorElementType();
365  SDValue StackPtr = DAG.CreateStackTemporary(VT);
366 
367  int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
368 
369  // Store the vector.
370  SDValue Ch = DAG.getStore(
371  DAG.getEntryNode(), dl, Tmp1, StackPtr,
372  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
373 
374  SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
375 
376  // Store the scalar value.
377  Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT);
378  // Load the updated vector.
379  return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
380  DAG.getMachineFunction(), SPFI));
381 }
382 
383 SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
384  SDValue Idx,
385  const SDLoc &dl) {
386  if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
387  // SCALAR_TO_VECTOR requires that the type of the value being inserted
388  // match the element type of the vector being created, except for
389  // integers in which case the inserted value can be over width.
390  EVT EltVT = Vec.getValueType().getVectorElementType();
391  if (Val.getValueType() == EltVT ||
392  (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
393  SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
394  Vec.getValueType(), Val);
395 
396  unsigned NumElts = Vec.getValueType().getVectorNumElements();
397  // We generate a shuffle of InVec and ScVec, so the shuffle mask
398  // should be 0,1,2,3,4,5... with the appropriate element replaced with
399  // elt 0 of the RHS.
400  SmallVector<int, 8> ShufOps;
401  for (unsigned i = 0; i != NumElts; ++i)
402  ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
403 
404  return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
405  }
406  }
407  return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
408 }
409 
410 SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
411  DEBUG(dbgs() << "Optimizing float store operations\n");
412  // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
413  // FIXME: We shouldn't do this for TargetConstantFP's.
414  // FIXME: move this to the DAG Combiner! Note that we can't regress due
415  // to phase ordering between legalized code and the dag combiner. This
416  // probably means that we need to integrate dag combiner and legalizer
417  // together.
418  // We generally can't do this one for long doubles.
419  SDValue Chain = ST->getChain();
420  SDValue Ptr = ST->getBasePtr();
421  unsigned Alignment = ST->getAlignment();
422  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
423  AAMDNodes AAInfo = ST->getAAInfo();
424  SDLoc dl(ST);
425  if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) {
426  if (CFP->getValueType(0) == MVT::f32 &&
427  TLI.isTypeLegal(MVT::i32)) {
428  SDValue Con = DAG.getConstant(CFP->getValueAPF().
429  bitcastToAPInt().zextOrTrunc(32),
430  SDLoc(CFP), MVT::i32);
431  return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), Alignment,
432  MMOFlags, AAInfo);
433  }
434 
435  if (CFP->getValueType(0) == MVT::f64) {
436  // If this target supports 64-bit registers, do a single 64-bit store.
437  if (TLI.isTypeLegal(MVT::i64)) {
438  SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
439  zextOrTrunc(64), SDLoc(CFP), MVT::i64);
440  return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
441  Alignment, MMOFlags, AAInfo);
442  }
443 
444  if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
445  // Otherwise, if the target supports 32-bit registers, use 2 32-bit
446  // stores. If the target supports neither 32- nor 64-bits, this
447  // xform is certainly not worth it.
448  const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
449  SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
450  SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
451  if (DAG.getDataLayout().isBigEndian())
452  std::swap(Lo, Hi);
453 
454  Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), Alignment,
455  MMOFlags, AAInfo);
456  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
457  DAG.getConstant(4, dl, Ptr.getValueType()));
458  Hi = DAG.getStore(Chain, dl, Hi, Ptr,
459  ST->getPointerInfo().getWithOffset(4),
460  MinAlign(Alignment, 4U), MMOFlags, AAInfo);
461 
462  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
463  }
464  }
465  }
466  return SDValue(nullptr, 0);
467 }
468 
469 void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
470  StoreSDNode *ST = cast<StoreSDNode>(Node);
471  SDValue Chain = ST->getChain();
472  SDValue Ptr = ST->getBasePtr();
473  SDLoc dl(Node);
474 
475  unsigned Alignment = ST->getAlignment();
476  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
477  AAMDNodes AAInfo = ST->getAAInfo();
478 
479  if (!ST->isTruncatingStore()) {
480  DEBUG(dbgs() << "Legalizing store operation\n");
481  if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
482  ReplaceNode(ST, OptStore);
483  return;
484  }
485 
486  SDValue Value = ST->getValue();
487  MVT VT = Value.getSimpleValueType();
488  switch (TLI.getOperationAction(ISD::STORE, VT)) {
489  default: llvm_unreachable("This action is not supported yet!");
490  case TargetLowering::Legal: {
491  // If this is an unaligned store and the target doesn't support it,
492  // expand it.
493  EVT MemVT = ST->getMemoryVT();
494  unsigned AS = ST->getAddressSpace();
495  unsigned Align = ST->getAlignment();
496  const DataLayout &DL = DAG.getDataLayout();
497  if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
498  DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
499  SDValue Result = TLI.expandUnalignedStore(ST, DAG);
500  ReplaceNode(SDValue(ST, 0), Result);
501  } else
502  DEBUG(dbgs() << "Legal store\n");
503  break;
504  }
505  case TargetLowering::Custom: {
506  DEBUG(dbgs() << "Trying custom lowering\n");
507  SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
508  if (Res && Res != SDValue(Node, 0))
509  ReplaceNode(SDValue(Node, 0), Res);
510  return;
511  }
513  MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
514  assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
515  "Can only promote stores to same size type");
516  Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
517  SDValue Result =
518  DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
519  Alignment, MMOFlags, AAInfo);
520  ReplaceNode(SDValue(Node, 0), Result);
521  break;
522  }
523  }
524  return;
525  }
526 
527  DEBUG(dbgs() << "Legalizing truncating store operations\n");
528  SDValue Value = ST->getValue();
529  EVT StVT = ST->getMemoryVT();
530  unsigned StWidth = StVT.getSizeInBits();
531  auto &DL = DAG.getDataLayout();
532 
533  if (StWidth != StVT.getStoreSizeInBits()) {
534  // Promote to a byte-sized store with upper bits zero if not
535  // storing an integral number of bytes. For example, promote
536  // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
537  EVT NVT = EVT::getIntegerVT(*DAG.getContext(),
538  StVT.getStoreSizeInBits());
539  Value = DAG.getZeroExtendInReg(Value, dl, StVT);
540  SDValue Result =
541  DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
542  Alignment, MMOFlags, AAInfo);
543  ReplaceNode(SDValue(Node, 0), Result);
544  } else if (StWidth & (StWidth - 1)) {
545  // If not storing a power-of-2 number of bits, expand as two stores.
546  assert(!StVT.isVector() && "Unsupported truncstore!");
547  unsigned RoundWidth = 1 << Log2_32(StWidth);
548  assert(RoundWidth < StWidth);
549  unsigned ExtraWidth = StWidth - RoundWidth;
550  assert(ExtraWidth < RoundWidth);
551  assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
552  "Store size not an integral number of bytes!");
553  EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
554  EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
555  SDValue Lo, Hi;
556  unsigned IncrementSize;
557 
558  if (DL.isLittleEndian()) {
559  // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
560  // Store the bottom RoundWidth bits.
561  Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
562  RoundVT, Alignment, MMOFlags, AAInfo);
563 
564  // Store the remaining ExtraWidth bits.
565  IncrementSize = RoundWidth / 8;
566  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
567  DAG.getConstant(IncrementSize, dl,
568  Ptr.getValueType()));
569  Hi = DAG.getNode(
570  ISD::SRL, dl, Value.getValueType(), Value,
571  DAG.getConstant(RoundWidth, dl,
572  TLI.getShiftAmountTy(Value.getValueType(), DL)));
573  Hi = DAG.getTruncStore(
574  Chain, dl, Hi, Ptr,
575  ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
576  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
577  } else {
578  // Big endian - avoid unaligned stores.
579  // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
580  // Store the top RoundWidth bits.
581  Hi = DAG.getNode(
582  ISD::SRL, dl, Value.getValueType(), Value,
583  DAG.getConstant(ExtraWidth, dl,
584  TLI.getShiftAmountTy(Value.getValueType(), DL)));
585  Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(),
586  RoundVT, Alignment, MMOFlags, AAInfo);
587 
588  // Store the remaining ExtraWidth bits.
589  IncrementSize = RoundWidth / 8;
590  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
591  DAG.getConstant(IncrementSize, dl,
592  Ptr.getValueType()));
593  Lo = DAG.getTruncStore(
594  Chain, dl, Value, Ptr,
595  ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
596  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
597  }
598 
599  // The order of the stores doesn't matter.
600  SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
601  ReplaceNode(SDValue(Node, 0), Result);
602  } else {
603  switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
604  default: llvm_unreachable("This action is not supported yet!");
605  case TargetLowering::Legal: {
606  EVT MemVT = ST->getMemoryVT();
607  unsigned AS = ST->getAddressSpace();
608  unsigned Align = ST->getAlignment();
609  // If this is an unaligned store and the target doesn't support it,
610  // expand it.
611  if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
612  SDValue Result = TLI.expandUnalignedStore(ST, DAG);
613  ReplaceNode(SDValue(ST, 0), Result);
614  }
615  break;
616  }
617  case TargetLowering::Custom: {
618  SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
619  if (Res && Res != SDValue(Node, 0))
620  ReplaceNode(SDValue(Node, 0), Res);
621  return;
622  }
624  assert(!StVT.isVector() &&
625  "Vector Stores are handled in LegalizeVectorOps");
626 
627  // TRUNCSTORE:i16 i32 -> STORE i16
628  assert(TLI.isTypeLegal(StVT) &&
629  "Do not know how to expand this store!");
630  Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
631  SDValue Result =
632  DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
633  Alignment, MMOFlags, AAInfo);
634  ReplaceNode(SDValue(Node, 0), Result);
635  break;
636  }
637  }
638 }
639 
640 void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
641  LoadSDNode *LD = cast<LoadSDNode>(Node);
642  SDValue Chain = LD->getChain(); // The chain.
643  SDValue Ptr = LD->getBasePtr(); // The base pointer.
644  SDValue Value; // The value returned by the load op.
645  SDLoc dl(Node);
646 
647  ISD::LoadExtType ExtType = LD->getExtensionType();
648  if (ExtType == ISD::NON_EXTLOAD) {
649  DEBUG(dbgs() << "Legalizing non-extending load operation\n");
650  MVT VT = Node->getSimpleValueType(0);
651  SDValue RVal = SDValue(Node, 0);
652  SDValue RChain = SDValue(Node, 1);
653 
654  switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
655  default: llvm_unreachable("This action is not supported yet!");
656  case TargetLowering::Legal: {
657  EVT MemVT = LD->getMemoryVT();
658  unsigned AS = LD->getAddressSpace();
659  unsigned Align = LD->getAlignment();
660  const DataLayout &DL = DAG.getDataLayout();
661  // If this is an unaligned load and the target doesn't support it,
662  // expand it.
663  if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
664  std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
665  }
666  break;
667  }
669  if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
670  RVal = Res;
671  RChain = Res.getValue(1);
672  }
673  break;
674 
676  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
677  assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
678  "Can only promote loads to same size type");
679 
680  SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
681  RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
682  RChain = Res.getValue(1);
683  break;
684  }
685  }
686  if (RChain.getNode() != Node) {
687  assert(RVal.getNode() != Node && "Load must be completely replaced");
688  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
689  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
690  if (UpdatedNodes) {
691  UpdatedNodes->insert(RVal.getNode());
692  UpdatedNodes->insert(RChain.getNode());
693  }
694  ReplacedNode(Node);
695  }
696  return;
697  }
698 
699  DEBUG(dbgs() << "Legalizing extending load operation\n");
700  EVT SrcVT = LD->getMemoryVT();
701  unsigned SrcWidth = SrcVT.getSizeInBits();
702  unsigned Alignment = LD->getAlignment();
703  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
704  AAMDNodes AAInfo = LD->getAAInfo();
705 
706  if (SrcWidth != SrcVT.getStoreSizeInBits() &&
707  // Some targets pretend to have an i1 loading operation, and actually
708  // load an i8. This trick is correct for ZEXTLOAD because the top 7
709  // bits are guaranteed to be zero; it helps the optimizers understand
710  // that these bits are zero. It is also useful for EXTLOAD, since it
711  // tells the optimizers that those bits are undefined. It would be
712  // nice to have an effective generic way of getting these benefits...
713  // Until such a way is found, don't insist on promoting i1 here.
714  (SrcVT != MVT::i1 ||
715  TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
717  // Promote to a byte-sized load if not loading an integral number of
718  // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
719  unsigned NewWidth = SrcVT.getStoreSizeInBits();
720  EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
721  SDValue Ch;
722 
723  // The extra bits are guaranteed to be zero, since we stored them that
724  // way. A zext load from NVT thus automatically gives zext from SrcVT.
725 
726  ISD::LoadExtType NewExtType =
728 
729  SDValue Result =
730  DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), Chain, Ptr,
731  LD->getPointerInfo(), NVT, Alignment, MMOFlags, AAInfo);
732 
733  Ch = Result.getValue(1); // The chain.
734 
735  if (ExtType == ISD::SEXTLOAD)
736  // Having the top bits zero doesn't help when sign extending.
737  Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
738  Result.getValueType(),
739  Result, DAG.getValueType(SrcVT));
740  else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
741  // All the top bits are guaranteed to be zero - inform the optimizers.
742  Result = DAG.getNode(ISD::AssertZext, dl,
743  Result.getValueType(), Result,
744  DAG.getValueType(SrcVT));
745 
746  Value = Result;
747  Chain = Ch;
748  } else if (SrcWidth & (SrcWidth - 1)) {
749  // If not loading a power-of-2 number of bits, expand as two loads.
750  assert(!SrcVT.isVector() && "Unsupported extload!");
751  unsigned RoundWidth = 1 << Log2_32(SrcWidth);
752  assert(RoundWidth < SrcWidth);
753  unsigned ExtraWidth = SrcWidth - RoundWidth;
754  assert(ExtraWidth < RoundWidth);
755  assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
756  "Load size not an integral number of bytes!");
757  EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
758  EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
759  SDValue Lo, Hi, Ch;
760  unsigned IncrementSize;
761  auto &DL = DAG.getDataLayout();
762 
763  if (DL.isLittleEndian()) {
764  // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
765  // Load the bottom RoundWidth bits.
766  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
767  LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
768  AAInfo);
769 
770  // Load the remaining ExtraWidth bits.
771  IncrementSize = RoundWidth / 8;
772  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
773  DAG.getConstant(IncrementSize, dl,
774  Ptr.getValueType()));
775  Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
776  LD->getPointerInfo().getWithOffset(IncrementSize),
777  ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
778  AAInfo);
779 
780  // Build a factor node to remember that this load is independent of
781  // the other one.
782  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
783  Hi.getValue(1));
784 
785  // Move the top bits to the right place.
786  Hi = DAG.getNode(
787  ISD::SHL, dl, Hi.getValueType(), Hi,
788  DAG.getConstant(RoundWidth, dl,
789  TLI.getShiftAmountTy(Hi.getValueType(), DL)));
790 
791  // Join the hi and lo parts.
792  Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
793  } else {
794  // Big endian - avoid unaligned loads.
795  // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
796  // Load the top RoundWidth bits.
797  Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
798  LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
799  AAInfo);
800 
801  // Load the remaining ExtraWidth bits.
802  IncrementSize = RoundWidth / 8;
803  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
804  DAG.getConstant(IncrementSize, dl,
805  Ptr.getValueType()));
806  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
807  LD->getPointerInfo().getWithOffset(IncrementSize),
808  ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
809  AAInfo);
810 
811  // Build a factor node to remember that this load is independent of
812  // the other one.
813  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
814  Hi.getValue(1));
815 
816  // Move the top bits to the right place.
817  Hi = DAG.getNode(
818  ISD::SHL, dl, Hi.getValueType(), Hi,
819  DAG.getConstant(ExtraWidth, dl,
820  TLI.getShiftAmountTy(Hi.getValueType(), DL)));
821 
822  // Join the hi and lo parts.
823  Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
824  }
825 
826  Chain = Ch;
827  } else {
828  bool isCustom = false;
829  switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
830  SrcVT.getSimpleVT())) {
831  default: llvm_unreachable("This action is not supported yet!");
833  isCustom = true;
836  Value = SDValue(Node, 0);
837  Chain = SDValue(Node, 1);
838 
839  if (isCustom) {
840  if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
841  Value = Res;
842  Chain = Res.getValue(1);
843  }
844  } else {
845  // If this is an unaligned load and the target doesn't support it,
846  // expand it.
847  EVT MemVT = LD->getMemoryVT();
848  unsigned AS = LD->getAddressSpace();
849  unsigned Align = LD->getAlignment();
850  const DataLayout &DL = DAG.getDataLayout();
851  if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
852  std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
853  }
854  }
855  break;
856 
857  case TargetLowering::Expand: {
858  EVT DestVT = Node->getValueType(0);
859  if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
860  // If the source type is not legal, see if there is a legal extload to
861  // an intermediate type that we can then extend further.
862  EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
863  if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
864  TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) {
865  // If we are loading a legal type, this is a non-extload followed by a
866  // full extend.
867  ISD::LoadExtType MidExtType =
868  (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
869 
870  SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
871  SrcVT, LD->getMemOperand());
872  unsigned ExtendOp =
873  ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
874  Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
875  Chain = Load.getValue(1);
876  break;
877  }
878 
879  // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
880  // normal undefined upper bits behavior to allow using an in-reg extend
881  // with the illegal FP type, so load as an integer and do the
882  // from-integer conversion.
883  if (SrcVT.getScalarType() == MVT::f16) {
884  EVT ISrcVT = SrcVT.changeTypeToInteger();
885  EVT IDestVT = DestVT.changeTypeToInteger();
886  EVT LoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
887 
888  SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, LoadVT,
889  Chain, Ptr, ISrcVT,
890  LD->getMemOperand());
891  Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result);
892  Chain = Result.getValue(1);
893  break;
894  }
895  }
896 
897  assert(!SrcVT.isVector() &&
898  "Vector Loads are handled in LegalizeVectorOps");
899 
900  // FIXME: This does not work for vectors on most targets. Sign-
901  // and zero-extend operations are currently folded into extending
902  // loads, whether they are legal or not, and then we end up here
903  // without any support for legalizing them.
904  assert(ExtType != ISD::EXTLOAD &&
905  "EXTLOAD should always be supported!");
906  // Turn the unsupported load into an EXTLOAD followed by an
907  // explicit zero/sign extend inreg.
908  SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
909  Node->getValueType(0),
910  Chain, Ptr, SrcVT,
911  LD->getMemOperand());
912  SDValue ValRes;
913  if (ExtType == ISD::SEXTLOAD)
914  ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
915  Result.getValueType(),
916  Result, DAG.getValueType(SrcVT));
917  else
918  ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType());
919  Value = ValRes;
920  Chain = Result.getValue(1);
921  break;
922  }
923  }
924  }
925 
926  // Since loads produce two values, make sure to remember that we legalized
927  // both of them.
928  if (Chain.getNode() != Node) {
929  assert(Value.getNode() != Node && "Load must be completely replaced");
930  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
931  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
932  if (UpdatedNodes) {
933  UpdatedNodes->insert(Value.getNode());
934  UpdatedNodes->insert(Chain.getNode());
935  }
936  ReplacedNode(Node);
937  }
938 }
939 
941 getStrictFPOpcodeAction(const TargetLowering &TLI, unsigned Opcode, EVT VT) {
942  unsigned EqOpc;
943  switch (Opcode) {
944  default: llvm_unreachable("Unexpected FP pseudo-opcode");
945  case ISD::STRICT_FSQRT: EqOpc = ISD::FSQRT; break;
946  case ISD::STRICT_FPOW: EqOpc = ISD::FPOW; break;
947  case ISD::STRICT_FPOWI: EqOpc = ISD::FPOWI; break;
948  case ISD::STRICT_FMA: EqOpc = ISD::FMA; break;
949  case ISD::STRICT_FSIN: EqOpc = ISD::FSIN; break;
950  case ISD::STRICT_FCOS: EqOpc = ISD::FCOS; break;
951  case ISD::STRICT_FEXP: EqOpc = ISD::FEXP; break;
952  case ISD::STRICT_FEXP2: EqOpc = ISD::FEXP2; break;
953  case ISD::STRICT_FLOG: EqOpc = ISD::FLOG; break;
954  case ISD::STRICT_FLOG10: EqOpc = ISD::FLOG10; break;
955  case ISD::STRICT_FLOG2: EqOpc = ISD::FLOG2; break;
956  case ISD::STRICT_FRINT: EqOpc = ISD::FRINT; break;
957  case ISD::STRICT_FNEARBYINT: EqOpc = ISD::FNEARBYINT; break;
958  }
959 
960  auto Action = TLI.getOperationAction(EqOpc, VT);
961 
962  // We don't currently handle Custom or Promote for strict FP pseudo-ops.
963  // For now, we just expand for those cases.
964  if (Action != TargetLowering::Legal)
965  Action = TargetLowering::Expand;
966 
967  return Action;
968 }
969 
970 /// Return a legal replacement for the given operation, with all legal operands.
971 void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
972  DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
973 
974  // Allow illegal target nodes and illegal registers.
975  if (Node->getOpcode() == ISD::TargetConstant ||
976  Node->getOpcode() == ISD::Register)
977  return;
978 
979 #ifndef NDEBUG
980  for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
981  assert((TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
983  TLI.isTypeLegal(Node->getValueType(i))) &&
984  "Unexpected illegal type!");
985 
986  for (const SDValue &Op : Node->op_values())
987  assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
989  TLI.isTypeLegal(Op.getValueType()) ||
990  Op.getOpcode() == ISD::TargetConstant ||
991  Op.getOpcode() == ISD::Register) &&
992  "Unexpected illegal type!");
993 #endif
994 
995  // Figure out the correct action; the way to query this varies by opcode
997  bool SimpleFinishLegalizing = true;
998  switch (Node->getOpcode()) {
1001  case ISD::INTRINSIC_VOID:
1002  case ISD::STACKSAVE:
1003  Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1004  break;
1006  Action = TLI.getOperationAction(Node->getOpcode(),
1007  Node->getValueType(0));
1008  break;
1009  case ISD::VAARG:
1010  Action = TLI.getOperationAction(Node->getOpcode(),
1011  Node->getValueType(0));
1012  if (Action != TargetLowering::Promote)
1013  Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1014  break;
1015  case ISD::FP_TO_FP16:
1016  case ISD::SINT_TO_FP:
1017  case ISD::UINT_TO_FP:
1019  Action = TLI.getOperationAction(Node->getOpcode(),
1020  Node->getOperand(0).getValueType());
1021  break;
1022  case ISD::FP_ROUND_INREG:
1023  case ISD::SIGN_EXTEND_INREG: {
1024  EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1025  Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1026  break;
1027  }
1028  case ISD::ATOMIC_STORE:
1029  Action = TLI.getOperationAction(Node->getOpcode(),
1030  Node->getOperand(2).getValueType());
1031  break;
1032  case ISD::SELECT_CC:
1033  case ISD::SETCC:
1034  case ISD::BR_CC: {
1035  unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
1036  Node->getOpcode() == ISD::SETCC ? 2 :
1037  Node->getOpcode() == ISD::SETCCE ? 3 : 1;
1038  unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0;
1039  MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1040  ISD::CondCode CCCode =
1041  cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1042  Action = TLI.getCondCodeAction(CCCode, OpVT);
1043  if (Action == TargetLowering::Legal) {
1044  if (Node->getOpcode() == ISD::SELECT_CC)
1045  Action = TLI.getOperationAction(Node->getOpcode(),
1046  Node->getValueType(0));
1047  else
1048  Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1049  }
1050  break;
1051  }
1052  case ISD::LOAD:
1053  case ISD::STORE:
1054  // FIXME: Model these properly. LOAD and STORE are complicated, and
1055  // STORE expects the unlegalized operand in some cases.
1056  SimpleFinishLegalizing = false;
1057  break;
1058  case ISD::CALLSEQ_START:
1059  case ISD::CALLSEQ_END:
1060  // FIXME: This shouldn't be necessary. These nodes have special properties
1061  // dealing with the recursive nature of legalization. Removing this
1062  // special case should be done as part of making LegalizeDAG non-recursive.
1063  SimpleFinishLegalizing = false;
1064  break;
1065  case ISD::EXTRACT_ELEMENT:
1066  case ISD::FLT_ROUNDS_:
1067  case ISD::MERGE_VALUES:
1068  case ISD::EH_RETURN:
1070  case ISD::EH_DWARF_CFA:
1071  case ISD::EH_SJLJ_SETJMP:
1072  case ISD::EH_SJLJ_LONGJMP:
1074  // These operations lie about being legal: when they claim to be legal,
1075  // they should actually be expanded.
1076  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1077  if (Action == TargetLowering::Legal)
1078  Action = TargetLowering::Expand;
1079  break;
1080  case ISD::INIT_TRAMPOLINE:
1082  case ISD::FRAMEADDR:
1083  case ISD::RETURNADDR:
1084  case ISD::ADDROFRETURNADDR:
1085  // These operations lie about being legal: when they claim to be legal,
1086  // they should actually be custom-lowered.
1087  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1088  if (Action == TargetLowering::Legal)
1089  Action = TargetLowering::Custom;
1090  break;
1091  case ISD::READCYCLECOUNTER:
1092  // READCYCLECOUNTER returns an i64, even if type legalization might have
1093  // expanded that to several smaller types.
1094  Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1095  break;
1096  case ISD::READ_REGISTER:
1097  case ISD::WRITE_REGISTER:
1098  // Named register is legal in the DAG, but blocked by register name
1099  // selection if not implemented by target (to chose the correct register)
1100  // They'll be converted to Copy(To/From)Reg.
1101  Action = TargetLowering::Legal;
1102  break;
1103  case ISD::DEBUGTRAP:
1104  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1105  if (Action == TargetLowering::Expand) {
1106  // replace ISD::DEBUGTRAP with ISD::TRAP
1107  SDValue NewVal;
1108  NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1109  Node->getOperand(0));
1110  ReplaceNode(Node, NewVal.getNode());
1111  LegalizeOp(NewVal.getNode());
1112  return;
1113  }
1114  break;
1115  case ISD::STRICT_FSQRT:
1116  case ISD::STRICT_FMA:
1117  case ISD::STRICT_FPOW:
1118  case ISD::STRICT_FPOWI:
1119  case ISD::STRICT_FSIN:
1120  case ISD::STRICT_FCOS:
1121  case ISD::STRICT_FEXP:
1122  case ISD::STRICT_FEXP2:
1123  case ISD::STRICT_FLOG:
1124  case ISD::STRICT_FLOG10:
1125  case ISD::STRICT_FLOG2:
1126  case ISD::STRICT_FRINT:
1128  // These pseudo-ops get legalized as if they were their non-strict
1129  // equivalent. For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT
1130  // is also legal, but if ISD::FSQRT requires expansion then so does
1131  // ISD::STRICT_FSQRT.
1132  Action = getStrictFPOpcodeAction(TLI, Node->getOpcode(),
1133  Node->getValueType(0));
1134  break;
1135  default:
1136  if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1137  Action = TargetLowering::Legal;
1138  } else {
1139  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1140  }
1141  break;
1142  }
1143 
1144  if (SimpleFinishLegalizing) {
1145  SDNode *NewNode = Node;
1146  switch (Node->getOpcode()) {
1147  default: break;
1148  case ISD::SHL:
1149  case ISD::SRL:
1150  case ISD::SRA:
1151  case ISD::ROTL:
1152  case ISD::ROTR: {
1153  // Legalizing shifts/rotates requires adjusting the shift amount
1154  // to the appropriate width.
1155  SDValue Op0 = Node->getOperand(0);
1156  SDValue Op1 = Node->getOperand(1);
1157  if (!Op1.getValueType().isVector()) {
1158  SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1159  // The getShiftAmountOperand() may create a new operand node or
1160  // return the existing one. If new operand is created we need
1161  // to update the parent node.
1162  // Do not try to legalize SAO here! It will be automatically legalized
1163  // in the next round.
1164  if (SAO != Op1)
1165  NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1166  }
1167  }
1168  break;
1169  case ISD::SRL_PARTS:
1170  case ISD::SRA_PARTS:
1171  case ISD::SHL_PARTS: {
1172  // Legalizing shifts/rotates requires adjusting the shift amount
1173  // to the appropriate width.
1174  SDValue Op0 = Node->getOperand(0);
1175  SDValue Op1 = Node->getOperand(1);
1176  SDValue Op2 = Node->getOperand(2);
1177  if (!Op2.getValueType().isVector()) {
1178  SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1179  // The getShiftAmountOperand() may create a new operand node or
1180  // return the existing one. If new operand is created we need
1181  // to update the parent node.
1182  if (SAO != Op2)
1183  NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1184  }
1185  break;
1186  }
1187  }
1188 
1189  if (NewNode != Node) {
1190  ReplaceNode(Node, NewNode);
1191  Node = NewNode;
1192  }
1193  switch (Action) {
1194  case TargetLowering::Legal:
1195  DEBUG(dbgs() << "Legal node: nothing to do\n");
1196  return;
1198  DEBUG(dbgs() << "Trying custom legalization\n");
1199  // FIXME: The handling for custom lowering with multiple results is
1200  // a complete mess.
1201  if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1202  if (!(Res.getNode() != Node || Res.getResNo() != 0))
1203  return;
1204 
1205  if (Node->getNumValues() == 1) {
1206  DEBUG(dbgs() << "Successfully custom legalized node\n");
1207  // We can just directly replace this node with the lowered value.
1208  ReplaceNode(SDValue(Node, 0), Res);
1209  return;
1210  }
1211 
1212  SmallVector<SDValue, 8> ResultVals;
1213  for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
1214  ResultVals.push_back(Res.getValue(i));
1215  DEBUG(dbgs() << "Successfully custom legalized node\n");
1216  ReplaceNode(Node, ResultVals.data());
1217  return;
1218  }
1219  DEBUG(dbgs() << "Could not custom legalize node\n");
1222  if (ExpandNode(Node))
1223  return;
1226  ConvertNodeToLibcall(Node);
1227  return;
1229  PromoteNode(Node);
1230  return;
1231  }
1232  }
1233 
1234  switch (Node->getOpcode()) {
1235  default:
1236 #ifndef NDEBUG
1237  dbgs() << "NODE: ";
1238  Node->dump( &DAG);
1239  dbgs() << "\n";
1240 #endif
1241  llvm_unreachable("Do not know how to legalize this operator!");
1242 
1243  case ISD::CALLSEQ_START:
1244  case ISD::CALLSEQ_END:
1245  break;
1246  case ISD::LOAD:
1247  return LegalizeLoadOps(Node);
1248  case ISD::STORE:
1249  return LegalizeStoreOps(Node);
1250  }
1251 }
1252 
1253 SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1254  SDValue Vec = Op.getOperand(0);
1255  SDValue Idx = Op.getOperand(1);
1256  SDLoc dl(Op);
1257 
1258  // Before we generate a new store to a temporary stack slot, see if there is
1259  // already one that we can use. There often is because when we scalarize
1260  // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1261  // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1262  // the vector. If all are expanded here, we don't want one store per vector
1263  // element.
1264 
1265  // Caches for hasPredecessorHelper
1268  Worklist.push_back(Idx.getNode());
1269  SDValue StackPtr, Ch;
1270  for (SDNode::use_iterator UI = Vec.getNode()->use_begin(),
1271  UE = Vec.getNode()->use_end(); UI != UE; ++UI) {
1272  SDNode *User = *UI;
1273  if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1274  if (ST->isIndexed() || ST->isTruncatingStore() ||
1275  ST->getValue() != Vec)
1276  continue;
1277 
1278  // Make sure that nothing else could have stored into the destination of
1279  // this store.
1280  if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1281  continue;
1282 
1283  // If the index is dependent on the store we will introduce a cycle when
1284  // creating the load (the load uses the index, and by replacing the chain
1285  // we will make the index dependent on the load). Also, the store might be
1286  // dependent on the extractelement and introduce a cycle when creating
1287  // the load.
1288  if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1289  ST->hasPredecessor(Op.getNode()))
1290  continue;
1291 
1292  StackPtr = ST->getBasePtr();
1293  Ch = SDValue(ST, 0);
1294  break;
1295  }
1296  }
1297 
1298  EVT VecVT = Vec.getValueType();
1299 
1300  if (!Ch.getNode()) {
1301  // Store the value to a temporary stack slot, then LOAD the returned part.
1302  StackPtr = DAG.CreateStackTemporary(VecVT);
1303  Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1304  MachinePointerInfo());
1305  }
1306 
1307  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1308 
1309  SDValue NewLoad;
1310 
1311  if (Op.getValueType().isVector())
1312  NewLoad =
1313  DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo());
1314  else
1315  NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1317  VecVT.getVectorElementType());
1318 
1319  // Replace the chain going out of the store, by the one out of the load.
1320  DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1321 
1322  // We introduced a cycle though, so update the loads operands, making sure
1323  // to use the original store's chain as an incoming chain.
1324  SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1325  NewLoad->op_end());
1326  NewLoadOperands[0] = Ch;
1327  NewLoad =
1328  SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1329  return NewLoad;
1330 }
1331 
1332 SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1333  assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1334 
1335  SDValue Vec = Op.getOperand(0);
1336  SDValue Part = Op.getOperand(1);
1337  SDValue Idx = Op.getOperand(2);
1338  SDLoc dl(Op);
1339 
1340  // Store the value to a temporary stack slot, then LOAD the returned part.
1341  EVT VecVT = Vec.getValueType();
1342  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1343  int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1344  MachinePointerInfo PtrInfo =
1345  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1346 
1347  // First store the whole vector.
1348  SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1349 
1350  // Then store the inserted part.
1351  SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1352 
1353  // Store the subvector.
1354  Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, MachinePointerInfo());
1355 
1356  // Finally, load the updated vector.
1357  return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1358 }
1359 
1360 SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1361  // We can't handle this case efficiently. Allocate a sufficiently
1362  // aligned object on the stack, store each element into it, then load
1363  // the result as a vector.
1364  // Create the stack frame object.
1365  EVT VT = Node->getValueType(0);
1366  EVT EltVT = VT.getVectorElementType();
1367  SDLoc dl(Node);
1368  SDValue FIPtr = DAG.CreateStackTemporary(VT);
1369  int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1370  MachinePointerInfo PtrInfo =
1371  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1372 
1373  // Emit a store of each element to the stack slot.
1374  SmallVector<SDValue, 8> Stores;
1375  unsigned TypeByteSize = EltVT.getSizeInBits() / 8;
1376  // Store (in the right endianness) the elements to memory.
1377  for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1378  // Ignore undef elements.
1379  if (Node->getOperand(i).isUndef()) continue;
1380 
1381  unsigned Offset = TypeByteSize*i;
1382 
1383  SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType());
1384  Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx);
1385 
1386  // If the destination vector element type is narrower than the source
1387  // element type, only store the bits necessary.
1388  if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) {
1389  Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1390  Node->getOperand(i), Idx,
1391  PtrInfo.getWithOffset(Offset), EltVT));
1392  } else
1393  Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1394  Idx, PtrInfo.getWithOffset(Offset)));
1395  }
1396 
1397  SDValue StoreChain;
1398  if (!Stores.empty()) // Not all undef elements?
1399  StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1400  else
1401  StoreChain = DAG.getEntryNode();
1402 
1403  // Result is a load from the stack slot.
1404  return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1405 }
1406 
1407 /// Bitcast a floating-point value to an integer value. Only bitcast the part
1408 /// containing the sign bit if the target has no integer value capable of
1409 /// holding all bits of the floating-point value.
1410 void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1411  const SDLoc &DL,
1412  SDValue Value) const {
1413  EVT FloatVT = Value.getValueType();
1414  unsigned NumBits = FloatVT.getSizeInBits();
1415  State.FloatVT = FloatVT;
1416  EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1417  // Convert to an integer of the same size.
1418  if (TLI.isTypeLegal(IVT)) {
1419  State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1420  State.SignMask = APInt::getSignMask(NumBits);
1421  State.SignBit = NumBits - 1;
1422  return;
1423  }
1424 
1425  auto &DataLayout = DAG.getDataLayout();
1426  // Store the float to memory, then load the sign part out as an integer.
1427  MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8);
1428  // First create a temporary that is aligned for both the load and store.
1429  SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1430  int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1431  // Then store the float to it.
1432  State.FloatPtr = StackPtr;
1433  MachineFunction &MF = DAG.getMachineFunction();
1434  State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1435  State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1436  State.FloatPointerInfo);
1437 
1438  SDValue IntPtr;
1439  if (DataLayout.isBigEndian()) {
1440  assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1441  // Load out a legal integer with the same sign bit as the float.
1442  IntPtr = StackPtr;
1443  State.IntPointerInfo = State.FloatPointerInfo;
1444  } else {
1445  // Advance the pointer so that the loaded byte will contain the sign bit.
1446  unsigned ByteOffset = (FloatVT.getSizeInBits() / 8) - 1;
1447  IntPtr = DAG.getNode(ISD::ADD, DL, StackPtr.getValueType(), StackPtr,
1448  DAG.getConstant(ByteOffset, DL, StackPtr.getValueType()));
1449  State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1450  ByteOffset);
1451  }
1452 
1453  State.IntPtr = IntPtr;
1454  State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1455  State.IntPointerInfo, MVT::i8);
1456  State.SignMask = APInt::getOneBitSet(LoadTy.getSizeInBits(), 7);
1457  State.SignBit = 7;
1458 }
1459 
1460 /// Replace the integer value produced by getSignAsIntValue() with a new value
1461 /// and cast the result back to a floating-point type.
1462 SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1463  const SDLoc &DL,
1464  SDValue NewIntValue) const {
1465  if (!State.Chain)
1466  return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1467 
1468  // Override the part containing the sign bit in the value stored on the stack.
1469  SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1470  State.IntPointerInfo, MVT::i8);
1471  return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1472  State.FloatPointerInfo);
1473 }
1474 
1475 SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1476  SDLoc DL(Node);
1477  SDValue Mag = Node->getOperand(0);
1478  SDValue Sign = Node->getOperand(1);
1479 
1480  // Get sign bit into an integer value.
1481  FloatSignAsInt SignAsInt;
1482  getSignAsIntValue(SignAsInt, DL, Sign);
1483 
1484  EVT IntVT = SignAsInt.IntValue.getValueType();
1485  SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1486  SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1487  SignMask);
1488 
1489  // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1490  EVT FloatVT = Mag.getValueType();
1491  if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1492  TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1493  SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1494  SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1495  SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1496  DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1497  return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1498  }
1499 
1500  // Transform Mag value to integer, and clear the sign bit.
1501  FloatSignAsInt MagAsInt;
1502  getSignAsIntValue(MagAsInt, DL, Mag);
1503  EVT MagVT = MagAsInt.IntValue.getValueType();
1504  SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1505  SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1506  ClearSignMask);
1507 
1508  // Get the signbit at the right position for MagAsInt.
1509  int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1510  if (SignBit.getValueSizeInBits() > ClearedSign.getValueSizeInBits()) {
1511  if (ShiftAmount > 0) {
1512  SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, IntVT);
1513  SignBit = DAG.getNode(ISD::SRL, DL, IntVT, SignBit, ShiftCnst);
1514  } else if (ShiftAmount < 0) {
1515  SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, IntVT);
1516  SignBit = DAG.getNode(ISD::SHL, DL, IntVT, SignBit, ShiftCnst);
1517  }
1518  SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1519  } else if (SignBit.getValueSizeInBits() < ClearedSign.getValueSizeInBits()) {
1520  SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1521  if (ShiftAmount > 0) {
1522  SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, MagVT);
1523  SignBit = DAG.getNode(ISD::SRL, DL, MagVT, SignBit, ShiftCnst);
1524  } else if (ShiftAmount < 0) {
1525  SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, MagVT);
1526  SignBit = DAG.getNode(ISD::SHL, DL, MagVT, SignBit, ShiftCnst);
1527  }
1528  }
1529 
1530  // Store the part with the modified sign and convert back to float.
1531  SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1532  return modifySignAsInt(MagAsInt, DL, CopiedSign);
1533 }
1534 
1535 SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1536  SDLoc DL(Node);
1537  SDValue Value = Node->getOperand(0);
1538 
1539  // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1540  EVT FloatVT = Value.getValueType();
1541  if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1542  SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1543  return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1544  }
1545 
1546  // Transform value to integer, clear the sign bit and transform back.
1547  FloatSignAsInt ValueAsInt;
1548  getSignAsIntValue(ValueAsInt, DL, Value);
1549  EVT IntVT = ValueAsInt.IntValue.getValueType();
1550  SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1551  SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1552  ClearSignMask);
1553  return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1554 }
1555 
1556 void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1558  unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
1559  assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1560  " not tell us which reg is the stack pointer!");
1561  SDLoc dl(Node);
1562  EVT VT = Node->getValueType(0);
1563  SDValue Tmp1 = SDValue(Node, 0);
1564  SDValue Tmp2 = SDValue(Node, 1);
1565  SDValue Tmp3 = Node->getOperand(2);
1566  SDValue Chain = Tmp1.getOperand(0);
1567 
1568  // Chain the dynamic stack allocation so that it doesn't modify the stack
1569  // pointer when other instructions are using the stack.
1570  Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1571 
1572  SDValue Size = Tmp2.getOperand(1);
1573  SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1574  Chain = SP.getValue(1);
1575  unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue();
1576  unsigned StackAlign =
1577  DAG.getSubtarget().getFrameLowering()->getStackAlignment();
1578  Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
1579  if (Align > StackAlign)
1580  Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1581  DAG.getConstant(-(uint64_t)Align, dl, VT));
1582  Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1583 
1584  Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
1585  DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
1586 
1587  Results.push_back(Tmp1);
1588  Results.push_back(Tmp2);
1589 }
1590 
1591 /// Legalize a SETCC with given LHS and RHS and condition code CC on the current
1592 /// target.
1593 ///
1594 /// If the SETCC has been legalized using AND / OR, then the legalized node
1595 /// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert
1596 /// will be set to false.
1597 ///
1598 /// If the SETCC has been legalized by using getSetCCSwappedOperands(),
1599 /// then the values of LHS and RHS will be swapped, CC will be set to the
1600 /// new condition, and NeedInvert will be set to false.
1601 ///
1602 /// If the SETCC has been legalized using the inverse condcode, then LHS and
1603 /// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert
1604 /// will be set to true. The caller must invert the result of the SETCC with
1605 /// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect
1606 /// of a true/false result.
1607 ///
1608 /// \returns true if the SetCC has been legalized, false if it hasn't.
1609 bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS,
1610  SDValue &RHS, SDValue &CC,
1611  bool &NeedInvert,
1612  const SDLoc &dl) {
1613  MVT OpVT = LHS.getSimpleValueType();
1614  ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
1615  NeedInvert = false;
1616  switch (TLI.getCondCodeAction(CCCode, OpVT)) {
1617  default: llvm_unreachable("Unknown condition code action!");
1618  case TargetLowering::Legal:
1619  // Nothing to do.
1620  break;
1621  case TargetLowering::Expand: {
1623  if (TLI.isCondCodeLegal(InvCC, OpVT)) {
1624  std::swap(LHS, RHS);
1625  CC = DAG.getCondCode(InvCC);
1626  return true;
1627  }
1629  unsigned Opc = 0;
1630  switch (CCCode) {
1631  default: llvm_unreachable("Don't know how to expand this condition!");
1632  case ISD::SETO:
1633  assert(TLI.getCondCodeAction(ISD::SETOEQ, OpVT)
1635  && "If SETO is expanded, SETOEQ must be legal!");
1636  CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break;
1637  case ISD::SETUO:
1638  assert(TLI.getCondCodeAction(ISD::SETUNE, OpVT)
1640  && "If SETUO is expanded, SETUNE must be legal!");
1641  CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break;
1642  case ISD::SETOEQ:
1643  case ISD::SETOGT:
1644  case ISD::SETOGE:
1645  case ISD::SETOLT:
1646  case ISD::SETOLE:
1647  case ISD::SETONE:
1648  case ISD::SETUEQ:
1649  case ISD::SETUNE:
1650  case ISD::SETUGT:
1651  case ISD::SETUGE:
1652  case ISD::SETULT:
1653  case ISD::SETULE:
1654  // If we are floating point, assign and break, otherwise fall through.
1655  if (!OpVT.isInteger()) {
1656  // We can use the 4th bit to tell if we are the unordered
1657  // or ordered version of the opcode.
1658  CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
1659  Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
1660  CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
1661  break;
1662  }
1663  // Fallthrough if we are unsigned integer.
1665  case ISD::SETLE:
1666  case ISD::SETGT:
1667  case ISD::SETGE:
1668  case ISD::SETLT:
1669  // We only support using the inverted operation, which is computed above
1670  // and not a different manner of supporting expanding these cases.
1671  llvm_unreachable("Don't know how to expand this condition!");
1672  case ISD::SETNE:
1673  case ISD::SETEQ:
1674  // Try inverting the result of the inverse condition.
1675  InvCC = CCCode == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ;
1676  if (TLI.isCondCodeLegal(InvCC, OpVT)) {
1677  CC = DAG.getCondCode(InvCC);
1678  NeedInvert = true;
1679  return true;
1680  }
1681  // If inverting the condition didn't work then we have no means to expand
1682  // the condition.
1683  llvm_unreachable("Don't know how to expand this condition!");
1684  }
1685 
1686  SDValue SetCC1, SetCC2;
1687  if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
1688  // If we aren't the ordered or unorder operation,
1689  // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
1690  SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1);
1691  SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2);
1692  } else {
1693  // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
1694  SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1);
1695  SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2);
1696  }
1697  LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
1698  RHS = SDValue();
1699  CC = SDValue();
1700  return true;
1701  }
1702  }
1703  return false;
1704 }
1705 
1706 /// Emit a store/load combination to the stack. This stores
1707 /// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1708 /// a load from the stack slot to DestVT, extending it if needed.
1709 /// The resultant code need not be legal.
1710 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1711  EVT DestVT, const SDLoc &dl) {
1712  // Create the stack frame object.
1713  unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment(
1714  SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1715  SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign);
1716 
1717  FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1718  int SPFI = StackPtrFI->getIndex();
1719  MachinePointerInfo PtrInfo =
1720  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1721 
1722  unsigned SrcSize = SrcOp.getValueSizeInBits();
1723  unsigned SlotSize = SlotVT.getSizeInBits();
1724  unsigned DestSize = DestVT.getSizeInBits();
1725  Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1726  unsigned DestAlign = DAG.getDataLayout().getPrefTypeAlignment(DestType);
1727 
1728  // Emit a store to the stack slot. Use a truncstore if the input value is
1729  // later than DestVT.
1730  SDValue Store;
1731 
1732  if (SrcSize > SlotSize)
1733  Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo,
1734  SlotVT, SrcAlign);
1735  else {
1736  assert(SrcSize == SlotSize && "Invalid store");
1737  Store =
1738  DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1739  }
1740 
1741  // Result is a load from the stack slot.
1742  if (SlotSize == DestSize)
1743  return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1744 
1745  assert(SlotSize < DestSize && "Unknown extension!");
1746  return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1747  DestAlign);
1748 }
1749 
1750 SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1751  SDLoc dl(Node);
1752  // Create a vector sized/aligned stack slot, store the value to element #0,
1753  // then load the whole vector back out.
1754  SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1755 
1756  FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1757  int SPFI = StackPtrFI->getIndex();
1758 
1759  SDValue Ch = DAG.getTruncStore(
1760  DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1761  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1762  Node->getValueType(0).getVectorElementType());
1763  return DAG.getLoad(
1764  Node->getValueType(0), dl, Ch, StackPtr,
1765  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1766 }
1767 
1768 static bool
1770  const TargetLowering &TLI, SDValue &Res) {
1771  unsigned NumElems = Node->getNumOperands();
1772  SDLoc dl(Node);
1773  EVT VT = Node->getValueType(0);
1774 
1775  // Try to group the scalars into pairs, shuffle the pairs together, then
1776  // shuffle the pairs of pairs together, etc. until the vector has
1777  // been built. This will work only if all of the necessary shuffle masks
1778  // are legal.
1779 
1780  // We do this in two phases; first to check the legality of the shuffles,
1781  // and next, assuming that all shuffles are legal, to create the new nodes.
1782  for (int Phase = 0; Phase < 2; ++Phase) {
1784  NewIntermedVals;
1785  for (unsigned i = 0; i < NumElems; ++i) {
1786  SDValue V = Node->getOperand(i);
1787  if (V.isUndef())
1788  continue;
1789 
1790  SDValue Vec;
1791  if (Phase)
1792  Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1793  IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1794  }
1795 
1796  while (IntermedVals.size() > 2) {
1797  NewIntermedVals.clear();
1798  for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1799  // This vector and the next vector are shuffled together (simply to
1800  // append the one to the other).
1801  SmallVector<int, 16> ShuffleVec(NumElems, -1);
1802 
1803  SmallVector<int, 16> FinalIndices;
1804  FinalIndices.reserve(IntermedVals[i].second.size() +
1805  IntermedVals[i+1].second.size());
1806 
1807  int k = 0;
1808  for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1809  ++j, ++k) {
1810  ShuffleVec[k] = j;
1811  FinalIndices.push_back(IntermedVals[i].second[j]);
1812  }
1813  for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1814  ++j, ++k) {
1815  ShuffleVec[k] = NumElems + j;
1816  FinalIndices.push_back(IntermedVals[i+1].second[j]);
1817  }
1818 
1819  SDValue Shuffle;
1820  if (Phase)
1821  Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1822  IntermedVals[i+1].first,
1823  ShuffleVec);
1824  else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1825  return false;
1826  NewIntermedVals.push_back(
1827  std::make_pair(Shuffle, std::move(FinalIndices)));
1828  }
1829 
1830  // If we had an odd number of defined values, then append the last
1831  // element to the array of new vectors.
1832  if ((IntermedVals.size() & 1) != 0)
1833  NewIntermedVals.push_back(IntermedVals.back());
1834 
1835  IntermedVals.swap(NewIntermedVals);
1836  }
1837 
1838  assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1839  "Invalid number of intermediate vectors");
1840  SDValue Vec1 = IntermedVals[0].first;
1841  SDValue Vec2;
1842  if (IntermedVals.size() > 1)
1843  Vec2 = IntermedVals[1].first;
1844  else if (Phase)
1845  Vec2 = DAG.getUNDEF(VT);
1846 
1847  SmallVector<int, 16> ShuffleVec(NumElems, -1);
1848  for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1849  ShuffleVec[IntermedVals[0].second[i]] = i;
1850  for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1851  ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1852 
1853  if (Phase)
1854  Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1855  else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1856  return false;
1857  }
1858 
1859  return true;
1860 }
1861 
1862 /// Expand a BUILD_VECTOR node on targets that don't
1863 /// support the operation, but do support the resultant vector type.
1864 SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1865  unsigned NumElems = Node->getNumOperands();
1866  SDValue Value1, Value2;
1867  SDLoc dl(Node);
1868  EVT VT = Node->getValueType(0);
1869  EVT OpVT = Node->getOperand(0).getValueType();
1870  EVT EltVT = VT.getVectorElementType();
1871 
1872  // If the only non-undef value is the low element, turn this into a
1873  // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1874  bool isOnlyLowElement = true;
1875  bool MoreThanTwoValues = false;
1876  bool isConstant = true;
1877  for (unsigned i = 0; i < NumElems; ++i) {
1878  SDValue V = Node->getOperand(i);
1879  if (V.isUndef())
1880  continue;
1881  if (i > 0)
1882  isOnlyLowElement = false;
1883  if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1884  isConstant = false;
1885 
1886  if (!Value1.getNode()) {
1887  Value1 = V;
1888  } else if (!Value2.getNode()) {
1889  if (V != Value1)
1890  Value2 = V;
1891  } else if (V != Value1 && V != Value2) {
1892  MoreThanTwoValues = true;
1893  }
1894  }
1895 
1896  if (!Value1.getNode())
1897  return DAG.getUNDEF(VT);
1898 
1899  if (isOnlyLowElement)
1900  return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1901 
1902  // If all elements are constants, create a load from the constant pool.
1903  if (isConstant) {
1905  for (unsigned i = 0, e = NumElems; i != e; ++i) {
1906  if (ConstantFPSDNode *V =
1907  dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1908  CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1909  } else if (ConstantSDNode *V =
1910  dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1911  if (OpVT==EltVT)
1912  CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1913  else {
1914  // If OpVT and EltVT don't match, EltVT is not legal and the
1915  // element values have been promoted/truncated earlier. Undo this;
1916  // we don't want a v16i8 to become a v16i32 for example.
1917  const ConstantInt *CI = V->getConstantIntValue();
1918  CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1919  CI->getZExtValue()));
1920  }
1921  } else {
1922  assert(Node->getOperand(i).isUndef());
1923  Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1924  CV.push_back(UndefValue::get(OpNTy));
1925  }
1926  }
1927  Constant *CP = ConstantVector::get(CV);
1928  SDValue CPIdx =
1929  DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1930  unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
1931  return DAG.getLoad(
1932  VT, dl, DAG.getEntryNode(), CPIdx,
1933  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
1934  Alignment);
1935  }
1936 
1937  SmallSet<SDValue, 16> DefinedValues;
1938  for (unsigned i = 0; i < NumElems; ++i) {
1939  if (Node->getOperand(i).isUndef())
1940  continue;
1941  DefinedValues.insert(Node->getOperand(i));
1942  }
1943 
1944  if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
1945  if (!MoreThanTwoValues) {
1946  SmallVector<int, 8> ShuffleVec(NumElems, -1);
1947  for (unsigned i = 0; i < NumElems; ++i) {
1948  SDValue V = Node->getOperand(i);
1949  if (V.isUndef())
1950  continue;
1951  ShuffleVec[i] = V == Value1 ? 0 : NumElems;
1952  }
1953  if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
1954  // Get the splatted value into the low element of a vector register.
1955  SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
1956  SDValue Vec2;
1957  if (Value2.getNode())
1958  Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
1959  else
1960  Vec2 = DAG.getUNDEF(VT);
1961 
1962  // Return shuffle(LowValVec, undef, <0,0,0,0>)
1963  return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1964  }
1965  } else {
1966  SDValue Res;
1967  if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
1968  return Res;
1969  }
1970  }
1971 
1972  // Otherwise, we can't handle this case efficiently.
1973  return ExpandVectorBuildThroughStack(Node);
1974 }
1975 
1976 // Expand a node into a call to a libcall. If the result value
1977 // does not fit into a register, return the lo part and set the hi part to the
1978 // by-reg argument. If it does fit into a single register, return the result
1979 // and leave the Hi part unset.
1980 SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
1981  bool isSigned) {
1983  TargetLowering::ArgListEntry Entry;
1984  for (const SDValue &Op : Node->op_values()) {
1985  EVT ArgVT = Op.getValueType();
1986  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
1987  Entry.Node = Op;
1988  Entry.Ty = ArgTy;
1989  Entry.IsSExt = isSigned;
1990  Entry.IsZExt = !isSigned;
1991  Args.push_back(Entry);
1992  }
1993  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
1994  TLI.getPointerTy(DAG.getDataLayout()));
1995 
1996  Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
1997 
1998  // By default, the input chain to this libcall is the entry node of the
1999  // function. If the libcall is going to be emitted as a tail call then
2000  // TLI.isUsedByReturnOnly will change it to the right chain if the return
2001  // node which is being folded has a non-entry input chain.
2002  SDValue InChain = DAG.getEntryNode();
2003 
2004  // isTailCall may be true since the callee does not reference caller stack
2005  // frame. Check if it's in the right position and that the return types match.
2006  SDValue TCChain = InChain;
2007  const Function *F = DAG.getMachineFunction().getFunction();
2008  bool isTailCall =
2009  TLI.isInTailCallPosition(DAG, Node, TCChain) &&
2010  (RetTy == F->getReturnType() || F->getReturnType()->isVoidTy());
2011  if (isTailCall)
2012  InChain = TCChain;
2013 
2015  CLI.setDebugLoc(SDLoc(Node))
2016  .setChain(InChain)
2017  .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2018  std::move(Args))
2019  .setTailCall(isTailCall)
2020  .setSExtResult(isSigned)
2021  .setZExtResult(!isSigned)
2023 
2024  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2025 
2026  if (!CallInfo.second.getNode()) {
2027  DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump());
2028  // It's a tailcall, return the chain (which is the DAG root).
2029  return DAG.getRoot();
2030  }
2031 
2032  DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump());
2033  return CallInfo.first;
2034 }
2035 
2036 /// Generate a libcall taking the given operands as arguments
2037 /// and returning a result of type RetVT.
2038 SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT,
2039  const SDValue *Ops, unsigned NumOps,
2040  bool isSigned, const SDLoc &dl) {
2042  Args.reserve(NumOps);
2043 
2044  TargetLowering::ArgListEntry Entry;
2045  for (unsigned i = 0; i != NumOps; ++i) {
2046  Entry.Node = Ops[i];
2047  Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
2048  Entry.IsSExt = isSigned;
2049  Entry.IsZExt = !isSigned;
2050  Args.push_back(Entry);
2051  }
2052  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2053  TLI.getPointerTy(DAG.getDataLayout()));
2054 
2055  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2056 
2058  CLI.setDebugLoc(dl)
2059  .setChain(DAG.getEntryNode())
2060  .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2061  std::move(Args))
2062  .setSExtResult(isSigned)
2063  .setZExtResult(!isSigned)
2065 
2066  std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(CLI);
2067 
2068  return CallInfo.first;
2069 }
2070 
2071 // Expand a node into a call to a libcall. Similar to
2072 // ExpandLibCall except that the first operand is the in-chain.
2073 std::pair<SDValue, SDValue>
2074 SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC,
2075  SDNode *Node,
2076  bool isSigned) {
2077  SDValue InChain = Node->getOperand(0);
2078 
2080  TargetLowering::ArgListEntry Entry;
2081  for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) {
2082  EVT ArgVT = Node->getOperand(i).getValueType();
2083  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2084  Entry.Node = Node->getOperand(i);
2085  Entry.Ty = ArgTy;
2086  Entry.IsSExt = isSigned;
2087  Entry.IsZExt = !isSigned;
2088  Args.push_back(Entry);
2089  }
2090  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2091  TLI.getPointerTy(DAG.getDataLayout()));
2092 
2093  Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
2094 
2096  CLI.setDebugLoc(SDLoc(Node))
2097  .setChain(InChain)
2098  .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2099  std::move(Args))
2100  .setSExtResult(isSigned)
2101  .setZExtResult(!isSigned);
2102 
2103  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2104 
2105  return CallInfo;
2106 }
2107 
2108 SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2109  RTLIB::Libcall Call_F32,
2110  RTLIB::Libcall Call_F64,
2111  RTLIB::Libcall Call_F80,
2112  RTLIB::Libcall Call_F128,
2113  RTLIB::Libcall Call_PPCF128) {
2114  if (Node->isStrictFPOpcode())
2115  Node = DAG.mutateStrictFPToFP(Node);
2116 
2117  RTLIB::Libcall LC;
2118  switch (Node->getSimpleValueType(0).SimpleTy) {
2119  default: llvm_unreachable("Unexpected request for libcall!");
2120  case MVT::f32: LC = Call_F32; break;
2121  case MVT::f64: LC = Call_F64; break;
2122  case MVT::f80: LC = Call_F80; break;
2123  case MVT::f128: LC = Call_F128; break;
2124  case MVT::ppcf128: LC = Call_PPCF128; break;
2125  }
2126  return ExpandLibCall(LC, Node, false);
2127 }
2128 
2129 SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2130  RTLIB::Libcall Call_I8,
2131  RTLIB::Libcall Call_I16,
2132  RTLIB::Libcall Call_I32,
2133  RTLIB::Libcall Call_I64,
2134  RTLIB::Libcall Call_I128) {
2135  RTLIB::Libcall LC;
2136  switch (Node->getSimpleValueType(0).SimpleTy) {
2137  default: llvm_unreachable("Unexpected request for libcall!");
2138  case MVT::i8: LC = Call_I8; break;
2139  case MVT::i16: LC = Call_I16; break;
2140  case MVT::i32: LC = Call_I32; break;
2141  case MVT::i64: LC = Call_I64; break;
2142  case MVT::i128: LC = Call_I128; break;
2143  }
2144  return ExpandLibCall(LC, Node, isSigned);
2145 }
2146 
2147 /// Issue libcalls to __{u}divmod to compute div / rem pairs.
2148 void
2149 SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2150  SmallVectorImpl<SDValue> &Results) {
2151  unsigned Opcode = Node->getOpcode();
2152  bool isSigned = Opcode == ISD::SDIVREM;
2153 
2154  RTLIB::Libcall LC;
2155  switch (Node->getSimpleValueType(0).SimpleTy) {
2156  default: llvm_unreachable("Unexpected request for libcall!");
2157  case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2158  case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2159  case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2160  case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2161  case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2162  }
2163 
2164  // The input chain to this libcall is the entry node of the function.
2165  // Legalizing the call will automatically add the previous call to the
2166  // dependence.
2167  SDValue InChain = DAG.getEntryNode();
2168 
2169  EVT RetVT = Node->getValueType(0);
2170  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2171 
2173  TargetLowering::ArgListEntry Entry;
2174  for (const SDValue &Op : Node->op_values()) {
2175  EVT ArgVT = Op.getValueType();
2176  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2177  Entry.Node = Op;
2178  Entry.Ty = ArgTy;
2179  Entry.IsSExt = isSigned;
2180  Entry.IsZExt = !isSigned;
2181  Args.push_back(Entry);
2182  }
2183 
2184  // Also pass the return address of the remainder.
2185  SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2186  Entry.Node = FIPtr;
2187  Entry.Ty = RetTy->getPointerTo();
2188  Entry.IsSExt = isSigned;
2189  Entry.IsZExt = !isSigned;
2190  Args.push_back(Entry);
2191 
2192  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2193  TLI.getPointerTy(DAG.getDataLayout()));
2194 
2195  SDLoc dl(Node);
2197  CLI.setDebugLoc(dl)
2198  .setChain(InChain)
2199  .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2200  std::move(Args))
2201  .setSExtResult(isSigned)
2202  .setZExtResult(!isSigned);
2203 
2204  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2205 
2206  // Remainder is loaded back from the stack frame.
2207  SDValue Rem =
2208  DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2209  Results.push_back(CallInfo.first);
2210  Results.push_back(Rem);
2211 }
2212 
2213 /// Return true if sincos libcall is available.
2214 static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2215  RTLIB::Libcall LC;
2216  switch (Node->getSimpleValueType(0).SimpleTy) {
2217  default: llvm_unreachable("Unexpected request for libcall!");
2218  case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2219  case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2220  case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2221  case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2222  case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2223  }
2224  return TLI.getLibcallName(LC) != nullptr;
2225 }
2226 
2227 /// Only issue sincos libcall if both sin and cos are needed.
2228 static bool useSinCos(SDNode *Node) {
2229  unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2230  ? ISD::FCOS : ISD::FSIN;
2231 
2232  SDValue Op0 = Node->getOperand(0);
2233  for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
2234  UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
2235  SDNode *User = *UI;
2236  if (User == Node)
2237  continue;
2238  // The other user might have been turned into sincos already.
2239  if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2240  return true;
2241  }
2242  return false;
2243 }
2244 
2245 /// Issue libcalls to sincos to compute sin / cos pairs.
2246 void
2247 SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2248  SmallVectorImpl<SDValue> &Results) {
2249  RTLIB::Libcall LC;
2250  switch (Node->getSimpleValueType(0).SimpleTy) {
2251  default: llvm_unreachable("Unexpected request for libcall!");
2252  case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2253  case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2254  case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2255  case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2256  case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2257  }
2258 
2259  // The input chain to this libcall is the entry node of the function.
2260  // Legalizing the call will automatically add the previous call to the
2261  // dependence.
2262  SDValue InChain = DAG.getEntryNode();
2263 
2264  EVT RetVT = Node->getValueType(0);
2265  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2266 
2268  TargetLowering::ArgListEntry Entry;
2269 
2270  // Pass the argument.
2271  Entry.Node = Node->getOperand(0);
2272  Entry.Ty = RetTy;
2273  Entry.IsSExt = false;
2274  Entry.IsZExt = false;
2275  Args.push_back(Entry);
2276 
2277  // Pass the return address of sin.
2278  SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2279  Entry.Node = SinPtr;
2280  Entry.Ty = RetTy->getPointerTo();
2281  Entry.IsSExt = false;
2282  Entry.IsZExt = false;
2283  Args.push_back(Entry);
2284 
2285  // Also pass the return address of the cos.
2286  SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2287  Entry.Node = CosPtr;
2288  Entry.Ty = RetTy->getPointerTo();
2289  Entry.IsSExt = false;
2290  Entry.IsZExt = false;
2291  Args.push_back(Entry);
2292 
2293  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2294  TLI.getPointerTy(DAG.getDataLayout()));
2295 
2296  SDLoc dl(Node);
2298  CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2299  TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2300  std::move(Args));
2301 
2302  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2303 
2304  Results.push_back(
2305  DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2306  Results.push_back(
2307  DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2308 }
2309 
2310 /// This function is responsible for legalizing a
2311 /// INT_TO_FP operation of the specified operand when the target requests that
2312 /// we expand it. At this point, we know that the result and operand types are
2313 /// legal for the target.
2314 SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0,
2315  EVT DestVT,
2316  const SDLoc &dl) {
2317  // TODO: Should any fast-math-flags be set for the created nodes?
2318  DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2319  if (Op0.getValueType() == MVT::i32 && TLI.isTypeLegal(MVT::f64)) {
2320  DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2321  "expansion\n");
2322 
2323  // Get the stack frame index of a 8 byte buffer.
2324  SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2325 
2326  // word offset constant for Hi/Lo address computation
2327  SDValue WordOff = DAG.getConstant(sizeof(int), dl,
2328  StackSlot.getValueType());
2329  // set up Hi and Lo (into buffer) address based on endian
2330  SDValue Hi = StackSlot;
2331  SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(),
2332  StackSlot, WordOff);
2333  if (DAG.getDataLayout().isLittleEndian())
2334  std::swap(Hi, Lo);
2335 
2336  // if signed map to unsigned space
2337  SDValue Op0Mapped;
2338  if (isSigned) {
2339  // constant used to invert sign bit (signed to unsigned mapping)
2340  SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32);
2341  Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit);
2342  } else {
2343  Op0Mapped = Op0;
2344  }
2345  // store the lo of the constructed double - based on integer input
2346  SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op0Mapped, Lo,
2347  MachinePointerInfo());
2348  // initial hi portion of constructed double
2349  SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2350  // store the hi of the constructed double - biased exponent
2351  SDValue Store2 =
2352  DAG.getStore(Store1, dl, InitialHi, Hi, MachinePointerInfo());
2353  // load the constructed double
2354  SDValue Load =
2355  DAG.getLoad(MVT::f64, dl, Store2, StackSlot, MachinePointerInfo());
2356  // FP constant to bias correct the final result
2357  SDValue Bias = DAG.getConstantFP(isSigned ?
2358  BitsToDouble(0x4330000080000000ULL) :
2359  BitsToDouble(0x4330000000000000ULL),
2360  dl, MVT::f64);
2361  // subtract the bias
2362  SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2363  // final result
2364  SDValue Result;
2365  // handle final rounding
2366  if (DestVT == MVT::f64) {
2367  // do nothing
2368  Result = Sub;
2369  } else if (DestVT.bitsLT(MVT::f64)) {
2370  Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub,
2371  DAG.getIntPtrConstant(0, dl));
2372  } else if (DestVT.bitsGT(MVT::f64)) {
2373  Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub);
2374  }
2375  return Result;
2376  }
2377  assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet");
2378  // Code below here assumes !isSigned without checking again.
2379 
2380  // Implementation of unsigned i64 to f64 following the algorithm in
2381  // __floatundidf in compiler_rt. This implementation has the advantage
2382  // of performing rounding correctly, both in the default rounding mode
2383  // and in all alternate rounding modes.
2384  // TODO: Generalize this for use with other types.
2385  if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f64) {
2386  DEBUG(dbgs() << "Converting unsigned i64 to f64\n");
2387  SDValue TwoP52 =
2388  DAG.getConstant(UINT64_C(0x4330000000000000), dl, MVT::i64);
2389  SDValue TwoP84PlusTwoP52 =
2390  DAG.getConstantFP(BitsToDouble(UINT64_C(0x4530000000100000)), dl,
2391  MVT::f64);
2392  SDValue TwoP84 =
2393  DAG.getConstant(UINT64_C(0x4530000000000000), dl, MVT::i64);
2394 
2395  SDValue Lo = DAG.getZeroExtendInReg(Op0, dl, MVT::i32);
2396  SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0,
2397  DAG.getConstant(32, dl, MVT::i64));
2398  SDValue LoOr = DAG.getNode(ISD::OR, dl, MVT::i64, Lo, TwoP52);
2399  SDValue HiOr = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, TwoP84);
2400  SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, LoOr);
2401  SDValue HiFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, HiOr);
2402  SDValue HiSub = DAG.getNode(ISD::FSUB, dl, MVT::f64, HiFlt,
2403  TwoP84PlusTwoP52);
2404  return DAG.getNode(ISD::FADD, dl, MVT::f64, LoFlt, HiSub);
2405  }
2406 
2407  // TODO: Generalize this for use with other types.
2408  if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f32) {
2409  DEBUG(dbgs() << "Converting unsigned i64 to f32\n");
2410  // For unsigned conversions, convert them to signed conversions using the
2411  // algorithm from the x86_64 __floatundidf in compiler_rt.
2412  if (!isSigned) {
2413  SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0);
2414 
2415  SDValue ShiftConst = DAG.getConstant(
2416  1, dl, TLI.getShiftAmountTy(Op0.getValueType(), DAG.getDataLayout()));
2417  SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst);
2418  SDValue AndConst = DAG.getConstant(1, dl, MVT::i64);
2419  SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, AndConst);
2420  SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr);
2421 
2422  SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Or);
2423  SDValue Slow = DAG.getNode(ISD::FADD, dl, MVT::f32, SignCvt, SignCvt);
2424 
2425  // TODO: This really should be implemented using a branch rather than a
2426  // select. We happen to get lucky and machinesink does the right
2427  // thing most of the time. This would be a good candidate for a
2428  //pseudo-op, or, even better, for whole-function isel.
2429  SDValue SignBitTest = DAG.getSetCC(dl, getSetCCResultType(MVT::i64),
2430  Op0, DAG.getConstant(0, dl, MVT::i64), ISD::SETLT);
2431  return DAG.getSelect(dl, MVT::f32, SignBitTest, Slow, Fast);
2432  }
2433 
2434  // Otherwise, implement the fully general conversion.
2435 
2436  SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2437  DAG.getConstant(UINT64_C(0xfffffffffffff800), dl, MVT::i64));
2438  SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And,
2439  DAG.getConstant(UINT64_C(0x800), dl, MVT::i64));
2440  SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2441  DAG.getConstant(UINT64_C(0x7ff), dl, MVT::i64));
2442  SDValue Ne = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), And2,
2443  DAG.getConstant(UINT64_C(0), dl, MVT::i64),
2444  ISD::SETNE);
2445  SDValue Sel = DAG.getSelect(dl, MVT::i64, Ne, Or, Op0);
2446  SDValue Ge = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), Op0,
2447  DAG.getConstant(UINT64_C(0x0020000000000000), dl,
2448  MVT::i64),
2449  ISD::SETUGE);
2450  SDValue Sel2 = DAG.getSelect(dl, MVT::i64, Ge, Sel, Op0);
2451  EVT SHVT = TLI.getShiftAmountTy(Sel2.getValueType(), DAG.getDataLayout());
2452 
2453  SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2,
2454  DAG.getConstant(32, dl, SHVT));
2455  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh);
2456  SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc);
2457  SDValue TwoP32 =
2458  DAG.getConstantFP(BitsToDouble(UINT64_C(0x41f0000000000000)), dl,
2459  MVT::f64);
2460  SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt);
2461  SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2);
2462  SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo);
2463  SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2);
2464  return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd,
2465  DAG.getIntPtrConstant(0, dl));
2466  }
2467 
2468  SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2469 
2470  SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(Op0.getValueType()),
2471  Op0,
2472  DAG.getConstant(0, dl, Op0.getValueType()),
2473  ISD::SETLT);
2474  SDValue Zero = DAG.getIntPtrConstant(0, dl),
2475  Four = DAG.getIntPtrConstant(4, dl);
2476  SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2477  SignSet, Four, Zero);
2478 
2479  // If the sign bit of the integer is set, the large number will be treated
2480  // as a negative number. To counteract this, the dynamic code adds an
2481  // offset depending on the data type.
2482  uint64_t FF;
2483  switch (Op0.getSimpleValueType().SimpleTy) {
2484  default: llvm_unreachable("Unsupported integer type!");
2485  case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2486  case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2487  case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2488  case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2489  }
2490  if (DAG.getDataLayout().isLittleEndian())
2491  FF <<= 32;
2492  Constant *FudgeFactor = ConstantInt::get(
2493  Type::getInt64Ty(*DAG.getContext()), FF);
2494 
2495  SDValue CPIdx =
2496  DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2497  unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
2498  CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2499  Alignment = std::min(Alignment, 4u);
2500  SDValue FudgeInReg;
2501  if (DestVT == MVT::f32)
2502  FudgeInReg = DAG.getLoad(
2503  MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2504  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2505  Alignment);
2506  else {
2507  SDValue Load = DAG.getExtLoad(
2508  ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2509  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2510  Alignment);
2511  HandleSDNode Handle(Load);
2512  LegalizeOp(Load.getNode());
2513  FudgeInReg = Handle.getValue();
2514  }
2515 
2516  return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2517 }
2518 
2519 /// This function is responsible for legalizing a
2520 /// *INT_TO_FP operation of the specified operand when the target requests that
2521 /// we promote it. At this point, we know that the result and operand types are
2522 /// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2523 /// operation that takes a larger input.
2524 SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT,
2525  bool isSigned,
2526  const SDLoc &dl) {
2527  // First step, figure out the appropriate *INT_TO_FP operation to use.
2528  EVT NewInTy = LegalOp.getValueType();
2529 
2530  unsigned OpToUse = 0;
2531 
2532  // Scan for the appropriate larger type to use.
2533  while (true) {
2534  NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2535  assert(NewInTy.isInteger() && "Ran out of possibilities!");
2536 
2537  // If the target supports SINT_TO_FP of this type, use it.
2538  if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) {
2539  OpToUse = ISD::SINT_TO_FP;
2540  break;
2541  }
2542  if (isSigned) continue;
2543 
2544  // If the target supports UINT_TO_FP of this type, use it.
2545  if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) {
2546  OpToUse = ISD::UINT_TO_FP;
2547  break;
2548  }
2549 
2550  // Otherwise, try a larger type.
2551  }
2552 
2553  // Okay, we found the operation and type to use. Zero extend our input to the
2554  // desired type then run the operation on it.
2555  return DAG.getNode(OpToUse, dl, DestVT,
2556  DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2557  dl, NewInTy, LegalOp));
2558 }
2559 
2560 /// This function is responsible for legalizing a
2561 /// FP_TO_*INT operation of the specified operand when the target requests that
2562 /// we promote it. At this point, we know that the result and operand types are
2563 /// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2564 /// operation that returns a larger result.
2565 SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT,
2566  bool isSigned,
2567  const SDLoc &dl) {
2568  // First step, figure out the appropriate FP_TO*INT operation to use.
2569  EVT NewOutTy = DestVT;
2570 
2571  unsigned OpToUse = 0;
2572 
2573  // Scan for the appropriate larger type to use.
2574  while (true) {
2575  NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2576  assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2577 
2578  // A larger signed type can hold all unsigned values of the requested type,
2579  // so using FP_TO_SINT is valid
2580  if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) {
2581  OpToUse = ISD::FP_TO_SINT;
2582  break;
2583  }
2584 
2585  // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2586  if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) {
2587  OpToUse = ISD::FP_TO_UINT;
2588  break;
2589  }
2590 
2591  // Otherwise, try a larger type.
2592  }
2593 
2594  // Okay, we found the operation and type to use.
2595  SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2596 
2597  // Truncate the result of the extended FP_TO_*INT operation to the desired
2598  // size.
2599  return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2600 }
2601 
2602 /// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts.
2603 SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) {
2604  EVT VT = Op.getValueType();
2605  EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2606  unsigned Sz = VT.getScalarSizeInBits();
2607 
2608  SDValue Tmp, Tmp2, Tmp3;
2609 
2610  // If we can, perform BSWAP first and then the mask+swap the i4, then i2
2611  // and finally the i1 pairs.
2612  // TODO: We can easily support i4/i2 legal types if any target ever does.
2613  if (Sz >= 8 && isPowerOf2_32(Sz)) {
2614  // Create the masks - repeating the pattern every byte.
2615  APInt MaskHi4(Sz, 0), MaskHi2(Sz, 0), MaskHi1(Sz, 0);
2616  APInt MaskLo4(Sz, 0), MaskLo2(Sz, 0), MaskLo1(Sz, 0);
2617  for (unsigned J = 0; J != Sz; J += 8) {
2618  MaskHi4 = MaskHi4 | (0xF0ull << J);
2619  MaskLo4 = MaskLo4 | (0x0Full << J);
2620  MaskHi2 = MaskHi2 | (0xCCull << J);
2621  MaskLo2 = MaskLo2 | (0x33ull << J);
2622  MaskHi1 = MaskHi1 | (0xAAull << J);
2623  MaskLo1 = MaskLo1 | (0x55ull << J);
2624  }
2625 
2626  // BSWAP if the type is wider than a single byte.
2627  Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);
2628 
2629  // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4)
2630  Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT));
2631  Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT));
2632  Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, VT));
2633  Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, VT));
2634  Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2635 
2636  // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2)
2637  Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT));
2638  Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT));
2639  Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, VT));
2640  Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, VT));
2641  Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2642 
2643  // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1)
2644  Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT));
2645  Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT));
2646  Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, VT));
2647  Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, VT));
2648  Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2649  return Tmp;
2650  }
2651 
2652  Tmp = DAG.getConstant(0, dl, VT);
2653  for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
2654  if (I < J)
2655  Tmp2 =
2656  DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
2657  else
2658  Tmp2 =
2659  DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));
2660 
2661  APInt Shift(Sz, 1);
2662  Shift <<= J;
2663  Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
2664  Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
2665  }
2666 
2667  return Tmp;
2668 }
2669 
2670 /// Open code the operations for BSWAP of the specified operation.
2671 SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) {
2672  EVT VT = Op.getValueType();
2673  EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2674  SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
2675  switch (VT.getSimpleVT().getScalarType().SimpleTy) {
2676  default: llvm_unreachable("Unhandled Expand type in BSWAP!");
2677  case MVT::i16:
2678  Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2679  Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2680  return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
2681  case MVT::i32:
2682  Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2683  Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2684  Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2685  Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2686  Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2687  DAG.getConstant(0xFF0000, dl, VT));
2688  Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
2689  Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2690  Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2691  return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2692  case MVT::i64:
2693  Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2694  Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2695  Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2696  Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2697  Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2698  Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2699  Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2700  Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2701  Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,
2702  DAG.getConstant(255ULL<<48, dl, VT));
2703  Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,
2704  DAG.getConstant(255ULL<<40, dl, VT));
2705  Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,
2706  DAG.getConstant(255ULL<<32, dl, VT));
2707  Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
2708  DAG.getConstant(255ULL<<24, dl, VT));
2709  Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2710  DAG.getConstant(255ULL<<16, dl, VT));
2711  Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
2712  DAG.getConstant(255ULL<<8 , dl, VT));
2713  Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
2714  Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
2715  Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2716  Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2717  Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
2718  Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2719  return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
2720  }
2721 }
2722 
2723 /// Expand the specified bitcount instruction into operations.
2724 SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op,
2725  const SDLoc &dl) {
2726  switch (Opc) {
2727  default: llvm_unreachable("Cannot expand this yet!");
2728  case ISD::CTPOP: {
2729  EVT VT = Op.getValueType();
2730  EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2731  unsigned Len = VT.getSizeInBits();
2732 
2733  assert(VT.isInteger() && Len <= 128 && Len % 8 == 0 &&
2734  "CTPOP not implemented for this type.");
2735 
2736  // This is the "best" algorithm from
2737  // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
2738 
2739  SDValue Mask55 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)),
2740  dl, VT);
2741  SDValue Mask33 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)),
2742  dl, VT);
2743  SDValue Mask0F = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)),
2744  dl, VT);
2745  SDValue Mask01 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)),
2746  dl, VT);
2747 
2748  // v = v - ((v >> 1) & 0x55555555...)
2749  Op = DAG.getNode(ISD::SUB, dl, VT, Op,
2750  DAG.getNode(ISD::AND, dl, VT,
2751  DAG.getNode(ISD::SRL, dl, VT, Op,
2752  DAG.getConstant(1, dl, ShVT)),
2753  Mask55));
2754  // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
2755  Op = DAG.getNode(ISD::ADD, dl, VT,
2756  DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
2757  DAG.getNode(ISD::AND, dl, VT,
2758  DAG.getNode(ISD::SRL, dl, VT, Op,
2759  DAG.getConstant(2, dl, ShVT)),
2760  Mask33));
2761  // v = (v + (v >> 4)) & 0x0F0F0F0F...
2762  Op = DAG.getNode(ISD::AND, dl, VT,
2763  DAG.getNode(ISD::ADD, dl, VT, Op,
2764  DAG.getNode(ISD::SRL, dl, VT, Op,
2765  DAG.getConstant(4, dl, ShVT))),
2766  Mask0F);
2767  // v = (v * 0x01010101...) >> (Len - 8)
2768  Op = DAG.getNode(ISD::SRL, dl, VT,
2769  DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
2770  DAG.getConstant(Len - 8, dl, ShVT));
2771 
2772  return Op;
2773  }
2774  case ISD::CTLZ_ZERO_UNDEF:
2775  // This trivially expands to CTLZ.
2776  return DAG.getNode(ISD::CTLZ, dl, Op.getValueType(), Op);
2777  case ISD::CTLZ: {
2778  EVT VT = Op.getValueType();
2779  unsigned Len = VT.getSizeInBits();
2780 
2781  if (TLI.isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)) {
2782  EVT SetCCVT = getSetCCResultType(VT);
2783  SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op);
2784  SDValue Zero = DAG.getConstant(0, dl, VT);
2785  SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
2786  return DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
2787  DAG.getConstant(Len, dl, VT), CTLZ);
2788  }
2789 
2790  // for now, we do this:
2791  // x = x | (x >> 1);
2792  // x = x | (x >> 2);
2793  // ...
2794  // x = x | (x >>16);
2795  // x = x | (x >>32); // for 64-bit input
2796  // return popcount(~x);
2797  //
2798  // Ref: "Hacker's Delight" by Henry Warren
2799  EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2800  for (unsigned i = 0; (1U << i) <= (Len / 2); ++i) {
2801  SDValue Tmp3 = DAG.getConstant(1ULL << i, dl, ShVT);
2802  Op = DAG.getNode(ISD::OR, dl, VT, Op,
2803  DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3));
2804  }
2805  Op = DAG.getNOT(dl, Op, VT);
2806  return DAG.getNode(ISD::CTPOP, dl, VT, Op);
2807  }
2808  case ISD::CTTZ_ZERO_UNDEF:
2809  // This trivially expands to CTTZ.
2810  return DAG.getNode(ISD::CTTZ, dl, Op.getValueType(), Op);
2811  case ISD::CTTZ: {
2812  EVT VT = Op.getValueType();
2813  unsigned Len = VT.getSizeInBits();
2814 
2815  if (TLI.isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) {
2816  EVT SetCCVT = getSetCCResultType(VT);
2817  SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op);
2818  SDValue Zero = DAG.getConstant(0, dl, VT);
2819  SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
2820  return DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
2821  DAG.getConstant(Len, dl, VT), CTTZ);
2822  }
2823 
2824  // for now, we use: { return popcount(~x & (x - 1)); }
2825  // unless the target has ctlz but not ctpop, in which case we use:
2826  // { return 32 - nlz(~x & (x-1)); }
2827  // Ref: "Hacker's Delight" by Henry Warren
2828  SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT,
2829  DAG.getNOT(dl, Op, VT),
2830  DAG.getNode(ISD::SUB, dl, VT, Op,
2831  DAG.getConstant(1, dl, VT)));
2832  // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
2833  if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) &&
2834  TLI.isOperationLegalOrCustom(ISD::CTLZ, VT))
2835  return DAG.getNode(ISD::SUB, dl, VT,
2836  DAG.getConstant(VT.getSizeInBits(), dl, VT),
2837  DAG.getNode(ISD::CTLZ, dl, VT, Tmp3));
2838  return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3);
2839  }
2840  }
2841 }
2842 
2843 bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2844  DEBUG(dbgs() << "Trying to expand node\n");
2846  SDLoc dl(Node);
2847  SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2848  bool NeedInvert;
2849  switch (Node->getOpcode()) {
2850  case ISD::CTPOP:
2851  case ISD::CTLZ:
2852  case ISD::CTLZ_ZERO_UNDEF:
2853  case ISD::CTTZ:
2854  case ISD::CTTZ_ZERO_UNDEF:
2855  Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl);
2856  Results.push_back(Tmp1);
2857  break;
2858  case ISD::BITREVERSE:
2859  Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl));
2860  break;
2861  case ISD::BSWAP:
2862  Results.push_back(ExpandBSWAP(Node->getOperand(0), dl));
2863  break;
2864  case ISD::FRAMEADDR:
2865  case ISD::RETURNADDR:
2867  Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
2868  break;
2869  case ISD::EH_DWARF_CFA: {
2870  SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
2871  TLI.getPointerTy(DAG.getDataLayout()));
2872  SDValue Offset = DAG.getNode(ISD::ADD, dl,
2873  CfaArg.getValueType(),
2874  DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
2875  CfaArg.getValueType()),
2876  CfaArg);
2877  SDValue FA = DAG.getNode(
2878  ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
2879  DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
2880  Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
2881  FA, Offset));
2882  break;
2883  }
2884  case ISD::FLT_ROUNDS_:
2885  Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
2886  break;
2887  case ISD::EH_RETURN:
2888  case ISD::EH_LABEL:
2889  case ISD::PREFETCH:
2890  case ISD::VAEND:
2891  case ISD::EH_SJLJ_LONGJMP:
2892  // If the target didn't expand these, there's nothing to do, so just
2893  // preserve the chain and be done.
2894  Results.push_back(Node->getOperand(0));
2895  break;
2896  case ISD::READCYCLECOUNTER:
2897  // If the target didn't expand this, just return 'zero' and preserve the
2898  // chain.
2899  Results.append(Node->getNumValues() - 1,
2900  DAG.getConstant(0, dl, Node->getValueType(0)));
2901  Results.push_back(Node->getOperand(0));
2902  break;
2903  case ISD::EH_SJLJ_SETJMP:
2904  // If the target didn't expand this, just return 'zero' and preserve the
2905  // chain.
2906  Results.push_back(DAG.getConstant(0, dl, MVT::i32));
2907  Results.push_back(Node->getOperand(0));
2908  break;
2909  case ISD::ATOMIC_LOAD: {
2910  // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
2911  SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
2912  SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2913  SDValue Swap = DAG.getAtomicCmpSwap(
2914  ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2915  Node->getOperand(0), Node->getOperand(1), Zero, Zero,
2916  cast<AtomicSDNode>(Node)->getMemOperand());
2917  Results.push_back(Swap.getValue(0));
2918  Results.push_back(Swap.getValue(1));
2919  break;
2920  }
2921  case ISD::ATOMIC_STORE: {
2922  // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
2923  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2924  cast<AtomicSDNode>(Node)->getMemoryVT(),
2925  Node->getOperand(0),
2926  Node->getOperand(1), Node->getOperand(2),
2927  cast<AtomicSDNode>(Node)->getMemOperand());
2928  Results.push_back(Swap.getValue(1));
2929  break;
2930  }
2932  // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
2933  // splits out the success value as a comparison. Expanding the resulting
2934  // ATOMIC_CMP_SWAP will produce a libcall.
2935  SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2936  SDValue Res = DAG.getAtomicCmpSwap(
2937  ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2938  Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
2939  Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
2940 
2941  SDValue ExtRes = Res;
2942  SDValue LHS = Res;
2943  SDValue RHS = Node->getOperand(1);
2944 
2945  EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
2946  EVT OuterType = Node->getValueType(0);
2947  switch (TLI.getExtendForAtomicOps()) {
2948  case ISD::SIGN_EXTEND:
2949  LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
2950  DAG.getValueType(AtomicType));
2951  RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
2952  Node->getOperand(2), DAG.getValueType(AtomicType));
2953  ExtRes = LHS;
2954  break;
2955  case ISD::ZERO_EXTEND:
2956  LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
2957  DAG.getValueType(AtomicType));
2958  RHS = DAG.getNode(ISD::ZERO_EXTEND, dl, OuterType, Node->getOperand(2));
2959  ExtRes = LHS;
2960  break;
2961  case ISD::ANY_EXTEND:
2962  LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
2963  RHS = DAG.getNode(ISD::ZERO_EXTEND, dl, OuterType, Node->getOperand(2));
2964  break;
2965  default:
2966  llvm_unreachable("Invalid atomic op extension");
2967  }
2968 
2969  SDValue Success =
2970  DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
2971 
2972  Results.push_back(ExtRes.getValue(0));
2973  Results.push_back(Success);
2974  Results.push_back(Res.getValue(1));
2975  break;
2976  }
2978  ExpandDYNAMIC_STACKALLOC(Node, Results);
2979  break;
2980  case ISD::MERGE_VALUES:
2981  for (unsigned i = 0; i < Node->getNumValues(); i++)
2982  Results.push_back(Node->getOperand(i));
2983  break;
2984  case ISD::UNDEF: {
2985  EVT VT = Node->getValueType(0);
2986  if (VT.isInteger())
2987  Results.push_back(DAG.getConstant(0, dl, VT));
2988  else {
2989  assert(VT.isFloatingPoint() && "Unknown value type!");
2990  Results.push_back(DAG.getConstantFP(0, dl, VT));
2991  }
2992  break;
2993  }
2994  case ISD::FP_ROUND:
2995  case ISD::BITCAST:
2996  Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
2997  Node->getValueType(0), dl);
2998  Results.push_back(Tmp1);
2999  break;
3000  case ISD::FP_EXTEND:
3001  Tmp1 = EmitStackConvert(Node->getOperand(0),
3002  Node->getOperand(0).getValueType(),
3003  Node->getValueType(0), dl);
3004  Results.push_back(Tmp1);
3005  break;
3006  case ISD::SIGN_EXTEND_INREG: {
3007  EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
3008  EVT VT = Node->getValueType(0);
3009 
3010  // An in-register sign-extend of a boolean is a negation:
3011  // 'true' (1) sign-extended is -1.
3012  // 'false' (0) sign-extended is 0.
3013  // However, we must mask the high bits of the source operand because the
3014  // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
3015 
3016  // TODO: Do this for vectors too?
3017  if (ExtraVT.getSizeInBits() == 1) {
3018  SDValue One = DAG.getConstant(1, dl, VT);
3019  SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
3020  SDValue Zero = DAG.getConstant(0, dl, VT);
3021  SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
3022  Results.push_back(Neg);
3023  break;
3024  }
3025 
3026  // NOTE: we could fall back on load/store here too for targets without
3027  // SRA. However, it is doubtful that any exist.
3028  EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3029  unsigned BitsDiff = VT.getScalarSizeInBits() -
3030  ExtraVT.getScalarSizeInBits();
3031  SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
3032  Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
3033  Node->getOperand(0), ShiftCst);
3034  Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
3035  Results.push_back(Tmp1);
3036  break;
3037  }
3038  case ISD::FP_ROUND_INREG: {
3039  // The only way we can lower this is to turn it into a TRUNCSTORE,
3040  // EXTLOAD pair, targeting a temporary location (a stack slot).
3041 
3042  // NOTE: there is a choice here between constantly creating new stack
3043  // slots and always reusing the same one. We currently always create
3044  // new ones, as reuse may inhibit scheduling.
3045  EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
3046  Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT,
3047  Node->getValueType(0), dl);
3048  Results.push_back(Tmp1);
3049  break;
3050  }
3051  case ISD::SINT_TO_FP:
3052  case ISD::UINT_TO_FP:
3053  Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP,
3054  Node->getOperand(0), Node->getValueType(0), dl);
3055  Results.push_back(Tmp1);
3056  break;
3057  case ISD::FP_TO_SINT:
3058  if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
3059  Results.push_back(Tmp1);
3060  break;
3061  case ISD::FP_TO_UINT: {
3062  SDValue True, False;
3063  EVT VT = Node->getOperand(0).getValueType();
3064  EVT NVT = Node->getValueType(0);
3065  APFloat apf(DAG.EVTToAPFloatSemantics(VT),
3068  (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven);
3069  Tmp1 = DAG.getConstantFP(apf, dl, VT);
3070  Tmp2 = DAG.getSetCC(dl, getSetCCResultType(VT),
3071  Node->getOperand(0),
3072  Tmp1, ISD::SETLT);
3073  True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0));
3074  // TODO: Should any fast-math-flags be set for the FSUB?
3075  False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT,
3076  DAG.getNode(ISD::FSUB, dl, VT,
3077  Node->getOperand(0), Tmp1));
3078  False = DAG.getNode(ISD::XOR, dl, NVT, False,
3079  DAG.getConstant(x, dl, NVT));
3080  Tmp1 = DAG.getSelect(dl, NVT, Tmp2, True, False);
3081  Results.push_back(Tmp1);
3082  break;
3083  }
3084  case ISD::VAARG:
3085  Results.push_back(DAG.expandVAArg(Node));
3086  Results.push_back(Results[0].getValue(1));
3087  break;
3088  case ISD::VACOPY:
3089  Results.push_back(DAG.expandVACopy(Node));
3090  break;
3092  if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
3093  // This must be an access of the only element. Return it.
3094  Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
3095  Node->getOperand(0));
3096  else
3097  Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
3098  Results.push_back(Tmp1);
3099  break;
3101  Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
3102  break;
3103  case ISD::INSERT_SUBVECTOR:
3104  Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
3105  break;
3106  case ISD::CONCAT_VECTORS:
3107  Results.push_back(ExpandVectorBuildThroughStack(Node));
3108  break;
3109  case ISD::SCALAR_TO_VECTOR:
3110  Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
3111  break;
3113  Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
3114  Node->getOperand(1),
3115  Node->getOperand(2), dl));
3116  break;
3117  case ISD::VECTOR_SHUFFLE: {
3118  SmallVector<int, 32> NewMask;
3119  ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
3120 
3121  EVT VT = Node->getValueType(0);
3122  EVT EltVT = VT.getVectorElementType();
3123  SDValue Op0 = Node->getOperand(0);
3124  SDValue Op1 = Node->getOperand(1);
3125  if (!TLI.isTypeLegal(EltVT)) {
3126  EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3127 
3128  // BUILD_VECTOR operands are allowed to be wider than the element type.
3129  // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3130  // it.
3131  if (NewEltVT.bitsLT(EltVT)) {
3132  // Convert shuffle node.
3133  // If original node was v4i64 and the new EltVT is i32,
3134  // cast operands to v8i32 and re-build the mask.
3135 
3136  // Calculate new VT, the size of the new VT should be equal to original.
3137  EVT NewVT =
3138  EVT::getVectorVT(*DAG.getContext(), NewEltVT,
3139  VT.getSizeInBits() / NewEltVT.getSizeInBits());
3140  assert(NewVT.bitsEq(VT));
3141 
3142  // cast operands to new VT
3143  Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
3144  Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
3145 
3146  // Convert the shuffle mask
3147  unsigned int factor =
3149 
3150  // EltVT gets smaller
3151  assert(factor > 0);
3152 
3153  for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3154  if (Mask[i] < 0) {
3155  for (unsigned fi = 0; fi < factor; ++fi)
3156  NewMask.push_back(Mask[i]);
3157  }
3158  else {
3159  for (unsigned fi = 0; fi < factor; ++fi)
3160  NewMask.push_back(Mask[i]*factor+fi);
3161  }
3162  }
3163  Mask = NewMask;
3164  VT = NewVT;
3165  }
3166  EltVT = NewEltVT;
3167  }
3168  unsigned NumElems = VT.getVectorNumElements();
3170  for (unsigned i = 0; i != NumElems; ++i) {
3171  if (Mask[i] < 0) {
3172  Ops.push_back(DAG.getUNDEF(EltVT));
3173  continue;
3174  }
3175  unsigned Idx = Mask[i];
3176  if (Idx < NumElems)
3177  Ops.push_back(DAG.getNode(
3178  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
3179  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
3180  else
3181  Ops.push_back(DAG.getNode(
3182  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
3183  DAG.getConstant(Idx - NumElems, dl,
3184  TLI.getVectorIdxTy(DAG.getDataLayout()))));
3185  }
3186 
3187  Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3188  // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3189  Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3190  Results.push_back(Tmp1);
3191  break;
3192  }
3193  case ISD::EXTRACT_ELEMENT: {
3194  EVT OpTy = Node->getOperand(0).getValueType();
3195  if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
3196  // 1 -> Hi
3197  Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3198  DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
3199  TLI.getShiftAmountTy(
3200  Node->getOperand(0).getValueType(),
3201  DAG.getDataLayout())));
3202  Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3203  } else {
3204  // 0 -> Lo
3205  Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3206  Node->getOperand(0));
3207  }
3208  Results.push_back(Tmp1);
3209  break;
3210  }
3211  case ISD::STACKSAVE:
3212  // Expand to CopyFromReg if the target set
3213  // StackPointerRegisterToSaveRestore.
3214  if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3215  Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3216  Node->getValueType(0)));
3217  Results.push_back(Results[0].getValue(1));
3218  } else {
3219  Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3220  Results.push_back(Node->getOperand(0));
3221  }
3222  break;
3223  case ISD::STACKRESTORE:
3224  // Expand to CopyToReg if the target set
3225  // StackPointerRegisterToSaveRestore.
3226  if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3227  Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3228  Node->getOperand(1)));
3229  } else {
3230  Results.push_back(Node->getOperand(0));
3231  }
3232  break;
3234  Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3235  Results.push_back(Results[0].getValue(0));
3236  break;
3237  case ISD::FCOPYSIGN:
3238  Results.push_back(ExpandFCOPYSIGN(Node));
3239  break;
3240  case ISD::FNEG:
3241  // Expand Y = FNEG(X) -> Y = SUB -0.0, X
3242  Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0));
3243  // TODO: If FNEG has fast-math-flags, propagate them to the FSUB.
3244  Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1,
3245  Node->getOperand(0));
3246  Results.push_back(Tmp1);
3247  break;
3248  case ISD::FABS:
3249  Results.push_back(ExpandFABS(Node));
3250  break;
3251  case ISD::SMIN:
3252  case ISD::SMAX:
3253  case ISD::UMIN:
3254  case ISD::UMAX: {
3255  // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3256  ISD::CondCode Pred;
3257  switch (Node->getOpcode()) {
3258  default: llvm_unreachable("How did we get here?");
3259  case ISD::SMAX: Pred = ISD::SETGT; break;
3260  case ISD::SMIN: Pred = ISD::SETLT; break;
3261  case ISD::UMAX: Pred = ISD::SETUGT; break;
3262  case ISD::UMIN: Pred = ISD::SETULT; break;
3263  }
3264  Tmp1 = Node->getOperand(0);
3265  Tmp2 = Node->getOperand(1);
3266  Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3267  Results.push_back(Tmp1);
3268  break;
3269  }
3270 
3271  case ISD::FSIN:
3272  case ISD::FCOS: {
3273  EVT VT = Node->getValueType(0);
3274  // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3275  // fcos which share the same operand and both are used.
3276  if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3277  isSinCosLibcallAvailable(Node, TLI))
3278  && useSinCos(Node)) {
3279  SDVTList VTs = DAG.getVTList(VT, VT);
3280  Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3281  if (Node->getOpcode() == ISD::FCOS)
3282  Tmp1 = Tmp1.getValue(1);
3283  Results.push_back(Tmp1);
3284  }
3285  break;
3286  }
3287  case ISD::FMAD:
3288  llvm_unreachable("Illegal fmad should never be formed");
3289 
3290  case ISD::FP16_TO_FP:
3291  if (Node->getValueType(0) != MVT::f32) {
3292  // We can extend to types bigger than f32 in two steps without changing
3293  // the result. Since "f16 -> f32" is much more commonly available, give
3294  // CodeGen the option of emitting that before resorting to a libcall.
3295  SDValue Res =
3296  DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3297  Results.push_back(
3298  DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3299  }
3300  break;
3301  case ISD::FP_TO_FP16:
3302  DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
3303  if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3304  SDValue Op = Node->getOperand(0);
3305  MVT SVT = Op.getSimpleValueType();
3306  if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3307  TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3308  // Under fastmath, we can expand this node into a fround followed by
3309  // a float-half conversion.
3310  SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3311  DAG.getIntPtrConstant(0, dl));
3312  Results.push_back(
3313  DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3314  }
3315  }
3316  break;
3317  case ISD::ConstantFP: {
3318  ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3319  // Check to see if this FP immediate is already legal.
3320  // If this is a legal constant, turn it into a TargetConstantFP node.
3321  if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0)))
3322  Results.push_back(ExpandConstantFP(CFP, true));
3323  break;
3324  }
3325  case ISD::Constant: {
3326  ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3327  Results.push_back(ExpandConstant(CP));
3328  break;
3329  }
3330  case ISD::FSUB: {
3331  EVT VT = Node->getValueType(0);
3332  if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3333  TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3334  const SDNodeFlags Flags = Node->getFlags();
3335  Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3336  Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3337  Results.push_back(Tmp1);
3338  }
3339  break;
3340  }
3341  case ISD::SUB: {
3342  EVT VT = Node->getValueType(0);
3343  assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3344  TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3345  "Don't know how to expand this subtraction!");
3346  Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
3347  DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
3348  VT));
3349  Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3350  Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3351  break;
3352  }
3353  case ISD::UREM:
3354  case ISD::SREM: {
3355  EVT VT = Node->getValueType(0);
3356  bool isSigned = Node->getOpcode() == ISD::SREM;
3357  unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
3358  unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3359  Tmp2 = Node->getOperand(0);
3360  Tmp3 = Node->getOperand(1);
3361  if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3362  SDVTList VTs = DAG.getVTList(VT, VT);
3363  Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1);
3364  Results.push_back(Tmp1);
3365  } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) {
3366  // X % Y -> X-X/Y*Y
3367  Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3);
3368  Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3);
3369  Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1);
3370  Results.push_back(Tmp1);
3371  }
3372  break;
3373  }
3374  case ISD::UDIV:
3375  case ISD::SDIV: {
3376  bool isSigned = Node->getOpcode() == ISD::SDIV;
3377  unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3378  EVT VT = Node->getValueType(0);
3379  if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3380  SDVTList VTs = DAG.getVTList(VT, VT);
3381  Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3382  Node->getOperand(1));
3383  Results.push_back(Tmp1);
3384  }
3385  break;
3386  }
3387  case ISD::MULHU:
3388  case ISD::MULHS: {
3389  unsigned ExpandOpcode =
3391  EVT VT = Node->getValueType(0);
3392  SDVTList VTs = DAG.getVTList(VT, VT);
3393 
3394  Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3395  Node->getOperand(1));
3396  Results.push_back(Tmp1.getValue(1));
3397  break;
3398  }
3399  case ISD::UMUL_LOHI:
3400  case ISD::SMUL_LOHI: {
3401  SDValue LHS = Node->getOperand(0);
3402  SDValue RHS = Node->getOperand(1);
3403  MVT VT = LHS.getSimpleValueType();
3404  unsigned MULHOpcode =
3406 
3407  if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3408  Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3409  Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3410  break;
3411  }
3412 
3413  SmallVector<SDValue, 4> Halves;
3414  EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3415  assert(TLI.isTypeLegal(HalfType));
3416  if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves,
3417  HalfType, DAG,
3419  for (unsigned i = 0; i < 2; ++i) {
3420  SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3421  SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3422  SDValue Shift = DAG.getConstant(
3423  HalfType.getScalarSizeInBits(), dl,
3424  TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3425  Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3426  Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3427  }
3428  break;
3429  }
3430  break;
3431  }
3432  case ISD::MUL: {
3433  EVT VT = Node->getValueType(0);
3434  SDVTList VTs = DAG.getVTList(VT, VT);
3435  // See if multiply or divide can be lowered using two-result operations.
3436  // We just need the low half of the multiply; try both the signed
3437  // and unsigned forms. If the target supports both SMUL_LOHI and
3438  // UMUL_LOHI, form a preference by checking which forms of plain
3439  // MULH it supports.
3440  bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3441  bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3442  bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3443  bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3444  unsigned OpToUse = 0;
3445  if (HasSMUL_LOHI && !HasMULHS) {
3446  OpToUse = ISD::SMUL_LOHI;
3447  } else if (HasUMUL_LOHI && !HasMULHU) {
3448  OpToUse = ISD::UMUL_LOHI;
3449  } else if (HasSMUL_LOHI) {
3450  OpToUse = ISD::SMUL_LOHI;
3451  } else if (HasUMUL_LOHI) {
3452  OpToUse = ISD::UMUL_LOHI;
3453  }
3454  if (OpToUse) {
3455  Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3456  Node->getOperand(1)));
3457  break;
3458  }
3459 
3460  SDValue Lo, Hi;
3461  EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3462  if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3463  TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3464  TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3465  TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3466  TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3468  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3469  Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3470  SDValue Shift =
3471  DAG.getConstant(HalfType.getSizeInBits(), dl,
3472  TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3473  Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3474  Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3475  }
3476  break;
3477  }
3478  case ISD::SADDO:
3479  case ISD::SSUBO: {
3480  SDValue LHS = Node->getOperand(0);
3481  SDValue RHS = Node->getOperand(1);
3482  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3483  ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3484  LHS, RHS);
3485  Results.push_back(Sum);
3486  EVT ResultType = Node->getValueType(1);
3487  EVT OType = getSetCCResultType(Node->getValueType(0));
3488 
3489  SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
3490 
3491  // LHSSign -> LHS >= 0
3492  // RHSSign -> RHS >= 0
3493  // SumSign -> Sum >= 0
3494  //
3495  // Add:
3496  // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3497  // Sub:
3498  // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3499  SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3500  SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3501  SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
3502  Node->getOpcode() == ISD::SADDO ?
3504 
3505  SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3506  SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3507 
3508  SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3509  Results.push_back(DAG.getBoolExtOrTrunc(Cmp, dl, ResultType, ResultType));
3510  break;
3511  }
3512  case ISD::UADDO:
3513  case ISD::USUBO: {
3514  SDValue LHS = Node->getOperand(0);
3515  SDValue RHS = Node->getOperand(1);
3516  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ?
3517  ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3518  LHS, RHS);
3519  Results.push_back(Sum);
3520 
3521  EVT ResultType = Node->getValueType(1);
3522  EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3523  ISD::CondCode CC
3524  = Node->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
3525  SDValue SetCC = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3526 
3527  Results.push_back(DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType));
3528  break;
3529  }
3530  case ISD::UMULO:
3531  case ISD::SMULO: {
3532  EVT VT = Node->getValueType(0);
3533  EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2);
3534  SDValue LHS = Node->getOperand(0);
3535  SDValue RHS = Node->getOperand(1);
3536  SDValue BottomHalf;
3537  SDValue TopHalf;
3538  static const unsigned Ops[2][3] =
3541  bool isSigned = Node->getOpcode() == ISD::SMULO;
3542  if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
3543  BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
3544  TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);
3545  } else if (TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
3546  BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,
3547  RHS);
3548  TopHalf = BottomHalf.getValue(1);
3549  } else if (TLI.isTypeLegal(WideVT)) {
3550  LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
3551  RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
3552  Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
3553  BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3554  DAG.getIntPtrConstant(0, dl));
3555  TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3556  DAG.getIntPtrConstant(1, dl));
3557  } else {
3558  // We can fall back to a libcall with an illegal type for the MUL if we
3559  // have a libcall big enough.
3560  // Also, we can fall back to a division in some cases, but that's a big
3561  // performance hit in the general case.
3562  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3563  if (WideVT == MVT::i16)
3564  LC = RTLIB::MUL_I16;
3565  else if (WideVT == MVT::i32)
3566  LC = RTLIB::MUL_I32;
3567  else if (WideVT == MVT::i64)
3568  LC = RTLIB::MUL_I64;
3569  else if (WideVT == MVT::i128)
3570  LC = RTLIB::MUL_I128;
3571  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!");
3572 
3573  SDValue HiLHS;
3574  SDValue HiRHS;
3575  if (isSigned) {
3576  // The high part is obtained by SRA'ing all but one of the bits of low
3577  // part.
3578  unsigned LoSize = VT.getSizeInBits();
3579  HiLHS =
3580  DAG.getNode(ISD::SRA, dl, VT, LHS,
3581  DAG.getConstant(LoSize - 1, dl,
3582  TLI.getPointerTy(DAG.getDataLayout())));
3583  HiRHS =
3584  DAG.getNode(ISD::SRA, dl, VT, RHS,
3585  DAG.getConstant(LoSize - 1, dl,
3586  TLI.getPointerTy(DAG.getDataLayout())));
3587  } else {
3588  HiLHS = DAG.getConstant(0, dl, VT);
3589  HiRHS = DAG.getConstant(0, dl, VT);
3590  }
3591 
3592  // Here we're passing the 2 arguments explicitly as 4 arguments that are
3593  // pre-lowered to the correct types. This all depends upon WideVT not
3594  // being a legal type for the architecture and thus has to be split to
3595  // two arguments.
3596  SDValue Ret;
3597  if(DAG.getDataLayout().isLittleEndian()) {
3598  // Halves of WideVT are packed into registers in different order
3599  // depending on platform endianness. This is usually handled by
3600  // the C calling convention, but we can't defer to it in
3601  // the legalizer.
3602  SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
3603  Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
3604  } else {
3605  SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };
3606  Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
3607  }
3608  assert(Ret.getOpcode() == ISD::MERGE_VALUES &&
3609  "Ret value is a collection of constituent nodes holding result.");
3610  BottomHalf = Ret.getOperand(0);
3611  TopHalf = Ret.getOperand(1);
3612  }
3613 
3614  if (isSigned) {
3615  Tmp1 = DAG.getConstant(
3616  VT.getSizeInBits() - 1, dl,
3617  TLI.getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));
3618  Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1);
3619  TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, Tmp1,
3620  ISD::SETNE);
3621  } else {
3622  TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf,
3623  DAG.getConstant(0, dl, VT), ISD::SETNE);
3624  }
3625 
3626  // Truncate the result if SetCC returns a larger type than needed.
3627  EVT RType = Node->getValueType(1);
3628  if (RType.getSizeInBits() < TopHalf.getValueSizeInBits())
3629  TopHalf = DAG.getNode(ISD::TRUNCATE, dl, RType, TopHalf);
3630 
3631  assert(RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&
3632  "Unexpected result type for S/UMULO legalization");
3633 
3634  Results.push_back(BottomHalf);
3635  Results.push_back(TopHalf);
3636  break;
3637  }
3638  case ISD::BUILD_PAIR: {
3639  EVT PairTy = Node->getValueType(0);
3640  Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3641  Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3642  Tmp2 = DAG.getNode(
3643  ISD::SHL, dl, PairTy, Tmp2,
3644  DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3645  TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3646  Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3647  break;
3648  }
3649  case ISD::SELECT:
3650  Tmp1 = Node->getOperand(0);
3651  Tmp2 = Node->getOperand(1);
3652  Tmp3 = Node->getOperand(2);
3653  if (Tmp1.getOpcode() == ISD::SETCC) {
3654  Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3655  Tmp2, Tmp3,
3656  cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3657  } else {
3658  Tmp1 = DAG.getSelectCC(dl, Tmp1,
3659  DAG.getConstant(0, dl, Tmp1.getValueType()),
3660  Tmp2, Tmp3, ISD::SETNE);
3661  }
3662  Results.push_back(Tmp1);
3663  break;
3664  case ISD::BR_JT: {
3665  SDValue Chain = Node->getOperand(0);
3666  SDValue Table = Node->getOperand(1);
3667  SDValue Index = Node->getOperand(2);
3668 
3669  const DataLayout &TD = DAG.getDataLayout();
3670  EVT PTy = TLI.getPointerTy(TD);
3671 
3672  unsigned EntrySize =
3673  DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3674 
3675  Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3676  DAG.getConstant(EntrySize, dl, Index.getValueType()));
3677  SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3678  Index, Table);
3679 
3680  EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3681  SDValue LD = DAG.getExtLoad(
3682  ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3683  MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3684  Addr = LD;
3685  if (TLI.isJumpTableRelative()) {
3686  // For PIC, the sequence is:
3687  // BRIND(load(Jumptable + index) + RelocBase)
3688  // RelocBase can be JumpTable, GOT or some sort of global base.
3689  Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3690  TLI.getPICJumpTableRelocBase(Table, DAG));
3691  }
3692  Tmp1 = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr);
3693  Results.push_back(Tmp1);
3694  break;
3695  }
3696  case ISD::BRCOND:
3697  // Expand brcond's setcc into its constituent parts and create a BR_CC
3698  // Node.
3699  Tmp1 = Node->getOperand(0);
3700  Tmp2 = Node->getOperand(1);
3701  if (Tmp2.getOpcode() == ISD::SETCC) {
3702  Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
3703  Tmp1, Tmp2.getOperand(2),
3704  Tmp2.getOperand(0), Tmp2.getOperand(1),
3705  Node->getOperand(2));
3706  } else {
3707  // We test only the i1 bit. Skip the AND if UNDEF or another AND.
3708  if (Tmp2.isUndef() ||
3709  (Tmp2.getOpcode() == ISD::AND &&
3710  isa<ConstantSDNode>(Tmp2.getOperand(1)) &&
3711  dyn_cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1))
3712  Tmp3 = Tmp2;
3713  else
3714  Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3715  DAG.getConstant(1, dl, Tmp2.getValueType()));
3716  Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3717  DAG.getCondCode(ISD::SETNE), Tmp3,
3718  DAG.getConstant(0, dl, Tmp3.getValueType()),
3719  Node->getOperand(2));
3720  }
3721  Results.push_back(Tmp1);
3722  break;
3723  case ISD::SETCC: {
3724  Tmp1 = Node->getOperand(0);
3725  Tmp2 = Node->getOperand(1);
3726  Tmp3 = Node->getOperand(2);
3727  bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2,
3728  Tmp3, NeedInvert, dl);
3729 
3730  if (Legalized) {
3731  // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3732  // condition code, create a new SETCC node.
3733  if (Tmp3.getNode())
3734  Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
3735  Tmp1, Tmp2, Tmp3);
3736 
3737  // If we expanded the SETCC by inverting the condition code, then wrap
3738  // the existing SETCC in a NOT to restore the intended condition.
3739  if (NeedInvert)
3740  Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
3741 
3742  Results.push_back(Tmp1);
3743  break;
3744  }
3745 
3746  // Otherwise, SETCC for the given comparison type must be completely
3747  // illegal; expand it into a SELECT_CC.
3748  EVT VT = Node->getValueType(0);
3749  int TrueValue;
3750  switch (TLI.getBooleanContents(Tmp1->getValueType(0))) {
3753  TrueValue = 1;
3754  break;
3756  TrueValue = -1;
3757  break;
3758  }
3759  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
3760  DAG.getConstant(TrueValue, dl, VT),
3761  DAG.getConstant(0, dl, VT),
3762  Tmp3);
3763  Results.push_back(Tmp1);
3764  break;
3765  }
3766  case ISD::SELECT_CC: {
3767  Tmp1 = Node->getOperand(0); // LHS
3768  Tmp2 = Node->getOperand(1); // RHS
3769  Tmp3 = Node->getOperand(2); // True
3770  Tmp4 = Node->getOperand(3); // False
3771  EVT VT = Node->getValueType(0);
3772  SDValue CC = Node->getOperand(4);
3773  ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
3774 
3775  if (TLI.isCondCodeLegal(CCOp, Tmp1.getSimpleValueType())) {
3776  // If the condition code is legal, then we need to expand this
3777  // node using SETCC and SELECT.
3778  EVT CmpVT = Tmp1.getValueType();
3779  assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
3780  "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
3781  "expanded.");
3782  EVT CCVT =
3783  TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT);
3784  SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC);
3785  Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
3786  break;
3787  }
3788 
3789  // SELECT_CC is legal, so the condition code must not be.
3790  bool Legalized = false;
3791  // Try to legalize by inverting the condition. This is for targets that
3792  // might support an ordered version of a condition, but not the unordered
3793  // version (or vice versa).
3794  ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp,
3795  Tmp1.getValueType().isInteger());
3796  if (TLI.isCondCodeLegal(InvCC, Tmp1.getSimpleValueType())) {
3797  // Use the new condition code and swap true and false
3798  Legalized = true;
3799  Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
3800  } else {
3801  // If The inverse is not legal, then try to swap the arguments using
3802  // the inverse condition code.
3803  ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
3804  if (TLI.isCondCodeLegal(SwapInvCC, Tmp1.getSimpleValueType())) {
3805  // The swapped inverse condition is legal, so swap true and false,
3806  // lhs and rhs.
3807  Legalized = true;
3808  Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
3809  }
3810  }
3811 
3812  if (!Legalized) {
3813  Legalized = LegalizeSetCCCondCode(
3814  getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert,
3815  dl);
3816 
3817  assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
3818 
3819  // If we expanded the SETCC by inverting the condition code, then swap
3820  // the True/False operands to match.
3821  if (NeedInvert)
3822  std::swap(Tmp3, Tmp4);
3823 
3824  // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3825  // condition code, create a new SELECT_CC node.
3826  if (CC.getNode()) {
3827  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
3828  Tmp1, Tmp2, Tmp3, Tmp4, CC);
3829  } else {
3830  Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
3831  CC = DAG.getCondCode(ISD::SETNE);
3832  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
3833  Tmp2, Tmp3, Tmp4, CC);
3834  }
3835  }
3836  Results.push_back(Tmp1);
3837  break;
3838  }
3839  case ISD::BR_CC: {
3840  Tmp1 = Node->getOperand(0); // Chain
3841  Tmp2 = Node->getOperand(2); // LHS
3842  Tmp3 = Node->getOperand(3); // RHS
3843  Tmp4 = Node->getOperand(1); // CC
3844 
3845  bool Legalized = LegalizeSetCCCondCode(getSetCCResultType(
3846  Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl);
3847  (void)Legalized;
3848  assert(Legalized && "Can't legalize BR_CC with legal condition!");
3849 
3850  // If we expanded the SETCC by inverting the condition code, then wrap
3851  // the existing SETCC in a NOT to restore the intended condition.
3852  if (NeedInvert)
3853  Tmp4 = DAG.getNOT(dl, Tmp4, Tmp4->getValueType(0));
3854 
3855  // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
3856  // node.
3857  if (Tmp4.getNode()) {
3858  Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
3859  Tmp4, Tmp2, Tmp3, Node->getOperand(4));
3860  } else {
3861  Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
3862  Tmp4 = DAG.getCondCode(ISD::SETNE);
3863  Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
3864  Tmp2, Tmp3, Node->getOperand(4));
3865  }
3866  Results.push_back(Tmp1);
3867  break;
3868  }
3869  case ISD::BUILD_VECTOR:
3870  Results.push_back(ExpandBUILD_VECTOR(Node));
3871  break;
3872  case ISD::SRA:
3873  case ISD::SRL:
3874  case ISD::SHL: {
3875  // Scalarize vector SRA/SRL/SHL.
3876  EVT VT = Node->getValueType(0);
3877  assert(VT.isVector() && "Unable to legalize non-vector shift");
3878  assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
3879  unsigned NumElem = VT.getVectorNumElements();
3880 
3881  SmallVector<SDValue, 8> Scalars;
3882  for (unsigned Idx = 0; Idx < NumElem; Idx++) {
3883  SDValue Ex = DAG.getNode(
3884  ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0),
3885  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3886  SDValue Sh = DAG.getNode(
3887  ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1),
3888  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3889  Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
3890  VT.getScalarType(), Ex, Sh));
3891  }
3892 
3893  SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
3894  ReplaceNode(SDValue(Node, 0), Result);
3895  break;
3896  }
3898  case ISD::GlobalAddress:
3899  case ISD::GlobalTLSAddress:
3900  case ISD::ExternalSymbol:
3901  case ISD::ConstantPool:
3902  case ISD::JumpTable:
3905  case ISD::INTRINSIC_VOID:
3906  // FIXME: Custom lowering for these operations shouldn't return null!
3907  break;
3908  }
3909 
3910  // Replace the original node with the legalized result.
3911  if (Results.empty()) {
3912  DEBUG(dbgs() << "Cannot expand node\n");
3913  return false;
3914  }
3915 
3916  DEBUG(dbgs() << "Succesfully expanded node\n");
3917  ReplaceNode(Node, Results.data());
3918  return true;
3919 }
3920 
3921 void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
3922  DEBUG(dbgs() << "Trying to convert node to libcall\n");
3924  SDLoc dl(Node);
3925  unsigned Opc = Node->getOpcode();
3926  switch (Opc) {
3927  case ISD::ATOMIC_FENCE: {
3928  // If the target didn't lower this, lower it to '__sync_synchronize()' call
3929  // FIXME: handle "fence singlethread" more efficiently.
3931 
3933  CLI.setDebugLoc(dl)
3934  .setChain(Node->getOperand(0))
3935  .setLibCallee(
3936  CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3937  DAG.getExternalSymbol("__sync_synchronize",
3938  TLI.getPointerTy(DAG.getDataLayout())),
3939  std::move(Args));
3940 
3941  std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3942 
3943  Results.push_back(CallResult.second);
3944  break;
3945  }
3946  // By default, atomic intrinsics are marked Legal and lowered. Targets
3947  // which don't support them directly, however, may want libcalls, in which
3948  // case they mark them Expand, and we get here.
3949  case ISD::ATOMIC_SWAP:
3950  case ISD::ATOMIC_LOAD_ADD:
3951  case ISD::ATOMIC_LOAD_SUB:
3952  case ISD::ATOMIC_LOAD_AND:
3953  case ISD::ATOMIC_LOAD_OR:
3954  case ISD::ATOMIC_LOAD_XOR:
3955  case ISD::ATOMIC_LOAD_NAND:
3956  case ISD::ATOMIC_LOAD_MIN:
3957  case ISD::ATOMIC_LOAD_MAX:
3958  case ISD::ATOMIC_LOAD_UMIN:
3959  case ISD::ATOMIC_LOAD_UMAX:
3960  case ISD::ATOMIC_CMP_SWAP: {
3961  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
3962  RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
3963  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
3964 
3965  std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false);
3966  Results.push_back(Tmp.first);
3967  Results.push_back(Tmp.second);
3968  break;
3969  }
3970  case ISD::TRAP: {
3971  // If this operation is not supported, lower it to 'abort()' call
3974  CLI.setDebugLoc(dl)
3975  .setChain(Node->getOperand(0))
3976  .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3977  DAG.getExternalSymbol(
3978  "abort", TLI.getPointerTy(DAG.getDataLayout())),
3979  std::move(Args));
3980  std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3981 
3982  Results.push_back(CallResult.second);
3983  break;
3984  }
3985  case ISD::FMINNUM:
3986  Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
3987  RTLIB::FMIN_F80, RTLIB::FMIN_F128,
3988  RTLIB::FMIN_PPCF128));
3989  break;
3990  case ISD::FMAXNUM:
3991  Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
3992  RTLIB::FMAX_F80, RTLIB::FMAX_F128,
3993  RTLIB::FMAX_PPCF128));
3994  break;
3995  case ISD::FSQRT:
3996  case ISD::STRICT_FSQRT:
3997  Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
3998  RTLIB::SQRT_F80, RTLIB::SQRT_F128,
3999  RTLIB::SQRT_PPCF128));
4000  break;
4001  case ISD::FSIN:
4002  case ISD::STRICT_FSIN:
4003  Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4004  RTLIB::SIN_F80, RTLIB::SIN_F128,
4005  RTLIB::SIN_PPCF128));
4006  break;
4007  case ISD::FCOS:
4008  case ISD::STRICT_FCOS:
4009  Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
4010  RTLIB::COS_F80, RTLIB::COS_F128,
4011  RTLIB::COS_PPCF128));
4012  break;
4013  case ISD::FSINCOS:
4014  // Expand into sincos libcall.
4015  ExpandSinCosLibCall(Node, Results);
4016  break;
4017  case ISD::FLOG:
4018  case ISD::STRICT_FLOG:
4019  Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64,
4020  RTLIB::LOG_F80, RTLIB::LOG_F128,
4021  RTLIB::LOG_PPCF128));
4022  break;
4023  case ISD::FLOG2:
4024  case ISD::STRICT_FLOG2:
4025  Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64,
4026  RTLIB::LOG2_F80, RTLIB::LOG2_F128,
4027  RTLIB::LOG2_PPCF128));
4028  break;
4029  case ISD::FLOG10:
4030  case ISD::STRICT_FLOG10:
4031  Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64,
4032  RTLIB::LOG10_F80, RTLIB::LOG10_F128,
4033  RTLIB::LOG10_PPCF128));
4034  break;
4035  case ISD::FEXP:
4036  case ISD::STRICT_FEXP:
4037  Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64,
4038  RTLIB::EXP_F80, RTLIB::EXP_F128,
4039  RTLIB::EXP_PPCF128));
4040  break;
4041  case ISD::FEXP2:
4042  case ISD::STRICT_FEXP2:
4043  Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64,
4044  RTLIB::EXP2_F80, RTLIB::EXP2_F128,
4045  RTLIB::EXP2_PPCF128));
4046  break;
4047  case ISD::FTRUNC:
4048  Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4049  RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4050  RTLIB::TRUNC_PPCF128));
4051  break;
4052  case ISD::FFLOOR:
4053  Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4054  RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4055  RTLIB::FLOOR_PPCF128));
4056  break;
4057  case ISD::FCEIL:
4058  Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4059  RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4060  RTLIB::CEIL_PPCF128));
4061  break;
4062  case ISD::FRINT:
4063  case ISD::STRICT_FRINT:
4064  Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4065  RTLIB::RINT_F80, RTLIB::RINT_F128,
4066  RTLIB::RINT_PPCF128));
4067  break;
4068  case ISD::FNEARBYINT:
4070  Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
4071  RTLIB::NEARBYINT_F64,
4072  RTLIB::NEARBYINT_F80,
4073  RTLIB::NEARBYINT_F128,
4074  RTLIB::NEARBYINT_PPCF128));
4075  break;
4076  case ISD::FROUND:
4077  Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32,
4078  RTLIB::ROUND_F64,
4079  RTLIB::ROUND_F80,
4080  RTLIB::ROUND_F128,
4081  RTLIB::ROUND_PPCF128));
4082  break;
4083  case ISD::FPOWI:
4084  case ISD::STRICT_FPOWI:
4085  Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64,
4086  RTLIB::POWI_F80, RTLIB::POWI_F128,
4087  RTLIB::POWI_PPCF128));
4088  break;
4089  case ISD::FPOW:
4090  case ISD::STRICT_FPOW:
4091  Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64,
4092  RTLIB::POW_F80, RTLIB::POW_F128,
4093  RTLIB::POW_PPCF128));
4094  break;
4095  case ISD::FDIV:
4096  Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4097  RTLIB::DIV_F80, RTLIB::DIV_F128,
4098  RTLIB::DIV_PPCF128));
4099  break;
4100  case ISD::FREM:
4101  Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
4102  RTLIB::REM_F80, RTLIB::REM_F128,
4103  RTLIB::REM_PPCF128));
4104  break;
4105  case ISD::FMA:
4106  case ISD::STRICT_FMA:
4107  Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4108  RTLIB::FMA_F80, RTLIB::FMA_F128,
4109  RTLIB::FMA_PPCF128));
4110  break;
4111  case ISD::FADD:
4112  Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4113  RTLIB::ADD_F80, RTLIB::ADD_F128,
4114  RTLIB::ADD_PPCF128));
4115  break;
4116  case ISD::FMUL:
4117  Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4118  RTLIB::MUL_F80, RTLIB::MUL_F128,
4119  RTLIB::MUL_PPCF128));
4120  break;
4121  case ISD::FP16_TO_FP:
4122  if (Node->getValueType(0) == MVT::f32) {
4123  Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
4124  }
4125  break;
4126  case ISD::FP_TO_FP16: {
4127  RTLIB::Libcall LC =
4129  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
4130  Results.push_back(ExpandLibCall(LC, Node, false));
4131  break;
4132  }
4133  case ISD::FSUB:
4134  Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4135  RTLIB::SUB_F80, RTLIB::SUB_F128,
4136  RTLIB::SUB_PPCF128));
4137  break;
4138  case ISD::SREM:
4139  Results.push_back(ExpandIntLibCall(Node, true,
4140  RTLIB::SREM_I8,
4141  RTLIB::SREM_I16, RTLIB::SREM_I32,
4142  RTLIB::SREM_I64, RTLIB::SREM_I128));
4143  break;
4144  case ISD::UREM:
4145  Results.push_back(ExpandIntLibCall(Node, false,
4146  RTLIB::UREM_I8,
4147  RTLIB::UREM_I16, RTLIB::UREM_I32,
4148  RTLIB::UREM_I64, RTLIB::UREM_I128));
4149  break;
4150  case ISD::SDIV:
4151  Results.push_back(ExpandIntLibCall(Node, true,
4152  RTLIB::SDIV_I8,
4153  RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4154  RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4155  break;
4156  case ISD::UDIV:
4157  Results.push_back(ExpandIntLibCall(Node, false,
4158  RTLIB::UDIV_I8,
4159  RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4160  RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4161  break;
4162  case ISD::SDIVREM:
4163  case ISD::UDIVREM:
4164  // Expand into divrem libcall
4165  ExpandDivRemLibCall(Node, Results);
4166  break;
4167  case ISD::MUL:
4168  Results.push_back(ExpandIntLibCall(Node, false,
4169  RTLIB::MUL_I8,
4170  RTLIB::MUL_I16, RTLIB::MUL_I32,
4171  RTLIB::MUL_I64, RTLIB::MUL_I128));
4172  break;
4173  }
4174 
4175  // Replace the original node with the legalized result.
4176  if (!Results.empty()) {
4177  DEBUG(dbgs() << "Successfully converted node to libcall\n");
4178  ReplaceNode(Node, Results.data());
4179  } else
4180  DEBUG(dbgs() << "Could not convert node to libcall\n");
4181 }
4182 
4183 // Determine the vector type to use in place of an original scalar element when
4184 // promoting equally sized vectors.
4186  MVT EltVT, MVT NewEltVT) {
4187  unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4188  MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
4189  assert(TLI.isTypeLegal(MidVT) && "unexpected");
4190  return MidVT;
4191 }
4192 
4193 void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4194  DEBUG(dbgs() << "Trying to promote node\n");
4196  MVT OVT = Node->getSimpleValueType(0);
4197  if (Node->getOpcode() == ISD::UINT_TO_FP ||
4198  Node->getOpcode() == ISD::SINT_TO_FP ||
4199  Node->getOpcode() == ISD::SETCC ||
4200  Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4201  Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4202  OVT = Node->getOperand(0).getSimpleValueType();
4203  }
4204  if (Node->getOpcode() == ISD::BR_CC)
4205  OVT = Node->getOperand(2).getSimpleValueType();
4206  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4207  SDLoc dl(Node);
4208  SDValue Tmp1, Tmp2, Tmp3;
4209  switch (Node->getOpcode()) {
4210  case ISD::CTTZ:
4211  case ISD::CTTZ_ZERO_UNDEF:
4212  case ISD::CTLZ:
4213  case ISD::CTLZ_ZERO_UNDEF:
4214  case ISD::CTPOP:
4215  // Zero extend the argument.
4216  Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4217  if (Node->getOpcode() == ISD::CTTZ) {
4218  // The count is the same in the promoted type except if the original
4219  // value was zero. This can be handled by setting the bit just off
4220  // the top of the original type.
4221  auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
4222  OVT.getSizeInBits());
4223  Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
4224  DAG.getConstant(TopBit, dl, NVT));
4225  }
4226  // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4227  // already the correct result.
4228  Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4229  if (Node->getOpcode() == ISD::CTLZ ||
4230  Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
4231  // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4232  Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
4233  DAG.getConstant(NVT.getSizeInBits() -
4234  OVT.getSizeInBits(), dl, NVT));
4235  }
4236  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4237  break;
4238  case ISD::BITREVERSE:
4239  case ISD::BSWAP: {
4240  unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
4241  Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4242  Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4243  Tmp1 = DAG.getNode(
4244  ISD::SRL, dl, NVT, Tmp1,
4245  DAG.getConstant(DiffBits, dl,
4246  TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4247  Results.push_back(Tmp1);
4248  break;
4249  }
4250  case ISD::FP_TO_UINT:
4251  case ISD::FP_TO_SINT:
4252  Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0),
4253  Node->getOpcode() == ISD::FP_TO_SINT, dl);
4254  Results.push_back(Tmp1);
4255  break;
4256  case ISD::UINT_TO_FP:
4257  case ISD::SINT_TO_FP:
4258  Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0),
4259  Node->getOpcode() == ISD::SINT_TO_FP, dl);
4260  Results.push_back(Tmp1);
4261  break;
4262  case ISD::VAARG: {
4263  SDValue Chain = Node->getOperand(0); // Get the chain.
4264  SDValue Ptr = Node->getOperand(1); // Get the pointer.
4265 
4266  unsigned TruncOp;
4267  if (OVT.isVector()) {
4268  TruncOp = ISD::BITCAST;
4269  } else {
4270  assert(OVT.isInteger()
4271  && "VAARG promotion is supported only for vectors or integer types");
4272  TruncOp = ISD::TRUNCATE;
4273  }
4274 
4275  // Perform the larger operation, then convert back
4276  Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
4277  Node->getConstantOperandVal(3));
4278  Chain = Tmp1.getValue(1);
4279 
4280  Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
4281 
4282  // Modified the chain result - switch anything that used the old chain to
4283  // use the new one.
4284  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
4285  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
4286  if (UpdatedNodes) {
4287  UpdatedNodes->insert(Tmp2.getNode());
4288  UpdatedNodes->insert(Chain.getNode());
4289  }
4290  ReplacedNode(Node);
4291  break;
4292  }
4293  case ISD::MUL:
4294  case ISD::SDIV:
4295  case ISD::SREM:
4296  case ISD::UDIV:
4297  case ISD::UREM:
4298  case ISD::AND:
4299  case ISD::OR:
4300  case ISD::XOR: {
4301  unsigned ExtOp, TruncOp;
4302  if (OVT.isVector()) {
4303  ExtOp = ISD::BITCAST;
4304  TruncOp = ISD::BITCAST;
4305  } else {
4306  assert(OVT.isInteger() && "Cannot promote logic operation");
4307 
4308  switch (Node->getOpcode()) {
4309  default:
4310  ExtOp = ISD::ANY_EXTEND;
4311  break;
4312  case ISD::SDIV:
4313  case ISD::SREM:
4314  ExtOp = ISD::SIGN_EXTEND;
4315  break;
4316  case ISD::UDIV:
4317  case ISD::UREM:
4318  ExtOp = ISD::ZERO_EXTEND;
4319  break;
4320  }
4321  TruncOp = ISD::TRUNCATE;
4322  }
4323  // Promote each of the values to the new type.
4324  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4325  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4326  // Perform the larger operation, then convert back
4327  Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4328  Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
4329  break;
4330  }
4331  case ISD::UMUL_LOHI:
4332  case ISD::SMUL_LOHI: {
4333  // Promote to a multiply in a wider integer type.
4334  unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
4335  : ISD::SIGN_EXTEND;
4336  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4337  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4338  Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
4339 
4340  auto &DL = DAG.getDataLayout();
4341  unsigned OriginalSize = OVT.getScalarSizeInBits();
4342  Tmp2 = DAG.getNode(
4343  ISD::SRL, dl, NVT, Tmp1,
4344  DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
4345  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4346  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
4347  break;
4348  }
4349  case ISD::SELECT: {
4350  unsigned ExtOp, TruncOp;
4351  if (Node->getValueType(0).isVector() ||
4352  Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
4353  ExtOp = ISD::BITCAST;
4354  TruncOp = ISD::BITCAST;
4355  } else if (Node->getValueType(0).isInteger()) {
4356  ExtOp = ISD::ANY_EXTEND;
4357  TruncOp = ISD::TRUNCATE;
4358  } else {
4359  ExtOp = ISD::FP_EXTEND;
4360  TruncOp = ISD::FP_ROUND;
4361  }
4362  Tmp1 = Node->getOperand(0);
4363  // Promote each of the values to the new type.
4364  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4365  Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4366  // Perform the larger operation, then round down.
4367  Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
4368  if (TruncOp != ISD::FP_ROUND)
4369  Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
4370  else
4371  Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
4372  DAG.getIntPtrConstant(0, dl));
4373  Results.push_back(Tmp1);
4374  break;
4375  }
4376  case ISD::VECTOR_SHUFFLE: {
4377  ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
4378 
4379  // Cast the two input vectors.
4380  Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
4381  Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
4382 
4383  // Convert the shuffle mask to the right # elements.
4384  Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
4385  Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
4386  Results.push_back(Tmp1);
4387  break;
4388  }
4389  case ISD::SETCC: {
4390  unsigned ExtOp = ISD::FP_EXTEND;
4391  if (NVT.isInteger()) {
4392  ISD::CondCode CCCode =
4393  cast<CondCodeSDNode>(Node->getOperand(2))->get();
4395  }
4396  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4397  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4398  Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
4399  Tmp1, Tmp2, Node->getOperand(2)));
4400  break;
4401  }
4402  case ISD::BR_CC: {
4403  unsigned ExtOp = ISD::FP_EXTEND;
4404  if (NVT.isInteger()) {
4405  ISD::CondCode CCCode =
4406  cast<CondCodeSDNode>(Node->getOperand(1))->get();
4408  }
4409  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4410  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4411  Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
4412  Node->getOperand(0), Node->getOperand(1),
4413  Tmp1, Tmp2, Node->getOperand(4)));
4414  break;
4415  }
4416  case ISD::FADD:
4417  case ISD::FSUB:
4418  case ISD::FMUL:
4419  case ISD::FDIV:
4420  case ISD::FREM:
4421  case ISD::FMINNUM:
4422  case ISD::FMAXNUM:
4423  case ISD::FPOW:
4424  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4425  Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4426  Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
4427  Node->getFlags());
4428  Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4429  Tmp3, DAG.getIntPtrConstant(0, dl)));
4430  break;
4431  case ISD::FMA:
4432  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4433  Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4434  Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
4435  Results.push_back(
4436  DAG.getNode(ISD::FP_ROUND, dl, OVT,
4437  DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
4438  DAG.getIntPtrConstant(0, dl)));
4439  break;
4440  case ISD::FCOPYSIGN:
4441  case ISD::FPOWI: {
4442  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4443  Tmp2 = Node->getOperand(1);
4444  Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4445 
4446  // fcopysign doesn't change anything but the sign bit, so
4447  // (fp_round (fcopysign (fpext a), b))
4448  // is as precise as
4449  // (fp_round (fpext a))
4450  // which is a no-op. Mark it as a TRUNCating FP_ROUND.
4451  const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
4452  Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4453  Tmp3, DAG.getIntPtrConstant(isTrunc, dl)));
4454  break;
4455  }
4456  case ISD::FFLOOR:
4457  case ISD::FCEIL:
4458  case ISD::FRINT:
4459  case ISD::FNEARBYINT:
4460  case ISD::FROUND:
4461  case ISD::FTRUNC:
4462  case ISD::FNEG:
4463  case ISD::FSQRT:
4464  case ISD::FSIN:
4465  case ISD::FCOS:
4466  case ISD::FLOG:
4467  case ISD::FLOG2:
4468  case ISD::FLOG10:
4469  case ISD::FABS:
4470  case ISD::FEXP:
4471  case ISD::FEXP2:
4472  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4473  Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4474  Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4475  Tmp2, DAG.getIntPtrConstant(0, dl)));
4476  break;
4477  case ISD::BUILD_VECTOR: {
4478  MVT EltVT = OVT.getVectorElementType();
4479  MVT NewEltVT = NVT.getVectorElementType();
4480 
4481  // Handle bitcasts to a different vector type with the same total bit size
4482  //
4483  // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
4484  // =>
4485  // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
4486 
4487  assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4488  "Invalid promote type for build_vector");
4489  assert(NewEltVT.bitsLT(EltVT) && "not handled");
4490 
4491  MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4492 
4493  SmallVector<SDValue, 8> NewOps;
4494  for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
4495  SDValue Op = Node->getOperand(I);
4496  NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
4497  }
4498 
4499  SDLoc SL(Node);
4500  SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps);
4501  SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4502  Results.push_back(CvtVec);
4503  break;
4504  }
4505  case ISD::EXTRACT_VECTOR_ELT: {
4506  MVT EltVT = OVT.getVectorElementType();
4507  MVT NewEltVT = NVT.getVectorElementType();
4508 
4509  // Handle bitcasts to a different vector type with the same total bit size.
4510  //
4511  // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
4512  // =>
4513  // v4i32:castx = bitcast x:v2i64
4514  //
4515  // i64 = bitcast
4516  // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
4517  // (i32 (extract_vector_elt castx, (2 * y + 1)))
4518  //
4519 
4520  assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4521  "Invalid promote type for extract_vector_elt");
4522  assert(NewEltVT.bitsLT(EltVT) && "not handled");
4523 
4524  MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4525  unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4526 
4527  SDValue Idx = Node->getOperand(1);
4528  EVT IdxVT = Idx.getValueType();
4529  SDLoc SL(Node);
4530  SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
4531  SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4532 
4533  SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4534 
4535  SmallVector<SDValue, 8> NewOps;
4536  for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4537  SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4538  SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4539 
4540  SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4541  CastVec, TmpIdx);
4542  NewOps.push_back(Elt);
4543  }
4544 
4545  SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
4546  Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
4547  break;
4548  }
4549  case ISD::INSERT_VECTOR_ELT: {
4550  MVT EltVT = OVT.getVectorElementType();
4551  MVT NewEltVT = NVT.getVectorElementType();
4552 
4553  // Handle bitcasts to a different vector type with the same total bit size
4554  //
4555  // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
4556  // =>
4557  // v4i32:castx = bitcast x:v2i64
4558  // v2i32:casty = bitcast y:i64
4559  //
4560  // v2i64 = bitcast
4561  // (v4i32 insert_vector_elt
4562  // (v4i32 insert_vector_elt v4i32:castx,
4563  // (extract_vector_elt casty, 0), 2 * z),
4564  // (extract_vector_elt casty, 1), (2 * z + 1))
4565 
4566  assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4567  "Invalid promote type for insert_vector_elt");
4568  assert(NewEltVT.bitsLT(EltVT) && "not handled");
4569 
4570  MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4571  unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4572 
4573  SDValue Val = Node->getOperand(1);
4574  SDValue Idx = Node->getOperand(2);
4575  EVT IdxVT = Idx.getValueType();
4576  SDLoc SL(Node);
4577 
4578  SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
4579  SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4580 
4581  SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4582  SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4583 
4584  SDValue NewVec = CastVec;
4585  for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4586  SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4587  SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4588 
4589  SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4590  CastVal, IdxOffset);
4591 
4592  NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
4593  NewVec, Elt, InEltIdx);
4594  }
4595 
4596  Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
4597  break;
4598  }
4599  case ISD::SCALAR_TO_VECTOR: {
4600  MVT EltVT = OVT.getVectorElementType();
4601  MVT NewEltVT = NVT.getVectorElementType();
4602 
4603  // Handle bitcasts to different vector type with the same total bit size.
4604  //
4605  // e.g. v2i64 = scalar_to_vector x:i64
4606  // =>
4607  // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
4608  //
4609 
4610  MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4611  SDValue Val = Node->getOperand(0);
4612  SDLoc SL(Node);
4613 
4614  SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4615  SDValue Undef = DAG.getUNDEF(MidVT);
4616 
4617  SmallVector<SDValue, 8> NewElts;
4618  NewElts.push_back(CastVal);
4619  for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
4620  NewElts.push_back(Undef);
4621 
4622  SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
4623  SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4624  Results.push_back(CvtVec);
4625  break;
4626  }
4627  }
4628 
4629  // Replace the original node with the legalized result.
4630  if (!Results.empty()) {
4631  DEBUG(dbgs() << "Successfully promoted node\n");
4632  ReplaceNode(Node, Results.data());
4633  } else
4634  DEBUG(dbgs() << "Could not promote node\n");
4635 }
4636 
4637 /// This is the entry point for the file.
4639  AssignTopologicalOrder();
4640 
4641  SmallPtrSet<SDNode *, 16> LegalizedNodes;
4642  // Use a delete listener to remove nodes which were deleted during
4643  // legalization from LegalizeNodes. This is needed to handle the situation
4644  // where a new node is allocated by the object pool to the same address of a
4645  // previously deleted node.
4646  DAGNodeDeletedListener DeleteListener(
4647  *this,
4648  [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); });
4649 
4650  SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
4651 
4652  // Visit all the nodes. We start in topological order, so that we see
4653  // nodes with their original operands intact. Legalization can produce
4654  // new nodes which may themselves need to be legalized. Iterate until all
4655  // nodes have been legalized.
4656  while (true) {
4657  bool AnyLegalized = false;
4658  for (auto NI = allnodes_end(); NI != allnodes_begin();) {
4659  --NI;
4660 
4661  SDNode *N = &*NI;
4662  if (N->use_empty() && N != getRoot().getNode()) {
4663  ++NI;
4664  DeleteNode(N);
4665  continue;
4666  }
4667 
4668  if (LegalizedNodes.insert(N).second) {
4669  AnyLegalized = true;
4670  Legalizer.LegalizeOp(N);
4671 
4672  if (N->use_empty() && N != getRoot().getNode()) {
4673  ++NI;
4674  DeleteNode(N);
4675  }
4676  }
4677  }
4678  if (!AnyLegalized)
4679  break;
4680 
4681  }
4682 
4683  // Remove dead nodes now.
4684  RemoveDeadNodes();
4685 }
4686 
4688  SmallSetVector<SDNode *, 16> &UpdatedNodes) {
4689  SmallPtrSet<SDNode *, 16> LegalizedNodes;
4690  SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
4691 
4692  // Directly insert the node in question, and legalize it. This will recurse
4693  // as needed through operands.
4694  LegalizedNodes.insert(N);
4695  Legalizer.LegalizeOp(N);
4696 
4697  return LegalizedNodes.count(N);
4698 }
bool LegalizeOp(SDNode *N, SmallSetVector< SDNode *, 16 > &UpdatedNodes)
Transforms a SelectionDAG node and any operands to it into a node that is compatible with the target ...
Fast - This calling convention attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:43
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
Definition: ISDOpcodes.h:731
static Constant * getFPTrunc(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:1584
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:545
X = FP_ROUND(Y, TRUNC) - Rounding &#39;Y&#39; from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:512
void push_back(const T &Elt)
Definition: SmallVector.h:212
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:834
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:109
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:569
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isInteger() const
Return true if this is an integer or a vector integer type.
static bool isConstant(const MachineInstr &MI)
bool isUndef() const
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
Definition: APFloat.h:1077
static MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
Constrained versions of libm-equivalent floating point intrinsics.
Definition: ISDOpcodes.h:272
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant, which is required to be operand #1) half of the integer or float value specified as operand #0.
Definition: ISDOpcodes.h:184
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
Definition: APInt.h:555
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:333
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:617
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
static MVT getVectorVT(MVT VT, unsigned NumElements)
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:342
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:260
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:136
bool isVector() const
Return true if this is a vector value type.
CallLoweringInfo & setIsPostTypeLegalization(bool Value=true)
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none...
const SDValue & getBasePtr() const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode *> &Visited, SmallVectorImpl< const SDNode *> &Worklist, unsigned int MaxSteps=0)
Returns true if N is a predecessor of any node in Worklist.
const SDValue & getValue() const
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain...
Definition: ISDOpcodes.h:666
SDVTList getVTList() const
This file contains the declarations for metadata subclasses.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:253
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:250
unsigned getVectorNumElements() const
const SDValue & getChain() const
Function Alias Analysis Results
static MVT getPromotedVectorElementType(const TargetLowering &TLI, MVT EltVT, MVT NewEltVT)
unsigned getAlignment() const
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:769
unsigned second
APInt trunc(unsigned width) const
Truncate to new width.
Definition: APInt.cpp:818
static uint32_t Concat[]
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:141
F(f)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:177
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:359
const SDNodeFlags getFlags() const
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Same for subtraction.
Definition: ISDOpcodes.h:253
void reserve(size_type N)
Definition: SmallVector.h:380
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:212
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1 at the ...
Definition: ISDOpcodes.h:328
The address of the GOT.
Definition: ISDOpcodes.h:66
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
Definition: ISDOpcodes.h:748
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:404
std::pair< MCSymbol *, MachineModuleInfoImpl::StubValueTy > PairTy
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
const ConstantFP * getConstantFPValue() const
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:159
bool isTruncatingStore() const
Return true if the op does a truncation before store.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:344
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic...
Definition: ISDOpcodes.h:114
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:209
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:136
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations...
Definition: ISDOpcodes.h:433
static uint32_t getAlignment(const MCSectionCOFF &Sec)
Shift and rotation operations.
Definition: ISDOpcodes.h:379
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:205
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:639