LLVM  3.7.0
LegalizeDAG.cpp
Go to the documentation of this file.
1 //===-- LegalizeDAG.cpp - Implement SelectionDAG::Legalize ----------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the SelectionDAG::Legalize method.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "llvm/ADT/SetVector.h"
16 #include "llvm/ADT/SmallPtrSet.h"
17 #include "llvm/ADT/SmallSet.h"
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/ADT/Triple.h"
20 #include "llvm/CodeGen/Analysis.h"
23 #include "llvm/IR/CallingConv.h"
24 #include "llvm/IR/Constants.h"
25 #include "llvm/IR/DataLayout.h"
26 #include "llvm/IR/DebugInfo.h"
27 #include "llvm/IR/DerivedTypes.h"
28 #include "llvm/IR/Function.h"
29 #include "llvm/IR/LLVMContext.h"
30 #include "llvm/Support/Debug.h"
38 using namespace llvm;
39 
40 #define DEBUG_TYPE "legalizedag"
41 
42 //===----------------------------------------------------------------------===//
43 /// This takes an arbitrary SelectionDAG as input and
44 /// hacks on it until the target machine can handle it. This involves
45 /// eliminating value sizes the machine cannot handle (promoting small sizes to
46 /// large sizes or splitting up large values into small values) as well as
47 /// eliminating operations the machine cannot handle.
48 ///
49 /// This code also does a small amount of optimization and recognition of idioms
50 /// as part of its processing. For example, if a target does not support a
51 /// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
52 /// will attempt merge setcc and brc instructions into brcc's.
53 ///
54 namespace {
55 class SelectionDAGLegalize {
56  const TargetMachine &TM;
57  const TargetLowering &TLI;
58  SelectionDAG &DAG;
59 
60  /// \brief The set of nodes which have already been legalized. We hold a
61  /// reference to it in order to update as necessary on node deletion.
62  SmallPtrSetImpl<SDNode *> &LegalizedNodes;
63 
64  /// \brief A set of all the nodes updated during legalization.
65  SmallSetVector<SDNode *, 16> *UpdatedNodes;
66 
67  EVT getSetCCResultType(EVT VT) const {
68  return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
69  }
70 
71  // Libcall insertion helpers.
72 
73 public:
74  SelectionDAGLegalize(SelectionDAG &DAG,
75  SmallPtrSetImpl<SDNode *> &LegalizedNodes,
76  SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
77  : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
78  LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
79 
80  /// \brief Legalizes the given operation.
81  void LegalizeOp(SDNode *Node);
82 
83 private:
84  SDValue OptimizeFloatStore(StoreSDNode *ST);
85 
86  void LegalizeLoadOps(SDNode *Node);
87  void LegalizeStoreOps(SDNode *Node);
88 
89  /// Some targets cannot handle a variable
90  /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
91  /// is necessary to spill the vector being inserted into to memory, perform
92  /// the insert there, and then read the result back.
93  SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val,
94  SDValue Idx, SDLoc dl);
95  SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
96  SDValue Idx, SDLoc dl);
97 
98  /// Return a vector shuffle operation which
99  /// performs the same shuffe in terms of order or result bytes, but on a type
100  /// whose vector element type is narrower than the original shuffle type.
101  /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
102  SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, SDLoc dl,
103  SDValue N1, SDValue N2,
104  ArrayRef<int> Mask) const;
105 
106  bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC,
107  bool &NeedInvert, SDLoc dl);
108 
109  SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
110  SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops,
111  unsigned NumOps, bool isSigned, SDLoc dl);
112 
113  std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
114  SDNode *Node, bool isSigned);
115  SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
116  RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
117  RTLIB::Libcall Call_F128,
118  RTLIB::Libcall Call_PPCF128);
119  SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
120  RTLIB::Libcall Call_I8,
121  RTLIB::Libcall Call_I16,
122  RTLIB::Libcall Call_I32,
123  RTLIB::Libcall Call_I64,
124  RTLIB::Libcall Call_I128);
125  void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
126  void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
127 
128  SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, SDLoc dl);
129  SDValue ExpandBUILD_VECTOR(SDNode *Node);
130  SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
131  void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
132  SmallVectorImpl<SDValue> &Results);
133  SDValue ExpandFCOPYSIGN(SDNode *Node);
134  SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue LegalOp, EVT DestVT,
135  SDLoc dl);
136  SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned,
137  SDLoc dl);
138  SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned,
139  SDLoc dl);
140 
141  SDValue ExpandBSWAP(SDValue Op, SDLoc dl);
142  SDValue ExpandBitCount(unsigned Opc, SDValue Op, SDLoc dl);
143 
144  SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
145  SDValue ExpandInsertToVectorThroughStack(SDValue Op);
146  SDValue ExpandVectorBuildThroughStack(SDNode* Node);
147 
148  SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
149 
150  std::pair<SDValue, SDValue> ExpandAtomic(SDNode *Node);
151 
152  void ExpandNode(SDNode *Node);
153  void PromoteNode(SDNode *Node);
154 
155 public:
156  // Node replacement helpers
157  void ReplacedNode(SDNode *N) {
158  LegalizedNodes.erase(N);
159  if (UpdatedNodes)
160  UpdatedNodes->insert(N);
161  }
162  void ReplaceNode(SDNode *Old, SDNode *New) {
163  DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
164  dbgs() << " with: "; New->dump(&DAG));
165 
166  assert(Old->getNumValues() == New->getNumValues() &&
167  "Replacing one node with another that produces a different number "
168  "of values!");
169  DAG.ReplaceAllUsesWith(Old, New);
170  for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i)
171  DAG.TransferDbgValues(SDValue(Old, i), SDValue(New, i));
172  if (UpdatedNodes)
173  UpdatedNodes->insert(New);
174  ReplacedNode(Old);
175  }
176  void ReplaceNode(SDValue Old, SDValue New) {
177  DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
178  dbgs() << " with: "; New->dump(&DAG));
179 
180  DAG.ReplaceAllUsesWith(Old, New);
181  DAG.TransferDbgValues(Old, New);
182  if (UpdatedNodes)
183  UpdatedNodes->insert(New.getNode());
184  ReplacedNode(Old.getNode());
185  }
186  void ReplaceNode(SDNode *Old, const SDValue *New) {
187  DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
188 
189  DAG.ReplaceAllUsesWith(Old, New);
190  for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
191  DEBUG(dbgs() << (i == 0 ? " with: "
192  : " and: ");
193  New[i]->dump(&DAG));
194  DAG.TransferDbgValues(SDValue(Old, i), New[i]);
195  if (UpdatedNodes)
196  UpdatedNodes->insert(New[i].getNode());
197  }
198  ReplacedNode(Old);
199  }
200 };
201 }
202 
203 /// Return a vector shuffle operation which
204 /// performs the same shuffe in terms of order or result bytes, but on a type
205 /// whose vector element type is narrower than the original shuffle type.
206 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
207 SDValue
208 SelectionDAGLegalize::ShuffleWithNarrowerEltType(EVT NVT, EVT VT, SDLoc dl,
209  SDValue N1, SDValue N2,
210  ArrayRef<int> Mask) const {
211  unsigned NumMaskElts = VT.getVectorNumElements();
212  unsigned NumDestElts = NVT.getVectorNumElements();
213  unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
214 
215  assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
216 
217  if (NumEltsGrowth == 1)
218  return DAG.getVectorShuffle(NVT, dl, N1, N2, &Mask[0]);
219 
220  SmallVector<int, 8> NewMask;
221  for (unsigned i = 0; i != NumMaskElts; ++i) {
222  int Idx = Mask[i];
223  for (unsigned j = 0; j != NumEltsGrowth; ++j) {
224  if (Idx < 0)
225  NewMask.push_back(-1);
226  else
227  NewMask.push_back(Idx * NumEltsGrowth + j);
228  }
229  }
230  assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
231  assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
232  return DAG.getVectorShuffle(NVT, dl, N1, N2, &NewMask[0]);
233 }
234 
235 /// Expands the ConstantFP node to an integer constant or
236 /// a load from the constant pool.
237 SDValue
238 SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
239  bool Extend = false;
240  SDLoc dl(CFP);
241 
242  // If a FP immediate is precise when represented as a float and if the
243  // target can do an extending load from float to double, we put it into
244  // the constant pool as a float, even if it's is statically typed as a
245  // double. This shrinks FP constants and canonicalizes them for targets where
246  // an FP extending load is the same cost as a normal load (such as on the x87
247  // fp stack or PPC FP unit).
248  EVT VT = CFP->getValueType(0);
249  ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
250  if (!UseCP) {
251  assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
252  return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
253  (VT == MVT::f64) ? MVT::i64 : MVT::i32);
254  }
255 
256  EVT OrigVT = VT;
257  EVT SVT = VT;
258  while (SVT != MVT::f32 && SVT != MVT::f16) {
259  SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
261  // Only do this if the target has a native EXTLOAD instruction from
262  // smaller type.
263  TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
264  TLI.ShouldShrinkFPConstant(OrigVT)) {
265  Type *SType = SVT.getTypeForEVT(*DAG.getContext());
266  LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
267  VT = SVT;
268  Extend = true;
269  }
270  }
271 
272  SDValue CPIdx =
273  DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
274  unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
275  if (Extend) {
276  SDValue Result =
277  DAG.getExtLoad(ISD::EXTLOAD, dl, OrigVT,
278  DAG.getEntryNode(),
280  VT, false, false, false, Alignment);
281  return Result;
282  }
283  SDValue Result =
284  DAG.getLoad(OrigVT, dl, DAG.getEntryNode(), CPIdx,
285  MachinePointerInfo::getConstantPool(), false, false, false,
286  Alignment);
287  return Result;
288 }
289 
290 /// Expands an unaligned store to 2 half-size stores.
292  const TargetLowering &TLI,
293  SelectionDAGLegalize *DAGLegalize) {
294  assert(ST->getAddressingMode() == ISD::UNINDEXED &&
295  "unaligned indexed stores not implemented!");
296  SDValue Chain = ST->getChain();
297  SDValue Ptr = ST->getBasePtr();
298  SDValue Val = ST->getValue();
299  EVT VT = Val.getValueType();
300  int Alignment = ST->getAlignment();
301  unsigned AS = ST->getAddressSpace();
302 
303  SDLoc dl(ST);
304  if (ST->getMemoryVT().isFloatingPoint() ||
305  ST->getMemoryVT().isVector()) {
306  EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
307  if (TLI.isTypeLegal(intVT)) {
308  // Expand to a bitconvert of the value to the integer type of the
309  // same size, then a (misaligned) int store.
310  // FIXME: Does not handle truncating floating point stores!
311  SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val);
312  Result = DAG.getStore(Chain, dl, Result, Ptr, ST->getPointerInfo(),
313  ST->isVolatile(), ST->isNonTemporal(), Alignment);
314  DAGLegalize->ReplaceNode(SDValue(ST, 0), Result);
315  return;
316  }
317  // Do a (aligned) store to a stack slot, then copy from the stack slot
318  // to the final destination using (unaligned) integer loads and stores.
319  EVT StoredVT = ST->getMemoryVT();
320  MVT RegVT =
321  TLI.getRegisterType(*DAG.getContext(),
323  StoredVT.getSizeInBits()));
324  unsigned StoredBytes = StoredVT.getSizeInBits() / 8;
325  unsigned RegBytes = RegVT.getSizeInBits() / 8;
326  unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes;
327 
328  // Make sure the stack slot is also aligned for the register type.
329  SDValue StackPtr = DAG.CreateStackTemporary(StoredVT, RegVT);
330 
331  // Perform the original store, only redirected to the stack slot.
332  SDValue Store = DAG.getTruncStore(Chain, dl,
333  Val, StackPtr, MachinePointerInfo(),
334  StoredVT, false, false, 0);
335  SDValue Increment = DAG.getConstant(
336  RegBytes, dl, TLI.getPointerTy(DAG.getDataLayout(), AS));
338  unsigned Offset = 0;
339 
340  // Do all but one copies using the full register width.
341  for (unsigned i = 1; i < NumRegs; i++) {
342  // Load one integer register's worth from the stack slot.
343  SDValue Load = DAG.getLoad(RegVT, dl, Store, StackPtr,
345  false, false, false, 0);
346  // Store it to the final location. Remember the store.
347  Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, Ptr,
348  ST->getPointerInfo().getWithOffset(Offset),
349  ST->isVolatile(), ST->isNonTemporal(),
350  MinAlign(ST->getAlignment(), Offset)));
351  // Increment the pointers.
352  Offset += RegBytes;
353  StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
354  Increment);
355  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment);
356  }
357 
358  // The last store may be partial. Do a truncating store. On big-endian
359  // machines this requires an extending load from the stack slot to ensure
360  // that the bits are in the right place.
361  EVT MemVT = EVT::getIntegerVT(*DAG.getContext(),
362  8 * (StoredBytes - Offset));
363 
364  // Load from the stack slot.
365  SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Store, StackPtr,
367  MemVT, false, false, false, 0);
368 
369  Stores.push_back(DAG.getTruncStore(Load.getValue(1), dl, Load, Ptr,
370  ST->getPointerInfo()
371  .getWithOffset(Offset),
372  MemVT, ST->isVolatile(),
373  ST->isNonTemporal(),
374  MinAlign(ST->getAlignment(), Offset),
375  ST->getAAInfo()));
376  // The order of the stores doesn't matter - say it with a TokenFactor.
377  SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
378  DAGLegalize->ReplaceNode(SDValue(ST, 0), Result);
379  return;
380  }
381  assert(ST->getMemoryVT().isInteger() &&
382  !ST->getMemoryVT().isVector() &&
383  "Unaligned store of unknown type.");
384  // Get the half-size VT
385  EVT NewStoredVT = ST->getMemoryVT().getHalfSizedIntegerVT(*DAG.getContext());
386  int NumBits = NewStoredVT.getSizeInBits();
387  int IncrementSize = NumBits / 8;
388 
389  // Divide the stored value in two parts.
390  SDValue ShiftAmount =
391  DAG.getConstant(NumBits, dl, TLI.getShiftAmountTy(Val.getValueType(),
392  DAG.getDataLayout()));
393  SDValue Lo = Val;
394  SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount);
395 
396  // Store the two parts
397  SDValue Store1, Store2;
398  Store1 = DAG.getTruncStore(Chain, dl,
399  DAG.getDataLayout().isLittleEndian() ? Lo : Hi,
400  Ptr, ST->getPointerInfo(), NewStoredVT,
401  ST->isVolatile(), ST->isNonTemporal(), Alignment);
402 
403  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
404  DAG.getConstant(IncrementSize, dl,
405  TLI.getPointerTy(DAG.getDataLayout(), AS)));
406  Alignment = MinAlign(Alignment, IncrementSize);
407  Store2 = DAG.getTruncStore(
408  Chain, dl, DAG.getDataLayout().isLittleEndian() ? Hi : Lo, Ptr,
409  ST->getPointerInfo().getWithOffset(IncrementSize), NewStoredVT,
410  ST->isVolatile(), ST->isNonTemporal(), Alignment, ST->getAAInfo());
411 
412  SDValue Result =
413  DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
414  DAGLegalize->ReplaceNode(SDValue(ST, 0), Result);
415 }
416 
417 /// Expands an unaligned load to 2 half-size loads.
418 static void
420  const TargetLowering &TLI,
421  SDValue &ValResult, SDValue &ChainResult) {
422  assert(LD->getAddressingMode() == ISD::UNINDEXED &&
423  "unaligned indexed loads not implemented!");
424  SDValue Chain = LD->getChain();
425  SDValue Ptr = LD->getBasePtr();
426  EVT VT = LD->getValueType(0);
427  EVT LoadedVT = LD->getMemoryVT();
428  SDLoc dl(LD);
429  if (VT.isFloatingPoint() || VT.isVector()) {
430  EVT intVT = EVT::getIntegerVT(*DAG.getContext(), LoadedVT.getSizeInBits());
431  if (TLI.isTypeLegal(intVT) && TLI.isTypeLegal(LoadedVT)) {
432  // Expand to a (misaligned) integer load of the same size,
433  // then bitconvert to floating point or vector.
434  SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr,
435  LD->getMemOperand());
436  SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad);
437  if (LoadedVT != VT)
438  Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND :
439  ISD::ANY_EXTEND, dl, VT, Result);
440 
441  ValResult = Result;
442  ChainResult = Chain;
443  return;
444  }
445 
446  // Copy the value to a (aligned) stack slot using (unaligned) integer
447  // loads and stores, then do a (aligned) load from the stack slot.
448  MVT RegVT = TLI.getRegisterType(*DAG.getContext(), intVT);
449  unsigned LoadedBytes = LoadedVT.getSizeInBits() / 8;
450  unsigned RegBytes = RegVT.getSizeInBits() / 8;
451  unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes;
452 
453  // Make sure the stack slot is also aligned for the register type.
454  SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT);
455 
456  SDValue Increment =
457  DAG.getConstant(RegBytes, dl, TLI.getPointerTy(DAG.getDataLayout()));
459  SDValue StackPtr = StackBase;
460  unsigned Offset = 0;
461 
462  // Do all but one copies using the full register width.
463  for (unsigned i = 1; i < NumRegs; i++) {
464  // Load one integer register's worth from the original location.
465  SDValue Load = DAG.getLoad(RegVT, dl, Chain, Ptr,
466  LD->getPointerInfo().getWithOffset(Offset),
467  LD->isVolatile(), LD->isNonTemporal(),
468  LD->isInvariant(),
469  MinAlign(LD->getAlignment(), Offset),
470  LD->getAAInfo());
471  // Follow the load with a store to the stack slot. Remember the store.
472  Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, StackPtr,
473  MachinePointerInfo(), false, false, 0));
474  // Increment the pointers.
475  Offset += RegBytes;
476  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment);
477  StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
478  Increment);
479  }
480 
481  // The last copy may be partial. Do an extending load.
482  EVT MemVT = EVT::getIntegerVT(*DAG.getContext(),
483  8 * (LoadedBytes - Offset));
484  SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr,
485  LD->getPointerInfo().getWithOffset(Offset),
486  MemVT, LD->isVolatile(),
487  LD->isNonTemporal(),
488  LD->isInvariant(),
489  MinAlign(LD->getAlignment(), Offset),
490  LD->getAAInfo());
491  // Follow the load with a store to the stack slot. Remember the store.
492  // On big-endian machines this requires a truncating store to ensure
493  // that the bits end up in the right place.
494  Stores.push_back(DAG.getTruncStore(Load.getValue(1), dl, Load, StackPtr,
495  MachinePointerInfo(), MemVT,
496  false, false, 0));
497 
498  // The order of the stores doesn't matter - say it with a TokenFactor.
499  SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
500 
501  // Finally, perform the original load only redirected to the stack slot.
502  Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase,
503  MachinePointerInfo(), LoadedVT, false,false, false,
504  0);
505 
506  // Callers expect a MERGE_VALUES node.
507  ValResult = Load;
508  ChainResult = TF;
509  return;
510  }
511  assert(LoadedVT.isInteger() && !LoadedVT.isVector() &&
512  "Unaligned load of unsupported type.");
513 
514  // Compute the new VT that is half the size of the old one. This is an
515  // integer MVT.
516  unsigned NumBits = LoadedVT.getSizeInBits();
517  EVT NewLoadedVT;
518  NewLoadedVT = EVT::getIntegerVT(*DAG.getContext(), NumBits/2);
519  NumBits >>= 1;
520 
521  unsigned Alignment = LD->getAlignment();
522  unsigned IncrementSize = NumBits / 8;
523  ISD::LoadExtType HiExtType = LD->getExtensionType();
524 
525  // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD.
526  if (HiExtType == ISD::NON_EXTLOAD)
527  HiExtType = ISD::ZEXTLOAD;
528 
529  // Load the value in two parts
530  SDValue Lo, Hi;
531  if (DAG.getDataLayout().isLittleEndian()) {
532  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(),
533  NewLoadedVT, LD->isVolatile(),
534  LD->isNonTemporal(), LD->isInvariant(), Alignment,
535  LD->getAAInfo());
536  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
537  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
538  Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr,
539  LD->getPointerInfo().getWithOffset(IncrementSize),
540  NewLoadedVT, LD->isVolatile(),
541  LD->isNonTemporal(),LD->isInvariant(),
542  MinAlign(Alignment, IncrementSize), LD->getAAInfo());
543  } else {
544  Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(),
545  NewLoadedVT, LD->isVolatile(),
546  LD->isNonTemporal(), LD->isInvariant(), Alignment,
547  LD->getAAInfo());
548  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
549  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
550  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr,
551  LD->getPointerInfo().getWithOffset(IncrementSize),
552  NewLoadedVT, LD->isVolatile(),
553  LD->isNonTemporal(), LD->isInvariant(),
554  MinAlign(Alignment, IncrementSize), LD->getAAInfo());
555  }
556 
557  // aggregate the two parts
558  SDValue ShiftAmount =
559  DAG.getConstant(NumBits, dl, TLI.getShiftAmountTy(Hi.getValueType(),
560  DAG.getDataLayout()));
561  SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount);
562  Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo);
563 
564  SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
565  Hi.getValue(1));
566 
567  ValResult = Result;
568  ChainResult = TF;
569 }
570 
571 /// Some target cannot handle a variable insertion index for the
572 /// INSERT_VECTOR_ELT instruction. In this case, it
573 /// is necessary to spill the vector being inserted into to memory, perform
574 /// the insert there, and then read the result back.
575 SDValue SelectionDAGLegalize::
576 PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
577  SDLoc dl) {
578  SDValue Tmp1 = Vec;
579  SDValue Tmp2 = Val;
580  SDValue Tmp3 = Idx;
581 
582  // If the target doesn't support this, we have to spill the input vector
583  // to a temporary stack slot, update the element, then reload it. This is
584  // badness. We could also load the value into a vector register (either
585  // with a "move to register" or "extload into register" instruction, then
586  // permute it into place, if the idx is a constant and if the idx is
587  // supported by the target.
588  EVT VT = Tmp1.getValueType();
589  EVT EltVT = VT.getVectorElementType();
590  EVT IdxVT = Tmp3.getValueType();
591  EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
592  SDValue StackPtr = DAG.CreateStackTemporary(VT);
593 
594  int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
595 
596  // Store the vector.
597  SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Tmp1, StackPtr,
599  false, false, 0);
600 
601  // Truncate or zero extend offset to target pointer type.
602  unsigned CastOpc = IdxVT.bitsGT(PtrVT) ? ISD::TRUNCATE : ISD::ZERO_EXTEND;
603  Tmp3 = DAG.getNode(CastOpc, dl, PtrVT, Tmp3);
604  // Add the offset to the index.
605  unsigned EltSize = EltVT.getSizeInBits()/8;
606  Tmp3 = DAG.getNode(ISD::MUL, dl, IdxVT, Tmp3,
607  DAG.getConstant(EltSize, dl, IdxVT));
608  SDValue StackPtr2 = DAG.getNode(ISD::ADD, dl, IdxVT, Tmp3, StackPtr);
609  // Store the scalar value.
610  Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT,
611  false, false, 0);
612  // Load the updated vector.
613  return DAG.getLoad(VT, dl, Ch, StackPtr,
614  MachinePointerInfo::getFixedStack(SPFI), false, false,
615  false, 0);
616 }
617 
618 
619 SDValue SelectionDAGLegalize::
620 ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx, SDLoc dl) {
621  if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
622  // SCALAR_TO_VECTOR requires that the type of the value being inserted
623  // match the element type of the vector being created, except for
624  // integers in which case the inserted value can be over width.
625  EVT EltVT = Vec.getValueType().getVectorElementType();
626  if (Val.getValueType() == EltVT ||
627  (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
628  SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
629  Vec.getValueType(), Val);
630 
631  unsigned NumElts = Vec.getValueType().getVectorNumElements();
632  // We generate a shuffle of InVec and ScVec, so the shuffle mask
633  // should be 0,1,2,3,4,5... with the appropriate element replaced with
634  // elt 0 of the RHS.
635  SmallVector<int, 8> ShufOps;
636  for (unsigned i = 0; i != NumElts; ++i)
637  ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
638 
639  return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec,
640  &ShufOps[0]);
641  }
642  }
643  return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
644 }
645 
646 SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
647  // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
648  // FIXME: We shouldn't do this for TargetConstantFP's.
649  // FIXME: move this to the DAG Combiner! Note that we can't regress due
650  // to phase ordering between legalized code and the dag combiner. This
651  // probably means that we need to integrate dag combiner and legalizer
652  // together.
653  // We generally can't do this one for long doubles.
654  SDValue Chain = ST->getChain();
655  SDValue Ptr = ST->getBasePtr();
656  unsigned Alignment = ST->getAlignment();
657  bool isVolatile = ST->isVolatile();
658  bool isNonTemporal = ST->isNonTemporal();
659  AAMDNodes AAInfo = ST->getAAInfo();
660  SDLoc dl(ST);
661  if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) {
662  if (CFP->getValueType(0) == MVT::f32 &&
663  TLI.isTypeLegal(MVT::i32)) {
664  SDValue Con = DAG.getConstant(CFP->getValueAPF().
665  bitcastToAPInt().zextOrTrunc(32),
666  SDLoc(CFP), MVT::i32);
667  return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
668  isVolatile, isNonTemporal, Alignment, AAInfo);
669  }
670 
671  if (CFP->getValueType(0) == MVT::f64) {
672  // If this target supports 64-bit registers, do a single 64-bit store.
673  if (TLI.isTypeLegal(MVT::i64)) {
674  SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
675  zextOrTrunc(64), SDLoc(CFP), MVT::i64);
676  return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
677  isVolatile, isNonTemporal, Alignment, AAInfo);
678  }
679 
680  if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
681  // Otherwise, if the target supports 32-bit registers, use 2 32-bit
682  // stores. If the target supports neither 32- nor 64-bits, this
683  // xform is certainly not worth it.
684  const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
685  SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
686  SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
687  if (DAG.getDataLayout().isBigEndian())
688  std::swap(Lo, Hi);
689 
690  Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), isVolatile,
691  isNonTemporal, Alignment, AAInfo);
692  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
693  DAG.getConstant(4, dl, Ptr.getValueType()));
694  Hi = DAG.getStore(Chain, dl, Hi, Ptr,
695  ST->getPointerInfo().getWithOffset(4),
696  isVolatile, isNonTemporal, MinAlign(Alignment, 4U),
697  AAInfo);
698 
699  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
700  }
701  }
702  }
703  return SDValue(nullptr, 0);
704 }
705 
706 void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
707  StoreSDNode *ST = cast<StoreSDNode>(Node);
708  SDValue Chain = ST->getChain();
709  SDValue Ptr = ST->getBasePtr();
710  SDLoc dl(Node);
711 
712  unsigned Alignment = ST->getAlignment();
713  bool isVolatile = ST->isVolatile();
714  bool isNonTemporal = ST->isNonTemporal();
715  AAMDNodes AAInfo = ST->getAAInfo();
716 
717  if (!ST->isTruncatingStore()) {
718  if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
719  ReplaceNode(ST, OptStore);
720  return;
721  }
722 
723  {
724  SDValue Value = ST->getValue();
725  MVT VT = Value.getSimpleValueType();
726  switch (TLI.getOperationAction(ISD::STORE, VT)) {
727  default: llvm_unreachable("This action is not supported yet!");
728  case TargetLowering::Legal: {
729  // If this is an unaligned store and the target doesn't support it,
730  // expand it.
731  unsigned AS = ST->getAddressSpace();
732  unsigned Align = ST->getAlignment();
733  if (!TLI.allowsMisalignedMemoryAccesses(ST->getMemoryVT(), AS, Align)) {
734  Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext());
735  unsigned ABIAlignment = DAG.getDataLayout().getABITypeAlignment(Ty);
736  if (Align < ABIAlignment)
737  ExpandUnalignedStore(cast<StoreSDNode>(Node), DAG, TLI, this);
738  }
739  break;
740  }
741  case TargetLowering::Custom: {
742  SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
743  if (Res && Res != SDValue(Node, 0))
744  ReplaceNode(SDValue(Node, 0), Res);
745  return;
746  }
748  MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
749  assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
750  "Can only promote stores to same size type");
751  Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
752  SDValue Result =
753  DAG.getStore(Chain, dl, Value, Ptr,
754  ST->getPointerInfo(), isVolatile,
755  isNonTemporal, Alignment, AAInfo);
756  ReplaceNode(SDValue(Node, 0), Result);
757  break;
758  }
759  }
760  return;
761  }
762  } else {
763  SDValue Value = ST->getValue();
764 
765  EVT StVT = ST->getMemoryVT();
766  unsigned StWidth = StVT.getSizeInBits();
767  auto &DL = DAG.getDataLayout();
768 
769  if (StWidth != StVT.getStoreSizeInBits()) {
770  // Promote to a byte-sized store with upper bits zero if not
771  // storing an integral number of bytes. For example, promote
772  // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
773  EVT NVT = EVT::getIntegerVT(*DAG.getContext(),
774  StVT.getStoreSizeInBits());
775  Value = DAG.getZeroExtendInReg(Value, dl, StVT);
776  SDValue Result =
777  DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
778  NVT, isVolatile, isNonTemporal, Alignment, AAInfo);
779  ReplaceNode(SDValue(Node, 0), Result);
780  } else if (StWidth & (StWidth - 1)) {
781  // If not storing a power-of-2 number of bits, expand as two stores.
782  assert(!StVT.isVector() && "Unsupported truncstore!");
783  unsigned RoundWidth = 1 << Log2_32(StWidth);
784  assert(RoundWidth < StWidth);
785  unsigned ExtraWidth = StWidth - RoundWidth;
786  assert(ExtraWidth < RoundWidth);
787  assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
788  "Store size not an integral number of bytes!");
789  EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
790  EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
791  SDValue Lo, Hi;
792  unsigned IncrementSize;
793 
794  if (DL.isLittleEndian()) {
795  // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
796  // Store the bottom RoundWidth bits.
797  Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
798  RoundVT,
799  isVolatile, isNonTemporal, Alignment,
800  AAInfo);
801 
802  // Store the remaining ExtraWidth bits.
803  IncrementSize = RoundWidth / 8;
804  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
805  DAG.getConstant(IncrementSize, dl,
806  Ptr.getValueType()));
807  Hi = DAG.getNode(
808  ISD::SRL, dl, Value.getValueType(), Value,
809  DAG.getConstant(RoundWidth, dl,
810  TLI.getShiftAmountTy(Value.getValueType(), DL)));
811  Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr,
812  ST->getPointerInfo().getWithOffset(IncrementSize),
813  ExtraVT, isVolatile, isNonTemporal,
814  MinAlign(Alignment, IncrementSize), AAInfo);
815  } else {
816  // Big endian - avoid unaligned stores.
817  // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
818  // Store the top RoundWidth bits.
819  Hi = DAG.getNode(
820  ISD::SRL, dl, Value.getValueType(), Value,
821  DAG.getConstant(ExtraWidth, dl,
822  TLI.getShiftAmountTy(Value.getValueType(), DL)));
823  Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(),
824  RoundVT, isVolatile, isNonTemporal, Alignment,
825  AAInfo);
826 
827  // Store the remaining ExtraWidth bits.
828  IncrementSize = RoundWidth / 8;
829  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
830  DAG.getConstant(IncrementSize, dl,
831  Ptr.getValueType()));
832  Lo = DAG.getTruncStore(Chain, dl, Value, Ptr,
833  ST->getPointerInfo().getWithOffset(IncrementSize),
834  ExtraVT, isVolatile, isNonTemporal,
835  MinAlign(Alignment, IncrementSize), AAInfo);
836  }
837 
838  // The order of the stores doesn't matter.
839  SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
840  ReplaceNode(SDValue(Node, 0), Result);
841  } else {
842  switch (TLI.getTruncStoreAction(ST->getValue().getSimpleValueType(),
843  StVT.getSimpleVT())) {
844  default: llvm_unreachable("This action is not supported yet!");
845  case TargetLowering::Legal: {
846  unsigned AS = ST->getAddressSpace();
847  unsigned Align = ST->getAlignment();
848  // If this is an unaligned store and the target doesn't support it,
849  // expand it.
850  if (!TLI.allowsMisalignedMemoryAccesses(ST->getMemoryVT(), AS, Align)) {
851  Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext());
852  unsigned ABIAlignment = DL.getABITypeAlignment(Ty);
853  if (Align < ABIAlignment)
854  ExpandUnalignedStore(cast<StoreSDNode>(Node), DAG, TLI, this);
855  }
856  break;
857  }
858  case TargetLowering::Custom: {
859  SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
860  if (Res && Res != SDValue(Node, 0))
861  ReplaceNode(SDValue(Node, 0), Res);
862  return;
863  }
865  assert(!StVT.isVector() &&
866  "Vector Stores are handled in LegalizeVectorOps");
867 
868  // TRUNCSTORE:i16 i32 -> STORE i16
869  assert(TLI.isTypeLegal(StVT) &&
870  "Do not know how to expand this store!");
871  Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
872  SDValue Result =
873  DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
874  isVolatile, isNonTemporal, Alignment, AAInfo);
875  ReplaceNode(SDValue(Node, 0), Result);
876  break;
877  }
878  }
879  }
880 }
881 
882 void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
883  LoadSDNode *LD = cast<LoadSDNode>(Node);
884  SDValue Chain = LD->getChain(); // The chain.
885  SDValue Ptr = LD->getBasePtr(); // The base pointer.
886  SDValue Value; // The value returned by the load op.
887  SDLoc dl(Node);
888 
889  ISD::LoadExtType ExtType = LD->getExtensionType();
890  if (ExtType == ISD::NON_EXTLOAD) {
891  MVT VT = Node->getSimpleValueType(0);
892  SDValue RVal = SDValue(Node, 0);
893  SDValue RChain = SDValue(Node, 1);
894 
895  switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
896  default: llvm_unreachable("This action is not supported yet!");
897  case TargetLowering::Legal: {
898  unsigned AS = LD->getAddressSpace();
899  unsigned Align = LD->getAlignment();
900  // If this is an unaligned load and the target doesn't support it,
901  // expand it.
902  if (!TLI.allowsMisalignedMemoryAccesses(LD->getMemoryVT(), AS, Align)) {
903  Type *Ty = LD->getMemoryVT().getTypeForEVT(*DAG.getContext());
904  unsigned ABIAlignment = DAG.getDataLayout().getABITypeAlignment(Ty);
905  if (Align < ABIAlignment){
906  ExpandUnalignedLoad(cast<LoadSDNode>(Node), DAG, TLI, RVal, RChain);
907  }
908  }
909  break;
910  }
911  case TargetLowering::Custom: {
912  SDValue Res = TLI.LowerOperation(RVal, DAG);
913  if (Res.getNode()) {
914  RVal = Res;
915  RChain = Res.getValue(1);
916  }
917  break;
918  }
920  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
921  assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
922  "Can only promote loads to same size type");
923 
924  SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
925  RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
926  RChain = Res.getValue(1);
927  break;
928  }
929  }
930  if (RChain.getNode() != Node) {
931  assert(RVal.getNode() != Node && "Load must be completely replaced");
932  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
933  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
934  if (UpdatedNodes) {
935  UpdatedNodes->insert(RVal.getNode());
936  UpdatedNodes->insert(RChain.getNode());
937  }
938  ReplacedNode(Node);
939  }
940  return;
941  }
942 
943  EVT SrcVT = LD->getMemoryVT();
944  unsigned SrcWidth = SrcVT.getSizeInBits();
945  unsigned Alignment = LD->getAlignment();
946  bool isVolatile = LD->isVolatile();
947  bool isNonTemporal = LD->isNonTemporal();
948  bool isInvariant = LD->isInvariant();
949  AAMDNodes AAInfo = LD->getAAInfo();
950 
951  if (SrcWidth != SrcVT.getStoreSizeInBits() &&
952  // Some targets pretend to have an i1 loading operation, and actually
953  // load an i8. This trick is correct for ZEXTLOAD because the top 7
954  // bits are guaranteed to be zero; it helps the optimizers understand
955  // that these bits are zero. It is also useful for EXTLOAD, since it
956  // tells the optimizers that those bits are undefined. It would be
957  // nice to have an effective generic way of getting these benefits...
958  // Until such a way is found, don't insist on promoting i1 here.
959  (SrcVT != MVT::i1 ||
960  TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
962  // Promote to a byte-sized load if not loading an integral number of
963  // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
964  unsigned NewWidth = SrcVT.getStoreSizeInBits();
965  EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
966  SDValue Ch;
967 
968  // The extra bits are guaranteed to be zero, since we stored them that
969  // way. A zext load from NVT thus automatically gives zext from SrcVT.
970 
971  ISD::LoadExtType NewExtType =
973 
974  SDValue Result =
975  DAG.getExtLoad(NewExtType, dl, Node->getValueType(0),
976  Chain, Ptr, LD->getPointerInfo(),
977  NVT, isVolatile, isNonTemporal, isInvariant, Alignment,
978  AAInfo);
979 
980  Ch = Result.getValue(1); // The chain.
981 
982  if (ExtType == ISD::SEXTLOAD)
983  // Having the top bits zero doesn't help when sign extending.
984  Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
985  Result.getValueType(),
986  Result, DAG.getValueType(SrcVT));
987  else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
988  // All the top bits are guaranteed to be zero - inform the optimizers.
989  Result = DAG.getNode(ISD::AssertZext, dl,
990  Result.getValueType(), Result,
991  DAG.getValueType(SrcVT));
992 
993  Value = Result;
994  Chain = Ch;
995  } else if (SrcWidth & (SrcWidth - 1)) {
996  // If not loading a power-of-2 number of bits, expand as two loads.
997  assert(!SrcVT.isVector() && "Unsupported extload!");
998  unsigned RoundWidth = 1 << Log2_32(SrcWidth);
999  assert(RoundWidth < SrcWidth);
1000  unsigned ExtraWidth = SrcWidth - RoundWidth;
1001  assert(ExtraWidth < RoundWidth);
1002  assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
1003  "Load size not an integral number of bytes!");
1004  EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
1005  EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
1006  SDValue Lo, Hi, Ch;
1007  unsigned IncrementSize;
1008  auto &DL = DAG.getDataLayout();
1009 
1010  if (DL.isLittleEndian()) {
1011  // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
1012  // Load the bottom RoundWidth bits.
1013  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0),
1014  Chain, Ptr,
1015  LD->getPointerInfo(), RoundVT, isVolatile,
1016  isNonTemporal, isInvariant, Alignment, AAInfo);
1017 
1018  // Load the remaining ExtraWidth bits.
1019  IncrementSize = RoundWidth / 8;
1020  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
1021  DAG.getConstant(IncrementSize, dl,
1022  Ptr.getValueType()));
1023  Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
1024  LD->getPointerInfo().getWithOffset(IncrementSize),
1025  ExtraVT, isVolatile, isNonTemporal, isInvariant,
1026  MinAlign(Alignment, IncrementSize), AAInfo);
1027 
1028  // Build a factor node to remember that this load is independent of
1029  // the other one.
1030  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1031  Hi.getValue(1));
1032 
1033  // Move the top bits to the right place.
1034  Hi = DAG.getNode(
1035  ISD::SHL, dl, Hi.getValueType(), Hi,
1036  DAG.getConstant(RoundWidth, dl,
1037  TLI.getShiftAmountTy(Hi.getValueType(), DL)));
1038 
1039  // Join the hi and lo parts.
1040  Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
1041  } else {
1042  // Big endian - avoid unaligned loads.
1043  // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
1044  // Load the top RoundWidth bits.
1045  Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
1046  LD->getPointerInfo(), RoundVT, isVolatile,
1047  isNonTemporal, isInvariant, Alignment, AAInfo);
1048 
1049  // Load the remaining ExtraWidth bits.
1050  IncrementSize = RoundWidth / 8;
1051  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
1052  DAG.getConstant(IncrementSize, dl,
1053  Ptr.getValueType()));
1054  Lo = DAG.getExtLoad(ISD::ZEXTLOAD,
1055  dl, Node->getValueType(0), Chain, Ptr,
1056  LD->getPointerInfo().getWithOffset(IncrementSize),
1057  ExtraVT, isVolatile, isNonTemporal, isInvariant,
1058  MinAlign(Alignment, IncrementSize), AAInfo);
1059 
1060  // Build a factor node to remember that this load is independent of
1061  // the other one.
1062  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1063  Hi.getValue(1));
1064 
1065  // Move the top bits to the right place.
1066  Hi = DAG.getNode(
1067  ISD::SHL, dl, Hi.getValueType(), Hi,
1068  DAG.getConstant(ExtraWidth, dl,
1069  TLI.getShiftAmountTy(Hi.getValueType(), DL)));
1070 
1071  // Join the hi and lo parts.
1072  Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
1073  }
1074 
1075  Chain = Ch;
1076  } else {
1077  bool isCustom = false;
1078  switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
1079  SrcVT.getSimpleVT())) {
1080  default: llvm_unreachable("This action is not supported yet!");
1082  isCustom = true;
1083  // FALLTHROUGH
1084  case TargetLowering::Legal: {
1085  Value = SDValue(Node, 0);
1086  Chain = SDValue(Node, 1);
1087 
1088  if (isCustom) {
1089  SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
1090  if (Res.getNode()) {
1091  Value = Res;
1092  Chain = Res.getValue(1);
1093  }
1094  } else {
1095  // If this is an unaligned load and the target doesn't support
1096  // it, expand it.
1097  EVT MemVT = LD->getMemoryVT();
1098  unsigned AS = LD->getAddressSpace();
1099  unsigned Align = LD->getAlignment();
1100  if (!TLI.allowsMisalignedMemoryAccesses(MemVT, AS, Align)) {
1101  Type *Ty = LD->getMemoryVT().getTypeForEVT(*DAG.getContext());
1102  unsigned ABIAlignment = DAG.getDataLayout().getABITypeAlignment(Ty);
1103  if (Align < ABIAlignment){
1104  ExpandUnalignedLoad(cast<LoadSDNode>(Node), DAG, TLI, Value, Chain);
1105  }
1106  }
1107  }
1108  break;
1109  }
1111  if (!TLI.isLoadExtLegal(ISD::EXTLOAD, Node->getValueType(0), SrcVT)) {
1112  // If the source type is not legal, see if there is a legal extload to
1113  // an intermediate type that we can then extend further.
1114  EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
1115  if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
1116  TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) {
1117  // If we are loading a legal type, this is a non-extload followed by a
1118  // full extend.
1119  ISD::LoadExtType MidExtType =
1120  (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
1121 
1122  SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
1123  SrcVT, LD->getMemOperand());
1124  unsigned ExtendOp =
1125  ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
1126  Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
1127  Chain = Load.getValue(1);
1128  break;
1129  }
1130  }
1131 
1132  assert(!SrcVT.isVector() &&
1133  "Vector Loads are handled in LegalizeVectorOps");
1134 
1135  // FIXME: This does not work for vectors on most targets. Sign-
1136  // and zero-extend operations are currently folded into extending
1137  // loads, whether they are legal or not, and then we end up here
1138  // without any support for legalizing them.
1139  assert(ExtType != ISD::EXTLOAD &&
1140  "EXTLOAD should always be supported!");
1141  // Turn the unsupported load into an EXTLOAD followed by an
1142  // explicit zero/sign extend inreg.
1143  SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
1144  Node->getValueType(0),
1145  Chain, Ptr, SrcVT,
1146  LD->getMemOperand());
1147  SDValue ValRes;
1148  if (ExtType == ISD::SEXTLOAD)
1149  ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
1150  Result.getValueType(),
1151  Result, DAG.getValueType(SrcVT));
1152  else
1153  ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType());
1154  Value = ValRes;
1155  Chain = Result.getValue(1);
1156  break;
1157  }
1158  }
1159 
1160  // Since loads produce two values, make sure to remember that we legalized
1161  // both of them.
1162  if (Chain.getNode() != Node) {
1163  assert(Value.getNode() != Node && "Load must be completely replaced");
1164  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
1165  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
1166  if (UpdatedNodes) {
1167  UpdatedNodes->insert(Value.getNode());
1168  UpdatedNodes->insert(Chain.getNode());
1169  }
1170  ReplacedNode(Node);
1171  }
1172 }
1173 
1174 /// Return a legal replacement for the given operation, with all legal operands.
1175 void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
1176  DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
1177 
1178  if (Node->getOpcode() == ISD::TargetConstant) // Allow illegal target nodes.
1179  return;
1180 
1181 #ifndef NDEBUG
1182  for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
1183  assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
1185  "Unexpected illegal type!");
1186 
1187  for (const SDValue &Op : Node->op_values())
1188  assert((TLI.getTypeAction(*DAG.getContext(),
1189  Op.getValueType()) == TargetLowering::TypeLegal ||
1190  Op.getOpcode() == ISD::TargetConstant) &&
1191  "Unexpected illegal type!");
1192 #endif
1193 
1194  // Figure out the correct action; the way to query this varies by opcode
1196  bool SimpleFinishLegalizing = true;
1197  switch (Node->getOpcode()) {
1200  case ISD::INTRINSIC_VOID:
1201  case ISD::STACKSAVE:
1202  Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1203  break;
1204  case ISD::VAARG:
1205  Action = TLI.getOperationAction(Node->getOpcode(),
1206  Node->getValueType(0));
1207  if (Action != TargetLowering::Promote)
1208  Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1209  break;
1210  case ISD::FP_TO_FP16:
1211  case ISD::SINT_TO_FP:
1212  case ISD::UINT_TO_FP:
1214  Action = TLI.getOperationAction(Node->getOpcode(),
1215  Node->getOperand(0).getValueType());
1216  break;
1217  case ISD::FP_ROUND_INREG:
1218  case ISD::SIGN_EXTEND_INREG: {
1219  EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1220  Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1221  break;
1222  }
1223  case ISD::ATOMIC_STORE: {
1224  Action = TLI.getOperationAction(Node->getOpcode(),
1225  Node->getOperand(2).getValueType());
1226  break;
1227  }
1228  case ISD::SELECT_CC:
1229  case ISD::SETCC:
1230  case ISD::BR_CC: {
1231  unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
1232  Node->getOpcode() == ISD::SETCC ? 2 : 1;
1233  unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0;
1234  MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1235  ISD::CondCode CCCode =
1236  cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1237  Action = TLI.getCondCodeAction(CCCode, OpVT);
1238  if (Action == TargetLowering::Legal) {
1239  if (Node->getOpcode() == ISD::SELECT_CC)
1240  Action = TLI.getOperationAction(Node->getOpcode(),
1241  Node->getValueType(0));
1242  else
1243  Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1244  }
1245  break;
1246  }
1247  case ISD::LOAD:
1248  case ISD::STORE:
1249  // FIXME: Model these properly. LOAD and STORE are complicated, and
1250  // STORE expects the unlegalized operand in some cases.
1251  SimpleFinishLegalizing = false;
1252  break;
1253  case ISD::CALLSEQ_START:
1254  case ISD::CALLSEQ_END:
1255  // FIXME: This shouldn't be necessary. These nodes have special properties
1256  // dealing with the recursive nature of legalization. Removing this
1257  // special case should be done as part of making LegalizeDAG non-recursive.
1258  SimpleFinishLegalizing = false;
1259  break;
1260  case ISD::EXTRACT_ELEMENT:
1261  case ISD::FLT_ROUNDS_:
1262  case ISD::FPOWI:
1263  case ISD::MERGE_VALUES:
1264  case ISD::EH_RETURN:
1266  case ISD::EH_SJLJ_SETJMP:
1267  case ISD::EH_SJLJ_LONGJMP:
1268  // These operations lie about being legal: when they claim to be legal,
1269  // they should actually be expanded.
1270  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1271  if (Action == TargetLowering::Legal)
1272  Action = TargetLowering::Expand;
1273  break;
1274  case ISD::INIT_TRAMPOLINE:
1276  case ISD::FRAMEADDR:
1277  case ISD::RETURNADDR:
1278  // These operations lie about being legal: when they claim to be legal,
1279  // they should actually be custom-lowered.
1280  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1281  if (Action == TargetLowering::Legal)
1282  Action = TargetLowering::Custom;
1283  break;
1284  case ISD::READ_REGISTER:
1285  case ISD::WRITE_REGISTER:
1286  // Named register is legal in the DAG, but blocked by register name
1287  // selection if not implemented by target (to chose the correct register)
1288  // They'll be converted to Copy(To/From)Reg.
1289  Action = TargetLowering::Legal;
1290  break;
1291  case ISD::DEBUGTRAP:
1292  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1293  if (Action == TargetLowering::Expand) {
1294  // replace ISD::DEBUGTRAP with ISD::TRAP
1295  SDValue NewVal;
1296  NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1297  Node->getOperand(0));
1298  ReplaceNode(Node, NewVal.getNode());
1299  LegalizeOp(NewVal.getNode());
1300  return;
1301  }
1302  break;
1303 
1304  default:
1305  if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1306  Action = TargetLowering::Legal;
1307  } else {
1308  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1309  }
1310  break;
1311  }
1312 
1313  if (SimpleFinishLegalizing) {
1314  SDNode *NewNode = Node;
1315  switch (Node->getOpcode()) {
1316  default: break;
1317  case ISD::SHL:
1318  case ISD::SRL:
1319  case ISD::SRA:
1320  case ISD::ROTL:
1321  case ISD::ROTR:
1322  // Legalizing shifts/rotates requires adjusting the shift amount
1323  // to the appropriate width.
1324  if (!Node->getOperand(1).getValueType().isVector()) {
1325  SDValue SAO =
1326  DAG.getShiftAmountOperand(Node->getOperand(0).getValueType(),
1327  Node->getOperand(1));
1328  HandleSDNode Handle(SAO);
1329  LegalizeOp(SAO.getNode());
1330  NewNode = DAG.UpdateNodeOperands(Node, Node->getOperand(0),
1331  Handle.getValue());
1332  }
1333  break;
1334  case ISD::SRL_PARTS:
1335  case ISD::SRA_PARTS:
1336  case ISD::SHL_PARTS:
1337  // Legalizing shifts/rotates requires adjusting the shift amount
1338  // to the appropriate width.
1339  if (!Node->getOperand(2).getValueType().isVector()) {
1340  SDValue SAO =
1341  DAG.getShiftAmountOperand(Node->getOperand(0).getValueType(),
1342  Node->getOperand(2));
1343  HandleSDNode Handle(SAO);
1344  LegalizeOp(SAO.getNode());
1345  NewNode = DAG.UpdateNodeOperands(Node, Node->getOperand(0),
1346  Node->getOperand(1),
1347  Handle.getValue());
1348  }
1349  break;
1350  }
1351 
1352  if (NewNode != Node) {
1353  ReplaceNode(Node, NewNode);
1354  Node = NewNode;
1355  }
1356  switch (Action) {
1357  case TargetLowering::Legal:
1358  return;
1359  case TargetLowering::Custom: {
1360  // FIXME: The handling for custom lowering with multiple results is
1361  // a complete mess.
1362  SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
1363  if (Res.getNode()) {
1364  if (!(Res.getNode() != Node || Res.getResNo() != 0))
1365  return;
1366 
1367  if (Node->getNumValues() == 1) {
1368  // We can just directly replace this node with the lowered value.
1369  ReplaceNode(SDValue(Node, 0), Res);
1370  return;
1371  }
1372 
1373  SmallVector<SDValue, 8> ResultVals;
1374  for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
1375  ResultVals.push_back(Res.getValue(i));
1376  ReplaceNode(Node, ResultVals.data());
1377  return;
1378  }
1379  }
1380  // FALL THROUGH
1382  ExpandNode(Node);
1383  return;
1385  PromoteNode(Node);
1386  return;
1387  }
1388  }
1389 
1390  switch (Node->getOpcode()) {
1391  default:
1392 #ifndef NDEBUG
1393  dbgs() << "NODE: ";
1394  Node->dump( &DAG);
1395  dbgs() << "\n";
1396 #endif
1397  llvm_unreachable("Do not know how to legalize this operator!");
1398 
1399  case ISD::CALLSEQ_START:
1400  case ISD::CALLSEQ_END:
1401  break;
1402  case ISD::LOAD: {
1403  return LegalizeLoadOps(Node);
1404  }
1405  case ISD::STORE: {
1406  return LegalizeStoreOps(Node);
1407  }
1408  }
1409 }
1410 
1411 SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1412  SDValue Vec = Op.getOperand(0);
1413  SDValue Idx = Op.getOperand(1);
1414  SDLoc dl(Op);
1415 
1416  // Before we generate a new store to a temporary stack slot, see if there is
1417  // already one that we can use. There often is because when we scalarize
1418  // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1419  // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1420  // the vector. If all are expanded here, we don't want one store per vector
1421  // element.
1422  SDValue StackPtr, Ch;
1423  for (SDNode::use_iterator UI = Vec.getNode()->use_begin(),
1424  UE = Vec.getNode()->use_end(); UI != UE; ++UI) {
1425  SDNode *User = *UI;
1426  if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1427  if (ST->isIndexed() || ST->isTruncatingStore() ||
1428  ST->getValue() != Vec)
1429  continue;
1430 
1431  // Make sure that nothing else could have stored into the destination of
1432  // this store.
1433  if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1434  continue;
1435 
1436  StackPtr = ST->getBasePtr();
1437  Ch = SDValue(ST, 0);
1438  break;
1439  }
1440  }
1441 
1442  if (!Ch.getNode()) {
1443  // Store the value to a temporary stack slot, then LOAD the returned part.
1444  StackPtr = DAG.CreateStackTemporary(Vec.getValueType());
1445  Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1446  MachinePointerInfo(), false, false, 0);
1447  }
1448 
1449  // Add the offset to the index.
1450  unsigned EltSize =
1452  Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx,
1453  DAG.getConstant(EltSize, SDLoc(Vec), Idx.getValueType()));
1454 
1455  Idx = DAG.getZExtOrTrunc(Idx, dl, TLI.getPointerTy(DAG.getDataLayout()));
1456  StackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, StackPtr);
1457 
1458  SDValue NewLoad;
1459 
1460  if (Op.getValueType().isVector())
1461  NewLoad = DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr,
1462  MachinePointerInfo(), false, false, false, 0);
1463  else
1464  NewLoad = DAG.getExtLoad(
1465  ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr, MachinePointerInfo(),
1466  Vec.getValueType().getVectorElementType(), false, false, false, 0);
1467 
1468  // Replace the chain going out of the store, by the one out of the load.
1469  DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1470 
1471  // We introduced a cycle though, so update the loads operands, making sure
1472  // to use the original store's chain as an incoming chain.
1473  SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1474  NewLoad->op_end());
1475  NewLoadOperands[0] = Ch;
1476  NewLoad =
1477  SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1478  return NewLoad;
1479 }
1480 
1481 SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1482  assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1483 
1484  SDValue Vec = Op.getOperand(0);
1485  SDValue Part = Op.getOperand(1);
1486  SDValue Idx = Op.getOperand(2);
1487  SDLoc dl(Op);
1488 
1489  // Store the value to a temporary stack slot, then LOAD the returned part.
1490 
1491  SDValue StackPtr = DAG.CreateStackTemporary(Vec.getValueType());
1492  int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1494 
1495  // First store the whole vector.
1496  SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1497  false, false, 0);
1498 
1499  // Then store the inserted part.
1500 
1501  // Add the offset to the index.
1502  unsigned EltSize =
1503  Vec.getValueType().getVectorElementType().getSizeInBits()/8;
1504 
1505  Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx,
1506  DAG.getConstant(EltSize, SDLoc(Vec), Idx.getValueType()));
1507  Idx = DAG.getZExtOrTrunc(Idx, dl, TLI.getPointerTy(DAG.getDataLayout()));
1508 
1509  SDValue SubStackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx,
1510  StackPtr);
1511 
1512  // Store the subvector.
1513  Ch = DAG.getStore(Ch, dl, Part, SubStackPtr,
1514  MachinePointerInfo(), false, false, 0);
1515 
1516  // Finally, load the updated vector.
1517  return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo,
1518  false, false, false, 0);
1519 }
1520 
1521 SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1522  // We can't handle this case efficiently. Allocate a sufficiently
1523  // aligned object on the stack, store each element into it, then load
1524  // the result as a vector.
1525  // Create the stack frame object.
1526  EVT VT = Node->getValueType(0);
1527  EVT EltVT = VT.getVectorElementType();
1528  SDLoc dl(Node);
1529  SDValue FIPtr = DAG.CreateStackTemporary(VT);
1530  int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1532 
1533  // Emit a store of each element to the stack slot.
1534  SmallVector<SDValue, 8> Stores;
1535  unsigned TypeByteSize = EltVT.getSizeInBits() / 8;
1536  // Store (in the right endianness) the elements to memory.
1537  for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1538  // Ignore undef elements.
1539  if (Node->getOperand(i).getOpcode() == ISD::UNDEF) continue;
1540 
1541  unsigned Offset = TypeByteSize*i;
1542 
1543  SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType());
1544  Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx);
1545 
1546  // If the destination vector element type is narrower than the source
1547  // element type, only store the bits necessary.
1548  if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) {
1549  Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1550  Node->getOperand(i), Idx,
1551  PtrInfo.getWithOffset(Offset),
1552  EltVT, false, false, 0));
1553  } else
1554  Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl,
1555  Node->getOperand(i), Idx,
1556  PtrInfo.getWithOffset(Offset),
1557  false, false, 0));
1558  }
1559 
1560  SDValue StoreChain;
1561  if (!Stores.empty()) // Not all undef elements?
1562  StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1563  else
1564  StoreChain = DAG.getEntryNode();
1565 
1566  // Result is a load from the stack slot.
1567  return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo,
1568  false, false, false, 0);
1569 }
1570 
1571 SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode* Node) {
1572  SDLoc dl(Node);
1573  SDValue Tmp1 = Node->getOperand(0);
1574  SDValue Tmp2 = Node->getOperand(1);
1575 
1576  // Get the sign bit of the RHS. First obtain a value that has the same
1577  // sign as the sign bit, i.e. negative if and only if the sign bit is 1.
1578  SDValue SignBit;
1579  EVT FloatVT = Tmp2.getValueType();
1580  EVT IVT = EVT::getIntegerVT(*DAG.getContext(), FloatVT.getSizeInBits());
1581  if (TLI.isTypeLegal(IVT)) {
1582  // Convert to an integer with the same sign bit.
1583  SignBit = DAG.getNode(ISD::BITCAST, dl, IVT, Tmp2);
1584  } else {
1585  auto &DL = DAG.getDataLayout();
1586  // Store the float to memory, then load the sign part out as an integer.
1587  MVT LoadTy = TLI.getPointerTy(DL);
1588  // First create a temporary that is aligned for both the load and store.
1589  SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1590  // Then store the float to it.
1591  SDValue Ch =
1592  DAG.getStore(DAG.getEntryNode(), dl, Tmp2, StackPtr, MachinePointerInfo(),
1593  false, false, 0);
1594  if (DL.isBigEndian()) {
1595  assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1596  // Load out a legal integer with the same sign bit as the float.
1597  SignBit = DAG.getLoad(LoadTy, dl, Ch, StackPtr, MachinePointerInfo(),
1598  false, false, false, 0);
1599  } else { // Little endian
1600  SDValue LoadPtr = StackPtr;
1601  // The float may be wider than the integer we are going to load. Advance
1602  // the pointer so that the loaded integer will contain the sign bit.
1603  unsigned Strides = (FloatVT.getSizeInBits()-1)/LoadTy.getSizeInBits();
1604  unsigned ByteOffset = (Strides * LoadTy.getSizeInBits()) / 8;
1605  LoadPtr = DAG.getNode(ISD::ADD, dl, LoadPtr.getValueType(), LoadPtr,
1606  DAG.getConstant(ByteOffset, dl,
1607  LoadPtr.getValueType()));
1608  // Load a legal integer containing the sign bit.
1609  SignBit = DAG.getLoad(LoadTy, dl, Ch, LoadPtr, MachinePointerInfo(),
1610  false, false, false, 0);
1611  // Move the sign bit to the top bit of the loaded integer.
1612  unsigned BitShift = LoadTy.getSizeInBits() -
1613  (FloatVT.getSizeInBits() - 8 * ByteOffset);
1614  assert(BitShift < LoadTy.getSizeInBits() && "Pointer advanced wrong?");
1615  if (BitShift)
1616  SignBit = DAG.getNode(
1617  ISD::SHL, dl, LoadTy, SignBit,
1618  DAG.getConstant(BitShift, dl,
1619  TLI.getShiftAmountTy(SignBit.getValueType(), DL)));
1620  }
1621  }
1622  // Now get the sign bit proper, by seeing whether the value is negative.
1623  SignBit = DAG.getSetCC(dl, getSetCCResultType(SignBit.getValueType()),
1624  SignBit,
1625  DAG.getConstant(0, dl, SignBit.getValueType()),
1626  ISD::SETLT);
1627  // Get the absolute value of the result.
1628  SDValue AbsVal = DAG.getNode(ISD::FABS, dl, Tmp1.getValueType(), Tmp1);
1629  // Select between the nabs and abs value based on the sign bit of
1630  // the input.
1631  return DAG.getSelect(dl, AbsVal.getValueType(), SignBit,
1632  DAG.getNode(ISD::FNEG, dl, AbsVal.getValueType(), AbsVal),
1633  AbsVal);
1634 }
1635 
1636 void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1637  SmallVectorImpl<SDValue> &Results) {
1638  unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
1639  assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1640  " not tell us which reg is the stack pointer!");
1641  SDLoc dl(Node);
1642  EVT VT = Node->getValueType(0);
1643  SDValue Tmp1 = SDValue(Node, 0);
1644  SDValue Tmp2 = SDValue(Node, 1);
1645  SDValue Tmp3 = Node->getOperand(2);
1646  SDValue Chain = Tmp1.getOperand(0);
1647 
1648  // Chain the dynamic stack allocation so that it doesn't modify the stack
1649  // pointer when other instructions are using the stack.
1650  Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(0, dl, true), dl);
1651 
1652  SDValue Size = Tmp2.getOperand(1);
1653  SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1654  Chain = SP.getValue(1);
1655  unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue();
1656  unsigned StackAlign =
1657  DAG.getSubtarget().getFrameLowering()->getStackAlignment();
1658  Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
1659  if (Align > StackAlign)
1660  Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1661  DAG.getConstant(-(uint64_t)Align, dl, VT));
1662  Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1663 
1664  Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
1665  DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
1666 
1667  Results.push_back(Tmp1);
1668  Results.push_back(Tmp2);
1669 }
1670 
1671 /// Legalize a SETCC with given LHS and RHS and condition code CC on the current
1672 /// target.
1673 ///
1674 /// If the SETCC has been legalized using AND / OR, then the legalized node
1675 /// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert
1676 /// will be set to false.
1677 ///
1678 /// If the SETCC has been legalized by using getSetCCSwappedOperands(),
1679 /// then the values of LHS and RHS will be swapped, CC will be set to the
1680 /// new condition, and NeedInvert will be set to false.
1681 ///
1682 /// If the SETCC has been legalized using the inverse condcode, then LHS and
1683 /// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert
1684 /// will be set to true. The caller must invert the result of the SETCC with
1685 /// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect
1686 /// of a true/false result.
1687 ///
1688 /// \returns true if the SetCC has been legalized, false if it hasn't.
1689 bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT,
1690  SDValue &LHS, SDValue &RHS,
1691  SDValue &CC,
1692  bool &NeedInvert,
1693  SDLoc dl) {
1694  MVT OpVT = LHS.getSimpleValueType();
1695  ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
1696  NeedInvert = false;
1697  switch (TLI.getCondCodeAction(CCCode, OpVT)) {
1698  default: llvm_unreachable("Unknown condition code action!");
1699  case TargetLowering::Legal:
1700  // Nothing to do.
1701  break;
1702  case TargetLowering::Expand: {
1704  if (TLI.isCondCodeLegal(InvCC, OpVT)) {
1705  std::swap(LHS, RHS);
1706  CC = DAG.getCondCode(InvCC);
1707  return true;
1708  }
1710  unsigned Opc = 0;
1711  switch (CCCode) {
1712  default: llvm_unreachable("Don't know how to expand this condition!");
1713  case ISD::SETO:
1714  assert(TLI.getCondCodeAction(ISD::SETOEQ, OpVT)
1716  && "If SETO is expanded, SETOEQ must be legal!");
1717  CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break;
1718  case ISD::SETUO:
1719  assert(TLI.getCondCodeAction(ISD::SETUNE, OpVT)
1721  && "If SETUO is expanded, SETUNE must be legal!");
1722  CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break;
1723  case ISD::SETOEQ:
1724  case ISD::SETOGT:
1725  case ISD::SETOGE:
1726  case ISD::SETOLT:
1727  case ISD::SETOLE:
1728  case ISD::SETONE:
1729  case ISD::SETUEQ:
1730  case ISD::SETUNE:
1731  case ISD::SETUGT:
1732  case ISD::SETUGE:
1733  case ISD::SETULT:
1734  case ISD::SETULE:
1735  // If we are floating point, assign and break, otherwise fall through.
1736  if (!OpVT.isInteger()) {
1737  // We can use the 4th bit to tell if we are the unordered
1738  // or ordered version of the opcode.
1739  CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
1740  Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
1741  CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
1742  break;
1743  }
1744  // Fallthrough if we are unsigned integer.
1745  case ISD::SETLE:
1746  case ISD::SETGT:
1747  case ISD::SETGE:
1748  case ISD::SETLT:
1749  // We only support using the inverted operation, which is computed above
1750  // and not a different manner of supporting expanding these cases.
1751  llvm_unreachable("Don't know how to expand this condition!");
1752  case ISD::SETNE:
1753  case ISD::SETEQ:
1754  // Try inverting the result of the inverse condition.
1755  InvCC = CCCode == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ;
1756  if (TLI.isCondCodeLegal(InvCC, OpVT)) {
1757  CC = DAG.getCondCode(InvCC);
1758  NeedInvert = true;
1759  return true;
1760  }
1761  // If inverting the condition didn't work then we have no means to expand
1762  // the condition.
1763  llvm_unreachable("Don't know how to expand this condition!");
1764  }
1765 
1766  SDValue SetCC1, SetCC2;
1767  if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
1768  // If we aren't the ordered or unorder operation,
1769  // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
1770  SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1);
1771  SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2);
1772  } else {
1773  // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
1774  SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1);
1775  SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2);
1776  }
1777  LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
1778  RHS = SDValue();
1779  CC = SDValue();
1780  return true;
1781  }
1782  }
1783  return false;
1784 }
1785 
1786 /// Emit a store/load combination to the stack. This stores
1787 /// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1788 /// a load from the stack slot to DestVT, extending it if needed.
1789 /// The resultant code need not be legal.
1790 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp,
1791  EVT SlotVT,
1792  EVT DestVT,
1793  SDLoc dl) {
1794  // Create the stack frame object.
1795  unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment(
1796  SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1797  SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign);
1798 
1799  FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1800  int SPFI = StackPtrFI->getIndex();
1802 
1803  unsigned SrcSize = SrcOp.getValueType().getSizeInBits();
1804  unsigned SlotSize = SlotVT.getSizeInBits();
1805  unsigned DestSize = DestVT.getSizeInBits();
1806  Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1807  unsigned DestAlign = DAG.getDataLayout().getPrefTypeAlignment(DestType);
1808 
1809  // Emit a store to the stack slot. Use a truncstore if the input value is
1810  // later than DestVT.
1811  SDValue Store;
1812 
1813  if (SrcSize > SlotSize)
1814  Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr,
1815  PtrInfo, SlotVT, false, false, SrcAlign);
1816  else {
1817  assert(SrcSize == SlotSize && "Invalid store");
1818  Store = DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr,
1819  PtrInfo, false, false, SrcAlign);
1820  }
1821 
1822  // Result is a load from the stack slot.
1823  if (SlotSize == DestSize)
1824  return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo,
1825  false, false, false, DestAlign);
1826 
1827  assert(SlotSize < DestSize && "Unknown extension!");
1828  return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr,
1829  PtrInfo, SlotVT, false, false, false, DestAlign);
1830 }
1831 
1832 SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1833  SDLoc dl(Node);
1834  // Create a vector sized/aligned stack slot, store the value to element #0,
1835  // then load the whole vector back out.
1836  SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1837 
1838  FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1839  int SPFI = StackPtrFI->getIndex();
1840 
1841  SDValue Ch = DAG.getTruncStore(DAG.getEntryNode(), dl, Node->getOperand(0),
1842  StackPtr,
1845  false, false, 0);
1846  return DAG.getLoad(Node->getValueType(0), dl, Ch, StackPtr,
1848  false, false, false, 0);
1849 }
1850 
1851 static bool
1853  const TargetLowering &TLI, SDValue &Res) {
1854  unsigned NumElems = Node->getNumOperands();
1855  SDLoc dl(Node);
1856  EVT VT = Node->getValueType(0);
1857 
1858  // Try to group the scalars into pairs, shuffle the pairs together, then
1859  // shuffle the pairs of pairs together, etc. until the vector has
1860  // been built. This will work only if all of the necessary shuffle masks
1861  // are legal.
1862 
1863  // We do this in two phases; first to check the legality of the shuffles,
1864  // and next, assuming that all shuffles are legal, to create the new nodes.
1865  for (int Phase = 0; Phase < 2; ++Phase) {
1867  NewIntermedVals;
1868  for (unsigned i = 0; i < NumElems; ++i) {
1869  SDValue V = Node->getOperand(i);
1870  if (V.getOpcode() == ISD::UNDEF)
1871  continue;
1872 
1873  SDValue Vec;
1874  if (Phase)
1875  Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1876  IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1877  }
1878 
1879  while (IntermedVals.size() > 2) {
1880  NewIntermedVals.clear();
1881  for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1882  // This vector and the next vector are shuffled together (simply to
1883  // append the one to the other).
1884  SmallVector<int, 16> ShuffleVec(NumElems, -1);
1885 
1886  SmallVector<int, 16> FinalIndices;
1887  FinalIndices.reserve(IntermedVals[i].second.size() +
1888  IntermedVals[i+1].second.size());
1889 
1890  int k = 0;
1891  for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1892  ++j, ++k) {
1893  ShuffleVec[k] = j;
1894  FinalIndices.push_back(IntermedVals[i].second[j]);
1895  }
1896  for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1897  ++j, ++k) {
1898  ShuffleVec[k] = NumElems + j;
1899  FinalIndices.push_back(IntermedVals[i+1].second[j]);
1900  }
1901 
1902  SDValue Shuffle;
1903  if (Phase)
1904  Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1905  IntermedVals[i+1].first,
1906  ShuffleVec.data());
1907  else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1908  return false;
1909  NewIntermedVals.push_back(
1910  std::make_pair(Shuffle, std::move(FinalIndices)));
1911  }
1912 
1913  // If we had an odd number of defined values, then append the last
1914  // element to the array of new vectors.
1915  if ((IntermedVals.size() & 1) != 0)
1916  NewIntermedVals.push_back(IntermedVals.back());
1917 
1918  IntermedVals.swap(NewIntermedVals);
1919  }
1920 
1921  assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1922  "Invalid number of intermediate vectors");
1923  SDValue Vec1 = IntermedVals[0].first;
1924  SDValue Vec2;
1925  if (IntermedVals.size() > 1)
1926  Vec2 = IntermedVals[1].first;
1927  else if (Phase)
1928  Vec2 = DAG.getUNDEF(VT);
1929 
1930  SmallVector<int, 16> ShuffleVec(NumElems, -1);
1931  for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1932  ShuffleVec[IntermedVals[0].second[i]] = i;
1933  for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1934  ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1935 
1936  if (Phase)
1937  Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec.data());
1938  else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1939  return false;
1940  }
1941 
1942  return true;
1943 }
1944 
1945 /// Expand a BUILD_VECTOR node on targets that don't
1946 /// support the operation, but do support the resultant vector type.
1947 SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1948  unsigned NumElems = Node->getNumOperands();
1949  SDValue Value1, Value2;
1950  SDLoc dl(Node);
1951  EVT VT = Node->getValueType(0);
1952  EVT OpVT = Node->getOperand(0).getValueType();
1953  EVT EltVT = VT.getVectorElementType();
1954 
1955  // If the only non-undef value is the low element, turn this into a
1956  // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1957  bool isOnlyLowElement = true;
1958  bool MoreThanTwoValues = false;
1959  bool isConstant = true;
1960  for (unsigned i = 0; i < NumElems; ++i) {
1961  SDValue V = Node->getOperand(i);
1962  if (V.getOpcode() == ISD::UNDEF)
1963  continue;
1964  if (i > 0)
1965  isOnlyLowElement = false;
1966  if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1967  isConstant = false;
1968 
1969  if (!Value1.getNode()) {
1970  Value1 = V;
1971  } else if (!Value2.getNode()) {
1972  if (V != Value1)
1973  Value2 = V;
1974  } else if (V != Value1 && V != Value2) {
1975  MoreThanTwoValues = true;
1976  }
1977  }
1978 
1979  if (!Value1.getNode())
1980  return DAG.getUNDEF(VT);
1981 
1982  if (isOnlyLowElement)
1983  return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1984 
1985  // If all elements are constants, create a load from the constant pool.
1986  if (isConstant) {
1988  for (unsigned i = 0, e = NumElems; i != e; ++i) {
1989  if (ConstantFPSDNode *V =
1990  dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1991  CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1992  } else if (ConstantSDNode *V =
1993  dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1994  if (OpVT==EltVT)
1995  CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1996  else {
1997  // If OpVT and EltVT don't match, EltVT is not legal and the
1998  // element values have been promoted/truncated earlier. Undo this;
1999  // we don't want a v16i8 to become a v16i32 for example.
2000  const ConstantInt *CI = V->getConstantIntValue();
2001  CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
2002  CI->getZExtValue()));
2003  }
2004  } else {
2005  assert(Node->getOperand(i).getOpcode() == ISD::UNDEF);
2006  Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
2007  CV.push_back(UndefValue::get(OpNTy));
2008  }
2009  }
2011  SDValue CPIdx =
2012  DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
2013  unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
2014  return DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
2016  false, false, false, Alignment);
2017  }
2018 
2019  SmallSet<SDValue, 16> DefinedValues;
2020  for (unsigned i = 0; i < NumElems; ++i) {
2021  if (Node->getOperand(i).getOpcode() == ISD::UNDEF)
2022  continue;
2023  DefinedValues.insert(Node->getOperand(i));
2024  }
2025 
2026  if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
2027  if (!MoreThanTwoValues) {
2028  SmallVector<int, 8> ShuffleVec(NumElems, -1);
2029  for (unsigned i = 0; i < NumElems; ++i) {
2030  SDValue V = Node->getOperand(i);
2031  if (V.getOpcode() == ISD::UNDEF)
2032  continue;
2033  ShuffleVec[i] = V == Value1 ? 0 : NumElems;
2034  }
2035  if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
2036  // Get the splatted value into the low element of a vector register.
2037  SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
2038  SDValue Vec2;
2039  if (Value2.getNode())
2040  Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
2041  else
2042  Vec2 = DAG.getUNDEF(VT);
2043 
2044  // Return shuffle(LowValVec, undef, <0,0,0,0>)
2045  return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec.data());
2046  }
2047  } else {
2048  SDValue Res;
2049  if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
2050  return Res;
2051  }
2052  }
2053 
2054  // Otherwise, we can't handle this case efficiently.
2055  return ExpandVectorBuildThroughStack(Node);
2056 }
2057 
2058 // Expand a node into a call to a libcall. If the result value
2059 // does not fit into a register, return the lo part and set the hi part to the
2060 // by-reg argument. If it does fit into a single register, return the result
2061 // and leave the Hi part unset.
2062 SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2063  bool isSigned) {
2066  for (const SDValue &Op : Node->op_values()) {
2067  EVT ArgVT = Op.getValueType();
2068  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2069  Entry.Node = Op;
2070  Entry.Ty = ArgTy;
2071  Entry.isSExt = isSigned;
2072  Entry.isZExt = !isSigned;
2073  Args.push_back(Entry);
2074  }
2075  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2076  TLI.getPointerTy(DAG.getDataLayout()));
2077 
2078  Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
2079 
2080  // By default, the input chain to this libcall is the entry node of the
2081  // function. If the libcall is going to be emitted as a tail call then
2082  // TLI.isUsedByReturnOnly will change it to the right chain if the return
2083  // node which is being folded has a non-entry input chain.
2084  SDValue InChain = DAG.getEntryNode();
2085 
2086  // isTailCall may be true since the callee does not reference caller stack
2087  // frame. Check if it's in the right position.
2088  SDValue TCChain = InChain;
2089  bool isTailCall = TLI.isInTailCallPosition(DAG, Node, TCChain);
2090  if (isTailCall)
2091  InChain = TCChain;
2092 
2094  CLI.setDebugLoc(SDLoc(Node)).setChain(InChain)
2095  .setCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, std::move(Args), 0)
2096  .setTailCall(isTailCall).setSExtResult(isSigned).setZExtResult(!isSigned);
2097 
2098  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2099 
2100  if (!CallInfo.second.getNode())
2101  // It's a tailcall, return the chain (which is the DAG root).
2102  return DAG.getRoot();
2103 
2104  return CallInfo.first;
2105 }
2106 
2107 /// Generate a libcall taking the given operands as arguments
2108 /// and returning a result of type RetVT.
2109 SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT,
2110  const SDValue *Ops, unsigned NumOps,
2111  bool isSigned, SDLoc dl) {
2113  Args.reserve(NumOps);
2114 
2116  for (unsigned i = 0; i != NumOps; ++i) {
2117  Entry.Node = Ops[i];
2118  Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
2119  Entry.isSExt = isSigned;
2120  Entry.isZExt = !isSigned;
2121  Args.push_back(Entry);
2122  }
2123  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2124  TLI.getPointerTy(DAG.getDataLayout()));
2125 
2126  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2127 
2129  CLI.setDebugLoc(dl).setChain(DAG.getEntryNode())
2130  .setCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, std::move(Args), 0)
2131  .setSExtResult(isSigned).setZExtResult(!isSigned);
2132 
2133  std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(CLI);
2134 
2135  return CallInfo.first;
2136 }
2137 
2138 // Expand a node into a call to a libcall. Similar to
2139 // ExpandLibCall except that the first operand is the in-chain.
2140 std::pair<SDValue, SDValue>
2141 SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC,
2142  SDNode *Node,
2143  bool isSigned) {
2144  SDValue InChain = Node->getOperand(0);
2145 
2148  for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) {
2149  EVT ArgVT = Node->getOperand(i).getValueType();
2150  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2151  Entry.Node = Node->getOperand(i);
2152  Entry.Ty = ArgTy;
2153  Entry.isSExt = isSigned;
2154  Entry.isZExt = !isSigned;
2155  Args.push_back(Entry);
2156  }
2157  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2158  TLI.getPointerTy(DAG.getDataLayout()));
2159 
2160  Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
2161 
2163  CLI.setDebugLoc(SDLoc(Node)).setChain(InChain)
2164  .setCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, std::move(Args), 0)
2165  .setSExtResult(isSigned).setZExtResult(!isSigned);
2166 
2167  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2168 
2169  return CallInfo;
2170 }
2171 
2172 SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2173  RTLIB::Libcall Call_F32,
2174  RTLIB::Libcall Call_F64,
2175  RTLIB::Libcall Call_F80,
2176  RTLIB::Libcall Call_F128,
2177  RTLIB::Libcall Call_PPCF128) {
2178  RTLIB::Libcall LC;
2179  switch (Node->getSimpleValueType(0).SimpleTy) {
2180  default: llvm_unreachable("Unexpected request for libcall!");
2181  case MVT::f32: LC = Call_F32; break;
2182  case MVT::f64: LC = Call_F64; break;
2183  case MVT::f80: LC = Call_F80; break;
2184  case MVT::f128: LC = Call_F128; break;
2185  case MVT::ppcf128: LC = Call_PPCF128; break;
2186  }
2187  return ExpandLibCall(LC, Node, false);
2188 }
2189 
2190 SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2191  RTLIB::Libcall Call_I8,
2192  RTLIB::Libcall Call_I16,
2193  RTLIB::Libcall Call_I32,
2194  RTLIB::Libcall Call_I64,
2195  RTLIB::Libcall Call_I128) {
2196  RTLIB::Libcall LC;
2197  switch (Node->getSimpleValueType(0).SimpleTy) {
2198  default: llvm_unreachable("Unexpected request for libcall!");
2199  case MVT::i8: LC = Call_I8; break;
2200  case MVT::i16: LC = Call_I16; break;
2201  case MVT::i32: LC = Call_I32; break;
2202  case MVT::i64: LC = Call_I64; break;
2203  case MVT::i128: LC = Call_I128; break;
2204  }
2205  return ExpandLibCall(LC, Node, isSigned);
2206 }
2207 
2208 /// Return true if divmod libcall is available.
2209 static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned,
2210  const TargetLowering &TLI) {
2211  RTLIB::Libcall LC;
2212  switch (Node->getSimpleValueType(0).SimpleTy) {
2213  default: llvm_unreachable("Unexpected request for libcall!");
2214  case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2215  case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2216  case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2217  case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2218  case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2219  }
2220 
2221  return TLI.getLibcallName(LC) != nullptr;
2222 }
2223 
2224 /// Only issue divrem libcall if both quotient and remainder are needed.
2225 static bool useDivRem(SDNode *Node, bool isSigned, bool isDIV) {
2226  // The other use might have been replaced with a divrem already.
2227  unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
2228  unsigned OtherOpcode = 0;
2229  if (isSigned)
2230  OtherOpcode = isDIV ? ISD::SREM : ISD::SDIV;
2231  else
2232  OtherOpcode = isDIV ? ISD::UREM : ISD::UDIV;
2233 
2234  SDValue Op0 = Node->getOperand(0);
2235  SDValue Op1 = Node->getOperand(1);
2236  for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
2237  UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
2238  SDNode *User = *UI;
2239  if (User == Node)
2240  continue;
2241  if ((User->getOpcode() == OtherOpcode || User->getOpcode() == DivRemOpc) &&
2242  User->getOperand(0) == Op0 &&
2243  User->getOperand(1) == Op1)
2244  return true;
2245  }
2246  return false;
2247 }
2248 
2249 /// Issue libcalls to __{u}divmod to compute div / rem pairs.
2250 void
2251 SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2252  SmallVectorImpl<SDValue> &Results) {
2253  unsigned Opcode = Node->getOpcode();
2254  bool isSigned = Opcode == ISD::SDIVREM;
2255 
2256  RTLIB::Libcall LC;
2257  switch (Node->getSimpleValueType(0).SimpleTy) {
2258  default: llvm_unreachable("Unexpected request for libcall!");
2259  case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2260  case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2261  case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2262  case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2263  case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2264  }
2265 
2266  // The input chain to this libcall is the entry node of the function.
2267  // Legalizing the call will automatically add the previous call to the
2268  // dependence.
2269  SDValue InChain = DAG.getEntryNode();
2270 
2271  EVT RetVT = Node->getValueType(0);
2272  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2273 
2276  for (const SDValue &Op : Node->op_values()) {
2277  EVT ArgVT = Op.getValueType();
2278  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2279  Entry.Node = Op;
2280  Entry.Ty = ArgTy;
2281  Entry.isSExt = isSigned;
2282  Entry.isZExt = !isSigned;
2283  Args.push_back(Entry);
2284  }
2285 
2286  // Also pass the return address of the remainder.
2287  SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2288  Entry.Node = FIPtr;
2289  Entry.Ty = RetTy->getPointerTo();
2290  Entry.isSExt = isSigned;
2291  Entry.isZExt = !isSigned;
2292  Args.push_back(Entry);
2293 
2294  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2295  TLI.getPointerTy(DAG.getDataLayout()));
2296 
2297  SDLoc dl(Node);
2299  CLI.setDebugLoc(dl).setChain(InChain)
2300  .setCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, std::move(Args), 0)
2301  .setSExtResult(isSigned).setZExtResult(!isSigned);
2302 
2303  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2304 
2305  // Remainder is loaded back from the stack frame.
2306  SDValue Rem = DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr,
2307  MachinePointerInfo(), false, false, false, 0);
2308  Results.push_back(CallInfo.first);
2309  Results.push_back(Rem);
2310 }
2311 
2312 /// Return true if sincos libcall is available.
2313 static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2314  RTLIB::Libcall LC;
2315  switch (Node->getSimpleValueType(0).SimpleTy) {
2316  default: llvm_unreachable("Unexpected request for libcall!");
2317  case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2318  case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2319  case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2320  case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2321  case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2322  }
2323  return TLI.getLibcallName(LC) != nullptr;
2324 }
2325 
2326 /// Return true if sincos libcall is available and can be used to combine sin
2327 /// and cos.
2328 static bool canCombineSinCosLibcall(SDNode *Node, const TargetLowering &TLI,
2329  const TargetMachine &TM) {
2330  if (!isSinCosLibcallAvailable(Node, TLI))
2331  return false;
2332  // GNU sin/cos functions set errno while sincos does not. Therefore
2333  // combining sin and cos is only safe if unsafe-fpmath is enabled.
2334  bool isGNU = Triple(TM.getTargetTriple()).getEnvironment() == Triple::GNU;
2335  if (isGNU && !TM.Options.UnsafeFPMath)
2336  return false;
2337  return true;
2338 }
2339 
2340 /// Only issue sincos libcall if both sin and cos are needed.
2341 static bool useSinCos(SDNode *Node) {
2342  unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2343  ? ISD::FCOS : ISD::FSIN;
2344 
2345  SDValue Op0 = Node->getOperand(0);
2346  for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
2347  UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
2348  SDNode *User = *UI;
2349  if (User == Node)
2350  continue;
2351  // The other user might have been turned into sincos already.
2352  if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2353  return true;
2354  }
2355  return false;
2356 }
2357 
2358 /// Issue libcalls to sincos to compute sin / cos pairs.
2359 void
2360 SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2361  SmallVectorImpl<SDValue> &Results) {
2362  RTLIB::Libcall LC;
2363  switch (Node->getSimpleValueType(0).SimpleTy) {
2364  default: llvm_unreachable("Unexpected request for libcall!");
2365  case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2366  case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2367  case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2368  case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2369  case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2370  }
2371 
2372  // The input chain to this libcall is the entry node of the function.
2373  // Legalizing the call will automatically add the previous call to the
2374  // dependence.
2375  SDValue InChain = DAG.getEntryNode();
2376 
2377  EVT RetVT = Node->getValueType(0);
2378  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2379 
2382 
2383  // Pass the argument.
2384  Entry.Node = Node->getOperand(0);
2385  Entry.Ty = RetTy;
2386  Entry.isSExt = false;
2387  Entry.isZExt = false;
2388  Args.push_back(Entry);
2389 
2390  // Pass the return address of sin.
2391  SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2392  Entry.Node = SinPtr;
2393  Entry.Ty = RetTy->getPointerTo();
2394  Entry.isSExt = false;
2395  Entry.isZExt = false;
2396  Args.push_back(Entry);
2397 
2398  // Also pass the return address of the cos.
2399  SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2400  Entry.Node = CosPtr;
2401  Entry.Ty = RetTy->getPointerTo();
2402  Entry.isSExt = false;
2403  Entry.isZExt = false;
2404  Args.push_back(Entry);
2405 
2406  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2407  TLI.getPointerTy(DAG.getDataLayout()));
2408 
2409  SDLoc dl(Node);
2411  CLI.setDebugLoc(dl).setChain(InChain)
2412  .setCallee(TLI.getLibcallCallingConv(LC),
2413  Type::getVoidTy(*DAG.getContext()), Callee, std::move(Args), 0);
2414 
2415  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2416 
2417  Results.push_back(DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr,
2418  MachinePointerInfo(), false, false, false, 0));
2419  Results.push_back(DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr,
2420  MachinePointerInfo(), false, false, false, 0));
2421 }
2422 
2423 /// This function is responsible for legalizing a
2424 /// INT_TO_FP operation of the specified operand when the target requests that
2425 /// we expand it. At this point, we know that the result and operand types are
2426 /// legal for the target.
2427 SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned,
2428  SDValue Op0,
2429  EVT DestVT,
2430  SDLoc dl) {
2431  if (Op0.getValueType() == MVT::i32 && TLI.isTypeLegal(MVT::f64)) {
2432  // simple 32-bit [signed|unsigned] integer to float/double expansion
2433 
2434  // Get the stack frame index of a 8 byte buffer.
2435  SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2436 
2437  // word offset constant for Hi/Lo address computation
2438  SDValue WordOff = DAG.getConstant(sizeof(int), dl,
2439  StackSlot.getValueType());
2440  // set up Hi and Lo (into buffer) address based on endian
2441  SDValue Hi = StackSlot;
2442  SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(),
2443  StackSlot, WordOff);
2444  if (DAG.getDataLayout().isLittleEndian())
2445  std::swap(Hi, Lo);
2446 
2447  // if signed map to unsigned space
2448  SDValue Op0Mapped;
2449  if (isSigned) {
2450  // constant used to invert sign bit (signed to unsigned mapping)
2451  SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32);
2452  Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit);
2453  } else {
2454  Op0Mapped = Op0;
2455  }
2456  // store the lo of the constructed double - based on integer input
2457  SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl,
2458  Op0Mapped, Lo, MachinePointerInfo(),
2459  false, false, 0);
2460  // initial hi portion of constructed double
2461  SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2462  // store the hi of the constructed double - biased exponent
2463  SDValue Store2 = DAG.getStore(Store1, dl, InitialHi, Hi,
2465  false, false, 0);
2466  // load the constructed double
2467  SDValue Load = DAG.getLoad(MVT::f64, dl, Store2, StackSlot,
2468  MachinePointerInfo(), false, false, false, 0);
2469  // FP constant to bias correct the final result
2470  SDValue Bias = DAG.getConstantFP(isSigned ?
2471  BitsToDouble(0x4330000080000000ULL) :
2472  BitsToDouble(0x4330000000000000ULL),
2473  dl, MVT::f64);
2474  // subtract the bias
2475  SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2476  // final result
2477  SDValue Result;
2478  // handle final rounding
2479  if (DestVT == MVT::f64) {
2480  // do nothing
2481  Result = Sub;
2482  } else if (DestVT.bitsLT(MVT::f64)) {
2483  Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub,
2484  DAG.getIntPtrConstant(0, dl));
2485  } else if (DestVT.bitsGT(MVT::f64)) {
2486  Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub);
2487  }
2488  return Result;
2489  }
2490  assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet");
2491  // Code below here assumes !isSigned without checking again.
2492 
2493  // Implementation of unsigned i64 to f64 following the algorithm in
2494  // __floatundidf in compiler_rt. This implementation has the advantage
2495  // of performing rounding correctly, both in the default rounding mode
2496  // and in all alternate rounding modes.
2497  // TODO: Generalize this for use with other types.
2498  if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f64) {
2499  SDValue TwoP52 =
2500  DAG.getConstant(UINT64_C(0x4330000000000000), dl, MVT::i64);
2501  SDValue TwoP84PlusTwoP52 =
2502  DAG.getConstantFP(BitsToDouble(UINT64_C(0x4530000000100000)), dl,
2503  MVT::f64);
2504  SDValue TwoP84 =
2505  DAG.getConstant(UINT64_C(0x4530000000000000), dl, MVT::i64);
2506 
2507  SDValue Lo = DAG.getZeroExtendInReg(Op0, dl, MVT::i32);
2508  SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0,
2509  DAG.getConstant(32, dl, MVT::i64));
2510  SDValue LoOr = DAG.getNode(ISD::OR, dl, MVT::i64, Lo, TwoP52);
2511  SDValue HiOr = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, TwoP84);
2512  SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, LoOr);
2513  SDValue HiFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, HiOr);
2514  SDValue HiSub = DAG.getNode(ISD::FSUB, dl, MVT::f64, HiFlt,
2515  TwoP84PlusTwoP52);
2516  return DAG.getNode(ISD::FADD, dl, MVT::f64, LoFlt, HiSub);
2517  }
2518 
2519  // Implementation of unsigned i64 to f32.
2520  // TODO: Generalize this for use with other types.
2521  if (Op0.getValueType() == MVT::i64 && DestVT == MVT::f32) {
2522  // For unsigned conversions, convert them to signed conversions using the
2523  // algorithm from the x86_64 __floatundidf in compiler_rt.
2524  if (!isSigned) {
2525  SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0);
2526 
2527  SDValue ShiftConst = DAG.getConstant(
2528  1, dl, TLI.getShiftAmountTy(Op0.getValueType(), DAG.getDataLayout()));
2529  SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst);
2530  SDValue AndConst = DAG.getConstant(1, dl, MVT::i64);
2531  SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, AndConst);
2532  SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr);
2533 
2534  SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Or);
2535  SDValue Slow = DAG.getNode(ISD::FADD, dl, MVT::f32, SignCvt, SignCvt);
2536 
2537  // TODO: This really should be implemented using a branch rather than a
2538  // select. We happen to get lucky and machinesink does the right
2539  // thing most of the time. This would be a good candidate for a
2540  //pseudo-op, or, even better, for whole-function isel.
2541  SDValue SignBitTest = DAG.getSetCC(dl, getSetCCResultType(MVT::i64),
2542  Op0, DAG.getConstant(0, dl, MVT::i64), ISD::SETLT);
2543  return DAG.getSelect(dl, MVT::f32, SignBitTest, Slow, Fast);
2544  }
2545 
2546  // Otherwise, implement the fully general conversion.
2547 
2548  SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2549  DAG.getConstant(UINT64_C(0xfffffffffffff800), dl, MVT::i64));
2550  SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And,
2551  DAG.getConstant(UINT64_C(0x800), dl, MVT::i64));
2552  SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2553  DAG.getConstant(UINT64_C(0x7ff), dl, MVT::i64));
2554  SDValue Ne = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), And2,
2555  DAG.getConstant(UINT64_C(0), dl, MVT::i64),
2556  ISD::SETNE);
2557  SDValue Sel = DAG.getSelect(dl, MVT::i64, Ne, Or, Op0);
2558  SDValue Ge = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), Op0,
2559  DAG.getConstant(UINT64_C(0x0020000000000000), dl,
2560  MVT::i64),
2561  ISD::SETUGE);
2562  SDValue Sel2 = DAG.getSelect(dl, MVT::i64, Ge, Sel, Op0);
2563  EVT SHVT = TLI.getShiftAmountTy(Sel2.getValueType(), DAG.getDataLayout());
2564 
2565  SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2,
2566  DAG.getConstant(32, dl, SHVT));
2567  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh);
2568  SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc);
2569  SDValue TwoP32 =
2570  DAG.getConstantFP(BitsToDouble(UINT64_C(0x41f0000000000000)), dl,
2571  MVT::f64);
2572  SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt);
2573  SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2);
2574  SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo);
2575  SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2);
2576  return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd,
2577  DAG.getIntPtrConstant(0, dl));
2578  }
2579 
2580  SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2581 
2582  SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(Op0.getValueType()),
2583  Op0,
2584  DAG.getConstant(0, dl, Op0.getValueType()),
2585  ISD::SETLT);
2586  SDValue Zero = DAG.getIntPtrConstant(0, dl),
2587  Four = DAG.getIntPtrConstant(4, dl);
2588  SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2589  SignSet, Four, Zero);
2590 
2591  // If the sign bit of the integer is set, the large number will be treated
2592  // as a negative number. To counteract this, the dynamic code adds an
2593  // offset depending on the data type.
2594  uint64_t FF;
2595  switch (Op0.getSimpleValueType().SimpleTy) {
2596  default: llvm_unreachable("Unsupported integer type!");
2597  case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2598  case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2599  case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2600  case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2601  }
2602  if (DAG.getDataLayout().isLittleEndian())
2603  FF <<= 32;
2604  Constant *FudgeFactor = ConstantInt::get(
2605  Type::getInt64Ty(*DAG.getContext()), FF);
2606 
2607  SDValue CPIdx =
2608  DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2609  unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
2610  CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2611  Alignment = std::min(Alignment, 4u);
2612  SDValue FudgeInReg;
2613  if (DestVT == MVT::f32)
2614  FudgeInReg = DAG.getLoad(MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2616  false, false, false, Alignment);
2617  else {
2618  SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT,
2619  DAG.getEntryNode(), CPIdx,
2621  MVT::f32, false, false, false, Alignment);
2622  HandleSDNode Handle(Load);
2623  LegalizeOp(Load.getNode());
2624  FudgeInReg = Handle.getValue();
2625  }
2626 
2627  return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2628 }
2629 
2630 /// This function is responsible for legalizing a
2631 /// *INT_TO_FP operation of the specified operand when the target requests that
2632 /// we promote it. At this point, we know that the result and operand types are
2633 /// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2634 /// operation that takes a larger input.
2635 SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp,
2636  EVT DestVT,
2637  bool isSigned,
2638  SDLoc dl) {
2639  // First step, figure out the appropriate *INT_TO_FP operation to use.
2640  EVT NewInTy = LegalOp.getValueType();
2641 
2642  unsigned OpToUse = 0;
2643 
2644  // Scan for the appropriate larger type to use.
2645  while (1) {
2646  NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2647  assert(NewInTy.isInteger() && "Ran out of possibilities!");
2648 
2649  // If the target supports SINT_TO_FP of this type, use it.
2650  if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) {
2651  OpToUse = ISD::SINT_TO_FP;
2652  break;
2653  }
2654  if (isSigned) continue;
2655 
2656  // If the target supports UINT_TO_FP of this type, use it.
2657  if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) {
2658  OpToUse = ISD::UINT_TO_FP;
2659  break;
2660  }
2661 
2662  // Otherwise, try a larger type.
2663  }
2664 
2665  // Okay, we found the operation and type to use. Zero extend our input to the
2666  // desired type then run the operation on it.
2667  return DAG.getNode(OpToUse, dl, DestVT,
2668  DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2669  dl, NewInTy, LegalOp));
2670 }
2671 
2672 /// This function is responsible for legalizing a
2673 /// FP_TO_*INT operation of the specified operand when the target requests that
2674 /// we promote it. At this point, we know that the result and operand types are
2675 /// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2676 /// operation that returns a larger result.
2677 SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp,
2678  EVT DestVT,
2679  bool isSigned,
2680  SDLoc dl) {
2681  // First step, figure out the appropriate FP_TO*INT operation to use.
2682  EVT NewOutTy = DestVT;
2683 
2684  unsigned OpToUse = 0;
2685 
2686  // Scan for the appropriate larger type to use.
2687  while (1) {
2688  NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2689  assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2690 
2691  // A larger signed type can hold all unsigned values of the requested type,
2692  // so using FP_TO_SINT is valid
2693  if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) {
2694  OpToUse = ISD::FP_TO_SINT;
2695  break;
2696  }
2697 
2698  // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2699  if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) {
2700  OpToUse = ISD::FP_TO_UINT;
2701  break;
2702  }
2703 
2704  // Otherwise, try a larger type.
2705  }
2706 
2707 
2708  // Okay, we found the operation and type to use.
2709  SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2710 
2711  // Truncate the result of the extended FP_TO_*INT operation to the desired
2712  // size.
2713  return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2714 }
2715 
2716 /// Open code the operations for BSWAP of the specified operation.
2717 SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, SDLoc dl) {
2718  EVT VT = Op.getValueType();
2719  EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2720  SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
2721  switch (VT.getSimpleVT().SimpleTy) {
2722  default: llvm_unreachable("Unhandled Expand type in BSWAP!");
2723  case MVT::i16:
2724  Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2725  Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2726  return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
2727  case MVT::i32:
2728  Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2729  Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2730  Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2731  Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2732  Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2733  DAG.getConstant(0xFF0000, dl, VT));
2734  Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
2735  Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2736  Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2737  return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2738  case MVT::i64:
2739  Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2740  Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2741  Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2742  Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2743  Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2744  Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2745  Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2746  Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2747  Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,
2748  DAG.getConstant(255ULL<<48, dl, VT));
2749  Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,
2750  DAG.getConstant(255ULL<<40, dl, VT));
2751  Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,
2752  DAG.getConstant(255ULL<<32, dl, VT));
2753  Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
2754  DAG.getConstant(255ULL<<24, dl, VT));
2755  Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2756  DAG.getConstant(255ULL<<16, dl, VT));
2757  Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
2758  DAG.getConstant(255ULL<<8 , dl, VT));
2759  Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
2760  Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
2761  Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2762  Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2763  Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
2764  Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2765  return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
2766  }
2767 }
2768 
2769 /// Expand the specified bitcount instruction into operations.
2770 SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op,
2771  SDLoc dl) {
2772  switch (Opc) {
2773  default: llvm_unreachable("Cannot expand this yet!");
2774  case ISD::CTPOP: {
2775  EVT VT = Op.getValueType();
2776  EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2777  unsigned Len = VT.getSizeInBits();
2778 
2779  assert(VT.isInteger() && Len <= 128 && Len % 8 == 0 &&
2780  "CTPOP not implemented for this type.");
2781 
2782  // This is the "best" algorithm from
2783  // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
2784 
2785  SDValue Mask55 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)),
2786  dl, VT);
2787  SDValue Mask33 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)),
2788  dl, VT);
2789  SDValue Mask0F = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)),
2790  dl, VT);
2791  SDValue Mask01 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)),
2792  dl, VT);
2793 
2794  // v = v - ((v >> 1) & 0x55555555...)
2795  Op = DAG.getNode(ISD::SUB, dl, VT, Op,
2796  DAG.getNode(ISD::AND, dl, VT,
2797  DAG.getNode(ISD::SRL, dl, VT, Op,
2798  DAG.getConstant(1, dl, ShVT)),
2799  Mask55));
2800  // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
2801  Op = DAG.getNode(ISD::ADD, dl, VT,
2802  DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
2803  DAG.getNode(ISD::AND, dl, VT,
2804  DAG.getNode(ISD::SRL, dl, VT, Op,
2805  DAG.getConstant(2, dl, ShVT)),
2806  Mask33));
2807  // v = (v + (v >> 4)) & 0x0F0F0F0F...
2808  Op = DAG.getNode(ISD::AND, dl, VT,
2809  DAG.getNode(ISD::ADD, dl, VT, Op,
2810  DAG.getNode(ISD::SRL, dl, VT, Op,
2811  DAG.getConstant(4, dl, ShVT))),
2812  Mask0F);
2813  // v = (v * 0x01010101...) >> (Len - 8)
2814  Op = DAG.getNode(ISD::SRL, dl, VT,
2815  DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
2816  DAG.getConstant(Len - 8, dl, ShVT));
2817 
2818  return Op;
2819  }
2820  case ISD::CTLZ_ZERO_UNDEF:
2821  // This trivially expands to CTLZ.
2822  return DAG.getNode(ISD::CTLZ, dl, Op.getValueType(), Op);
2823  case ISD::CTLZ: {
2824  // for now, we do this:
2825  // x = x | (x >> 1);
2826  // x = x | (x >> 2);
2827  // ...
2828  // x = x | (x >>16);
2829  // x = x | (x >>32); // for 64-bit input
2830  // return popcount(~x);
2831  //
2832  // Ref: "Hacker's Delight" by Henry Warren
2833  EVT VT = Op.getValueType();
2834  EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2835  unsigned len = VT.getSizeInBits();
2836  for (unsigned i = 0; (1U << i) <= (len / 2); ++i) {
2837  SDValue Tmp3 = DAG.getConstant(1ULL << i, dl, ShVT);
2838  Op = DAG.getNode(ISD::OR, dl, VT, Op,
2839  DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3));
2840  }
2841  Op = DAG.getNOT(dl, Op, VT);
2842  return DAG.getNode(ISD::CTPOP, dl, VT, Op);
2843  }
2844  case ISD::CTTZ_ZERO_UNDEF:
2845  // This trivially expands to CTTZ.
2846  return DAG.getNode(ISD::CTTZ, dl, Op.getValueType(), Op);
2847  case ISD::CTTZ: {
2848  // for now, we use: { return popcount(~x & (x - 1)); }
2849  // unless the target has ctlz but not ctpop, in which case we use:
2850  // { return 32 - nlz(~x & (x-1)); }
2851  // Ref: "Hacker's Delight" by Henry Warren
2852  EVT VT = Op.getValueType();
2853  SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT,
2854  DAG.getNOT(dl, Op, VT),
2855  DAG.getNode(ISD::SUB, dl, VT, Op,
2856  DAG.getConstant(1, dl, VT)));
2857  // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
2858  if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) &&
2859  TLI.isOperationLegalOrCustom(ISD::CTLZ, VT))
2860  return DAG.getNode(ISD::SUB, dl, VT,
2861  DAG.getConstant(VT.getSizeInBits(), dl, VT),
2862  DAG.getNode(ISD::CTLZ, dl, VT, Tmp3));
2863  return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3);
2864  }
2865  }
2866 }
2867 
2868 std::pair <SDValue, SDValue> SelectionDAGLegalize::ExpandAtomic(SDNode *Node) {
2869  unsigned Opc = Node->getOpcode();
2870  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
2871  RTLIB::Libcall LC = RTLIB::getATOMIC(Opc, VT);
2872  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
2873 
2874  return ExpandChainLibCall(LC, Node, false);
2875 }
2876 
2877 void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2878  SmallVector<SDValue, 8> Results;
2879  SDLoc dl(Node);
2880  SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2881  bool NeedInvert;
2882  switch (Node->getOpcode()) {
2883  case ISD::CTPOP:
2884  case ISD::CTLZ:
2885  case ISD::CTLZ_ZERO_UNDEF:
2886  case ISD::CTTZ:
2887  case ISD::CTTZ_ZERO_UNDEF:
2888  Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl);
2889  Results.push_back(Tmp1);
2890  break;
2891  case ISD::BSWAP:
2892  Results.push_back(ExpandBSWAP(Node->getOperand(0), dl));
2893  break;
2894  case ISD::FRAMEADDR:
2895  case ISD::RETURNADDR:
2897  Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
2898  break;
2899  case ISD::FLT_ROUNDS_:
2900  Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
2901  break;
2902  case ISD::EH_RETURN:
2903  case ISD::EH_LABEL:
2904  case ISD::PREFETCH:
2905  case ISD::VAEND:
2906  case ISD::EH_SJLJ_LONGJMP:
2907  // If the target didn't expand these, there's nothing to do, so just
2908  // preserve the chain and be done.
2909  Results.push_back(Node->getOperand(0));
2910  break;
2911  case ISD::EH_SJLJ_SETJMP:
2912  // If the target didn't expand this, just return 'zero' and preserve the
2913  // chain.
2914  Results.push_back(DAG.getConstant(0, dl, MVT::i32));
2915  Results.push_back(Node->getOperand(0));
2916  break;
2917  case ISD::ATOMIC_FENCE: {
2918  // If the target didn't lower this, lower it to '__sync_synchronize()' call
2919  // FIXME: handle "fence singlethread" more efficiently.
2921 
2923  CLI.setDebugLoc(dl)
2924  .setChain(Node->getOperand(0))
2925  .setCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
2926  DAG.getExternalSymbol("__sync_synchronize",
2927  TLI.getPointerTy(DAG.getDataLayout())),
2928  std::move(Args), 0);
2929 
2930  std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
2931 
2932  Results.push_back(CallResult.second);
2933  break;
2934  }
2935  case ISD::ATOMIC_LOAD: {
2936  // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
2937  SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
2938  SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2939  SDValue Swap = DAG.getAtomicCmpSwap(
2940  ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2941  Node->getOperand(0), Node->getOperand(1), Zero, Zero,
2942  cast<AtomicSDNode>(Node)->getMemOperand(),
2943  cast<AtomicSDNode>(Node)->getOrdering(),
2944  cast<AtomicSDNode>(Node)->getOrdering(),
2945  cast<AtomicSDNode>(Node)->getSynchScope());
2946  Results.push_back(Swap.getValue(0));
2947  Results.push_back(Swap.getValue(1));
2948  break;
2949  }
2950  case ISD::ATOMIC_STORE: {
2951  // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
2952  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2953  cast<AtomicSDNode>(Node)->getMemoryVT(),
2954  Node->getOperand(0),
2955  Node->getOperand(1), Node->getOperand(2),
2956  cast<AtomicSDNode>(Node)->getMemOperand(),
2957  cast<AtomicSDNode>(Node)->getOrdering(),
2958  cast<AtomicSDNode>(Node)->getSynchScope());
2959  Results.push_back(Swap.getValue(1));
2960  break;
2961  }
2962  // By default, atomic intrinsics are marked Legal and lowered. Targets
2963  // which don't support them directly, however, may want libcalls, in which
2964  // case they mark them Expand, and we get here.
2965  case ISD::ATOMIC_SWAP:
2966  case ISD::ATOMIC_LOAD_ADD:
2967  case ISD::ATOMIC_LOAD_SUB:
2968  case ISD::ATOMIC_LOAD_AND:
2969  case ISD::ATOMIC_LOAD_OR:
2970  case ISD::ATOMIC_LOAD_XOR:
2971  case ISD::ATOMIC_LOAD_NAND:
2972  case ISD::ATOMIC_LOAD_MIN:
2973  case ISD::ATOMIC_LOAD_MAX:
2974  case ISD::ATOMIC_LOAD_UMIN:
2975  case ISD::ATOMIC_LOAD_UMAX:
2976  case ISD::ATOMIC_CMP_SWAP: {
2977  std::pair<SDValue, SDValue> Tmp = ExpandAtomic(Node);
2978  Results.push_back(Tmp.first);
2979  Results.push_back(Tmp.second);
2980  break;
2981  }
2983  // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
2984  // splits out the success value as a comparison. Expanding the resulting
2985  // ATOMIC_CMP_SWAP will produce a libcall.
2986  SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2987  SDValue Res = DAG.getAtomicCmpSwap(
2988  ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2989  Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
2990  Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand(),
2991  cast<AtomicSDNode>(Node)->getSuccessOrdering(),
2992  cast<AtomicSDNode>(Node)->getFailureOrdering(),
2993  cast<AtomicSDNode>(Node)->getSynchScope());
2994 
2995  SDValue Success = DAG.getSetCC(SDLoc(Node), Node->getValueType(1),
2996  Res, Node->getOperand(2), ISD::SETEQ);
2997 
2998  Results.push_back(Res.getValue(0));
2999  Results.push_back(Success);
3000  Results.push_back(Res.getValue(1));
3001  break;
3002  }
3004  ExpandDYNAMIC_STACKALLOC(Node, Results);
3005  break;
3006  case ISD::MERGE_VALUES:
3007  for (unsigned i = 0; i < Node->getNumValues(); i++)
3008  Results.push_back(Node->getOperand(i));
3009  break;
3010  case ISD::UNDEF: {
3011  EVT VT = Node->getValueType(0);
3012  if (VT.isInteger())
3013  Results.push_back(DAG.getConstant(0, dl, VT));
3014  else {
3015  assert(VT.isFloatingPoint() && "Unknown value type!");
3016  Results.push_back(DAG.getConstantFP(0, dl, VT));
3017  }
3018  break;
3019  }
3020  case ISD::TRAP: {
3021  // If this operation is not supported, lower it to 'abort()' call
3024  CLI.setDebugLoc(dl)
3025  .setChain(Node->getOperand(0))
3026  .setCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3027  DAG.getExternalSymbol("abort",
3028  TLI.getPointerTy(DAG.getDataLayout())),
3029  std::move(Args), 0);
3030  std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3031 
3032  Results.push_back(CallResult.second);
3033  break;
3034  }
3035  case ISD::FP_ROUND:
3036  case ISD::BITCAST:
3037  Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
3038  Node->getValueType(0), dl);
3039  Results.push_back(Tmp1);
3040  break;
3041  case ISD::FP_EXTEND:
3042  Tmp1 = EmitStackConvert(Node->getOperand(0),
3043  Node->getOperand(0).getValueType(),
3044  Node->getValueType(0), dl);
3045  Results.push_back(Tmp1);
3046  break;
3047  case ISD::SIGN_EXTEND_INREG: {
3048  // NOTE: we could fall back on load/store here too for targets without
3049  // SAR. However, it is doubtful that any exist.
3050  EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
3051  EVT VT = Node->getValueType(0);
3052  EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3053  if (VT.isVector())
3054  ShiftAmountTy = VT;
3055  unsigned BitsDiff = VT.getScalarType().getSizeInBits() -
3056  ExtraVT.getScalarType().getSizeInBits();
3057  SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
3058  Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
3059  Node->getOperand(0), ShiftCst);
3060  Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
3061  Results.push_back(Tmp1);
3062  break;
3063  }
3064  case ISD::FP_ROUND_INREG: {
3065  // The only way we can lower this is to turn it into a TRUNCSTORE,
3066  // EXTLOAD pair, targeting a temporary location (a stack slot).
3067 
3068  // NOTE: there is a choice here between constantly creating new stack
3069  // slots and always reusing the same one. We currently always create
3070  // new ones, as reuse may inhibit scheduling.
3071  EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
3072  Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT,
3073  Node->getValueType(0), dl);
3074  Results.push_back(Tmp1);
3075  break;
3076  }
3077  case ISD::SINT_TO_FP:
3078  case ISD::UINT_TO_FP:
3079  Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP,
3080  Node->getOperand(0), Node->getValueType(0), dl);
3081  Results.push_back(Tmp1);
3082  break;
3083  case ISD::FP_TO_SINT:
3084  if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
3085  Results.push_back(Tmp1);
3086  break;
3087  case ISD::FP_TO_UINT: {
3088  SDValue True, False;
3089  EVT VT = Node->getOperand(0).getValueType();
3090  EVT NVT = Node->getValueType(0);
3091  APFloat apf(DAG.EVTToAPFloatSemantics(VT),
3094  (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven);
3095  Tmp1 = DAG.getConstantFP(apf, dl, VT);
3096  Tmp2 = DAG.getSetCC(dl, getSetCCResultType(VT),
3097  Node->getOperand(0),
3098  Tmp1, ISD::SETLT);
3099  True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0));
3100  False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT,
3101  DAG.getNode(ISD::FSUB, dl, VT,
3102  Node->getOperand(0), Tmp1));
3103  False = DAG.getNode(ISD::XOR, dl, NVT, False,
3104  DAG.getConstant(x, dl, NVT));
3105  Tmp1 = DAG.getSelect(dl, NVT, Tmp2, True, False);
3106  Results.push_back(Tmp1);
3107  break;
3108  }
3109  case ISD::VAARG: {
3110  const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
3111  EVT VT = Node->getValueType(0);
3112  Tmp1 = Node->getOperand(0);
3113  Tmp2 = Node->getOperand(1);
3114  unsigned Align = Node->getConstantOperandVal(3);
3115 
3116  SDValue VAListLoad =
3117  DAG.getLoad(TLI.getPointerTy(DAG.getDataLayout()), dl, Tmp1, Tmp2,
3118  MachinePointerInfo(V), false, false, false, 0);
3119  SDValue VAList = VAListLoad;
3120 
3121  if (Align > TLI.getMinStackArgumentAlignment()) {
3122  assert(((Align & (Align-1)) == 0) && "Expected Align to be a power of 2");
3123 
3124  VAList = DAG.getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
3125  DAG.getConstant(Align - 1, dl,
3126  VAList.getValueType()));
3127 
3128  VAList = DAG.getNode(ISD::AND, dl, VAList.getValueType(), VAList,
3129  DAG.getConstant(-(int64_t)Align, dl,
3130  VAList.getValueType()));
3131  }
3132 
3133  // Increment the pointer, VAList, to the next vaarg
3134  Tmp3 = DAG.getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
3135  DAG.getConstant(DAG.getDataLayout().getTypeAllocSize(
3136  VT.getTypeForEVT(*DAG.getContext())),
3137  dl, VAList.getValueType()));
3138  // Store the incremented VAList to the legalized pointer
3139  Tmp3 = DAG.getStore(VAListLoad.getValue(1), dl, Tmp3, Tmp2,
3140  MachinePointerInfo(V), false, false, 0);
3141  // Load the actual argument out of the pointer VAList
3142  Results.push_back(DAG.getLoad(VT, dl, Tmp3, VAList, MachinePointerInfo(),
3143  false, false, false, 0));
3144  Results.push_back(Results[0].getValue(1));
3145  break;
3146  }
3147  case ISD::VACOPY: {
3148  // This defaults to loading a pointer from the input and storing it to the
3149  // output, returning the chain.
3150  const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
3151  const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
3152  Tmp1 = DAG.getLoad(TLI.getPointerTy(DAG.getDataLayout()), dl,
3153  Node->getOperand(0), Node->getOperand(2),
3154  MachinePointerInfo(VS), false, false, false, 0);
3155  Tmp1 = DAG.getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1),
3156  MachinePointerInfo(VD), false, false, 0);
3157  Results.push_back(Tmp1);
3158  break;
3159  }
3161  if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
3162  // This must be an access of the only element. Return it.
3163  Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
3164  Node->getOperand(0));
3165  else
3166  Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
3167  Results.push_back(Tmp1);
3168  break;
3170  Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
3171  break;
3172  case ISD::INSERT_SUBVECTOR:
3173  Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
3174  break;
3175  case ISD::CONCAT_VECTORS: {
3176  Results.push_back(ExpandVectorBuildThroughStack(Node));
3177  break;
3178  }
3179  case ISD::SCALAR_TO_VECTOR:
3180  Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
3181  break;
3183  Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
3184  Node->getOperand(1),
3185  Node->getOperand(2), dl));
3186  break;
3187  case ISD::VECTOR_SHUFFLE: {
3188  SmallVector<int, 32> NewMask;
3189  ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
3190 
3191  EVT VT = Node->getValueType(0);
3192  EVT EltVT = VT.getVectorElementType();
3193  SDValue Op0 = Node->getOperand(0);
3194  SDValue Op1 = Node->getOperand(1);
3195  if (!TLI.isTypeLegal(EltVT)) {
3196 
3197  EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3198 
3199  // BUILD_VECTOR operands are allowed to be wider than the element type.
3200  // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3201  // it.
3202  if (NewEltVT.bitsLT(EltVT)) {
3203 
3204  // Convert shuffle node.
3205  // If original node was v4i64 and the new EltVT is i32,
3206  // cast operands to v8i32 and re-build the mask.
3207 
3208  // Calculate new VT, the size of the new VT should be equal to original.
3209  EVT NewVT =
3210  EVT::getVectorVT(*DAG.getContext(), NewEltVT,
3211  VT.getSizeInBits() / NewEltVT.getSizeInBits());
3212  assert(NewVT.bitsEq(VT));
3213 
3214  // cast operands to new VT
3215  Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
3216  Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
3217 
3218  // Convert the shuffle mask
3219  unsigned int factor =
3221 
3222  // EltVT gets smaller
3223  assert(factor > 0);
3224 
3225  for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3226  if (Mask[i] < 0) {
3227  for (unsigned fi = 0; fi < factor; ++fi)
3228  NewMask.push_back(Mask[i]);
3229  }
3230  else {
3231  for (unsigned fi = 0; fi < factor; ++fi)
3232  NewMask.push_back(Mask[i]*factor+fi);
3233  }
3234  }
3235  Mask = NewMask;
3236  VT = NewVT;
3237  }
3238  EltVT = NewEltVT;
3239  }
3240  unsigned NumElems = VT.getVectorNumElements();
3242  for (unsigned i = 0; i != NumElems; ++i) {
3243  if (Mask[i] < 0) {
3244  Ops.push_back(DAG.getUNDEF(EltVT));
3245  continue;
3246  }
3247  unsigned Idx = Mask[i];
3248  if (Idx < NumElems)
3249  Ops.push_back(DAG.getNode(
3250  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
3251  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
3252  else
3253  Ops.push_back(DAG.getNode(
3254  ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
3255  DAG.getConstant(Idx - NumElems, dl,
3256  TLI.getVectorIdxTy(DAG.getDataLayout()))));
3257  }
3258 
3259  Tmp1 = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops);
3260  // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3261  Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3262  Results.push_back(Tmp1);
3263  break;
3264  }
3265  case ISD::EXTRACT_ELEMENT: {
3266  EVT OpTy = Node->getOperand(0).getValueType();
3267  if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
3268  // 1 -> Hi
3269  Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3270  DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
3271  TLI.getShiftAmountTy(
3272  Node->getOperand(0).getValueType(),
3273  DAG.getDataLayout())));
3274  Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3275  } else {
3276  // 0 -> Lo
3277  Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3278  Node->getOperand(0));
3279  }
3280  Results.push_back(Tmp1);
3281  break;
3282  }
3283  case ISD::STACKSAVE:
3284  // Expand to CopyFromReg if the target set
3285  // StackPointerRegisterToSaveRestore.
3286  if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3287  Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3288  Node->getValueType(0)));
3289  Results.push_back(Results[0].getValue(1));
3290  } else {
3291  Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3292  Results.push_back(Node->getOperand(0));
3293  }
3294  break;
3295  case ISD::STACKRESTORE:
3296  // Expand to CopyToReg if the target set
3297  // StackPointerRegisterToSaveRestore.
3298  if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3299  Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3300  Node->getOperand(1)));
3301  } else {
3302  Results.push_back(Node->getOperand(0));
3303  }
3304  break;
3305  case ISD::FCOPYSIGN:
3306  Results.push_back(ExpandFCOPYSIGN(Node));
3307  break;
3308  case ISD::FNEG:
3309  // Expand Y = FNEG(X) -> Y = SUB -0.0, X
3310  Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0));
3311  Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1,
3312  Node->getOperand(0));
3313  Results.push_back(Tmp1);
3314  break;
3315  case ISD::FABS: {
3316  // Expand Y = FABS(X) -> Y = (X >u 0.0) ? X : fneg(X).
3317  EVT VT = Node->getValueType(0);
3318  Tmp1 = Node->getOperand(0);
3319  Tmp2 = DAG.getConstantFP(0.0, dl, VT);
3320  Tmp2 = DAG.getSetCC(dl, getSetCCResultType(Tmp1.getValueType()),
3321  Tmp1, Tmp2, ISD::SETUGT);
3322  Tmp3 = DAG.getNode(ISD::FNEG, dl, VT, Tmp1);
3323  Tmp1 = DAG.getSelect(dl, VT, Tmp2, Tmp1, Tmp3);
3324  Results.push_back(Tmp1);
3325  break;
3326  }
3327  case ISD::SMIN:
3328  case ISD::SMAX:
3329  case ISD::UMIN:
3330  case ISD::UMAX: {
3331  // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3332  ISD::CondCode Pred;
3333  switch (Node->getOpcode()) {
3334  default: llvm_unreachable("How did we get here?");
3335  case ISD::SMAX: Pred = ISD::SETGT; break;
3336  case ISD::SMIN: Pred = ISD::SETLT; break;
3337  case ISD::UMAX: Pred = ISD::SETUGT; break;
3338  case ISD::UMIN: Pred = ISD::SETULT; break;
3339  }
3340  Tmp1 = Node->getOperand(0);
3341  Tmp2 = Node->getOperand(1);
3342  Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3343  Results.push_back(Tmp1);
3344  break;
3345  }
3346 
3347  case ISD::FMINNUM:
3348  Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
3351  break;
3352  case ISD::FMAXNUM:
3353  Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
3356  break;
3357  case ISD::FSQRT:
3358  Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
3361  break;
3362  case ISD::FSIN:
3363  case ISD::FCOS: {
3364  EVT VT = Node->getValueType(0);
3365  bool isSIN = Node->getOpcode() == ISD::FSIN;
3366  // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3367  // fcos which share the same operand and both are used.
3368  if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3369  canCombineSinCosLibcall(Node, TLI, TM))
3370  && useSinCos(Node)) {
3371  SDVTList VTs = DAG.getVTList(VT, VT);
3372  Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3373  if (!isSIN)
3374  Tmp1 = Tmp1.getValue(1);
3375  Results.push_back(Tmp1);
3376  } else if (isSIN) {
3377  Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
3380  } else {
3381  Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
3384  }
3385  break;
3386  }
3387  case ISD::FSINCOS:
3388  // Expand into sincos libcall.
3389  ExpandSinCosLibCall(Node, Results);
3390  break;
3391  case ISD::FLOG:
3392  Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64,
3395  break;
3396  case ISD::FLOG2:
3397  Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64,
3400  break;
3401  case ISD::FLOG10:
3402  Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64,
3405  break;
3406  case ISD::FEXP:
3407  Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64,
3410  break;
3411  case ISD::FEXP2:
3412  Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64,
3415  break;
3416  case ISD::FTRUNC:
3417  Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
3420  break;
3421  case ISD::FFLOOR:
3422  Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
3425  break;
3426  case ISD::FCEIL:
3427  Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
3430  break;
3431  case ISD::FRINT:
3432  Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
3435  break;
3436  case ISD::FNEARBYINT:
3437  Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
3442  break;
3443  case ISD::FROUND:
3444  Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32,
3449  break;
3450  case ISD::FPOWI:
3451  Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64,
3454  break;
3455  case ISD::FPOW:
3456  Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64,
3459  break;
3460  case ISD::FDIV:
3461  Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
3464  break;
3465  case ISD::FREM:
3466  Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
3469  break;
3470  case ISD::FMA:
3471  Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
3474  break;
3475  case ISD::FMAD:
3476  llvm_unreachable("Illegal fmad should never be formed");
3477 
3478  case ISD::FADD:
3479  Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
3482  break;
3483  case ISD::FMUL:
3484  Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
3487  break;
3488  case ISD::FP16_TO_FP: {
3489  if (Node->getValueType(0) == MVT::f32) {
3490  Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
3491  break;
3492  }
3493 
3494  // We can extend to types bigger than f32 in two steps without changing the
3495  // result. Since "f16 -> f32" is much more commonly available, give CodeGen
3496  // the option of emitting that before resorting to a libcall.
3497  SDValue Res =
3498  DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3499  Results.push_back(
3500  DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3501  break;
3502  }
3503  case ISD::FP_TO_FP16: {
3504  if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3505  SDValue Op = Node->getOperand(0);
3506  MVT SVT = Op.getSimpleValueType();
3507  if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3508  TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3509  // Under fastmath, we can expand this node into a fround followed by
3510  // a float-half conversion.
3511  SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3512  DAG.getIntPtrConstant(0, dl));
3513  Results.push_back(
3514  DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, FloatVal));
3515  break;
3516  }
3517  }
3518 
3519  RTLIB::Libcall LC =
3521  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
3522  Results.push_back(ExpandLibCall(LC, Node, false));
3523  break;
3524  }
3525  case ISD::ConstantFP: {
3526  ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3527  // Check to see if this FP immediate is already legal.
3528  // If this is a legal constant, turn it into a TargetConstantFP node.
3529  if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0)))
3530  Results.push_back(ExpandConstantFP(CFP, true));
3531  break;
3532  }
3533  case ISD::FSUB: {
3534  EVT VT = Node->getValueType(0);
3535  if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3536  TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3537  Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3538  Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1);
3539  Results.push_back(Tmp1);
3540  } else {
3541  Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
3544  }
3545  break;
3546  }
3547  case ISD::SUB: {
3548  EVT VT = Node->getValueType(0);
3549  assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3550  TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3551  "Don't know how to expand this subtraction!");
3552  Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
3553  DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
3554  VT));
3555  Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3556  Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3557  break;
3558  }
3559  case ISD::UREM:
3560  case ISD::SREM: {
3561  EVT VT = Node->getValueType(0);
3562  bool isSigned = Node->getOpcode() == ISD::SREM;
3563  unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
3564  unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3565  Tmp2 = Node->getOperand(0);
3566  Tmp3 = Node->getOperand(1);
3567  if (TLI.isOperationLegalOrCustom(DivRemOpc, VT) ||
3568  (isDivRemLibcallAvailable(Node, isSigned, TLI) &&
3569  // If div is legal, it's better to do the normal expansion
3570  !TLI.isOperationLegalOrCustom(DivOpc, Node->getValueType(0)) &&
3571  useDivRem(Node, isSigned, false))) {
3572  SDVTList VTs = DAG.getVTList(VT, VT);
3573  Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1);
3574  } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) {
3575  // X % Y -> X-X/Y*Y
3576  Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3);
3577  Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3);
3578  Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1);
3579  } else if (isSigned)
3580  Tmp1 = ExpandIntLibCall(Node, true,
3584  else
3585  Tmp1 = ExpandIntLibCall(Node, false,
3589  Results.push_back(Tmp1);
3590  break;
3591  }
3592  case ISD::UDIV:
3593  case ISD::SDIV: {
3594  bool isSigned = Node->getOpcode() == ISD::SDIV;
3595  unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3596  EVT VT = Node->getValueType(0);
3597  SDVTList VTs = DAG.getVTList(VT, VT);
3598  if (TLI.isOperationLegalOrCustom(DivRemOpc, VT) ||
3599  (isDivRemLibcallAvailable(Node, isSigned, TLI) &&
3600  useDivRem(Node, isSigned, true)))
3601  Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3602  Node->getOperand(1));
3603  else if (isSigned)
3604  Tmp1 = ExpandIntLibCall(Node, true,
3608  else
3609  Tmp1 = ExpandIntLibCall(Node, false,
3613  Results.push_back(Tmp1);
3614  break;
3615  }
3616  case ISD::MULHU:
3617  case ISD::MULHS: {
3618  unsigned ExpandOpcode = Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI :
3620  EVT VT = Node->getValueType(0);
3621  SDVTList VTs = DAG.getVTList(VT, VT);
3622  assert(TLI.isOperationLegalOrCustom(ExpandOpcode, VT) &&
3623  "If this wasn't legal, it shouldn't have been created!");
3624  Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3625  Node->getOperand(1));
3626  Results.push_back(Tmp1.getValue(1));
3627  break;
3628  }
3629  case ISD::SDIVREM:
3630  case ISD::UDIVREM:
3631  // Expand into divrem libcall
3632  ExpandDivRemLibCall(Node, Results);
3633  break;
3634  case ISD::MUL: {
3635  EVT VT = Node->getValueType(0);
3636  SDVTList VTs = DAG.getVTList(VT, VT);
3637  // See if multiply or divide can be lowered using two-result operations.
3638  // We just need the low half of the multiply; try both the signed
3639  // and unsigned forms. If the target supports both SMUL_LOHI and
3640  // UMUL_LOHI, form a preference by checking which forms of plain
3641  // MULH it supports.
3642  bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3643  bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3644  bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3645  bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3646  unsigned OpToUse = 0;
3647  if (HasSMUL_LOHI && !HasMULHS) {
3648  OpToUse = ISD::SMUL_LOHI;
3649  } else if (HasUMUL_LOHI && !HasMULHU) {
3650  OpToUse = ISD::UMUL_LOHI;
3651  } else if (HasSMUL_LOHI) {
3652  OpToUse = ISD::SMUL_LOHI;
3653  } else if (HasUMUL_LOHI) {
3654  OpToUse = ISD::UMUL_LOHI;
3655  }
3656  if (OpToUse) {
3657  Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3658  Node->getOperand(1)));
3659  break;
3660  }
3661 
3662  SDValue Lo, Hi;
3663  EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3664  if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3665  TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3666  TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3667  TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3668  TLI.expandMUL(Node, Lo, Hi, HalfType, DAG)) {
3669  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3670  Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3671  SDValue Shift =
3672  DAG.getConstant(HalfType.getSizeInBits(), dl,
3673  TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3674  Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3675  Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3676  break;
3677  }
3678 
3679  Tmp1 = ExpandIntLibCall(Node, false,
3680  RTLIB::MUL_I8,
3683  Results.push_back(Tmp1);
3684  break;
3685  }
3686  case ISD::SADDO:
3687  case ISD::SSUBO: {
3688  SDValue LHS = Node->getOperand(0);
3689  SDValue RHS = Node->getOperand(1);
3690  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3691  ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3692  LHS, RHS);
3693  Results.push_back(Sum);
3694  EVT ResultType = Node->getValueType(1);
3695  EVT OType = getSetCCResultType(Node->getValueType(0));
3696 
3697  SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
3698 
3699  // LHSSign -> LHS >= 0
3700  // RHSSign -> RHS >= 0
3701  // SumSign -> Sum >= 0
3702  //
3703  // Add:
3704  // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3705  // Sub:
3706  // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3707  //
3708  SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3709  SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3710  SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
3711  Node->getOpcode() == ISD::SADDO ?
3713 
3714  SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3715  SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3716 
3717  SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3718  Results.push_back(DAG.getBoolExtOrTrunc(Cmp, dl, ResultType, ResultType));
3719  break;
3720  }
3721  case ISD::UADDO:
3722  case ISD::USUBO: {
3723  SDValue LHS = Node->getOperand(0);
3724  SDValue RHS = Node->getOperand(1);
3725  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ?
3726  ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3727  LHS, RHS);
3728  Results.push_back(Sum);
3729 
3730  EVT ResultType = Node->getValueType(1);
3731  EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3732  ISD::CondCode CC
3733  = Node->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
3734  SDValue SetCC = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3735 
3736  Results.push_back(DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType));
3737  break;
3738  }
3739  case ISD::UMULO:
3740  case ISD::SMULO: {
3741  EVT VT = Node->getValueType(0);
3742  EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2);
3743  SDValue LHS = Node->getOperand(0);
3744  SDValue RHS = Node->getOperand(1);
3745  SDValue BottomHalf;
3746  SDValue TopHalf;
3747  static const unsigned Ops[2][3] =
3750  bool isSigned = Node->getOpcode() == ISD::SMULO;
3751  if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
3752  BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
3753  TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);
3754  } else if (TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
3755  BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,
3756  RHS);
3757  TopHalf = BottomHalf.getValue(1);
3758  } else if (TLI.isTypeLegal(WideVT)) {
3759  LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
3760  RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
3761  Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
3762  BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3763  DAG.getIntPtrConstant(0, dl));
3764  TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3765  DAG.getIntPtrConstant(1, dl));
3766  } else {
3767  // We can fall back to a libcall with an illegal type for the MUL if we
3768  // have a libcall big enough.
3769  // Also, we can fall back to a division in some cases, but that's a big
3770  // performance hit in the general case.
3772  if (WideVT == MVT::i16)
3773  LC = RTLIB::MUL_I16;
3774  else if (WideVT == MVT::i32)
3775  LC = RTLIB::MUL_I32;
3776  else if (WideVT == MVT::i64)
3777  LC = RTLIB::MUL_I64;
3778  else if (WideVT == MVT::i128)
3779  LC = RTLIB::MUL_I128;
3780  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!");
3781 
3782  // The high part is obtained by SRA'ing all but one of the bits of low
3783  // part.
3784  unsigned LoSize = VT.getSizeInBits();
3785  SDValue HiLHS =
3786  DAG.getNode(ISD::SRA, dl, VT, RHS,
3787  DAG.getConstant(LoSize - 1, dl,
3788  TLI.getPointerTy(DAG.getDataLayout())));
3789  SDValue HiRHS =
3790  DAG.getNode(ISD::SRA, dl, VT, LHS,
3791  DAG.getConstant(LoSize - 1, dl,
3792  TLI.getPointerTy(DAG.getDataLayout())));
3793 
3794  // Here we're passing the 2 arguments explicitly as 4 arguments that are
3795  // pre-lowered to the correct types. This all depends upon WideVT not
3796  // being a legal type for the architecture and thus has to be split to
3797  // two arguments.
3798  SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
3799  SDValue Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
3800  BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret,
3801  DAG.getIntPtrConstant(0, dl));
3802  TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret,
3803  DAG.getIntPtrConstant(1, dl));
3804  // Ret is a node with an illegal type. Because such things are not
3805  // generally permitted during this phase of legalization, make sure the
3806  // node has no more uses. The above EXTRACT_ELEMENT nodes should have been
3807  // folded.
3808  assert(Ret->use_empty() &&
3809  "Unexpected uses of illegally type from expanded lib call.");
3810  }
3811 
3812  if (isSigned) {
3813  Tmp1 = DAG.getConstant(
3814  VT.getSizeInBits() - 1, dl,
3815  TLI.getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));
3816  Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1);
3817  TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, Tmp1,
3818  ISD::SETNE);
3819  } else {
3820  TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf,
3821  DAG.getConstant(0, dl, VT), ISD::SETNE);
3822  }
3823  Results.push_back(BottomHalf);
3824  Results.push_back(TopHalf);
3825  break;
3826  }
3827  case ISD::BUILD_PAIR: {
3828  EVT PairTy = Node->getValueType(0);
3829  Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3830  Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3831  Tmp2 = DAG.getNode(
3832  ISD::SHL, dl, PairTy, Tmp2,
3833  DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3834  TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3835  Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3836  break;
3837  }
3838  case ISD::SELECT:
3839  Tmp1 = Node->getOperand(0);
3840  Tmp2 = Node->getOperand(1);
3841  Tmp3 = Node->getOperand(2);
3842  if (Tmp1.getOpcode() == ISD::SETCC) {
3843  Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3844  Tmp2, Tmp3,
3845  cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3846  } else {
3847  Tmp1 = DAG.getSelectCC(dl, Tmp1,
3848  DAG.getConstant(0, dl, Tmp1.getValueType()),
3849  Tmp2, Tmp3, ISD::SETNE);
3850  }
3851  Results.push_back(Tmp1);
3852  break;
3853  case ISD::BR_JT: {
3854  SDValue Chain = Node->getOperand(0);
3855  SDValue Table = Node->getOperand(1);
3856  SDValue Index = Node->getOperand(2);
3857 
3858  EVT PTy = TLI.getPointerTy(DAG.getDataLayout());
3859 
3860  const DataLayout &TD = DAG.getDataLayout();
3861  unsigned EntrySize =
3862  DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3863 
3864  Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3865  DAG.getConstant(EntrySize, dl, Index.getValueType()));
3866  SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3867  Index, Table);
3868 
3869  EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3870  SDValue LD = DAG.getExtLoad(ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3872  false, false, false, 0);
3873  Addr = LD;
3874  if (TM.getRelocationModel() == Reloc::PIC_) {
3875  // For PIC, the sequence is:
3876  // BRIND(load(Jumptable + index) + RelocBase)
3877  // RelocBase can be JumpTable, GOT or some sort of global base.
3878  Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3879  TLI.getPICJumpTableRelocBase(Table, DAG));
3880  }
3881  Tmp1 = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr);
3882  Results.push_back(Tmp1);
3883  break;
3884  }
3885  case ISD::BRCOND:
3886  // Expand brcond's setcc into its constituent parts and create a BR_CC
3887  // Node.
3888  Tmp1 = Node->getOperand(0);
3889  Tmp2 = Node->getOperand(1);
3890  if (Tmp2.getOpcode() == ISD::SETCC) {
3891  Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
3892  Tmp1, Tmp2.getOperand(2),
3893  Tmp2.getOperand(0), Tmp2.getOperand(1),
3894  Node->getOperand(2));
3895  } else {
3896  // We test only the i1 bit. Skip the AND if UNDEF.
3897  Tmp3 = (Tmp2.getOpcode() == ISD::UNDEF) ? Tmp2 :
3898  DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3899  DAG.getConstant(1, dl, Tmp2.getValueType()));
3900  Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3901  DAG.getCondCode(ISD::SETNE), Tmp3,
3902  DAG.getConstant(0, dl, Tmp3.getValueType()),
3903  Node->getOperand(2));
3904  }
3905  Results.push_back(Tmp1);
3906  break;
3907  case ISD::SETCC: {
3908  Tmp1 = Node->getOperand(0);
3909  Tmp2 = Node->getOperand(1);
3910  Tmp3 = Node->getOperand(2);
3911  bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2,
3912  Tmp3, NeedInvert, dl);
3913 
3914  if (Legalized) {
3915  // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3916  // condition code, create a new SETCC node.
3917  if (Tmp3.getNode())
3918  Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
3919  Tmp1, Tmp2, Tmp3);
3920 
3921  // If we expanded the SETCC by inverting the condition code, then wrap
3922  // the existing SETCC in a NOT to restore the intended condition.
3923  if (NeedInvert)
3924  Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
3925 
3926  Results.push_back(Tmp1);
3927  break;
3928  }
3929 
3930  // Otherwise, SETCC for the given comparison type must be completely
3931  // illegal; expand it into a SELECT_CC.
3932  EVT VT = Node->getValueType(0);
3933  int TrueValue;
3934  switch (TLI.getBooleanContents(Tmp1->getValueType(0))) {
3937  TrueValue = 1;
3938  break;
3940  TrueValue = -1;
3941  break;
3942  }
3943  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
3944  DAG.getConstant(TrueValue, dl, VT),
3945  DAG.getConstant(0, dl, VT),
3946  Tmp3);
3947  Results.push_back(Tmp1);
3948  break;
3949  }
3950  case ISD::SELECT_CC: {
3951  Tmp1 = Node->getOperand(0); // LHS
3952  Tmp2 = Node->getOperand(1); // RHS
3953  Tmp3 = Node->getOperand(2); // True
3954  Tmp4 = Node->getOperand(3); // False
3955  EVT VT = Node->getValueType(0);
3956  SDValue CC = Node->getOperand(4);
3957  ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
3958 
3959  if (TLI.isCondCodeLegal(CCOp, Tmp1.getSimpleValueType())) {
3960  // If the condition code is legal, then we need to expand this
3961  // node using SETCC and SELECT.
3962  EVT CmpVT = Tmp1.getValueType();
3963  assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
3964  "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
3965  "expanded.");
3966  EVT CCVT =
3967  TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT);
3968  SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC);
3969  Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
3970  break;
3971  }
3972 
3973  // SELECT_CC is legal, so the condition code must not be.
3974  bool Legalized = false;
3975  // Try to legalize by inverting the condition. This is for targets that
3976  // might support an ordered version of a condition, but not the unordered
3977  // version (or vice versa).
3978  ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp,
3979  Tmp1.getValueType().isInteger());
3980  if (TLI.isCondCodeLegal(InvCC, Tmp1.getSimpleValueType())) {
3981  // Use the new condition code and swap true and false
3982  Legalized = true;
3983  Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
3984  } else {
3985  // If The inverse is not legal, then try to swap the arguments using
3986  // the inverse condition code.
3987  ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
3988  if (TLI.isCondCodeLegal(SwapInvCC, Tmp1.getSimpleValueType())) {
3989  // The swapped inverse condition is legal, so swap true and false,
3990  // lhs and rhs.
3991  Legalized = true;
3992  Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
3993  }
3994  }
3995 
3996  if (!Legalized) {
3997  Legalized = LegalizeSetCCCondCode(
3998  getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert,
3999  dl);
4000 
4001  assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
4002 
4003  // If we expanded the SETCC by inverting the condition code, then swap
4004  // the True/False operands to match.
4005  if (NeedInvert)
4006  std::swap(Tmp3, Tmp4);
4007 
4008  // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4009  // condition code, create a new SELECT_CC node.
4010  if (CC.getNode()) {
4011  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
4012  Tmp1, Tmp2, Tmp3, Tmp4, CC);
4013  } else {
4014  Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
4015  CC = DAG.getCondCode(ISD::SETNE);
4016  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
4017  Tmp2, Tmp3, Tmp4, CC);
4018  }
4019  }
4020  Results.push_back(Tmp1);
4021  break;
4022  }
4023  case ISD::BR_CC: {
4024  Tmp1 = Node->getOperand(0); // Chain
4025  Tmp2 = Node->getOperand(2); // LHS
4026  Tmp3 = Node->getOperand(3); // RHS
4027  Tmp4 = Node->getOperand(1); // CC
4028 
4029  bool Legalized = LegalizeSetCCCondCode(getSetCCResultType(
4030  Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl);
4031  (void)Legalized;
4032  assert(Legalized && "Can't legalize BR_CC with legal condition!");
4033 
4034  // If we expanded the SETCC by inverting the condition code, then wrap
4035  // the existing SETCC in a NOT to restore the intended condition.
4036  if (NeedInvert)
4037  Tmp4 = DAG.getNOT(dl, Tmp4, Tmp4->getValueType(0));
4038 
4039  // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
4040  // node.
4041  if (Tmp4.getNode()) {
4042  Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
4043  Tmp4, Tmp2, Tmp3, Node->getOperand(4));
4044  } else {
4045  Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
4046  Tmp4 = DAG.getCondCode(ISD::SETNE);
4047  Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
4048  Tmp2, Tmp3, Node->getOperand(4));
4049  }
4050  Results.push_back(Tmp1);
4051  break;
4052  }
4053  case ISD::BUILD_VECTOR:
4054  Results.push_back(ExpandBUILD_VECTOR(Node));
4055  break;
4056  case ISD::SRA:
4057  case ISD::SRL:
4058  case ISD::SHL: {
4059  // Scalarize vector SRA/SRL/SHL.
4060  EVT VT = Node->getValueType(0);
4061  assert(VT.isVector() && "Unable to legalize non-vector shift");
4062  assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
4063  unsigned NumElem = VT.getVectorNumElements();
4064 
4065  SmallVector<SDValue, 8> Scalars;
4066  for (unsigned Idx = 0; Idx < NumElem; Idx++) {
4067  SDValue Ex = DAG.getNode(
4068  ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0),
4069  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4070  SDValue Sh = DAG.getNode(
4071  ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1),
4072  DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4073  Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
4074  VT.getScalarType(), Ex, Sh));
4075  }
4076  SDValue Result =
4077  DAG.getNode(ISD::BUILD_VECTOR, dl, Node->getValueType(0), Scalars);
4078  ReplaceNode(SDValue(Node, 0), Result);
4079  break;
4080  }
4082  case ISD::GlobalAddress:
4083  case ISD::GlobalTLSAddress:
4084  case ISD::ExternalSymbol:
4085  case ISD::ConstantPool:
4086  case ISD::JumpTable:
4089  case ISD::INTRINSIC_VOID:
4090  // FIXME: Custom lowering for these operations shouldn't return null!
4091  break;
4092  }
4093 
4094  // Replace the original node with the legalized result.
4095  if (!Results.empty())
4096  ReplaceNode(Node, Results.data());
4097 }
4098 
4099 void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4100  SmallVector<SDValue, 8> Results;
4101  MVT OVT = Node->getSimpleValueType(0);
4102  if (Node->getOpcode() == ISD::UINT_TO_FP ||
4103  Node->getOpcode() == ISD::SINT_TO_FP ||
4104  Node->getOpcode() == ISD::SETCC) {
4105  OVT = Node->getOperand(0).getSimpleValueType();
4106  }
4107  if (Node->getOpcode() == ISD::BR_CC)
4108  OVT = Node->getOperand(2).getSimpleValueType();
4109  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4110  SDLoc dl(Node);
4111  SDValue Tmp1, Tmp2, Tmp3;
4112  switch (Node->getOpcode()) {
4113  case ISD::CTTZ:
4114  case ISD::CTTZ_ZERO_UNDEF:
4115  case ISD::CTLZ:
4116  case ISD::CTLZ_ZERO_UNDEF:
4117  case ISD::CTPOP:
4118  // Zero extend the argument.
4119  Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4120  // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4121  // already the correct result.
4122  Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4123  if (Node->getOpcode() == ISD::CTTZ) {
4124  // FIXME: This should set a bit in the zero extended value instead.
4125  Tmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT),
4126  Tmp1, DAG.getConstant(NVT.getSizeInBits(), dl, NVT),
4127  ISD::SETEQ);
4128  Tmp1 = DAG.getSelect(dl, NVT, Tmp2,
4129  DAG.getConstant(OVT.getSizeInBits(), dl, NVT), Tmp1);
4130  } else if (Node->getOpcode() == ISD::CTLZ ||
4131  Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
4132  // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4133  Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
4134  DAG.getConstant(NVT.getSizeInBits() -
4135  OVT.getSizeInBits(), dl, NVT));
4136  }
4137  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4138  break;
4139  case ISD::BSWAP: {
4140  unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
4141  Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4142  Tmp1 = DAG.getNode(ISD::BSWAP, dl, NVT, Tmp1);
4143  Tmp1 = DAG.getNode(
4144  ISD::SRL, dl, NVT, Tmp1,
4145  DAG.getConstant(DiffBits, dl,
4146  TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4147  Results.push_back(Tmp1);
4148  break;
4149  }
4150  case ISD::FP_TO_UINT:
4151  case ISD::FP_TO_SINT:
4152  Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0),
4153  Node->getOpcode() == ISD::FP_TO_SINT, dl);
4154  Results.push_back(Tmp1);
4155  break;
4156  case ISD::UINT_TO_FP:
4157  case ISD::SINT_TO_FP:
4158  Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0),
4159  Node->getOpcode() == ISD::SINT_TO_FP, dl);
4160  Results.push_back(Tmp1);
4161  break;
4162  case ISD::VAARG: {
4163  SDValue Chain = Node->getOperand(0); // Get the chain.
4164  SDValue Ptr = Node->getOperand(1); // Get the pointer.
4165 
4166  unsigned TruncOp;
4167  if (OVT.isVector()) {
4168  TruncOp = ISD::BITCAST;
4169  } else {
4170  assert(OVT.isInteger()
4171  && "VAARG promotion is supported only for vectors or integer types");
4172  TruncOp = ISD::TRUNCATE;
4173  }
4174 
4175  // Perform the larger operation, then convert back
4176  Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
4177  Node->getConstantOperandVal(3));
4178  Chain = Tmp1.getValue(1);
4179 
4180  Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
4181 
4182  // Modified the chain result - switch anything that used the old chain to
4183  // use the new one.
4184  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
4185  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
4186  if (UpdatedNodes) {
4187  UpdatedNodes->insert(Tmp2.getNode());
4188  UpdatedNodes->insert(Chain.getNode());
4189  }
4190  ReplacedNode(Node);
4191  break;
4192  }
4193  case ISD::AND:
4194  case ISD::OR:
4195  case ISD::XOR: {
4196  unsigned ExtOp, TruncOp;
4197  if (OVT.isVector()) {
4198  ExtOp = ISD::BITCAST;
4199  TruncOp = ISD::BITCAST;
4200  } else {
4201  assert(OVT.isInteger() && "Cannot promote logic operation");
4202  ExtOp = ISD::ANY_EXTEND;
4203  TruncOp = ISD::TRUNCATE;
4204  }
4205  // Promote each of the values to the new type.
4206  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4207  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4208  // Perform the larger operation, then convert back
4209  Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4210  Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
4211  break;
4212  }
4213  case ISD::SELECT: {
4214  unsigned ExtOp, TruncOp;
4215  if (Node->getValueType(0).isVector() ||
4216  Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
4217  ExtOp = ISD::BITCAST;
4218  TruncOp = ISD::BITCAST;
4219  } else if (Node->getValueType(0).isInteger()) {
4220  ExtOp = ISD::ANY_EXTEND;
4221  TruncOp = ISD::TRUNCATE;
4222  } else {
4223  ExtOp = ISD::FP_EXTEND;
4224  TruncOp = ISD::FP_ROUND;
4225  }
4226  Tmp1 = Node->getOperand(0);
4227  // Promote each of the values to the new type.
4228  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4229  Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4230  // Perform the larger operation, then round down.
4231  Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
4232  if (TruncOp != ISD::FP_ROUND)
4233  Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
4234  else
4235  Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
4236  DAG.getIntPtrConstant(0, dl));
4237  Results.push_back(Tmp1);
4238  break;
4239  }
4240  case ISD::VECTOR_SHUFFLE: {
4241  ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
4242 
4243  // Cast the two input vectors.
4244  Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
4245  Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
4246 
4247  // Convert the shuffle mask to the right # elements.
4248  Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
4249  Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
4250  Results.push_back(Tmp1);
4251  break;
4252  }
4253  case ISD::SETCC: {
4254  unsigned ExtOp = ISD::FP_EXTEND;
4255  if (NVT.isInteger()) {
4256  ISD::CondCode CCCode =
4257  cast<CondCodeSDNode>(Node->getOperand(2))->get();
4259  }
4260  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4261  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4262  Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
4263  Tmp1, Tmp2, Node->getOperand(2)));
4264  break;
4265  }
4266  case ISD::BR_CC: {
4267  unsigned ExtOp = ISD::FP_EXTEND;
4268  if (NVT.isInteger()) {
4269  ISD::CondCode CCCode =
4270  cast<CondCodeSDNode>(Node->getOperand(1))->get();
4272  }
4273  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4274  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4275  Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
4276  Node->getOperand(0), Node->getOperand(1),
4277  Tmp1, Tmp2, Node->getOperand(4)));
4278  break;
4279  }
4280  case ISD::FADD:
4281  case ISD::FSUB:
4282  case ISD::FMUL:
4283  case ISD::FDIV:
4284  case ISD::FREM:
4285  case ISD::FMINNUM:
4286  case ISD::FMAXNUM:
4287  case ISD::FCOPYSIGN:
4288  case ISD::FPOW: {
4289  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4290  Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4291  Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4292  Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4293  Tmp3, DAG.getIntPtrConstant(0, dl)));
4294  break;
4295  }
4296  case ISD::FMA: {
4297  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4298  Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4299  Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
4300  Results.push_back(
4301  DAG.getNode(ISD::FP_ROUND, dl, OVT,
4302  DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
4303  DAG.getIntPtrConstant(0, dl)));
4304  break;
4305  }
4306  case ISD::FPOWI: {
4307  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4308  Tmp2 = Node->getOperand(1);
4309  Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4310  Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4311  Tmp3, DAG.getIntPtrConstant(0, dl)));
4312  break;
4313  }
4314  case ISD::FFLOOR:
4315  case ISD::FCEIL:
4316  case ISD::FRINT:
4317  case ISD::FNEARBYINT:
4318  case ISD::FROUND:
4319  case ISD::FTRUNC:
4320  case ISD::FNEG:
4321  case ISD::FSQRT:
4322  case ISD::FSIN:
4323  case ISD::FCOS:
4324  case ISD::FLOG:
4325  case ISD::FLOG2:
4326  case ISD::FLOG10:
4327  case ISD::FABS:
4328  case ISD::FEXP:
4329  case ISD::FEXP2: {
4330  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4331  Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4332  Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4333  Tmp2, DAG.getIntPtrConstant(0, dl)));
4334  break;
4335  }
4336  }
4337 
4338  // Replace the original node with the legalized result.
4339  if (!Results.empty())
4340  ReplaceNode(Node, Results.data());
4341 }
4342 
4343 /// This is the entry point for the file.
4346 
4347  SmallPtrSet<SDNode *, 16> LegalizedNodes;
4348  SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
4349 
4350  // Visit all the nodes. We start in topological order, so that we see
4351  // nodes with their original operands intact. Legalization can produce
4352  // new nodes which may themselves need to be legalized. Iterate until all
4353  // nodes have been legalized.
4354  for (;;) {
4355  bool AnyLegalized = false;
4356  for (auto NI = allnodes_end(); NI != allnodes_begin();) {
4357  --NI;
4358 
4359  SDNode *N = NI;
4360  if (N->use_empty() && N != getRoot().getNode()) {
4361  ++NI;
4362  DeleteNode(N);
4363  continue;
4364  }
4365 
4366  if (LegalizedNodes.insert(N).second) {
4367  AnyLegalized = true;
4368  Legalizer.LegalizeOp(N);
4369 
4370  if (N->use_empty() && N != getRoot().getNode()) {
4371  ++NI;
4372  DeleteNode(N);
4373  }
4374  }
4375  }
4376  if (!AnyLegalized)
4377  break;
4378 
4379  }
4380 
4381  // Remove dead nodes now.
4382  RemoveDeadNodes();
4383 }
4384 
4386  SmallSetVector<SDNode *, 16> &UpdatedNodes) {
4387  SmallPtrSet<SDNode *, 16> LegalizedNodes;
4388  SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
4389 
4390  // Directly insert the node in question, and legalize it. This will recurse
4391  // as needed through operands.
4392  LegalizedNodes.insert(N);
4393  Legalizer.LegalizeOp(N);
4394 
4395  return LegalizedNodes.count(N);
4396 }
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
bool LegalizeOp(SDNode *N, SmallSetVector< SDNode *, 16 > &UpdatedNodes)
Transforms a SelectionDAG node and any operands to it into a node that is compatible with the target ...
SDValue getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT TVT, bool isNonTemporal, bool isVolatile, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes())
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
Definition: ISDOpcodes.h:641
static Constant * getFPTrunc(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:1741
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:477
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:450
bool use_empty() const
Return true if there are no uses of this node.
void push_back(const T &Elt)
Definition: SmallVector.h:222
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:724
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:104
SDValue getValue(unsigned R) const
static APInt getSignBit(unsigned BitWidth)
Get the SignBit for a specific bit width.
Definition: APInt.h:446
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant, which is required to be operand #1) half of the integer or float value specified as operand #0.
Definition: ISDOpcodes.h:175
static void ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG, const TargetLowering &TLI, SDValue &ValResult, SDValue &ChainResult)
Expands an unaligned load to 2 half-size loads.
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
Definition: APInt.h:453
LLVMContext * getContext() const
Definition: SelectionDAG.h:289
void dump() const
Dump this node, for debugging.
NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:292
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:554
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:301
static MachinePointerInfo getJumpTable()
getJumpTable - Return a MachinePointerInfo record that refers to a jump table entry.
SDVTList getVTList() const
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:285
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain...
Definition: ISDOpcodes.h:585
size_type count(PtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:276
static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned, const TargetLowering &TLI)
Return true if divmod libcall is available.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:228
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
Type * getTypeForEVT(LLVMContext &Context) const
getTypeForEVT - This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:181
unsigned getSizeInBits() const
SDValue getLoad(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, bool isVolatile, bool isNonTemporal, bool isInvariant, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:679
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
static MachinePointerInfo getConstantPool()
getConstantPool - Return a MachinePointerInfo record that refers to the constant pool.
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:240
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:318
Same for subtraction.
Definition: ISDOpcodes.h:231
void reserve(size_type N)
Definition: SmallVector.h:401
void DeleteNode(SDNode *N)
Remove the specified node from the system.
const SDValue & getBasePtr() const
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1 at the ...
Definition: ISDOpcodes.h:287
The address of the GOT.
Definition: ISDOpcodes.h:66
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
Definition: ISDOpcodes.h:658
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:357
unsigned getResNo() const
get the index which selects a specific result in the SDNode
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
bool bitsLT(EVT VT) const
bitsLT - Return true if this has less bits than VT.
Definition: ValueTypes.h:189
static MachinePointerInfo getFixedStack(int FI, int64_t offset=0)
getFixedStack - Return a MachinePointerInfo record that refers to the the specified FrameIndex...
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:150
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:242
const Triple & getTargetTriple() const
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic...
Definition: ISDOpcodes.h:109
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:200
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations...
Definition: ISDOpcodes.h:371
bool isVector() const
isVector - Return true if this is a vector value type.
Definition: ValueTypes.h:115
SDValue getStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, bool isVolatile, bool isNonTemporal, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
Shift and rotation operations.
Definition: ISDOpcodes.h:332
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:98
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
APInt LLVM_ATTRIBUTE_UNUSED_RESULT lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition: APInt.cpp:1142
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:181
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APInt.h:33
static Constant * get(ArrayRef< Constant * > V)
Definition: Constants.cpp:1057
FLT_ROUNDS_ - Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest 2 Round to ...
Definition: ISDOpcodes.h:458
MachinePointerInfo getWithOffset(int64_t O) const
SimpleValueType SimpleTy
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence, and carry arbitrary information that target might want to know.
Definition: ISDOpcodes.h:592
EVT getScalarType() const
getScalarType - If this is a vector type, return the element type, otherwise return this...
Definition: ValueTypes.h:210
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt) For double-word atomic operations: ValLo, ValHi, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amtLo, amtHi) ValLo, ValHi, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amtLo, amtHi) These correspond to the atomicrmw instruction.
Definition: ISDOpcodes.h:687
Number of individual test Apply this number of consecutive mutations to each input exit after the first new interesting input is found the minimized corpus is saved into the first input directory Number of jobs to run If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
size_type size() const
Definition: SmallSet.h:48
allnodes_const_iterator allnodes_end() const
Definition: SelectionDAG.h:318
bool bitsGE(EVT VT) const
bitsGE - Return true if this has no less bits than VT.
Definition: ValueTypes.h:183
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG...
Definition: ISDOpcodes.h:73
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
Definition: ValueTypes.h:110
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:581
FRAME_TO_ARGS_OFFSET - This node represents offset from frame pointer to first (possible) on-stack ar...
Definition: ISDOpcodes.h:91
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:117
EVT getVectorElementType() const
getVectorElementType - Given a vector type, return the type of each element.
Definition: ValueTypes.h:216
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:393
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
Definition: SmallVector.h:57
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:191
SDValue getVectorShuffle(EVT VT, SDLoc dl, SDValue N1, SDValue N2, const int *MaskElts)
Return an ISD::VECTOR_SHUFFLE node.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: ArrayRef.h:31
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:217
const SDValue & getBasePtr() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
getHalfSizedIntegerVT - Finds the smallest simple value type that is greater than or equal to half th...
Definition: ValueTypes.h:270
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:659
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:804
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification, or lowering of the constant.
Definition: ISDOpcodes.h:116
EVT getMemoryVT() const
Return the type of the in-memory value.
static void ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG, const TargetLowering &TLI, SelectionDAGLegalize *DAGLegalize)
Expands an unaligned store to 2 half-size stores.
static bool useSinCos(SDNode *Node)
Only issue sincos libcall if both sin and cos are needed.
bool isSignedIntSetCC(CondCode Code)
isSignedIntSetCC - Return true if this is a setcc instruction that performs a signed comparison when ...
Definition: ISDOpcodes.h:837
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:142
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:284
UNDEF - An undefined node.
Definition: ISDOpcodes.h:169
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:436
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:262
static bool useDivRem(SDNode *Node, bool isSigned, bool isDIV)
Only issue divrem libcall if both quotient and remainder are needed.
SDNode * getNode() const
get the SDNode which holds the desired result
A self-contained host- and target-independent arbitrary-precision floating-point software implementat...
Definition: APFloat.h:122
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:157
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
unsigned getStoreSizeInBits() const
getStoreSizeInBits - Return the number of bits overwritten by a store of the specified value type...
Definition: ValueTypes.h:251
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
virtual bool isShuffleMaskLegal(const SmallVectorImpl< int > &, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations, those with specific masks.
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition: ISDOpcodes.h:85
APInt LLVM_ATTRIBUTE_UNUSED_RESULT trunc(unsigned width) const
Truncate to new width.
Definition: APInt.cpp:932
void Legalize()
This transforms the SelectionDAG into a SelectionDAG that is compatible with the target instruction s...
MVT - Machine Value Type.
const SDValue & getOperand(unsigned i) const
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
void swap(SmallVectorImpl &RHS)
Definition: SmallVector.h:693
Simple binary floating point operators.
Definition: ISDOpcodes.h:237
bool isNonTemporal() const
This is an important base class in LLVM.
Definition: Constant.h:41
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE...
Definition: ISDOpcodes.h:607
bool isVector() const
isVector - Return true if this is a vector value type.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:780
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:267
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:32
This file contains the declarations for the subclasses of Constant, which represent the different fla...
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
Definition: APInt.h:1895
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:233
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:264
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:225
This class provides iterator support for SDUse operands that use a specific SDNode.
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
Definition: ISDOpcodes.h:635
unsigned getOpcode() const
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:644
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
DEBUGTRAP - Trap intended to get the attention of a debugger.
Definition: ISDOpcodes.h:647
CondCode getSetCCSwappedOperands(CondCode Operation)
getSetCCSwappedOperands - Return the operation corresponding to (Y op X) when given the operation for...
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:69
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:598
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
bool isVolatile() const
const SDValue & getValue() const
static bool isValueValidForType(EVT VT, const APFloat &Val)
SDValue getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, bool isVolatile, bool isNonTemporal, bool isInvariant, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes())
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:338
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo...
Definition: ISDOpcodes.h:673
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:468
EVT - Extended Value Type.
Definition: ValueTypes.h:31
static UndefValue * get(Type *T)
get() - Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1473
std::vector< ArgListEntry > ArgListTy
const APFloat & getValueAPF() const
bool bitsEq(EVT VT) const
bitsEq - Return true if this has the same number of bits as VT.
Definition: ValueTypes.h:171
const ConstantFP * getConstantFPValue() const
This structure contains all information that is necessary for lowering calls.
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc, or post-dec.
PointerType * getPointerTo(unsigned AddrSpace=0)
getPointerTo - Return a pointer to the current type.
Definition: Type.cpp:764
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements)
getVectorVT - Returns the EVT that represents a vector NumElements in length, where each element is o...
Definition: ValueTypes.h:70
MachinePointerInfo - This class contains a discriminated union of information about pointers in memor...
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:335
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:576
const MachinePointerInfo & getPointerInfo() const
bool bitsGT(EVT VT) const
bitsGT - Return true if this has more bits than VT.
Definition: ValueTypes.h:177
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:50
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:217
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:299
This is the shared class of boolean and integer constants.
Definition: Constants.h:47
allnodes_const_iterator allnodes_begin() const
Definition: SelectionDAG.h:317
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false...
Definition: SmallPtrSet.h:271
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:273
bool isInvariant() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:179
X = FP_ROUND_INREG(Y, VT) - This operator takes an FP register, and rounds it to a floating point val...
Definition: ISDOpcodes.h:465
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:548
const SDValue & getChain() const
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:548
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:335
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:500
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:582
Represents one node in the SelectionDAG.
CondCode getSetCCInverse(CondCode Operation, bool isInteger)
getSetCCInverse - Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operat...
static cl::opt< AlignMode > Align(cl::desc("Load/store alignment support"), cl::Hidden, cl::init(NoStrictAlign), cl::values(clEnumValN(StrictAlign,"aarch64-strict-align","Disallow all unaligned memory accesses"), clEnumValN(NoStrictAlign,"aarch64-no-strict-align","Allow unaligned memory accesses"), clEnumValEnd))
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
double BitsToDouble(uint64_t Bits)
BitsToDouble - This function takes a 64-bit integer and returns the bit equivalent double...
Definition: MathExtras.h:504
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:123
unsigned Log2_32(uint32_t Value)
Log2_32 - This function returns the floor log base 2 of the specified value, -1 if the value is zero...
Definition: MathExtras.h:468
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:576
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
Definition: APInt.h:545
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Class for arbitrary precision integers.
Definition: APInt.h:73
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:342
op_iterator op_begin() const
static use_iterator use_end()
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:383
APInt bitcastToAPInt() const
Definition: APFloat.cpp:3084
SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:386
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:250
iterator_range< value_op_iterator > op_values() const
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
Definition: APInt.h:1890
unsigned getAddressSpace() const
Return the address space for the associated pointer.
#define Success
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:542
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer, a SRCVALUE for the destination, and a SRCVALUE for the source.
Definition: ISDOpcodes.h:603
uint64_t MinAlign(uint64_t A, uint64_t B)
MinAlign - A and B are either alignments or offsets.
Definition: MathExtras.h:552
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:321
pointer data()
Return a pointer to the vector's buffer, even if empty().
Definition: SmallVector.h:134
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:196
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:401
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:518
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
Definition: ValueTypes.h:233
void ReplaceAllUsesWith(SDValue From, SDValue Op)
Modify anything using 'From' to use 'To' instead.
#define N
Fast - This calling convention attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:42
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
op_iterator op_end() const
Same for multiplication.
Definition: ISDOpcodes.h:234
bool reachesChainWithoutSideEffects(SDValue Dest, unsigned Depth=2) const
Return true if this operand (which must be a chain) reaches the specified operand without crossing an...
FSINCOS - Compute both fsin and fcos as a single operation.
Definition: ISDOpcodes.h:512
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.setjmp intrinsic.
Definition: ISDOpcodes.h:103
EVT getValueType() const
Return the ValueType of the referenced return value.
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin...
Definition: ISDOpcodes.h:97
SDValue getConstant(uint64_t Val, SDLoc DL, EVT VT, bool isTarget=false, bool isOpaque=false)
const APFloat & getValueAPF() const
Definition: Constants.h:270
Libcall getATOMIC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none...
bool isByteSized() const
isByteSized - Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:160
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:279
bool isFloatingPoint() const
isFloatingPoint - Return true if this is a FP, or a vector FP type.
Definition: ValueTypes.h:105
This class is used to form a handle around another node that is persistent and is updated across invo...
static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI)
Return true if sincos libcall is available.
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
LLVM Value Representation.
Definition: Value.h:69
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:240
static bool ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, const TargetLowering &TLI, SDValue &Res)
bool isTruncatingStore() const
Return true if the op does a truncation before store.
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:653
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:287
#define DEBUG(X)
Definition: Debug.h:92
Primary interface to the complete machine description for the target machine.
C - The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations...
Definition: ISDOpcodes.h:244
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:365
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:188
Conversion operators.
Definition: ISDOpcodes.h:380
static bool isVolatile(Instruction *Inst)
static APInt getNullValue(unsigned numBits)
Get the '0' value.
Definition: APInt.h:460
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:666
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:389
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
unsigned getAlignment() const
void TransferDbgValues(SDValue From, SDValue To)
Transfer SDDbgValues.
static bool canCombineSinCosLibcall(SDNode *Node, const TargetLowering &TLI, const TargetMachine &TM)
Return true if sincos libcall is available and can be used to combine sin and cos.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR - Perform various unary floating point operations.
Definition: ISDOpcodes.h:506
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
getIntegerVT - Returns the EVT that represents an integer with the given number of bits...
Definition: ValueTypes.h:61
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:662
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:309
MVT getSimpleVT() const
getSimpleVT - Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:203
This file describes how to lower LLVM code to machine code.
BRIND - Indirect branch.
Definition: ISDOpcodes.h:538
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:314
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:225
This class is used to represent ISD::LOAD nodes.
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary...
Definition: ISDOpcodes.h:527