LLVM  3.7.0
SelectionDAG.cpp
Go to the documentation of this file.
1 //===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This implements the SelectionDAG class.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "SDNodeDbgValue.h"
16 #include "llvm/ADT/SetVector.h"
17 #include "llvm/ADT/SmallPtrSet.h"
18 #include "llvm/ADT/SmallSet.h"
19 #include "llvm/ADT/SmallVector.h"
20 #include "llvm/ADT/StringExtras.h"
26 #include "llvm/IR/CallingConv.h"
27 #include "llvm/IR/Constants.h"
28 #include "llvm/IR/DataLayout.h"
29 #include "llvm/IR/DebugInfo.h"
30 #include "llvm/IR/DerivedTypes.h"
31 #include "llvm/IR/Function.h"
32 #include "llvm/IR/GlobalAlias.h"
33 #include "llvm/IR/GlobalVariable.h"
34 #include "llvm/IR/Intrinsics.h"
36 #include "llvm/Support/Debug.h"
40 #include "llvm/Support/Mutex.h"
50 #include <algorithm>
51 #include <cmath>
52 #include <utility>
53 
54 using namespace llvm;
55 
56 /// makeVTList - Return an instance of the SDVTList struct initialized with the
57 /// specified members.
58 static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
59  SDVTList Res = {VTs, NumVTs};
60  return Res;
61 }
62 
63 // Default null implementations of the callbacks.
66 
67 //===----------------------------------------------------------------------===//
68 // ConstantFPSDNode Class
69 //===----------------------------------------------------------------------===//
70 
71 /// isExactlyValue - We don't rely on operator== working on double values, as
72 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
73 /// As such, this method can be used to do an exact bit-for-bit comparison of
74 /// two floating point values.
76  return getValueAPF().bitwiseIsEqual(V);
77 }
78 
80  const APFloat& Val) {
81  assert(VT.isFloatingPoint() && "Can only convert between FP types");
82 
83  // convert modifies in place, so make a copy.
84  APFloat Val2 = APFloat(Val);
85  bool losesInfo;
86  (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
88  &losesInfo);
89  return !losesInfo;
90 }
91 
92 //===----------------------------------------------------------------------===//
93 // ISD Namespace
94 //===----------------------------------------------------------------------===//
95 
96 /// isBuildVectorAllOnes - Return true if the specified node is a
97 /// BUILD_VECTOR where all of the elements are ~0 or undef.
99  // Look through a bit convert.
100  while (N->getOpcode() == ISD::BITCAST)
101  N = N->getOperand(0).getNode();
102 
103  if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
104 
105  unsigned i = 0, e = N->getNumOperands();
106 
107  // Skip over all of the undef values.
108  while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF)
109  ++i;
110 
111  // Do not accept an all-undef vector.
112  if (i == e) return false;
113 
114  // Do not accept build_vectors that aren't all constants or which have non-~0
115  // elements. We have to be a bit careful here, as the type of the constant
116  // may not be the same as the type of the vector elements due to type
117  // legalization (the elements are promoted to a legal type for the target and
118  // a vector of a type may be legal when the base element type is not).
119  // We only want to check enough bits to cover the vector elements, because
120  // we care if the resultant vector is all ones, not whether the individual
121  // constants are.
122  SDValue NotZero = N->getOperand(i);
123  unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
124  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
125  if (CN->getAPIntValue().countTrailingOnes() < EltSize)
126  return false;
127  } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
128  if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
129  return false;
130  } else
131  return false;
132 
133  // Okay, we have at least one ~0 value, check to see if the rest match or are
134  // undefs. Even with the above element type twiddling, this should be OK, as
135  // the same type legalization should have applied to all the elements.
136  for (++i; i != e; ++i)
137  if (N->getOperand(i) != NotZero &&
138  N->getOperand(i).getOpcode() != ISD::UNDEF)
139  return false;
140  return true;
141 }
142 
143 
144 /// isBuildVectorAllZeros - Return true if the specified node is a
145 /// BUILD_VECTOR where all of the elements are 0 or undef.
147  // Look through a bit convert.
148  while (N->getOpcode() == ISD::BITCAST)
149  N = N->getOperand(0).getNode();
150 
151  if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
152 
153  bool IsAllUndef = true;
154  for (const SDValue &Op : N->op_values()) {
155  if (Op.getOpcode() == ISD::UNDEF)
156  continue;
157  IsAllUndef = false;
158  // Do not accept build_vectors that aren't all constants or which have non-0
159  // elements. We have to be a bit careful here, as the type of the constant
160  // may not be the same as the type of the vector elements due to type
161  // legalization (the elements are promoted to a legal type for the target
162  // and a vector of a type may be legal when the base element type is not).
163  // We only want to check enough bits to cover the vector elements, because
164  // we care if the resultant vector is all zeros, not whether the individual
165  // constants are.
166  unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
167  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
168  if (CN->getAPIntValue().countTrailingZeros() < EltSize)
169  return false;
170  } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Op)) {
171  if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
172  return false;
173  } else
174  return false;
175  }
176 
177  // Do not accept an all-undef vector.
178  if (IsAllUndef)
179  return false;
180  return true;
181 }
182 
183 /// \brief Return true if the specified node is a BUILD_VECTOR node of
184 /// all ConstantSDNode or undef.
186  if (N->getOpcode() != ISD::BUILD_VECTOR)
187  return false;
188 
189  for (const SDValue &Op : N->op_values()) {
190  if (Op.getOpcode() == ISD::UNDEF)
191  continue;
192  if (!isa<ConstantSDNode>(Op))
193  return false;
194  }
195  return true;
196 }
197 
198 /// \brief Return true if the specified node is a BUILD_VECTOR node of
199 /// all ConstantFPSDNode or undef.
201  if (N->getOpcode() != ISD::BUILD_VECTOR)
202  return false;
203 
204  for (const SDValue &Op : N->op_values()) {
205  if (Op.getOpcode() == ISD::UNDEF)
206  continue;
207  if (!isa<ConstantFPSDNode>(Op))
208  return false;
209  }
210  return true;
211 }
212 
213 /// isScalarToVector - Return true if the specified node is a
214 /// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low
215 /// element is not an undef.
217  if (N->getOpcode() == ISD::SCALAR_TO_VECTOR)
218  return true;
219 
220  if (N->getOpcode() != ISD::BUILD_VECTOR)
221  return false;
222  if (N->getOperand(0).getOpcode() == ISD::UNDEF)
223  return false;
224  unsigned NumElems = N->getNumOperands();
225  if (NumElems == 1)
226  return false;
227  for (unsigned i = 1; i < NumElems; ++i) {
228  SDValue V = N->getOperand(i);
229  if (V.getOpcode() != ISD::UNDEF)
230  return false;
231  }
232  return true;
233 }
234 
235 /// allOperandsUndef - Return true if the node has at least one operand
236 /// and all operands of the specified node are ISD::UNDEF.
238  // Return false if the node has no operands.
239  // This is "logically inconsistent" with the definition of "all" but
240  // is probably the desired behavior.
241  if (N->getNumOperands() == 0)
242  return false;
243 
244  for (const SDValue &Op : N->op_values())
245  if (Op.getOpcode() != ISD::UNDEF)
246  return false;
247 
248  return true;
249 }
250 
252  switch (ExtType) {
253  case ISD::EXTLOAD:
254  return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
255  case ISD::SEXTLOAD:
256  return ISD::SIGN_EXTEND;
257  case ISD::ZEXTLOAD:
258  return ISD::ZERO_EXTEND;
259  default:
260  break;
261  }
262 
263  llvm_unreachable("Invalid LoadExtType");
264 }
265 
266 /// getSetCCSwappedOperands - Return the operation corresponding to (Y op X)
267 /// when given the operation for (X op Y).
269  // To perform this operation, we just need to swap the L and G bits of the
270  // operation.
271  unsigned OldL = (Operation >> 2) & 1;
272  unsigned OldG = (Operation >> 1) & 1;
273  return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
274  (OldL << 1) | // New G bit
275  (OldG << 2)); // New L bit.
276 }
277 
278 /// getSetCCInverse - Return the operation corresponding to !(X op Y), where
279 /// 'op' is a valid SetCC operation.
281  unsigned Operation = Op;
282  if (isInteger)
283  Operation ^= 7; // Flip L, G, E bits, but not U.
284  else
285  Operation ^= 15; // Flip all of the condition bits.
286 
287  if (Operation > ISD::SETTRUE2)
288  Operation &= ~8; // Don't let N and U bits get set.
289 
290  return ISD::CondCode(Operation);
291 }
292 
293 
294 /// isSignedOp - For an integer comparison, return 1 if the comparison is a
295 /// signed operation and 2 if the result is an unsigned comparison. Return zero
296 /// if the operation does not depend on the sign of the input (setne and seteq).
297 static int isSignedOp(ISD::CondCode Opcode) {
298  switch (Opcode) {
299  default: llvm_unreachable("Illegal integer setcc operation!");
300  case ISD::SETEQ:
301  case ISD::SETNE: return 0;
302  case ISD::SETLT:
303  case ISD::SETLE:
304  case ISD::SETGT:
305  case ISD::SETGE: return 1;
306  case ISD::SETULT:
307  case ISD::SETULE:
308  case ISD::SETUGT:
309  case ISD::SETUGE: return 2;
310  }
311 }
312 
313 /// getSetCCOrOperation - Return the result of a logical OR between different
314 /// comparisons of identical values: ((X op1 Y) | (X op2 Y)). This function
315 /// returns SETCC_INVALID if it is not possible to represent the resultant
316 /// comparison.
318  bool isInteger) {
319  if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
320  // Cannot fold a signed integer setcc with an unsigned integer setcc.
321  return ISD::SETCC_INVALID;
322 
323  unsigned Op = Op1 | Op2; // Combine all of the condition bits.
324 
325  // If the N and U bits get set then the resultant comparison DOES suddenly
326  // care about orderedness, and is true when ordered.
327  if (Op > ISD::SETTRUE2)
328  Op &= ~16; // Clear the U bit if the N bit is set.
329 
330  // Canonicalize illegal integer setcc's.
331  if (isInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
332  Op = ISD::SETNE;
333 
334  return ISD::CondCode(Op);
335 }
336 
337 /// getSetCCAndOperation - Return the result of a logical AND between different
338 /// comparisons of identical values: ((X op1 Y) & (X op2 Y)). This
339 /// function returns zero if it is not possible to represent the resultant
340 /// comparison.
342  bool isInteger) {
343  if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
344  // Cannot fold a signed setcc with an unsigned setcc.
345  return ISD::SETCC_INVALID;
346 
347  // Combine all of the condition bits.
348  ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
349 
350  // Canonicalize illegal integer setcc's.
351  if (isInteger) {
352  switch (Result) {
353  default: break;
354  case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
355  case ISD::SETOEQ: // SETEQ & SETU[LG]E
356  case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
357  case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
358  case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
359  }
360  }
361 
362  return Result;
363 }
364 
365 //===----------------------------------------------------------------------===//
366 // SDNode Profile Support
367 //===----------------------------------------------------------------------===//
368 
369 /// AddNodeIDOpcode - Add the node opcode to the NodeID data.
370 ///
371 static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
372  ID.AddInteger(OpC);
373 }
374 
375 /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
376 /// solely with their pointer.
378  ID.AddPointer(VTList.VTs);
379 }
380 
381 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
382 ///
384  ArrayRef<SDValue> Ops) {
385  for (auto& Op : Ops) {
386  ID.AddPointer(Op.getNode());
387  ID.AddInteger(Op.getResNo());
388  }
389 }
390 
391 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
392 ///
394  ArrayRef<SDUse> Ops) {
395  for (auto& Op : Ops) {
396  ID.AddPointer(Op.getNode());
397  ID.AddInteger(Op.getResNo());
398  }
399 }
400 /// Add logical or fast math flag values to FoldingSetNodeID value.
401 static void AddNodeIDFlags(FoldingSetNodeID &ID, unsigned Opcode,
402  const SDNodeFlags *Flags) {
403  if (!Flags || !isBinOpWithFlags(Opcode))
404  return;
405 
406  unsigned RawFlags = Flags->getRawFlags();
407  // If no flags are set, do not alter the ID. We must match the ID of nodes
408  // that were created without explicitly specifying flags. This also saves time
409  // and allows a gradual increase in API usage of the optional optimization
410  // flags.
411  if (RawFlags != 0)
412  ID.AddInteger(RawFlags);
413 }
414 
415 static void AddNodeIDFlags(FoldingSetNodeID &ID, const SDNode *N) {
416  if (auto *Node = dyn_cast<BinaryWithFlagsSDNode>(N))
417  AddNodeIDFlags(ID, Node->getOpcode(), &Node->Flags);
418 }
419 
420 static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
421  SDVTList VTList, ArrayRef<SDValue> OpList) {
422  AddNodeIDOpcode(ID, OpC);
423  AddNodeIDValueTypes(ID, VTList);
424  AddNodeIDOperands(ID, OpList);
425 }
426 
427 /// If this is an SDNode with special info, add this info to the NodeID data.
428 static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
429  switch (N->getOpcode()) {
431  case ISD::ExternalSymbol:
432  case ISD::MCSymbol:
433  llvm_unreachable("Should only be used on nodes with operands");
434  default: break; // Normal nodes don't need extra info.
435  case ISD::TargetConstant:
436  case ISD::Constant: {
437  const ConstantSDNode *C = cast<ConstantSDNode>(N);
439  ID.AddBoolean(C->isOpaque());
440  break;
441  }
443  case ISD::ConstantFP: {
444  ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
445  break;
446  }
448  case ISD::GlobalAddress:
450  case ISD::GlobalTLSAddress: {
451  const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
452  ID.AddPointer(GA->getGlobal());
453  ID.AddInteger(GA->getOffset());
454  ID.AddInteger(GA->getTargetFlags());
455  ID.AddInteger(GA->getAddressSpace());
456  break;
457  }
458  case ISD::BasicBlock:
459  ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
460  break;
461  case ISD::Register:
462  ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
463  break;
464  case ISD::RegisterMask:
465  ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
466  break;
467  case ISD::SRCVALUE:
468  ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
469  break;
470  case ISD::FrameIndex:
472  ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
473  break;
474  case ISD::JumpTable:
476  ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
477  ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
478  break;
479  case ISD::ConstantPool:
481  const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
482  ID.AddInteger(CP->getAlignment());
483  ID.AddInteger(CP->getOffset());
484  if (CP->isMachineConstantPoolEntry())
486  else
487  ID.AddPointer(CP->getConstVal());
488  ID.AddInteger(CP->getTargetFlags());
489  break;
490  }
491  case ISD::TargetIndex: {
492  const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
493  ID.AddInteger(TI->getIndex());
494  ID.AddInteger(TI->getOffset());
495  ID.AddInteger(TI->getTargetFlags());
496  break;
497  }
498  case ISD::LOAD: {
499  const LoadSDNode *LD = cast<LoadSDNode>(N);
500  ID.AddInteger(LD->getMemoryVT().getRawBits());
501  ID.AddInteger(LD->getRawSubclassData());
503  break;
504  }
505  case ISD::STORE: {
506  const StoreSDNode *ST = cast<StoreSDNode>(N);
507  ID.AddInteger(ST->getMemoryVT().getRawBits());
508  ID.AddInteger(ST->getRawSubclassData());
510  break;
511  }
514  case ISD::ATOMIC_SWAP:
518  case ISD::ATOMIC_LOAD_OR:
525  case ISD::ATOMIC_LOAD:
526  case ISD::ATOMIC_STORE: {
527  const AtomicSDNode *AT = cast<AtomicSDNode>(N);
528  ID.AddInteger(AT->getMemoryVT().getRawBits());
529  ID.AddInteger(AT->getRawSubclassData());
531  break;
532  }
533  case ISD::PREFETCH: {
534  const MemSDNode *PF = cast<MemSDNode>(N);
536  break;
537  }
538  case ISD::VECTOR_SHUFFLE: {
539  const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
540  for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
541  i != e; ++i)
542  ID.AddInteger(SVN->getMaskElt(i));
543  break;
544  }
546  case ISD::BlockAddress: {
547  const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
548  ID.AddPointer(BA->getBlockAddress());
549  ID.AddInteger(BA->getOffset());
550  ID.AddInteger(BA->getTargetFlags());
551  break;
552  }
553  } // end switch (N->getOpcode())
554 
555  AddNodeIDFlags(ID, N);
556 
557  // Target specific memory nodes could also have address spaces to check.
558  if (N->isTargetMemoryOpcode())
559  ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
560 }
561 
562 /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
563 /// data.
564 static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
565  AddNodeIDOpcode(ID, N->getOpcode());
566  // Add the return value info.
567  AddNodeIDValueTypes(ID, N->getVTList());
568  // Add the operand info.
569  AddNodeIDOperands(ID, N->ops());
570 
571  // Handle SDNode leafs with special info.
572  AddNodeIDCustom(ID, N);
573 }
574 
575 /// encodeMemSDNodeFlags - Generic routine for computing a value for use in
576 /// the CSE map that carries volatility, temporalness, indexing mode, and
577 /// extension/truncation information.
578 ///
579 static inline unsigned
581  bool isNonTemporal, bool isInvariant) {
582  assert((ConvType & 3) == ConvType &&
583  "ConvType may not require more than 2 bits!");
584  assert((AM & 7) == AM &&
585  "AM may not require more than 3 bits!");
586  return ConvType |
587  (AM << 2) |
588  (isVolatile << 5) |
589  (isNonTemporal << 6) |
590  (isInvariant << 7);
591 }
592 
593 //===----------------------------------------------------------------------===//
594 // SelectionDAG Class
595 //===----------------------------------------------------------------------===//
596 
597 /// doNotCSE - Return true if CSE should not be performed for this node.
598 static bool doNotCSE(SDNode *N) {
599  if (N->getValueType(0) == MVT::Glue)
600  return true; // Never CSE anything that produces a flag.
601 
602  switch (N->getOpcode()) {
603  default: break;
604  case ISD::HANDLENODE:
605  case ISD::EH_LABEL:
606  return true; // Never CSE these nodes.
607  }
608 
609  // Check that remaining values produced are not flags.
610  for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
611  if (N->getValueType(i) == MVT::Glue)
612  return true; // Never CSE anything that produces a flag.
613 
614  return false;
615 }
616 
617 /// RemoveDeadNodes - This method deletes all unreachable nodes in the
618 /// SelectionDAG.
620  // Create a dummy node (which is not added to allnodes), that adds a reference
621  // to the root node, preventing it from being deleted.
623 
624  SmallVector<SDNode*, 128> DeadNodes;
625 
626  // Add all obviously-dead nodes to the DeadNodes worklist.
627  for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I)
628  if (I->use_empty())
629  DeadNodes.push_back(I);
630 
631  RemoveDeadNodes(DeadNodes);
632 
633  // If the root changed (e.g. it was a dead load, update the root).
634  setRoot(Dummy.getValue());
635 }
636 
637 /// RemoveDeadNodes - This method deletes the unreachable nodes in the
638 /// given list, and any nodes that become unreachable as a result.
640 
641  // Process the worklist, deleting the nodes and adding their uses to the
642  // worklist.
643  while (!DeadNodes.empty()) {
644  SDNode *N = DeadNodes.pop_back_val();
645 
646  for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
647  DUL->NodeDeleted(N, nullptr);
648 
649  // Take the node out of the appropriate CSE map.
650  RemoveNodeFromCSEMaps(N);
651 
652  // Next, brutally remove the operand list. This is safe to do, as there are
653  // no cycles in the graph.
654  for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
655  SDUse &Use = *I++;
656  SDNode *Operand = Use.getNode();
657  Use.set(SDValue());
658 
659  // Now that we removed this operand, see if there are no uses of it left.
660  if (Operand->use_empty())
661  DeadNodes.push_back(Operand);
662  }
663 
664  DeallocateNode(N);
665  }
666 }
667 
669  SmallVector<SDNode*, 16> DeadNodes(1, N);
670 
671  // Create a dummy node that adds a reference to the root node, preventing
672  // it from being deleted. (This matters if the root is an operand of the
673  // dead node.)
675 
676  RemoveDeadNodes(DeadNodes);
677 }
678 
680  // First take this out of the appropriate CSE map.
681  RemoveNodeFromCSEMaps(N);
682 
683  // Finally, remove uses due to operands of this node, remove from the
684  // AllNodes list, and delete the node.
685  DeleteNodeNotInCSEMaps(N);
686 }
687 
688 void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
689  assert(N != AllNodes.begin() && "Cannot delete the entry node!");
690  assert(N->use_empty() && "Cannot delete a node that is not dead!");
691 
692  // Drop all of the operands and decrement used node's use counts.
693  N->DropOperands();
694 
695  DeallocateNode(N);
696 }
697 
698 void SDDbgInfo::erase(const SDNode *Node) {
699  DbgValMapType::iterator I = DbgValMap.find(Node);
700  if (I == DbgValMap.end())
701  return;
702  for (auto &Val: I->second)
703  Val->setIsInvalidated();
704  DbgValMap.erase(I);
705 }
706 
707 void SelectionDAG::DeallocateNode(SDNode *N) {
708  if (N->OperandsNeedDelete)
709  delete[] N->OperandList;
710 
711  // Set the opcode to DELETED_NODE to help catch bugs when node
712  // memory is reallocated.
713  N->NodeType = ISD::DELETED_NODE;
714 
715  NodeAllocator.Deallocate(AllNodes.remove(N));
716 
717  // If any of the SDDbgValue nodes refer to this SDNode, invalidate
718  // them and forget about that node.
719  DbgInfo->erase(N);
720 }
721 
722 #ifndef NDEBUG
723 /// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid.
724 static void VerifySDNode(SDNode *N) {
725  switch (N->getOpcode()) {
726  default:
727  break;
728  case ISD::BUILD_PAIR: {
729  EVT VT = N->getValueType(0);
730  assert(N->getNumValues() == 1 && "Too many results!");
731  assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
732  "Wrong return type!");
733  assert(N->getNumOperands() == 2 && "Wrong number of operands!");
734  assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
735  "Mismatched operand types!");
736  assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
737  "Wrong operand type!");
738  assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
739  "Wrong return type size");
740  break;
741  }
742  case ISD::BUILD_VECTOR: {
743  assert(N->getNumValues() == 1 && "Too many results!");
744  assert(N->getValueType(0).isVector() && "Wrong return type!");
745  assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
746  "Wrong number of operands!");
747  EVT EltVT = N->getValueType(0).getVectorElementType();
748  for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
749  assert((I->getValueType() == EltVT ||
750  (EltVT.isInteger() && I->getValueType().isInteger() &&
751  EltVT.bitsLE(I->getValueType()))) &&
752  "Wrong operand type!");
753  assert(I->getValueType() == N->getOperand(0).getValueType() &&
754  "Operands must all have the same type");
755  }
756  break;
757  }
758  }
759 }
760 #endif // NDEBUG
761 
762 /// \brief Insert a newly allocated node into the DAG.
763 ///
764 /// Handles insertion into the all nodes list and CSE map, as well as
765 /// verification and other common operations when a new node is allocated.
766 void SelectionDAG::InsertNode(SDNode *N) {
767  AllNodes.push_back(N);
768 #ifndef NDEBUG
769  VerifySDNode(N);
770 #endif
771 }
772 
773 /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
774 /// correspond to it. This is useful when we're about to delete or repurpose
775 /// the node. We don't want future request for structurally identical nodes
776 /// to return N anymore.
777 bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
778  bool Erased = false;
779  switch (N->getOpcode()) {
780  case ISD::HANDLENODE: return false; // noop.
781  case ISD::CONDCODE:
782  assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
783  "Cond code doesn't exist!");
784  Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
785  CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
786  break;
787  case ISD::ExternalSymbol:
788  Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
789  break;
791  ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
792  Erased = TargetExternalSymbols.erase(
793  std::pair<std::string,unsigned char>(ESN->getSymbol(),
794  ESN->getTargetFlags()));
795  break;
796  }
797  case ISD::MCSymbol: {
798  auto *MCSN = cast<MCSymbolSDNode>(N);
799  Erased = MCSymbols.erase(MCSN->getMCSymbol());
800  break;
801  }
802  case ISD::VALUETYPE: {
803  EVT VT = cast<VTSDNode>(N)->getVT();
804  if (VT.isExtended()) {
805  Erased = ExtendedValueTypeNodes.erase(VT);
806  } else {
807  Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
808  ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
809  }
810  break;
811  }
812  default:
813  // Remove it from the CSE Map.
814  assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
815  assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
816  Erased = CSEMap.RemoveNode(N);
817  break;
818  }
819 #ifndef NDEBUG
820  // Verify that the node was actually in one of the CSE maps, unless it has a
821  // flag result (which cannot be CSE'd) or is one of the special cases that are
822  // not subject to CSE.
823  if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
824  !N->isMachineOpcode() && !doNotCSE(N)) {
825  N->dump(this);
826  dbgs() << "\n";
827  llvm_unreachable("Node is not in map!");
828  }
829 #endif
830  return Erased;
831 }
832 
833 /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
834 /// maps and modified in place. Add it back to the CSE maps, unless an identical
835 /// node already exists, in which case transfer all its users to the existing
836 /// node. This transfer can potentially trigger recursive merging.
837 ///
838 void
839 SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
840  // For node types that aren't CSE'd, just act as if no identical node
841  // already exists.
842  if (!doNotCSE(N)) {
843  SDNode *Existing = CSEMap.GetOrInsertNode(N);
844  if (Existing != N) {
845  // If there was already an existing matching node, use ReplaceAllUsesWith
846  // to replace the dead one with the existing one. This can cause
847  // recursive merging of other unrelated nodes down the line.
848  ReplaceAllUsesWith(N, Existing);
849 
850  // N is now dead. Inform the listeners and delete it.
851  for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
852  DUL->NodeDeleted(N, Existing);
853  DeleteNodeNotInCSEMaps(N);
854  return;
855  }
856  }
857 
858  // If the node doesn't already exist, we updated it. Inform listeners.
859  for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
860  DUL->NodeUpdated(N);
861 }
862 
863 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
864 /// were replaced with those specified. If this node is never memoized,
865 /// return null, otherwise return a pointer to the slot it would take. If a
866 /// node already exists with these operands, the slot will be non-null.
867 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
868  void *&InsertPos) {
869  if (doNotCSE(N))
870  return nullptr;
871 
872  SDValue Ops[] = { Op };
874  AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
875  AddNodeIDCustom(ID, N);
876  SDNode *Node = FindNodeOrInsertPos(ID, N->getDebugLoc(), InsertPos);
877  return Node;
878 }
879 
880 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
881 /// were replaced with those specified. If this node is never memoized,
882 /// return null, otherwise return a pointer to the slot it would take. If a
883 /// node already exists with these operands, the slot will be non-null.
884 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
885  SDValue Op1, SDValue Op2,
886  void *&InsertPos) {
887  if (doNotCSE(N))
888  return nullptr;
889 
890  SDValue Ops[] = { Op1, Op2 };
892  AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
893  AddNodeIDCustom(ID, N);
894  SDNode *Node = FindNodeOrInsertPos(ID, N->getDebugLoc(), InsertPos);
895  return Node;
896 }
897 
898 
899 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
900 /// were replaced with those specified. If this node is never memoized,
901 /// return null, otherwise return a pointer to the slot it would take. If a
902 /// node already exists with these operands, the slot will be non-null.
903 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
904  void *&InsertPos) {
905  if (doNotCSE(N))
906  return nullptr;
907 
909  AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
910  AddNodeIDCustom(ID, N);
911  SDNode *Node = FindNodeOrInsertPos(ID, N->getDebugLoc(), InsertPos);
912  return Node;
913 }
914 
915 /// getEVTAlignment - Compute the default alignment value for the
916 /// given type.
917 ///
918 unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
919  Type *Ty = VT == MVT::iPTR ?
921  VT.getTypeForEVT(*getContext());
922 
923  return getDataLayout().getABITypeAlignment(Ty);
924 }
925 
926 // EntryNode could meaningfully have debug info if we can find it...
927 SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
928  : TM(tm), TSI(nullptr), TLI(nullptr), OptLevel(OL),
929  EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
931  UpdateListeners(nullptr) {
932  AllNodes.push_back(&EntryNode);
933  DbgInfo = new SDDbgInfo();
934 }
935 
937  MF = &mf;
940  Context = &mf.getFunction()->getContext();
941 }
942 
944  assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
945  allnodes_clear();
946  delete DbgInfo;
947 }
948 
949 void SelectionDAG::allnodes_clear() {
950  assert(&*AllNodes.begin() == &EntryNode);
951  AllNodes.remove(AllNodes.begin());
952  while (!AllNodes.empty())
953  DeallocateNode(AllNodes.begin());
954 }
955 
956 BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL,
957  SDVTList VTs, SDValue N1,
958  SDValue N2,
959  const SDNodeFlags *Flags) {
960  if (isBinOpWithFlags(Opcode)) {
961  // If no flags were passed in, use a default flags object.
962  SDNodeFlags F;
963  if (Flags == nullptr)
964  Flags = &F;
965 
966  BinaryWithFlagsSDNode *FN = new (NodeAllocator) BinaryWithFlagsSDNode(
967  Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2, *Flags);
968 
969  return FN;
970  }
971 
972  BinarySDNode *N = new (NodeAllocator)
973  BinarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
974  return N;
975 }
976 
977 SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
978  void *&InsertPos) {
979  SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
980  if (N) {
981  switch (N->getOpcode()) {
982  default: break;
983  case ISD::Constant:
984  case ISD::ConstantFP:
985  llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
986  "debug location. Use another overload.");
987  }
988  }
989  return N;
990 }
991 
992 SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
993  DebugLoc DL, void *&InsertPos) {
994  SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
995  if (N) {
996  switch (N->getOpcode()) {
997  default: break; // Process only regular (non-target) constant nodes.
998  case ISD::Constant:
999  case ISD::ConstantFP:
1000  // Erase debug location from the node if the node is used at several
1001  // different places to do not propagate one location to all uses as it
1002  // leads to incorrect debug info.
1003  if (N->getDebugLoc() != DL)
1004  N->setDebugLoc(DebugLoc());
1005  break;
1006  }
1007  }
1008  return N;
1009 }
1010 
1012  allnodes_clear();
1013  OperandAllocator.Reset();
1014  CSEMap.clear();
1015 
1016  ExtendedValueTypeNodes.clear();
1017  ExternalSymbols.clear();
1018  TargetExternalSymbols.clear();
1019  MCSymbols.clear();
1020  std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
1021  static_cast<CondCodeSDNode*>(nullptr));
1022  std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
1023  static_cast<SDNode*>(nullptr));
1024 
1025  EntryNode.UseList = nullptr;
1026  AllNodes.push_back(&EntryNode);
1027  Root = getEntryNode();
1028  DbgInfo->clear();
1029 }
1030 
1032  return VT.bitsGT(Op.getValueType()) ?
1033  getNode(ISD::ANY_EXTEND, DL, VT, Op) :
1034  getNode(ISD::TRUNCATE, DL, VT, Op);
1035 }
1036 
1038  return VT.bitsGT(Op.getValueType()) ?
1039  getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
1040  getNode(ISD::TRUNCATE, DL, VT, Op);
1041 }
1042 
1044  return VT.bitsGT(Op.getValueType()) ?
1045  getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
1046  getNode(ISD::TRUNCATE, DL, VT, Op);
1047 }
1048 
1050  EVT OpVT) {
1051  if (VT.bitsLE(Op.getValueType()))
1052  return getNode(ISD::TRUNCATE, SL, VT, Op);
1053 
1055  return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
1056 }
1057 
1059  assert(!VT.isVector() &&
1060  "getZeroExtendInReg should use the vector element type instead of "
1061  "the vector type!");
1062  if (Op.getValueType() == VT) return Op;
1063  unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
1064  APInt Imm = APInt::getLowBitsSet(BitWidth,
1065  VT.getSizeInBits());
1066  return getNode(ISD::AND, DL, Op.getValueType(), Op,
1067  getConstant(Imm, DL, Op.getValueType()));
1068 }
1069 
1071  assert(VT.isVector() && "This DAG node is restricted to vector types.");
1072  assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
1073  "The sizes of the input and result must match in order to perform the "
1074  "extend in-register.");
1076  "The destination vector type must have fewer lanes than the input.");
1077  return getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Op);
1078 }
1079 
1081  assert(VT.isVector() && "This DAG node is restricted to vector types.");
1082  assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
1083  "The sizes of the input and result must match in order to perform the "
1084  "extend in-register.");
1086  "The destination vector type must have fewer lanes than the input.");
1087  return getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Op);
1088 }
1089 
1091  assert(VT.isVector() && "This DAG node is restricted to vector types.");
1092  assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
1093  "The sizes of the input and result must match in order to perform the "
1094  "extend in-register.");
1096  "The destination vector type must have fewer lanes than the input.");
1097  return getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Op);
1098 }
1099 
1100 /// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1101 ///
1103  EVT EltVT = VT.getScalarType();
1104  SDValue NegOne =
1106  return getNode(ISD::XOR, DL, VT, Val, NegOne);
1107 }
1108 
1110  EVT EltVT = VT.getScalarType();
1111  SDValue TrueValue;
1112  switch (TLI->getBooleanContents(VT)) {
1115  TrueValue = getConstant(1, DL, VT);
1116  break;
1118  TrueValue = getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL,
1119  VT);
1120  break;
1121  }
1122  return getNode(ISD::XOR, DL, VT, Val, TrueValue);
1123 }
1124 
1125 SDValue SelectionDAG::getConstant(uint64_t Val, SDLoc DL, EVT VT, bool isT,
1126  bool isO) {
1127  EVT EltVT = VT.getScalarType();
1128  assert((EltVT.getSizeInBits() >= 64 ||
1129  (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
1130  "getConstant with a uint64_t value that doesn't fit in the type!");
1131  return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO);
1132 }
1133 
1134 SDValue SelectionDAG::getConstant(const APInt &Val, SDLoc DL, EVT VT, bool isT,
1135  bool isO)
1136 {
1137  return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO);
1138 }
1139 
1141  bool isT, bool isO) {
1142  assert(VT.isInteger() && "Cannot create FP integer constant!");
1143 
1144  EVT EltVT = VT.getScalarType();
1145  const ConstantInt *Elt = &Val;
1146 
1147  // In some cases the vector type is legal but the element type is illegal and
1148  // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1149  // inserted value (the type does not need to match the vector element type).
1150  // Any extra bits introduced will be truncated away.
1151  if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
1153  EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1154  APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits());
1155  Elt = ConstantInt::get(*getContext(), NewVal);
1156  }
1157  // In other cases the element type is illegal and needs to be expanded, for
1158  // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1159  // the value into n parts and use a vector type with n-times the elements.
1160  // Then bitcast to the type requested.
1161  // Legalizing constants too early makes the DAGCombiner's job harder so we
1162  // only legalize if the DAG tells us we must produce legal types.
1163  else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1164  TLI->getTypeAction(*getContext(), EltVT) ==
1166  APInt NewVal = Elt->getValue();
1167  EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1168  unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1169  unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1170  EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
1171 
1172  // Check the temporary vector is the correct size. If this fails then
1173  // getTypeToTransformTo() probably returned a type whose size (in bits)
1174  // isn't a power-of-2 factor of the requested type size.
1175  assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1176 
1177  SmallVector<SDValue, 2> EltParts;
1178  for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
1179  EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
1180  .trunc(ViaEltSizeInBits), DL,
1181  ViaEltVT, isT, isO));
1182  }
1183 
1184  // EltParts is currently in little endian order. If we actually want
1185  // big-endian order then reverse it now.
1186  if (getDataLayout().isBigEndian())
1187  std::reverse(EltParts.begin(), EltParts.end());
1188 
1189  // The elements must be reversed when the element order is different
1190  // to the endianness of the elements (because the BITCAST is itself a
1191  // vector shuffle in this situation). However, we do not need any code to
1192  // perform this reversal because getConstant() is producing a vector
1193  // splat.
1194  // This situation occurs in MIPS MSA.
1195 
1197  for (unsigned i = 0; i < VT.getVectorNumElements(); ++i)
1198  Ops.insert(Ops.end(), EltParts.begin(), EltParts.end());
1199 
1200  SDValue Result = getNode(ISD::BITCAST, SDLoc(), VT,
1201  getNode(ISD::BUILD_VECTOR, SDLoc(), ViaVecVT,
1202  Ops));
1203  return Result;
1204  }
1205 
1206  assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1207  "APInt size does not match type size!");
1208  unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1210  AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1211  ID.AddPointer(Elt);
1212  ID.AddBoolean(isO);
1213  void *IP = nullptr;
1214  SDNode *N = nullptr;
1215  if ((N = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)))
1216  if (!VT.isVector())
1217  return SDValue(N, 0);
1218 
1219  if (!N) {
1220  N = new (NodeAllocator) ConstantSDNode(isT, isO, Elt, DL.getDebugLoc(),
1221  EltVT);
1222  CSEMap.InsertNode(N, IP);
1223  InsertNode(N);
1224  }
1225 
1226  SDValue Result(N, 0);
1227  if (VT.isVector()) {
1229  Ops.assign(VT.getVectorNumElements(), Result);
1230  Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
1231  }
1232  return Result;
1233 }
1234 
1235 SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, SDLoc DL, bool isTarget) {
1236  return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget);
1237 }
1238 
1240  bool isTarget) {
1241  return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget);
1242 }
1243 
1245  bool isTarget){
1246  assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1247 
1248  EVT EltVT = VT.getScalarType();
1249 
1250  // Do the map lookup using the actual bit pattern for the floating point
1251  // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1252  // we don't have issues with SNANs.
1253  unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1255  AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1256  ID.AddPointer(&V);
1257  void *IP = nullptr;
1258  SDNode *N = nullptr;
1259  if ((N = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)))
1260  if (!VT.isVector())
1261  return SDValue(N, 0);
1262 
1263  if (!N) {
1264  N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, DL.getDebugLoc(),
1265  EltVT);
1266  CSEMap.InsertNode(N, IP);
1267  InsertNode(N);
1268  }
1269 
1270  SDValue Result(N, 0);
1271  if (VT.isVector()) {
1273  Ops.assign(VT.getVectorNumElements(), Result);
1274  Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
1275  }
1276  return Result;
1277 }
1278 
1280  bool isTarget) {
1281  EVT EltVT = VT.getScalarType();
1282  if (EltVT==MVT::f32)
1283  return getConstantFP(APFloat((float)Val), DL, VT, isTarget);
1284  else if (EltVT==MVT::f64)
1285  return getConstantFP(APFloat(Val), DL, VT, isTarget);
1286  else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 ||
1287  EltVT==MVT::f16) {
1288  bool ignored;
1289  APFloat apf = APFloat(Val);
1291  &ignored);
1292  return getConstantFP(apf, DL, VT, isTarget);
1293  } else
1294  llvm_unreachable("Unsupported type in getConstantFP");
1295 }
1296 
1298  EVT VT, int64_t Offset,
1299  bool isTargetGA,
1300  unsigned char TargetFlags) {
1301  assert((TargetFlags == 0 || isTargetGA) &&
1302  "Cannot set target flags on target-independent globals");
1303 
1304  // Truncate (with sign-extension) the offset value to the pointer size.
1305  unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1306  if (BitWidth < 64)
1307  Offset = SignExtend64(Offset, BitWidth);
1308 
1309  unsigned Opc;
1310  if (GV->isThreadLocal())
1312  else
1313  Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1314 
1316  AddNodeIDNode(ID, Opc, getVTList(VT), None);
1317  ID.AddPointer(GV);
1318  ID.AddInteger(Offset);
1319  ID.AddInteger(TargetFlags);
1320  ID.AddInteger(GV->getType()->getAddressSpace());
1321  void *IP = nullptr;
1322  if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
1323  return SDValue(E, 0);
1324 
1325  SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL.getIROrder(),
1326  DL.getDebugLoc(), GV, VT,
1327  Offset, TargetFlags);
1328  CSEMap.InsertNode(N, IP);
1329  InsertNode(N);
1330  return SDValue(N, 0);
1331 }
1332 
1333 SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1334  unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1336  AddNodeIDNode(ID, Opc, getVTList(VT), None);
1337  ID.AddInteger(FI);
1338  void *IP = nullptr;
1339  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1340  return SDValue(E, 0);
1341 
1342  SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget);
1343  CSEMap.InsertNode(N, IP);
1344  InsertNode(N);
1345  return SDValue(N, 0);
1346 }
1347 
1348 SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1349  unsigned char TargetFlags) {
1350  assert((TargetFlags == 0 || isTarget) &&
1351  "Cannot set target flags on target-independent jump tables");
1352  unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1354  AddNodeIDNode(ID, Opc, getVTList(VT), None);
1355  ID.AddInteger(JTI);
1356  ID.AddInteger(TargetFlags);
1357  void *IP = nullptr;
1358  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1359  return SDValue(E, 0);
1360 
1361  SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget,
1362  TargetFlags);
1363  CSEMap.InsertNode(N, IP);
1364  InsertNode(N);
1365  return SDValue(N, 0);
1366 }
1367 
1369  unsigned Alignment, int Offset,
1370  bool isTarget,
1371  unsigned char TargetFlags) {
1372  assert((TargetFlags == 0 || isTarget) &&
1373  "Cannot set target flags on target-independent globals");
1374  if (Alignment == 0)
1375  Alignment = getDataLayout().getPrefTypeAlignment(C->getType());
1376  unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1378  AddNodeIDNode(ID, Opc, getVTList(VT), None);
1379  ID.AddInteger(Alignment);
1380  ID.AddInteger(Offset);
1381  ID.AddPointer(C);
1382  ID.AddInteger(TargetFlags);
1383  void *IP = nullptr;
1384  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1385  return SDValue(E, 0);
1386 
1387  SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
1388  Alignment, TargetFlags);
1389  CSEMap.InsertNode(N, IP);
1390  InsertNode(N);
1391  return SDValue(N, 0);
1392 }
1393 
1394 
1396  unsigned Alignment, int Offset,
1397  bool isTarget,
1398  unsigned char TargetFlags) {
1399  assert((TargetFlags == 0 || isTarget) &&
1400  "Cannot set target flags on target-independent globals");
1401  if (Alignment == 0)
1402  Alignment = getDataLayout().getPrefTypeAlignment(C->getType());
1403  unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1405  AddNodeIDNode(ID, Opc, getVTList(VT), None);
1406  ID.AddInteger(Alignment);
1407  ID.AddInteger(Offset);
1408  C->addSelectionDAGCSEId(ID);
1409  ID.AddInteger(TargetFlags);
1410  void *IP = nullptr;
1411  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1412  return SDValue(E, 0);
1413 
1414  SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
1415  Alignment, TargetFlags);
1416  CSEMap.InsertNode(N, IP);
1417  InsertNode(N);
1418  return SDValue(N, 0);
1419 }
1420 
1421 SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
1422  unsigned char TargetFlags) {
1425  ID.AddInteger(Index);
1426  ID.AddInteger(Offset);
1427  ID.AddInteger(TargetFlags);
1428  void *IP = nullptr;
1429  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1430  return SDValue(E, 0);
1431 
1432  SDNode *N = new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset,
1433  TargetFlags);
1434  CSEMap.InsertNode(N, IP);
1435  InsertNode(N);
1436  return SDValue(N, 0);
1437 }
1438 
1442  ID.AddPointer(MBB);
1443  void *IP = nullptr;
1444  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1445  return SDValue(E, 0);
1446 
1447  SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB);
1448  CSEMap.InsertNode(N, IP);
1449  InsertNode(N);
1450  return SDValue(N, 0);
1451 }
1452 
1454  if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
1455  ValueTypeNodes.size())
1456  ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
1457 
1458  SDNode *&N = VT.isExtended() ?
1459  ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
1460 
1461  if (N) return SDValue(N, 0);
1462  N = new (NodeAllocator) VTSDNode(VT);
1463  InsertNode(N);
1464  return SDValue(N, 0);
1465 }
1466 
1468  SDNode *&N = ExternalSymbols[Sym];
1469  if (N) return SDValue(N, 0);
1470  N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT);
1471  InsertNode(N);
1472  return SDValue(N, 0);
1473 }
1474 
1476  SDNode *&N = MCSymbols[Sym];
1477  if (N)
1478  return SDValue(N, 0);
1479  N = new (NodeAllocator) MCSymbolSDNode(Sym, VT);
1480  InsertNode(N);
1481  return SDValue(N, 0);
1482 }
1483 
1485  unsigned char TargetFlags) {
1486  SDNode *&N =
1487  TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym,
1488  TargetFlags)];
1489  if (N) return SDValue(N, 0);
1490  N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT);
1491  InsertNode(N);
1492  return SDValue(N, 0);
1493 }
1494 
1496  if ((unsigned)Cond >= CondCodeNodes.size())
1497  CondCodeNodes.resize(Cond+1);
1498 
1499  if (!CondCodeNodes[Cond]) {
1500  CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond);
1501  CondCodeNodes[Cond] = N;
1502  InsertNode(N);
1503  }
1504 
1505  return SDValue(CondCodeNodes[Cond], 0);
1506 }
1507 
1508 // commuteShuffle - swaps the values of N1 and N2, and swaps all indices in
1509 // the shuffle mask M that point at N1 to point at N2, and indices that point
1510 // N2 to point at N1.
1512  std::swap(N1, N2);
1514 }
1515 
1517  SDValue N2, const int *Mask) {
1518  assert(VT == N1.getValueType() && VT == N2.getValueType() &&
1519  "Invalid VECTOR_SHUFFLE");
1520 
1521  // Canonicalize shuffle undef, undef -> undef
1522  if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF)
1523  return getUNDEF(VT);
1524 
1525  // Validate that all indices in Mask are within the range of the elements
1526  // input to the shuffle.
1527  unsigned NElts = VT.getVectorNumElements();
1528  SmallVector<int, 8> MaskVec;
1529  for (unsigned i = 0; i != NElts; ++i) {
1530  assert(Mask[i] < (int)(NElts * 2) && "Index out of range");
1531  MaskVec.push_back(Mask[i]);
1532  }
1533 
1534  // Canonicalize shuffle v, v -> v, undef
1535  if (N1 == N2) {
1536  N2 = getUNDEF(VT);
1537  for (unsigned i = 0; i != NElts; ++i)
1538  if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts;
1539  }
1540 
1541  // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
1542  if (N1.getOpcode() == ISD::UNDEF)
1543  commuteShuffle(N1, N2, MaskVec);
1544 
1545  // If shuffling a splat, try to blend the splat instead. We do this here so
1546  // that even when this arises during lowering we don't have to re-handle it.
1547  auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
1548  BitVector UndefElements;
1549  SDValue Splat = BV->getSplatValue(&UndefElements);
1550  if (!Splat)
1551  return;
1552 
1553  for (int i = 0; i < (int)NElts; ++i) {
1554  if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + (int)NElts))
1555  continue;
1556 
1557  // If this input comes from undef, mark it as such.
1558  if (UndefElements[MaskVec[i] - Offset]) {
1559  MaskVec[i] = -1;
1560  continue;
1561  }
1562 
1563  // If we can blend a non-undef lane, use that instead.
1564  if (!UndefElements[i])
1565  MaskVec[i] = i + Offset;
1566  }
1567  };
1568  if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
1569  BlendSplat(N1BV, 0);
1570  if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
1571  BlendSplat(N2BV, NElts);
1572 
1573  // Canonicalize all index into lhs, -> shuffle lhs, undef
1574  // Canonicalize all index into rhs, -> shuffle rhs, undef
1575  bool AllLHS = true, AllRHS = true;
1576  bool N2Undef = N2.getOpcode() == ISD::UNDEF;
1577  for (unsigned i = 0; i != NElts; ++i) {
1578  if (MaskVec[i] >= (int)NElts) {
1579  if (N2Undef)
1580  MaskVec[i] = -1;
1581  else
1582  AllLHS = false;
1583  } else if (MaskVec[i] >= 0) {
1584  AllRHS = false;
1585  }
1586  }
1587  if (AllLHS && AllRHS)
1588  return getUNDEF(VT);
1589  if (AllLHS && !N2Undef)
1590  N2 = getUNDEF(VT);
1591  if (AllRHS) {
1592  N1 = getUNDEF(VT);
1593  commuteShuffle(N1, N2, MaskVec);
1594  }
1595  // Reset our undef status after accounting for the mask.
1596  N2Undef = N2.getOpcode() == ISD::UNDEF;
1597  // Re-check whether both sides ended up undef.
1598  if (N1.getOpcode() == ISD::UNDEF && N2Undef)
1599  return getUNDEF(VT);
1600 
1601  // If Identity shuffle return that node.
1602  bool Identity = true, AllSame = true;
1603  for (unsigned i = 0; i != NElts; ++i) {
1604  if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false;
1605  if (MaskVec[i] != MaskVec[0]) AllSame = false;
1606  }
1607  if (Identity && NElts)
1608  return N1;
1609 
1610  // Shuffling a constant splat doesn't change the result.
1611  if (N2Undef) {
1612  SDValue V = N1;
1613 
1614  // Look through any bitcasts. We check that these don't change the number
1615  // (and size) of elements and just changes their types.
1616  while (V.getOpcode() == ISD::BITCAST)
1617  V = V->getOperand(0);
1618 
1619  // A splat should always show up as a build vector node.
1620  if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
1621  BitVector UndefElements;
1622  SDValue Splat = BV->getSplatValue(&UndefElements);
1623  // If this is a splat of an undef, shuffling it is also undef.
1624  if (Splat && Splat.getOpcode() == ISD::UNDEF)
1625  return getUNDEF(VT);
1626 
1627  bool SameNumElts =
1629 
1630  // We only have a splat which can skip shuffles if there is a splatted
1631  // value and no undef lanes rearranged by the shuffle.
1632  if (Splat && UndefElements.none()) {
1633  // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
1634  // number of elements match or the value splatted is a zero constant.
1635  if (SameNumElts)
1636  return N1;
1637  if (auto *C = dyn_cast<ConstantSDNode>(Splat))
1638  if (C->isNullValue())
1639  return N1;
1640  }
1641 
1642  // If the shuffle itself creates a splat, build the vector directly.
1643  if (AllSame && SameNumElts) {
1644  const SDValue &Splatted = BV->getOperand(MaskVec[0]);
1645  SmallVector<SDValue, 8> Ops(NElts, Splatted);
1646 
1647  EVT BuildVT = BV->getValueType(0);
1648  SDValue NewBV = getNode(ISD::BUILD_VECTOR, dl, BuildVT, Ops);
1649 
1650  // We may have jumped through bitcasts, so the type of the
1651  // BUILD_VECTOR may not match the type of the shuffle.
1652  if (BuildVT != VT)
1653  NewBV = getNode(ISD::BITCAST, dl, VT, NewBV);
1654  return NewBV;
1655  }
1656  }
1657  }
1658 
1660  SDValue Ops[2] = { N1, N2 };
1662  for (unsigned i = 0; i != NElts; ++i)
1663  ID.AddInteger(MaskVec[i]);
1664 
1665  void* IP = nullptr;
1666  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP))
1667  return SDValue(E, 0);
1668 
1669  // Allocate the mask array for the node out of the BumpPtrAllocator, since
1670  // SDNode doesn't have access to it. This memory will be "leaked" when
1671  // the node is deallocated, but recovered when the NodeAllocator is released.
1672  int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
1673  memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int));
1674 
1675  ShuffleVectorSDNode *N =
1676  new (NodeAllocator) ShuffleVectorSDNode(VT, dl.getIROrder(),
1677  dl.getDebugLoc(), N1, N2,
1678  MaskAlloc);
1679  CSEMap.InsertNode(N, IP);
1680  InsertNode(N);
1681  return SDValue(N, 0);
1682 }
1683 
1685  MVT VT = SV.getSimpleValueType(0);
1686  SmallVector<int, 8> MaskVec(SV.getMask().begin(), SV.getMask().end());
1688 
1689  SDValue Op0 = SV.getOperand(0);
1690  SDValue Op1 = SV.getOperand(1);
1691  return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, &MaskVec[0]);
1692 }
1693 
1695  SDValue Val, SDValue DTy,
1696  SDValue STy, SDValue Rnd, SDValue Sat,
1697  ISD::CvtCode Code) {
1698  // If the src and dest types are the same and the conversion is between
1699  // integer types of the same sign or two floats, no conversion is necessary.
1700  if (DTy == STy &&
1701  (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF))
1702  return Val;
1703 
1705  SDValue Ops[] = { Val, DTy, STy, Rnd, Sat };
1707  void* IP = nullptr;
1708  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP))
1709  return SDValue(E, 0);
1710 
1711  CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl.getIROrder(),
1712  dl.getDebugLoc(),
1713  Ops, Code);
1714  CSEMap.InsertNode(N, IP);
1715  InsertNode(N);
1716  return SDValue(N, 0);
1717 }
1718 
1722  ID.AddInteger(RegNo);
1723  void *IP = nullptr;
1724  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1725  return SDValue(E, 0);
1726 
1727  SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT);
1728  CSEMap.InsertNode(N, IP);
1729  InsertNode(N);
1730  return SDValue(N, 0);
1731 }
1732 
1733 SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
1736  ID.AddPointer(RegMask);
1737  void *IP = nullptr;
1738  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1739  return SDValue(E, 0);
1740 
1741  SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask);
1742  CSEMap.InsertNode(N, IP);
1743  InsertNode(N);
1744  return SDValue(N, 0);
1745 }
1746 
1749  SDValue Ops[] = { Root };
1751  ID.AddPointer(Label);
1752  void *IP = nullptr;
1753  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1754  return SDValue(E, 0);
1755 
1756  SDNode *N = new (NodeAllocator) EHLabelSDNode(dl.getIROrder(),
1757  dl.getDebugLoc(), Root, Label);
1758  CSEMap.InsertNode(N, IP);
1759  InsertNode(N);
1760  return SDValue(N, 0);
1761 }
1762 
1763 
1765  int64_t Offset,
1766  bool isTarget,
1767  unsigned char TargetFlags) {
1768  unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
1769 
1771  AddNodeIDNode(ID, Opc, getVTList(VT), None);
1772  ID.AddPointer(BA);
1773  ID.AddInteger(Offset);
1774  ID.AddInteger(TargetFlags);
1775  void *IP = nullptr;
1776  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1777  return SDValue(E, 0);
1778 
1779  SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset,
1780  TargetFlags);
1781  CSEMap.InsertNode(N, IP);
1782  InsertNode(N);
1783  return SDValue(N, 0);
1784 }
1785 
1787  assert((!V || V->getType()->isPointerTy()) &&
1788  "SrcValue is not a pointer?");
1789 
1792  ID.AddPointer(V);
1793 
1794  void *IP = nullptr;
1795  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1796  return SDValue(E, 0);
1797 
1798  SDNode *N = new (NodeAllocator) SrcValueSDNode(V);
1799  CSEMap.InsertNode(N, IP);
1800  InsertNode(N);
1801  return SDValue(N, 0);
1802 }
1803 
1804 /// getMDNode - Return an MDNodeSDNode which holds an MDNode.
1808  ID.AddPointer(MD);
1809 
1810  void *IP = nullptr;
1811  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1812  return SDValue(E, 0);
1813 
1814  SDNode *N = new (NodeAllocator) MDNodeSDNode(MD);
1815  CSEMap.InsertNode(N, IP);
1816  InsertNode(N);
1817  return SDValue(N, 0);
1818 }
1819 
1821  if (VT == V.getValueType())
1822  return V;
1823 
1824  return getNode(ISD::BITCAST, SDLoc(V), VT, V);
1825 }
1826 
1827 /// getAddrSpaceCast - Return an AddrSpaceCastSDNode.
1829  unsigned SrcAS, unsigned DestAS) {
1830  SDValue Ops[] = {Ptr};
1833  ID.AddInteger(SrcAS);
1834  ID.AddInteger(DestAS);
1835 
1836  void *IP = nullptr;
1837  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP))
1838  return SDValue(E, 0);
1839 
1840  SDNode *N = new (NodeAllocator) AddrSpaceCastSDNode(dl.getIROrder(),
1841  dl.getDebugLoc(),
1842  VT, Ptr, SrcAS, DestAS);
1843  CSEMap.InsertNode(N, IP);
1844  InsertNode(N);
1845  return SDValue(N, 0);
1846 }
1847 
1848 /// getShiftAmountOperand - Return the specified value casted to
1849 /// the target's desired shift amount type.
1851  EVT OpTy = Op.getValueType();
1852  EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout());
1853  if (OpTy == ShTy || OpTy.isVector()) return Op;
1854 
1855  ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ? ISD::TRUNCATE : ISD::ZERO_EXTEND;
1856  return getNode(Opcode, SDLoc(Op), ShTy, Op);
1857 }
1858 
1859 /// CreateStackTemporary - Create a stack temporary, suitable for holding the
1860 /// specified value type.
1863  unsigned ByteSize = VT.getStoreSize();
1864  Type *Ty = VT.getTypeForEVT(*getContext());
1865  unsigned StackAlign =
1866  std::max((unsigned)getDataLayout().getPrefTypeAlignment(Ty), minAlign);
1867 
1868  int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false);
1869  return getFrameIndex(FrameIdx, TLI->getPointerTy(getDataLayout()));
1870 }
1871 
1872 /// CreateStackTemporary - Create a stack temporary suitable for holding
1873 /// either of the specified value types.
1875  unsigned Bytes = std::max(VT1.getStoreSizeInBits(),
1876  VT2.getStoreSizeInBits())/8;
1877  Type *Ty1 = VT1.getTypeForEVT(*getContext());
1878  Type *Ty2 = VT2.getTypeForEVT(*getContext());
1879  const DataLayout &DL = getDataLayout();
1880  unsigned Align =
1881  std::max(DL.getPrefTypeAlignment(Ty1), DL.getPrefTypeAlignment(Ty2));
1882 
1884  int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false);
1885  return getFrameIndex(FrameIdx, TLI->getPointerTy(getDataLayout()));
1886 }
1887 
1889  SDValue N2, ISD::CondCode Cond, SDLoc dl) {
1890  // These setcc operations always fold.
1891  switch (Cond) {
1892  default: break;
1893  case ISD::SETFALSE:
1894  case ISD::SETFALSE2: return getConstant(0, dl, VT);
1895  case ISD::SETTRUE:
1896  case ISD::SETTRUE2: {
1898  TLI->getBooleanContents(N1->getValueType(0));
1899  return getConstant(
1900  Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, dl,
1901  VT);
1902  }
1903 
1904  case ISD::SETOEQ:
1905  case ISD::SETOGT:
1906  case ISD::SETOGE:
1907  case ISD::SETOLT:
1908  case ISD::SETOLE:
1909  case ISD::SETONE:
1910  case ISD::SETO:
1911  case ISD::SETUO:
1912  case ISD::SETUEQ:
1913  case ISD::SETUNE:
1914  assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!");
1915  break;
1916  }
1917 
1918  if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2)) {
1919  const APInt &C2 = N2C->getAPIntValue();
1920  if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
1921  const APInt &C1 = N1C->getAPIntValue();
1922 
1923  switch (Cond) {
1924  default: llvm_unreachable("Unknown integer setcc!");
1925  case ISD::SETEQ: return getConstant(C1 == C2, dl, VT);
1926  case ISD::SETNE: return getConstant(C1 != C2, dl, VT);
1927  case ISD::SETULT: return getConstant(C1.ult(C2), dl, VT);
1928  case ISD::SETUGT: return getConstant(C1.ugt(C2), dl, VT);
1929  case ISD::SETULE: return getConstant(C1.ule(C2), dl, VT);
1930  case ISD::SETUGE: return getConstant(C1.uge(C2), dl, VT);
1931  case ISD::SETLT: return getConstant(C1.slt(C2), dl, VT);
1932  case ISD::SETGT: return getConstant(C1.sgt(C2), dl, VT);
1933  case ISD::SETLE: return getConstant(C1.sle(C2), dl, VT);
1934  case ISD::SETGE: return getConstant(C1.sge(C2), dl, VT);
1935  }
1936  }
1937  }
1938  if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1)) {
1939  if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2)) {
1940  APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF());
1941  switch (Cond) {
1942  default: break;
1943  case ISD::SETEQ: if (R==APFloat::cmpUnordered)
1944  return getUNDEF(VT);
1945  // fall through
1946  case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, dl, VT);
1947  case ISD::SETNE: if (R==APFloat::cmpUnordered)
1948  return getUNDEF(VT);
1949  // fall through
1951  R==APFloat::cmpLessThan, dl, VT);
1952  case ISD::SETLT: if (R==APFloat::cmpUnordered)
1953  return getUNDEF(VT);
1954  // fall through
1955  case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, dl, VT);
1956  case ISD::SETGT: if (R==APFloat::cmpUnordered)
1957  return getUNDEF(VT);
1958  // fall through
1959  case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, dl, VT);
1960  case ISD::SETLE: if (R==APFloat::cmpUnordered)
1961  return getUNDEF(VT);
1962  // fall through
1963  case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan ||
1964  R==APFloat::cmpEqual, dl, VT);
1965  case ISD::SETGE: if (R==APFloat::cmpUnordered)
1966  return getUNDEF(VT);
1967  // fall through
1969  R==APFloat::cmpEqual, dl, VT);
1970  case ISD::SETO: return getConstant(R!=APFloat::cmpUnordered, dl, VT);
1971  case ISD::SETUO: return getConstant(R==APFloat::cmpUnordered, dl, VT);
1972  case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered ||
1973  R==APFloat::cmpEqual, dl, VT);
1974  case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, dl, VT);
1975  case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered ||
1976  R==APFloat::cmpLessThan, dl, VT);
1978  R==APFloat::cmpUnordered, dl, VT);
1979  case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, dl, VT);
1980  case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, dl, VT);
1981  }
1982  } else {
1983  // Ensure that the constant occurs on the RHS.
1984  ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
1985  MVT CompVT = N1.getValueType().getSimpleVT();
1986  if (!TLI->isCondCodeLegal(SwappedCond, CompVT))
1987  return SDValue();
1988 
1989  return getSetCC(dl, VT, N2, N1, SwappedCond);
1990  }
1991  }
1992 
1993  // Could not fold it.
1994  return SDValue();
1995 }
1996 
1997 /// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
1998 /// use this predicate to simplify operations downstream.
1999 bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2000  // This predicate is not safe for vector operations.
2001  if (Op.getValueType().isVector())
2002  return false;
2003 
2004  unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
2005  return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth);
2006 }
2007 
2008 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2009 /// this predicate to simplify operations downstream. Mask is known to be zero
2010 /// for bits that V cannot have.
2012  unsigned Depth) const {
2013  APInt KnownZero, KnownOne;
2014  computeKnownBits(Op, KnownZero, KnownOne, Depth);
2015  return (KnownZero & Mask) == Mask;
2016 }
2017 
2018 /// Determine which bits of Op are known to be either zero or one and return
2019 /// them in the KnownZero/KnownOne bitsets.
2021  APInt &KnownOne, unsigned Depth) const {
2022  unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
2023 
2024  KnownZero = KnownOne = APInt(BitWidth, 0); // Don't know anything.
2025  if (Depth == 6)
2026  return; // Limit search depth.
2027 
2028  APInt KnownZero2, KnownOne2;
2029 
2030  switch (Op.getOpcode()) {
2031  case ISD::Constant:
2032  // We know all of the bits for a constant!
2033  KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue();
2034  KnownZero = ~KnownOne;
2035  break;
2036  case ISD::AND:
2037  // If either the LHS or the RHS are Zero, the result is zero.
2038  computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
2039  computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
2040 
2041  // Output known-1 bits are only known if set in both the LHS & RHS.
2042  KnownOne &= KnownOne2;
2043  // Output known-0 are known to be clear if zero in either the LHS | RHS.
2044  KnownZero |= KnownZero2;
2045  break;
2046  case ISD::OR:
2047  computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
2048  computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
2049 
2050  // Output known-0 bits are only known if clear in both the LHS & RHS.
2051  KnownZero &= KnownZero2;
2052  // Output known-1 are known to be set if set in either the LHS | RHS.
2053  KnownOne |= KnownOne2;
2054  break;
2055  case ISD::XOR: {
2056  computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
2057  computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
2058 
2059  // Output known-0 bits are known if clear or set in both the LHS & RHS.
2060  APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2);
2061  // Output known-1 are known to be set if set in only one of the LHS, RHS.
2062  KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2);
2063  KnownZero = KnownZeroOut;
2064  break;
2065  }
2066  case ISD::MUL: {
2067  computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
2068  computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
2069 
2070  // If low bits are zero in either operand, output low known-0 bits.
2071  // Also compute a conserative estimate for high known-0 bits.
2072  // More trickiness is possible, but this is sufficient for the
2073  // interesting case of alignment computation.
2074  KnownOne.clearAllBits();
2075  unsigned TrailZ = KnownZero.countTrailingOnes() +
2076  KnownZero2.countTrailingOnes();
2077  unsigned LeadZ = std::max(KnownZero.countLeadingOnes() +
2078  KnownZero2.countLeadingOnes(),
2079  BitWidth) - BitWidth;
2080 
2081  TrailZ = std::min(TrailZ, BitWidth);
2082  LeadZ = std::min(LeadZ, BitWidth);
2083  KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) |
2084  APInt::getHighBitsSet(BitWidth, LeadZ);
2085  break;
2086  }
2087  case ISD::UDIV: {
2088  // For the purposes of computing leading zeros we can conservatively
2089  // treat a udiv as a logical right shift by the power of 2 known to
2090  // be less than the denominator.
2091  computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
2092  unsigned LeadZ = KnownZero2.countLeadingOnes();
2093 
2094  KnownOne2.clearAllBits();
2095  KnownZero2.clearAllBits();
2096  computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
2097  unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros();
2098  if (RHSUnknownLeadingOnes != BitWidth)
2099  LeadZ = std::min(BitWidth,
2100  LeadZ + BitWidth - RHSUnknownLeadingOnes - 1);
2101 
2102  KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ);
2103  break;
2104  }
2105  case ISD::SELECT:
2106  computeKnownBits(Op.getOperand(2), KnownZero, KnownOne, Depth+1);
2107  computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
2108 
2109  // Only known if known in both the LHS and RHS.
2110  KnownOne &= KnownOne2;
2111  KnownZero &= KnownZero2;
2112  break;
2113  case ISD::SELECT_CC:
2114  computeKnownBits(Op.getOperand(3), KnownZero, KnownOne, Depth+1);
2115  computeKnownBits(Op.getOperand(2), KnownZero2, KnownOne2, Depth+1);
2116 
2117  // Only known if known in both the LHS and RHS.
2118  KnownOne &= KnownOne2;
2119  KnownZero &= KnownZero2;
2120  break;
2121  case ISD::SADDO:
2122  case ISD::UADDO:
2123  case ISD::SSUBO:
2124  case ISD::USUBO:
2125  case ISD::SMULO:
2126  case ISD::UMULO:
2127  if (Op.getResNo() != 1)
2128  break;
2129  // The boolean result conforms to getBooleanContents.
2130  // If we know the result of a setcc has the top bits zero, use this info.
2131  // We know that we have an integer-based boolean since these operations
2132  // are only available for integer.
2133  if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
2135  BitWidth > 1)
2136  KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
2137  break;
2138  case ISD::SETCC:
2139  // If we know the result of a setcc has the top bits zero, use this info.
2140  if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
2142  BitWidth > 1)
2143  KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
2144  break;
2145  case ISD::SHL:
2146  // (shl X, C1) & C2 == 0 iff (X & C2 >>u C1) == 0
2147  if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2148  unsigned ShAmt = SA->getZExtValue();
2149 
2150  // If the shift count is an invalid immediate, don't do anything.
2151  if (ShAmt >= BitWidth)
2152  break;
2153 
2154  computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2155  KnownZero <<= ShAmt;
2156  KnownOne <<= ShAmt;
2157  // low bits known zero.
2158  KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt);
2159  }
2160  break;
2161  case ISD::SRL:
2162  // (ushr X, C1) & C2 == 0 iff (-1 >> C1) & C2 == 0
2163  if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2164  unsigned ShAmt = SA->getZExtValue();
2165 
2166  // If the shift count is an invalid immediate, don't do anything.
2167  if (ShAmt >= BitWidth)
2168  break;
2169 
2170  computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2171  KnownZero = KnownZero.lshr(ShAmt);
2172  KnownOne = KnownOne.lshr(ShAmt);
2173 
2174  APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
2175  KnownZero |= HighBits; // High bits known zero.
2176  }
2177  break;
2178  case ISD::SRA:
2179  if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2180  unsigned ShAmt = SA->getZExtValue();
2181 
2182  // If the shift count is an invalid immediate, don't do anything.
2183  if (ShAmt >= BitWidth)
2184  break;
2185 
2186  // If any of the demanded bits are produced by the sign extension, we also
2187  // demand the input sign bit.
2188  APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
2189 
2190  computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2191  KnownZero = KnownZero.lshr(ShAmt);
2192  KnownOne = KnownOne.lshr(ShAmt);
2193 
2194  // Handle the sign bits.
2195  APInt SignBit = APInt::getSignBit(BitWidth);
2196  SignBit = SignBit.lshr(ShAmt); // Adjust to where it is now in the mask.
2197 
2198  if (KnownZero.intersects(SignBit)) {
2199  KnownZero |= HighBits; // New bits are known zero.
2200  } else if (KnownOne.intersects(SignBit)) {
2201  KnownOne |= HighBits; // New bits are known one.
2202  }
2203  }
2204  break;
2205  case ISD::SIGN_EXTEND_INREG: {
2206  EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2207  unsigned EBits = EVT.getScalarType().getSizeInBits();
2208 
2209  // Sign extension. Compute the demanded bits in the result that are not
2210  // present in the input.
2211  APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits);
2212 
2213  APInt InSignBit = APInt::getSignBit(EBits);
2214  APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits);
2215 
2216  // If the sign extended bits are demanded, we know that the sign
2217  // bit is demanded.
2218  InSignBit = InSignBit.zext(BitWidth);
2219  if (NewBits.getBoolValue())
2220  InputDemandedBits |= InSignBit;
2221 
2222  computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2223  KnownOne &= InputDemandedBits;
2224  KnownZero &= InputDemandedBits;
2225 
2226  // If the sign bit of the input is known set or clear, then we know the
2227  // top bits of the result.
2228  if (KnownZero.intersects(InSignBit)) { // Input sign bit known clear
2229  KnownZero |= NewBits;
2230  KnownOne &= ~NewBits;
2231  } else if (KnownOne.intersects(InSignBit)) { // Input sign bit known set
2232  KnownOne |= NewBits;
2233  KnownZero &= ~NewBits;
2234  } else { // Input sign bit unknown
2235  KnownZero &= ~NewBits;
2236  KnownOne &= ~NewBits;
2237  }
2238  break;
2239  }
2240  case ISD::CTTZ:
2241  case ISD::CTTZ_ZERO_UNDEF:
2242  case ISD::CTLZ:
2243  case ISD::CTLZ_ZERO_UNDEF:
2244  case ISD::CTPOP: {
2245  unsigned LowBits = Log2_32(BitWidth)+1;
2246  KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits);
2247  KnownOne.clearAllBits();
2248  break;
2249  }
2250  case ISD::LOAD: {
2251  LoadSDNode *LD = cast<LoadSDNode>(Op);
2252  // If this is a ZEXTLoad and we are looking at the loaded value.
2253  if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
2254  EVT VT = LD->getMemoryVT();
2255  unsigned MemBits = VT.getScalarType().getSizeInBits();
2256  KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
2257  } else if (const MDNode *Ranges = LD->getRanges()) {
2259  }
2260  break;
2261  }
2262  case ISD::ZERO_EXTEND: {
2263  EVT InVT = Op.getOperand(0).getValueType();
2264  unsigned InBits = InVT.getScalarType().getSizeInBits();
2265  APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
2266  KnownZero = KnownZero.trunc(InBits);
2267  KnownOne = KnownOne.trunc(InBits);
2268  computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2269  KnownZero = KnownZero.zext(BitWidth);
2270  KnownOne = KnownOne.zext(BitWidth);
2271  KnownZero |= NewBits;
2272  break;
2273  }
2274  case ISD::SIGN_EXTEND: {
2275  EVT InVT = Op.getOperand(0).getValueType();
2276  unsigned InBits = InVT.getScalarType().getSizeInBits();
2277  APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
2278 
2279  KnownZero = KnownZero.trunc(InBits);
2280  KnownOne = KnownOne.trunc(InBits);
2281  computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2282 
2283  // Note if the sign bit is known to be zero or one.
2284  bool SignBitKnownZero = KnownZero.isNegative();
2285  bool SignBitKnownOne = KnownOne.isNegative();
2286 
2287  KnownZero = KnownZero.zext(BitWidth);
2288  KnownOne = KnownOne.zext(BitWidth);
2289 
2290  // If the sign bit is known zero or one, the top bits match.
2291  if (SignBitKnownZero)
2292  KnownZero |= NewBits;
2293  else if (SignBitKnownOne)
2294  KnownOne |= NewBits;
2295  break;
2296  }
2297  case ISD::ANY_EXTEND: {
2298  EVT InVT = Op.getOperand(0).getValueType();
2299  unsigned InBits = InVT.getScalarType().getSizeInBits();
2300  KnownZero = KnownZero.trunc(InBits);
2301  KnownOne = KnownOne.trunc(InBits);
2302  computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2303  KnownZero = KnownZero.zext(BitWidth);
2304  KnownOne = KnownOne.zext(BitWidth);
2305  break;
2306  }
2307  case ISD::TRUNCATE: {
2308  EVT InVT = Op.getOperand(0).getValueType();
2309  unsigned InBits = InVT.getScalarType().getSizeInBits();
2310  KnownZero = KnownZero.zext(InBits);
2311  KnownOne = KnownOne.zext(InBits);
2312  computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2313  KnownZero = KnownZero.trunc(BitWidth);
2314  KnownOne = KnownOne.trunc(BitWidth);
2315  break;
2316  }
2317  case ISD::AssertZext: {
2318  EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2319  APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
2320  computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2321  KnownZero |= (~InMask);
2322  KnownOne &= (~KnownZero);
2323  break;
2324  }
2325  case ISD::FGETSIGN:
2326  // All bits are zero except the low bit.
2327  KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1);
2328  break;
2329 
2330  case ISD::SUB: {
2331  if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) {
2332  // We know that the top bits of C-X are clear if X contains less bits
2333  // than C (i.e. no wrap-around can happen). For example, 20-X is
2334  // positive if we can prove that X is >= 0 and < 16.
2335  if (CLHS->getAPIntValue().isNonNegative()) {
2336  unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros();
2337  // NLZ can't be BitWidth with no sign bit
2338  APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1);
2339  computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
2340 
2341  // If all of the MaskV bits are known to be zero, then we know the
2342  // output top bits are zero, because we now know that the output is
2343  // from [0-C].
2344  if ((KnownZero2 & MaskV) == MaskV) {
2345  unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros();
2346  // Top bits known zero.
2347  KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2);
2348  }
2349  }
2350  }
2351  }
2352  // fall through
2353  case ISD::ADD:
2354  case ISD::ADDE: {
2355  // Output known-0 bits are known if clear or set in both the low clear bits
2356  // common to both LHS & RHS. For example, 8+(X<<3) is known to have the
2357  // low 3 bits clear.
2358  // Output known-0 bits are also known if the top bits of each input are
2359  // known to be clear. For example, if one input has the top 10 bits clear
2360  // and the other has the top 8 bits clear, we know the top 7 bits of the
2361  // output must be clear.
2362  computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
2363  unsigned KnownZeroHigh = KnownZero2.countLeadingOnes();
2364  unsigned KnownZeroLow = KnownZero2.countTrailingOnes();
2365 
2366  computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
2367  KnownZeroHigh = std::min(KnownZeroHigh,
2368  KnownZero2.countLeadingOnes());
2369  KnownZeroLow = std::min(KnownZeroLow,
2370  KnownZero2.countTrailingOnes());
2371 
2372  if (Op.getOpcode() == ISD::ADD) {
2373  KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroLow);
2374  if (KnownZeroHigh > 1)
2375  KnownZero |= APInt::getHighBitsSet(BitWidth, KnownZeroHigh - 1);
2376  break;
2377  }
2378 
2379  // With ADDE, a carry bit may be added in, so we can only use this
2380  // information if we know (at least) that the low two bits are clear. We
2381  // then return to the caller that the low bit is unknown but that other bits
2382  // are known zero.
2383  if (KnownZeroLow >= 2) // ADDE
2384  KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroLow);
2385  break;
2386  }
2387  case ISD::SREM:
2388  if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2389  const APInt &RA = Rem->getAPIntValue().abs();
2390  if (RA.isPowerOf2()) {
2391  APInt LowBits = RA - 1;
2392  computeKnownBits(Op.getOperand(0), KnownZero2,KnownOne2,Depth+1);
2393 
2394  // The low bits of the first operand are unchanged by the srem.
2395  KnownZero = KnownZero2 & LowBits;
2396  KnownOne = KnownOne2 & LowBits;
2397 
2398  // If the first operand is non-negative or has all low bits zero, then
2399  // the upper bits are all zero.
2400  if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits))
2401  KnownZero |= ~LowBits;
2402 
2403  // If the first operand is negative and not all low bits are zero, then
2404  // the upper bits are all one.
2405  if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0))
2406  KnownOne |= ~LowBits;
2407  assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?");
2408  }
2409  }
2410  break;
2411  case ISD::UREM: {
2412  if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2413  const APInt &RA = Rem->getAPIntValue();
2414  if (RA.isPowerOf2()) {
2415  APInt LowBits = (RA - 1);
2416  computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth + 1);
2417 
2418  // The upper bits are all zero, the lower ones are unchanged.
2419  KnownZero = KnownZero2 | ~LowBits;
2420  KnownOne = KnownOne2 & LowBits;
2421  break;
2422  }
2423  }
2424 
2425  // Since the result is less than or equal to either operand, any leading
2426  // zero bits in either operand must also exist in the result.
2427  computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2428  computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
2429 
2430  uint32_t Leaders = std::max(KnownZero.countLeadingOnes(),
2431  KnownZero2.countLeadingOnes());
2432  KnownOne.clearAllBits();
2433  KnownZero = APInt::getHighBitsSet(BitWidth, Leaders);
2434  break;
2435  }
2436  case ISD::EXTRACT_ELEMENT: {
2437  computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2438  const unsigned Index =
2439  cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
2440  const unsigned BitWidth = Op.getValueType().getSizeInBits();
2441 
2442  // Remove low part of known bits mask
2443  KnownZero = KnownZero.getHiBits(KnownZero.getBitWidth() - Index * BitWidth);
2444  KnownOne = KnownOne.getHiBits(KnownOne.getBitWidth() - Index * BitWidth);
2445 
2446  // Remove high part of known bit mask
2447  KnownZero = KnownZero.trunc(BitWidth);
2448  KnownOne = KnownOne.trunc(BitWidth);
2449  break;
2450  }
2451  case ISD::SMIN:
2452  case ISD::SMAX:
2453  case ISD::UMIN:
2454  case ISD::UMAX: {
2455  APInt Op0Zero, Op0One;
2456  APInt Op1Zero, Op1One;
2457  computeKnownBits(Op.getOperand(0), Op0Zero, Op0One, Depth);
2458  computeKnownBits(Op.getOperand(1), Op1Zero, Op1One, Depth);
2459 
2460  KnownZero = Op0Zero & Op1Zero;
2461  KnownOne = Op0One & Op1One;
2462  break;
2463  }
2464  case ISD::FrameIndex:
2465  case ISD::TargetFrameIndex:
2466  if (unsigned Align = InferPtrAlignment(Op)) {
2467  // The low bits are known zero if the pointer is aligned.
2468  KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align));
2469  break;
2470  }
2471  break;
2472 
2473  default:
2474  if (Op.getOpcode() < ISD::BUILTIN_OP_END)
2475  break;
2476  // Fallthrough
2479  case ISD::INTRINSIC_VOID:
2480  // Allow the target to implement this method for its nodes.
2481  TLI->computeKnownBitsForTargetNode(Op, KnownZero, KnownOne, *this, Depth);
2482  break;
2483  }
2484 
2485  assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
2486 }
2487 
2488 /// ComputeNumSignBits - Return the number of times the sign bit of the
2489 /// register is replicated into the other bits. We know that at least 1 bit
2490 /// is always equal to the sign bit (itself), but other cases can give us
2491 /// information. For example, immediately after an "SRA X, 2", we know that
2492 /// the top 3 bits are all equal to each other, so we return 3.
2493 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{
2494  EVT VT = Op.getValueType();
2495  assert(VT.isInteger() && "Invalid VT!");
2496  unsigned VTBits = VT.getScalarType().getSizeInBits();
2497  unsigned Tmp, Tmp2;
2498  unsigned FirstAnswer = 1;
2499 
2500  if (Depth == 6)
2501  return 1; // Limit search depth.
2502 
2503  switch (Op.getOpcode()) {
2504  default: break;
2505  case ISD::AssertSext:
2506  Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
2507  return VTBits-Tmp+1;
2508  case ISD::AssertZext:
2509  Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
2510  return VTBits-Tmp;
2511 
2512  case ISD::Constant: {
2513  const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue();
2514  return Val.getNumSignBits();
2515  }
2516 
2517  case ISD::SIGN_EXTEND:
2518  Tmp =
2520  return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp;
2521 
2523  // Max of the input and what this extends.
2524  Tmp =
2525  cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits();
2526  Tmp = VTBits-Tmp+1;
2527 
2528  Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2529  return std::max(Tmp, Tmp2);
2530 
2531  case ISD::SRA:
2532  Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2533  // SRA X, C -> adds C sign bits.
2534  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2535  Tmp += C->getZExtValue();
2536  if (Tmp > VTBits) Tmp = VTBits;
2537  }
2538  return Tmp;
2539  case ISD::SHL:
2540  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2541  // shl destroys sign bits.
2542  Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2543  if (C->getZExtValue() >= VTBits || // Bad shift.
2544  C->getZExtValue() >= Tmp) break; // Shifted all sign bits out.
2545  return Tmp - C->getZExtValue();
2546  }
2547  break;
2548  case ISD::AND:
2549  case ISD::OR:
2550  case ISD::XOR: // NOT is handled here.
2551  // Logical binary ops preserve the number of sign bits at the worst.
2552  Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2553  if (Tmp != 1) {
2554  Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
2555  FirstAnswer = std::min(Tmp, Tmp2);
2556  // We computed what we know about the sign bits as our first
2557  // answer. Now proceed to the generic code that uses
2558  // computeKnownBits, and pick whichever answer is better.
2559  }
2560  break;
2561 
2562  case ISD::SELECT:
2563  Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1);
2564  if (Tmp == 1) return 1; // Early out.
2565  Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1);
2566  return std::min(Tmp, Tmp2);
2567  case ISD::SMIN:
2568  case ISD::SMAX:
2569  case ISD::UMIN:
2570  case ISD::UMAX:
2571  Tmp = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
2572  if (Tmp == 1)
2573  return 1; // Early out.
2574  Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
2575  return std::min(Tmp, Tmp2);
2576  case ISD::SADDO:
2577  case ISD::UADDO:
2578  case ISD::SSUBO:
2579  case ISD::USUBO:
2580  case ISD::SMULO:
2581  case ISD::UMULO:
2582  if (Op.getResNo() != 1)
2583  break;
2584  // The boolean result conforms to getBooleanContents. Fall through.
2585  // If setcc returns 0/-1, all bits are sign bits.
2586  // We know that we have an integer-based boolean since these operations
2587  // are only available for integer.
2588  if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
2590  return VTBits;
2591  break;
2592  case ISD::SETCC:
2593  // If setcc returns 0/-1, all bits are sign bits.
2594  if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
2596  return VTBits;
2597  break;
2598  case ISD::ROTL:
2599  case ISD::ROTR:
2600  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2601  unsigned RotAmt = C->getZExtValue() & (VTBits-1);
2602 
2603  // Handle rotate right by N like a rotate left by 32-N.
2604  if (Op.getOpcode() == ISD::ROTR)
2605  RotAmt = (VTBits-RotAmt) & (VTBits-1);
2606 
2607  // If we aren't rotating out all of the known-in sign bits, return the
2608  // number that are left. This handles rotl(sext(x), 1) for example.
2609  Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2610  if (Tmp > RotAmt+1) return Tmp-RotAmt;
2611  }
2612  break;
2613  case ISD::ADD:
2614  // Add can have at most one carry bit. Thus we know that the output
2615  // is, at worst, one more bit than the inputs.
2616  Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2617  if (Tmp == 1) return 1; // Early out.
2618 
2619  // Special case decrementing a value (ADD X, -1):
2620  if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
2621  if (CRHS->isAllOnesValue()) {
2622  APInt KnownZero, KnownOne;
2623  computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
2624 
2625  // If the input is known to be 0 or 1, the output is 0/-1, which is all
2626  // sign bits set.
2627  if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
2628  return VTBits;
2629 
2630  // If we are subtracting one from a positive number, there is no carry
2631  // out of the result.
2632  if (KnownZero.isNegative())
2633  return Tmp;
2634  }
2635 
2636  Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
2637  if (Tmp2 == 1) return 1;
2638  return std::min(Tmp, Tmp2)-1;
2639 
2640  case ISD::SUB:
2641  Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
2642  if (Tmp2 == 1) return 1;
2643 
2644  // Handle NEG.
2645  if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0)))
2646  if (CLHS->isNullValue()) {
2647  APInt KnownZero, KnownOne;
2648  computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
2649  // If the input is known to be 0 or 1, the output is 0/-1, which is all
2650  // sign bits set.
2651  if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
2652  return VTBits;
2653 
2654  // If the input is known to be positive (the sign bit is known clear),
2655  // the output of the NEG has the same number of sign bits as the input.
2656  if (KnownZero.isNegative())
2657  return Tmp2;
2658 
2659  // Otherwise, we treat this like a SUB.
2660  }
2661 
2662  // Sub can have at most one carry bit. Thus we know that the output
2663  // is, at worst, one more bit than the inputs.
2664  Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2665  if (Tmp == 1) return 1; // Early out.
2666  return std::min(Tmp, Tmp2)-1;
2667  case ISD::TRUNCATE:
2668  // FIXME: it's tricky to do anything useful for this, but it is an important
2669  // case for targets like X86.
2670  break;
2671  case ISD::EXTRACT_ELEMENT: {
2672  const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);
2673  const int BitWidth = Op.getValueType().getSizeInBits();
2674  const int Items =
2675  Op.getOperand(0).getValueType().getSizeInBits() / BitWidth;
2676 
2677  // Get reverse index (starting from 1), Op1 value indexes elements from
2678  // little end. Sign starts at big end.
2679  const int rIndex = Items - 1 -
2680  cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
2681 
2682  // If the sign portion ends in our element the substraction gives correct
2683  // result. Otherwise it gives either negative or > bitwidth result
2684  return std::max(std::min(KnownSign - rIndex * BitWidth, BitWidth), 0);
2685  }
2686  }
2687 
2688  // If we are looking at the loaded value of the SDNode.
2689  if (Op.getResNo() == 0) {
2690  // Handle LOADX separately here. EXTLOAD case will fallthrough.
2691  if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
2692  unsigned ExtType = LD->getExtensionType();
2693  switch (ExtType) {
2694  default: break;
2695  case ISD::SEXTLOAD: // '17' bits known
2696  Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
2697  return VTBits-Tmp+1;
2698  case ISD::ZEXTLOAD: // '16' bits known
2699  Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
2700  return VTBits-Tmp;
2701  }
2702  }
2703  }
2704 
2705  // Allow the target to implement this method for its nodes.
2706  if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
2709  Op.getOpcode() == ISD::INTRINSIC_VOID) {
2710  unsigned NumBits = TLI->ComputeNumSignBitsForTargetNode(Op, *this, Depth);
2711  if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits);
2712  }
2713 
2714  // Finally, if we can prove that the top bits of the result are 0's or 1's,
2715  // use this information.
2716  APInt KnownZero, KnownOne;
2717  computeKnownBits(Op, KnownZero, KnownOne, Depth);
2718 
2719  APInt Mask;
2720  if (KnownZero.isNegative()) { // sign bit is 0
2721  Mask = KnownZero;
2722  } else if (KnownOne.isNegative()) { // sign bit is 1;
2723  Mask = KnownOne;
2724  } else {
2725  // Nothing known.
2726  return FirstAnswer;
2727  }
2728 
2729  // Okay, we know that the sign bit in Mask is set. Use CLZ to determine
2730  // the number of identical bits in the top of the input value.
2731  Mask = ~Mask;
2732  Mask <<= Mask.getBitWidth()-VTBits;
2733  // Return # leading zeros. We use 'min' here in case Val was zero before
2734  // shifting. We don't want to return '64' as for an i32 "0".
2735  return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros()));
2736 }
2737 
2738 /// isBaseWithConstantOffset - Return true if the specified operand is an
2739 /// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an
2740 /// ISD::OR with a ConstantSDNode that is guaranteed to have the same
2741 /// semantics as an ADD. This handles the equivalence:
2742 /// X|Cst == X+Cst iff X&Cst = 0.
2744  if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
2745  !isa<ConstantSDNode>(Op.getOperand(1)))
2746  return false;
2747 
2748  if (Op.getOpcode() == ISD::OR &&
2750  cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue()))
2751  return false;
2752 
2753  return true;
2754 }
2755 
2756 
2758  // If we're told that NaNs won't happen, assume they won't.
2759  if (getTarget().Options.NoNaNsFPMath)
2760  return true;
2761 
2762  // If the value is a constant, we can obviously see if it is a NaN or not.
2763  if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
2764  return !C->getValueAPF().isNaN();
2765 
2766  // TODO: Recognize more cases here.
2767 
2768  return false;
2769 }
2770 
2772  // If the value is a constant, we can obviously see if it is a zero or not.
2773  if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
2774  return !C->isZero();
2775 
2776  // TODO: Recognize more cases here.
2777  switch (Op.getOpcode()) {
2778  default: break;
2779  case ISD::OR:
2780  if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
2781  return !C->isNullValue();
2782  break;
2783  }
2784 
2785  return false;
2786 }
2787 
2789  // Check the obvious case.
2790  if (A == B) return true;
2791 
2792  // For for negative and positive zero.
2793  if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
2794  if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
2795  if (CA->isZero() && CB->isZero()) return true;
2796 
2797  // Otherwise they may not be equal.
2798  return false;
2799 }
2800 
2801 /// getNode - Gets or creates the specified node.
2802 ///
2803 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT) {
2805  AddNodeIDNode(ID, Opcode, getVTList(VT), None);
2806  void *IP = nullptr;
2807  if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
2808  return SDValue(E, 0);
2809 
2810  SDNode *N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(),
2811  DL.getDebugLoc(), getVTList(VT));
2812  CSEMap.InsertNode(N, IP);
2813 
2814  InsertNode(N);
2815  return SDValue(N, 0);
2816 }
2817 
2819  EVT VT, SDValue Operand) {
2820  // Constant fold unary operations with an integer constant operand. Even
2821  // opaque constant will be folded, because the folding of unary operations
2822  // doesn't create new constants with different values. Nevertheless, the
2823  // opaque flag is preserved during folding to prevent future folding with
2824  // other constants.
2825  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand)) {
2826  const APInt &Val = C->getAPIntValue();
2827  switch (Opcode) {
2828  default: break;
2829  case ISD::SIGN_EXTEND:
2830  return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
2831  C->isTargetOpcode(), C->isOpaque());
2832  case ISD::ANY_EXTEND:
2833  case ISD::ZERO_EXTEND:
2834  case ISD::TRUNCATE:
2835  return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
2836  C->isTargetOpcode(), C->isOpaque());
2837  case ISD::UINT_TO_FP:
2838  case ISD::SINT_TO_FP: {
2841  (void)apf.convertFromAPInt(Val,
2842  Opcode==ISD::SINT_TO_FP,
2844  return getConstantFP(apf, DL, VT);
2845  }
2846  case ISD::BITCAST:
2847  if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
2848  return getConstantFP(APFloat(APFloat::IEEEhalf, Val), DL, VT);
2849  if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
2850  return getConstantFP(APFloat(APFloat::IEEEsingle, Val), DL, VT);
2851  else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
2852  return getConstantFP(APFloat(APFloat::IEEEdouble, Val), DL, VT);
2853  break;
2854  case ISD::BSWAP:
2855  return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(),
2856  C->isOpaque());
2857  case ISD::CTPOP:
2858  return getConstant(Val.countPopulation(), DL, VT, C->isTargetOpcode(),
2859  C->isOpaque());
2860  case ISD::CTLZ:
2861  case ISD::CTLZ_ZERO_UNDEF:
2862  return getConstant(Val.countLeadingZeros(), DL, VT, C->isTargetOpcode(),
2863  C->isOpaque());
2864  case ISD::CTTZ:
2865  case ISD::CTTZ_ZERO_UNDEF:
2866  return getConstant(Val.countTrailingZeros(), DL, VT, C->isTargetOpcode(),
2867  C->isOpaque());
2868  }
2869  }
2870 
2871  // Constant fold unary operations with a floating point constant operand.
2872  if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand)) {
2873  APFloat V = C->getValueAPF(); // make copy
2874  switch (Opcode) {
2875  case ISD::FNEG:
2876  V.changeSign();
2877  return getConstantFP(V, DL, VT);
2878  case ISD::FABS:
2879  V.clearSign();
2880  return getConstantFP(V, DL, VT);
2881  case ISD::FCEIL: {
2883  if (fs == APFloat::opOK || fs == APFloat::opInexact)
2884  return getConstantFP(V, DL, VT);
2885  break;
2886  }
2887  case ISD::FTRUNC: {
2889  if (fs == APFloat::opOK || fs == APFloat::opInexact)
2890  return getConstantFP(V, DL, VT);
2891  break;
2892  }
2893  case ISD::FFLOOR: {
2895  if (fs == APFloat::opOK || fs == APFloat::opInexact)
2896  return getConstantFP(V, DL, VT);
2897  break;
2898  }
2899  case ISD::FP_EXTEND: {
2900  bool ignored;
2901  // This can return overflow, underflow, or inexact; we don't care.
2902  // FIXME need to be more flexible about rounding mode.
2903  (void)V.convert(EVTToAPFloatSemantics(VT),
2904  APFloat::rmNearestTiesToEven, &ignored);
2905  return getConstantFP(V, DL, VT);
2906  }
2907  case ISD::FP_TO_SINT:
2908  case ISD::FP_TO_UINT: {
2909  integerPart x[2];
2910  bool ignored;
2911  static_assert(integerPartWidth >= 64, "APFloat parts too small!");
2912  // FIXME need to be more flexible about rounding mode.
2914  Opcode==ISD::FP_TO_SINT,
2915  APFloat::rmTowardZero, &ignored);
2916  if (s==APFloat::opInvalidOp) // inexact is OK, in fact usual
2917  break;
2918  APInt api(VT.getSizeInBits(), x);
2919  return getConstant(api, DL, VT);
2920  }
2921  case ISD::BITCAST:
2922  if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
2923  return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
2924  else if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
2925  return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
2926  else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
2927  return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
2928  break;
2929  }
2930  }
2931 
2932  // Constant fold unary operations with a vector integer or float operand.
2933  if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand)) {
2934  if (BV->isConstant()) {
2935  switch (Opcode) {
2936  default:
2937  // FIXME: Entirely reasonable to perform folding of other unary
2938  // operations here as the need arises.
2939  break;
2940  case ISD::FNEG:
2941  case ISD::FABS:
2942  case ISD::FCEIL:
2943  case ISD::FTRUNC:
2944  case ISD::FFLOOR:
2945  case ISD::FP_EXTEND:
2946  case ISD::FP_TO_SINT:
2947  case ISD::FP_TO_UINT:
2948  case ISD::TRUNCATE:
2949  case ISD::UINT_TO_FP:
2950  case ISD::SINT_TO_FP:
2951  case ISD::BSWAP:
2952  case ISD::CTLZ:
2953  case ISD::CTLZ_ZERO_UNDEF:
2954  case ISD::CTTZ:
2955  case ISD::CTTZ_ZERO_UNDEF:
2956  case ISD::CTPOP: {
2957  EVT SVT = VT.getScalarType();
2958  EVT InVT = BV->getValueType(0);
2959  EVT InSVT = InVT.getScalarType();
2960 
2961  // Find legal integer scalar type for constant promotion and
2962  // ensure that its scalar size is at least as large as source.
2963  EVT LegalSVT = SVT;
2964  if (SVT.isInteger()) {
2965  LegalSVT = TLI->getTypeToTransformTo(*getContext(), SVT);
2966  if (LegalSVT.bitsLT(SVT)) break;
2967  }
2968 
2969  // Let the above scalar folding handle the folding of each element.
2971  for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
2972  SDValue OpN = BV->getOperand(i);
2973  EVT OpVT = OpN.getValueType();
2974 
2975  // Build vector (integer) scalar operands may need implicit
2976  // truncation - do this before constant folding.
2977  if (OpVT.isInteger() && OpVT.bitsGT(InSVT))
2978  OpN = getNode(ISD::TRUNCATE, DL, InSVT, OpN);
2979 
2980  OpN = getNode(Opcode, DL, SVT, OpN);
2981 
2982  // Legalize the (integer) scalar constant if necessary.
2983  if (LegalSVT != SVT)
2984  OpN = getNode(ISD::ANY_EXTEND, DL, LegalSVT, OpN);
2985 
2986  if (OpN.getOpcode() != ISD::UNDEF &&
2987  OpN.getOpcode() != ISD::Constant &&
2988  OpN.getOpcode() != ISD::ConstantFP)
2989  break;
2990  Ops.push_back(OpN);
2991  }
2992  if (Ops.size() == VT.getVectorNumElements())
2993  return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
2994  break;
2995  }
2996  }
2997  }
2998  }
2999 
3000  unsigned OpOpcode = Operand.getNode()->getOpcode();
3001  switch (Opcode) {
3002  case ISD::TokenFactor:
3003  case ISD::MERGE_VALUES:
3004  case ISD::CONCAT_VECTORS:
3005  return Operand; // Factor, merge or concat of one node? No need.
3006  case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
3007  case ISD::FP_EXTEND:
3008  assert(VT.isFloatingPoint() &&
3009  Operand.getValueType().isFloatingPoint() && "Invalid FP cast!");
3010  if (Operand.getValueType() == VT) return Operand; // noop conversion.
3011  assert((!VT.isVector() ||
3012  VT.getVectorNumElements() ==
3013  Operand.getValueType().getVectorNumElements()) &&
3014  "Vector element count mismatch!");
3015  if (Operand.getOpcode() == ISD::UNDEF)
3016  return getUNDEF(VT);
3017  break;
3018  case ISD::SIGN_EXTEND:
3019  assert(VT.isInteger() && Operand.getValueType().isInteger() &&
3020  "Invalid SIGN_EXTEND!");
3021  if (Operand.getValueType() == VT) return Operand; // noop extension
3022  assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
3023  "Invalid sext node, dst < src!");
3024  assert((!VT.isVector() ||
3025  VT.getVectorNumElements() ==
3026  Operand.getValueType().getVectorNumElements()) &&
3027  "Vector element count mismatch!");
3028  if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
3029  return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
3030  else if (OpOpcode == ISD::UNDEF)
3031  // sext(undef) = 0, because the top bits will all be the same.
3032  return getConstant(0, DL, VT);
3033  break;
3034  case ISD::ZERO_EXTEND:
3035  assert(VT.isInteger() && Operand.getValueType().isInteger() &&
3036  "Invalid ZERO_EXTEND!");
3037  if (Operand.getValueType() == VT) return Operand; // noop extension
3038  assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
3039  "Invalid zext node, dst < src!");
3040  assert((!VT.isVector() ||
3041  VT.getVectorNumElements() ==
3042  Operand.getValueType().getVectorNumElements()) &&
3043  "Vector element count mismatch!");
3044  if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x)
3045  return getNode(ISD::ZERO_EXTEND, DL, VT,
3046  Operand.getNode()->getOperand(0));
3047  else if (OpOpcode == ISD::UNDEF)
3048  // zext(undef) = 0, because the top bits will be zero.
3049  return getConstant(0, DL, VT);
3050  break;
3051  case ISD::ANY_EXTEND:
3052  assert(VT.isInteger() && Operand.getValueType().isInteger() &&
3053  "Invalid ANY_EXTEND!");
3054  if (Operand.getValueType() == VT) return Operand; // noop extension
3055  assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
3056  "Invalid anyext node, dst < src!");
3057  assert((!VT.isVector() ||
3058  VT.getVectorNumElements() ==
3059  Operand.getValueType().getVectorNumElements()) &&
3060  "Vector element count mismatch!");
3061 
3062  if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
3063  OpOpcode == ISD::ANY_EXTEND)
3064  // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
3065  return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
3066  else if (OpOpcode == ISD::UNDEF)
3067  return getUNDEF(VT);
3068 
3069  // (ext (trunx x)) -> x
3070  if (OpOpcode == ISD::TRUNCATE) {
3071  SDValue OpOp = Operand.getNode()->getOperand(0);
3072  if (OpOp.getValueType() == VT)
3073  return OpOp;
3074  }
3075  break;
3076  case ISD::TRUNCATE:
3077  assert(VT.isInteger() && Operand.getValueType().isInteger() &&
3078  "Invalid TRUNCATE!");
3079  if (Operand.getValueType() == VT) return Operand; // noop truncate
3080  assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) &&
3081  "Invalid truncate node, src < dst!");
3082  assert((!VT.isVector() ||
3083  VT.getVectorNumElements() ==
3084  Operand.getValueType().getVectorNumElements()) &&
3085  "Vector element count mismatch!");
3086  if (OpOpcode == ISD::TRUNCATE)
3087  return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
3088  if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
3089  OpOpcode == ISD::ANY_EXTEND) {
3090  // If the source is smaller than the dest, we still need an extend.
3091  if (Operand.getNode()->getOperand(0).getValueType().getScalarType()
3092  .bitsLT(VT.getScalarType()))
3093  return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
3094  if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT))
3095  return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
3096  return Operand.getNode()->getOperand(0);
3097  }
3098  if (OpOpcode == ISD::UNDEF)
3099  return getUNDEF(VT);
3100  break;
3101  case ISD::BSWAP:
3102  assert(VT.isInteger() && VT == Operand.getValueType() &&
3103  "Invalid BSWAP!");
3104  assert((VT.getScalarSizeInBits() % 16 == 0) &&
3105  "BSWAP types must be a multiple of 16 bits!");
3106  if (OpOpcode == ISD::UNDEF)
3107  return getUNDEF(VT);
3108  break;
3109  case ISD::BITCAST:
3110  // Basic sanity checking.
3111  assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits()
3112  && "Cannot BITCAST between types of different sizes!");
3113  if (VT == Operand.getValueType()) return Operand; // noop conversion.
3114  if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
3115  return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
3116  if (OpOpcode == ISD::UNDEF)
3117  return getUNDEF(VT);
3118  break;
3119  case ISD::SCALAR_TO_VECTOR:
3120  assert(VT.isVector() && !Operand.getValueType().isVector() &&
3121  (VT.getVectorElementType() == Operand.getValueType() ||
3122  (VT.getVectorElementType().isInteger() &&
3123  Operand.getValueType().isInteger() &&
3124  VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&
3125  "Illegal SCALAR_TO_VECTOR node!");
3126  if (OpOpcode == ISD::UNDEF)
3127  return getUNDEF(VT);
3128  // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
3129  if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
3130  isa<ConstantSDNode>(Operand.getOperand(1)) &&
3131  Operand.getConstantOperandVal(1) == 0 &&
3132  Operand.getOperand(0).getValueType() == VT)
3133  return Operand.getOperand(0);
3134  break;
3135  case ISD::FNEG:
3136  // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0
3137  if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB)
3138  return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1),
3139  Operand.getNode()->getOperand(0));
3140  if (OpOpcode == ISD::FNEG) // --X -> X
3141  return Operand.getNode()->getOperand(0);
3142  break;
3143  case ISD::FABS:
3144  if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
3145  return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0));
3146  break;
3147  }
3148 
3149  SDNode *N;
3150  SDVTList VTs = getVTList(VT);
3151  if (VT != MVT::Glue) { // Don't CSE flag producing nodes
3153  SDValue Ops[1] = { Operand };
3154  AddNodeIDNode(ID, Opcode, VTs, Ops);
3155  void *IP = nullptr;
3156  if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
3157  return SDValue(E, 0);
3158 
3159  N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
3160  DL.getDebugLoc(), VTs, Operand);
3161  CSEMap.InsertNode(N, IP);
3162  } else {
3163  N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
3164  DL.getDebugLoc(), VTs, Operand);
3165  }
3166 
3167  InsertNode(N);
3168  return SDValue(N, 0);
3169 }
3170 
3171 static std::pair<APInt, bool> FoldValue(unsigned Opcode, const APInt &C1,
3172  const APInt &C2) {
3173  switch (Opcode) {
3174  case ISD::ADD: return std::make_pair(C1 + C2, true);
3175  case ISD::SUB: return std::make_pair(C1 - C2, true);
3176  case ISD::MUL: return std::make_pair(C1 * C2, true);
3177  case ISD::AND: return std::make_pair(C1 & C2, true);
3178  case ISD::OR: return std::make_pair(C1 | C2, true);
3179  case ISD::XOR: return std::make_pair(C1 ^ C2, true);
3180  case ISD::SHL: return std::make_pair(C1 << C2, true);
3181  case ISD::SRL: return std::make_pair(C1.lshr(C2), true);
3182  case ISD::SRA: return std::make_pair(C1.ashr(C2), true);
3183  case ISD::ROTL: return std::make_pair(C1.rotl(C2), true);
3184  case ISD::ROTR: return std::make_pair(C1.rotr(C2), true);
3185  case ISD::UDIV:
3186  if (!C2.getBoolValue())
3187  break;
3188  return std::make_pair(C1.udiv(C2), true);
3189  case ISD::UREM:
3190  if (!C2.getBoolValue())
3191  break;
3192  return std::make_pair(C1.urem(C2), true);
3193  case ISD::SDIV:
3194  if (!C2.getBoolValue())
3195  break;
3196  return std::make_pair(C1.sdiv(C2), true);
3197  case ISD::SREM:
3198  if (!C2.getBoolValue())
3199  break;
3200  return std::make_pair(C1.srem(C2), true);
3201  }
3202  return std::make_pair(APInt(1, 0), false);
3203 }
3204 
3206  const ConstantSDNode *Cst1,
3207  const ConstantSDNode *Cst2) {
3208  if (Cst1->isOpaque() || Cst2->isOpaque())
3209  return SDValue();
3210 
3211  std::pair<APInt, bool> Folded = FoldValue(Opcode, Cst1->getAPIntValue(),
3212  Cst2->getAPIntValue());
3213  if (!Folded.second)
3214  return SDValue();
3215  return getConstant(Folded.first, DL, VT);
3216 }
3217 
3219  SDNode *Cst1, SDNode *Cst2) {
3220  // If the opcode is a target-specific ISD node, there's nothing we can
3221  // do here and the operand rules may not line up with the below, so
3222  // bail early.
3223  if (Opcode >= ISD::BUILTIN_OP_END)
3224  return SDValue();
3225 
3226  // Handle the case of two scalars.
3227  if (const ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1)) {
3228  if (const ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2)) {
3229  if (SDValue Folded =
3230  FoldConstantArithmetic(Opcode, DL, VT, Scalar1, Scalar2)) {
3231  if (!VT.isVector())
3232  return Folded;
3233  SmallVector<SDValue, 4> Outputs;
3234  // We may have a vector type but a scalar result. Create a splat.
3235  Outputs.resize(VT.getVectorNumElements(), Outputs.back());
3236  // Build a big vector out of the scalar elements we generated.
3237  return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs);
3238  } else {
3239  return SDValue();
3240  }
3241  }
3242  }
3243 
3244  // For vectors extract each constant element into Inputs so we can constant
3245  // fold them individually.
3248  if (!BV1 || !BV2)
3249  return SDValue();
3250 
3251  assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!");
3252 
3253  EVT SVT = VT.getScalarType();
3254  SmallVector<SDValue, 4> Outputs;
3255  for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) {
3257  ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I));
3258  if (!V1 || !V2) // Not a constant, bail.
3259  return SDValue();
3260 
3261  if (V1->isOpaque() || V2->isOpaque())
3262  return SDValue();
3263 
3264  // Avoid BUILD_VECTOR nodes that perform implicit truncation.
3265  // FIXME: This is valid and could be handled by truncating the APInts.
3266  if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
3267  return SDValue();
3268 
3269  // Fold one vector element.
3270  std::pair<APInt, bool> Folded = FoldValue(Opcode, V1->getAPIntValue(),
3271  V2->getAPIntValue());
3272  if (!Folded.second)
3273  return SDValue();
3274  Outputs.push_back(getConstant(Folded.first, DL, SVT));
3275  }
3276 
3277  assert(VT.getVectorNumElements() == Outputs.size() &&
3278  "Vector size mismatch!");
3279 
3280  // We may have a vector type but a scalar result. Create a splat.
3281  Outputs.resize(VT.getVectorNumElements(), Outputs.back());
3282 
3283  // Build a big vector out of the scalar elements we generated.
3284  return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs);
3285 }
3286 
3287 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
3288  SDValue N2, const SDNodeFlags *Flags) {
3291  switch (Opcode) {
3292  default: break;
3293  case ISD::TokenFactor:
3294  assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
3295  N2.getValueType() == MVT::Other && "Invalid token factor!");
3296  // Fold trivial token factors.
3297  if (N1.getOpcode() == ISD::EntryToken) return N2;
3298  if (N2.getOpcode() == ISD::EntryToken) return N1;
3299  if (N1 == N2) return N1;
3300  break;
3301  case ISD::CONCAT_VECTORS:
3302  // Concat of UNDEFs is UNDEF.
3303  if (N1.getOpcode() == ISD::UNDEF &&
3304  N2.getOpcode() == ISD::UNDEF)
3305  return getUNDEF(VT);
3306 
3307  // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
3308  // one big BUILD_VECTOR.
3309  if (N1.getOpcode() == ISD::BUILD_VECTOR &&
3310  N2.getOpcode() == ISD::BUILD_VECTOR) {
3312  N1.getNode()->op_end());
3313  Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
3314 
3315  // BUILD_VECTOR requires all inputs to be of the same type, find the
3316  // maximum type and extend them all.
3317  EVT SVT = VT.getScalarType();
3318  for (SDValue Op : Elts)
3319  SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);
3320  if (SVT.bitsGT(VT.getScalarType()))
3321  for (SDValue &Op : Elts)
3322  Op = TLI->isZExtFree(Op.getValueType(), SVT)
3323  ? getZExtOrTrunc(Op, DL, SVT)
3324  : getSExtOrTrunc(Op, DL, SVT);
3325 
3326  return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
3327  }
3328  break;
3329  case ISD::AND:
3330  assert(VT.isInteger() && "This operator does not apply to FP types!");
3331  assert(N1.getValueType() == N2.getValueType() &&
3332  N1.getValueType() == VT && "Binary operator types must match!");
3333  // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
3334  // worth handling here.
3335  if (N2C && N2C->isNullValue())
3336  return N2;
3337  if (N2C && N2C->isAllOnesValue()) // X & -1 -> X
3338  return N1;
3339  break;
3340  case ISD::OR:
3341  case ISD::XOR:
3342  case ISD::ADD:
3343  case ISD::SUB:
3344  assert(VT.isInteger() && "This operator does not apply to FP types!");
3345  assert(N1.getValueType() == N2.getValueType() &&
3346  N1.getValueType() == VT && "Binary operator types must match!");
3347  // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
3348  // it's worth handling here.
3349  if (N2C && N2C->isNullValue())
3350  return N1;
3351  break;
3352  case ISD::UDIV:
3353  case ISD::UREM:
3354  case ISD::MULHU:
3355  case ISD::MULHS:
3356  case ISD::MUL:
3357  case ISD::SDIV:
3358  case ISD::SREM:
3359  assert(VT.isInteger() && "This operator does not apply to FP types!");
3360  assert(N1.getValueType() == N2.getValueType() &&
3361  N1.getValueType() == VT && "Binary operator types must match!");
3362  break;
3363  case ISD::FADD:
3364  case ISD::FSUB:
3365  case ISD::FMUL:
3366  case ISD::FDIV:
3367  case ISD::FREM:
3368  if (getTarget().Options.UnsafeFPMath) {
3369  if (Opcode == ISD::FADD) {
3370  // 0+x --> x
3371  if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1))
3372  if (CFP->getValueAPF().isZero())
3373  return N2;
3374  // x+0 --> x
3375  if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
3376  if (CFP->getValueAPF().isZero())
3377  return N1;
3378  } else if (Opcode == ISD::FSUB) {
3379  // x-0 --> x
3380  if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
3381  if (CFP->getValueAPF().isZero())
3382  return N1;
3383  } else if (Opcode == ISD::FMUL) {
3385  SDValue V = N2;
3386 
3387  // If the first operand isn't the constant, try the second
3388  if (!CFP) {
3389  CFP = dyn_cast<ConstantFPSDNode>(N2);
3390  V = N1;
3391  }
3392 
3393  if (CFP) {
3394  // 0*x --> 0
3395  if (CFP->isZero())
3396  return SDValue(CFP,0);
3397  // 1*x --> x
3398  if (CFP->isExactlyValue(1.0))
3399  return V;
3400  }
3401  }
3402  }
3403  assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
3404  assert(N1.getValueType() == N2.getValueType() &&
3405  N1.getValueType() == VT && "Binary operator types must match!");
3406  break;
3407  case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
3408  assert(N1.getValueType() == VT &&
3409  N1.getValueType().isFloatingPoint() &&
3410  N2.getValueType().isFloatingPoint() &&
3411  "Invalid FCOPYSIGN!");
3412  break;
3413  case ISD::SHL:
3414  case ISD::SRA:
3415  case ISD::SRL:
3416  case ISD::ROTL:
3417  case ISD::ROTR:
3418  assert(VT == N1.getValueType() &&
3419  "Shift operators return type must be the same as their first arg");
3420  assert(VT.isInteger() && N2.getValueType().isInteger() &&
3421  "Shifts only work on integers");
3422  assert((!VT.isVector() || VT == N2.getValueType()) &&
3423  "Vector shift amounts must be in the same as their first arg");
3424  // Verify that the shift amount VT is bit enough to hold valid shift
3425  // amounts. This catches things like trying to shift an i1024 value by an
3426  // i8, which is easy to fall into in generic code that uses
3427  // TLI.getShiftAmount().
3428  assert(N2.getValueType().getSizeInBits() >=
3430  "Invalid use of small shift amount with oversized value!");
3431 
3432  // Always fold shifts of i1 values so the code generator doesn't need to
3433  // handle them. Since we know the size of the shift has to be less than the
3434  // size of the value, the shift/rotate count is guaranteed to be zero.
3435  if (VT == MVT::i1)
3436  return N1;
3437  if (N2C && N2C->isNullValue())
3438  return N1;
3439  break;
3440  case ISD::FP_ROUND_INREG: {
3441  EVT EVT = cast<VTSDNode>(N2)->getVT();
3442  assert(VT == N1.getValueType() && "Not an inreg round!");
3443  assert(VT.isFloatingPoint() && EVT.isFloatingPoint() &&
3444  "Cannot FP_ROUND_INREG integer types");
3445  assert(EVT.isVector() == VT.isVector() &&
3446  "FP_ROUND_INREG type should be vector iff the operand "
3447  "type is vector!");
3448  assert((!EVT.isVector() ||
3449  EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
3450  "Vector element counts must match in FP_ROUND_INREG");
3451  assert(EVT.bitsLE(VT) && "Not rounding down!");
3452  (void)EVT;
3453  if (cast<VTSDNode>(N2)->getVT() == VT) return N1; // Not actually rounding.
3454  break;
3455  }
3456  case ISD::FP_ROUND:
3457  assert(VT.isFloatingPoint() &&
3458  N1.getValueType().isFloatingPoint() &&
3459  VT.bitsLE(N1.getValueType()) &&
3460  isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!");
3461  if (N1.getValueType() == VT) return N1; // noop conversion.
3462  break;
3463  case ISD::AssertSext:
3464  case ISD::AssertZext: {
3465  EVT EVT = cast<VTSDNode>(N2)->getVT();
3466  assert(VT == N1.getValueType() && "Not an inreg extend!");
3467  assert(VT.isInteger() && EVT.isInteger() &&
3468  "Cannot *_EXTEND_INREG FP types");
3469  assert(!EVT.isVector() &&
3470  "AssertSExt/AssertZExt type should be the vector element type "
3471  "rather than the vector type!");
3472  assert(EVT.bitsLE(VT) && "Not extending!");
3473  if (VT == EVT) return N1; // noop assertion.
3474  break;
3475  }
3476  case ISD::SIGN_EXTEND_INREG: {
3477  EVT EVT = cast<VTSDNode>(N2)->getVT();
3478  assert(VT == N1.getValueType() && "Not an inreg extend!");
3479  assert(VT.isInteger() && EVT.isInteger() &&
3480  "Cannot *_EXTEND_INREG FP types");
3481  assert(EVT.isVector() == VT.isVector() &&
3482  "SIGN_EXTEND_INREG type should be vector iff the operand "
3483  "type is vector!");
3484  assert((!EVT.isVector() ||
3485  EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
3486  "Vector element counts must match in SIGN_EXTEND_INREG");
3487  assert(EVT.bitsLE(VT) && "Not extending!");
3488  if (EVT == VT) return N1; // Not actually extending
3489 
3490  auto SignExtendInReg = [&](APInt Val) {
3491  unsigned FromBits = EVT.getScalarType().getSizeInBits();
3492  Val <<= Val.getBitWidth() - FromBits;
3493  Val = Val.ashr(Val.getBitWidth() - FromBits);
3494  return getConstant(Val, DL, VT.getScalarType());
3495  };
3496 
3497  if (N1C) {
3498  APInt Val = N1C->getAPIntValue();
3499  return SignExtendInReg(Val);
3500  }
3503  for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
3504  SDValue Op = N1.getOperand(i);
3505  if (Op.getValueType() != VT.getScalarType()) break;
3506  if (Op.getOpcode() == ISD::UNDEF) {
3507  Ops.push_back(Op);
3508  continue;
3509  }
3510  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
3511  APInt Val = C->getAPIntValue();
3512  Ops.push_back(SignExtendInReg(Val));
3513  continue;
3514  }
3515  break;
3516  }
3517  if (Ops.size() == VT.getVectorNumElements())
3518  return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
3519  }
3520  break;
3521  }
3523  // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF.
3524  if (N1.getOpcode() == ISD::UNDEF)
3525  return getUNDEF(VT);
3526 
3527  // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF
3528  if (N2C && N2C->getZExtValue() >= N1.getValueType().getVectorNumElements())
3529  return getUNDEF(VT);
3530 
3531  // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
3532  // expanding copies of large vectors from registers.
3533  if (N2C &&
3534  N1.getOpcode() == ISD::CONCAT_VECTORS &&
3535  N1.getNumOperands() > 0) {
3536  unsigned Factor =
3538  return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
3539  N1.getOperand(N2C->getZExtValue() / Factor),
3540  getConstant(N2C->getZExtValue() % Factor, DL,
3541  N2.getValueType()));
3542  }
3543 
3544  // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
3545  // expanding large vector constants.
3546  if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) {
3547  SDValue Elt = N1.getOperand(N2C->getZExtValue());
3548 
3549  if (VT != Elt.getValueType())
3550  // If the vector element type is not legal, the BUILD_VECTOR operands
3551  // are promoted and implicitly truncated, and the result implicitly
3552  // extended. Make that explicit here.
3553  Elt = getAnyExtOrTrunc(Elt, DL, VT);
3554 
3555  return Elt;
3556  }
3557 
3558  // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
3559  // operations are lowered to scalars.
3560  if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
3561  // If the indices are the same, return the inserted element else
3562  // if the indices are known different, extract the element from
3563  // the original vector.
3564  SDValue N1Op2 = N1.getOperand(2);
3565  ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2);
3566 
3567  if (N1Op2C && N2C) {
3568  if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
3569  if (VT == N1.getOperand(1).getValueType())
3570  return N1.getOperand(1);
3571  else
3572  return getSExtOrTrunc(N1.getOperand(1), DL, VT);
3573  }
3574 
3575  return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
3576  }
3577  }
3578  break;
3579  case ISD::EXTRACT_ELEMENT:
3580  assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
3581  assert(!N1.getValueType().isVector() && !VT.isVector() &&
3582  (N1.getValueType().isInteger() == VT.isInteger()) &&
3583  N1.getValueType() != VT &&
3584  "Wrong types for EXTRACT_ELEMENT!");
3585 
3586  // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
3587  // 64-bit integers into 32-bit parts. Instead of building the extract of
3588  // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
3589  if (N1.getOpcode() == ISD::BUILD_PAIR)
3590  return N1.getOperand(N2C->getZExtValue());
3591 
3592  // EXTRACT_ELEMENT of a constant int is also very common.
3593  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
3594  unsigned ElementSize = VT.getSizeInBits();
3595  unsigned Shift = ElementSize * N2C->getZExtValue();
3596  APInt ShiftedVal = C->getAPIntValue().lshr(Shift);
3597  return getConstant(ShiftedVal.trunc(ElementSize), DL, VT);
3598  }
3599  break;
3600  case ISD::EXTRACT_SUBVECTOR: {
3601  SDValue Index = N2;
3602  if (VT.isSimple() && N1.getValueType().isSimple()) {
3603  assert(VT.isVector() && N1.getValueType().isVector() &&
3604  "Extract subvector VTs must be a vectors!");
3605  assert(VT.getVectorElementType() ==
3607  "Extract subvector VTs must have the same element type!");
3608  assert(VT.getSimpleVT() <= N1.getSimpleValueType() &&
3609  "Extract subvector must be from larger vector to smaller vector!");
3610 
3611  if (isa<ConstantSDNode>(Index)) {
3612  assert((VT.getVectorNumElements() +
3613  cast<ConstantSDNode>(Index)->getZExtValue()
3615  && "Extract subvector overflow!");
3616  }
3617 
3618  // Trivial extraction.
3619  if (VT.getSimpleVT() == N1.getSimpleValueType())
3620  return N1;
3621  }
3622  break;
3623  }
3624  }
3625 
3626  // Perform trivial constant folding.
3627  if (SDValue SV =
3628  FoldConstantArithmetic(Opcode, DL, VT, N1.getNode(), N2.getNode()))
3629  return SV;
3630 
3631  // Canonicalize constant to RHS if commutative.
3632  if (N1C && !N2C && isCommutativeBinOp(Opcode)) {
3633  std::swap(N1C, N2C);
3634  std::swap(N1, N2);
3635  }
3636 
3637  // Constant fold FP operations.
3638  bool HasFPExceptions = TLI->hasFloatingPointExceptions();
3641  if (N1CFP) {
3642  if (!N2CFP && isCommutativeBinOp(Opcode)) {
3643  // Canonicalize constant to RHS if commutative.
3644  std::swap(N1CFP, N2CFP);
3645  std::swap(N1, N2);
3646  } else if (N2CFP) {
3647  APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF();
3649  switch (Opcode) {
3650  case ISD::FADD:
3652  if (!HasFPExceptions || s != APFloat::opInvalidOp)
3653  return getConstantFP(V1, DL, VT);
3654  break;
3655  case ISD::FSUB:
3657  if (!HasFPExceptions || s!=APFloat::opInvalidOp)
3658  return getConstantFP(V1, DL, VT);
3659  break;
3660  case ISD::FMUL:
3662  if (!HasFPExceptions || s!=APFloat::opInvalidOp)
3663  return getConstantFP(V1, DL, VT);
3664  break;
3665  case ISD::FDIV:
3667  if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
3668  s!=APFloat::opDivByZero)) {
3669  return getConstantFP(V1, DL, VT);
3670  }
3671  break;
3672  case ISD::FREM :
3674  if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
3675  s!=APFloat::opDivByZero)) {
3676  return getConstantFP(V1, DL, VT);
3677  }
3678  break;
3679  case ISD::FCOPYSIGN:
3680  V1.copySign(V2);
3681  return getConstantFP(V1, DL, VT);
3682  default: break;
3683  }
3684  }
3685 
3686  if (Opcode == ISD::FP_ROUND) {
3687  APFloat V = N1CFP->getValueAPF(); // make copy
3688  bool ignored;
3689  // This can return overflow, underflow, or inexact; we don't care.
3690  // FIXME need to be more flexible about rounding mode.
3691  (void)V.convert(EVTToAPFloatSemantics(VT),
3692  APFloat::rmNearestTiesToEven, &ignored);
3693  return getConstantFP(V, DL, VT);
3694  }
3695  }
3696 
3697  // Canonicalize an UNDEF to the RHS, even over a constant.
3698  if (N1.getOpcode() == ISD::UNDEF) {
3699  if (isCommutativeBinOp(Opcode)) {
3700  std::swap(N1, N2);
3701  } else {
3702  switch (Opcode) {
3703  case ISD::FP_ROUND_INREG:
3705  case ISD::SUB:
3706  case ISD::FSUB:
3707  case ISD::FDIV:
3708  case ISD::FREM:
3709  case ISD::SRA:
3710  return N1; // fold op(undef, arg2) -> undef
3711  case ISD::UDIV:
3712  case ISD::SDIV:
3713  case ISD::UREM:
3714  case ISD::SREM:
3715  case ISD::SRL:
3716  case ISD::SHL:
3717  if (!VT.isVector())
3718  return getConstant(0, DL, VT); // fold op(undef, arg2) -> 0
3719  // For vectors, we can't easily build an all zero vector, just return
3720  // the LHS.
3721  return N2;
3722  }
3723  }
3724  }
3725 
3726  // Fold a bunch of operators when the RHS is undef.
3727  if (N2.getOpcode() == ISD::UNDEF) {
3728  switch (Opcode) {
3729  case ISD::XOR:
3730  if (N1.getOpcode() == ISD::UNDEF)
3731  // Handle undef ^ undef -> 0 special case. This is a common
3732  // idiom (misuse).
3733  return getConstant(0, DL, VT);
3734  // fallthrough
3735  case ISD::ADD:
3736  case ISD::ADDC:
3737  case ISD::ADDE:
3738  case ISD::SUB:
3739  case ISD::UDIV:
3740  case ISD::SDIV:
3741  case ISD::UREM:
3742  case ISD::SREM:
3743  return N2; // fold op(arg1, undef) -> undef
3744  case ISD::FADD:
3745  case ISD::FSUB:
3746  case ISD::FMUL:
3747  case ISD::FDIV:
3748  case ISD::FREM:
3750  return N2;
3751  break;
3752  case ISD::MUL:
3753  case ISD::AND:
3754  case ISD::SRL:
3755  case ISD::SHL:
3756  if (!VT.isVector())
3757  return getConstant(0, DL, VT); // fold op(arg1, undef) -> 0
3758  // For vectors, we can't easily build an all zero vector, just return
3759  // the LHS.
3760  return N1;
3761  case ISD::OR:
3762  if (!VT.isVector())
3763  return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), DL, VT);
3764  // For vectors, we can't easily build an all one vector, just return
3765  // the LHS.
3766  return N1;
3767  case ISD::SRA:
3768  return N1;
3769  }
3770  }
3771 
3772  // Memoize this node if possible.
3773  BinarySDNode *N;
3774  SDVTList VTs = getVTList(VT);
3775  if (VT != MVT::Glue) {
3776  SDValue Ops[] = {N1, N2};
3778  AddNodeIDNode(ID, Opcode, VTs, Ops);
3779  AddNodeIDFlags(ID, Opcode, Flags);
3780  void *IP = nullptr;
3781  if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
3782  return SDValue(E, 0);
3783 
3784  N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags);
3785 
3786  CSEMap.InsertNode(N, IP);
3787  } else {
3788  N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags);
3789  }
3790 
3791  InsertNode(N);
3792  return SDValue(N, 0);
3793 }
3794 
3795 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
3796  SDValue N1, SDValue N2, SDValue N3) {
3797  // Perform various simplifications.
3799  switch (Opcode) {
3800  case ISD::FMA: {
3804  if (N1CFP && N2CFP && N3CFP) {
3805  APFloat V1 = N1CFP->getValueAPF();
3806  const APFloat &V2 = N2CFP->getValueAPF();
3807  const APFloat &V3 = N3CFP->getValueAPF();
3808  APFloat::opStatus s =
3811  return getConstantFP(V1, DL, VT);
3812  }
3813  break;
3814  }
3815  case ISD::CONCAT_VECTORS:
3816  // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
3817  // one big BUILD_VECTOR.
3818  if (N1.getOpcode() == ISD::BUILD_VECTOR &&
3819  N2.getOpcode() == ISD::BUILD_VECTOR &&
3820  N3.getOpcode() == ISD::BUILD_VECTOR) {
3822  N1.getNode()->op_end());
3823  Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
3824  Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end());
3825  return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
3826  }
3827  break;
3828  case ISD::SETCC: {
3829  // Use FoldSetCC to simplify SETCC's.
3830  SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL);
3831  if (Simp.getNode()) return Simp;
3832  break;
3833  }
3834  case ISD::SELECT:
3835  if (N1C) {
3836  if (N1C->getZExtValue())
3837  return N2; // select true, X, Y -> X
3838  return N3; // select false, X, Y -> Y
3839  }
3840 
3841  if (N2 == N3) return N2; // select C, X, X -> X
3842  break;
3843  case ISD::VECTOR_SHUFFLE:
3844  llvm_unreachable("should use getVectorShuffle constructor!");
3845  case ISD::INSERT_SUBVECTOR: {
3846  SDValue Index = N3;
3847  if (VT.isSimple() && N1.getValueType().isSimple()
3848  && N2.getValueType().isSimple()) {
3849  assert(VT.isVector() && N1.getValueType().isVector() &&
3850  N2.getValueType().isVector() &&
3851  "Insert subvector VTs must be a vectors");
3852  assert(VT == N1.getValueType() &&
3853  "Dest and insert subvector source types must match!");
3854  assert(N2.getSimpleValueType() <= N1.getSimpleValueType() &&
3855  "Insert subvector must be from smaller vector to larger vector!");
3856  if (isa<ConstantSDNode>(Index)) {
3857  assert((N2.getValueType().getVectorNumElements() +
3858  cast<ConstantSDNode>(Index)->getZExtValue()
3859  <= VT.getVectorNumElements())
3860  && "Insert subvector overflow!");
3861  }
3862 
3863  // Trivial insertion.
3864  if (VT.getSimpleVT() == N2.getSimpleValueType())
3865  return N2;
3866  }
3867  break;
3868  }
3869  case ISD::BITCAST:
3870  // Fold bit_convert nodes from a type to themselves.
3871  if (N1.getValueType() == VT)
3872  return N1;
3873  break;
3874  }
3875 
3876  // Memoize node if it doesn't produce a flag.
3877  SDNode *N;
3878  SDVTList VTs = getVTList(VT);
3879  if (VT != MVT::Glue) {
3880  SDValue Ops[] = { N1, N2, N3 };
3882  AddNodeIDNode(ID, Opcode, VTs, Ops);
3883  void *IP = nullptr;
3884  if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
3885  return SDValue(E, 0);
3886 
3887  N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
3888  DL.getDebugLoc(), VTs, N1, N2, N3);
3889  CSEMap.InsertNode(N, IP);
3890  } else {
3891  N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
3892  DL.getDebugLoc(), VTs, N1, N2, N3);
3893  }
3894 
3895  InsertNode(N);
3896  return SDValue(N, 0);
3897 }
3898 
3899 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
3900  SDValue N1, SDValue N2, SDValue N3,
3901  SDValue N4) {
3902  SDValue Ops[] = { N1, N2, N3, N4 };
3903  return getNode(Opcode, DL, VT, Ops);
3904 }
3905 
3906 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
3907  SDValue N1, SDValue N2, SDValue N3,
3908  SDValue N4, SDValue N5) {
3909  SDValue Ops[] = { N1, N2, N3, N4, N5 };
3910  return getNode(Opcode, DL, VT, Ops);
3911 }
3912 
3913 /// getStackArgumentTokenFactor - Compute a TokenFactor to force all
3914 /// the incoming stack arguments to be loaded from the stack.
3916  SmallVector<SDValue, 8> ArgChains;
3917 
3918  // Include the original chain at the beginning of the list. When this is
3919  // used by target LowerCall hooks, this helps legalize find the
3920  // CALLSEQ_BEGIN node.
3921  ArgChains.push_back(Chain);
3922 
3923  // Add a chain value for each stack argument.
3924  for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(),
3925  UE = getEntryNode().getNode()->use_end(); U != UE; ++U)
3926  if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
3927  if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
3928  if (FI->getIndex() < 0)
3929  ArgChains.push_back(SDValue(L, 1));
3930 
3931  // Build a tokenfactor for all the chains.
3932  return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
3933 }
3934 
3935 /// getMemsetValue - Vectorized representation of the memset value
3936 /// operand.
3938  SDLoc dl) {
3939  assert(Value.getOpcode() != ISD::UNDEF);
3940 
3941  unsigned NumBits = VT.getScalarType().getSizeInBits();
3942  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
3943  assert(C->getAPIntValue().getBitWidth() == 8);
3944  APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
3945  if (VT.isInteger())
3946  return DAG.getConstant(Val, dl, VT);
3947  return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), dl,
3948  VT);
3949  }
3950 
3951  assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
3952  EVT IntVT = VT.getScalarType();
3953  if (!IntVT.isInteger())
3954  IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits());
3955 
3956  Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value);
3957  if (NumBits > 8) {
3958  // Use a multiplication with 0x010101... to extend the input to the
3959  // required length.
3960  APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
3961  Value = DAG.getNode(ISD::MUL, dl, IntVT, Value,
3962  DAG.getConstant(Magic, dl, IntVT));
3963  }
3964 
3965  if (VT != Value.getValueType() && !VT.isInteger())
3966  Value = DAG.getNode(ISD::BITCAST, dl, VT.getScalarType(), Value);
3967  if (VT != Value.getValueType()) {
3968  assert(VT.getVectorElementType() == Value.getValueType() &&
3969  "value type should be one vector element here");
3970  SmallVector<SDValue, 8> BVOps(VT.getVectorNumElements(), Value);
3971  Value = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, BVOps);
3972  }
3973 
3974  return Value;
3975 }
3976 
3977 /// getMemsetStringVal - Similar to getMemsetValue. Except this is only
3978 /// used when a memcpy is turned into a memset when the source is a constant
3979 /// string ptr.
3981  const TargetLowering &TLI, StringRef Str) {
3982  // Handle vector with all elements zero.
3983  if (Str.empty()) {
3984  if (VT.isInteger())
3985  return DAG.getConstant(0, dl, VT);
3986  else if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
3987  return DAG.getConstantFP(0.0, dl, VT);
3988  else if (VT.isVector()) {
3989  unsigned NumElts = VT.getVectorNumElements();
3990  MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
3991  return DAG.getNode(ISD::BITCAST, dl, VT,
3992  DAG.getConstant(0, dl,
3994  EltVT, NumElts)));
3995  } else
3996  llvm_unreachable("Expected type!");
3997  }
3998 
3999  assert(!VT.isVector() && "Can't handle vector type here!");
4000  unsigned NumVTBits = VT.getSizeInBits();
4001  unsigned NumVTBytes = NumVTBits / 8;
4002  unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size()));
4003 
4004  APInt Val(NumVTBits, 0);
4005  if (DAG.getDataLayout().isLittleEndian()) {
4006  for (unsigned i = 0; i != NumBytes; ++i)
4007  Val |= (uint64_t)(unsigned char)Str[i] << i*8;
4008  } else {
4009  for (unsigned i = 0; i != NumBytes; ++i)
4010  Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8;
4011  }
4012 
4013  // If the "cost" of materializing the integer immediate is less than the cost
4014  // of a load, then it is cost effective to turn the load into the immediate.
4015  Type *Ty = VT.getTypeForEVT(*DAG.getContext());
4016  if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
4017  return DAG.getConstant(Val, dl, VT);
4018  return SDValue(nullptr, 0);
4019 }
4020 
4021 /// getMemBasePlusOffset - Returns base and offset node for the
4022 ///
4023 static SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, SDLoc dl,
4024  SelectionDAG &DAG) {
4025  EVT VT = Base.getValueType();
4026  return DAG.getNode(ISD::ADD, dl,
4027  VT, Base, DAG.getConstant(Offset, dl, VT));
4028 }
4029 
4030 /// isMemSrcFromString - Returns true if memcpy source is a string constant.
4031 ///
4032 static bool isMemSrcFromString(SDValue Src, StringRef &Str) {
4033  unsigned SrcDelta = 0;
4034  GlobalAddressSDNode *G = nullptr;
4035  if (Src.getOpcode() == ISD::GlobalAddress)
4036  G = cast<GlobalAddressSDNode>(Src);
4037  else if (Src.getOpcode() == ISD::ADD &&
4038  Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
4039  Src.getOperand(1).getOpcode() == ISD::Constant) {
4040  G = cast<GlobalAddressSDNode>(Src.getOperand(0));
4041  SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue();
4042  }
4043  if (!G)
4044  return false;
4045 
4046  return getConstantStringInfo(G->getGlobal(), Str, SrcDelta, false);
4047 }
4048 
4049 /// Determines the optimal series of memory ops to replace the memset / memcpy.
4050 /// Return true if the number of memory ops is below the threshold (Limit).
4051 /// It returns the types of the sequence of memory ops to perform
4052 /// memset / memcpy by reference.
4053 static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
4054  unsigned Limit, uint64_t Size,
4055  unsigned DstAlign, unsigned SrcAlign,
4056  bool IsMemset,
4057  bool ZeroMemset,
4058  bool MemcpyStrSrc,
4059  bool AllowOverlap,
4060  SelectionDAG &DAG,
4061  const TargetLowering &TLI) {
4062  assert((SrcAlign == 0 || SrcAlign >= DstAlign) &&
4063  "Expecting memcpy / memset source to meet alignment requirement!");
4064  // If 'SrcAlign' is zero, that means the memory operation does not need to
4065  // load the value, i.e. memset or memcpy from constant string. Otherwise,
4066  // it's the inferred alignment of the source. 'DstAlign', on the other hand,
4067  // is the specified alignment of the memory operation. If it is zero, that
4068  // means it's possible to change the alignment of the destination.
4069  // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does
4070  // not need to be loaded.
4071  EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign,
4072  IsMemset, ZeroMemset, MemcpyStrSrc,
4073  DAG.getMachineFunction());
4074 
4075  if (VT == MVT::Other) {
4076  unsigned AS = 0;
4077  if (DstAlign >= DAG.getDataLayout().getPointerPrefAlignment(AS) ||
4078  TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign)) {
4079  VT = TLI.getPointerTy(DAG.getDataLayout());
4080  } else {
4081  switch (DstAlign & 7) {
4082  case 0: VT = MVT::i64; break;
4083  case 4: VT = MVT::i32; break;
4084  case 2: VT = MVT::i16; break;
4085  default: VT = MVT::i8; break;
4086  }
4087  }
4088 
4089  MVT LVT = MVT::i64;
4090  while (!TLI.isTypeLegal(LVT))
4091  LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
4092  assert(LVT.isInteger());
4093 
4094  if (VT.bitsGT(LVT))
4095  VT = LVT;
4096  }
4097 
4098  unsigned NumMemOps = 0;
4099  while (Size != 0) {
4100  unsigned VTSize = VT.getSizeInBits() / 8;
4101  while (VTSize > Size) {
4102  // For now, only use non-vector load / store's for the left-over pieces.
4103  EVT NewVT = VT;
4104  unsigned NewVTSize;
4105 
4106  bool Found = false;
4107  if (VT.isVector() || VT.isFloatingPoint()) {
4108  NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
4109  if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) &&
4110  TLI.isSafeMemOpType(NewVT.getSimpleVT()))
4111  Found = true;
4112  else if (NewVT == MVT::i64 &&
4114  TLI.isSafeMemOpType(MVT::f64)) {
4115  // i64 is usually not legal on 32-bit targets, but f64 may be.
4116  NewVT = MVT::f64;
4117  Found = true;
4118  }
4119  }
4120 
4121  if (!Found) {
4122  do {
4123  NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
4124  if (NewVT == MVT::i8)
4125  break;
4126  } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT()));
4127  }
4128  NewVTSize = NewVT.getSizeInBits() / 8;
4129 
4130  // If the new VT cannot cover all of the remaining bits, then consider
4131  // issuing a (or a pair of) unaligned and overlapping load / store.
4132  // FIXME: Only does this for 64-bit or more since we don't have proper
4133  // cost model for unaligned load / store.
4134  bool Fast;
4135  unsigned AS = 0;
4136  if (NumMemOps && AllowOverlap &&
4137  VTSize >= 8 && NewVTSize < Size &&
4138  TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign, &Fast) && Fast)
4139  VTSize = Size;
4140  else {
4141  VT = NewVT;
4142  VTSize = NewVTSize;
4143  }
4144  }
4145 
4146  if (++NumMemOps > Limit)
4147  return false;
4148 
4149  MemOps.push_back(VT);
4150  Size -= VTSize;
4151  }
4152 
4153  return true;
4154 }
4155 
4157  SDValue Chain, SDValue Dst,
4158  SDValue Src, uint64_t Size,
4159  unsigned Align, bool isVol,
4160  bool AlwaysInline,
4161  MachinePointerInfo DstPtrInfo,
4162  MachinePointerInfo SrcPtrInfo) {
4163  // Turn a memcpy of undef to nop.
4164  if (Src.getOpcode() == ISD::UNDEF)
4165  return Chain;
4166 
4167  // Expand memcpy to a series of load and store ops if the size operand falls
4168  // below a certain threshold.
4169  // TODO: In the AlwaysInline case, if the size is big then generate a loop
4170  // rather than maybe a humongous number of loads and stores.
4171  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4172  std::vector<EVT> MemOps;
4173  bool DstAlignCanChange = false;
4174  MachineFunction &MF = DAG.getMachineFunction();
4175  MachineFrameInfo *MFI = MF.getFrameInfo();
4178  if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
4179  DstAlignCanChange = true;
4180  unsigned SrcAlign = DAG.InferPtrAlignment(Src);
4181  if (Align > SrcAlign)
4182  SrcAlign = Align;
4183  StringRef Str;
4184  bool CopyFromStr = isMemSrcFromString(Src, Str);
4185  bool isZeroStr = CopyFromStr && Str.empty();
4186  unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
4187 
4188  if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
4189  (DstAlignCanChange ? 0 : Align),
4190  (isZeroStr ? 0 : SrcAlign),
4191  false, false, CopyFromStr, true, DAG, TLI))
4192  return SDValue();
4193 
4194  if (DstAlignCanChange) {
4195  Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
4196  unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty);
4197 
4198  // Don't promote to an alignment that would require dynamic stack
4199  // realignment.
4200  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
4201  if (!TRI->needsStackRealignment(MF))
4202  while (NewAlign > Align &&
4204  NewAlign /= 2;
4205 
4206  if (NewAlign > Align) {
4207  // Give the stack frame object a larger alignment if needed.
4208  if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
4209  MFI->setObjectAlignment(FI->getIndex(), NewAlign);
4210  Align = NewAlign;
4211  }
4212  }
4213 
4214  SmallVector<SDValue, 8> OutChains;
4215  unsigned NumMemOps = MemOps.size();
4216  uint64_t SrcOff = 0, DstOff = 0;
4217  for (unsigned i = 0; i != NumMemOps; ++i) {
4218  EVT VT = MemOps[i];
4219  unsigned VTSize = VT.getSizeInBits() / 8;
4220  SDValue Value, Store;
4221 
4222  if (VTSize > Size) {
4223  // Issuing an unaligned load / store pair that overlaps with the previous
4224  // pair. Adjust the offset accordingly.
4225  assert(i == NumMemOps-1 && i != 0);
4226  SrcOff -= VTSize - Size;
4227  DstOff -= VTSize - Size;
4228  }
4229 
4230  if (CopyFromStr &&
4231  (isZeroStr || (VT.isInteger() && !VT.isVector()))) {
4232  // It's unlikely a store of a vector immediate can be done in a single
4233  // instruction. It would require a load from a constantpool first.
4234  // We only handle zero vectors here.
4235  // FIXME: Handle other cases where store of vector immediate is done in
4236  // a single instruction.
4237  Value = getMemsetStringVal(VT, dl, DAG, TLI, Str.substr(SrcOff));
4238  if (Value.getNode())
4239  Store = DAG.getStore(Chain, dl, Value,
4240  getMemBasePlusOffset(Dst, DstOff, dl, DAG),
4241  DstPtrInfo.getWithOffset(DstOff), isVol,
4242  false, Align);
4243  }
4244 
4245  if (!Store.getNode()) {
4246  // The type might not be legal for the target. This should only happen
4247  // if the type is smaller than a legal type, as on PPC, so the right
4248  // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
4249  // to Load/Store if NVT==VT.
4250  // FIXME does the case above also need this?
4251  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4252  assert(NVT.bitsGE(VT));
4253  Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain,
4254  getMemBasePlusOffset(Src, SrcOff, dl, DAG),
4255  SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false,
4256  false, MinAlign(SrcAlign, SrcOff));
4257  Store = DAG.getTruncStore(Chain, dl, Value,
4258  getMemBasePlusOffset(Dst, DstOff, dl, DAG),
4259  DstPtrInfo.getWithOffset(DstOff), VT, isVol,
4260  false, Align);
4261  }
4262  OutChains.push_back(Store);
4263  SrcOff += VTSize;
4264  DstOff += VTSize;
4265  Size -= VTSize;
4266  }
4267 
4268  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
4269 }
4270 
4272  SDValue Chain, SDValue Dst,
4273  SDValue Src, uint64_t Size,
4274  unsigned Align, bool isVol,
4275  bool AlwaysInline,
4276  MachinePointerInfo DstPtrInfo,
4277  MachinePointerInfo SrcPtrInfo) {
4278  // Turn a memmove of undef to nop.
4279  if (Src.getOpcode() == ISD::UNDEF)
4280  return Chain;
4281 
4282  // Expand memmove to a series of load and store ops if the size operand falls
4283  // below a certain threshold.
4284  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4285  std::vector<EVT> MemOps;
4286  bool DstAlignCanChange = false;
4287  MachineFunction &MF = DAG.getMachineFunction();
4288  MachineFrameInfo *MFI = MF.getFrameInfo();
4291  if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
4292  DstAlignCanChange = true;
4293  unsigned SrcAlign = DAG.InferPtrAlignment(Src);
4294  if (Align > SrcAlign)
4295  SrcAlign = Align;
4296  unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
4297 
4298  if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
4299  (DstAlignCanChange ? 0 : Align), SrcAlign,
4300  false, false, false, false, DAG, TLI))
4301  return SDValue();
4302 
4303  if (DstAlignCanChange) {
4304  Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
4305  unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty);
4306  if (NewAlign > Align) {
4307  // Give the stack frame object a larger alignment if needed.
4308  if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
4309  MFI->setObjectAlignment(FI->getIndex(), NewAlign);
4310  Align = NewAlign;
4311  }
4312  }
4313 
4314  uint64_t SrcOff = 0, DstOff = 0;
4315  SmallVector<SDValue, 8> LoadValues;
4316  SmallVector<SDValue, 8> LoadChains;
4317  SmallVector<SDValue, 8> OutChains;
4318  unsigned NumMemOps = MemOps.size();
4319  for (unsigned i = 0; i < NumMemOps; i++) {
4320  EVT VT = MemOps[i];
4321  unsigned VTSize = VT.getSizeInBits() / 8;
4322  SDValue Value;
4323 
4324  Value = DAG.getLoad(VT, dl, Chain,
4325  getMemBasePlusOffset(Src, SrcOff, dl, DAG),
4326  SrcPtrInfo.getWithOffset(SrcOff), isVol,
4327  false, false, SrcAlign);
4328  LoadValues.push_back(Value);
4329  LoadChains.push_back(Value.getValue(1));
4330  SrcOff += VTSize;
4331  }
4332  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
4333  OutChains.clear();
4334  for (unsigned i = 0; i < NumMemOps; i++) {
4335  EVT VT = MemOps[i];
4336  unsigned VTSize = VT.getSizeInBits() / 8;
4337  SDValue Store;
4338 
4339  Store = DAG.getStore(Chain, dl, LoadValues[i],
4340  getMemBasePlusOffset(Dst, DstOff, dl, DAG),
4341  DstPtrInfo.getWithOffset(DstOff), isVol, false, Align);
4342  OutChains.push_back(Store);
4343  DstOff += VTSize;
4344  }
4345 
4346  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
4347 }
4348 
4349 /// \brief Lower the call to 'memset' intrinsic function into a series of store
4350 /// operations.
4351 ///
4352 /// \param DAG Selection DAG where lowered code is placed.
4353 /// \param dl Link to corresponding IR location.
4354 /// \param Chain Control flow dependency.
4355 /// \param Dst Pointer to destination memory location.
4356 /// \param Src Value of byte to write into the memory.
4357 /// \param Size Number of bytes to write.
4358 /// \param Align Alignment of the destination in bytes.
4359 /// \param isVol True if destination is volatile.
4360 /// \param DstPtrInfo IR information on the memory pointer.
4361 /// \returns New head in the control flow, if lowering was successful, empty
4362 /// SDValue otherwise.
4363 ///
4364 /// The function tries to replace 'llvm.memset' intrinsic with several store
4365 /// operations and value calculation code. This is usually profitable for small
4366 /// memory size.
4368  SDValue Chain, SDValue Dst,
4369  SDValue Src, uint64_t Size,
4370  unsigned Align, bool isVol,
4371  MachinePointerInfo DstPtrInfo) {
4372  // Turn a memset of undef to nop.
4373  if (Src.getOpcode() == ISD::UNDEF)
4374  return Chain;
4375 
4376  // Expand memset to a series of load/store ops if the size operand
4377  // falls below a certain threshold.
4378  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4379  std::vector<EVT> MemOps;
4380  bool DstAlignCanChange = false;
4381  MachineFunction &MF = DAG.getMachineFunction();
4382  MachineFrameInfo *MFI = MF.getFrameInfo();
4385  if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
4386  DstAlignCanChange = true;
4387  bool IsZeroVal =
4388  isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
4389  if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize),
4390  Size, (DstAlignCanChange ? 0 : Align), 0,
4391  true, IsZeroVal, false, true, DAG, TLI))
4392  return SDValue();
4393 
4394  if (DstAlignCanChange) {
4395  Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
4396  unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty);
4397  if (NewAlign > Align) {
4398  // Give the stack frame object a larger alignment if needed.
4399  if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
4400  MFI->setObjectAlignment(FI->getIndex(), NewAlign);
4401  Align = NewAlign;
4402  }
4403  }
4404 
4405  SmallVector<SDValue, 8> OutChains;
4406  uint64_t DstOff = 0;
4407  unsigned NumMemOps = MemOps.size();
4408 
4409  // Find the largest store and generate the bit pattern for it.
4410  EVT LargestVT = MemOps[0];
4411  for (unsigned i = 1; i < NumMemOps; i++)
4412  if (MemOps[i].bitsGT(LargestVT))
4413  LargestVT = MemOps[i];
4414  SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
4415 
4416  for (unsigned i = 0; i < NumMemOps; i++) {
4417  EVT VT = MemOps[i];
4418  unsigned VTSize = VT.getSizeInBits() / 8;
4419  if (VTSize > Size) {
4420  // Issuing an unaligned load / store pair that overlaps with the previous
4421  // pair. Adjust the offset accordingly.
4422  assert(i == NumMemOps-1 && i != 0);
4423  DstOff -= VTSize - Size;
4424  }
4425 
4426  // If this store is smaller than the largest store see whether we can get
4427  // the smaller value for free with a truncate.
4428  SDValue Value = MemSetValue;
4429  if (VT.bitsLT(LargestVT)) {
4430  if (!LargestVT.isVector() && !VT.isVector() &&
4431  TLI.isTruncateFree(LargestVT, VT))
4432  Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
4433  else
4434  Value = getMemsetValue(Src, VT, DAG, dl);
4435  }
4436  assert(Value.getValueType() == VT && "Value with wrong type.");
4437  SDValue Store = DAG.getStore(Chain, dl, Value,
4438  getMemBasePlusOffset(Dst, DstOff, dl, DAG),
4439  DstPtrInfo.getWithOffset(DstOff),
4440  isVol, false, Align);
4441  OutChains.push_back(Store);
4442  DstOff += VT.getSizeInBits() / 8;
4443  Size -= VTSize;
4444  }
4445 
4446  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
4447 }
4448 
4450  SDValue Src, SDValue Size,
4451  unsigned Align, bool isVol, bool AlwaysInline,
4452  bool isTailCall, MachinePointerInfo DstPtrInfo,
4453  MachinePointerInfo SrcPtrInfo) {
4454  assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
4455 
4456  // Check to see if we should lower the memcpy to loads and stores first.
4457  // For cases within the target-specified limits, this is the best choice.
4458  ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
4459  if (ConstantSize) {
4460  // Memcpy with size zero? Just return the original chain.
4461  if (ConstantSize->isNullValue())
4462  return Chain;
4463 
4464  SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
4465  ConstantSize->getZExtValue(),Align,
4466  isVol, false, DstPtrInfo, SrcPtrInfo);
4467  if (Result.getNode())
4468  return Result;
4469  }
4470 
4471  // Then check to see if we should lower the memcpy with target-specific
4472  // code. If the target chooses to do this, this is the next best.
4473  if (TSI) {
4474  SDValue Result = TSI->EmitTargetCodeForMemcpy(
4475  *this, dl, Chain, Dst, Src, Size, Align, isVol, AlwaysInline,
4476  DstPtrInfo, SrcPtrInfo);
4477  if (Result.getNode())
4478  return Result;
4479  }
4480 
4481  // If we really need inline code and the target declined to provide it,
4482  // use a (potentially long) sequence of loads and stores.
4483  if (AlwaysInline) {
4484  assert(ConstantSize && "AlwaysInline requires a constant size!");
4485  return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
4486  ConstantSize->getZExtValue(), Align, isVol,
4487  true, DstPtrInfo, SrcPtrInfo);
4488  }
4489 
4490  // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
4491  // memcpy is not guaranteed to be safe. libc memcpys aren't required to
4492  // respect volatile, so they may do things like read or write memory
4493  // beyond the given memory regions. But fixing this isn't easy, and most
4494  // people don't care.
4495 
4496  // Emit a library call.
4499  Entry.Ty = getDataLayout().getIntPtrType(*getContext());
4500  Entry.Node = Dst; Args.push_back(Entry);
4501  Entry.Node = Src; Args.push_back(Entry);
4502  Entry.Node = Size; Args.push_back(Entry);
4503  // FIXME: pass in SDLoc
4505  CLI.setDebugLoc(dl)
4506  .setChain(Chain)
4510  TLI->getPointerTy(getDataLayout())),
4511  std::move(Args), 0)
4512  .setDiscardResult()
4513  .setTailCall(isTailCall);
4514 
4515  std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
4516  return CallResult.second;
4517 }
4518 
4520  SDValue Src, SDValue Size,
4521  unsigned Align, bool isVol, bool isTailCall,
4522  MachinePointerInfo DstPtrInfo,
4523  MachinePointerInfo SrcPtrInfo) {
4524  assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
4525 
4526  // Check to see if we should lower the memmove to loads and stores first.
4527  // For cases within the target-specified limits, this is the best choice.
4528  ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
4529  if (ConstantSize) {
4530  // Memmove with size zero? Just return the original chain.
4531  if (ConstantSize->isNullValue())
4532  return Chain;
4533 
4534  SDValue Result =
4535  getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src,
4536  ConstantSize->getZExtValue(), Align, isVol,
4537  false, DstPtrInfo, SrcPtrInfo);
4538  if (Result.getNode())
4539  return Result;
4540  }
4541 
4542  // Then check to see if we should lower the memmove with target-specific
4543  // code. If the target chooses to do this, this is the next best.
4544  if (TSI) {
4545  SDValue Result = TSI->EmitTargetCodeForMemmove(
4546  *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo, SrcPtrInfo);
4547  if (Result.getNode())
4548  return Result;
4549  }
4550 
4551  // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
4552  // not be safe. See memcpy above for more details.
4553 
4554  // Emit a library call.
4557  Entry.Ty = getDataLayout().getIntPtrType(*getContext());
4558  Entry.Node = Dst; Args.push_back(Entry);
4559  Entry.Node = Src; Args.push_back(Entry);
4560  Entry.Node = Size; Args.push_back(Entry);
4561  // FIXME: pass in SDLoc
4563  CLI.setDebugLoc(dl)
4564  .setChain(Chain)
4568  TLI->getPointerTy(getDataLayout())),
4569  std::move(Args), 0)
4570  .setDiscardResult()
4571  .setTailCall(isTailCall);
4572 
4573  std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
4574  return CallResult.second;
4575 }
4576 
4578  SDValue Src, SDValue Size,
4579  unsigned Align, bool isVol, bool isTailCall,
4580  MachinePointerInfo DstPtrInfo) {
4581  assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
4582 
4583  // Check to see if we should lower the memset to stores first.
4584  // For cases within the target-specified limits, this is the best choice.
4585  ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
4586  if (ConstantSize) {
4587  // Memset with size zero? Just return the original chain.
4588  if (ConstantSize->isNullValue())
4589  return Chain;
4590 
4591  SDValue Result =
4592  getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
4593  Align, isVol, DstPtrInfo);
4594 
4595  if (Result.getNode())
4596  return Result;
4597  }
4598 
4599  // Then check to see if we should lower the memset with target-specific
4600  // code. If the target chooses to do this, this is the next best.
4601  if (TSI) {
4602  SDValue Result = TSI->EmitTargetCodeForMemset(
4603  *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo);
4604  if (Result.getNode())
4605  return Result;
4606  }
4607 
4608  // Emit a library call.
4609  Type *IntPtrTy = getDataLayout().getIntPtrType(*getContext());
4612  Entry.Node = Dst; Entry.Ty = IntPtrTy;
4613  Args.push_back(Entry);
4614  Entry.Node = Src;
4615  Entry.Ty = Src.getValueType().getTypeForEVT(*getContext());
4616  Args.push_back(Entry);
4617  Entry.Node = Size;
4618  Entry.Ty = IntPtrTy;
4619  Args.push_back(Entry);
4620 
4621  // FIXME: pass in SDLoc
4623  CLI.setDebugLoc(dl)
4624  .setChain(Chain)
4628  TLI->getPointerTy(getDataLayout())),
4629  std::move(Args), 0)
4630  .setDiscardResult()
4631  .setTailCall(isTailCall);
4632 
4633  std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
4634  return CallResult.second;
4635 }
4636 
4637 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4638  SDVTList VTList, ArrayRef<SDValue> Ops,
4639  MachineMemOperand *MMO,
4640  AtomicOrdering SuccessOrdering,
4641  AtomicOrdering FailureOrdering,
4642  SynchronizationScope SynchScope) {
4644  ID.AddInteger(MemVT.getRawBits());
4645  AddNodeIDNode(ID, Opcode, VTList, Ops);
4646  ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
4647  void* IP = nullptr;
4648  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
4649  cast<AtomicSDNode>(E)->refineAlignment(MMO);
4650  return SDValue(E, 0);
4651  }
4652 
4653  // Allocate the operands array for the node out of the BumpPtrAllocator, since
4654  // SDNode doesn't have access to it. This memory will be "leaked" when
4655  // the node is deallocated, but recovered when the allocator is released.
4656  // If the number of operands is less than 5 we use AtomicSDNode's internal
4657  // storage.
4658  unsigned NumOps = Ops.size();
4659  SDUse *DynOps = NumOps > 4 ? OperandAllocator.Allocate<SDUse>(NumOps)
4660  : nullptr;
4661 
4662  SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(),
4663  dl.getDebugLoc(), VTList, MemVT,
4664  Ops.data(), DynOps, NumOps, MMO,
4665  SuccessOrdering, FailureOrdering,
4666  SynchScope);
4667  CSEMap.InsertNode(N, IP);
4668  InsertNode(N);
4669  return SDValue(N, 0);
4670 }
4671 
4672 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4673  SDVTList VTList, ArrayRef<SDValue> Ops,
4674  MachineMemOperand *MMO,
4675  AtomicOrdering Ordering,
4676  SynchronizationScope SynchScope) {
4677  return getAtomic(Opcode, dl, MemVT, VTList, Ops, MMO, Ordering,
4678  Ordering, SynchScope);
4679 }
4680 
4682  unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs, SDValue Chain,
4683  SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo,
4684  unsigned Alignment, AtomicOrdering SuccessOrdering,
4685  AtomicOrdering FailureOrdering, SynchronizationScope SynchScope) {
4686  assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
4688  assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
4689 
4690  if (Alignment == 0) // Ensure that codegen never sees alignment 0
4691  Alignment = getEVTAlignment(MemVT);
4692 
4694 
4695  // FIXME: Volatile isn't really correct; we should keep track of atomic
4696  // orderings in the memoperand.
4697  unsigned Flags = MachineMemOperand::MOVolatile;
4698  Flags |= MachineMemOperand::MOLoad;
4699  Flags |= MachineMemOperand::MOStore;
4700 
4701  MachineMemOperand *MMO =
4702  MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
4703 
4704  return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO,
4705  SuccessOrdering, FailureOrdering, SynchScope);
4706 }
4707 
4709  SDVTList VTs, SDValue Chain, SDValue Ptr,
4710  SDValue Cmp, SDValue Swp,
4711  MachineMemOperand *MMO,
4712  AtomicOrdering SuccessOrdering,
4713  AtomicOrdering FailureOrdering,
4714  SynchronizationScope SynchScope) {
4715  assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
4717  assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
4718 
4719  SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
4720  return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO,
4721  SuccessOrdering, FailureOrdering, SynchScope);
4722 }
4723 
4724 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4725  SDValue Chain,
4726  SDValue Ptr, SDValue Val,
4727  const Value* PtrVal,
4728  unsigned Alignment,
4729  AtomicOrdering Ordering,
4730  SynchronizationScope SynchScope) {
4731  if (Alignment == 0) // Ensure that codegen never sees alignment 0
4732  Alignment = getEVTAlignment(MemVT);
4733 
4735  // An atomic store does not load. An atomic load does not store.
4736  // (An atomicrmw obviously both loads and stores.)
4737  // For now, atomics are considered to be volatile always, and they are
4738  // chained as such.
4739  // FIXME: Volatile isn't really correct; we should keep track of atomic
4740  // orderings in the memoperand.
4741  unsigned Flags = MachineMemOperand::MOVolatile;
4742  if (Opcode != ISD::ATOMIC_STORE)
4743  Flags |= MachineMemOperand::MOLoad;
4744  if (Opcode != ISD::ATOMIC_LOAD)
4745  Flags |= MachineMemOperand::MOStore;
4746 
4747  MachineMemOperand *MMO =
4748  MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
4749  MemVT.getStoreSize(), Alignment);
4750 
4751  return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO,
4752  Ordering, SynchScope);
4753 }
4754 
4755 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4756  SDValue Chain,
4757  SDValue Ptr, SDValue Val,
4758  MachineMemOperand *MMO,
4759  AtomicOrdering Ordering,
4760  SynchronizationScope SynchScope) {
4761  assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
4762  Opcode == ISD::ATOMIC_LOAD_SUB ||
4763  Opcode == ISD::ATOMIC_LOAD_AND ||
4764  Opcode == ISD::ATOMIC_LOAD_OR ||
4765  Opcode == ISD::ATOMIC_LOAD_XOR ||
4766  Opcode == ISD::ATOMIC_LOAD_NAND ||
4767  Opcode == ISD::ATOMIC_LOAD_MIN ||
4768  Opcode == ISD::ATOMIC_LOAD_MAX ||
4769  Opcode == ISD::ATOMIC_LOAD_UMIN ||
4770  Opcode == ISD::ATOMIC_LOAD_UMAX ||
4771  Opcode == ISD::ATOMIC_SWAP ||
4772  Opcode == ISD::ATOMIC_STORE) &&
4773  "Invalid Atomic Op");
4774 
4775  EVT VT = Val.getValueType();
4776 
4777  SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
4778  getVTList(VT, MVT::Other);
4779  SDValue Ops[] = {Chain, Ptr, Val};
4780  return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
4781 }
4782 
4783 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
4784  EVT VT, SDValue Chain,
4785  SDValue Ptr,
4786  MachineMemOperand *MMO,
4787  AtomicOrdering Ordering,
4788  SynchronizationScope SynchScope) {
4789  assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
4790 
4791  SDVTList VTs = getVTList(VT, MVT::Other);
4792  SDValue Ops[] = {Chain, Ptr};
4793  return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
4794 }
4795 
4796 /// getMergeValues - Create a MERGE_VALUES node from the given operands.
4798  if (Ops.size() == 1)
4799  return Ops[0];
4800 
4801  SmallVector<EVT, 4> VTs;
4802  VTs.reserve(Ops.size());
4803  for (unsigned i = 0; i < Ops.size(); ++i)
4804  VTs.push_back(Ops[i].getValueType());
4805  return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
4806 }
4807 
4808 SDValue
4810  ArrayRef<SDValue> Ops,
4811  EVT MemVT, MachinePointerInfo PtrInfo,
4812  unsigned Align, bool Vol,
4813  bool ReadMem, bool WriteMem, unsigned Size) {
4814  if (Align == 0) // Ensure that codegen never sees alignment 0
4815  Align = getEVTAlignment(MemVT);
4816 
4818  unsigned Flags = 0;
4819  if (WriteMem)
4820  Flags |= MachineMemOperand::MOStore;
4821  if (ReadMem)
4822  Flags |= MachineMemOperand::MOLoad;
4823  if (Vol)
4825  if (!Size)
4826  Size = MemVT.getStoreSize();
4827  MachineMemOperand *MMO =
4828  MF.getMachineMemOperand(PtrInfo, Flags, Size, Align);
4829 
4830  return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
4831 }
4832 
4833 SDValue
4835  ArrayRef<SDValue> Ops, EVT MemVT,
4836  MachineMemOperand *MMO) {
4837  assert((Opcode == ISD::INTRINSIC_VOID ||
4838  Opcode == ISD::INTRINSIC_W_CHAIN ||
4839  Opcode == ISD::PREFETCH ||
4840  Opcode == ISD::LIFETIME_START ||
4841  Opcode == ISD::LIFETIME_END ||
4842  (Opcode <= INT_MAX &&
4843  (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
4844  "Opcode is not a memory-accessing opcode!");
4845 
4846  // Memoize the node unless it returns a flag.
4848  if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
4850  AddNodeIDNode(ID, Opcode, VTList, Ops);
4851  ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
4852  void *IP = nullptr;
4853  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
4854  cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
4855  return SDValue(E, 0);
4856  }
4857 
4858  N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
4859  dl.getDebugLoc(), VTList, Ops,
4860  MemVT, MMO);
4861  CSEMap.InsertNode(N, IP);
4862  } else {
4863  N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
4864  dl.getDebugLoc(), VTList, Ops,
4865  MemVT, MMO);
4866  }
4867  InsertNode(N);
4868  return SDValue(N, 0);
4869 }
4870 
4871 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
4872 /// MachinePointerInfo record from it. This is particularly useful because the
4873 /// code generator has many cases where it doesn't bother passing in a
4874 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
4875 static MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) {
4876  // If this is FI+Offset, we can model it.
4877  if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
4878  return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset);
4879 
4880  // If this is (FI+Offset1)+Offset2, we can model it.
4881  if (Ptr.getOpcode() != ISD::ADD ||
4882  !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
4883  !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
4884  return MachinePointerInfo();
4885 
4886  int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
4887  return MachinePointerInfo::getFixedStack(FI, Offset+
4888  cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
4889 }
4890 
4891 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
4892 /// MachinePointerInfo record from it. This is particularly useful because the
4893 /// code generator has many cases where it doesn't bother passing in a
4894 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
4896  // If the 'Offset' value isn't a constant, we can't handle this.
4897  if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
4898  return InferPointerInfo(Ptr, OffsetNode->getSExtValue());
4899  if (OffsetOp.getOpcode() == ISD::UNDEF)
4900  return InferPointerInfo(Ptr);
4901  return MachinePointerInfo();
4902 }
4903 
4904 
4905 SDValue
4907  EVT VT, SDLoc dl, SDValue Chain,
4908  SDValue Ptr, SDValue Offset,
4909  MachinePointerInfo PtrInfo, EVT MemVT,
4910  bool isVolatile, bool isNonTemporal, bool isInvariant,
4911  unsigned Alignment, const AAMDNodes &AAInfo,
4912  const MDNode *Ranges) {
4913  assert(Chain.getValueType() == MVT::Other &&
4914  "Invalid chain type");
4915  if (Alignment == 0) // Ensure that codegen never sees alignment 0
4916  Alignment = getEVTAlignment(VT);
4917 
4918  unsigned Flags = MachineMemOperand::MOLoad;
4919  if (isVolatile)
4921  if (isNonTemporal)
4923  if (isInvariant)
4925 
4926  // If we don't have a PtrInfo, infer the trivial frame index case to simplify
4927  // clients.
4928  if (PtrInfo.V.isNull())
4929  PtrInfo = InferPointerInfo(Ptr, Offset);
4930 
4932  MachineMemOperand *MMO =
4933  MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment,
4934  AAInfo, Ranges);
4935  return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
4936 }
4937 
4938 SDValue
4940  EVT VT, SDLoc dl, SDValue Chain,
4941  SDValue Ptr, SDValue Offset, EVT MemVT,
4942  MachineMemOperand *MMO) {
4943  if (VT == MemVT) {
4944  ExtType = ISD::NON_EXTLOAD;
4945  } else if (ExtType == ISD::NON_EXTLOAD) {
4946  assert(VT == MemVT && "Non-extending load from different memory type!");
4947  } else {
4948  // Extending load.
4949  assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
4950  "Should only be an extending load, not truncating!");
4951  assert(VT.isInteger() == MemVT.isInteger() &&
4952  "Cannot convert from FP to Int or Int -> FP!");
4953  assert(VT.isVector() == MemVT.isVector() &&
4954  "Cannot use an ext load to convert to or from a vector!");
4955  assert((!VT.isVector() ||
4956  VT.getVectorNumElements() == MemVT.getVectorNumElements()) &&
4957  "Cannot use an ext load to change the number of vector elements!");
4958  }
4959 
4960  bool Indexed = AM != ISD::UNINDEXED;
4961  assert((Indexed || Offset.getOpcode() == ISD::UNDEF) &&
4962  "Unindexed load with an offset!");
4963 
4964  SDVTList VTs = Indexed ?
4966  SDValue Ops[] = { Chain, Ptr, Offset };
4968  AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
4969  ID.AddInteger(MemVT.getRawBits());
4970  ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(),
4971  MMO->isNonTemporal(),
4972  MMO->isInvariant()));
4973  ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
4974  void *IP = nullptr;
4975  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
4976  cast<LoadSDNode>(E)->refineAlignment(MMO);
4977  return SDValue(E, 0);
4978  }
4979  SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl.getIROrder(),
4980  dl.getDebugLoc(), VTs, AM, ExtType,
4981  MemVT, MMO);
4982  CSEMap.InsertNode(N, IP);
4983  InsertNode(N);
4984  return SDValue(N, 0);
4985 }
4986 
4988  SDValue Chain, SDValue Ptr,
4989  MachinePointerInfo PtrInfo,
4990  bool isVolatile, bool isNonTemporal,
4991  bool isInvariant, unsigned Alignment,
4992  const AAMDNodes &AAInfo,
4993  const MDNode *Ranges) {
4995  return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
4996  PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment,
4997  AAInfo, Ranges);
4998 }
4999 
5001  SDValue Chain, SDValue Ptr,
5002  MachineMemOperand *MMO) {
5004  return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
5005  VT, MMO);
5006 }
5007 
5009  SDValue Chain, SDValue Ptr,
5010  MachinePointerInfo PtrInfo, EVT MemVT,
5011  bool isVolatile, bool isNonTemporal,
5012  bool isInvariant, unsigned Alignment,
5013  const AAMDNodes &AAInfo) {
5015  return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
5016  PtrInfo, MemVT, isVolatile, isNonTemporal, isInvariant,
5017  Alignment, AAInfo);
5018 }
5019 
5020 
5022  SDValue Chain, SDValue Ptr, EVT MemVT,
5023  MachineMemOperand *MMO) {
5025  return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
5026  MemVT, MMO);
5027 }
5028 
5029 SDValue
5031  SDValue Offset, ISD::MemIndexedMode AM) {
5032  LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
5033  assert(LD->getOffset().getOpcode() == ISD::UNDEF &&
5034  "Load is already a indexed load!");
5035  return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
5036  LD->getChain(), Base, Offset, LD->getPointerInfo(),
5037  LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(),
5038  false, LD->getAlignment());
5039 }
5040 
5042  SDValue Ptr, MachinePointerInfo PtrInfo,
5043  bool isVolatile, bool isNonTemporal,
5044  unsigned Alignment, const AAMDNodes &AAInfo) {
5045  assert(Chain.getValueType() == MVT::Other &&
5046  "Invalid chain type");
5047  if (Alignment == 0) // Ensure that codegen never sees alignment 0
5048  Alignment = getEVTAlignment(Val.getValueType());
5049 
5050  unsigned Flags = MachineMemOperand::MOStore;
5051  if (isVolatile)
5053  if (isNonTemporal)
5055 
5056  if (PtrInfo.V.isNull())
5057  PtrInfo = InferPointerInfo(Ptr);
5058 
5060  MachineMemOperand *MMO =
5061  MF.getMachineMemOperand(PtrInfo, Flags,
5062  Val.getValueType().getStoreSize(), Alignment,
5063  AAInfo);
5064 
5065  return getStore(Chain, dl, Val, Ptr, MMO);
5066 }
5067 
5069  SDValue Ptr, MachineMemOperand *MMO) {
5070  assert(Chain.getValueType() == MVT::Other &&
5071  "Invalid chain type");
5072  EVT VT = Val.getValueType();
5073  SDVTList VTs = getVTList(MVT::Other);
5075  SDValue Ops[] = { Chain, Val, Ptr, Undef };
5077  AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
5078  ID.AddInteger(VT.getRawBits());
5080  MMO->isNonTemporal(), MMO->isInvariant()));
5081  ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
5082  void *IP = nullptr;
5083  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
5084  cast<StoreSDNode>(E)->refineAlignment(MMO);
5085  return SDValue(E, 0);
5086  }
5087  SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
5088  dl.getDebugLoc(), VTs,
5089  ISD::UNINDEXED, false, VT, MMO);
5090  CSEMap.InsertNode(N, IP);
5091  InsertNode(N);
5092  return SDValue(N, 0);
5093 }
5094 
5096  SDValue Ptr, MachinePointerInfo PtrInfo,
5097  EVT SVT,bool isVolatile, bool isNonTemporal,
5098  unsigned Alignment,
5099  const AAMDNodes &AAInfo) {
5100  assert(Chain.getValueType() == MVT::Other &&
5101  "Invalid chain type");
5102  if (Alignment == 0) // Ensure that codegen never sees alignment 0
5103  Alignment = getEVTAlignment(SVT);
5104 
5105  unsigned Flags = MachineMemOperand::MOStore;
5106  if (isVolatile)
5108  if (isNonTemporal)
5110 
5111  if (PtrInfo.V.isNull())
5112  PtrInfo = InferPointerInfo(Ptr);
5113 
5115  MachineMemOperand *MMO =
5116  MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment,
5117  AAInfo);
5118 
5119  return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
5120 }
5121 
5123  SDValue Ptr, EVT SVT,
5124  MachineMemOperand *MMO) {
5125  EVT VT = Val.getValueType();
5126 
5127  assert(Chain.getValueType() == MVT::Other &&
5128  "Invalid chain type");
5129  if (VT == SVT)
5130  return getStore(Chain, dl, Val, Ptr, MMO);
5131 
5132  assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
5133  "Should only be a truncating store, not extending!");
5134  assert(VT.isInteger() == SVT.isInteger() &&
5135  "Can't do FP-INT conversion!");
5136  assert(VT.isVector() == SVT.isVector() &&
5137  "Cannot use trunc store to convert to or from a vector!");
5138  assert((!VT.isVector() ||
5139  VT.getVectorNumElements() == SVT.getVectorNumElements()) &&
5140  "Cannot use trunc store to change the number of vector elements!");
5141 
5142  SDVTList VTs = getVTList(MVT::Other);
5144  SDValue Ops[] = { Chain, Val, Ptr, Undef };
5146  AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
5147  ID.AddInteger(SVT.getRawBits());
5149  MMO->isNonTemporal(), MMO->isInvariant()));
5150  ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
5151  void *IP = nullptr;
5152  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
5153  cast<StoreSDNode>(E)->refineAlignment(MMO);
5154  return SDValue(E, 0);
5155  }
5156  SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
5157  dl.getDebugLoc(), VTs,
5158  ISD::UNINDEXED, true, SVT, MMO);
5159  CSEMap.InsertNode(N, IP);
5160  InsertNode(N);
5161  return SDValue(N, 0);
5162 }
5163 
5164 SDValue
5166  SDValue Offset, ISD::MemIndexedMode AM) {
5167  StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
5168  assert(ST->getOffset().getOpcode() == ISD::UNDEF &&
5169  "Store is already a indexed store!");
5170  SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
5171  SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
5173  AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
5174  ID.AddInteger(ST->getMemoryVT().getRawBits());
5175  ID.AddInteger(ST->getRawSubclassData());
5177  void *IP = nullptr;
5178  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP))
5179  return SDValue(E, 0);
5180 
5181  SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
5182  dl.getDebugLoc(), VTs, AM,
5183  ST->isTruncatingStore(),
5184  ST->getMemoryVT(),
5185  ST->getMemOperand());
5186  CSEMap.InsertNode(N, IP);
5187  InsertNode(N);
5188  return SDValue(N, 0);
5189 }
5190 
5191 SDValue
5193  SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT,
5194  MachineMemOperand *MMO, ISD::LoadExtType ExtTy) {
5195 
5196  SDVTList VTs = getVTList(VT, MVT::Other);
5197  SDValue Ops[] = { Chain, Ptr, Mask, Src0 };
5199  AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops);
5200  ID.AddInteger(VT.getRawBits());
5202  MMO->isVolatile(),
5203  MMO->isNonTemporal(),
5204  MMO->isInvariant()));
5205  ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
5206  void *IP = nullptr;
5207  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
5208  cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
5209  return SDValue(E, 0);
5210  }
5211  SDNode *N = new (NodeAllocator) MaskedLoadSDNode(dl.getIROrder(),
5212  dl.getDebugLoc(), Ops, 4, VTs,
5213  ExtTy, MemVT, MMO);
5214  CSEMap.InsertNode(N, IP);
5215  InsertNode(N);
5216  return SDValue(N, 0);
5217 }
5218 
5220  SDValue Ptr, SDValue Mask, EVT MemVT,
5221  MachineMemOperand *MMO, bool isTrunc) {
5222  assert(Chain.getValueType() == MVT::Other &&
5223  "Invalid chain type");
5224  EVT VT = Val.getValueType();
5225  SDVTList VTs = getVTList(MVT::Other);
5226  SDValue Ops[] = { Chain, Ptr, Mask, Val };
5228  AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops);
5229  ID.AddInteger(VT.getRawBits());
5231  MMO->isNonTemporal(), MMO->isInvariant()));
5232  ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
5233  void *IP = nullptr;
5234  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
5235  cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
5236  return SDValue(E, 0);
5237  }
5238  SDNode *N = new (NodeAllocator) MaskedStoreSDNode(dl.getIROrder(),
5239  dl.getDebugLoc(), Ops, 4,
5240  VTs, isTrunc, MemVT, MMO);
5241  CSEMap.InsertNode(N, IP);
5242  InsertNode(N);
5243  return SDValue(N, 0);
5244 }
5245 
5246 SDValue
5248  ArrayRef<SDValue> Ops,
5249  MachineMemOperand *MMO) {
5250 
5252  AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);
5253  ID.AddInteger(VT.getRawBits());
5255  MMO->isVolatile(),
5256  MMO->isNonTemporal(),
5257  MMO->isInvariant()));
5258  ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
5259  void *IP = nullptr;
5260  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
5261  cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
5262  return SDValue(E, 0);
5263  }
5264  MaskedGatherSDNode *N =
5265  new (NodeAllocator) MaskedGatherSDNode(dl.getIROrder(), dl.getDebugLoc(),
5266  Ops, VTs, VT, MMO);
5267  CSEMap.InsertNode(N, IP);
5268  InsertNode(N);
5269  return SDValue(N, 0);
5270 }
5271 
5273  ArrayRef<SDValue> Ops,
5274  MachineMemOperand *MMO) {
5276  AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops);
5277  ID.AddInteger(VT.getRawBits());
5279  MMO->isNonTemporal(),
5280  MMO->isInvariant()));
5281  ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
5282  void *IP = nullptr;
5283  if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) {
5284  cast<MaskedScatterSDNode>(E)->refineAlignment(MMO);
5285  return SDValue(E, 0);
5286  }
5287  SDNode *N =
5288  new (NodeAllocator) MaskedScatterSDNode(dl.getIROrder(), dl.getDebugLoc(),
5289  Ops, VTs, VT, MMO);
5290  CSEMap.InsertNode(N, IP);
5291  InsertNode(N);
5292  return SDValue(N, 0);
5293 }
5294 
5296  SDValue Chain, SDValue Ptr,
5297  SDValue SV,
5298  unsigned Align) {
5299  SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) };
5300  return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
5301 }
5302 
5303 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
5304  ArrayRef<SDUse> Ops) {
5305  switch (Ops.size()) {
5306  case 0: return getNode(Opcode, DL, VT);
5307  case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
5308  case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
5309  case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
5310  default: break;
5311  }
5312 
5313  // Copy from an SDUse array into an SDValue array for use with
5314  // the regular getNode logic.
5315  SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
5316  return getNode(Opcode, DL, VT, NewOps);
5317 }
5318 
5319 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
5320  ArrayRef<SDValue> Ops) {
5321  unsigned NumOps = Ops.size();
5322  switch (NumOps) {
5323  case 0: return getNode(Opcode, DL, VT);
5324  case 1: return getNode(Opcode, DL, VT, Ops[0]);
5325  case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
5326  case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
5327  default: break;
5328  }
5329 
5330  switch (Opcode) {
5331  default: break;
5332  case ISD::SELECT_CC: {
5333  assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
5334  assert(Ops[0].getValueType() == Ops[1].getValueType() &&
5335  "LHS and RHS of condition must have same type!");
5336  assert(Ops[2].getValueType() == Ops[3].getValueType() &&
5337  "True and False arms of SelectCC must have same type!");
5338  assert(Ops[2].getValueType() == VT &&
5339  "select_cc node must be of same type as true and false value!");
5340  break;
5341  }
5342  case ISD::BR_CC: {
5343  assert(NumOps == 5 && "BR_CC takes 5 operands!");
5344  assert(Ops[2].getValueType() == Ops[3].getValueType() &&
5345  "LHS/RHS of comparison should match types!");
5346  break;
5347  }
5348  }
5349 
5350  // Memoize nodes.
5351  SDNode *N;
5352  SDVTList VTs = getVTList(VT);
5353 
5354  if (VT != MVT::Glue) {
5356  AddNodeIDNode(ID, Opcode, VTs, Ops);
5357  void *IP = nullptr;
5358 
5359  if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
5360  return SDValue(E, 0);
5361 
5362  N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
5363  VTs, Ops);
5364  CSEMap.InsertNode(N, IP);
5365  } else {
5366  N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
5367  VTs, Ops);
5368  }
5369 
5370  InsertNode(N);
5371  return SDValue(N, 0);
5372 }
5373 
5375  ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
5376  return getNode(Opcode, DL, getVTList(ResultTys), Ops);
5377 }
5378 
5379 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
5380  ArrayRef<SDValue> Ops) {
5381  if (VTList.NumVTs == 1)
5382  return getNode(Opcode, DL, VTList.VTs[0], Ops);
5383 
5384 #if 0
5385  switch (Opcode) {
5386  // FIXME: figure out how to safely handle things like
5387  // int foo(int x) { return 1 << (x & 255); }
5388  // int bar() { return foo(256); }
5389  case ISD::SRA_PARTS:
5390  case ISD::SRL_PARTS:
5391  case ISD::SHL_PARTS:
5392  if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
5393  cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
5394  return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
5395  else if (N3.getOpcode() == ISD::AND)
5396  if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
5397  // If the and is only masking out bits that cannot effect the shift,
5398  // eliminate the and.
5399  unsigned NumBits = VT.getScalarType().getSizeInBits()*2;
5400  if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
5401  return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
5402  }
5403  break;
5404  }
5405 #endif
5406 
5407  // Memoize the node unless it returns a flag.
5408  SDNode *N;
5409  unsigned NumOps = Ops.size();
5410  if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
5412  AddNodeIDNode(ID, Opcode, VTList, Ops);
5413  void *IP = nullptr;
5414  if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
5415  return SDValue(E, 0);
5416 
5417  if (NumOps == 1) {
5418  N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
5419  DL.getDebugLoc(), VTList, Ops[0]);
5420  } else if (NumOps == 2) {
5421  N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
5422  DL.getDebugLoc(), VTList, Ops[0],
5423  Ops[1]);
5424  } else if (NumOps == 3) {
5425  N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
5426  DL.getDebugLoc(), VTList, Ops[0],
5427  Ops[1], Ops[2]);
5428  } else {
5429  N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
5430  VTList, Ops);
5431  }
5432  CSEMap.InsertNode(N, IP);
5433  } else {
5434  if (NumOps == 1) {
5435  N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
5436  DL.getDebugLoc(), VTList, Ops[0]);
5437  } else if (NumOps == 2) {
5438  N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
5439  DL.getDebugLoc(), VTList, Ops[0],
5440  Ops[1]);
5441  } else if (NumOps == 3) {
5442  N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
5443  DL.getDebugLoc(), VTList, Ops[0],
5444  Ops[1], Ops[2]);
5445  } else {
5446  N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
5447  VTList, Ops);
5448  }
5449  }
5450  InsertNode(N);
5451  return SDValue(N, 0);
5452 }
5453 
5454 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList) {
5455  return getNode(Opcode, DL, VTList, None);
5456 }
5457 
5458 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
5459  SDValue N1) {
5460  SDValue Ops[] = { N1 };
5461  return getNode(Opcode, DL, VTList, Ops);
5462 }
5463 
5464 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
5465  SDValue N1, SDValue N2) {
5466  SDValue Ops[] = { N1, N2 };
5467  return getNode(Opcode, DL, VTList, Ops);
5468 }
5469 
5470 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
5471  SDValue N1, SDValue N2, SDValue N3) {
5472  SDValue Ops[] = { N1, N2, N3 };
5473  return getNode(Opcode, DL, VTList, Ops);
5474 }
5475 
5476 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
5477  SDValue N1, SDValue N2, SDValue N3,
5478  SDValue N4) {
5479  SDValue Ops[] = { N1, N2, N3, N4 };
5480  return getNode(Opcode, DL, VTList, Ops);
5481 }
5482 
5483 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
5484  SDValue N1, SDValue N2, SDValue N3,
5485  SDValue N4, SDValue N5) {
5486  SDValue Ops[] = { N1, N2, N3, N4, N5 };
5487  return getNode(Opcode, DL, VTList, Ops);
5488 }
5489 
5491  return makeVTList(SDNode::getValueTypeList(VT), 1);
5492 }
5493 
5496  ID.AddInteger(2U);
5497  ID.AddInteger(VT1.getRawBits());
5498  ID.AddInteger(VT2.getRawBits());
5499 
5500  void *IP = nullptr;
5501  SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
5502  if (!Result) {
5503  EVT *Array = Allocator.Allocate<EVT>(2);
5504  Array[0] = VT1;
5505  Array[1] = VT2;
5506  Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
5507  VTListMap.InsertNode(Result, IP);
5508  }
5509  return Result->getSDVTList();
5510 }
5511 
5514  ID.AddInteger(3U);
5515  ID.AddInteger(VT1.getRawBits());
5516  ID.AddInteger(VT2.getRawBits());
5517  ID.AddInteger(VT3.getRawBits());
5518 
5519  void *IP = nullptr;
5520  SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
5521  if (!Result) {
5522  EVT *Array = Allocator.Allocate<EVT>(3);
5523  Array[0] = VT1;
5524  Array[1] = VT2;
5525  Array[2] = VT3;
5526  Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
5527  VTListMap.InsertNode(Result, IP);
5528  }
5529  return Result->getSDVTList();
5530 }
5531 
5534  ID.AddInteger(4U);
5535  ID.AddInteger(VT1.getRawBits());
5536  ID.AddInteger(VT2.getRawBits());
5537  ID.AddInteger(VT3.getRawBits());
5538  ID.AddInteger(VT4.getRawBits());
5539 
5540  void *IP = nullptr;
5541  SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
5542  if (!Result) {
5543  EVT *Array = Allocator.Allocate<EVT>(4);
5544  Array[0] = VT1;
5545  Array[1] = VT2;
5546  Array[2] = VT3;
5547  Array[3] = VT4;
5548  Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
5549  VTListMap.InsertNode(Result, IP);
5550  }
5551  return Result->getSDVTList();
5552 }
5553 
5555  unsigned NumVTs = VTs.size();
5557  ID.AddInteger(NumVTs);
5558  for (unsigned index = 0; index < NumVTs; index++) {
5559  ID.AddInteger(VTs[index].getRawBits());
5560  }
5561 
5562  void *IP = nullptr;
5563  SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
5564  if (!Result) {
5565  EVT *Array = Allocator.Allocate<EVT>(NumVTs);
5566  std::copy(VTs.begin(), VTs.end(), Array);
5567  Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
5568  VTListMap.InsertNode(Result, IP);
5569  }
5570  return Result->getSDVTList();
5571 }
5572 
5573 
5574 /// UpdateNodeOperands - *Mutate* the specified node in-place to have the
5575 /// specified operands. If the resultant node already exists in the DAG,
5576 /// this does not modify the specified node, instead it returns the node that
5577 /// already exists. If the resultant node does not exist in the DAG, the
5578 /// input node is returned. As a degenerate case, if you specify the same
5579 /// input operands as the node already has, the input node is returned.
5581  assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
5582 
5583  // Check to see if there is no change.
5584  if (Op == N->getOperand(0)) return N;
5585 
5586  // See if the modified node already exists.
5587  void *InsertPos = nullptr;
5588  if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
5589  return Existing;
5590 
5591  // Nope it doesn't. Remove the node from its current place in the maps.
5592  if (InsertPos)
5593  if (!RemoveNodeFromCSEMaps(N))
5594  InsertPos = nullptr;
5595 
5596  // Now we update the operands.
5597  N->OperandList[0].set(Op);
5598 
5599  // If this gets put into a CSE map, add it.
5600  if (InsertPos) CSEMap.InsertNode(N, InsertPos);
5601  return N;
5602 }
5603 
5605  assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
5606 
5607  // Check to see if there is no change.
5608  if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
5609  return N; // No operands changed, just return the input node.
5610 
5611  // See if the modified node already exists.
5612  void *InsertPos = nullptr;
5613  if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
5614  return Existing;
5615 
5616  // Nope it doesn't. Remove the node from its current place in the maps.
5617  if (InsertPos)
5618  if (!RemoveNodeFromCSEMaps(N))
5619  InsertPos = nullptr;
5620 
5621  // Now we update the operands.
5622  if (N->OperandList[0] != Op1)
5623  N->OperandList[0].set(Op1);
5624  if (N->OperandList[1] != Op2)
5625  N->OperandList[1].set(Op2);
5626 
5627  // If this gets put into a CSE map, add it.
5628  if (InsertPos) CSEMap.InsertNode(N, InsertPos);
5629  return N;
5630 }
5631 
5634  SDValue Ops[] = { Op1, Op2, Op3 };
5635  return UpdateNodeOperands(N, Ops);
5636 }
5637 
5640  SDValue Op3, SDValue Op4) {
5641  SDValue Ops[] = { Op1, Op2, Op3, Op4 };
5642  return UpdateNodeOperands(N, Ops);
5643 }
5644 
5647  SDValue Op3, SDValue Op4, SDValue Op5) {
5648  SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
5649  return UpdateNodeOperands(N, Ops);
5650 }
5651 
5654  unsigned NumOps = Ops.size();
5655  assert(N->getNumOperands() == NumOps &&
5656  "Update with wrong number of operands");
5657 
5658  // If no operands changed just return the input node.
5659  if (Ops.empty() || std::equal(Ops.begin(), Ops.end(), N->op_begin()))
5660  return N;
5661 
5662  // See if the modified node already exists.
5663  void *InsertPos = nullptr;
5664  if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
5665  return Existing;
5666 
5667  // Nope it doesn't. Remove the node from its current place in the maps.
5668  if (InsertPos)
5669  if (!RemoveNodeFromCSEMaps(N))
5670  InsertPos = nullptr;
5671 
5672  // Now we update the operands.
5673  for (unsigned i = 0; i != NumOps; ++i)
5674  if (N->OperandList[i] != Ops[i])
5675  N->OperandList[i].set(Ops[i]);
5676 
5677  // If this gets put into a CSE map, add it.
5678  if (InsertPos) CSEMap.InsertNode(N, InsertPos);
5679  return N;
5680 }
5681 
5682 /// DropOperands - Release the operands and set this node to have
5683 /// zero operands.
5685  // Unlike the code in MorphNodeTo that does this, we don't need to
5686  // watch for dead nodes here.
5687  for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
5688  SDUse &Use = *I++;
5689  Use.set(SDValue());
5690  }
5691 }
5692 
5693 /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
5694 /// machine opcode.
5695 ///
5696 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5697  EVT VT) {
5698  SDVTList VTs = getVTList(VT);
5699  return SelectNodeTo(N, MachineOpc, VTs, None);
5700 }
5701 
5702 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5703  EVT VT, SDValue Op1) {
5704  SDVTList VTs = getVTList(VT);
5705  SDValue Ops[] = { Op1 };
5706  return SelectNodeTo(N, MachineOpc, VTs, Ops);
5707 }
5708 
5709 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5710  EVT VT, SDValue Op1,
5711  SDValue Op2) {
5712  SDVTList VTs = getVTList(VT);
5713  SDValue Ops[] = { Op1, Op2 };
5714  return SelectNodeTo(N, MachineOpc, VTs, Ops);
5715 }
5716 
5717 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5718  EVT VT, SDValue Op1,
5719  SDValue Op2, SDValue Op3) {
5720  SDVTList VTs = getVTList(VT);
5721  SDValue Ops[] = { Op1, Op2, Op3 };
5722  return SelectNodeTo(N, MachineOpc, VTs, Ops);
5723 }
5724 
5725 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5726  EVT VT, ArrayRef<SDValue> Ops) {
5727  SDVTList VTs = getVTList(VT);
5728  return SelectNodeTo(N, MachineOpc, VTs, Ops);
5729 }
5730 
5731 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5732  EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
5733  SDVTList VTs = getVTList(VT1, VT2);
5734  return SelectNodeTo(N, MachineOpc, VTs, Ops);
5735 }
5736 
5737 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5738  EVT VT1, EVT VT2) {
5739  SDVTList VTs = getVTList(VT1, VT2);
5740  return SelectNodeTo(N, MachineOpc, VTs, None);
5741 }
5742 
5743 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5744  EVT VT1, EVT VT2, EVT VT3,
5745  ArrayRef<SDValue> Ops) {
5746  SDVTList VTs = getVTList(VT1, VT2, VT3);
5747  return SelectNodeTo(N, MachineOpc, VTs, Ops);
5748 }
5749 
5750 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5751  EVT VT1, EVT VT2, EVT VT3, EVT VT4,
5752  ArrayRef<SDValue> Ops) {
5753  SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
5754  return SelectNodeTo(N, MachineOpc, VTs, Ops);
5755 }
5756 
5757 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5758  EVT VT1, EVT VT2,
5759  SDValue Op1) {
5760  SDVTList VTs = getVTList(VT1, VT2);
5761  SDValue Ops[] = { Op1 };
5762  return SelectNodeTo(N, MachineOpc, VTs, Ops);
5763 }
5764 
5765 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5766  EVT VT1, EVT VT2,
5767  SDValue Op1, SDValue Op2) {
5768  SDVTList VTs = getVTList(VT1, VT2);
5769  SDValue Ops[] = { Op1, Op2 };
5770  return SelectNodeTo(N, MachineOpc, VTs, Ops);
5771 }
5772 
5773 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5774  EVT VT1, EVT VT2,
5775  SDValue Op1, SDValue Op2,
5776  SDValue Op3) {
5777  SDVTList VTs = getVTList(VT1, VT2);
5778  SDValue Ops[] = { Op1, Op2, Op3 };
5779  return SelectNodeTo(N, MachineOpc, VTs, Ops);
5780 }
5781 
5782 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5783  EVT VT1, EVT VT2, EVT VT3,
5784  SDValue Op1, SDValue Op2,
5785  SDValue Op3) {
5786  SDVTList VTs = getVTList(VT1, VT2, VT3);
5787  SDValue Ops[] = { Op1, Op2, Op3 };
5788  return SelectNodeTo(N, MachineOpc, VTs, Ops);
5789 }
5790 
5791 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
5792  SDVTList VTs,ArrayRef<SDValue> Ops) {
5793  N = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
5794  // Reset the NodeID to -1.
5795  N->setNodeId(-1);
5796  return N;
5797 }
5798 
5799 /// UpdadeSDLocOnMergedSDNode - If the opt level is -O0 then it throws away
5800 /// the line number information on the merged node since it is not possible to
5801 /// preserve the information that operation is associated with multiple lines.
5802 /// This will make the debugger working better at -O0, were there is a higher
5803 /// probability having other instructions associated with that line.
5804 ///
5805 /// For IROrder, we keep the smaller of the two
5806 SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) {
5807  DebugLoc NLoc = N->getDebugLoc();
5808  if (NLoc && OptLevel == CodeGenOpt::None && OLoc.getDebugLoc() != NLoc) {
5809  N->setDebugLoc(DebugLoc());
5810  }
5811  unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
5812  N->setIROrder(Order);
5813  return N;
5814 }
5815 
5816 /// MorphNodeTo - This *mutates* the specified node to have the specified
5817 /// return type, opcode, and operands.
5818 ///
5819 /// Note that MorphNodeTo returns the resultant node. If there is already a
5820 /// node of the specified opcode and operands, it returns that node instead of
5821 /// the current one. Note that the SDLoc need not be the same.
5822 ///
5823 /// Using MorphNodeTo is faster than creating a new node and swapping it in
5824 /// with ReplaceAllUsesWith both because it often avoids allocating a new
5825 /// node, and because it doesn't require CSE recalculation for any of
5826 /// the node's users.
5827 ///
5828 /// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
5829 /// As a consequence it isn't appropriate to use from within the DAG combiner or
5830 /// the legalizer which maintain worklists that would need to be updated when
5831 /// deleting things.
5833  SDVTList VTs, ArrayRef<SDValue> Ops) {
5834  unsigned NumOps = Ops.size();
5835  // If an identical node already exists, use it.
5836  void *IP = nullptr;
5837  if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
5839  AddNodeIDNode(ID, Opc, VTs, Ops);
5840  if (SDNode *ON = FindNodeOrInsertPos(ID, N->getDebugLoc(), IP))
5841  return UpdadeSDLocOnMergedSDNode(ON, SDLoc(N));
5842  }
5843 
5844  if (!RemoveNodeFromCSEMaps(N))
5845  IP = nullptr;
5846 
5847  // Start the morphing.
5848  N->NodeType = Opc;
5849  N->ValueList = VTs.VTs;
5850  N->NumValues = VTs.NumVTs;
5851 
5852  // Clear the operands list, updating used nodes to remove this from their
5853  // use list. Keep track of any operands that become dead as a result.
5854  SmallPtrSet<SDNode*, 16> DeadNodeSet;
5855  for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
5856  SDUse &Use = *I++;
5857  SDNode *Used = Use.getNode();
5858  Use.set(SDValue());
5859  if (Used->use_empty())
5860  DeadNodeSet.insert(Used);
5861  }
5862 
5863  if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) {
5864  // Initialize the memory references information.
5865  MN->setMemRefs(nullptr, nullptr);
5866  // If NumOps is larger than the # of operands we can have in a
5867  // MachineSDNode, reallocate the operand list.
5868  if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) {
5869  if (MN->OperandsNeedDelete)
5870  delete[] MN->OperandList;
5871  if (NumOps > array_lengthof(MN->LocalOperands))
5872  // We're creating a final node that will live unmorphed for the
5873  // remainder of the current SelectionDAG iteration, so we can allocate
5874  // the operands directly out of a pool with no recycling metadata.
5875  MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
5876  Ops.data(), NumOps);
5877  else
5878  MN->InitOperands(MN->LocalOperands, Ops.data(), NumOps);
5879  MN->OperandsNeedDelete = false;
5880  } else
5881  MN->InitOperands(MN->OperandList, Ops.data(), NumOps);
5882  } else {
5883  // If NumOps is larger than the # of operands we currently have, reallocate
5884  // the operand list.
5885  if (NumOps > N->NumOperands) {
5886  if (N->OperandsNeedDelete)
5887  delete[] N->OperandList;
5888  N->InitOperands(new SDUse[NumOps], Ops.data(), NumOps);
5889  N->OperandsNeedDelete = true;
5890  } else
5891  N->InitOperands(N->OperandList, Ops.data(), NumOps);
5892  }
5893 
5894  // Delete any nodes that are still dead after adding the uses for the
5895  // new operands.
5896  if (!DeadNodeSet.empty()) {
5897  SmallVector<SDNode *, 16> DeadNodes;
5898  for (SDNode *N : DeadNodeSet)
5899  if (N->use_empty())
5900  DeadNodes.push_back(N);
5901  RemoveDeadNodes(DeadNodes);
5902  }
5903 
5904  if (IP)
5905  CSEMap.InsertNode(N, IP); // Memoize the new node.
5906  return N;
5907 }
5908 
5909 
5910 /// getMachineNode - These are used for target selectors to create a new node
5911 /// with specified return type(s), MachineInstr opcode, and operands.
5912 ///
5913 /// Note that getMachineNode returns the resultant node. If there is already a
5914 /// node of the specified opcode and operands, it returns that node instead of
5915 /// the current one.
5916 MachineSDNode *
5917 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT) {
5918  SDVTList VTs = getVTList(VT);
5919  return getMachineNode(Opcode, dl, VTs, None);
5920 }
5921 
5922 MachineSDNode *
5923 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, SDValue Op1) {
5924  SDVTList VTs = getVTList(VT);
5925  SDValue Ops[] = { Op1 };
5926  return getMachineNode(Opcode, dl, VTs, Ops);
5927 }
5928 
5929 MachineSDNode *
5930 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
5931  SDValue Op1, SDValue Op2) {
5932  SDVTList VTs = getVTList(VT);
5933  SDValue Ops[] = { Op1, Op2 };
5934  return getMachineNode(Opcode, dl, VTs, Ops);
5935 }
5936 
5937 MachineSDNode *
5938 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
5939  SDValue Op1, SDValue Op2, SDValue Op3) {
5940  SDVTList VTs = getVTList(VT);
5941  SDValue Ops[] = { Op1, Op2, Op3 };
5942  return getMachineNode(Opcode, dl, VTs, Ops);
5943 }
5944 
5945 MachineSDNode *
5946 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
5947  ArrayRef<SDValue> Ops) {
5948  SDVTList VTs = getVTList(VT);
5949  return getMachineNode(Opcode, dl, VTs, Ops);
5950 }
5951 
5952 MachineSDNode *
5953 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, EVT VT2) {
5954  SDVTList VTs = getVTList(VT1, VT2);
5955  return getMachineNode(Opcode, dl, VTs, None);
5956 }
5957 
5958 MachineSDNode *
5960  EVT VT1, EVT VT2, SDValue Op1) {
5961  SDVTList VTs = getVTList(VT1, VT2);
5962  SDValue Ops[] = { Op1 };
5963  return getMachineNode(Opcode, dl, VTs, Ops);
5964 }
5965 
5966 MachineSDNode *
5968  EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) {
5969  SDVTList VTs = getVTList(VT1, VT2);
5970  SDValue Ops[] = { Op1, Op2 };
5971  return getMachineNode(Opcode, dl, VTs, Ops);
5972 }
5973 
5974 MachineSDNode *
5976  EVT VT1, EVT VT2, SDValue Op1,
5977  SDValue Op2, SDValue Op3) {
5978  SDVTList VTs = getVTList(VT1, VT2);
5979  SDValue Ops[] = { Op1, Op2, Op3 };
5980  return getMachineNode(Opcode, dl, VTs, Ops);
5981 }
5982 
5983 MachineSDNode *
5985  EVT VT1, EVT VT2,
5986  ArrayRef<SDValue> Ops) {
5987  SDVTList VTs = getVTList(VT1, VT2);
5988  return getMachineNode(Opcode, dl, VTs, Ops);
5989 }
5990 
5991 MachineSDNode *
5993  EVT VT1, EVT VT2, EVT VT3,
5994  SDValue Op1, SDValue Op2) {
5995  SDVTList VTs = getVTList(VT1, VT2, VT3);
5996  SDValue Ops[] = { Op1, Op2 };
5997  return getMachineNode(Opcode, dl, VTs, Ops);
5998 }
5999 
6000 MachineSDNode *
6002  EVT VT1, EVT VT2, EVT VT3,
6003  SDValue Op1, SDValue Op2, SDValue Op3) {
6004  SDVTList VTs = getVTList(VT1, VT2, VT3);
6005  SDValue Ops[] = { Op1, Op2, Op3 };
6006  return getMachineNode(Opcode, dl, VTs, Ops);
6007 }
6008 
6009 MachineSDNode *
6011  EVT VT1, EVT VT2, EVT VT3,
6012  ArrayRef<SDValue> Ops) {
6013  SDVTList VTs = getVTList(VT1, VT2, VT3);
6014  return getMachineNode(Opcode, dl, VTs, Ops);
6015 }
6016 
6017 MachineSDNode *
6018 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1,
6019  EVT VT2, EVT VT3, EVT VT4,
6020  ArrayRef<SDValue> Ops) {
6021  SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
6022  return getMachineNode(Opcode, dl, VTs, Ops);
6023 }
6024 
6025 MachineSDNode *
6027  ArrayRef<EVT> ResultTys,
6028  ArrayRef<SDValue> Ops) {
6029  SDVTList VTs = getVTList(ResultTys);
6030  return getMachineNode(Opcode, dl, VTs, Ops);
6031 }
6032 
6033 MachineSDNode *
6035  ArrayRef<SDValue> OpsArray) {
6036  bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
6037  MachineSDNode *N;
6038  void *IP = nullptr;
6039  const SDValue *Ops = OpsArray.data();
6040  unsigned NumOps = OpsArray.size();
6041 
6042  if (DoCSE) {
6044  AddNodeIDNode(ID, ~Opcode, VTs, OpsArray);
6045  IP = nullptr;
6046  if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) {
6047  return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL));
6048  }
6049  }
6050 
6051  // Allocate a new MachineSDNode.
6052  N = new (NodeAllocator) MachineSDNode(~Opcode, DL.getIROrder(),
6053  DL.getDebugLoc(), VTs);
6054 
6055  // Initialize the operands list.
6056  if (NumOps > array_lengthof(N->LocalOperands))
6057  // We're creating a final node that will live unmorphed for the
6058  // remainder of the current SelectionDAG iteration, so we can allocate
6059  // the operands directly out of a pool with no recycling metadata.
6060  N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
6061  Ops, NumOps);
6062  else
6063  N->InitOperands(N->LocalOperands, Ops, NumOps);
6064  N->OperandsNeedDelete = false;
6065 
6066  if (DoCSE)
6067  CSEMap.InsertNode(N, IP);
6068 
6069  InsertNode(N);
6070  return N;
6071 }
6072 
6073 /// getTargetExtractSubreg - A convenience function for creating
6074 /// TargetOpcode::EXTRACT_SUBREG nodes.
6075 SDValue
6077  SDValue Operand) {
6078  SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
6080  VT, Operand, SRIdxVal);
6081  return SDValue(Subreg, 0);
6082 }
6083 
6084 /// getTargetInsertSubreg - A convenience function for creating
6085 /// TargetOpcode::INSERT_SUBREG nodes.
6086 SDValue
6088  SDValue Operand, SDValue Subreg) {
6089  SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
6091  VT, Operand, Subreg, SRIdxVal);
6092  return SDValue(Result, 0);
6093 }
6094 
6095 /// getNodeIfExists - Get the specified node if it's already available, or
6096 /// else return NULL.
6098  ArrayRef<SDValue> Ops,
6099  const SDNodeFlags *Flags) {
6100  if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
6102  AddNodeIDNode(ID, Opcode, VTList, Ops);
6103  AddNodeIDFlags(ID, Opcode, Flags);
6104  void *IP = nullptr;
6105  if (SDNode *E = FindNodeOrInsertPos(ID, DebugLoc(), IP))
6106  return E;
6107  }
6108  return nullptr;
6109 }
6110 
6111 /// getDbgValue - Creates a SDDbgValue node.
6112 ///
6113 /// SDNode
6115  unsigned R, bool IsIndirect, uint64_t Off,
6116  DebugLoc DL, unsigned O) {
6117  assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
6118  "Expected inlined-at fields to agree");
6119  return new (DbgInfo->getAlloc())
6120  SDDbgValue(Var, Expr, N, R, IsIndirect, Off, DL, O);
6121 }
6122 
6123 /// Constant
6125  const Value *C, uint64_t Off,
6126  DebugLoc DL, unsigned O) {
6127  assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
6128  "Expected inlined-at fields to agree");
6129  return new (DbgInfo->getAlloc()) SDDbgValue(Var, Expr, C, Off, DL, O);
6130 }
6131 
6132 /// FrameIndex
6134  unsigned FI, uint64_t Off,
6135  DebugLoc DL, unsigned O) {
6136  assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
6137  "Expected inlined-at fields to agree");
6138  return new (DbgInfo->getAlloc()) SDDbgValue(Var, Expr, FI, Off, DL, O);
6139 }
6140 
6141 namespace {
6142 
6143 /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
6144 /// pointed to by a use iterator is deleted, increment the use iterator
6145 /// so that it doesn't dangle.
6146 ///
6147 class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
6150 
6151  void NodeDeleted(SDNode *N, SDNode *E) override {
6152  // Increment the iterator as needed.
6153  while (UI != UE && N == *UI)
6154  ++UI;
6155  }
6156 
6157 public:
6158  RAUWUpdateListener(SelectionDAG &d,
6161  : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
6162 };
6163 
6164 }
6165 
6166 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
6167 /// This can cause recursive merging of nodes in the DAG.
6168 ///
6169 /// This version assumes From has a single result value.
6170 ///
6172  SDNode *From = FromN.getNode();
6173  assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
6174  "Cannot replace with this method!");
6175  assert(From != To.getNode() && "Cannot replace uses of with self");
6176 
6177  // Iterate over all the existing uses of From. New uses will be added
6178  // to the beginning of the use list, which we avoid visiting.
6179  // This specifically avoids visiting uses of From that arise while the
6180  // replacement is happening, because any such uses would be the result
6181  // of CSE: If an existing node looks like From after one of its operands
6182  // is replaced by To, we don't want to replace of all its users with To
6183  // too. See PR3018 for more info.
6184  SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
6185  RAUWUpdateListener Listener(*this, UI, UE);
6186  while (UI != UE) {
6187  SDNode *User = *UI;
6188 
6189  // This node is about to morph, remove its old self from the CSE maps.
6190  RemoveNodeFromCSEMaps(User);
6191 
6192  // A user can appear in a use list multiple times, and when this
6193  // happens the uses are usually next to each other in the list.
6194  // To help reduce the number of CSE recomputations, process all
6195  // the uses of this user that we can find this way.
6196  do {
6197  SDUse &Use = UI.getUse();
6198  ++UI;
6199  Use.set(To);
6200  } while (UI != UE && *UI == User);
6201 
6202  // Now that we have modified User, add it back to the CSE maps. If it
6203  // already exists there, recursively merge the results together.
6204  AddModifiedNodeToCSEMaps(User);
6205  }
6206 
6207  // If we just RAUW'd the root, take note.
6208  if (FromN == getRoot())
6209  setRoot(To);
6210 }
6211 
6212 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
6213 /// This can cause recursive merging of nodes in the DAG.
6214 ///
6215 /// This version assumes that for each value of From, there is a
6216 /// corresponding value in To in the same position with the same type.
6217 ///
6219 #ifndef NDEBUG
6220  for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
6221  assert((!From->hasAnyUseOfValue(i) ||
6222  From->getValueType(i) == To->getValueType(i)) &&
6223  "Cannot use this version of ReplaceAllUsesWith!");
6224 #endif
6225 
6226  // Handle the trivial case.
6227  if (From == To)
6228  return;
6229 
6230  // Iterate over just the existing users of From. See the comments in
6231  // the ReplaceAllUsesWith above.
6232  SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
6233  RAUWUpdateListener Listener(*this, UI, UE);
6234  while (UI != UE) {
6235  SDNode *User = *UI;
6236 
6237  // This node is about to morph, remove its old self from the CSE maps.
6238  RemoveNodeFromCSEMaps(User);
6239 
6240  // A user can appear in a use list multiple times, and when this
6241  // happens the uses are usually next to each other in the list.
6242  // To help reduce the number of CSE recomputations, process all
6243  // the uses of this user that we can find this way.
6244  do {
6245  SDUse &Use = UI.getUse();
6246  ++UI;
6247  Use.setNode(To);
6248  } while (UI != UE && *UI == User);
6249 
6250  // Now that we have modified User, add it back to the CSE maps. If it
6251  // already exists there, recursively merge the results together.
6252  AddModifiedNodeToCSEMaps(User);
6253  }
6254 
6255  // If we just RAUW'd the root, take note.
6256  if (From == getRoot().getNode())
6257  setRoot(SDValue(To, getRoot().getResNo()));
6258 }
6259 
6260 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
6261 /// This can cause recursive merging of nodes in the DAG.
6262 ///
6263 /// This version can replace From with any result values. To must match the
6264 /// number and types of values returned by From.
6266  if (From->getNumValues() == 1) // Handle the simple case efficiently.
6267  return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
6268 
6269  // Iterate over just the existing users of From. See the comments in
6270  // the ReplaceAllUsesWith above.
6271  SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
6272  RAUWUpdateListener Listener(*this, UI, UE);
6273  while (UI != UE) {
6274  SDNode *User = *UI;
6275 
6276  // This node is about to morph, remove its old self from the CSE maps.
6277  RemoveNodeFromCSEMaps(User);
6278 
6279  // A user can appear in a use list multiple times, and when this
6280  // happens the uses are usually next to each other in the list.
6281  // To help reduce the number of CSE recomputations, process all
6282  // the uses of this user that we can find this way.
6283  do {
6284  SDUse &Use = UI.getUse();
6285  const SDValue &ToOp = To[Use.getResNo()];
6286  ++UI;
6287  Use.set(ToOp);
6288  } while (UI != UE && *UI == User);
6289 
6290  // Now that we have modified User, add it back to the CSE maps. If it
6291  // already exists there, recursively merge the results together.
6292  AddModifiedNodeToCSEMaps(User);
6293  }
6294 
6295  // If we just RAUW'd the root, take note.
6296  if (From == getRoot().getNode())
6297  setRoot(SDValue(To[getRoot().getResNo()]));
6298 }
6299 
6300 /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
6301 /// uses of other values produced by From.getNode() alone. The Deleted
6302 /// vector is handled the same way as for ReplaceAllUsesWith.
6304  // Handle the really simple, really trivial case efficiently.
6305  if (From == To) return;
6306 
6307  // Handle the simple, trivial, case efficiently.
6308  if (From.getNode()->getNumValues() == 1) {
6309  ReplaceAllUsesWith(From, To);
6310  return;
6311  }
6312 
6313  // Iterate over just the existing users of From. See the comments in
6314  // the ReplaceAllUsesWith above.
6315  SDNode::use_iterator UI = From.getNode()->use_begin(),
6316  UE = From.getNode()->use_end();
6317  RAUWUpdateListener Listener(*this, UI, UE);
6318  while (UI != UE) {
6319  SDNode *User = *UI;
6320  bool UserRemovedFromCSEMaps = false;
6321 
6322  // A user can appear in a use list multiple times, and when this
6323  // happens the uses are usually next to each other in the list.
6324  // To help reduce the number of CSE recomputations, process all
6325  // the uses of this user that we can find this way.
6326  do {
6327  SDUse &Use = UI.getUse();
6328 
6329  // Skip uses of different values from the same node.
6330  if (Use.getResNo() != From.getResNo()) {
6331  ++UI;
6332  continue;
6333  }
6334 
6335  // If this node hasn't been modified yet, it's still in the CSE maps,
6336  // so remove its old self from the CSE maps.
6337  if (!UserRemovedFromCSEMaps) {
6338  RemoveNodeFromCSEMaps(User);
6339  UserRemovedFromCSEMaps = true;
6340  }
6341 
6342  ++UI;
6343  Use.set(To);
6344  } while (UI != UE && *UI == User);
6345 
6346  // We are iterating over all uses of the From node, so if a use
6347  // doesn't use the specific value, no changes are made.
6348  if (!UserRemovedFromCSEMaps)
6349  continue;
6350 
6351  // Now that we have modified User, add it back to the CSE maps. If it
6352  // already exists there, recursively merge the results together.
6353  AddModifiedNodeToCSEMaps(User);
6354  }
6355 
6356  // If we just RAUW'd the root, take note.
6357  if (From == getRoot())
6358  setRoot(To);
6359 }
6360 
6361 namespace {
6362  /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
6363  /// to record information about a use.
6364  struct UseMemo {
6365  SDNode *User;
6366  unsigned Index;
6367  SDUse *Use;
6368  };
6369 
6370  /// operator< - Sort Memos by User.
6371  bool operator<(const UseMemo &L, const UseMemo &R) {
6372  return (intptr_t)L.User < (intptr_t)R.User;
6373  }
6374 }
6375 
6376 /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
6377 /// uses of other values produced by From.getNode() alone. The same value
6378 /// may appear in both the From and To list. The Deleted vector is
6379 /// handled the same way as for ReplaceAllUsesWith.
6381  const SDValue *To,
6382  unsigned Num){
6383  // Handle the simple, trivial case efficiently.
6384  if (Num == 1)
6385  return ReplaceAllUsesOfValueWith(*From, *To);
6386 
6387  // Read up all the uses and make records of them. This helps
6388  // processing new uses that are introduced during the
6389  // replacement process.
6391  for (unsigned i = 0; i != Num; ++i) {
6392  unsigned FromResNo = From[i].getResNo();
6393  SDNode *FromNode = From[i].getNode();
6394  for (SDNode::use_iterator UI = FromNode->use_begin(),
6395  E = FromNode->use_end(); UI != E; ++UI) {
6396  SDUse &Use = UI.getUse();
6397  if (Use.getResNo() == FromResNo) {
6398  UseMemo Memo = { *UI, i, &Use };
6399  Uses.push_back(Memo);
6400  }
6401  }
6402  }
6403 
6404  // Sort the uses, so that all the uses from a given User are together.
6405  std::sort(Uses.begin(), Uses.end());
6406 
6407  for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
6408  UseIndex != UseIndexEnd; ) {
6409  // We know that this user uses some value of From. If it is the right
6410  // value, update it.
6411  SDNode *User = Uses[UseIndex].User;
6412 
6413  // This node is about to morph, remove its old self from the CSE maps.
6414  RemoveNodeFromCSEMaps(User);
6415 
6416  // The Uses array is sorted, so all the uses for a given User
6417  // are next to each other in the list.
6418  // To help reduce the number of CSE recomputations, process all
6419  // the uses of this user that we can find this way.
6420  do {
6421  unsigned i = Uses[UseIndex].Index;
6422  SDUse &Use = *Uses[UseIndex].Use;
6423  ++UseIndex;
6424 
6425  Use.set(To[i]);
6426  } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
6427 
6428  // Now that we have modified User, add it back to the CSE maps. If it
6429  // already exists there, recursively merge the results together.
6430  AddModifiedNodeToCSEMaps(User);
6431  }
6432 }
6433 
6434 /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
6435 /// based on their topological order. It returns the maximum id and a vector
6436 /// of the SDNodes* in assigned order by reference.
6438 
6439  unsigned DAGSize = 0;
6440 
6441  // SortedPos tracks the progress of the algorithm. Nodes before it are
6442  // sorted, nodes after it are unsorted. When the algorithm completes
6443  // it is at the end of the list.
6444  allnodes_iterator SortedPos = allnodes_begin();
6445 
6446  // Visit all the nodes. Move nodes with no operands to the front of
6447  // the list immediately. Annotate nodes that do have operands with their
6448  // operand count. Before we do this, the Node Id fields of the nodes
6449  // may contain arbitrary values. After, the Node Id fields for nodes
6450  // before SortedPos will contain the topological sort index, and the
6451  // Node Id fields for nodes At SortedPos and after will contain the
6452  // count of outstanding operands.
6453  for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
6454  SDNode *N = I++;
6455  checkForCycles(N, this);
6456  unsigned Degree = N->getNumOperands();
6457  if (Degree == 0) {
6458  // A node with no uses, add it to the result array immediately.
6459  N->setNodeId(DAGSize++);
6460  allnodes_iterator Q = N;
6461  if (Q != SortedPos)
6462  SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
6463  assert(SortedPos != AllNodes.end() && "Overran node list");
6464  ++SortedPos;
6465  } else {
6466  // Temporarily use the Node Id as scratch space for the degree count.
6467  N->setNodeId(Degree);
6468  }
6469  }
6470 
6471  // Visit all the nodes. As we iterate, move nodes into sorted order,
6472  // such that by the time the end is reached all nodes will be sorted.
6473  for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) {
6474  SDNode *N = I;
6475  checkForCycles(N, this);
6476  // N is in sorted position, so all its uses have one less operand
6477  // that needs to be sorted.
6478  for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
6479  UI != UE; ++UI) {
6480  SDNode *P = *UI;
6481  unsigned Degree = P->getNodeId();
6482  assert(Degree != 0 && "Invalid node degree");
6483  --Degree;
6484  if (Degree == 0) {
6485  // All of P's operands are sorted, so P may sorted now.
6486  P->setNodeId(DAGSize++);
6487  if (P != SortedPos)
6488  SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
6489  assert(SortedPos != AllNodes.end() && "Overran node list");
6490  ++SortedPos;
6491  } else {
6492  // Update P's outstanding operand count.
6493  P->setNodeId(Degree);
6494  }
6495  }
6496  if (I == SortedPos) {
6497 #ifndef NDEBUG
6498  SDNode *S = ++I;
6499  dbgs() << "Overran sorted position:\n";
6500  S->dumprFull(this); dbgs() << "\n";
6501  dbgs() << "Checking if this is due to cycles\n";
6502  checkForCycles(this, true);
6503 #endif
6504  llvm_unreachable(nullptr);
6505  }
6506  }
6507 
6508  assert(SortedPos == AllNodes.end() &&
6509  "Topological sort incomplete!");
6510  assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
6511  "First node in topological sort is not the entry token!");
6512  assert(AllNodes.front().getNodeId() == 0 &&
6513  "First node in topological sort has non-zero id!");
6514  assert(AllNodes.front().getNumOperands() == 0 &&
6515  "First node in topological sort has operands!");
6516  assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
6517  "Last node in topologic sort has unexpected id!");
6518  assert(AllNodes.back().use_empty() &&
6519  "Last node in topologic sort has users!");
6520  assert(DAGSize == allnodes_size() && "Node count mismatch!");
6521  return DAGSize;
6522 }
6523 
6524 /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
6525 /// value is produced by SD.
6526 void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) {
6527  if (SD) {
6528  assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
6529  SD->setHasDebugValue(true);
6530  }
6531  DbgInfo->add(DB, SD, isParameter);
6532 }
6533 
6534 /// TransferDbgValues - Transfer SDDbgValues.
6536  if (From == To || !From.getNode()->getHasDebugValue())
6537  return;
6538  SDNode *FromNode = From.getNode();
6539  SDNode *ToNode = To.getNode();
6540  ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode);
6541  SmallVector<SDDbgValue *, 2> ClonedDVs;
6542  for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end();
6543  I != E; ++I) {
6544  SDDbgValue *Dbg = *I;
6545  if (Dbg->getKind() == SDDbgValue::SDNODE) {
6546  SDDbgValue *Clone =
6547  getDbgValue(Dbg->getVariable(), Dbg->getExpression(), ToNode,
6548  To.getResNo(), Dbg->isIndirect(), Dbg->getOffset(),
6549  Dbg->getDebugLoc(), Dbg->getOrder());
6550  ClonedDVs.push_back(Clone);
6551  }
6552  }
6553  for (SmallVectorImpl<SDDbgValue *>::iterator I = ClonedDVs.begin(),
6554  E = ClonedDVs.end(); I != E; ++I)
6555  AddDbgValue(*I, ToNode, false);
6556 }
6557 
6558 //===----------------------------------------------------------------------===//
6559 // SDNode Class
6560 //===----------------------------------------------------------------------===//
6561 
6563  DropOperands();
6564 }
6565 
6566 GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
6567  DebugLoc DL, const GlobalValue *GA,
6568  EVT VT, int64_t o, unsigned char TF)
6569  : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
6570  TheGlobal = GA;
6571 }
6572 
6574  SDValue X, unsigned SrcAS,
6575  unsigned DestAS)
6576  : UnarySDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT), X),
6577  SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
6578 
6579 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
6580  EVT memvt, MachineMemOperand *mmo)
6581  : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
6583  MMO->isNonTemporal(), MMO->isInvariant());
6584  assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
6585  assert(isNonTemporal() == MMO->isNonTemporal() &&
6586  "Non-temporal encoding error!");
6587  // We check here that the size of the memory operand fits within the size of
6588  // the MMO. This is because the MMO might indicate only a possible address
6589  // range instead of specifying the affected memory addresses precisely.
6590  assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
6591 }
6592 
6593 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
6594  ArrayRef<SDValue> Ops, EVT memvt, MachineMemOperand *mmo)
6595  : SDNode(Opc, Order, dl, VTs, Ops),
6596  MemoryVT(memvt), MMO(mmo) {
6598  MMO->isNonTemporal(), MMO->isInvariant());
6599  assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
6600  assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
6601 }
6602 
6603 /// Profile - Gather unique data for the node.
6604 ///
6606  AddNodeIDNode(ID, this);
6607 }
6608 
6609 namespace {
6610  struct EVTArray {
6611  std::vector<EVT> VTs;
6612 
6613  EVTArray() {
6614  VTs.reserve(MVT::LAST_VALUETYPE);
6615  for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i)
6616  VTs.push_back(MVT((MVT::SimpleValueType)i));
6617  }
6618  };
6619 }
6620 
6624 
6625 /// getValueTypeList - Return a pointer to the specified value type.
6626 ///
6627 const EVT *SDNode::getValueTypeList(EVT VT) {
6628  if (VT.isExtended()) {
6630  return &(*EVTs->insert(VT).first);
6631  } else {
6632  assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE &&
6633  "Value type out of range!");
6634  return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy];
6635  }
6636 }
6637 
6638 /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
6639 /// indicated value. This method ignores uses of other values defined by this
6640 /// operation.
6641 bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
6642  assert(Value < getNumValues() && "Bad value!");
6643 
6644  // TODO: Only iterate over uses of a given value of the node
6645  for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
6646  if (UI.getUse().getResNo() == Value) {
6647  if (NUses == 0)
6648  return false;
6649  --NUses;
6650  }
6651  }
6652 
6653  // Found exactly the right number of uses?
6654  return NUses == 0;
6655 }
6656 
6657 
6658 /// hasAnyUseOfValue - Return true if there are any use of the indicated
6659 /// value. This method ignores uses of other values defined by this operation.
6660 bool SDNode::hasAnyUseOfValue(unsigned Value) const {
6661  assert(Value < getNumValues() && "Bad value!");
6662 
6663  for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI)
6664  if (UI.getUse().getResNo() == Value)
6665  return true;
6666 
6667  return false;
6668 }
6669 
6670 
6671 /// isOnlyUserOf - Return true if this node is the only use of N.
6672 ///
6673 bool SDNode::isOnlyUserOf(const SDNode *N) const {
6674  bool Seen = false;
6675  for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
6676  SDNode *User = *I;
6677  if (User == this)
6678  Seen = true;
6679  else
6680  return false;
6681  }
6682 
6683  return Seen;
6684 }
6685 
6686 /// isOperand - Return true if this node is an operand of N.
6687 ///
6688 bool SDValue::isOperandOf(const SDNode *N) const {
6689  for (const SDValue &Op : N->op_values())
6690  if (*this == Op)
6691  return true;
6692  return false;
6693 }
6694 
6695 bool SDNode::isOperandOf(const SDNode *N) const {
6696  for (const SDValue &Op : N->op_values())
6697  if (this == Op.getNode())
6698  return true;
6699  return false;
6700 }
6701 
6702 /// reachesChainWithoutSideEffects - Return true if this operand (which must
6703 /// be a chain) reaches the specified operand without crossing any
6704 /// side-effecting instructions on any chain path. In practice, this looks
6705 /// through token factors and non-volatile loads. In order to remain efficient,
6706 /// this only looks a couple of nodes in, it does not do an exhaustive search.
6708  unsigned Depth) const {
6709  if (*this == Dest) return true;
6710 
6711  // Don't search too deeply, we just want to be able to see through
6712  // TokenFactor's etc.
6713  if (Depth == 0) return false;
6714 
6715  // If this is a token factor, all inputs to the TF happen in parallel. If any
6716  // of the operands of the TF does not reach dest, then we cannot do the xform.
6717  if (getOpcode() == ISD::TokenFactor) {
6718  for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
6719  if (!getOperand(i).reachesChainWithoutSideEffects(Dest, Depth-1))
6720  return false;
6721  return true;
6722  }
6723 
6724  // Loads don't have side effects, look through them.
6725  if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) {
6726  if (!Ld->isVolatile())
6727  return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1);
6728  }
6729  return false;
6730 }
6731 
6732 /// hasPredecessor - Return true if N is a predecessor of this node.
6733 /// N is either an operand of this node, or can be reached by recursively
6734 /// traversing up the operands.
6735 /// NOTE: This is an expensive method. Use it carefully.
6736 bool SDNode::hasPredecessor(const SDNode *N) const {
6739  return hasPredecessorHelper(N, Visited, Worklist);
6740 }
6741 
6742 bool
6745  SmallVectorImpl<const SDNode *> &Worklist) const {
6746  if (Visited.empty()) {
6747  Worklist.push_back(this);
6748  } else {
6749  // Take a look in the visited set. If we've already encountered this node
6750  // we needn't search further.
6751  if (Visited.count(N))
6752  return true;
6753  }
6754 
6755  // Haven't visited N yet. Continue the search.
6756  while (!Worklist.empty()) {
6757  const SDNode *M = Worklist.pop_back_val();
6758  for (const SDValue &OpV : M->op_values()) {
6759  SDNode *Op = OpV.getNode();
6760  if (Visited.insert(Op).second)
6761  Worklist.push_back(Op);
6762  if (Op == N)
6763  return true;
6764  }
6765  }
6766 
6767  return false;
6768 }
6769 
6770 uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
6771  assert(Num < NumOperands && "Invalid child # of SDNode!");
6772  return cast<ConstantSDNode>(OperandList[Num])->getZExtValue();
6773 }
6774 
6776  assert(N->getNumValues() == 1 &&
6777  "Can't unroll a vector with multiple results!");
6778 
6779  EVT VT = N->getValueType(0);
6780  unsigned NE = VT.getVectorNumElements();
6781  EVT EltVT = VT.getVectorElementType();
6782  SDLoc dl(N);
6783 
6784  SmallVector<SDValue, 8> Scalars;
6786 
6787  // If ResNE is 0, fully unroll the vector op.
6788  if (ResNE == 0)
6789  ResNE = NE;
6790  else if (NE > ResNE)
6791  NE = ResNE;
6792 
6793  unsigned i;
6794  for (i= 0; i != NE; ++i) {
6795  for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
6796  SDValue Operand = N->getOperand(j);
6797  EVT OperandVT = Operand.getValueType();
6798  if (OperandVT.isVector()) {
6799  // A vector operand; extract a single element.
6800  EVT OperandEltVT = OperandVT.getVectorElementType();
6801  Operands[j] =
6802  getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, Operand,
6803  getConstant(i, dl, TLI->getVectorIdxTy(getDataLayout())));
6804  } else {
6805  // A scalar operand; just use it as is.
6806  Operands[j] = Operand;
6807  }
6808  }
6809 
6810  switch (N->getOpcode()) {
6811  default:
6812  Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands));
6813  break;
6814  case ISD::VSELECT:
6815  Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands));
6816  break;
6817  case ISD::SHL:
6818  case ISD::SRA:
6819  case ISD::SRL:
6820  case ISD::ROTL:
6821  case ISD::ROTR:
6822  Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0],
6823  getShiftAmountOperand(Operands[0].getValueType(),
6824  Operands[1])));
6825  break;
6827  case ISD::FP_ROUND_INREG: {
6828  EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType();
6829  Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
6830  Operands[0],
6831  getValueType(ExtVT)));
6832  }
6833  }
6834  }
6835 
6836  for (; i < ResNE; ++i)
6837  Scalars.push_back(getUNDEF(EltVT));
6838 
6839  return getNode(ISD::BUILD_VECTOR, dl,
6840  EVT::getVectorVT(*getContext(), EltVT, ResNE), Scalars);
6841 }
6842 
6843 
6844 /// isConsecutiveLoad - Return true if LD is loading 'Bytes' bytes from a
6845 /// location that is 'Dist' units away from the location that the 'Base' load
6846 /// is loading from.
6848  unsigned Bytes, int Dist) const {
6849  if (LD->getChain() != Base->getChain())
6850  return false;
6851  EVT VT = LD->getValueType(0);
6852  if (VT.getSizeInBits() / 8 != Bytes)
6853  return false;
6854 
6855  SDValue Loc = LD->getOperand(1);
6856  SDValue BaseLoc = Base->getOperand(1);
6857  if (Loc.getOpcode() == ISD::FrameIndex) {
6858  if (BaseLoc.getOpcode() != ISD::FrameIndex)
6859  return false;
6861  int FI = cast<FrameIndexSDNode>(Loc)->getIndex();
6862  int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex();
6863  int FS = MFI->getObjectSize(FI);
6864  int BFS = MFI->getObjectSize(BFI);
6865  if (FS != BFS || FS != (int)Bytes) return false;
6866  return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Bytes);
6867  }
6868 
6869  // Handle X + C.
6870  if (isBaseWithConstantOffset(Loc)) {
6871  int64_t LocOffset = cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue();
6872  if (Loc.getOperand(0) == BaseLoc) {
6873  // If the base location is a simple address with no offset itself, then
6874  // the second load's first add operand should be the base address.
6875  if (LocOffset == Dist * (int)Bytes)
6876  return true;
6877  } else if (isBaseWithConstantOffset(BaseLoc)) {
6878  // The base location itself has an offset, so subtract that value from the
6879  // second load's offset before comparing to distance * size.
6880  int64_t BOffset =
6881  cast<ConstantSDNode>(BaseLoc.getOperand(1))->getSExtValue();
6882  if (Loc.getOperand(0) == BaseLoc.getOperand(0)) {
6883  if ((LocOffset - BOffset) == Dist * (int)Bytes)
6884  return true;
6885  }
6886  }
6887  }
6888  const GlobalValue *GV1 = nullptr;
6889  const GlobalValue *GV2 = nullptr;
6890  int64_t Offset1 = 0;
6891  int64_t Offset2 = 0;
6892  bool isGA1 = TLI->isGAPlusOffset(Loc.getNode(), GV1, Offset1);
6893  bool isGA2 = TLI->isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2);
6894  if (isGA1 && isGA2 && GV1 == GV2)
6895  return Offset1 == (Offset2 + Dist*Bytes);
6896  return false;
6897 }
6898 
6899 
6900 /// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if
6901 /// it cannot be inferred.
6903  // If this is a GlobalAddress + cst, return the alignment.
6904  const GlobalValue *GV;
6905  int64_t GVOffset = 0;
6906  if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) {
6907  unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
6908  APInt KnownZero(PtrWidth, 0), KnownOne(PtrWidth, 0);
6909  llvm::computeKnownBits(const_cast<GlobalValue *>(GV), KnownZero, KnownOne,
6910  getDataLayout());
6911  unsigned AlignBits = KnownZero.countTrailingOnes();
6912  unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0;
6913  if (Align)
6914  return MinAlign(Align, GVOffset);
6915  }
6916 
6917  // If this is a direct reference to a stack slot, use information about the
6918  // stack slot's alignment.
6919  int FrameIdx = 1 << 31;
6920  int64_t FrameOffset = 0;
6921  if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) {
6922  FrameIdx = FI->getIndex();
6923  } else if (isBaseWithConstantOffset(Ptr) &&
6924  isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
6925  // Handle FI+Cst
6926  FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
6927  FrameOffset = Ptr.getConstantOperandVal(1);
6928  }
6929 
6930  if (FrameIdx != (1 << 31)) {
6932  unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx),
6933  FrameOffset);
6934  return FIInfoAlign;
6935  }
6936 
6937  return 0;
6938 }
6939 
6940 /// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
6941 /// which is split (or expanded) into two not necessarily identical pieces.
6942 std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
6943  // Currently all types are split in half.
6944  EVT LoVT, HiVT;
6945  if (!VT.isVector()) {
6946  LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT);
6947  } else {
6948  unsigned NumElements = VT.getVectorNumElements();
6949  assert(!(NumElements & 1) && "Splitting vector, but not in half!");
6950  LoVT = HiVT = EVT::getVectorVT(*getContext(), VT.getVectorElementType(),
6951  NumElements/2);
6952  }
6953  return std::make_pair(LoVT, HiVT);
6954 }
6955 
6956 /// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
6957 /// low/high part.
6958 std::pair<SDValue, SDValue>
6959 SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
6960  const EVT &HiVT) {
6961  assert(LoVT.getVectorNumElements() + HiVT.getVectorNumElements() <=
6963  "More vector elements requested than available!");
6964  SDValue Lo, Hi;
6965  Lo = getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N,
6966  getConstant(0, DL, TLI->getVectorIdxTy(getDataLayout())));
6967  Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N,
6969  TLI->getVectorIdxTy(getDataLayout())));
6970  return std::make_pair(Lo, Hi);
6971 }
6972 
6975  unsigned Start, unsigned Count) {
6976  EVT VT = Op.getValueType();
6977  if (Count == 0)
6978  Count = VT.getVectorNumElements();
6979 
6980  EVT EltVT = VT.getVectorElementType();
6981  EVT IdxTy = TLI->getVectorIdxTy(getDataLayout());
6982  SDLoc SL(Op);
6983  for (unsigned i = Start, e = Start + Count; i != e; ++i) {
6984  Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
6985  Op, getConstant(i, SL, IdxTy)));
6986  }
6987 }
6988 
6989 // getAddressSpace - Return the address space this GlobalAddress belongs to.
6991  return getGlobal()->getType()->getAddressSpace();
6992 }
6993 
6994 
6997  return Val.MachineCPVal->getType();
6998  return Val.ConstVal->getType();
6999 }
7000 
7002  APInt &SplatUndef,
7003  unsigned &SplatBitSize,
7004  bool &HasAnyUndefs,
7005  unsigned MinSplatBits,
7006  bool isBigEndian) const {
7007  EVT VT = getValueType(0);
7008  assert(VT.isVector() && "Expected a vector type");
7009  unsigned sz = VT.getSizeInBits();
7010  if (MinSplatBits > sz)
7011  return false;
7012 
7013  SplatValue = APInt(sz, 0);
7014  SplatUndef = APInt(sz, 0);
7015 
7016  // Get the bits. Bits with undefined values (when the corresponding element
7017  // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
7018  // in SplatValue. If any of the values are not constant, give up and return
7019  // false.
7020  unsigned int nOps = getNumOperands();
7021  assert(nOps > 0 && "isConstantSplat has 0-size build vector");
7022  unsigned EltBitSize = VT.getVectorElementType().getSizeInBits();
7023 
7024  for (unsigned j = 0; j < nOps; ++j) {
7025  unsigned i = isBigEndian ? nOps-1-j : j;
7026  SDValue OpVal = getOperand(i);
7027  unsigned BitPos = j * EltBitSize;
7028 
7029  if (OpVal.getOpcode() == ISD::UNDEF)
7030  SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos + EltBitSize);
7031  else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal))
7032  SplatValue |= CN->getAPIntValue().zextOrTrunc(EltBitSize).
7033  zextOrTrunc(sz) << BitPos;
7034  else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal))
7035  SplatValue |= CN->getValueAPF().bitcastToAPInt().zextOrTrunc(sz) <<BitPos;
7036  else
7037  return false;
7038  }
7039 
7040  // The build_vector is all constants or undefs. Find the smallest element
7041  // size that splats the vector.
7042 
7043  HasAnyUndefs = (SplatUndef != 0);
7044  while (sz > 8) {
7045 
7046  unsigned HalfSize = sz / 2;
7047  APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize);
7048  APInt LowValue = SplatValue.trunc(HalfSize);
7049  APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize);
7050  APInt LowUndef = SplatUndef.trunc(HalfSize);
7051 
7052  // If the two halves do not match (ignoring undef bits), stop here.
7053  if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
7054  MinSplatBits > HalfSize)
7055  break;
7056 
7057  SplatValue = HighValue | LowValue;
7058  SplatUndef = HighUndef & LowUndef;
7059 
7060  sz = HalfSize;
7061  }
7062 
7063  SplatBitSize = sz;
7064  return true;
7065 }
7066 
7068  if (UndefElements) {
7069  UndefElements->clear();
7070  UndefElements->resize(getNumOperands());
7071  }
7072  SDValue Splatted;
7073  for (unsigned i = 0, e = getNumOperands(); i != e; ++i) {
7074  SDValue Op = getOperand(i);
7075  if (Op.getOpcode() == ISD::UNDEF) {
7076  if (UndefElements)
7077  (*UndefElements)[i] = true;
7078  } else if (!Splatted) {
7079  Splatted = Op;
7080  } else if (Splatted != Op) {
7081  return SDValue();
7082  }
7083  }
7084 
7085  if (!Splatted) {
7086  assert(getOperand(0).getOpcode() == ISD::UNDEF &&
7087  "Can only have a splat without a constant for all undefs.");
7088  return getOperand(0);
7089  }
7090 
7091  return Splatted;
7092 }
7093 
7096  return dyn_cast_or_null<ConstantSDNode>(getSplatValue(UndefElements));
7097 }
7098 
7101  return dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements));
7102 }
7103 
7105  for (const SDValue &Op : op_values()) {
7106  unsigned Opc = Op.getOpcode();
7107  if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP)
7108  return false;
7109  }
7110  return true;
7111 }
7112 
7113 bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) {
7114  // Find the first non-undef value in the shuffle mask.
7115  unsigned i, e;
7116  for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i)
7117  /* search */;
7118 
7119  assert(i != e && "VECTOR_SHUFFLE node with all undef indices!");
7120 
7121  // Make sure all remaining elements are either undef or the same as the first
7122  // non-undef value.
7123  for (int Idx = Mask[i]; i != e; ++i)
7124  if (Mask[i] >= 0 && Mask[i] != Idx)
7125  return false;
7126  return true;
7127 }
7128 
7129 #ifndef NDEBUG
7130 static void checkForCyclesHelper(const SDNode *N,
7133  const llvm::SelectionDAG *DAG) {
7134  // If this node has already been checked, don't check it again.
7135  if (Checked.count(N))
7136  return;
7137 
7138  // If a node has already been visited on this depth-first walk, reject it as
7139  // a cycle.
7140  if (!Visited.insert(N).second) {
7141  errs() << "Detected cycle in SelectionDAG\n";
7142  dbgs() << "Offending node:\n";
7143  N->dumprFull(DAG); dbgs() << "\n";
7144  abort();
7145  }
7146 
7147  for (const SDValue &Op : N->op_values())
7148  checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
7149 
7150  Checked.insert(N);
7151  Visited.erase(N);
7152 }
7153 #endif
7154 
7156  const llvm::SelectionDAG *DAG,
7157  bool force) {
7158 #ifndef NDEBUG
7159  bool check = force;
7160 #ifdef XDEBUG
7161  check = true;
7162 #endif // XDEBUG
7163  if (check) {
7164  assert(N && "Checking nonexistent SDNode");
7167  checkForCyclesHelper(N, visited, checked, DAG);
7168  }
7169 #endif // !NDEBUG
7170 }
7171 
7172 void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
7173  checkForCycles(DAG->getRoot().getNode(), DAG, force);
7174 }
void clearAllBits()
Set every bit to 0.
Definition: APInt.h:1239
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
SDValue getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT TVT, bool isNonTemporal, bool isVolatile, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes())
SDNode * MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef< SDValue > Ops)
This mutates the specified node to have the specified return type, opcode, and operands.
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
Definition: BitVector.h:192
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:477
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:450
APInt LLVM_ATTRIBUTE_UNUSED_RESULT ashr(unsigned shiftAmt) const
Arithmetic right-shift function.
Definition: APInt.cpp:1051
void AddPointer(const void *Ptr)
Add* - Add various data types to Bit data.
Definition: FoldingSet.cpp:52
bool use_empty() const
Return true if there are no uses of this node.
opStatus divide(const APFloat &, roundingMode)
Definition: APFloat.cpp:1709
unsigned Log2_32_Ceil(uint32_t Value)
Log2_32_Ceil - This function returns the ceil log base 2 of the specified value, 32 if the value is z...
Definition: MathExtras.h:481
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:724
The memory access reads data.
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:104
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
unsigned getRawFlags() const
Return a raw encoding of the flags.
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, unsigned FixedArgs=-1)
SDValue getValue(unsigned R) const
virtual SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, SDLoc dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, unsigned Align, bool isVolatile, MachinePointerInfo DstPtrInfo) const
EmitTargetCodeForMemset - Emit target-specific code that performs a memset.
static APInt getSignBit(unsigned BitWidth)
Get the SignBit for a specific bit width.
Definition: APInt.h:446
APInt LLVM_ATTRIBUTE_UNUSED_RESULT byteSwap() const
Definition: APInt.cpp:790
The memory access writes data.
ConstantFPSDNode * getConstantFPSplatNode(BitVector *UndefElements=nullptr) const
Returns the splatted constant FP or null if this is not a constant FP splat.
raw_ostream & errs()
This returns a reference to a raw_ostream for standard error.
const char * getSymbol() const
APInt LLVM_ATTRIBUTE_UNUSED_RESULT abs() const
Get the absolute value;.
Definition: APInt.h:1571
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand...
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant, which is required to be operand #1) half of the integer or float value specified as operand #0.
Definition: ISDOpcodes.h:175
static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, SDLoc dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, unsigned Align, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
SDValue getSplatValue(BitVector *UndefElements=nullptr) const
Returns the splatted value or a null value if this is not a splat.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:223
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
Definition: APInt.h:453
LLVMContext * getContext() const
Definition: SelectionDAG.h:289
SDValue getTargetIndex(int Index, EVT VT, int64_t Offset=0, unsigned char TargetFlags=0)
Keeps track of dbg_value information through SDISel.
Definition: SelectionDAG.h:114
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef...
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1327
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
Definition: SmallPtrSet.h:78
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, unsigned Align, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
void dumprFull(const SelectionDAG *G=nullptr) const
printrFull to dbgs().
SDValue getIndexedLoad(SDValue OrigLoad, SDLoc dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
void dump() const
Dump this node, for debugging.
NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition: ISDOpcodes.h:42
MDNODE_SDNODE - This is a node that holdes an MDNode*, which is used to reference metadata in the IR...
Definition: ISDOpcodes.h:615
unsigned getMaxStoresPerMemcpy(bool OptSize) const
Get maximum # of store operations permitted for llvm.memcpy.
size_t size() const
size - Get the string size.
Definition: StringRef.h:113
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:292
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
bool isKnownNeverNaN(SDValue Op) const
Test whether the given SDValue is known to never be NaN.
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:554
static void Found()
bool none() const
none - Returns true if none of the bits are set.
Definition: BitVector.h:150
Various leaf nodes.
Definition: ISDOpcodes.h:60
SDValue getBoolExtOrTrunc(SDValue Op, SDLoc SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
static const fltSemantics IEEEdouble
Definition: APFloat.h:133
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:301
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:39
This class is used for three-operand SDNodes.
SDVTList getVTList() const
SDValue getVAArg(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input...
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:432
SDValue getZeroExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT)
Return an operation which will zero extend the low lanes of the operand into the specified vector typ...
CallLoweringInfo & setDebugLoc(SDLoc dl)
unsigned getBitWidth() const
getBitWidth - Return the bitwidth of this constant.
Definition: Constants.h:111
SDValue getMergeValues(ArrayRef< SDValue > Ops, SDLoc dl)
Create a MERGE_VALUES node from the given operands.
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:210
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:285
unsigned getPointerTypeSizeInBits(Type *) const
Layout pointer size, in bits, based on the type.
Definition: DataLayout.cpp:602
bool isExtended() const
isExtended - Test if the given EVT is extended (as opposed to being simple).
Definition: ValueTypes.h:100
unsigned getPrefTypeAlignment(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:684
Clients of various APIs that cause global effects on the DAG can optionally implement this interface...
Definition: SelectionDAG.h:225
iterator end() const
Definition: ArrayRef.h:123
static void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl< int > &M)
static bool isBinOpWithFlags(unsigned Opcode)
Returns true if the opcode is a binary operation with flags.
friend struct DAGUpdateListener
DAGUpdateListener is a friend so it can manipulate the listener stack.
Definition: SelectionDAG.h:257
const TargetSubtargetInfo & getSubtarget() const
Definition: SelectionDAG.h:286
unsigned InferPtrAlignment(SDValue Ptr) const
Infer alignment of a load / store address.
void computeKnownBits(SDValue Op, APInt &KnownZero, APInt &KnownOne, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in the KnownZero/KnownO...
unsigned getPointerPrefAlignment(unsigned AS=0) const
Return target's alignment for stack-based pointers FIXME: The defaults need to be removed once all of...
Definition: DataLayout.cpp:584
size_type count(PtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:276
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
CondCode getSetCCAndOperation(CondCode Op1, CondCode Op2, bool isInteger)
getSetCCAndOperation - Return the result of a logical AND between different comparisons of identical ...
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
Definition: APInt.h:531
const GlobalValue * getGlobal() const
virtual void addSelectionDAGCSEId(FoldingSetNodeID &ID)=0
void computeKnownBits(Value *V, APInt &KnownZero, APInt &KnownOne, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
static PointerType * get(Type *ElementType, unsigned AddressSpace)
PointerType::get - This constructs a pointer to an object of the specified type in a numbered address...
Definition: Type.cpp:738
SDValue getBasicBlock(MachineBasicBlock *MBB)
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:228
Completely target-dependent object reference.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
static ISD::NodeType getExtendForContent(BooleanContent Content)
bool getHasDebugValue() const
Get this bit.
StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:405
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
Definition: SelectionDAG.h:253
Type * getTypeForEVT(LLVMContext &Context) const
getTypeForEVT - This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:181
opStatus
IEEE-754R 7: Default exception handling.
Definition: APFloat.h:166
SDValue getLoad(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, bool isVolatile, bool isNonTemporal, bool isInvariant, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
SDValue getZeroExtendInReg(SDValue Op, SDLoc DL, EVT SrcTy)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value...
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
Definition: ISDOpcodes.h:122
void clearSign()
Definition: APFloat.cpp:1630
unsigned getRawSubclassData() const
Return the SubclassData value, which contains an encoding of the volatile flag, as well as bits used ...
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:679
unsigned getNumOperands() const
Return the number of values used by this operation.
unsigned getNumOperands() const
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
void Deallocate(SubClass *E)
Deallocate - Release storage for the pointed-to object.
virtual bool isZExtFree(Type *, Type *) const
Return true if any actual instruction that defines a value of type Ty1 implicitly zero-extends the va...
This class is an extension of BinarySDNode used from those opcodes that have associated extra flags...
A debug info location.
Definition: DebugLoc.h:34
Metadata node.
Definition: Metadata.h:740
const SDValue & getOperand(unsigned Num) const
F(f)
const Function * getFunction() const
getFunction - Return the LLVM function that this machine code represents
SDValue getMaskedGather(SDVTList VTs, EVT VT, SDLoc dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, unsigned Align=1, bool *=nullptr) const
Determine if the target supports unaligned memory accesses.
unsigned getAddressSpace() const
Return the address space of the Pointer type.
Definition: DerivedTypes.h:472
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:423
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:318
void setNodeId(int Id)
Set unique node id.
static bool isCommutativeBinOp(unsigned Opcode)
Returns true if the opcode is a commutative binary operation.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned char TargetFlags=0)
ConstantSDNode * getConstantSplatNode(BitVector *UndefElements=nullptr) const
Returns the splatted constant or null if this is not a constant splat.
CvtCode
CvtCode enum - This enum defines the various converts CONVERT_RNDSAT supports.
Definition: ISDOpcodes.h:885
const MachinePointerInfo & getPointerInfo() const
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:344
Same for subtraction.
Definition: ISDOpcodes.h:231
void reserve(size_type N)
Definition: SmallVector.h:401
unsigned getNumSignBits() const
Computes the number of leading bits of this APInt that are equal to its sign bit. ...
Definition: APInt.h:1381
const SDValue & getValue() const
void DeleteNode(SDNode *N)
Remove the specified node from the system.
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1 at the ...
Definition: ISDOpcodes.h:287
SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align=0, int Offs=0, bool isT=false, unsigned char TargetFlags=0)
APInt LLVM_ATTRIBUTE_UNUSED_RESULT zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
Definition: APInt.cpp:1015
std::size_t countLeadingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the most significant bit to the least stopping at the first 1...
Definition: MathExtras.h:178
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, unsigned f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
getMachineMemOperand - Allocate a new MachineMemOperand.
unsigned int NumVTs
EntryToken - This is the marker used to indicate the start of a region.
Definition: ISDOpcodes.h:45
virtual SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, SDLoc dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, unsigned Align, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const
EmitTargetCodeForMemcpy - Emit target-specific code that performs a memcpy.
static sys::Mutex Lock
void AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter)
Add a dbg_value SDNode.
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:357
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:39
unsigned getResNo() const
get the index which selects a specific result in the SDNode
SDValue getMCSymbol(MCSymbol *Sym, EVT VT)
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
bool bitsLT(EVT VT) const
bitsLT - Return true if this has less bits than VT.
Definition: ValueTypes.h:189
virtual void NodeUpdated(SDNode *N)
The node N that was updated.
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
Definition: ISDOpcodes.h:254
void computeKnownBitsFromRangeMetadata(const MDNode &Ranges, APInt &KnownZero)
Compute known bits from the range metadata.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
SDValue getExternalSymbol(const char *Sym, EVT VT)
SDValue getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst, SDValue Src, SDValue Size, unsigned Align, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
static MachinePointerInfo getFixedStack(int FI, int64_t offset=0)
getFixedStack - Return a MachinePointerInfo record that refers to the the specified FrameIndex...
NOTE: avoid using this node as this may disappear in the future and most targets don't support it...
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:150
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:242
bool isKnownNeverZero(SDValue Op) const
Test whether the given SDValue is known to never be positive or negative Zero.
bool isNegative() const
Determine sign of this APInt.
Definition: APInt.h:319
APInt LLVM_ATTRIBUTE_UNUSED_RESULT rotl(unsigned rotateAmt) const
Rotate left by rotateAmt.
Definition: APInt.cpp:1271
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations...
Definition: ISDOpcodes.h:371
bool isVector() const
isVector - Return true if this is a vector value type.
Definition: ValueTypes.h:115
APInt LLVM_ATTRIBUTE_UNUSED_RESULT urem(const APInt &RHS) const
Unsigned remainder operation.
Definition: APInt.cpp:1887
BlockAddress - The address of a basic block.
Definition: Constants.h:802
void clear()
Clear state and free memory necessary to make this SelectionDAG ready to process a new block...
SDValue getStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, bool isVolatile, bool isNonTemporal, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
MachineMemOperand - A description of a memory reference used in the backend.
void clear()
clear - Clear all bits.
Definition: BitVector.h:187
SDValue getMemset(SDValue Chain, SDLoc dl, SDValue Dst, SDValue Src, SDValue Size, unsigned Align, bool isVol, bool isTailCall, MachinePointerInfo DstPtrInfo)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
void Reset()
Deallocate all but the current slab and reset the current pointer to the beginning of it...
Definition: Allocator.h:189
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:106
Shift and rotation operations.
Definition: ISDOpcodes.h:332
unsigned getIROrder()
T LLVM_ATTRIBUTE_UNUSED_RESULT pop_back_val()
Definition: SmallVector.h:406
opStatus convertToInteger(integerPart *, unsigned int, bool, roundingMode, bool *) const
Definition: APFloat.cpp:2191
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:98
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef...
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:69
bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist) const
Return true if N is a predecessor of this node.
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:283
bool isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base, unsigned Bytes, int Dist) const
Return true if LD is loading 'Bytes' bytes from a location that is 'Dist' units away from the locatio...
virtual void computeKnownBitsForTargetNode(const SDValue Op, APInt &KnownZero, APInt &KnownOne, const SelectionDAG &DAG, unsigned Depth=0) const
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
APInt LLVM_ATTRIBUTE_UNUSED_RESULT lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition: APInt.cpp:1142
CallLoweringInfo & setChain(SDValue InChain)
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:181
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APInt.h:33
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
unsigned getAddressSpace() const
MachinePointerInfo getWithOffset(int64_t O) const
SimpleValueType SimpleTy
SDDbgValue * getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N, unsigned R, bool IsIndirect, uint64_t Off, DebugLoc DL, unsigned O)
Creates a SDDbgValue node.
void AddInteger(signed I)
Definition: FoldingSet.cpp:60
EVT getScalarType() const
getScalarType - If this is a vector type, return the element type, otherwise return this...
Definition: ValueTypes.h:210
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt) For double-word atomic operations: ValLo, ValHi, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amtLo, amtHi) ValLo, ValHi, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amtLo, amtHi) These correspond to the atomicrmw instruction.
Definition: ISDOpcodes.h:687
Number of individual test Apply this number of consecutive mutations to each input exit after the first new interesting input is found the minimized corpus is saved into the first input directory Number of jobs to run If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
SynchronizationScope
Definition: Instructions.h:49
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
allnodes_const_iterator allnodes_end() const
Definition: SelectionDAG.h:318
bool bitsGE(EVT VT) const
bitsGE - Return true if this has no less bits than VT.
Definition: ValueTypes.h:183
int getMaskElt(unsigned Idx) const
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
This class is used to represent EVT's, which are used to parameterize some operations.
The memory access is volatile.
This is an SDNode representing atomic operations.
#define false
Definition: ConvertUTF.c:65
MDNode * getExpression() const
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue getAnyExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT)
Return an operation which will any-extend the low lanes of the operand into the specified vector type...
void copySign(const APFloat &)
Definition: APFloat.cpp:1637
ELFYAML::ELF_STO Other
Definition: ELFYAML.cpp:591
#define G(x, y, z)
Definition: MD5.cpp:52
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
Definition: ValueTypes.h:110
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
unsigned getEVTAlignment(EVT MemoryVT) const
getEVTAlignment - Compute the default alignment value for the given type.
bool getBoolValue() const
Convert APInt to a boolean value.
Definition: APInt.h:400
This class is used to represent an MSTORE node.
AtomicOrdering
Definition: Instructions.h:38
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getVectorElementType() const
getVectorElementType - Given a vector type, return the type of each element.
Definition: ValueTypes.h:216
void assign(size_type NumElts, const T &Elt)
Definition: SmallVector.h:442
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
static void VerifySDNode(SDNode *N)
VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector...
void checkForCycles(const SelectionDAG *DAG, bool force=false)
unsigned getIROrder() const
Return the node ordering.
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:393
bool isIndirect() const
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
Definition: SmallVector.h:57
SDValue getRegisterMask(const uint32_t *RegMask)
DAGUpdateListener *const Next
Definition: SelectionDAG.h:226
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:351
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:191
SDValue getVectorShuffle(EVT VT, SDLoc dl, SDValue N1, SDValue N2, const int *MaskElts)
Return an ISD::VECTOR_SHUFFLE node.
bool sgt(const APInt &RHS) const
Signed greather than comparison.
Definition: APInt.h:1119
DbgValueKind getKind() const
static std::error_code check(std::error_code Err)
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: ArrayRef.h:31
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:217
MachineConstantPoolValue * getMachineCPVal() const
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:659
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:804
const APInt & getAPIntValue() const
uint16_t SubclassData
This member is defined by this class, but is not used for anything.
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification, or lowering of the constant.
Definition: ISDOpcodes.h:116
EVT getMemoryVT() const
Return the type of the in-memory value.
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:410
const ConstantInt * getConstantIntValue() const
SDValue getAnyExtOrTrunc(SDValue Op, SDLoc DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
void setIROrder(unsigned Order)
Set the node ordering.
virtual bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const
Return true if it is beneficial to convert a load of a constant to just the constant itself...
static unsigned encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile, bool isNonTemporal, bool isInvariant)
encodeMemSDNodeFlags - Generic routine for computing a value for use in the CSE map that carries vola...
void init(MachineFunction &mf)
Prepare this SelectionDAG to process code in the given MachineFunction.
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:142
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:134
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:284
bool bitsLE(EVT VT) const
bitsLE - Return true if this has no more bits than VT.
Definition: ValueTypes.h:195
unsigned char getTargetFlags() const
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition: APInt.cpp:520
UNDEF - An undefined node.
Definition: ISDOpcodes.h:169
SDValue getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS)
Return an AddrSpaceCastSDNode.
This class is used to represent ISD::STORE nodes.
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:436
static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList)
AddNodeIDValueTypes - Value type lists are intern'd so we can represent them solely with their pointe...
opStatus mod(const APFloat &, roundingMode)
C fmod, or llvm frem.
Definition: APFloat.cpp:1765
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a vector with the specified, possibly variable...
Definition: ISDOpcodes.h:262
This corresponds to the llvm.lifetime.
Definition: ISDOpcodes.h:711
opStatus convertFromAPInt(const APInt &, bool, roundingMode)
Definition: APFloat.cpp:2272
LLVM_CONSTEXPR size_t array_lengthof(T(&)[N])
Find the length of an array.
Definition: STLExtras.h:247
The memory access is non-temporal.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Get a value with high bits set.
Definition: APInt.h:513
SDNode * getNode() const
get the SDNode which holds the desired result
FoldingSetNodeID - This class is used to gather all the unique data bits of a node.
Definition: FoldingSet.h:297
A self-contained host- and target-independent arbitrary-precision floating-point software implementat...
Definition: APFloat.h:122
bool hasPredecessor(const SDNode *N) const
Return true if N is a predecessor of this node.
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:239
unsigned getStoreSize() const
getStoreSize - Return the number of bytes overwritten by a store of the specified value type...
Definition: ValueTypes.h:245
SDValue getMDNode(const MDNode *MD)
Return an MDNodeSDNode which holds an MDNode.
#define P(N)
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
SDValue getMaskedScatter(SDVTList VTs, EVT VT, SDLoc dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO)
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:157
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
unsigned getStoreSizeInBits() const
getStoreSizeInBits - Return the number of bits overwritten by a store of the specified value type...
Definition: ValueTypes.h:251
bool isInteger() const
isInteger - Return true if this is an integer, or a vector integer type.
SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV)
Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to the shuffle node in input but with swa...
bool isZero() const
Return true if the value is positive or negative zero.
AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT, SDValue X, unsigned SrcAS, unsigned DestAS)
APInt LLVM_ATTRIBUTE_UNUSED_RESULT trunc(unsigned width) const
Truncate to new width.
Definition: APInt.cpp:932
bool hasFloatingPointExceptions() const
Return true if target supports floating point exceptions.
bool isMachineConstantPoolEntry() const
MVT - Machine Value Type.
static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC, SDVTList VTList, ArrayRef< SDValue > OpList)
virtual bool needsStackRealignment(const MachineFunction &MF) const
needsStackRealignment - true if storage within the function requires the stack pointer to be aligned ...
const SDValue & getOperand(unsigned i) const
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
Simple binary floating point operators.
Definition: ISDOpcodes.h:237
bool isNonTemporal() const
static SDValue getMemsetStores(SelectionDAG &DAG, SDLoc dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, unsigned Align, bool isVol, MachinePointerInfo DstPtrInfo)
Lower the call to 'memset' intrinsic function into a series of store operations.
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
static bool doNotCSE(SDNode *N)
doNotCSE - Return true if CSE should not be performed for this node.
bool sge(const APInt &RHS) const
Signed greather or equal comparison.
Definition: APInt.h:1153
void DropOperands()
Release the operands and set this node to have zero operands.
virtual EVT getOptimalMemOpType(uint64_t, unsigned, unsigned, bool, bool, bool, MachineFunction &) const
Returns the target specific optimal type for load and store operations as a result of memset...
PointerUnion< const Value *, const PseudoSourceValue * > V
V - This is the IR pointer value for the access, or it is null if unknown.
This is an important base class in LLVM.
Definition: Constant.h:41
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:780
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL...
Definition: ISDOpcodes.h:267
const Constant * getConstVal() const
This file contains the declarations for the subclasses of Constant, which represent the different fla...
LLVM_ATTRIBUTE_RETURNS_NONNULL LLVM_ATTRIBUTE_RETURNS_NOALIAS void * Allocate(size_t Size, size_t Alignment)
Allocate space at the specified alignment.
Definition: Allocator.h:208
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:219
virtual bool isSafeMemOpType(MVT) const
Returns true if it's safe to use load / store of the specified type to expand memcpy / memset inline...
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:233
const unsigned int integerPartWidth
Definition: APInt.h:40
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:264
opStatus fusedMultiplyAdd(const APFloat &, const APFloat &, roundingMode)
Definition: APFloat.cpp:1805
unsigned getMaxStoresPerMemmove(bool OptSize) const
Get maximum # of store operations permitted for llvm.memmove.
INSERT_SUBREG - This instruction takes three operands: a register that has subregisters, a register providing an insert value, and a subregister index.
Definition: TargetOpcodes.h:49
void AddBoolean(bool B)
Definition: FoldingSet.h:317
static SDValue getMemsetStringVal(EVT VT, SDLoc dl, SelectionDAG &DAG, const TargetLowering &TLI, StringRef Str)
getMemsetStringVal - Similar to getMemsetValue.
bool sle(const APInt &RHS) const
Signed less or equal comparison.
Definition: APInt.h:1085
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
This class is used for two-operand SDNodes.
virtual bool isTruncateFree(Type *, Type *) const
Return true if it's free to truncate a value of type Ty1 to type Ty2.
static void AddNodeIDOperands(FoldingSetNodeID &ID, ArrayRef< SDValue > Ops)
AddNodeIDOperands - Various routines for adding operands to the NodeID data.
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:225
This class provides iterator support for SDUse operands that use a specific SDNode.
SDValue getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
BumpPtrAllocator & getAlloc()
Definition: SelectionDAG.h:145
ArrayRef< SDDbgValue * > GetDbgValues(const SDNode *SD)
Get the debug values which reference the given SDNode.
static ManagedStatic< std::set< EVT, EVT::compareRawBits > > EVTs
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang","erlang-compatible garbage collector")
SDValue getTargetConstant(uint64_t Val, SDLoc DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:436
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1273
iterator begin() const
Definition: ArrayRef.h:122
opStatus convert(const fltSemantics &, roundingMode, bool *)
APFloat::convert - convert a value of one floating point type to another.
Definition: APFloat.cpp:1972
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
Definition: APInt.h:1137
unsigned getOpcode() const
TargetIndex - Like a constant pool entry, but with completely target-dependent semantics.
Definition: ISDOpcodes.h:135
void InitOperands(SDUse *Ops, const SDValue &Op0)
Initialize the operands list of this with 1 operand.
APInt LLVM_ATTRIBUTE_UNUSED_RESULT sdiv(const APInt &RHS) const
Signed division function for APInt.
Definition: APInt.cpp:1876
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:57
unsigned char getTargetFlags() const
void changeSign()
Definition: APFloat.cpp:1623
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
This class is used for single-operand SDNodes.
SI Fold Operands
CondCode getSetCCSwappedOperands(CondCode Operation)
getSetCCSwappedOperands - Return the operation corresponding to (Y op X) when given the operation for...
unsigned countPopulation() const
Count the number of bits set.
Definition: APInt.h:1415
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:598
ilist< SDNode >::size_type allnodes_size() const
Definition: SelectionDAG.h:322
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:129
bool isVolatile() const
MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO)
const SDValue & getValue() const
static bool isValueValidForType(EVT VT, const APFloat &Val)
SDValue getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, bool isVolatile, bool isNonTemporal, bool isInvariant, unsigned Alignment, const AAMDNodes &AAInfo=AAMDNodes())
bool isThreadLocal() const
If the value is "Thread Local", its value isn't shared by the threads.
Definition: GlobalValue.h:152
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:338
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo...
Definition: ISDOpcodes.h:673
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:468
static ManagedStatic< sys::SmartMutex< true > > VTMutex
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:416
ArrayRef< SDDbgValue * > getSDDbgValues(const SDNode *Node)
Definition: SelectionDAG.h:151
EVT - Extended Value Type.
Definition: ValueTypes.h:31
bool isPointerTy() const
isPointerTy - True if this is an instance of PointerType.
Definition: Type.h:217
std::vector< ArgListEntry > ArgListTy
const APFloat & getValueAPF() const
SDValue getGlobalAddress(const GlobalValue *GV, SDLoc DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned char TargetFlags=0)
SDValue getMaskedLoad(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType)
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
bool isEqualTo(SDValue A, SDValue B) const
Test whether two SDValues are known to compare equal.
Abstract base class for all machine specific constantpool value subclasses.
This structure contains all information that is necessary for lowering calls.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
DebugLoc getDebugLoc()
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements)
getVectorVT - Returns the EVT that represents a vector NumElements in length, where each element is o...
Definition: ValueTypes.h:70
MachinePointerInfo - This class contains a discriminated union of information about pointers in memor...
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition: APInt.h:386
static const fltSemantics IEEEhalf
Definition: APFloat.h:131
HANDLENODE node - Used as a handle for various purposes.
Definition: ISDOpcodes.h:627
APInt LLVM_ATTRIBUTE_UNUSED_RESULT rotr(unsigned rotateAmt) const
Rotate right by rotateAmt.
Definition: APInt.cpp:1282
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:335
SDValue getMaskedStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, bool IsTrunc)
const BlockAddress * getBlockAddress() const
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
bool isNull() const
isNull - Return true if the pointer held in the union is null, regardless of which type it is...
Definition: PointerUnion.h:109
virtual const TargetSelectionDAGInfo * getSelectionDAGInfo() const
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:576
The memory access is invariant.
APInt LLVM_ATTRIBUTE_UNUSED_RESULT srem(const APInt &RHS) const
Function for signed remainder operation.
Definition: APInt.cpp:1924
const MachinePointerInfo & getPointerInfo() const
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
MachineMemOperand * MMO
Memory reference information.
const MDNode * getRanges() const
Returns the Ranges that describes the dereference.
static const char *const Magic
Definition: Archive.cpp:26
const SDValue & getOffset() const
bool bitsGT(EVT VT) const
bitsGT - Return true if this has more bits than VT.
Definition: ValueTypes.h:177
ArrayRef< int > getMask() const
bool ugt(const APInt &RHS) const
Unsigned greather than comparison.
Definition: APInt.h:1101
static void commuteMask(SmallVectorImpl< int > &Mask)
Change values in a shuffle permute mask assuming the two vector operands have swapped position...
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type...
unsigned countTrailingZeros() const
Count the number of trailing zero bits.
Definition: APInt.cpp:749
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space...
Definition: DataLayout.cpp:694
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:50
unsigned getABITypeAlignment(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Definition: DataLayout.cpp:674
bool isBuildVectorAllOnes(const SDNode *N)
Node predicates.
SDValue getConvertRndSat(EVT VT, SDLoc dl, SDValue Val, SDValue DTy, SDValue STy, SDValue Rnd, SDValue Sat, ISD::CvtCode Code)
Returns the ConvertRndSat Note: Avoid using this node because it may disappear in the future and most...
SDValue FoldConstantArithmetic(unsigned Opcode, SDLoc DL, EVT VT, SDNode *Cst1, SDNode *Cst2)
EXTRACT_SUBREG - This instruction takes two operands: a register that has subregisters, and a subregister index.
Definition: TargetOpcodes.h:41
SDDbgValue * getConstantDbgValue(MDNode *Var, MDNode *Expr, const Value *C, uint64_t Off, DebugLoc DL, unsigned O)
Constant.
SDValue getNOT(SDLoc DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
static bool isMemSrcFromString(SDValue Src, StringRef &Str)
isMemSrcFromString - Returns true if memcpy source is a string constant.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:299
This is the shared class of boolean and integer constants.
Definition: Constants.h:47
int64_t SignExtend64(uint64_t x)
SignExtend64 - Sign extend B-bit number x to 64-bit int.
Definition: MathExtras.h:634
allnodes_const_iterator allnodes_begin() const
Definition: SelectionDAG.h:317
virtual void NodeDeleted(SDNode *N, SDNode *E)
The node N that was deleted and, if E is not null, an equivalent node E that replaced it...
bool slt(const APInt &RHS) const
Signed less than comparison.
Definition: APInt.cpp:552
bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side...
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false...
Definition: SmallPtrSet.h:271
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:273
opStatus add(const APFloat &, roundingMode)
Definition: APFloat.cpp:1676
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provides VTs and return the low/high part...
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:179
X = FP_ROUND_INREG(Y, VT) - This operator takes an FP register, and rounds it to a floating point val...
Definition: ISDOpcodes.h:465
SDValue getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDNode * SelectNodeTo(SDNode *N, unsigned TargetOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type...
ADDRSPACECAST - This operator converts between pointers of different address spaces.
Definition: ISDOpcodes.h:481
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
unsigned getObjectAlignment(int ObjectIdx) const
Return the alignment of the specified stack object.
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:222
virtual const TargetLowering * getTargetLowering() const
bool allOperandsUndef(const SDNode *N)
Return true if the node has at least one operand and all operands of the specified node are ISD::UNDE...
opStatus multiply(const APFloat &, roundingMode)
Definition: APFloat.cpp:1690
An SDNode that represents everything that will be needed to construct a MachineInstr.
const SDValue & getChain() const
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:548
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:335
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
This is an abstract virtual class for memory operations.
unsigned char getTargetFlags() const
Unsigned from Signed.
Definition: ISDOpcodes.h:894
SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, const Value *PtrVal, unsigned Alignment, AtomicOrdering Ordering, SynchronizationScope SynchScope)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static void AddNodeIDFlags(FoldingSetNodeID &ID, unsigned Opcode, const SDNodeFlags *Flags)
Add logical or fast math flag values to FoldingSetNodeID value.
unsigned getAddrSpace() const
getAddrSpace - Return the LLVM IR address space number that this pointer points into.
void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0)
Append the extracted elements from Start to Count out of the vector Op in Args.
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:582
virtual bool isGAPlusOffset(SDNode *N, const GlobalValue *&GA, int64_t &Offset) const
Returns true (and the GlobalValue and the offset) if the node is a GlobalAddress + offset...
static const int FIRST_TARGET_MEMORY_OPCODE
FIRST_TARGET_MEMORY_OPCODE - Target-specific pre-isel operations which do not reference a specific me...
Definition: ISDOpcodes.h:731
MachineFrameInfo * getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Represents one node in the SelectionDAG.
static Constant * get(Type *Ty, double V)
get() - This returns a ConstantFP, or a vector containing a splat of a ConstantFP, for the specified value in the specified type.
Definition: Constants.cpp:652
CondCode getSetCCInverse(CondCode Operation, bool isInteger)
getSetCCInverse - Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operat...
Type * getType() const
getType - get type of this MachineConstantPoolValue.
static cl::opt< AlignMode > Align(cl::desc("Load/store alignment support"), cl::Hidden, cl::init(NoStrictAlign), cl::values(clEnumValN(StrictAlign,"aarch64-strict-align","Disallow all unaligned memory accesses"), clEnumValN(NoStrictAlign,"aarch64-no-strict-align","Allow unaligned memory accesses"), clEnumValEnd))
virtual unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const SelectionDAG &DAG, unsigned Depth=0) const
This method can be implemented by targets that want to expose additional information about sign bits ...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:123
unsigned Log2_32(uint32_t Value)
Log2_32 - This function returns the floor log base 2 of the specified value, -1 if the value is zero...
Definition: MathExtras.h:468
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:576
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
Definition: APInt.h:545
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Class for arbitrary precision integers.
Definition: APInt.h:73
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef...
bool exceedsNaturalStackAlignment(unsigned Align) const
Returns true if the given alignment exceeds the natural stack alignment.
Definition: DataLayout.h:249
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
A "pseudo-class" with methods for operating on BUILD_VECTORs.
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:342
op_iterator op_begin() const
static use_iterator use_end()
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:383
APInt bitcastToAPInt() const
Definition: APFloat.cpp:3084
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:285
SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:386
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:481
CondCode getSetCCOrOperation(CondCode Op1, CondCode Op2, bool isInteger)
getSetCCOrOperation - Return the result of a logical OR between different comparisons of identical va...
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:250
static bool FindOptimalMemOpLowering(std::vector< EVT > &MemOps, unsigned Limit, uint64_t Size, unsigned DstAlign, unsigned SrcAlign, bool IsMemset, bool ZeroMemset, bool MemcpyStrSrc, bool AllowOverlap, SelectionDAG &DAG, const TargetLowering &TLI)
Determines the optimal series of memory ops to replace the memset / memcpy.
SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op)
Return the specified value casted to the target's desired shift amount type.
SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, bool isTarget=false, unsigned char TargetFlags=0)
SDDbgValue * getFrameIndexDbgValue(MDNode *Var, MDNode *Expr, unsigned FI, uint64_t Off, DebugLoc DL, unsigned O)
FrameIndex.
iterator_range< value_op_iterator > op_values() const
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
Definition: APInt.h:498
SI Fix SGPR Live Ranges
CallLoweringInfo & setTailCall(bool Value=true)
unsigned char getTargetFlags() const
static SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, SDLoc dl, SelectionDAG &DAG)
getMemBasePlusOffset - Returns base and offset node for the
uint64_t MinAlign(uint64_t A, uint64_t B)
MinAlign - A and B are either alignments or offsets.
Definition: MathExtras.h:552
static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, SDLoc dl)
getMemsetValue - Vectorized representation of the memset value operand.
unsigned countLeadingOnes() const
Count the number of leading one bits.
Definition: APInt.cpp:722
SDValue getStackArgumentTokenFactor(SDValue Chain)
Compute a TokenFactor to force all the incoming stack arguments to be loaded from the stack...
These are IR-level optimization flags that may be propagated to SDNodes.
bool isScalarToVector(const SDNode *N)
Return true if the specified node is a ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only t...
PointerType * getType() const
Global values are always pointers.
Definition: GlobalValue.h:185
Represents a use of a SDNode.
uint64_t getConstantOperandVal(unsigned i) const
static const fltSemantics IEEEsingle
Definition: APFloat.h:132
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:321
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
static void checkForCyclesHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallPtrSetImpl< const SDNode * > &Checked, const llvm::SelectionDAG *DAG)
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.h:217
APInt LLVM_ATTRIBUTE_UNUSED_RESULT udiv(const APInt &RHS) const
Unsigned division operation.
Definition: APInt.cpp:1839
static MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset=0)
InferPointerInfo - If the specified ptr/offset is a frame index, infer a MachinePointerInfo record fr...
opStatus roundToIntegral(roundingMode)
Definition: APFloat.cpp:1849
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:401
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:518
unsigned getMaxStoresPerMemset(bool OptSize) const
Get maximum # of store operations permitted for llvm.memset.
unsigned getSizeInBits() const
getSizeInBits - Return the size of the specified value type in bits.
Definition: ValueTypes.h:233
void ReplaceAllUsesWith(SDValue From, SDValue Op)
Modify anything using 'From' to use 'To' instead.
SDValue getIndexedStore(SDValue OrigStoe, SDLoc dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
#define I(x, y, z)
Definition: MD5.cpp:54
#define N
ArrayRef< SDUse > ops() const
Fast - This calling convention attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:42
bool isOperandOf(const SDNode *N) const
isOperand - Return true if this node is an operand of N.
void Profile(FoldingSetNodeID &ID) const
Gather unique data for the node.
SDValue getLogicalNOT(SDLoc DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
static std::pair< APInt, bool > FoldValue(unsigned Opcode, const APInt &C1, const APInt &C2)
op_iterator op_end() const
An SDNode that holds an arbitrary LLVM IR Value.
const SDValue & getOffset() const
unsigned countTrailingOnes() const
Count the number of trailing one bits.
Definition: APInt.h:1403
intptr_t getRawBits() const
Definition: ValueTypes.h:323
MachineSDNode * getMachineNode(unsigned Opcode, SDLoc dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s), MachineInstr opcode, and operands.
Same for multiplication.
Definition: ISDOpcodes.h:234
bool reachesChainWithoutSideEffects(SDValue Dest, unsigned Depth=2) const
Return true if this operand (which must be a chain) reaches the specified operand without crossing an...
SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, SDLoc dl)
Constant fold a setcc to true or false.
static int isSignedOp(ISD::CondCode Opcode)
isSignedOp - For an integer comparison, return 1 if the comparison is a signed operation and 2 if the...
This class is used to represent an MSCATTER node.
DebugLoc getDebugLoc() const
int getNodeId() const
Return the unique node id.
SDValue getMemmove(SDValue Chain, SDLoc dl, SDValue Dst, SDValue Src, SDValue Size, unsigned Align, bool isVol, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
int CreateStackObject(uint64_t Size, unsigned Alignment, bool isSS, const AllocaInst *Alloca=nullptr)
Create a new statically sized stack object, returning a nonnegative identifier to represent it...
EVT getValueType() const
Return the ValueType of the referenced return value.
SDValue getCondCode(ISD::CondCode Cond)
SDNode * getNode() const
Convenience function for get().getNode().
SDValue getConstant(uint64_t Val, SDLoc DL, EVT VT, bool isTarget=false, bool isOpaque=false)
bool getConstantStringInfo(const Value *V, StringRef &Str, uint64_t Offset=0, bool TrimAtNul=true)
getConstantStringInfo - This function computes the length of a null-terminated C string pointed to by...
CONVERT_RNDSAT - This operator is used to support various conversions between various types (float...
Definition: ISDOpcodes.h:494
const DebugLoc & getDebugLoc() const
Return the source location info.
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:279
bool isFloatingPoint() const
isFloatingPoint - Return true if this is a FP, or a vector FP type.
Definition: ValueTypes.h:105
SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTs, ArrayRef< SDValue > Ops, const SDNodeFlags *Flags=nullptr)
Get the specified node if it's already available, or else return NULL.
SDValue getJumpTable(int JTI, EVT VT, bool isTarget=false, unsigned char TargetFlags=0)
This class is used to form a handle around another node that is persistent and is updated across invo...
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
This class is used to represent an MLOAD node.
bool isSimple() const
isSimple - Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:94
unsigned getAlignment() const
bool operator<(int64_t V1, const APSInt &V2)
Definition: APSInt.h:332
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
SDValue getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
uint64_t getOffset() const
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
LLVM Value Representation.
Definition: Value.h:69
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:240
SDValue getRegister(unsigned Reg, EVT VT)
APInt LLVM_ATTRIBUTE_UNUSED_RESULT sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
Definition: APInt.cpp:1023
SDValue getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label)
bool isTruncatingStore() const
Return true if the op does a truncation before store.
SDVTList getSDVTList()
Definition: SelectionDAG.h:60
void setDebugLoc(DebugLoc dl)
Set source location info.
This class is used to represent an MGATHER node.
SDValue getValueType(EVT)
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:653
void setHasDebugValue(bool b)
Set this bit.
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits...
uint64_t getSize() const
getSize - Return the size in bytes of the memory reference.
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:287
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.Val alone...
unsigned countLeadingZeros() const
The APInt version of the countLeadingZeros functions in MathExtras.h.
Definition: APInt.h:1361
Primary interface to the complete machine description for the target machine.
void add(SDDbgValue *V, const SDNode *Node, bool isParameter)
Definition: SelectionDAG.h:126
SDValue getSignExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT)
Return an operation which will sign extend the low lanes of the operand into the specified vector typ...
FoldingSetNodeIDRef Intern(BumpPtrAllocator &Allocator) const
Intern - Copy this node's data to a memory region allocated from the given allocator and return a Fol...
Definition: FoldingSet.cpp:176
SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:40
APInt LLVM_ATTRIBUTE_UNUSED_RESULT zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:996
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:365
ManagedStatic - This transparently changes the behavior of global statics to be lazily constructed on...
Definition: ManagedStatic.h:61
SDValue getConstantFP(double Val, SDLoc DL, EVT VT, bool isTarget=false)
SDValue getSetCC(SDLoc DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:726
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:188
static bool isVolatile(Instruction *Inst)
Conversion operators.
Definition: ISDOpcodes.h:380
BooleanContent
Enum that describes how the target represents true/false values.
static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC)
AddNodeIDOpcode - Add the node opcode to the NodeID data.
static APInt getNullValue(unsigned numBits)
Get the '0' value.
Definition: APInt.h:460
static ManagedStatic< EVTArray > SimpleVTArray
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:338
Unsigned from Float.
Definition: ISDOpcodes.h:891
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:666
MDNode * getVariable() const
bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
bool isOperandOf(const SDNode *N) const
Return true if this node is an operand of N.
cmpResult
IEEE-754R 5.11: Floating Point Comparison Relations.
Definition: APFloat.h:147
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:389
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
bool isTargetMemoryOpcode() const
Test if this node has a target-specific memory-referencing opcode (in the <target>ISD namespace and g...
unsigned getAlignment() const
void TransferDbgValues(SDValue From, SDValue To)
Transfer SDDbgValues.
virtual SDValue EmitTargetCodeForMemmove(SelectionDAG &DAG, SDLoc dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, unsigned Align, bool isVolatile, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const
EmitTargetCodeForMemmove - Emit target-specific code that performs a memmove.
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal on this target.
void erase(const SDNode *Node)
Invalidate all DbgValues attached to the node and remove it from the Node-to-DbgValues map...
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
bool isBigEndian() const
Definition: DataLayout.h:218
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
SDDbgValue - Holds the information from a dbg_value node through SDISel.
FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR - Perform various unary floating point operations.
Definition: ISDOpcodes.h:506
void setObjectAlignment(int ObjectIdx, unsigned Align)
setObjectAlignment - Change the alignment of the specified stack object.
const T * data() const
Definition: ArrayRef.h:131
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
getIntegerVT - Returns the EVT that represents an integer with the given number of bits...
Definition: ValueTypes.h:61
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
Definition: APInt.h:1069
static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N)
If this is an SDNode with special info, add this info to the NodeID data.
SDValue getAtomicCmpSwap(unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, unsigned Alignment, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SynchronizationScope SynchScope)
Gets a node for an atomic cmpxchg op.
static IntegerType * getInt8Ty(LLVMContext &C)
Definition: Type.cpp:237
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:662
opStatus subtract(const APFloat &, roundingMode)
Definition: APFloat.cpp:1683
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:309
static bool isSplatMask(const int *Mask, EVT VT)
unsigned getResNo() const
Convenience function for get().getResNo().
MVT getSimpleVT() const
getSimpleVT - Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:203
SDValue getIntPtrConstant(uint64_t Val, SDLoc DL, bool isTarget=false)
SDValue getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align=0, bool Vol=false, bool ReadMem=true, bool WriteMem=true, unsigned Size=0)
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
This file describes how to lower LLVM code to machine code.
unsigned char getTargetFlags() const
bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:110
unsigned getOrder() const
uint64_t getZExtValue() const
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
Definition: ISDOpcodes.h:761
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:314
uint64_t integerPart
Definition: APInt.h:33
static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs)
makeVTList - Return an instance of the SDVTList struct initialized with the specified members...
unsigned getVectorNumElements() const
getVectorNumElements - Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:225
SRCVALUE - This is a node type that holds a Value* that is used to make reference to a value in the L...
Definition: ISDOpcodes.h:611
void resize(size_type N)
Definition: SmallVector.h:376
This class is used to represent ISD::LOAD nodes.