LLVM API Documentation

SelectionDAG.cpp
Go to the documentation of this file.
00001 //===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This implements the SelectionDAG class.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "llvm/CodeGen/SelectionDAG.h"
00015 #include "SDNodeDbgValue.h"
00016 #include "llvm/ADT/SetVector.h"
00017 #include "llvm/ADT/SmallPtrSet.h"
00018 #include "llvm/ADT/SmallSet.h"
00019 #include "llvm/ADT/SmallVector.h"
00020 #include "llvm/ADT/StringExtras.h"
00021 #include "llvm/Analysis/ValueTracking.h"
00022 #include "llvm/CodeGen/MachineBasicBlock.h"
00023 #include "llvm/CodeGen/MachineConstantPool.h"
00024 #include "llvm/CodeGen/MachineFrameInfo.h"
00025 #include "llvm/CodeGen/MachineModuleInfo.h"
00026 #include "llvm/IR/CallingConv.h"
00027 #include "llvm/IR/Constants.h"
00028 #include "llvm/IR/DataLayout.h"
00029 #include "llvm/IR/DebugInfo.h"
00030 #include "llvm/IR/DerivedTypes.h"
00031 #include "llvm/IR/Function.h"
00032 #include "llvm/IR/GlobalAlias.h"
00033 #include "llvm/IR/GlobalVariable.h"
00034 #include "llvm/IR/Intrinsics.h"
00035 #include "llvm/Support/CommandLine.h"
00036 #include "llvm/Support/Debug.h"
00037 #include "llvm/Support/ErrorHandling.h"
00038 #include "llvm/Support/ManagedStatic.h"
00039 #include "llvm/Support/MathExtras.h"
00040 #include "llvm/Support/Mutex.h"
00041 #include "llvm/Support/raw_ostream.h"
00042 #include "llvm/Target/TargetInstrInfo.h"
00043 #include "llvm/Target/TargetIntrinsicInfo.h"
00044 #include "llvm/Target/TargetLowering.h"
00045 #include "llvm/Target/TargetMachine.h"
00046 #include "llvm/Target/TargetOptions.h"
00047 #include "llvm/Target/TargetRegisterInfo.h"
00048 #include "llvm/Target/TargetSelectionDAGInfo.h"
00049 #include <algorithm>
00050 #include <cmath>
00051 
00052 using namespace llvm;
00053 
00054 /// makeVTList - Return an instance of the SDVTList struct initialized with the
00055 /// specified members.
00056 static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
00057   SDVTList Res = {VTs, NumVTs};
00058   return Res;
00059 }
00060 
00061 // Default null implementations of the callbacks.
00062 void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
00063 void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
00064 
00065 //===----------------------------------------------------------------------===//
00066 //                              ConstantFPSDNode Class
00067 //===----------------------------------------------------------------------===//
00068 
00069 /// isExactlyValue - We don't rely on operator== working on double values, as
00070 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
00071 /// As such, this method can be used to do an exact bit-for-bit comparison of
00072 /// two floating point values.
00073 bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
00074   return getValueAPF().bitwiseIsEqual(V);
00075 }
00076 
00077 bool ConstantFPSDNode::isValueValidForType(EVT VT,
00078                                            const APFloat& Val) {
00079   assert(VT.isFloatingPoint() && "Can only convert between FP types");
00080 
00081   // convert modifies in place, so make a copy.
00082   APFloat Val2 = APFloat(Val);
00083   bool losesInfo;
00084   (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
00085                       APFloat::rmNearestTiesToEven,
00086                       &losesInfo);
00087   return !losesInfo;
00088 }
00089 
00090 //===----------------------------------------------------------------------===//
00091 //                              ISD Namespace
00092 //===----------------------------------------------------------------------===//
00093 
00094 /// isBuildVectorAllOnes - Return true if the specified node is a
00095 /// BUILD_VECTOR where all of the elements are ~0 or undef.
00096 bool ISD::isBuildVectorAllOnes(const SDNode *N) {
00097   // Look through a bit convert.
00098   if (N->getOpcode() == ISD::BITCAST)
00099     N = N->getOperand(0).getNode();
00100 
00101   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
00102 
00103   unsigned i = 0, e = N->getNumOperands();
00104 
00105   // Skip over all of the undef values.
00106   while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF)
00107     ++i;
00108 
00109   // Do not accept an all-undef vector.
00110   if (i == e) return false;
00111 
00112   // Do not accept build_vectors that aren't all constants or which have non-~0
00113   // elements. We have to be a bit careful here, as the type of the constant
00114   // may not be the same as the type of the vector elements due to type
00115   // legalization (the elements are promoted to a legal type for the target and
00116   // a vector of a type may be legal when the base element type is not).
00117   // We only want to check enough bits to cover the vector elements, because
00118   // we care if the resultant vector is all ones, not whether the individual
00119   // constants are.
00120   SDValue NotZero = N->getOperand(i);
00121   unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
00122   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
00123     if (CN->getAPIntValue().countTrailingOnes() < EltSize)
00124       return false;
00125   } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
00126     if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
00127       return false;
00128   } else
00129     return false;
00130 
00131   // Okay, we have at least one ~0 value, check to see if the rest match or are
00132   // undefs. Even with the above element type twiddling, this should be OK, as
00133   // the same type legalization should have applied to all the elements.
00134   for (++i; i != e; ++i)
00135     if (N->getOperand(i) != NotZero &&
00136         N->getOperand(i).getOpcode() != ISD::UNDEF)
00137       return false;
00138   return true;
00139 }
00140 
00141 
00142 /// isBuildVectorAllZeros - Return true if the specified node is a
00143 /// BUILD_VECTOR where all of the elements are 0 or undef.
00144 bool ISD::isBuildVectorAllZeros(const SDNode *N) {
00145   // Look through a bit convert.
00146   if (N->getOpcode() == ISD::BITCAST)
00147     N = N->getOperand(0).getNode();
00148 
00149   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
00150 
00151   bool IsAllUndef = true;
00152   for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) {
00153     if (N->getOperand(i).getOpcode() == ISD::UNDEF)
00154       continue;
00155     IsAllUndef = false;
00156     // Do not accept build_vectors that aren't all constants or which have non-0
00157     // elements. We have to be a bit careful here, as the type of the constant
00158     // may not be the same as the type of the vector elements due to type
00159     // legalization (the elements are promoted to a legal type for the target
00160     // and a vector of a type may be legal when the base element type is not).
00161     // We only want to check enough bits to cover the vector elements, because
00162     // we care if the resultant vector is all zeros, not whether the individual
00163     // constants are.
00164     SDValue Zero = N->getOperand(i);
00165     unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
00166     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Zero)) {
00167       if (CN->getAPIntValue().countTrailingZeros() < EltSize)
00168         return false;
00169     } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Zero)) {
00170       if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
00171         return false;
00172     } else
00173       return false;
00174   }
00175 
00176   // Do not accept an all-undef vector.
00177   if (IsAllUndef)
00178     return false;
00179   return true;
00180 }
00181 
00182 /// \brief Return true if the specified node is a BUILD_VECTOR node of
00183 /// all ConstantSDNode or undef.
00184 bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
00185   if (N->getOpcode() != ISD::BUILD_VECTOR)
00186     return false;
00187 
00188   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
00189     SDValue Op = N->getOperand(i);
00190     if (Op.getOpcode() == ISD::UNDEF)
00191       continue;
00192     if (!isa<ConstantSDNode>(Op))
00193       return false;
00194   }
00195   return true;
00196 }
00197 
00198 /// isScalarToVector - Return true if the specified node is a
00199 /// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low
00200 /// element is not an undef.
00201 bool ISD::isScalarToVector(const SDNode *N) {
00202   if (N->getOpcode() == ISD::SCALAR_TO_VECTOR)
00203     return true;
00204 
00205   if (N->getOpcode() != ISD::BUILD_VECTOR)
00206     return false;
00207   if (N->getOperand(0).getOpcode() == ISD::UNDEF)
00208     return false;
00209   unsigned NumElems = N->getNumOperands();
00210   if (NumElems == 1)
00211     return false;
00212   for (unsigned i = 1; i < NumElems; ++i) {
00213     SDValue V = N->getOperand(i);
00214     if (V.getOpcode() != ISD::UNDEF)
00215       return false;
00216   }
00217   return true;
00218 }
00219 
00220 /// allOperandsUndef - Return true if the node has at least one operand
00221 /// and all operands of the specified node are ISD::UNDEF.
00222 bool ISD::allOperandsUndef(const SDNode *N) {
00223   // Return false if the node has no operands.
00224   // This is "logically inconsistent" with the definition of "all" but
00225   // is probably the desired behavior.
00226   if (N->getNumOperands() == 0)
00227     return false;
00228 
00229   for (unsigned i = 0, e = N->getNumOperands(); i != e ; ++i)
00230     if (N->getOperand(i).getOpcode() != ISD::UNDEF)
00231       return false;
00232 
00233   return true;
00234 }
00235 
00236 ISD::NodeType ISD::getExtForLoadExtType(ISD::LoadExtType ExtType) {
00237   switch (ExtType) {
00238   case ISD::EXTLOAD:
00239     return ISD::ANY_EXTEND;
00240   case ISD::SEXTLOAD:
00241     return ISD::SIGN_EXTEND;
00242   case ISD::ZEXTLOAD:
00243     return ISD::ZERO_EXTEND;
00244   default:
00245     break;
00246   }
00247 
00248   llvm_unreachable("Invalid LoadExtType");
00249 }
00250 
00251 /// getSetCCSwappedOperands - Return the operation corresponding to (Y op X)
00252 /// when given the operation for (X op Y).
00253 ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
00254   // To perform this operation, we just need to swap the L and G bits of the
00255   // operation.
00256   unsigned OldL = (Operation >> 2) & 1;
00257   unsigned OldG = (Operation >> 1) & 1;
00258   return ISD::CondCode((Operation & ~6) |  // Keep the N, U, E bits
00259                        (OldL << 1) |       // New G bit
00260                        (OldG << 2));       // New L bit.
00261 }
00262 
00263 /// getSetCCInverse - Return the operation corresponding to !(X op Y), where
00264 /// 'op' is a valid SetCC operation.
00265 ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) {
00266   unsigned Operation = Op;
00267   if (isInteger)
00268     Operation ^= 7;   // Flip L, G, E bits, but not U.
00269   else
00270     Operation ^= 15;  // Flip all of the condition bits.
00271 
00272   if (Operation > ISD::SETTRUE2)
00273     Operation &= ~8;  // Don't let N and U bits get set.
00274 
00275   return ISD::CondCode(Operation);
00276 }
00277 
00278 
00279 /// isSignedOp - For an integer comparison, return 1 if the comparison is a
00280 /// signed operation and 2 if the result is an unsigned comparison.  Return zero
00281 /// if the operation does not depend on the sign of the input (setne and seteq).
00282 static int isSignedOp(ISD::CondCode Opcode) {
00283   switch (Opcode) {
00284   default: llvm_unreachable("Illegal integer setcc operation!");
00285   case ISD::SETEQ:
00286   case ISD::SETNE: return 0;
00287   case ISD::SETLT:
00288   case ISD::SETLE:
00289   case ISD::SETGT:
00290   case ISD::SETGE: return 1;
00291   case ISD::SETULT:
00292   case ISD::SETULE:
00293   case ISD::SETUGT:
00294   case ISD::SETUGE: return 2;
00295   }
00296 }
00297 
00298 /// getSetCCOrOperation - Return the result of a logical OR between different
00299 /// comparisons of identical values: ((X op1 Y) | (X op2 Y)).  This function
00300 /// returns SETCC_INVALID if it is not possible to represent the resultant
00301 /// comparison.
00302 ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
00303                                        bool isInteger) {
00304   if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
00305     // Cannot fold a signed integer setcc with an unsigned integer setcc.
00306     return ISD::SETCC_INVALID;
00307 
00308   unsigned Op = Op1 | Op2;  // Combine all of the condition bits.
00309 
00310   // If the N and U bits get set then the resultant comparison DOES suddenly
00311   // care about orderedness, and is true when ordered.
00312   if (Op > ISD::SETTRUE2)
00313     Op &= ~16;     // Clear the U bit if the N bit is set.
00314 
00315   // Canonicalize illegal integer setcc's.
00316   if (isInteger && Op == ISD::SETUNE)  // e.g. SETUGT | SETULT
00317     Op = ISD::SETNE;
00318 
00319   return ISD::CondCode(Op);
00320 }
00321 
00322 /// getSetCCAndOperation - Return the result of a logical AND between different
00323 /// comparisons of identical values: ((X op1 Y) & (X op2 Y)).  This
00324 /// function returns zero if it is not possible to represent the resultant
00325 /// comparison.
00326 ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
00327                                         bool isInteger) {
00328   if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
00329     // Cannot fold a signed setcc with an unsigned setcc.
00330     return ISD::SETCC_INVALID;
00331 
00332   // Combine all of the condition bits.
00333   ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
00334 
00335   // Canonicalize illegal integer setcc's.
00336   if (isInteger) {
00337     switch (Result) {
00338     default: break;
00339     case ISD::SETUO : Result = ISD::SETFALSE; break;  // SETUGT & SETULT
00340     case ISD::SETOEQ:                                 // SETEQ  & SETU[LG]E
00341     case ISD::SETUEQ: Result = ISD::SETEQ   ; break;  // SETUGE & SETULE
00342     case ISD::SETOLT: Result = ISD::SETULT  ; break;  // SETULT & SETNE
00343     case ISD::SETOGT: Result = ISD::SETUGT  ; break;  // SETUGT & SETNE
00344     }
00345   }
00346 
00347   return Result;
00348 }
00349 
00350 //===----------------------------------------------------------------------===//
00351 //                           SDNode Profile Support
00352 //===----------------------------------------------------------------------===//
00353 
00354 /// AddNodeIDOpcode - Add the node opcode to the NodeID data.
00355 ///
00356 static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC)  {
00357   ID.AddInteger(OpC);
00358 }
00359 
00360 /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
00361 /// solely with their pointer.
00362 static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
00363   ID.AddPointer(VTList.VTs);
00364 }
00365 
00366 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
00367 ///
00368 static void AddNodeIDOperands(FoldingSetNodeID &ID,
00369                               ArrayRef<SDValue> Ops) {
00370   for (auto& Op : Ops) {
00371     ID.AddPointer(Op.getNode());
00372     ID.AddInteger(Op.getResNo());
00373   }
00374 }
00375 
00376 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
00377 ///
00378 static void AddNodeIDOperands(FoldingSetNodeID &ID,
00379                               ArrayRef<SDUse> Ops) {
00380   for (auto& Op : Ops) {
00381     ID.AddPointer(Op.getNode());
00382     ID.AddInteger(Op.getResNo());
00383   }
00384 }
00385 
00386 static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, bool nuw, bool nsw,
00387                                   bool exact) {
00388   ID.AddBoolean(nuw);
00389   ID.AddBoolean(nsw);
00390   ID.AddBoolean(exact);
00391 }
00392 
00393 /// AddBinaryNodeIDCustom - Add BinarySDNodes special infos
00394 static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, unsigned Opcode,
00395                                   bool nuw, bool nsw, bool exact) {
00396   if (isBinOpWithFlags(Opcode))
00397     AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
00398 }
00399 
00400 static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
00401                           SDVTList VTList, ArrayRef<SDValue> OpList) {
00402   AddNodeIDOpcode(ID, OpC);
00403   AddNodeIDValueTypes(ID, VTList);
00404   AddNodeIDOperands(ID, OpList);
00405 }
00406 
00407 /// AddNodeIDCustom - If this is an SDNode with special info, add this info to
00408 /// the NodeID data.
00409 static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
00410   switch (N->getOpcode()) {
00411   case ISD::TargetExternalSymbol:
00412   case ISD::ExternalSymbol:
00413     llvm_unreachable("Should only be used on nodes with operands");
00414   default: break;  // Normal nodes don't need extra info.
00415   case ISD::TargetConstant:
00416   case ISD::Constant: {
00417     const ConstantSDNode *C = cast<ConstantSDNode>(N);
00418     ID.AddPointer(C->getConstantIntValue());
00419     ID.AddBoolean(C->isOpaque());
00420     break;
00421   }
00422   case ISD::TargetConstantFP:
00423   case ISD::ConstantFP: {
00424     ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
00425     break;
00426   }
00427   case ISD::TargetGlobalAddress:
00428   case ISD::GlobalAddress:
00429   case ISD::TargetGlobalTLSAddress:
00430   case ISD::GlobalTLSAddress: {
00431     const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
00432     ID.AddPointer(GA->getGlobal());
00433     ID.AddInteger(GA->getOffset());
00434     ID.AddInteger(GA->getTargetFlags());
00435     ID.AddInteger(GA->getAddressSpace());
00436     break;
00437   }
00438   case ISD::BasicBlock:
00439     ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
00440     break;
00441   case ISD::Register:
00442     ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
00443     break;
00444   case ISD::RegisterMask:
00445     ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
00446     break;
00447   case ISD::SRCVALUE:
00448     ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
00449     break;
00450   case ISD::FrameIndex:
00451   case ISD::TargetFrameIndex:
00452     ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
00453     break;
00454   case ISD::JumpTable:
00455   case ISD::TargetJumpTable:
00456     ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
00457     ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
00458     break;
00459   case ISD::ConstantPool:
00460   case ISD::TargetConstantPool: {
00461     const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
00462     ID.AddInteger(CP->getAlignment());
00463     ID.AddInteger(CP->getOffset());
00464     if (CP->isMachineConstantPoolEntry())
00465       CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
00466     else
00467       ID.AddPointer(CP->getConstVal());
00468     ID.AddInteger(CP->getTargetFlags());
00469     break;
00470   }
00471   case ISD::TargetIndex: {
00472     const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
00473     ID.AddInteger(TI->getIndex());
00474     ID.AddInteger(TI->getOffset());
00475     ID.AddInteger(TI->getTargetFlags());
00476     break;
00477   }
00478   case ISD::LOAD: {
00479     const LoadSDNode *LD = cast<LoadSDNode>(N);
00480     ID.AddInteger(LD->getMemoryVT().getRawBits());
00481     ID.AddInteger(LD->getRawSubclassData());
00482     ID.AddInteger(LD->getPointerInfo().getAddrSpace());
00483     break;
00484   }
00485   case ISD::STORE: {
00486     const StoreSDNode *ST = cast<StoreSDNode>(N);
00487     ID.AddInteger(ST->getMemoryVT().getRawBits());
00488     ID.AddInteger(ST->getRawSubclassData());
00489     ID.AddInteger(ST->getPointerInfo().getAddrSpace());
00490     break;
00491   }
00492   case ISD::SDIV:
00493   case ISD::UDIV:
00494   case ISD::SRA:
00495   case ISD::SRL:
00496   case ISD::MUL:
00497   case ISD::ADD:
00498   case ISD::SUB:
00499   case ISD::SHL: {
00500     const BinaryWithFlagsSDNode *BinNode = cast<BinaryWithFlagsSDNode>(N);
00501     AddBinaryNodeIDCustom(ID, N->getOpcode(), BinNode->hasNoUnsignedWrap(),
00502                           BinNode->hasNoSignedWrap(), BinNode->isExact());
00503     break;
00504   }
00505   case ISD::ATOMIC_CMP_SWAP:
00506   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
00507   case ISD::ATOMIC_SWAP:
00508   case ISD::ATOMIC_LOAD_ADD:
00509   case ISD::ATOMIC_LOAD_SUB:
00510   case ISD::ATOMIC_LOAD_AND:
00511   case ISD::ATOMIC_LOAD_OR:
00512   case ISD::ATOMIC_LOAD_XOR:
00513   case ISD::ATOMIC_LOAD_NAND:
00514   case ISD::ATOMIC_LOAD_MIN:
00515   case ISD::ATOMIC_LOAD_MAX:
00516   case ISD::ATOMIC_LOAD_UMIN:
00517   case ISD::ATOMIC_LOAD_UMAX:
00518   case ISD::ATOMIC_LOAD:
00519   case ISD::ATOMIC_STORE: {
00520     const AtomicSDNode *AT = cast<AtomicSDNode>(N);
00521     ID.AddInteger(AT->getMemoryVT().getRawBits());
00522     ID.AddInteger(AT->getRawSubclassData());
00523     ID.AddInteger(AT->getPointerInfo().getAddrSpace());
00524     break;
00525   }
00526   case ISD::PREFETCH: {
00527     const MemSDNode *PF = cast<MemSDNode>(N);
00528     ID.AddInteger(PF->getPointerInfo().getAddrSpace());
00529     break;
00530   }
00531   case ISD::VECTOR_SHUFFLE: {
00532     const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
00533     for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
00534          i != e; ++i)
00535       ID.AddInteger(SVN->getMaskElt(i));
00536     break;
00537   }
00538   case ISD::TargetBlockAddress:
00539   case ISD::BlockAddress: {
00540     const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
00541     ID.AddPointer(BA->getBlockAddress());
00542     ID.AddInteger(BA->getOffset());
00543     ID.AddInteger(BA->getTargetFlags());
00544     break;
00545   }
00546   } // end switch (N->getOpcode())
00547 
00548   // Target specific memory nodes could also have address spaces to check.
00549   if (N->isTargetMemoryOpcode())
00550     ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
00551 }
00552 
00553 /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
00554 /// data.
00555 static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
00556   AddNodeIDOpcode(ID, N->getOpcode());
00557   // Add the return value info.
00558   AddNodeIDValueTypes(ID, N->getVTList());
00559   // Add the operand info.
00560   AddNodeIDOperands(ID, N->ops());
00561 
00562   // Handle SDNode leafs with special info.
00563   AddNodeIDCustom(ID, N);
00564 }
00565 
00566 /// encodeMemSDNodeFlags - Generic routine for computing a value for use in
00567 /// the CSE map that carries volatility, temporalness, indexing mode, and
00568 /// extension/truncation information.
00569 ///
00570 static inline unsigned
00571 encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile,
00572                      bool isNonTemporal, bool isInvariant) {
00573   assert((ConvType & 3) == ConvType &&
00574          "ConvType may not require more than 2 bits!");
00575   assert((AM & 7) == AM &&
00576          "AM may not require more than 3 bits!");
00577   return ConvType |
00578          (AM << 2) |
00579          (isVolatile << 5) |
00580          (isNonTemporal << 6) |
00581          (isInvariant << 7);
00582 }
00583 
00584 //===----------------------------------------------------------------------===//
00585 //                              SelectionDAG Class
00586 //===----------------------------------------------------------------------===//
00587 
00588 /// doNotCSE - Return true if CSE should not be performed for this node.
00589 static bool doNotCSE(SDNode *N) {
00590   if (N->getValueType(0) == MVT::Glue)
00591     return true; // Never CSE anything that produces a flag.
00592 
00593   switch (N->getOpcode()) {
00594   default: break;
00595   case ISD::HANDLENODE:
00596   case ISD::EH_LABEL:
00597     return true;   // Never CSE these nodes.
00598   }
00599 
00600   // Check that remaining values produced are not flags.
00601   for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
00602     if (N->getValueType(i) == MVT::Glue)
00603       return true; // Never CSE anything that produces a flag.
00604 
00605   return false;
00606 }
00607 
00608 /// RemoveDeadNodes - This method deletes all unreachable nodes in the
00609 /// SelectionDAG.
00610 void SelectionDAG::RemoveDeadNodes() {
00611   // Create a dummy node (which is not added to allnodes), that adds a reference
00612   // to the root node, preventing it from being deleted.
00613   HandleSDNode Dummy(getRoot());
00614 
00615   SmallVector<SDNode*, 128> DeadNodes;
00616 
00617   // Add all obviously-dead nodes to the DeadNodes worklist.
00618   for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I)
00619     if (I->use_empty())
00620       DeadNodes.push_back(I);
00621 
00622   RemoveDeadNodes(DeadNodes);
00623 
00624   // If the root changed (e.g. it was a dead load, update the root).
00625   setRoot(Dummy.getValue());
00626 }
00627 
00628 /// RemoveDeadNodes - This method deletes the unreachable nodes in the
00629 /// given list, and any nodes that become unreachable as a result.
00630 void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
00631 
00632   // Process the worklist, deleting the nodes and adding their uses to the
00633   // worklist.
00634   while (!DeadNodes.empty()) {
00635     SDNode *N = DeadNodes.pop_back_val();
00636 
00637     for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00638       DUL->NodeDeleted(N, nullptr);
00639 
00640     // Take the node out of the appropriate CSE map.
00641     RemoveNodeFromCSEMaps(N);
00642 
00643     // Next, brutally remove the operand list.  This is safe to do, as there are
00644     // no cycles in the graph.
00645     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
00646       SDUse &Use = *I++;
00647       SDNode *Operand = Use.getNode();
00648       Use.set(SDValue());
00649 
00650       // Now that we removed this operand, see if there are no uses of it left.
00651       if (Operand->use_empty())
00652         DeadNodes.push_back(Operand);
00653     }
00654 
00655     DeallocateNode(N);
00656   }
00657 }
00658 
00659 void SelectionDAG::RemoveDeadNode(SDNode *N){
00660   SmallVector<SDNode*, 16> DeadNodes(1, N);
00661 
00662   // Create a dummy node that adds a reference to the root node, preventing
00663   // it from being deleted.  (This matters if the root is an operand of the
00664   // dead node.)
00665   HandleSDNode Dummy(getRoot());
00666 
00667   RemoveDeadNodes(DeadNodes);
00668 }
00669 
00670 void SelectionDAG::DeleteNode(SDNode *N) {
00671   // First take this out of the appropriate CSE map.
00672   RemoveNodeFromCSEMaps(N);
00673 
00674   // Finally, remove uses due to operands of this node, remove from the
00675   // AllNodes list, and delete the node.
00676   DeleteNodeNotInCSEMaps(N);
00677 }
00678 
00679 void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
00680   assert(N != AllNodes.begin() && "Cannot delete the entry node!");
00681   assert(N->use_empty() && "Cannot delete a node that is not dead!");
00682 
00683   // Drop all of the operands and decrement used node's use counts.
00684   N->DropOperands();
00685 
00686   DeallocateNode(N);
00687 }
00688 
00689 void SelectionDAG::DeallocateNode(SDNode *N) {
00690   if (N->OperandsNeedDelete)
00691     delete[] N->OperandList;
00692 
00693   // Set the opcode to DELETED_NODE to help catch bugs when node
00694   // memory is reallocated.
00695   N->NodeType = ISD::DELETED_NODE;
00696 
00697   NodeAllocator.Deallocate(AllNodes.remove(N));
00698 
00699   // If any of the SDDbgValue nodes refer to this SDNode, invalidate them.
00700   ArrayRef<SDDbgValue*> DbgVals = DbgInfo->getSDDbgValues(N);
00701   for (unsigned i = 0, e = DbgVals.size(); i != e; ++i)
00702     DbgVals[i]->setIsInvalidated();
00703 }
00704 
00705 #ifndef NDEBUG
00706 /// VerifySDNode - Sanity check the given SDNode.  Aborts if it is invalid.
00707 static void VerifySDNode(SDNode *N) {
00708   switch (N->getOpcode()) {
00709   default:
00710     break;
00711   case ISD::BUILD_PAIR: {
00712     EVT VT = N->getValueType(0);
00713     assert(N->getNumValues() == 1 && "Too many results!");
00714     assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
00715            "Wrong return type!");
00716     assert(N->getNumOperands() == 2 && "Wrong number of operands!");
00717     assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
00718            "Mismatched operand types!");
00719     assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
00720            "Wrong operand type!");
00721     assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
00722            "Wrong return type size");
00723     break;
00724   }
00725   case ISD::BUILD_VECTOR: {
00726     assert(N->getNumValues() == 1 && "Too many results!");
00727     assert(N->getValueType(0).isVector() && "Wrong return type!");
00728     assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
00729            "Wrong number of operands!");
00730     EVT EltVT = N->getValueType(0).getVectorElementType();
00731     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
00732       assert((I->getValueType() == EltVT ||
00733              (EltVT.isInteger() && I->getValueType().isInteger() &&
00734               EltVT.bitsLE(I->getValueType()))) &&
00735             "Wrong operand type!");
00736       assert(I->getValueType() == N->getOperand(0).getValueType() &&
00737              "Operands must all have the same type");
00738     }
00739     break;
00740   }
00741   }
00742 }
00743 #endif // NDEBUG
00744 
00745 /// \brief Insert a newly allocated node into the DAG.
00746 ///
00747 /// Handles insertion into the all nodes list and CSE map, as well as
00748 /// verification and other common operations when a new node is allocated.
00749 void SelectionDAG::InsertNode(SDNode *N) {
00750   AllNodes.push_back(N);
00751 #ifndef NDEBUG
00752   VerifySDNode(N);
00753 #endif
00754 }
00755 
00756 /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
00757 /// correspond to it.  This is useful when we're about to delete or repurpose
00758 /// the node.  We don't want future request for structurally identical nodes
00759 /// to return N anymore.
00760 bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
00761   bool Erased = false;
00762   switch (N->getOpcode()) {
00763   case ISD::HANDLENODE: return false;  // noop.
00764   case ISD::CONDCODE:
00765     assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
00766            "Cond code doesn't exist!");
00767     Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
00768     CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
00769     break;
00770   case ISD::ExternalSymbol:
00771     Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
00772     break;
00773   case ISD::TargetExternalSymbol: {
00774     ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
00775     Erased = TargetExternalSymbols.erase(
00776                std::pair<std::string,unsigned char>(ESN->getSymbol(),
00777                                                     ESN->getTargetFlags()));
00778     break;
00779   }
00780   case ISD::VALUETYPE: {
00781     EVT VT = cast<VTSDNode>(N)->getVT();
00782     if (VT.isExtended()) {
00783       Erased = ExtendedValueTypeNodes.erase(VT);
00784     } else {
00785       Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
00786       ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
00787     }
00788     break;
00789   }
00790   default:
00791     // Remove it from the CSE Map.
00792     assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
00793     assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
00794     Erased = CSEMap.RemoveNode(N);
00795     break;
00796   }
00797 #ifndef NDEBUG
00798   // Verify that the node was actually in one of the CSE maps, unless it has a
00799   // flag result (which cannot be CSE'd) or is one of the special cases that are
00800   // not subject to CSE.
00801   if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
00802       !N->isMachineOpcode() && !doNotCSE(N)) {
00803     N->dump(this);
00804     dbgs() << "\n";
00805     llvm_unreachable("Node is not in map!");
00806   }
00807 #endif
00808   return Erased;
00809 }
00810 
00811 /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
00812 /// maps and modified in place. Add it back to the CSE maps, unless an identical
00813 /// node already exists, in which case transfer all its users to the existing
00814 /// node. This transfer can potentially trigger recursive merging.
00815 ///
00816 void
00817 SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
00818   // For node types that aren't CSE'd, just act as if no identical node
00819   // already exists.
00820   if (!doNotCSE(N)) {
00821     SDNode *Existing = CSEMap.GetOrInsertNode(N);
00822     if (Existing != N) {
00823       // If there was already an existing matching node, use ReplaceAllUsesWith
00824       // to replace the dead one with the existing one.  This can cause
00825       // recursive merging of other unrelated nodes down the line.
00826       ReplaceAllUsesWith(N, Existing);
00827 
00828       // N is now dead. Inform the listeners and delete it.
00829       for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00830         DUL->NodeDeleted(N, Existing);
00831       DeleteNodeNotInCSEMaps(N);
00832       return;
00833     }
00834   }
00835 
00836   // If the node doesn't already exist, we updated it.  Inform listeners.
00837   for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00838     DUL->NodeUpdated(N);
00839 }
00840 
00841 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00842 /// were replaced with those specified.  If this node is never memoized,
00843 /// return null, otherwise return a pointer to the slot it would take.  If a
00844 /// node already exists with these operands, the slot will be non-null.
00845 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
00846                                            void *&InsertPos) {
00847   if (doNotCSE(N))
00848     return nullptr;
00849 
00850   SDValue Ops[] = { Op };
00851   FoldingSetNodeID ID;
00852   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00853   AddNodeIDCustom(ID, N);
00854   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00855   return Node;
00856 }
00857 
00858 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00859 /// were replaced with those specified.  If this node is never memoized,
00860 /// return null, otherwise return a pointer to the slot it would take.  If a
00861 /// node already exists with these operands, the slot will be non-null.
00862 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
00863                                            SDValue Op1, SDValue Op2,
00864                                            void *&InsertPos) {
00865   if (doNotCSE(N))
00866     return nullptr;
00867 
00868   SDValue Ops[] = { Op1, Op2 };
00869   FoldingSetNodeID ID;
00870   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00871   AddNodeIDCustom(ID, N);
00872   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00873   return Node;
00874 }
00875 
00876 
00877 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00878 /// were replaced with those specified.  If this node is never memoized,
00879 /// return null, otherwise return a pointer to the slot it would take.  If a
00880 /// node already exists with these operands, the slot will be non-null.
00881 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
00882                                            void *&InsertPos) {
00883   if (doNotCSE(N))
00884     return nullptr;
00885 
00886   FoldingSetNodeID ID;
00887   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00888   AddNodeIDCustom(ID, N);
00889   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00890   return Node;
00891 }
00892 
00893 /// getEVTAlignment - Compute the default alignment value for the
00894 /// given type.
00895 ///
00896 unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
00897   Type *Ty = VT == MVT::iPTR ?
00898                    PointerType::get(Type::getInt8Ty(*getContext()), 0) :
00899                    VT.getTypeForEVT(*getContext());
00900 
00901   return TM.getTargetLowering()->getDataLayout()->getABITypeAlignment(Ty);
00902 }
00903 
00904 // EntryNode could meaningfully have debug info if we can find it...
00905 SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
00906   : TM(tm), TSI(*tm.getSelectionDAGInfo()), TLI(nullptr), OptLevel(OL),
00907     EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
00908     Root(getEntryNode()), NewNodesMustHaveLegalTypes(false),
00909     UpdateListeners(nullptr) {
00910   AllNodes.push_back(&EntryNode);
00911   DbgInfo = new SDDbgInfo();
00912 }
00913 
00914 void SelectionDAG::init(MachineFunction &mf, const TargetLowering *tli) {
00915   MF = &mf;
00916   TLI = tli;
00917   Context = &mf.getFunction()->getContext();
00918 }
00919 
00920 SelectionDAG::~SelectionDAG() {
00921   assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
00922   allnodes_clear();
00923   delete DbgInfo;
00924 }
00925 
00926 void SelectionDAG::allnodes_clear() {
00927   assert(&*AllNodes.begin() == &EntryNode);
00928   AllNodes.remove(AllNodes.begin());
00929   while (!AllNodes.empty())
00930     DeallocateNode(AllNodes.begin());
00931 }
00932 
00933 BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL,
00934                                             SDVTList VTs, SDValue N1,
00935                                             SDValue N2, bool nuw, bool nsw,
00936                                             bool exact) {
00937   if (isBinOpWithFlags(Opcode)) {
00938     BinaryWithFlagsSDNode *FN = new (NodeAllocator) BinaryWithFlagsSDNode(
00939         Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
00940     FN->setHasNoUnsignedWrap(nuw);
00941     FN->setHasNoSignedWrap(nsw);
00942     FN->setIsExact(exact);
00943 
00944     return FN;
00945   }
00946 
00947   BinarySDNode *N = new (NodeAllocator)
00948       BinarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
00949   return N;
00950 }
00951 
00952 void SelectionDAG::clear() {
00953   allnodes_clear();
00954   OperandAllocator.Reset();
00955   CSEMap.clear();
00956 
00957   ExtendedValueTypeNodes.clear();
00958   ExternalSymbols.clear();
00959   TargetExternalSymbols.clear();
00960   std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
00961             static_cast<CondCodeSDNode*>(nullptr));
00962   std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
00963             static_cast<SDNode*>(nullptr));
00964 
00965   EntryNode.UseList = nullptr;
00966   AllNodes.push_back(&EntryNode);
00967   Root = getEntryNode();
00968   DbgInfo->clear();
00969 }
00970 
00971 SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00972   return VT.bitsGT(Op.getValueType()) ?
00973     getNode(ISD::ANY_EXTEND, DL, VT, Op) :
00974     getNode(ISD::TRUNCATE, DL, VT, Op);
00975 }
00976 
00977 SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00978   return VT.bitsGT(Op.getValueType()) ?
00979     getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
00980     getNode(ISD::TRUNCATE, DL, VT, Op);
00981 }
00982 
00983 SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00984   return VT.bitsGT(Op.getValueType()) ?
00985     getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
00986     getNode(ISD::TRUNCATE, DL, VT, Op);
00987 }
00988 
00989 SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, SDLoc SL, EVT VT,
00990                                         EVT OpVT) {
00991   if (VT.bitsLE(Op.getValueType()))
00992     return getNode(ISD::TRUNCATE, SL, VT, Op);
00993 
00994   TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
00995   return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
00996 }
00997 
00998 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, SDLoc DL, EVT VT) {
00999   assert(!VT.isVector() &&
01000          "getZeroExtendInReg should use the vector element type instead of "
01001          "the vector type!");
01002   if (Op.getValueType() == VT) return Op;
01003   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01004   APInt Imm = APInt::getLowBitsSet(BitWidth,
01005                                    VT.getSizeInBits());
01006   return getNode(ISD::AND, DL, Op.getValueType(), Op,
01007                  getConstant(Imm, Op.getValueType()));
01008 }
01009 
01010 SDValue SelectionDAG::getAnyExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01011   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01012   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01013          "The sizes of the input and result must match in order to perform the "
01014          "extend in-register.");
01015   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01016          "The destination vector type must have fewer lanes than the input.");
01017   return getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Op);
01018 }
01019 
01020 SDValue SelectionDAG::getSignExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01021   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01022   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01023          "The sizes of the input and result must match in order to perform the "
01024          "extend in-register.");
01025   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01026          "The destination vector type must have fewer lanes than the input.");
01027   return getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Op);
01028 }
01029 
01030 SDValue SelectionDAG::getZeroExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01031   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01032   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01033          "The sizes of the input and result must match in order to perform the "
01034          "extend in-register.");
01035   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01036          "The destination vector type must have fewer lanes than the input.");
01037   return getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Op);
01038 }
01039 
01040 /// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
01041 ///
01042 SDValue SelectionDAG::getNOT(SDLoc DL, SDValue Val, EVT VT) {
01043   EVT EltVT = VT.getScalarType();
01044   SDValue NegOne =
01045     getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT);
01046   return getNode(ISD::XOR, DL, VT, Val, NegOne);
01047 }
01048 
01049 SDValue SelectionDAG::getLogicalNOT(SDLoc DL, SDValue Val, EVT VT) {
01050   EVT EltVT = VT.getScalarType();
01051   SDValue TrueValue;
01052   switch (TLI->getBooleanContents(VT)) {
01053     case TargetLowering::ZeroOrOneBooleanContent:
01054     case TargetLowering::UndefinedBooleanContent:
01055       TrueValue = getConstant(1, VT);
01056       break;
01057     case TargetLowering::ZeroOrNegativeOneBooleanContent:
01058       TrueValue = getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()),
01059                               VT);
01060       break;
01061   }
01062   return getNode(ISD::XOR, DL, VT, Val, TrueValue);
01063 }
01064 
01065 SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT, bool isO) {
01066   EVT EltVT = VT.getScalarType();
01067   assert((EltVT.getSizeInBits() >= 64 ||
01068          (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
01069          "getConstant with a uint64_t value that doesn't fit in the type!");
01070   return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT, isO);
01071 }
01072 
01073 SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT, bool isO)
01074 {
01075   return getConstant(*ConstantInt::get(*Context, Val), VT, isT, isO);
01076 }
01077 
01078 SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT,
01079                                   bool isO) {
01080   assert(VT.isInteger() && "Cannot create FP integer constant!");
01081 
01082   EVT EltVT = VT.getScalarType();
01083   const ConstantInt *Elt = &Val;
01084 
01085   const TargetLowering *TLI = TM.getTargetLowering();
01086 
01087   // In some cases the vector type is legal but the element type is illegal and
01088   // needs to be promoted, for example v8i8 on ARM.  In this case, promote the
01089   // inserted value (the type does not need to match the vector element type).
01090   // Any extra bits introduced will be truncated away.
01091   if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
01092       TargetLowering::TypePromoteInteger) {
01093    EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
01094    APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits());
01095    Elt = ConstantInt::get(*getContext(), NewVal);
01096   }
01097   // In other cases the element type is illegal and needs to be expanded, for
01098   // example v2i64 on MIPS32. In this case, find the nearest legal type, split
01099   // the value into n parts and use a vector type with n-times the elements.
01100   // Then bitcast to the type requested.
01101   // Legalizing constants too early makes the DAGCombiner's job harder so we
01102   // only legalize if the DAG tells us we must produce legal types.
01103   else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
01104            TLI->getTypeAction(*getContext(), EltVT) ==
01105            TargetLowering::TypeExpandInteger) {
01106     APInt NewVal = Elt->getValue();
01107     EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
01108     unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
01109     unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
01110     EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
01111 
01112     // Check the temporary vector is the correct size. If this fails then
01113     // getTypeToTransformTo() probably returned a type whose size (in bits)
01114     // isn't a power-of-2 factor of the requested type size.
01115     assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
01116 
01117     SmallVector<SDValue, 2> EltParts;
01118     for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
01119       EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
01120                                            .trunc(ViaEltSizeInBits),
01121                                      ViaEltVT, isT, isO));
01122     }
01123 
01124     // EltParts is currently in little endian order. If we actually want
01125     // big-endian order then reverse it now.
01126     if (TLI->isBigEndian())
01127       std::reverse(EltParts.begin(), EltParts.end());
01128 
01129     // The elements must be reversed when the element order is different
01130     // to the endianness of the elements (because the BITCAST is itself a
01131     // vector shuffle in this situation). However, we do not need any code to
01132     // perform this reversal because getConstant() is producing a vector
01133     // splat.
01134     // This situation occurs in MIPS MSA.
01135 
01136     SmallVector<SDValue, 8> Ops;
01137     for (unsigned i = 0; i < VT.getVectorNumElements(); ++i)
01138       Ops.insert(Ops.end(), EltParts.begin(), EltParts.end());
01139 
01140     SDValue Result = getNode(ISD::BITCAST, SDLoc(), VT,
01141                              getNode(ISD::BUILD_VECTOR, SDLoc(), ViaVecVT,
01142                                      Ops));
01143     return Result;
01144   }
01145 
01146   assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
01147          "APInt size does not match type size!");
01148   unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
01149   FoldingSetNodeID ID;
01150   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
01151   ID.AddPointer(Elt);
01152   ID.AddBoolean(isO);
01153   void *IP = nullptr;
01154   SDNode *N = nullptr;
01155   if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
01156     if (!VT.isVector())
01157       return SDValue(N, 0);
01158 
01159   if (!N) {
01160     N = new (NodeAllocator) ConstantSDNode(isT, isO, Elt, EltVT);
01161     CSEMap.InsertNode(N, IP);
01162     InsertNode(N);
01163   }
01164 
01165   SDValue Result(N, 0);
01166   if (VT.isVector()) {
01167     SmallVector<SDValue, 8> Ops;
01168     Ops.assign(VT.getVectorNumElements(), Result);
01169     Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
01170   }
01171   return Result;
01172 }
01173 
01174 SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) {
01175   return getConstant(Val, TM.getTargetLowering()->getPointerTy(), isTarget);
01176 }
01177 
01178 
01179 SDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) {
01180   return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget);
01181 }
01182 
01183 SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){
01184   assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
01185 
01186   EVT EltVT = VT.getScalarType();
01187 
01188   // Do the map lookup using the actual bit pattern for the floating point
01189   // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
01190   // we don't have issues with SNANs.
01191   unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
01192   FoldingSetNodeID ID;
01193   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
01194   ID.AddPointer(&V);
01195   void *IP = nullptr;
01196   SDNode *N = nullptr;
01197   if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
01198     if (!VT.isVector())
01199       return SDValue(N, 0);
01200 
01201   if (!N) {
01202     N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT);
01203     CSEMap.InsertNode(N, IP);
01204     InsertNode(N);
01205   }
01206 
01207   SDValue Result(N, 0);
01208   if (VT.isVector()) {
01209     SmallVector<SDValue, 8> Ops;
01210     Ops.assign(VT.getVectorNumElements(), Result);
01211     // FIXME SDLoc info might be appropriate here
01212     Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
01213   }
01214   return Result;
01215 }
01216 
01217 SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) {
01218   EVT EltVT = VT.getScalarType();
01219   if (EltVT==MVT::f32)
01220     return getConstantFP(APFloat((float)Val), VT, isTarget);
01221   else if (EltVT==MVT::f64)
01222     return getConstantFP(APFloat(Val), VT, isTarget);
01223   else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 ||
01224            EltVT==MVT::f16) {
01225     bool ignored;
01226     APFloat apf = APFloat(Val);
01227     apf.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
01228                 &ignored);
01229     return getConstantFP(apf, VT, isTarget);
01230   } else
01231     llvm_unreachable("Unsupported type in getConstantFP");
01232 }
01233 
01234 SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, SDLoc DL,
01235                                        EVT VT, int64_t Offset,
01236                                        bool isTargetGA,
01237                                        unsigned char TargetFlags) {
01238   assert((TargetFlags == 0 || isTargetGA) &&
01239          "Cannot set target flags on target-independent globals");
01240   const TargetLowering *TLI = TM.getTargetLowering();
01241 
01242   // Truncate (with sign-extension) the offset value to the pointer size.
01243   unsigned BitWidth = TLI->getPointerTypeSizeInBits(GV->getType());
01244   if (BitWidth < 64)
01245     Offset = SignExtend64(Offset, BitWidth);
01246 
01247   unsigned Opc;
01248   if (GV->isThreadLocal())
01249     Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
01250   else
01251     Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
01252 
01253   FoldingSetNodeID ID;
01254   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01255   ID.AddPointer(GV);
01256   ID.AddInteger(Offset);
01257   ID.AddInteger(TargetFlags);
01258   ID.AddInteger(GV->getType()->getAddressSpace());
01259   void *IP = nullptr;
01260   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01261     return SDValue(E, 0);
01262 
01263   SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL.getIROrder(),
01264                                                       DL.getDebugLoc(), GV, VT,
01265                                                       Offset, TargetFlags);
01266   CSEMap.InsertNode(N, IP);
01267     InsertNode(N);
01268   return SDValue(N, 0);
01269 }
01270 
01271 SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
01272   unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
01273   FoldingSetNodeID ID;
01274   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01275   ID.AddInteger(FI);
01276   void *IP = nullptr;
01277   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01278     return SDValue(E, 0);
01279 
01280   SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget);
01281   CSEMap.InsertNode(N, IP);
01282   InsertNode(N);
01283   return SDValue(N, 0);
01284 }
01285 
01286 SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
01287                                    unsigned char TargetFlags) {
01288   assert((TargetFlags == 0 || isTarget) &&
01289          "Cannot set target flags on target-independent jump tables");
01290   unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
01291   FoldingSetNodeID ID;
01292   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01293   ID.AddInteger(JTI);
01294   ID.AddInteger(TargetFlags);
01295   void *IP = nullptr;
01296   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01297     return SDValue(E, 0);
01298 
01299   SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget,
01300                                                   TargetFlags);
01301   CSEMap.InsertNode(N, IP);
01302   InsertNode(N);
01303   return SDValue(N, 0);
01304 }
01305 
01306 SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
01307                                       unsigned Alignment, int Offset,
01308                                       bool isTarget,
01309                                       unsigned char TargetFlags) {
01310   assert((TargetFlags == 0 || isTarget) &&
01311          "Cannot set target flags on target-independent globals");
01312   if (Alignment == 0)
01313     Alignment =
01314     TM.getTargetLowering()->getDataLayout()->getPrefTypeAlignment(C->getType());
01315   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
01316   FoldingSetNodeID ID;
01317   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01318   ID.AddInteger(Alignment);
01319   ID.AddInteger(Offset);
01320   ID.AddPointer(C);
01321   ID.AddInteger(TargetFlags);
01322   void *IP = nullptr;
01323   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01324     return SDValue(E, 0);
01325 
01326   SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
01327                                                      Alignment, TargetFlags);
01328   CSEMap.InsertNode(N, IP);
01329   InsertNode(N);
01330   return SDValue(N, 0);
01331 }
01332 
01333 
01334 SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
01335                                       unsigned Alignment, int Offset,
01336                                       bool isTarget,
01337                                       unsigned char TargetFlags) {
01338   assert((TargetFlags == 0 || isTarget) &&
01339          "Cannot set target flags on target-independent globals");
01340   if (Alignment == 0)
01341     Alignment =
01342     TM.getTargetLowering()->getDataLayout()->getPrefTypeAlignment(C->getType());
01343   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
01344   FoldingSetNodeID ID;
01345   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01346   ID.AddInteger(Alignment);
01347   ID.AddInteger(Offset);
01348   C->addSelectionDAGCSEId(ID);
01349   ID.AddInteger(TargetFlags);
01350   void *IP = nullptr;
01351   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01352     return SDValue(E, 0);
01353 
01354   SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
01355                                                      Alignment, TargetFlags);
01356   CSEMap.InsertNode(N, IP);
01357   InsertNode(N);
01358   return SDValue(N, 0);
01359 }
01360 
01361 SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
01362                                      unsigned char TargetFlags) {
01363   FoldingSetNodeID ID;
01364   AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None);
01365   ID.AddInteger(Index);
01366   ID.AddInteger(Offset);
01367   ID.AddInteger(TargetFlags);
01368   void *IP = nullptr;
01369   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01370     return SDValue(E, 0);
01371 
01372   SDNode *N = new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset,
01373                                                     TargetFlags);
01374   CSEMap.InsertNode(N, IP);
01375   InsertNode(N);
01376   return SDValue(N, 0);
01377 }
01378 
01379 SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
01380   FoldingSetNodeID ID;
01381   AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
01382   ID.AddPointer(MBB);
01383   void *IP = nullptr;
01384   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01385     return SDValue(E, 0);
01386 
01387   SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB);
01388   CSEMap.InsertNode(N, IP);
01389   InsertNode(N);
01390   return SDValue(N, 0);
01391 }
01392 
01393 SDValue SelectionDAG::getValueType(EVT VT) {
01394   if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
01395       ValueTypeNodes.size())
01396     ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
01397 
01398   SDNode *&N = VT.isExtended() ?
01399     ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
01400 
01401   if (N) return SDValue(N, 0);
01402   N = new (NodeAllocator) VTSDNode(VT);
01403   InsertNode(N);
01404   return SDValue(N, 0);
01405 }
01406 
01407 SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
01408   SDNode *&N = ExternalSymbols[Sym];
01409   if (N) return SDValue(N, 0);
01410   N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT);
01411   InsertNode(N);
01412   return SDValue(N, 0);
01413 }
01414 
01415 SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
01416                                               unsigned char TargetFlags) {
01417   SDNode *&N =
01418     TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym,
01419                                                                TargetFlags)];
01420   if (N) return SDValue(N, 0);
01421   N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT);
01422   InsertNode(N);
01423   return SDValue(N, 0);
01424 }
01425 
01426 SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
01427   if ((unsigned)Cond >= CondCodeNodes.size())
01428     CondCodeNodes.resize(Cond+1);
01429 
01430   if (!CondCodeNodes[Cond]) {
01431     CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond);
01432     CondCodeNodes[Cond] = N;
01433     InsertNode(N);
01434   }
01435 
01436   return SDValue(CondCodeNodes[Cond], 0);
01437 }
01438 
01439 // commuteShuffle - swaps the values of N1 and N2, and swaps all indices in
01440 // the shuffle mask M that point at N1 to point at N2, and indices that point
01441 // N2 to point at N1.
01442 static void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) {
01443   std::swap(N1, N2);
01444   int NElts = M.size();
01445   for (int i = 0; i != NElts; ++i) {
01446     if (M[i] >= NElts)
01447       M[i] -= NElts;
01448     else if (M[i] >= 0)
01449       M[i] += NElts;
01450   }
01451 }
01452 
01453 SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1,
01454                                        SDValue N2, const int *Mask) {
01455   assert(VT == N1.getValueType() && VT == N2.getValueType() &&
01456          "Invalid VECTOR_SHUFFLE");
01457 
01458   // Canonicalize shuffle undef, undef -> undef
01459   if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF)
01460     return getUNDEF(VT);
01461 
01462   // Validate that all indices in Mask are within the range of the elements
01463   // input to the shuffle.
01464   unsigned NElts = VT.getVectorNumElements();
01465   SmallVector<int, 8> MaskVec;
01466   for (unsigned i = 0; i != NElts; ++i) {
01467     assert(Mask[i] < (int)(NElts * 2) && "Index out of range");
01468     MaskVec.push_back(Mask[i]);
01469   }
01470 
01471   // Canonicalize shuffle v, v -> v, undef
01472   if (N1 == N2) {
01473     N2 = getUNDEF(VT);
01474     for (unsigned i = 0; i != NElts; ++i)
01475       if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts;
01476   }
01477 
01478   // Canonicalize shuffle undef, v -> v, undef.  Commute the shuffle mask.
01479   if (N1.getOpcode() == ISD::UNDEF)
01480     commuteShuffle(N1, N2, MaskVec);
01481 
01482   // Canonicalize all index into lhs, -> shuffle lhs, undef
01483   // Canonicalize all index into rhs, -> shuffle rhs, undef
01484   bool AllLHS = true, AllRHS = true;
01485   bool N2Undef = N2.getOpcode() == ISD::UNDEF;
01486   for (unsigned i = 0; i != NElts; ++i) {
01487     if (MaskVec[i] >= (int)NElts) {
01488       if (N2Undef)
01489         MaskVec[i] = -1;
01490       else
01491         AllLHS = false;
01492     } else if (MaskVec[i] >= 0) {
01493       AllRHS = false;
01494     }
01495   }
01496   if (AllLHS && AllRHS)
01497     return getUNDEF(VT);
01498   if (AllLHS && !N2Undef)
01499     N2 = getUNDEF(VT);
01500   if (AllRHS) {
01501     N1 = getUNDEF(VT);
01502     commuteShuffle(N1, N2, MaskVec);
01503   }
01504   // Reset our undef status after accounting for the mask.
01505   N2Undef = N2.getOpcode() == ISD::UNDEF;
01506   // Re-check whether both sides ended up undef.
01507   if (N1.getOpcode() == ISD::UNDEF && N2Undef)
01508     return getUNDEF(VT);
01509 
01510   // If Identity shuffle return that node.
01511   bool Identity = true;
01512   for (unsigned i = 0; i != NElts; ++i) {
01513     if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false;
01514   }
01515   if (Identity && NElts)
01516     return N1;
01517 
01518   // Shuffling a constant splat doesn't change the result.
01519   if (N2Undef) {
01520     SDValue V = N1;
01521 
01522     // Look through any bitcasts. We check that these don't change the number
01523     // (and size) of elements and just changes their types.
01524     while (V.getOpcode() == ISD::BITCAST)
01525       V = V->getOperand(0);
01526 
01527     // A splat should always show up as a build vector node.
01528     if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
01529       BitVector UndefElements;
01530       SDValue Splat = BV->getSplatValue(&UndefElements);
01531       // If this is a splat of an undef, shuffling it is also undef.
01532       if (Splat && Splat.getOpcode() == ISD::UNDEF)
01533         return getUNDEF(VT);
01534 
01535       // We only have a splat which can skip shuffles if there is a splatted
01536       // value and no undef lanes rearranged by the shuffle.
01537       if (Splat && UndefElements.none()) {
01538         // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
01539         // number of elements match or the value splatted is a zero constant.
01540         if (V.getValueType().getVectorNumElements() ==
01541             VT.getVectorNumElements())
01542           return N1;
01543         if (auto *C = dyn_cast<ConstantSDNode>(Splat))
01544           if (C->isNullValue())
01545             return N1;
01546       }
01547     }
01548   }
01549 
01550   FoldingSetNodeID ID;
01551   SDValue Ops[2] = { N1, N2 };
01552   AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
01553   for (unsigned i = 0; i != NElts; ++i)
01554     ID.AddInteger(MaskVec[i]);
01555 
01556   void* IP = nullptr;
01557   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01558     return SDValue(E, 0);
01559 
01560   // Allocate the mask array for the node out of the BumpPtrAllocator, since
01561   // SDNode doesn't have access to it.  This memory will be "leaked" when
01562   // the node is deallocated, but recovered when the NodeAllocator is released.
01563   int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
01564   memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int));
01565 
01566   ShuffleVectorSDNode *N =
01567     new (NodeAllocator) ShuffleVectorSDNode(VT, dl.getIROrder(),
01568                                             dl.getDebugLoc(), N1, N2,
01569                                             MaskAlloc);
01570   CSEMap.InsertNode(N, IP);
01571   InsertNode(N);
01572   return SDValue(N, 0);
01573 }
01574 
01575 SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
01576   MVT VT = SV.getSimpleValueType(0);
01577   unsigned NumElems = VT.getVectorNumElements();
01578   SmallVector<int, 8> MaskVec;
01579 
01580   for (unsigned i = 0; i != NumElems; ++i) {
01581     int Idx = SV.getMaskElt(i);
01582     if (Idx >= 0) {
01583       if (Idx < (int)NumElems)
01584         Idx += NumElems;
01585       else
01586         Idx -= NumElems;
01587     }
01588     MaskVec.push_back(Idx);
01589   }
01590 
01591   SDValue Op0 = SV.getOperand(0);
01592   SDValue Op1 = SV.getOperand(1);
01593   return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, &MaskVec[0]);
01594 }
01595 
01596 SDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl,
01597                                        SDValue Val, SDValue DTy,
01598                                        SDValue STy, SDValue Rnd, SDValue Sat,
01599                                        ISD::CvtCode Code) {
01600   // If the src and dest types are the same and the conversion is between
01601   // integer types of the same sign or two floats, no conversion is necessary.
01602   if (DTy == STy &&
01603       (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF))
01604     return Val;
01605 
01606   FoldingSetNodeID ID;
01607   SDValue Ops[] = { Val, DTy, STy, Rnd, Sat };
01608   AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), Ops);
01609   void* IP = nullptr;
01610   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01611     return SDValue(E, 0);
01612 
01613   CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl.getIROrder(),
01614                                                            dl.getDebugLoc(),
01615                                                            Ops, Code);
01616   CSEMap.InsertNode(N, IP);
01617   InsertNode(N);
01618   return SDValue(N, 0);
01619 }
01620 
01621 SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
01622   FoldingSetNodeID ID;
01623   AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
01624   ID.AddInteger(RegNo);
01625   void *IP = nullptr;
01626   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01627     return SDValue(E, 0);
01628 
01629   SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT);
01630   CSEMap.InsertNode(N, IP);
01631   InsertNode(N);
01632   return SDValue(N, 0);
01633 }
01634 
01635 SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
01636   FoldingSetNodeID ID;
01637   AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
01638   ID.AddPointer(RegMask);
01639   void *IP = nullptr;
01640   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01641     return SDValue(E, 0);
01642 
01643   SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask);
01644   CSEMap.InsertNode(N, IP);
01645   InsertNode(N);
01646   return SDValue(N, 0);
01647 }
01648 
01649 SDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) {
01650   FoldingSetNodeID ID;
01651   SDValue Ops[] = { Root };
01652   AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), Ops);
01653   ID.AddPointer(Label);
01654   void *IP = nullptr;
01655   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01656     return SDValue(E, 0);
01657 
01658   SDNode *N = new (NodeAllocator) EHLabelSDNode(dl.getIROrder(),
01659                                                 dl.getDebugLoc(), Root, Label);
01660   CSEMap.InsertNode(N, IP);
01661   InsertNode(N);
01662   return SDValue(N, 0);
01663 }
01664 
01665 
01666 SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
01667                                       int64_t Offset,
01668                                       bool isTarget,
01669                                       unsigned char TargetFlags) {
01670   unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
01671 
01672   FoldingSetNodeID ID;
01673   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01674   ID.AddPointer(BA);
01675   ID.AddInteger(Offset);
01676   ID.AddInteger(TargetFlags);
01677   void *IP = nullptr;
01678   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01679     return SDValue(E, 0);
01680 
01681   SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset,
01682                                                      TargetFlags);
01683   CSEMap.InsertNode(N, IP);
01684   InsertNode(N);
01685   return SDValue(N, 0);
01686 }
01687 
01688 SDValue SelectionDAG::getSrcValue(const Value *V) {
01689   assert((!V || V->getType()->isPointerTy()) &&
01690          "SrcValue is not a pointer?");
01691 
01692   FoldingSetNodeID ID;
01693   AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None);
01694   ID.AddPointer(V);
01695 
01696   void *IP = nullptr;
01697   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01698     return SDValue(E, 0);
01699 
01700   SDNode *N = new (NodeAllocator) SrcValueSDNode(V);
01701   CSEMap.InsertNode(N, IP);
01702   InsertNode(N);
01703   return SDValue(N, 0);
01704 }
01705 
01706 /// getMDNode - Return an MDNodeSDNode which holds an MDNode.
01707 SDValue SelectionDAG::getMDNode(const MDNode *MD) {
01708   FoldingSetNodeID ID;
01709   AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None);
01710   ID.AddPointer(MD);
01711 
01712   void *IP = nullptr;
01713   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01714     return SDValue(E, 0);
01715 
01716   SDNode *N = new (NodeAllocator) MDNodeSDNode(MD);
01717   CSEMap.InsertNode(N, IP);
01718   InsertNode(N);
01719   return SDValue(N, 0);
01720 }
01721 
01722 /// getAddrSpaceCast - Return an AddrSpaceCastSDNode.
01723 SDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr,
01724                                        unsigned SrcAS, unsigned DestAS) {
01725   SDValue Ops[] = {Ptr};
01726   FoldingSetNodeID ID;
01727   AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
01728   ID.AddInteger(SrcAS);
01729   ID.AddInteger(DestAS);
01730 
01731   void *IP = nullptr;
01732   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01733     return SDValue(E, 0);
01734 
01735   SDNode *N = new (NodeAllocator) AddrSpaceCastSDNode(dl.getIROrder(),
01736                                                       dl.getDebugLoc(),
01737                                                       VT, Ptr, SrcAS, DestAS);
01738   CSEMap.InsertNode(N, IP);
01739   InsertNode(N);
01740   return SDValue(N, 0);
01741 }
01742 
01743 /// getShiftAmountOperand - Return the specified value casted to
01744 /// the target's desired shift amount type.
01745 SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
01746   EVT OpTy = Op.getValueType();
01747   EVT ShTy = TM.getTargetLowering()->getShiftAmountTy(LHSTy);
01748   if (OpTy == ShTy || OpTy.isVector()) return Op;
01749 
01750   ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ?  ISD::TRUNCATE : ISD::ZERO_EXTEND;
01751   return getNode(Opcode, SDLoc(Op), ShTy, Op);
01752 }
01753 
01754 /// CreateStackTemporary - Create a stack temporary, suitable for holding the
01755 /// specified value type.
01756 SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
01757   MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
01758   unsigned ByteSize = VT.getStoreSize();
01759   Type *Ty = VT.getTypeForEVT(*getContext());
01760   const TargetLowering *TLI = TM.getTargetLowering();
01761   unsigned StackAlign =
01762   std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty), minAlign);
01763 
01764   int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false);
01765   return getFrameIndex(FrameIdx, TLI->getPointerTy());
01766 }
01767 
01768 /// CreateStackTemporary - Create a stack temporary suitable for holding
01769 /// either of the specified value types.
01770 SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
01771   unsigned Bytes = std::max(VT1.getStoreSizeInBits(),
01772                             VT2.getStoreSizeInBits())/8;
01773   Type *Ty1 = VT1.getTypeForEVT(*getContext());
01774   Type *Ty2 = VT2.getTypeForEVT(*getContext());
01775   const TargetLowering *TLI = TM.getTargetLowering();
01776   const DataLayout *TD = TLI->getDataLayout();
01777   unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1),
01778                             TD->getPrefTypeAlignment(Ty2));
01779 
01780   MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
01781   int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false);
01782   return getFrameIndex(FrameIdx, TLI->getPointerTy());
01783 }
01784 
01785 SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1,
01786                                 SDValue N2, ISD::CondCode Cond, SDLoc dl) {
01787   // These setcc operations always fold.
01788   switch (Cond) {
01789   default: break;
01790   case ISD::SETFALSE:
01791   case ISD::SETFALSE2: return getConstant(0, VT);
01792   case ISD::SETTRUE:
01793   case ISD::SETTRUE2: {
01794     const TargetLowering *TLI = TM.getTargetLowering();
01795     TargetLowering::BooleanContent Cnt =
01796         TLI->getBooleanContents(N1->getValueType(0));
01797     return getConstant(
01798         Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, VT);
01799   }
01800 
01801   case ISD::SETOEQ:
01802   case ISD::SETOGT:
01803   case ISD::SETOGE:
01804   case ISD::SETOLT:
01805   case ISD::SETOLE:
01806   case ISD::SETONE:
01807   case ISD::SETO:
01808   case ISD::SETUO:
01809   case ISD::SETUEQ:
01810   case ISD::SETUNE:
01811     assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!");
01812     break;
01813   }
01814 
01815   if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) {
01816     const APInt &C2 = N2C->getAPIntValue();
01817     if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
01818       const APInt &C1 = N1C->getAPIntValue();
01819 
01820       switch (Cond) {
01821       default: llvm_unreachable("Unknown integer setcc!");
01822       case ISD::SETEQ:  return getConstant(C1 == C2, VT);
01823       case ISD::SETNE:  return getConstant(C1 != C2, VT);
01824       case ISD::SETULT: return getConstant(C1.ult(C2), VT);
01825       case ISD::SETUGT: return getConstant(C1.ugt(C2), VT);
01826       case ISD::SETULE: return getConstant(C1.ule(C2), VT);
01827       case ISD::SETUGE: return getConstant(C1.uge(C2), VT);
01828       case ISD::SETLT:  return getConstant(C1.slt(C2), VT);
01829       case ISD::SETGT:  return getConstant(C1.sgt(C2), VT);
01830       case ISD::SETLE:  return getConstant(C1.sle(C2), VT);
01831       case ISD::SETGE:  return getConstant(C1.sge(C2), VT);
01832       }
01833     }
01834   }
01835   if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) {
01836     if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) {
01837       APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF());
01838       switch (Cond) {
01839       default: break;
01840       case ISD::SETEQ:  if (R==APFloat::cmpUnordered)
01841                           return getUNDEF(VT);
01842                         // fall through
01843       case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT);
01844       case ISD::SETNE:  if (R==APFloat::cmpUnordered)
01845                           return getUNDEF(VT);
01846                         // fall through
01847       case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan ||
01848                                            R==APFloat::cmpLessThan, VT);
01849       case ISD::SETLT:  if (R==APFloat::cmpUnordered)
01850                           return getUNDEF(VT);
01851                         // fall through
01852       case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT);
01853       case ISD::SETGT:  if (R==APFloat::cmpUnordered)
01854                           return getUNDEF(VT);
01855                         // fall through
01856       case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT);
01857       case ISD::SETLE:  if (R==APFloat::cmpUnordered)
01858                           return getUNDEF(VT);
01859                         // fall through
01860       case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan ||
01861                                            R==APFloat::cmpEqual, VT);
01862       case ISD::SETGE:  if (R==APFloat::cmpUnordered)
01863                           return getUNDEF(VT);
01864                         // fall through
01865       case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan ||
01866                                            R==APFloat::cmpEqual, VT);
01867       case ISD::SETO:   return getConstant(R!=APFloat::cmpUnordered, VT);
01868       case ISD::SETUO:  return getConstant(R==APFloat::cmpUnordered, VT);
01869       case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered ||
01870                                            R==APFloat::cmpEqual, VT);
01871       case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT);
01872       case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered ||
01873                                            R==APFloat::cmpLessThan, VT);
01874       case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan ||
01875                                            R==APFloat::cmpUnordered, VT);
01876       case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT);
01877       case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT);
01878       }
01879     } else {
01880       // Ensure that the constant occurs on the RHS.
01881       ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
01882       MVT CompVT = N1.getValueType().getSimpleVT();
01883       if (!TM.getTargetLowering()->isCondCodeLegal(SwappedCond, CompVT))
01884         return SDValue();
01885 
01886       return getSetCC(dl, VT, N2, N1, SwappedCond);
01887     }
01888   }
01889 
01890   // Could not fold it.
01891   return SDValue();
01892 }
01893 
01894 /// SignBitIsZero - Return true if the sign bit of Op is known to be zero.  We
01895 /// use this predicate to simplify operations downstream.
01896 bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
01897   // This predicate is not safe for vector operations.
01898   if (Op.getValueType().isVector())
01899     return false;
01900 
01901   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01902   return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth);
01903 }
01904 
01905 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero.  We use
01906 /// this predicate to simplify operations downstream.  Mask is known to be zero
01907 /// for bits that V cannot have.
01908 bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask,
01909                                      unsigned Depth) const {
01910   APInt KnownZero, KnownOne;
01911   computeKnownBits(Op, KnownZero, KnownOne, Depth);
01912   return (KnownZero & Mask) == Mask;
01913 }
01914 
01915 /// Determine which bits of Op are known to be either zero or one and return
01916 /// them in the KnownZero/KnownOne bitsets.
01917 void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
01918                                     APInt &KnownOne, unsigned Depth) const {
01919   const TargetLowering *TLI = TM.getTargetLowering();
01920   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01921 
01922   KnownZero = KnownOne = APInt(BitWidth, 0);   // Don't know anything.
01923   if (Depth == 6)
01924     return;  // Limit search depth.
01925 
01926   APInt KnownZero2, KnownOne2;
01927 
01928   switch (Op.getOpcode()) {
01929   case ISD::Constant:
01930     // We know all of the bits for a constant!
01931     KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue();
01932     KnownZero = ~KnownOne;
01933     break;
01934   case ISD::AND:
01935     // If either the LHS or the RHS are Zero, the result is zero.
01936     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01937     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01938 
01939     // Output known-1 bits are only known if set in both the LHS & RHS.
01940     KnownOne &= KnownOne2;
01941     // Output known-0 are known to be clear if zero in either the LHS | RHS.
01942     KnownZero |= KnownZero2;
01943     break;
01944   case ISD::OR:
01945     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01946     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01947 
01948     // Output known-0 bits are only known if clear in both the LHS & RHS.
01949     KnownZero &= KnownZero2;
01950     // Output known-1 are known to be set if set in either the LHS | RHS.
01951     KnownOne |= KnownOne2;
01952     break;
01953   case ISD::XOR: {
01954     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01955     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01956 
01957     // Output known-0 bits are known if clear or set in both the LHS & RHS.
01958     APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2);
01959     // Output known-1 are known to be set if set in only one of the LHS, RHS.
01960     KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2);
01961     KnownZero = KnownZeroOut;
01962     break;
01963   }
01964   case ISD::MUL: {
01965     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01966     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01967 
01968     // If low bits are zero in either operand, output low known-0 bits.
01969     // Also compute a conserative estimate for high known-0 bits.
01970     // More trickiness is possible, but this is sufficient for the
01971     // interesting case of alignment computation.
01972     KnownOne.clearAllBits();
01973     unsigned TrailZ = KnownZero.countTrailingOnes() +
01974                       KnownZero2.countTrailingOnes();
01975     unsigned LeadZ =  std::max(KnownZero.countLeadingOnes() +
01976                                KnownZero2.countLeadingOnes(),
01977                                BitWidth) - BitWidth;
01978 
01979     TrailZ = std::min(TrailZ, BitWidth);
01980     LeadZ = std::min(LeadZ, BitWidth);
01981     KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) |
01982                 APInt::getHighBitsSet(BitWidth, LeadZ);
01983     break;
01984   }
01985   case ISD::UDIV: {
01986     // For the purposes of computing leading zeros we can conservatively
01987     // treat a udiv as a logical right shift by the power of 2 known to
01988     // be less than the denominator.
01989     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01990     unsigned LeadZ = KnownZero2.countLeadingOnes();
01991 
01992     KnownOne2.clearAllBits();
01993     KnownZero2.clearAllBits();
01994     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
01995     unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros();
01996     if (RHSUnknownLeadingOnes != BitWidth)
01997       LeadZ = std::min(BitWidth,
01998                        LeadZ + BitWidth - RHSUnknownLeadingOnes - 1);
01999 
02000     KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ);
02001     break;
02002   }
02003   case ISD::SELECT:
02004     computeKnownBits(Op.getOperand(2), KnownZero, KnownOne, Depth+1);
02005     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02006 
02007     // Only known if known in both the LHS and RHS.
02008     KnownOne &= KnownOne2;
02009     KnownZero &= KnownZero2;
02010     break;
02011   case ISD::SELECT_CC:
02012     computeKnownBits(Op.getOperand(3), KnownZero, KnownOne, Depth+1);
02013     computeKnownBits(Op.getOperand(2), KnownZero2, KnownOne2, Depth+1);
02014 
02015     // Only known if known in both the LHS and RHS.
02016     KnownOne &= KnownOne2;
02017     KnownZero &= KnownZero2;
02018     break;
02019   case ISD::SADDO:
02020   case ISD::UADDO:
02021   case ISD::SSUBO:
02022   case ISD::USUBO:
02023   case ISD::SMULO:
02024   case ISD::UMULO:
02025     if (Op.getResNo() != 1)
02026       break;
02027     // The boolean result conforms to getBooleanContents.
02028     // If we know the result of a setcc has the top bits zero, use this info.
02029     // We know that we have an integer-based boolean since these operations
02030     // are only available for integer.
02031     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
02032             TargetLowering::ZeroOrOneBooleanContent &&
02033         BitWidth > 1)
02034       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02035     break;
02036   case ISD::SETCC:
02037     // If we know the result of a setcc has the top bits zero, use this info.
02038     if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
02039             TargetLowering::ZeroOrOneBooleanContent &&
02040         BitWidth > 1)
02041       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02042     break;
02043   case ISD::SHL:
02044     // (shl X, C1) & C2 == 0   iff   (X & C2 >>u C1) == 0
02045     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02046       unsigned ShAmt = SA->getZExtValue();
02047 
02048       // If the shift count is an invalid immediate, don't do anything.
02049       if (ShAmt >= BitWidth)
02050         break;
02051 
02052       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02053       KnownZero <<= ShAmt;
02054       KnownOne  <<= ShAmt;
02055       // low bits known zero.
02056       KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt);
02057     }
02058     break;
02059   case ISD::SRL:
02060     // (ushr X, C1) & C2 == 0   iff  (-1 >> C1) & C2 == 0
02061     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02062       unsigned ShAmt = SA->getZExtValue();
02063 
02064       // If the shift count is an invalid immediate, don't do anything.
02065       if (ShAmt >= BitWidth)
02066         break;
02067 
02068       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02069       KnownZero = KnownZero.lshr(ShAmt);
02070       KnownOne  = KnownOne.lshr(ShAmt);
02071 
02072       APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
02073       KnownZero |= HighBits;  // High bits known zero.
02074     }
02075     break;
02076   case ISD::SRA:
02077     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02078       unsigned ShAmt = SA->getZExtValue();
02079 
02080       // If the shift count is an invalid immediate, don't do anything.
02081       if (ShAmt >= BitWidth)
02082         break;
02083 
02084       // If any of the demanded bits are produced by the sign extension, we also
02085       // demand the input sign bit.
02086       APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
02087 
02088       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02089       KnownZero = KnownZero.lshr(ShAmt);
02090       KnownOne  = KnownOne.lshr(ShAmt);
02091 
02092       // Handle the sign bits.
02093       APInt SignBit = APInt::getSignBit(BitWidth);
02094       SignBit = SignBit.lshr(ShAmt);  // Adjust to where it is now in the mask.
02095 
02096       if (KnownZero.intersects(SignBit)) {
02097         KnownZero |= HighBits;  // New bits are known zero.
02098       } else if (KnownOne.intersects(SignBit)) {
02099         KnownOne  |= HighBits;  // New bits are known one.
02100       }
02101     }
02102     break;
02103   case ISD::SIGN_EXTEND_INREG: {
02104     EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
02105     unsigned EBits = EVT.getScalarType().getSizeInBits();
02106 
02107     // Sign extension.  Compute the demanded bits in the result that are not
02108     // present in the input.
02109     APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits);
02110 
02111     APInt InSignBit = APInt::getSignBit(EBits);
02112     APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits);
02113 
02114     // If the sign extended bits are demanded, we know that the sign
02115     // bit is demanded.
02116     InSignBit = InSignBit.zext(BitWidth);
02117     if (NewBits.getBoolValue())
02118       InputDemandedBits |= InSignBit;
02119 
02120     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02121     KnownOne &= InputDemandedBits;
02122     KnownZero &= InputDemandedBits;
02123 
02124     // If the sign bit of the input is known set or clear, then we know the
02125     // top bits of the result.
02126     if (KnownZero.intersects(InSignBit)) {         // Input sign bit known clear
02127       KnownZero |= NewBits;
02128       KnownOne  &= ~NewBits;
02129     } else if (KnownOne.intersects(InSignBit)) {   // Input sign bit known set
02130       KnownOne  |= NewBits;
02131       KnownZero &= ~NewBits;
02132     } else {                              // Input sign bit unknown
02133       KnownZero &= ~NewBits;
02134       KnownOne  &= ~NewBits;
02135     }
02136     break;
02137   }
02138   case ISD::CTTZ:
02139   case ISD::CTTZ_ZERO_UNDEF:
02140   case ISD::CTLZ:
02141   case ISD::CTLZ_ZERO_UNDEF:
02142   case ISD::CTPOP: {
02143     unsigned LowBits = Log2_32(BitWidth)+1;
02144     KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits);
02145     KnownOne.clearAllBits();
02146     break;
02147   }
02148   case ISD::LOAD: {
02149     LoadSDNode *LD = cast<LoadSDNode>(Op);
02150     // If this is a ZEXTLoad and we are looking at the loaded value.
02151     if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
02152       EVT VT = LD->getMemoryVT();
02153       unsigned MemBits = VT.getScalarType().getSizeInBits();
02154       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
02155     } else if (const MDNode *Ranges = LD->getRanges()) {
02156       computeKnownBitsFromRangeMetadata(*Ranges, KnownZero);
02157     }
02158     break;
02159   }
02160   case ISD::ZERO_EXTEND: {
02161     EVT InVT = Op.getOperand(0).getValueType();
02162     unsigned InBits = InVT.getScalarType().getSizeInBits();
02163     APInt NewBits   = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
02164     KnownZero = KnownZero.trunc(InBits);
02165     KnownOne = KnownOne.trunc(InBits);
02166     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02167     KnownZero = KnownZero.zext(BitWidth);
02168     KnownOne = KnownOne.zext(BitWidth);
02169     KnownZero |= NewBits;
02170     break;
02171   }
02172   case ISD::SIGN_EXTEND: {
02173     EVT InVT = Op.getOperand(0).getValueType();
02174     unsigned InBits = InVT.getScalarType().getSizeInBits();
02175     APInt NewBits   = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
02176 
02177     KnownZero = KnownZero.trunc(InBits);
02178     KnownOne = KnownOne.trunc(InBits);
02179     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02180 
02181     // Note if the sign bit is known to be zero or one.
02182     bool SignBitKnownZero = KnownZero.isNegative();
02183     bool SignBitKnownOne  = KnownOne.isNegative();
02184 
02185     KnownZero = KnownZero.zext(BitWidth);
02186     KnownOne = KnownOne.zext(BitWidth);
02187 
02188     // If the sign bit is known zero or one, the top bits match.
02189     if (SignBitKnownZero)
02190       KnownZero |= NewBits;
02191     else if (SignBitKnownOne)
02192       KnownOne  |= NewBits;
02193     break;
02194   }
02195   case ISD::ANY_EXTEND: {
02196     EVT InVT = Op.getOperand(0).getValueType();
02197     unsigned InBits = InVT.getScalarType().getSizeInBits();
02198     KnownZero = KnownZero.trunc(InBits);
02199     KnownOne = KnownOne.trunc(InBits);
02200     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02201     KnownZero = KnownZero.zext(BitWidth);
02202     KnownOne = KnownOne.zext(BitWidth);
02203     break;
02204   }
02205   case ISD::TRUNCATE: {
02206     EVT InVT = Op.getOperand(0).getValueType();
02207     unsigned InBits = InVT.getScalarType().getSizeInBits();
02208     KnownZero = KnownZero.zext(InBits);
02209     KnownOne = KnownOne.zext(InBits);
02210     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02211     KnownZero = KnownZero.trunc(BitWidth);
02212     KnownOne = KnownOne.trunc(BitWidth);
02213     break;
02214   }
02215   case ISD::AssertZext: {
02216     EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
02217     APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
02218     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02219     KnownZero |= (~InMask);
02220     KnownOne  &= (~KnownZero);
02221     break;
02222   }
02223   case ISD::FGETSIGN:
02224     // All bits are zero except the low bit.
02225     KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02226     break;
02227 
02228   case ISD::SUB: {
02229     if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) {
02230       // We know that the top bits of C-X are clear if X contains less bits
02231       // than C (i.e. no wrap-around can happen).  For example, 20-X is
02232       // positive if we can prove that X is >= 0 and < 16.
02233       if (CLHS->getAPIntValue().isNonNegative()) {
02234         unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros();
02235         // NLZ can't be BitWidth with no sign bit
02236         APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1);
02237         computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02238 
02239         // If all of the MaskV bits are known to be zero, then we know the
02240         // output top bits are zero, because we now know that the output is
02241         // from [0-C].
02242         if ((KnownZero2 & MaskV) == MaskV) {
02243           unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros();
02244           // Top bits known zero.
02245           KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2);
02246         }
02247       }
02248     }
02249   }
02250   // fall through
02251   case ISD::ADD:
02252   case ISD::ADDE: {
02253     // Output known-0 bits are known if clear or set in both the low clear bits
02254     // common to both LHS & RHS.  For example, 8+(X<<3) is known to have the
02255     // low 3 bits clear.
02256     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
02257     unsigned KnownZeroOut = KnownZero2.countTrailingOnes();
02258 
02259     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02260     KnownZeroOut = std::min(KnownZeroOut,
02261                             KnownZero2.countTrailingOnes());
02262 
02263     if (Op.getOpcode() == ISD::ADD) {
02264       KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut);
02265       break;
02266     }
02267 
02268     // With ADDE, a carry bit may be added in, so we can only use this
02269     // information if we know (at least) that the low two bits are clear.  We
02270     // then return to the caller that the low bit is unknown but that other bits
02271     // are known zero.
02272     if (KnownZeroOut >= 2) // ADDE
02273       KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut);
02274     break;
02275   }
02276   case ISD::SREM:
02277     if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02278       const APInt &RA = Rem->getAPIntValue().abs();
02279       if (RA.isPowerOf2()) {
02280         APInt LowBits = RA - 1;
02281         computeKnownBits(Op.getOperand(0), KnownZero2,KnownOne2,Depth+1);
02282 
02283         // The low bits of the first operand are unchanged by the srem.
02284         KnownZero = KnownZero2 & LowBits;
02285         KnownOne = KnownOne2 & LowBits;
02286 
02287         // If the first operand is non-negative or has all low bits zero, then
02288         // the upper bits are all zero.
02289         if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits))
02290           KnownZero |= ~LowBits;
02291 
02292         // If the first operand is negative and not all low bits are zero, then
02293         // the upper bits are all one.
02294         if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0))
02295           KnownOne |= ~LowBits;
02296         assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?");
02297       }
02298     }
02299     break;
02300   case ISD::UREM: {
02301     if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02302       const APInt &RA = Rem->getAPIntValue();
02303       if (RA.isPowerOf2()) {
02304         APInt LowBits = (RA - 1);
02305         computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth + 1);
02306 
02307         // The upper bits are all zero, the lower ones are unchanged.
02308         KnownZero = KnownZero2 | ~LowBits;
02309         KnownOne = KnownOne2 & LowBits;
02310         break;
02311       }
02312     }
02313 
02314     // Since the result is less than or equal to either operand, any leading
02315     // zero bits in either operand must also exist in the result.
02316     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02317     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02318 
02319     uint32_t Leaders = std::max(KnownZero.countLeadingOnes(),
02320                                 KnownZero2.countLeadingOnes());
02321     KnownOne.clearAllBits();
02322     KnownZero = APInt::getHighBitsSet(BitWidth, Leaders);
02323     break;
02324   }
02325   case ISD::FrameIndex:
02326   case ISD::TargetFrameIndex:
02327     if (unsigned Align = InferPtrAlignment(Op)) {
02328       // The low bits are known zero if the pointer is aligned.
02329       KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align));
02330       break;
02331     }
02332     break;
02333 
02334   default:
02335     if (Op.getOpcode() < ISD::BUILTIN_OP_END)
02336       break;
02337     // Fallthrough
02338   case ISD::INTRINSIC_WO_CHAIN:
02339   case ISD::INTRINSIC_W_CHAIN:
02340   case ISD::INTRINSIC_VOID:
02341     // Allow the target to implement this method for its nodes.
02342     TLI->computeKnownBitsForTargetNode(Op, KnownZero, KnownOne, *this, Depth);
02343     break;
02344   }
02345 
02346   assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
02347 }
02348 
02349 /// ComputeNumSignBits - Return the number of times the sign bit of the
02350 /// register is replicated into the other bits.  We know that at least 1 bit
02351 /// is always equal to the sign bit (itself), but other cases can give us
02352 /// information.  For example, immediately after an "SRA X, 2", we know that
02353 /// the top 3 bits are all equal to each other, so we return 3.
02354 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{
02355   const TargetLowering *TLI = TM.getTargetLowering();
02356   EVT VT = Op.getValueType();
02357   assert(VT.isInteger() && "Invalid VT!");
02358   unsigned VTBits = VT.getScalarType().getSizeInBits();
02359   unsigned Tmp, Tmp2;
02360   unsigned FirstAnswer = 1;
02361 
02362   if (Depth == 6)
02363     return 1;  // Limit search depth.
02364 
02365   switch (Op.getOpcode()) {
02366   default: break;
02367   case ISD::AssertSext:
02368     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
02369     return VTBits-Tmp+1;
02370   case ISD::AssertZext:
02371     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
02372     return VTBits-Tmp;
02373 
02374   case ISD::Constant: {
02375     const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue();
02376     return Val.getNumSignBits();
02377   }
02378 
02379   case ISD::SIGN_EXTEND:
02380     Tmp =
02381         VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits();
02382     return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp;
02383 
02384   case ISD::SIGN_EXTEND_INREG:
02385     // Max of the input and what this extends.
02386     Tmp =
02387       cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits();
02388     Tmp = VTBits-Tmp+1;
02389 
02390     Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02391     return std::max(Tmp, Tmp2);
02392 
02393   case ISD::SRA:
02394     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02395     // SRA X, C   -> adds C sign bits.
02396     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02397       Tmp += C->getZExtValue();
02398       if (Tmp > VTBits) Tmp = VTBits;
02399     }
02400     return Tmp;
02401   case ISD::SHL:
02402     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02403       // shl destroys sign bits.
02404       Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02405       if (C->getZExtValue() >= VTBits ||      // Bad shift.
02406           C->getZExtValue() >= Tmp) break;    // Shifted all sign bits out.
02407       return Tmp - C->getZExtValue();
02408     }
02409     break;
02410   case ISD::AND:
02411   case ISD::OR:
02412   case ISD::XOR:    // NOT is handled here.
02413     // Logical binary ops preserve the number of sign bits at the worst.
02414     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02415     if (Tmp != 1) {
02416       Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02417       FirstAnswer = std::min(Tmp, Tmp2);
02418       // We computed what we know about the sign bits as our first
02419       // answer. Now proceed to the generic code that uses
02420       // computeKnownBits, and pick whichever answer is better.
02421     }
02422     break;
02423 
02424   case ISD::SELECT:
02425     Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02426     if (Tmp == 1) return 1;  // Early out.
02427     Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1);
02428     return std::min(Tmp, Tmp2);
02429 
02430   case ISD::SADDO:
02431   case ISD::UADDO:
02432   case ISD::SSUBO:
02433   case ISD::USUBO:
02434   case ISD::SMULO:
02435   case ISD::UMULO:
02436     if (Op.getResNo() != 1)
02437       break;
02438     // The boolean result conforms to getBooleanContents.  Fall through.
02439     // If setcc returns 0/-1, all bits are sign bits.
02440     // We know that we have an integer-based boolean since these operations
02441     // are only available for integer.
02442     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
02443         TargetLowering::ZeroOrNegativeOneBooleanContent)
02444       return VTBits;
02445     break;
02446   case ISD::SETCC:
02447     // If setcc returns 0/-1, all bits are sign bits.
02448     if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
02449         TargetLowering::ZeroOrNegativeOneBooleanContent)
02450       return VTBits;
02451     break;
02452   case ISD::ROTL:
02453   case ISD::ROTR:
02454     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02455       unsigned RotAmt = C->getZExtValue() & (VTBits-1);
02456 
02457       // Handle rotate right by N like a rotate left by 32-N.
02458       if (Op.getOpcode() == ISD::ROTR)
02459         RotAmt = (VTBits-RotAmt) & (VTBits-1);
02460 
02461       // If we aren't rotating out all of the known-in sign bits, return the
02462       // number that are left.  This handles rotl(sext(x), 1) for example.
02463       Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02464       if (Tmp > RotAmt+1) return Tmp-RotAmt;
02465     }
02466     break;
02467   case ISD::ADD:
02468     // Add can have at most one carry bit.  Thus we know that the output
02469     // is, at worst, one more bit than the inputs.
02470     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02471     if (Tmp == 1) return 1;  // Early out.
02472 
02473     // Special case decrementing a value (ADD X, -1):
02474     if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
02475       if (CRHS->isAllOnesValue()) {
02476         APInt KnownZero, KnownOne;
02477         computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02478 
02479         // If the input is known to be 0 or 1, the output is 0/-1, which is all
02480         // sign bits set.
02481         if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
02482           return VTBits;
02483 
02484         // If we are subtracting one from a positive number, there is no carry
02485         // out of the result.
02486         if (KnownZero.isNegative())
02487           return Tmp;
02488       }
02489 
02490     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02491     if (Tmp2 == 1) return 1;
02492     return std::min(Tmp, Tmp2)-1;
02493 
02494   case ISD::SUB:
02495     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02496     if (Tmp2 == 1) return 1;
02497 
02498     // Handle NEG.
02499     if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0)))
02500       if (CLHS->isNullValue()) {
02501         APInt KnownZero, KnownOne;
02502         computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
02503         // If the input is known to be 0 or 1, the output is 0/-1, which is all
02504         // sign bits set.
02505         if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
02506           return VTBits;
02507 
02508         // If the input is known to be positive (the sign bit is known clear),
02509         // the output of the NEG has the same number of sign bits as the input.
02510         if (KnownZero.isNegative())
02511           return Tmp2;
02512 
02513         // Otherwise, we treat this like a SUB.
02514       }
02515 
02516     // Sub can have at most one carry bit.  Thus we know that the output
02517     // is, at worst, one more bit than the inputs.
02518     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02519     if (Tmp == 1) return 1;  // Early out.
02520     return std::min(Tmp, Tmp2)-1;
02521   case ISD::TRUNCATE:
02522     // FIXME: it's tricky to do anything useful for this, but it is an important
02523     // case for targets like X86.
02524     break;
02525   }
02526 
02527   // If we are looking at the loaded value of the SDNode.
02528   if (Op.getResNo() == 0) {
02529     // Handle LOADX separately here. EXTLOAD case will fallthrough.
02530     if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
02531       unsigned ExtType = LD->getExtensionType();
02532       switch (ExtType) {
02533         default: break;
02534         case ISD::SEXTLOAD:    // '17' bits known
02535           Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
02536           return VTBits-Tmp+1;
02537         case ISD::ZEXTLOAD:    // '16' bits known
02538           Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
02539           return VTBits-Tmp;
02540       }
02541     }
02542   }
02543 
02544   // Allow the target to implement this method for its nodes.
02545   if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
02546       Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
02547       Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
02548       Op.getOpcode() == ISD::INTRINSIC_VOID) {
02549     unsigned NumBits = TLI->ComputeNumSignBitsForTargetNode(Op, *this, Depth);
02550     if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits);
02551   }
02552 
02553   // Finally, if we can prove that the top bits of the result are 0's or 1's,
02554   // use this information.
02555   APInt KnownZero, KnownOne;
02556   computeKnownBits(Op, KnownZero, KnownOne, Depth);
02557 
02558   APInt Mask;
02559   if (KnownZero.isNegative()) {        // sign bit is 0
02560     Mask = KnownZero;
02561   } else if (KnownOne.isNegative()) {  // sign bit is 1;
02562     Mask = KnownOne;
02563   } else {
02564     // Nothing known.
02565     return FirstAnswer;
02566   }
02567 
02568   // Okay, we know that the sign bit in Mask is set.  Use CLZ to determine
02569   // the number of identical bits in the top of the input value.
02570   Mask = ~Mask;
02571   Mask <<= Mask.getBitWidth()-VTBits;
02572   // Return # leading zeros.  We use 'min' here in case Val was zero before
02573   // shifting.  We don't want to return '64' as for an i32 "0".
02574   return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros()));
02575 }
02576 
02577 /// isBaseWithConstantOffset - Return true if the specified operand is an
02578 /// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an
02579 /// ISD::OR with a ConstantSDNode that is guaranteed to have the same
02580 /// semantics as an ADD.  This handles the equivalence:
02581 ///     X|Cst == X+Cst iff X&Cst = 0.
02582 bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
02583   if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
02584       !isa<ConstantSDNode>(Op.getOperand(1)))
02585     return false;
02586 
02587   if (Op.getOpcode() == ISD::OR &&
02588       !MaskedValueIsZero(Op.getOperand(0),
02589                      cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue()))
02590     return false;
02591 
02592   return true;
02593 }
02594 
02595 
02596 bool SelectionDAG::isKnownNeverNaN(SDValue Op) const {
02597   // If we're told that NaNs won't happen, assume they won't.
02598   if (getTarget().Options.NoNaNsFPMath)
02599     return true;
02600 
02601   // If the value is a constant, we can obviously see if it is a NaN or not.
02602   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
02603     return !C->getValueAPF().isNaN();
02604 
02605   // TODO: Recognize more cases here.
02606 
02607   return false;
02608 }
02609 
02610 bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
02611   // If the value is a constant, we can obviously see if it is a zero or not.
02612   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
02613     return !C->isZero();
02614 
02615   // TODO: Recognize more cases here.
02616   switch (Op.getOpcode()) {
02617   default: break;
02618   case ISD::OR:
02619     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
02620       return !C->isNullValue();
02621     break;
02622   }
02623 
02624   return false;
02625 }
02626 
02627 bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
02628   // Check the obvious case.
02629   if (A == B) return true;
02630 
02631   // For for negative and positive zero.
02632   if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
02633     if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
02634       if (CA->isZero() && CB->isZero()) return true;
02635 
02636   // Otherwise they may not be equal.
02637   return false;
02638 }
02639 
02640 /// getNode - Gets or creates the specified node.
02641 ///
02642 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT) {
02643   FoldingSetNodeID ID;
02644   AddNodeIDNode(ID, Opcode, getVTList(VT), None);
02645   void *IP = nullptr;
02646   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
02647     return SDValue(E, 0);
02648 
02649   SDNode *N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(),
02650                                          DL.getDebugLoc(), getVTList(VT));
02651   CSEMap.InsertNode(N, IP);
02652 
02653   InsertNode(N);
02654   return SDValue(N, 0);
02655 }
02656 
02657 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
02658                               EVT VT, SDValue Operand) {
02659   // Constant fold unary operations with an integer constant operand. Even
02660   // opaque constant will be folded, because the folding of unary operations
02661   // doesn't create new constants with different values. Nevertheless, the
02662   // opaque flag is preserved during folding to prevent future folding with
02663   // other constants.
02664   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) {
02665     const APInt &Val = C->getAPIntValue();
02666     switch (Opcode) {
02667     default: break;
02668     case ISD::SIGN_EXTEND:
02669       return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT,
02670                          C->isTargetOpcode(), C->isOpaque());
02671     case ISD::ANY_EXTEND:
02672     case ISD::ZERO_EXTEND:
02673     case ISD::TRUNCATE:
02674       return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT,
02675                          C->isTargetOpcode(), C->isOpaque());
02676     case ISD::UINT_TO_FP:
02677     case ISD::SINT_TO_FP: {
02678       APFloat apf(EVTToAPFloatSemantics(VT),
02679                   APInt::getNullValue(VT.getSizeInBits()));
02680       (void)apf.convertFromAPInt(Val,
02681                                  Opcode==ISD::SINT_TO_FP,
02682                                  APFloat::rmNearestTiesToEven);
02683       return getConstantFP(apf, VT);
02684     }
02685     case ISD::BITCAST:
02686       if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
02687         return getConstantFP(APFloat(APFloat::IEEEsingle, Val), VT);
02688       else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
02689         return getConstantFP(APFloat(APFloat::IEEEdouble, Val), VT);
02690       break;
02691     case ISD::BSWAP:
02692       return getConstant(Val.byteSwap(), VT, C->isTargetOpcode(),
02693                          C->isOpaque());
02694     case ISD::CTPOP:
02695       return getConstant(Val.countPopulation(), VT, C->isTargetOpcode(),
02696                          C->isOpaque());
02697     case ISD::CTLZ:
02698     case ISD::CTLZ_ZERO_UNDEF:
02699       return getConstant(Val.countLeadingZeros(), VT, C->isTargetOpcode(),
02700                          C->isOpaque());
02701     case ISD::CTTZ:
02702     case ISD::CTTZ_ZERO_UNDEF:
02703       return getConstant(Val.countTrailingZeros(), VT, C->isTargetOpcode(),
02704                          C->isOpaque());
02705     }
02706   }
02707 
02708   // Constant fold unary operations with a floating point constant operand.
02709   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) {
02710     APFloat V = C->getValueAPF();    // make copy
02711     switch (Opcode) {
02712     case ISD::FNEG:
02713       V.changeSign();
02714       return getConstantFP(V, VT);
02715     case ISD::FABS:
02716       V.clearSign();
02717       return getConstantFP(V, VT);
02718     case ISD::FCEIL: {
02719       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
02720       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02721         return getConstantFP(V, VT);
02722       break;
02723     }
02724     case ISD::FTRUNC: {
02725       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
02726       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02727         return getConstantFP(V, VT);
02728       break;
02729     }
02730     case ISD::FFLOOR: {
02731       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
02732       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02733         return getConstantFP(V, VT);
02734       break;
02735     }
02736     case ISD::FP_EXTEND: {
02737       bool ignored;
02738       // This can return overflow, underflow, or inexact; we don't care.
02739       // FIXME need to be more flexible about rounding mode.
02740       (void)V.convert(EVTToAPFloatSemantics(VT),
02741                       APFloat::rmNearestTiesToEven, &ignored);
02742       return getConstantFP(V, VT);
02743     }
02744     case ISD::FP_TO_SINT:
02745     case ISD::FP_TO_UINT: {
02746       integerPart x[2];
02747       bool ignored;
02748       assert(integerPartWidth >= 64);
02749       // FIXME need to be more flexible about rounding mode.
02750       APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(),
02751                             Opcode==ISD::FP_TO_SINT,
02752                             APFloat::rmTowardZero, &ignored);
02753       if (s==APFloat::opInvalidOp)     // inexact is OK, in fact usual
02754         break;
02755       APInt api(VT.getSizeInBits(), x);
02756       return getConstant(api, VT);
02757     }
02758     case ISD::BITCAST:
02759       if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
02760         return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT);
02761       else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
02762         return getConstant(V.bitcastToAPInt().getZExtValue(), VT);
02763       break;
02764     }
02765   }
02766 
02767   // Constant fold unary operations with a vector integer operand.
02768   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand.getNode())) {
02769     if (BV->isConstant()) {
02770       switch (Opcode) {
02771       default:
02772         // FIXME: Entirely reasonable to perform folding of other unary
02773         // operations here as the need arises.
02774         break;
02775       case ISD::UINT_TO_FP:
02776       case ISD::SINT_TO_FP: {
02777         SmallVector<SDValue, 8> Ops;
02778         for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
02779           SDValue OpN = BV->getOperand(i);
02780           // Let the above scalar folding handle the conversion of each
02781           // element.
02782           OpN = getNode(ISD::SINT_TO_FP, DL, VT.getVectorElementType(),
02783                         OpN);
02784           Ops.push_back(OpN);
02785         }
02786         return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
02787       }
02788       }
02789     }
02790   }
02791 
02792   unsigned OpOpcode = Operand.getNode()->getOpcode();
02793   switch (Opcode) {
02794   case ISD::TokenFactor:
02795   case ISD::MERGE_VALUES:
02796   case ISD::CONCAT_VECTORS:
02797     return Operand;         // Factor, merge or concat of one node?  No need.
02798   case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
02799   case ISD::FP_EXTEND:
02800     assert(VT.isFloatingPoint() &&
02801            Operand.getValueType().isFloatingPoint() && "Invalid FP cast!");
02802     if (Operand.getValueType() == VT) return Operand;  // noop conversion.
02803     assert((!VT.isVector() ||
02804             VT.getVectorNumElements() ==
02805             Operand.getValueType().getVectorNumElements()) &&
02806            "Vector element count mismatch!");
02807     if (Operand.getOpcode() == ISD::UNDEF)
02808       return getUNDEF(VT);
02809     break;
02810   case ISD::SIGN_EXTEND:
02811     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02812            "Invalid SIGN_EXTEND!");
02813     if (Operand.getValueType() == VT) return Operand;   // noop extension
02814     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02815            "Invalid sext node, dst < src!");
02816     assert((!VT.isVector() ||
02817             VT.getVectorNumElements() ==
02818             Operand.getValueType().getVectorNumElements()) &&
02819            "Vector element count mismatch!");
02820     if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
02821       return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02822     else if (OpOpcode == ISD::UNDEF)
02823       // sext(undef) = 0, because the top bits will all be the same.
02824       return getConstant(0, VT);
02825     break;
02826   case ISD::ZERO_EXTEND:
02827     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02828            "Invalid ZERO_EXTEND!");
02829     if (Operand.getValueType() == VT) return Operand;   // noop extension
02830     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02831            "Invalid zext node, dst < src!");
02832     assert((!VT.isVector() ||
02833             VT.getVectorNumElements() ==
02834             Operand.getValueType().getVectorNumElements()) &&
02835            "Vector element count mismatch!");
02836     if (OpOpcode == ISD::ZERO_EXTEND)   // (zext (zext x)) -> (zext x)
02837       return getNode(ISD::ZERO_EXTEND, DL, VT,
02838                      Operand.getNode()->getOperand(0));
02839     else if (OpOpcode == ISD::UNDEF)
02840       // zext(undef) = 0, because the top bits will be zero.
02841       return getConstant(0, VT);
02842     break;
02843   case ISD::ANY_EXTEND:
02844     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02845            "Invalid ANY_EXTEND!");
02846     if (Operand.getValueType() == VT) return Operand;   // noop extension
02847     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02848            "Invalid anyext node, dst < src!");
02849     assert((!VT.isVector() ||
02850             VT.getVectorNumElements() ==
02851             Operand.getValueType().getVectorNumElements()) &&
02852            "Vector element count mismatch!");
02853 
02854     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
02855         OpOpcode == ISD::ANY_EXTEND)
02856       // (ext (zext x)) -> (zext x)  and  (ext (sext x)) -> (sext x)
02857       return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02858     else if (OpOpcode == ISD::UNDEF)
02859       return getUNDEF(VT);
02860 
02861     // (ext (trunx x)) -> x
02862     if (OpOpcode == ISD::TRUNCATE) {
02863       SDValue OpOp = Operand.getNode()->getOperand(0);
02864       if (OpOp.getValueType() == VT)
02865         return OpOp;
02866     }
02867     break;
02868   case ISD::TRUNCATE:
02869     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02870            "Invalid TRUNCATE!");
02871     if (Operand.getValueType() == VT) return Operand;   // noop truncate
02872     assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) &&
02873            "Invalid truncate node, src < dst!");
02874     assert((!VT.isVector() ||
02875             VT.getVectorNumElements() ==
02876             Operand.getValueType().getVectorNumElements()) &&
02877            "Vector element count mismatch!");
02878     if (OpOpcode == ISD::TRUNCATE)
02879       return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
02880     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
02881         OpOpcode == ISD::ANY_EXTEND) {
02882       // If the source is smaller than the dest, we still need an extend.
02883       if (Operand.getNode()->getOperand(0).getValueType().getScalarType()
02884             .bitsLT(VT.getScalarType()))
02885         return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02886       if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT))
02887         return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
02888       return Operand.getNode()->getOperand(0);
02889     }
02890     if (OpOpcode == ISD::UNDEF)
02891       return getUNDEF(VT);
02892     break;
02893   case ISD::BITCAST:
02894     // Basic sanity checking.
02895     assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits()
02896            && "Cannot BITCAST between types of different sizes!");
02897     if (VT == Operand.getValueType()) return Operand;  // noop conversion.
02898     if (OpOpcode == ISD::BITCAST)  // bitconv(bitconv(x)) -> bitconv(x)
02899       return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
02900     if (OpOpcode == ISD::UNDEF)
02901       return getUNDEF(VT);
02902     break;
02903   case ISD::SCALAR_TO_VECTOR:
02904     assert(VT.isVector() && !Operand.getValueType().isVector() &&
02905            (VT.getVectorElementType() == Operand.getValueType() ||
02906             (VT.getVectorElementType().isInteger() &&
02907              Operand.getValueType().isInteger() &&
02908              VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&
02909            "Illegal SCALAR_TO_VECTOR node!");
02910     if (OpOpcode == ISD::UNDEF)
02911       return getUNDEF(VT);
02912     // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
02913     if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
02914         isa<ConstantSDNode>(Operand.getOperand(1)) &&
02915         Operand.getConstantOperandVal(1) == 0 &&
02916         Operand.getOperand(0).getValueType() == VT)
02917       return Operand.getOperand(0);
02918     break;
02919   case ISD::FNEG:
02920     // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0
02921     if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB)
02922       return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1),
02923                      Operand.getNode()->getOperand(0));
02924     if (OpOpcode == ISD::FNEG)  // --X -> X
02925       return Operand.getNode()->getOperand(0);
02926     break;
02927   case ISD::FABS:
02928     if (OpOpcode == ISD::FNEG)  // abs(-X) -> abs(X)
02929       return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0));
02930     break;
02931   }
02932 
02933   SDNode *N;
02934   SDVTList VTs = getVTList(VT);
02935   if (VT != MVT::Glue) { // Don't CSE flag producing nodes
02936     FoldingSetNodeID ID;
02937     SDValue Ops[1] = { Operand };
02938     AddNodeIDNode(ID, Opcode, VTs, Ops);
02939     void *IP = nullptr;
02940     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
02941       return SDValue(E, 0);
02942 
02943     N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
02944                                         DL.getDebugLoc(), VTs, Operand);
02945     CSEMap.InsertNode(N, IP);
02946   } else {
02947     N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
02948                                         DL.getDebugLoc(), VTs, Operand);
02949   }
02950 
02951   InsertNode(N);
02952   return SDValue(N, 0);
02953 }
02954 
02955 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT,
02956                                              SDNode *Cst1, SDNode *Cst2) {
02957   // If the opcode is a target-specific ISD node, there's nothing we can
02958   // do here and the operand rules may not line up with the below, so
02959   // bail early.
02960   if (Opcode >= ISD::BUILTIN_OP_END)
02961     return SDValue();
02962 
02963   SmallVector<std::pair<ConstantSDNode *, ConstantSDNode *>, 4> Inputs;
02964   SmallVector<SDValue, 4> Outputs;
02965   EVT SVT = VT.getScalarType();
02966 
02967   ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1);
02968   ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2);
02969   if (Scalar1 && Scalar2 && (Scalar1->isOpaque() || Scalar2->isOpaque()))
02970     return SDValue();
02971 
02972   if (Scalar1 && Scalar2)
02973     // Scalar instruction.
02974     Inputs.push_back(std::make_pair(Scalar1, Scalar2));
02975   else {
02976     // For vectors extract each constant element into Inputs so we can constant
02977     // fold them individually.
02978     BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1);
02979     BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2);
02980     if (!BV1 || !BV2)
02981       return SDValue();
02982 
02983     assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!");
02984 
02985     for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) {
02986       ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I));
02987       ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I));
02988       if (!V1 || !V2) // Not a constant, bail.
02989         return SDValue();
02990 
02991       if (V1->isOpaque() || V2->isOpaque())
02992         return SDValue();
02993 
02994       // Avoid BUILD_VECTOR nodes that perform implicit truncation.
02995       // FIXME: This is valid and could be handled by truncating the APInts.
02996       if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
02997         return SDValue();
02998 
02999       Inputs.push_back(std::make_pair(V1, V2));
03000     }
03001   }
03002 
03003   // We have a number of constant values, constant fold them element by element.
03004   for (unsigned I = 0, E = Inputs.size(); I != E; ++I) {
03005     const APInt &C1 = Inputs[I].first->getAPIntValue();
03006     const APInt &C2 = Inputs[I].second->getAPIntValue();
03007 
03008     switch (Opcode) {
03009     case ISD::ADD:
03010       Outputs.push_back(getConstant(C1 + C2, SVT));
03011       break;
03012     case ISD::SUB:
03013       Outputs.push_back(getConstant(C1 - C2, SVT));
03014       break;
03015     case ISD::MUL:
03016       Outputs.push_back(getConstant(C1 * C2, SVT));
03017       break;
03018     case ISD::UDIV:
03019       if (!C2.getBoolValue())
03020         return SDValue();
03021       Outputs.push_back(getConstant(C1.udiv(C2), SVT));
03022       break;
03023     case ISD::UREM:
03024       if (!C2.getBoolValue())
03025         return SDValue();
03026       Outputs.push_back(getConstant(C1.urem(C2), SVT));
03027       break;
03028     case ISD::SDIV:
03029       if (!C2.getBoolValue())
03030         return SDValue();
03031       Outputs.push_back(getConstant(C1.sdiv(C2), SVT));
03032       break;
03033     case ISD::SREM:
03034       if (!C2.getBoolValue())
03035         return SDValue();
03036       Outputs.push_back(getConstant(C1.srem(C2), SVT));
03037       break;
03038     case ISD::AND:
03039       Outputs.push_back(getConstant(C1 & C2, SVT));
03040       break;
03041     case ISD::OR:
03042       Outputs.push_back(getConstant(C1 | C2, SVT));
03043       break;
03044     case ISD::XOR:
03045       Outputs.push_back(getConstant(C1 ^ C2, SVT));
03046       break;
03047     case ISD::SHL:
03048       Outputs.push_back(getConstant(C1 << C2, SVT));
03049       break;
03050     case ISD::SRL:
03051       Outputs.push_back(getConstant(C1.lshr(C2), SVT));
03052       break;
03053     case ISD::SRA:
03054       Outputs.push_back(getConstant(C1.ashr(C2), SVT));
03055       break;
03056     case ISD::ROTL:
03057       Outputs.push_back(getConstant(C1.rotl(C2), SVT));
03058       break;
03059     case ISD::ROTR:
03060       Outputs.push_back(getConstant(C1.rotr(C2), SVT));
03061       break;
03062     default:
03063       return SDValue();
03064     }
03065   }
03066 
03067   assert((Scalar1 && Scalar2) || (VT.getVectorNumElements() == Outputs.size() &&
03068                                   "Expected a scalar or vector!"));
03069 
03070   // Handle the scalar case first.
03071   if (!VT.isVector())
03072     return Outputs.back();
03073 
03074   // We may have a vector type but a scalar result. Create a splat.
03075   Outputs.resize(VT.getVectorNumElements(), Outputs.back());
03076 
03077   // Build a big vector out of the scalar elements we generated.
03078   return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs);
03079 }
03080 
03081 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
03082                               SDValue N2, bool nuw, bool nsw, bool exact) {
03083   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
03084   ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode());
03085   switch (Opcode) {
03086   default: break;
03087   case ISD::TokenFactor:
03088     assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
03089            N2.getValueType() == MVT::Other && "Invalid token factor!");
03090     // Fold trivial token factors.
03091     if (N1.getOpcode() == ISD::EntryToken) return N2;
03092     if (N2.getOpcode() == ISD::EntryToken) return N1;
03093     if (N1 == N2) return N1;
03094     break;
03095   case ISD::CONCAT_VECTORS:
03096     // Concat of UNDEFs is UNDEF.
03097     if (N1.getOpcode() == ISD::UNDEF &&
03098         N2.getOpcode() == ISD::UNDEF)
03099       return getUNDEF(VT);
03100 
03101     // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
03102     // one big BUILD_VECTOR.
03103     if (N1.getOpcode() == ISD::BUILD_VECTOR &&
03104         N2.getOpcode() == ISD::BUILD_VECTOR) {
03105       SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
03106                                     N1.getNode()->op_end());
03107       Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
03108       return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
03109     }
03110     break;
03111   case ISD::AND:
03112     assert(VT.isInteger() && "This operator does not apply to FP types!");
03113     assert(N1.getValueType() == N2.getValueType() &&
03114            N1.getValueType() == VT && "Binary operator types must match!");
03115     // (X & 0) -> 0.  This commonly occurs when legalizing i64 values, so it's
03116     // worth handling here.
03117     if (N2C && N2C->isNullValue())
03118       return N2;
03119     if (N2C && N2C->isAllOnesValue())  // X & -1 -> X
03120       return N1;
03121     break;
03122   case ISD::OR:
03123   case ISD::XOR:
03124   case ISD::ADD:
03125   case ISD::SUB:
03126     assert(VT.isInteger() && "This operator does not apply to FP types!");
03127     assert(N1.getValueType() == N2.getValueType() &&
03128            N1.getValueType() == VT && "Binary operator types must match!");
03129     // (X ^|+- 0) -> X.  This commonly occurs when legalizing i64 values, so
03130     // it's worth handling here.
03131     if (N2C && N2C->isNullValue())
03132       return N1;
03133     break;
03134   case ISD::UDIV:
03135   case ISD::UREM:
03136   case ISD::MULHU:
03137   case ISD::MULHS:
03138   case ISD::MUL:
03139   case ISD::SDIV:
03140   case ISD::SREM:
03141     assert(VT.isInteger() && "This operator does not apply to FP types!");
03142     assert(N1.getValueType() == N2.getValueType() &&
03143            N1.getValueType() == VT && "Binary operator types must match!");
03144     break;
03145   case ISD::FADD:
03146   case ISD::FSUB:
03147   case ISD::FMUL:
03148   case ISD::FDIV:
03149   case ISD::FREM:
03150     if (getTarget().Options.UnsafeFPMath) {
03151       if (Opcode == ISD::FADD) {
03152         // 0+x --> x
03153         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1))
03154           if (CFP->getValueAPF().isZero())
03155             return N2;
03156         // x+0 --> x
03157         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
03158           if (CFP->getValueAPF().isZero())
03159             return N1;
03160       } else if (Opcode == ISD::FSUB) {
03161         // x-0 --> x
03162         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
03163           if (CFP->getValueAPF().isZero())
03164             return N1;
03165       } else if (Opcode == ISD::FMUL) {
03166         ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1);
03167         SDValue V = N2;
03168 
03169         // If the first operand isn't the constant, try the second
03170         if (!CFP) {
03171           CFP = dyn_cast<ConstantFPSDNode>(N2);
03172           V = N1;
03173         }
03174 
03175         if (CFP) {
03176           // 0*x --> 0
03177           if (CFP->isZero())
03178             return SDValue(CFP,0);
03179           // 1*x --> x
03180           if (CFP->isExactlyValue(1.0))
03181             return V;
03182         }
03183       }
03184     }
03185     assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
03186     assert(N1.getValueType() == N2.getValueType() &&
03187            N1.getValueType() == VT && "Binary operator types must match!");
03188     break;
03189   case ISD::FCOPYSIGN:   // N1 and result must match.  N1/N2 need not match.
03190     assert(N1.getValueType() == VT &&
03191            N1.getValueType().isFloatingPoint() &&
03192            N2.getValueType().isFloatingPoint() &&
03193            "Invalid FCOPYSIGN!");
03194     break;
03195   case ISD::SHL:
03196   case ISD::SRA:
03197   case ISD::SRL:
03198   case ISD::ROTL:
03199   case ISD::ROTR:
03200     assert(VT == N1.getValueType() &&
03201            "Shift operators return type must be the same as their first arg");
03202     assert(VT.isInteger() && N2.getValueType().isInteger() &&
03203            "Shifts only work on integers");
03204     assert((!VT.isVector() || VT == N2.getValueType()) &&
03205            "Vector shift amounts must be in the same as their first arg");
03206     // Verify that the shift amount VT is bit enough to hold valid shift
03207     // amounts.  This catches things like trying to shift an i1024 value by an
03208     // i8, which is easy to fall into in generic code that uses
03209     // TLI.getShiftAmount().
03210     assert(N2.getValueType().getSizeInBits() >=
03211                    Log2_32_Ceil(N1.getValueType().getSizeInBits()) &&
03212            "Invalid use of small shift amount with oversized value!");
03213 
03214     // Always fold shifts of i1 values so the code generator doesn't need to
03215     // handle them.  Since we know the size of the shift has to be less than the
03216     // size of the value, the shift/rotate count is guaranteed to be zero.
03217     if (VT == MVT::i1)
03218       return N1;
03219     if (N2C && N2C->isNullValue())
03220       return N1;
03221     break;
03222   case ISD::FP_ROUND_INREG: {
03223     EVT EVT = cast<VTSDNode>(N2)->getVT();
03224     assert(VT == N1.getValueType() && "Not an inreg round!");
03225     assert(VT.isFloatingPoint() && EVT.isFloatingPoint() &&
03226            "Cannot FP_ROUND_INREG integer types");
03227     assert(EVT.isVector() == VT.isVector() &&
03228            "FP_ROUND_INREG type should be vector iff the operand "
03229            "type is vector!");
03230     assert((!EVT.isVector() ||
03231             EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
03232            "Vector element counts must match in FP_ROUND_INREG");
03233     assert(EVT.bitsLE(VT) && "Not rounding down!");
03234     (void)EVT;
03235     if (cast<VTSDNode>(N2)->getVT() == VT) return N1;  // Not actually rounding.
03236     break;
03237   }
03238   case ISD::FP_ROUND:
03239     assert(VT.isFloatingPoint() &&
03240            N1.getValueType().isFloatingPoint() &&
03241            VT.bitsLE(N1.getValueType()) &&
03242            isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!");
03243     if (N1.getValueType() == VT) return N1;  // noop conversion.
03244     break;
03245   case ISD::AssertSext:
03246   case ISD::AssertZext: {
03247     EVT EVT = cast<VTSDNode>(N2)->getVT();
03248     assert(VT == N1.getValueType() && "Not an inreg extend!");
03249     assert(VT.isInteger() && EVT.isInteger() &&
03250            "Cannot *_EXTEND_INREG FP types");
03251     assert(!EVT.isVector() &&
03252            "AssertSExt/AssertZExt type should be the vector element type "
03253            "rather than the vector type!");
03254     assert(EVT.bitsLE(VT) && "Not extending!");
03255     if (VT == EVT) return N1; // noop assertion.
03256     break;
03257   }
03258   case ISD::SIGN_EXTEND_INREG: {
03259     EVT EVT = cast<VTSDNode>(N2)->getVT();
03260     assert(VT == N1.getValueType() && "Not an inreg extend!");
03261     assert(VT.isInteger() && EVT.isInteger() &&
03262            "Cannot *_EXTEND_INREG FP types");
03263     assert(EVT.isVector() == VT.isVector() &&
03264            "SIGN_EXTEND_INREG type should be vector iff the operand "
03265            "type is vector!");
03266     assert((!EVT.isVector() ||
03267             EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
03268            "Vector element counts must match in SIGN_EXTEND_INREG");
03269     assert(EVT.bitsLE(VT) && "Not extending!");
03270     if (EVT == VT) return N1;  // Not actually extending
03271 
03272     if (N1C) {
03273       APInt Val = N1C->getAPIntValue();
03274       unsigned FromBits = EVT.getScalarType().getSizeInBits();
03275       Val <<= Val.getBitWidth()-FromBits;
03276       Val = Val.ashr(Val.getBitWidth()-FromBits);
03277       return getConstant(Val, VT);
03278     }
03279     break;
03280   }
03281   case ISD::EXTRACT_VECTOR_ELT:
03282     // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF.
03283     if (N1.getOpcode() == ISD::UNDEF)
03284       return getUNDEF(VT);
03285 
03286     // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
03287     // expanding copies of large vectors from registers.
03288     if (N2C &&
03289         N1.getOpcode() == ISD::CONCAT_VECTORS &&
03290         N1.getNumOperands() > 0) {
03291       unsigned Factor =
03292         N1.getOperand(0).getValueType().getVectorNumElements();
03293       return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
03294                      N1.getOperand(N2C->getZExtValue() / Factor),
03295                      getConstant(N2C->getZExtValue() % Factor,
03296                                  N2.getValueType()));
03297     }
03298 
03299     // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
03300     // expanding large vector constants.
03301     if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) {
03302       SDValue Elt = N1.getOperand(N2C->getZExtValue());
03303 
03304       if (VT != Elt.getValueType())
03305         // If the vector element type is not legal, the BUILD_VECTOR operands
03306         // are promoted and implicitly truncated, and the result implicitly
03307         // extended. Make that explicit here.
03308         Elt = getAnyExtOrTrunc(Elt, DL, VT);
03309 
03310       return Elt;
03311     }
03312 
03313     // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
03314     // operations are lowered to scalars.
03315     if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
03316       // If the indices are the same, return the inserted element else
03317       // if the indices are known different, extract the element from
03318       // the original vector.
03319       SDValue N1Op2 = N1.getOperand(2);
03320       ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode());
03321 
03322       if (N1Op2C && N2C) {
03323         if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
03324           if (VT == N1.getOperand(1).getValueType())
03325             return N1.getOperand(1);
03326           else
03327             return getSExtOrTrunc(N1.getOperand(1), DL, VT);
03328         }
03329 
03330         return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
03331       }
03332     }
03333     break;
03334   case ISD::EXTRACT_ELEMENT:
03335     assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
03336     assert(!N1.getValueType().isVector() && !VT.isVector() &&
03337            (N1.getValueType().isInteger() == VT.isInteger()) &&
03338            N1.getValueType() != VT &&
03339            "Wrong types for EXTRACT_ELEMENT!");
03340 
03341     // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
03342     // 64-bit integers into 32-bit parts.  Instead of building the extract of
03343     // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
03344     if (N1.getOpcode() == ISD::BUILD_PAIR)
03345       return N1.getOperand(N2C->getZExtValue());
03346 
03347     // EXTRACT_ELEMENT of a constant int is also very common.
03348     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
03349       unsigned ElementSize = VT.getSizeInBits();
03350       unsigned Shift = ElementSize * N2C->getZExtValue();
03351       APInt ShiftedVal = C->getAPIntValue().lshr(Shift);
03352       return getConstant(ShiftedVal.trunc(ElementSize), VT);
03353     }
03354     break;
03355   case ISD::EXTRACT_SUBVECTOR: {
03356     SDValue Index = N2;
03357     if (VT.isSimple() && N1.getValueType().isSimple()) {
03358       assert(VT.isVector() && N1.getValueType().isVector() &&
03359              "Extract subvector VTs must be a vectors!");
03360       assert(VT.getVectorElementType() ==
03361              N1.getValueType().getVectorElementType() &&
03362              "Extract subvector VTs must have the same element type!");
03363       assert(VT.getSimpleVT() <= N1.getSimpleValueType() &&
03364              "Extract subvector must be from larger vector to smaller vector!");
03365 
03366       if (isa<ConstantSDNode>(Index.getNode())) {
03367         assert((VT.getVectorNumElements() +
03368                 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
03369                 <= N1.getValueType().getVectorNumElements())
03370                && "Extract subvector overflow!");
03371       }
03372 
03373       // Trivial extraction.
03374       if (VT.getSimpleVT() == N1.getSimpleValueType())
03375         return N1;
03376     }
03377     break;
03378   }
03379   }
03380 
03381   // Perform trivial constant folding.
03382   SDValue SV = FoldConstantArithmetic(Opcode, VT, N1.getNode(), N2.getNode());
03383   if (SV.getNode()) return SV;
03384 
03385   // Canonicalize constant to RHS if commutative.
03386   if (N1C && !N2C && isCommutativeBinOp(Opcode)) {
03387     std::swap(N1C, N2C);
03388     std::swap(N1, N2);
03389   }
03390 
03391   // Constant fold FP operations.
03392   ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode());
03393   ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode());
03394   if (N1CFP) {
03395     if (!N2CFP && isCommutativeBinOp(Opcode)) {
03396       // Canonicalize constant to RHS if commutative.
03397       std::swap(N1CFP, N2CFP);
03398       std::swap(N1, N2);
03399     } else if (N2CFP) {
03400       APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF();
03401       APFloat::opStatus s;
03402       switch (Opcode) {
03403       case ISD::FADD:
03404         s = V1.add(V2, APFloat::rmNearestTiesToEven);
03405         if (s != APFloat::opInvalidOp)
03406           return getConstantFP(V1, VT);
03407         break;
03408       case ISD::FSUB:
03409         s = V1.subtract(V2, APFloat::rmNearestTiesToEven);
03410         if (s!=APFloat::opInvalidOp)
03411           return getConstantFP(V1, VT);
03412         break;
03413       case ISD::FMUL:
03414         s = V1.multiply(V2, APFloat::rmNearestTiesToEven);
03415         if (s!=APFloat::opInvalidOp)
03416           return getConstantFP(V1, VT);
03417         break;
03418       case ISD::FDIV:
03419         s = V1.divide(V2, APFloat::rmNearestTiesToEven);
03420         if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero)
03421           return getConstantFP(V1, VT);
03422         break;
03423       case ISD::FREM :
03424         s = V1.mod(V2, APFloat::rmNearestTiesToEven);
03425         if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero)
03426           return getConstantFP(V1, VT);
03427         break;
03428       case ISD::FCOPYSIGN:
03429         V1.copySign(V2);
03430         return getConstantFP(V1, VT);
03431       default: break;
03432       }
03433     }
03434 
03435     if (Opcode == ISD::FP_ROUND) {
03436       APFloat V = N1CFP->getValueAPF();    // make copy
03437       bool ignored;
03438       // This can return overflow, underflow, or inexact; we don't care.
03439       // FIXME need to be more flexible about rounding mode.
03440       (void)V.convert(EVTToAPFloatSemantics(VT),
03441                       APFloat::rmNearestTiesToEven, &ignored);
03442       return getConstantFP(V, VT);
03443     }
03444   }
03445 
03446   // Canonicalize an UNDEF to the RHS, even over a constant.
03447   if (N1.getOpcode() == ISD::UNDEF) {
03448     if (isCommutativeBinOp(Opcode)) {
03449       std::swap(N1, N2);
03450     } else {
03451       switch (Opcode) {
03452       case ISD::FP_ROUND_INREG:
03453       case ISD::SIGN_EXTEND_INREG:
03454       case ISD::SUB:
03455       case ISD::FSUB:
03456       case ISD::FDIV:
03457       case ISD::FREM:
03458       case ISD::SRA:
03459         return N1;     // fold op(undef, arg2) -> undef
03460       case ISD::UDIV:
03461       case ISD::SDIV:
03462       case ISD::UREM:
03463       case ISD::SREM:
03464       case ISD::SRL:
03465       case ISD::SHL:
03466         if (!VT.isVector())
03467           return getConstant(0, VT);    // fold op(undef, arg2) -> 0
03468         // For vectors, we can't easily build an all zero vector, just return
03469         // the LHS.
03470         return N2;
03471       }
03472     }
03473   }
03474 
03475   // Fold a bunch of operators when the RHS is undef.
03476   if (N2.getOpcode() == ISD::UNDEF) {
03477     switch (Opcode) {
03478     case ISD::XOR:
03479       if (N1.getOpcode() == ISD::UNDEF)
03480         // Handle undef ^ undef -> 0 special case. This is a common
03481         // idiom (misuse).
03482         return getConstant(0, VT);
03483       // fallthrough
03484     case ISD::ADD:
03485     case ISD::ADDC:
03486     case ISD::ADDE:
03487     case ISD::SUB:
03488     case ISD::UDIV:
03489     case ISD::SDIV:
03490     case ISD::UREM:
03491     case ISD::SREM:
03492       return N2;       // fold op(arg1, undef) -> undef
03493     case ISD::FADD:
03494     case ISD::FSUB:
03495     case ISD::FMUL:
03496     case ISD::FDIV:
03497     case ISD::FREM:
03498       if (getTarget().Options.UnsafeFPMath)
03499         return N2;
03500       break;
03501     case ISD::MUL:
03502     case ISD::AND:
03503     case ISD::SRL:
03504     case ISD::SHL:
03505       if (!VT.isVector())
03506         return getConstant(0, VT);  // fold op(arg1, undef) -> 0
03507       // For vectors, we can't easily build an all zero vector, just return
03508       // the LHS.
03509       return N1;
03510     case ISD::OR:
03511       if (!VT.isVector())
03512         return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT);
03513       // For vectors, we can't easily build an all one vector, just return
03514       // the LHS.
03515       return N1;
03516     case ISD::SRA:
03517       return N1;
03518     }
03519   }
03520 
03521   // Memoize this node if possible.
03522   BinarySDNode *N;
03523   SDVTList VTs = getVTList(VT);
03524   const bool BinOpHasFlags = isBinOpWithFlags(Opcode);
03525   if (VT != MVT::Glue) {
03526     SDValue Ops[] = {N1, N2};
03527     FoldingSetNodeID ID;
03528     AddNodeIDNode(ID, Opcode, VTs, Ops);
03529     if (BinOpHasFlags)
03530       AddBinaryNodeIDCustom(ID, Opcode, nuw, nsw, exact);
03531     void *IP = nullptr;
03532     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
03533       return SDValue(E, 0);
03534 
03535     N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
03536 
03537     CSEMap.InsertNode(N, IP);
03538   } else {
03539 
03540     N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
03541   }
03542 
03543   InsertNode(N);
03544   return SDValue(N, 0);
03545 }
03546 
03547 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03548                               SDValue N1, SDValue N2, SDValue N3) {
03549   // Perform various simplifications.
03550   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
03551   switch (Opcode) {
03552   case ISD::FMA: {
03553     ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
03554     ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
03555     ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
03556     if (N1CFP && N2CFP && N3CFP) {
03557       APFloat  V1 = N1CFP->getValueAPF();
03558       const APFloat &V2 = N2CFP->getValueAPF();
03559       const APFloat &V3 = N3CFP->getValueAPF();
03560       APFloat::opStatus s =
03561         V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
03562       if (s != APFloat::opInvalidOp)
03563         return getConstantFP(V1, VT);
03564     }
03565     break;
03566   }
03567   case ISD::CONCAT_VECTORS:
03568     // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
03569     // one big BUILD_VECTOR.
03570     if (N1.getOpcode() == ISD::BUILD_VECTOR &&
03571         N2.getOpcode() == ISD::BUILD_VECTOR &&
03572         N3.getOpcode() == ISD::BUILD_VECTOR) {
03573       SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
03574                                     N1.getNode()->op_end());
03575       Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
03576       Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end());
03577       return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
03578     }
03579     break;
03580   case ISD::SETCC: {
03581     // Use FoldSetCC to simplify SETCC's.
03582     SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL);
03583     if (Simp.getNode()) return Simp;
03584     break;
03585   }
03586   case ISD::SELECT:
03587     if (N1C) {
03588      if (N1C->getZExtValue())
03589        return N2;             // select true, X, Y -> X
03590      return N3;             // select false, X, Y -> Y
03591     }
03592 
03593     if (N2 == N3) return N2;   // select C, X, X -> X
03594     break;
03595   case ISD::VECTOR_SHUFFLE:
03596     llvm_unreachable("should use getVectorShuffle constructor!");
03597   case ISD::INSERT_SUBVECTOR: {
03598     SDValue Index = N3;
03599     if (VT.isSimple() && N1.getValueType().isSimple()
03600         && N2.getValueType().isSimple()) {
03601       assert(VT.isVector() && N1.getValueType().isVector() &&
03602              N2.getValueType().isVector() &&
03603              "Insert subvector VTs must be a vectors");
03604       assert(VT == N1.getValueType() &&
03605              "Dest and insert subvector source types must match!");
03606       assert(N2.getSimpleValueType() <= N1.getSimpleValueType() &&
03607              "Insert subvector must be from smaller vector to larger vector!");
03608       if (isa<ConstantSDNode>(Index.getNode())) {
03609         assert((N2.getValueType().getVectorNumElements() +
03610                 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
03611                 <= VT.getVectorNumElements())
03612                && "Insert subvector overflow!");
03613       }
03614 
03615       // Trivial insertion.
03616       if (VT.getSimpleVT() == N2.getSimpleValueType())
03617         return N2;
03618     }
03619     break;
03620   }
03621   case ISD::BITCAST:
03622     // Fold bit_convert nodes from a type to themselves.
03623     if (N1.getValueType() == VT)
03624       return N1;
03625     break;
03626   }
03627 
03628   // Memoize node if it doesn't produce a flag.
03629   SDNode *N;
03630   SDVTList VTs = getVTList(VT);
03631   if (VT != MVT::Glue) {
03632     SDValue Ops[] = { N1, N2, N3 };
03633     FoldingSetNodeID ID;
03634     AddNodeIDNode(ID, Opcode, VTs, Ops);
03635     void *IP = nullptr;
03636     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
03637       return SDValue(E, 0);
03638 
03639     N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
03640                                           DL.getDebugLoc(), VTs, N1, N2, N3);
03641     CSEMap.InsertNode(N, IP);
03642   } else {
03643     N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
03644                                           DL.getDebugLoc(), VTs, N1, N2, N3);
03645   }
03646 
03647   InsertNode(N);
03648   return SDValue(N, 0);
03649 }
03650 
03651 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03652                               SDValue N1, SDValue N2, SDValue N3,
03653                               SDValue N4) {
03654   SDValue Ops[] = { N1, N2, N3, N4 };
03655   return getNode(Opcode, DL, VT, Ops);
03656 }
03657 
03658 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03659                               SDValue N1, SDValue N2, SDValue N3,
03660                               SDValue N4, SDValue N5) {
03661   SDValue Ops[] = { N1, N2, N3, N4, N5 };
03662   return getNode(Opcode, DL, VT, Ops);
03663 }
03664 
03665 /// getStackArgumentTokenFactor - Compute a TokenFactor to force all
03666 /// the incoming stack arguments to be loaded from the stack.
03667 SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
03668   SmallVector<SDValue, 8> ArgChains;
03669 
03670   // Include the original chain at the beginning of the list. When this is
03671   // used by target LowerCall hooks, this helps legalize find the
03672   // CALLSEQ_BEGIN node.
03673   ArgChains.push_back(Chain);
03674 
03675   // Add a chain value for each stack argument.
03676   for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(),
03677        UE = getEntryNode().getNode()->use_end(); U != UE; ++U)
03678     if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
03679       if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
03680         if (FI->getIndex() < 0)
03681           ArgChains.push_back(SDValue(L, 1));
03682 
03683   // Build a tokenfactor for all the chains.
03684   return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
03685 }
03686 
03687 /// getMemsetValue - Vectorized representation of the memset value
03688 /// operand.
03689 static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
03690                               SDLoc dl) {
03691   assert(Value.getOpcode() != ISD::UNDEF);
03692 
03693   unsigned NumBits = VT.getScalarType().getSizeInBits();
03694   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
03695     assert(C->getAPIntValue().getBitWidth() == 8);
03696     APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
03697     if (VT.isInteger())
03698       return DAG.getConstant(Val, VT);
03699     return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), VT);
03700   }
03701 
03702   Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value);
03703   if (NumBits > 8) {
03704     // Use a multiplication with 0x010101... to extend the input to the
03705     // required length.
03706     APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
03707     Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT));
03708   }
03709 
03710   return Value;
03711 }
03712 
03713 /// getMemsetStringVal - Similar to getMemsetValue. Except this is only
03714 /// used when a memcpy is turned into a memset when the source is a constant
03715 /// string ptr.
03716 static SDValue getMemsetStringVal(EVT VT, SDLoc dl, SelectionDAG &DAG,
03717                                   const TargetLowering &TLI, StringRef Str) {
03718   // Handle vector with all elements zero.
03719   if (Str.empty()) {
03720     if (VT.isInteger())
03721       return DAG.getConstant(0, VT);
03722     else if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
03723       return DAG.getConstantFP(0.0, VT);
03724     else if (VT.isVector()) {
03725       unsigned NumElts = VT.getVectorNumElements();
03726       MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
03727       return DAG.getNode(ISD::BITCAST, dl, VT,
03728                          DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(),
03729                                                              EltVT, NumElts)));
03730     } else
03731       llvm_unreachable("Expected type!");
03732   }
03733 
03734   assert(!VT.isVector() && "Can't handle vector type here!");
03735   unsigned NumVTBits = VT.getSizeInBits();
03736   unsigned NumVTBytes = NumVTBits / 8;
03737   unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size()));
03738 
03739   APInt Val(NumVTBits, 0);
03740   if (TLI.isLittleEndian()) {
03741     for (unsigned i = 0; i != NumBytes; ++i)
03742       Val |= (uint64_t)(unsigned char)Str[i] << i*8;
03743   } else {
03744     for (unsigned i = 0; i != NumBytes; ++i)
03745       Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8;
03746   }
03747 
03748   // If the "cost" of materializing the integer immediate is less than the cost
03749   // of a load, then it is cost effective to turn the load into the immediate.
03750   Type *Ty = VT.getTypeForEVT(*DAG.getContext());
03751   if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
03752     return DAG.getConstant(Val, VT);
03753   return SDValue(nullptr, 0);
03754 }
03755 
03756 /// getMemBasePlusOffset - Returns base and offset node for the
03757 ///
03758 static SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, SDLoc dl,
03759                                       SelectionDAG &DAG) {
03760   EVT VT = Base.getValueType();
03761   return DAG.getNode(ISD::ADD, dl,
03762                      VT, Base, DAG.getConstant(Offset, VT));
03763 }
03764 
03765 /// isMemSrcFromString - Returns true if memcpy source is a string constant.
03766 ///
03767 static bool isMemSrcFromString(SDValue Src, StringRef &Str) {
03768   unsigned SrcDelta = 0;
03769   GlobalAddressSDNode *G = nullptr;
03770   if (Src.getOpcode() == ISD::GlobalAddress)
03771     G = cast<GlobalAddressSDNode>(Src);
03772   else if (Src.getOpcode() == ISD::ADD &&
03773            Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
03774            Src.getOperand(1).getOpcode() == ISD::Constant) {
03775     G = cast<GlobalAddressSDNode>(Src.getOperand(0));
03776     SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue();
03777   }
03778   if (!G)
03779     return false;
03780 
03781   return getConstantStringInfo(G->getGlobal(), Str, SrcDelta, false);
03782 }
03783 
03784 /// FindOptimalMemOpLowering - Determines the optimial series memory ops
03785 /// to replace the memset / memcpy. Return true if the number of memory ops
03786 /// is below the threshold. It returns the types of the sequence of
03787 /// memory ops to perform memset / memcpy by reference.
03788 static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
03789                                      unsigned Limit, uint64_t Size,
03790                                      unsigned DstAlign, unsigned SrcAlign,
03791                                      bool IsMemset,
03792                                      bool ZeroMemset,
03793                                      bool MemcpyStrSrc,
03794                                      bool AllowOverlap,
03795                                      SelectionDAG &DAG,
03796                                      const TargetLowering &TLI) {
03797   assert((SrcAlign == 0 || SrcAlign >= DstAlign) &&
03798          "Expecting memcpy / memset source to meet alignment requirement!");
03799   // If 'SrcAlign' is zero, that means the memory operation does not need to
03800   // load the value, i.e. memset or memcpy from constant string. Otherwise,
03801   // it's the inferred alignment of the source. 'DstAlign', on the other hand,
03802   // is the specified alignment of the memory operation. If it is zero, that
03803   // means it's possible to change the alignment of the destination.
03804   // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does
03805   // not need to be loaded.
03806   EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign,
03807                                    IsMemset, ZeroMemset, MemcpyStrSrc,
03808                                    DAG.getMachineFunction());
03809 
03810   if (VT == MVT::Other) {
03811     unsigned AS = 0;
03812     if (DstAlign >= TLI.getDataLayout()->getPointerPrefAlignment(AS) ||
03813         TLI.allowsUnalignedMemoryAccesses(VT, AS)) {
03814       VT = TLI.getPointerTy();
03815     } else {
03816       switch (DstAlign & 7) {
03817       case 0:  VT = MVT::i64; break;
03818       case 4:  VT = MVT::i32; break;
03819       case 2:  VT = MVT::i16; break;
03820       default: VT = MVT::i8;  break;
03821       }
03822     }
03823 
03824     MVT LVT = MVT::i64;
03825     while (!TLI.isTypeLegal(LVT))
03826       LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
03827     assert(LVT.isInteger());
03828 
03829     if (VT.bitsGT(LVT))
03830       VT = LVT;
03831   }
03832 
03833   unsigned NumMemOps = 0;
03834   while (Size != 0) {
03835     unsigned VTSize = VT.getSizeInBits() / 8;
03836     while (VTSize > Size) {
03837       // For now, only use non-vector load / store's for the left-over pieces.
03838       EVT NewVT = VT;
03839       unsigned NewVTSize;
03840 
03841       bool Found = false;
03842       if (VT.isVector() || VT.isFloatingPoint()) {
03843         NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
03844         if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) &&
03845             TLI.isSafeMemOpType(NewVT.getSimpleVT()))
03846           Found = true;
03847         else if (NewVT == MVT::i64 &&
03848                  TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
03849                  TLI.isSafeMemOpType(MVT::f64)) {
03850           // i64 is usually not legal on 32-bit targets, but f64 may be.
03851           NewVT = MVT::f64;
03852           Found = true;
03853         }
03854       }
03855 
03856       if (!Found) {
03857         do {
03858           NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
03859           if (NewVT == MVT::i8)
03860             break;
03861         } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT()));
03862       }
03863       NewVTSize = NewVT.getSizeInBits() / 8;
03864 
03865       // If the new VT cannot cover all of the remaining bits, then consider
03866       // issuing a (or a pair of) unaligned and overlapping load / store.
03867       // FIXME: Only does this for 64-bit or more since we don't have proper
03868       // cost model for unaligned load / store.
03869       bool Fast;
03870       unsigned AS = 0;
03871       if (NumMemOps && AllowOverlap &&
03872           VTSize >= 8 && NewVTSize < Size &&
03873           TLI.allowsUnalignedMemoryAccesses(VT, AS, &Fast) && Fast)
03874         VTSize = Size;
03875       else {
03876         VT = NewVT;
03877         VTSize = NewVTSize;
03878       }
03879     }
03880 
03881     if (++NumMemOps > Limit)
03882       return false;
03883 
03884     MemOps.push_back(VT);
03885     Size -= VTSize;
03886   }
03887 
03888   return true;
03889 }
03890 
03891 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
03892                                        SDValue Chain, SDValue Dst,
03893                                        SDValue Src, uint64_t Size,
03894                                        unsigned Align, bool isVol,
03895                                        bool AlwaysInline,
03896                                        MachinePointerInfo DstPtrInfo,
03897                                        MachinePointerInfo SrcPtrInfo) {
03898   // Turn a memcpy of undef to nop.
03899   if (Src.getOpcode() == ISD::UNDEF)
03900     return Chain;
03901 
03902   // Expand memcpy to a series of load and store ops if the size operand falls
03903   // below a certain threshold.
03904   // TODO: In the AlwaysInline case, if the size is big then generate a loop
03905   // rather than maybe a humongous number of loads and stores.
03906   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03907   std::vector<EVT> MemOps;
03908   bool DstAlignCanChange = false;
03909   MachineFunction &MF = DAG.getMachineFunction();
03910   MachineFrameInfo *MFI = MF.getFrameInfo();
03911   bool OptSize =
03912     MF.getFunction()->getAttributes().
03913       hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
03914   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
03915   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
03916     DstAlignCanChange = true;
03917   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
03918   if (Align > SrcAlign)
03919     SrcAlign = Align;
03920   StringRef Str;
03921   bool CopyFromStr = isMemSrcFromString(Src, Str);
03922   bool isZeroStr = CopyFromStr && Str.empty();
03923   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
03924 
03925   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
03926                                 (DstAlignCanChange ? 0 : Align),
03927                                 (isZeroStr ? 0 : SrcAlign),
03928                                 false, false, CopyFromStr, true, DAG, TLI))
03929     return SDValue();
03930 
03931   if (DstAlignCanChange) {
03932     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
03933     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
03934 
03935     // Don't promote to an alignment that would require dynamic stack
03936     // realignment.
03937     const TargetRegisterInfo *TRI = MF.getTarget().getRegisterInfo();
03938     if (!TRI->needsStackRealignment(MF))
03939        while (NewAlign > Align &&
03940              TLI.getDataLayout()->exceedsNaturalStackAlignment(NewAlign))
03941           NewAlign /= 2;
03942 
03943     if (NewAlign > Align) {
03944       // Give the stack frame object a larger alignment if needed.
03945       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
03946         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
03947       Align = NewAlign;
03948     }
03949   }
03950 
03951   SmallVector<SDValue, 8> OutChains;
03952   unsigned NumMemOps = MemOps.size();
03953   uint64_t SrcOff = 0, DstOff = 0;
03954   for (unsigned i = 0; i != NumMemOps; ++i) {
03955     EVT VT = MemOps[i];
03956     unsigned VTSize = VT.getSizeInBits() / 8;
03957     SDValue Value, Store;
03958 
03959     if (VTSize > Size) {
03960       // Issuing an unaligned load / store pair  that overlaps with the previous
03961       // pair. Adjust the offset accordingly.
03962       assert(i == NumMemOps-1 && i != 0);
03963       SrcOff -= VTSize - Size;
03964       DstOff -= VTSize - Size;
03965     }
03966 
03967     if (CopyFromStr &&
03968         (isZeroStr || (VT.isInteger() && !VT.isVector()))) {
03969       // It's unlikely a store of a vector immediate can be done in a single
03970       // instruction. It would require a load from a constantpool first.
03971       // We only handle zero vectors here.
03972       // FIXME: Handle other cases where store of vector immediate is done in
03973       // a single instruction.
03974       Value = getMemsetStringVal(VT, dl, DAG, TLI, Str.substr(SrcOff));
03975       if (Value.getNode())
03976         Store = DAG.getStore(Chain, dl, Value,
03977                              getMemBasePlusOffset(Dst, DstOff, dl, DAG),
03978                              DstPtrInfo.getWithOffset(DstOff), isVol,
03979                              false, Align);
03980     }
03981 
03982     if (!Store.getNode()) {
03983       // The type might not be legal for the target.  This should only happen
03984       // if the type is smaller than a legal type, as on PPC, so the right
03985       // thing to do is generate a LoadExt/StoreTrunc pair.  These simplify
03986       // to Load/Store if NVT==VT.
03987       // FIXME does the case above also need this?
03988       EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
03989       assert(NVT.bitsGE(VT));
03990       Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain,
03991                              getMemBasePlusOffset(Src, SrcOff, dl, DAG),
03992                              SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false,
03993                              MinAlign(SrcAlign, SrcOff));
03994       Store = DAG.getTruncStore(Chain, dl, Value,
03995                                 getMemBasePlusOffset(Dst, DstOff, dl, DAG),
03996                                 DstPtrInfo.getWithOffset(DstOff), VT, isVol,
03997                                 false, Align);
03998     }
03999     OutChains.push_back(Store);
04000     SrcOff += VTSize;
04001     DstOff += VTSize;
04002     Size -= VTSize;
04003   }
04004 
04005   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04006 }
04007 
04008 static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
04009                                         SDValue Chain, SDValue Dst,
04010                                         SDValue Src, uint64_t Size,
04011                                         unsigned Align,  bool isVol,
04012                                         bool AlwaysInline,
04013                                         MachinePointerInfo DstPtrInfo,
04014                                         MachinePointerInfo SrcPtrInfo) {
04015   // Turn a memmove of undef to nop.
04016   if (Src.getOpcode() == ISD::UNDEF)
04017     return Chain;
04018 
04019   // Expand memmove to a series of load and store ops if the size operand falls
04020   // below a certain threshold.
04021   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04022   std::vector<EVT> MemOps;
04023   bool DstAlignCanChange = false;
04024   MachineFunction &MF = DAG.getMachineFunction();
04025   MachineFrameInfo *MFI = MF.getFrameInfo();
04026   bool OptSize = MF.getFunction()->getAttributes().
04027     hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
04028   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
04029   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
04030     DstAlignCanChange = true;
04031   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
04032   if (Align > SrcAlign)
04033     SrcAlign = Align;
04034   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
04035 
04036   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
04037                                 (DstAlignCanChange ? 0 : Align), SrcAlign,
04038                                 false, false, false, false, DAG, TLI))
04039     return SDValue();
04040 
04041   if (DstAlignCanChange) {
04042     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
04043     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
04044     if (NewAlign > Align) {
04045       // Give the stack frame object a larger alignment if needed.
04046       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
04047         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
04048       Align = NewAlign;
04049     }
04050   }
04051 
04052   uint64_t SrcOff = 0, DstOff = 0;
04053   SmallVector<SDValue, 8> LoadValues;
04054   SmallVector<SDValue, 8> LoadChains;
04055   SmallVector<SDValue, 8> OutChains;
04056   unsigned NumMemOps = MemOps.size();
04057   for (unsigned i = 0; i < NumMemOps; i++) {
04058     EVT VT = MemOps[i];
04059     unsigned VTSize = VT.getSizeInBits() / 8;
04060     SDValue Value;
04061 
04062     Value = DAG.getLoad(VT, dl, Chain,
04063                         getMemBasePlusOffset(Src, SrcOff, dl, DAG),
04064                         SrcPtrInfo.getWithOffset(SrcOff), isVol,
04065                         false, false, SrcAlign);
04066     LoadValues.push_back(Value);
04067     LoadChains.push_back(Value.getValue(1));
04068     SrcOff += VTSize;
04069   }
04070   Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
04071   OutChains.clear();
04072   for (unsigned i = 0; i < NumMemOps; i++) {
04073     EVT VT = MemOps[i];
04074     unsigned VTSize = VT.getSizeInBits() / 8;
04075     SDValue Store;
04076 
04077     Store = DAG.getStore(Chain, dl, LoadValues[i],
04078                          getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04079                          DstPtrInfo.getWithOffset(DstOff), isVol, false, Align);
04080     OutChains.push_back(Store);
04081     DstOff += VTSize;
04082   }
04083 
04084   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04085 }
04086 
04087 /// \brief Lower the call to 'memset' intrinsic function into a series of store
04088 /// operations.
04089 ///
04090 /// \param DAG Selection DAG where lowered code is placed.
04091 /// \param dl Link to corresponding IR location.
04092 /// \param Chain Control flow dependency.
04093 /// \param Dst Pointer to destination memory location.
04094 /// \param Src Value of byte to write into the memory.
04095 /// \param Size Number of bytes to write.
04096 /// \param Align Alignment of the destination in bytes.
04097 /// \param isVol True if destination is volatile.
04098 /// \param DstPtrInfo IR information on the memory pointer.
04099 /// \returns New head in the control flow, if lowering was successful, empty
04100 /// SDValue otherwise.
04101 ///
04102 /// The function tries to replace 'llvm.memset' intrinsic with several store
04103 /// operations and value calculation code. This is usually profitable for small
04104 /// memory size.
04105 static SDValue getMemsetStores(SelectionDAG &DAG, SDLoc dl,
04106                                SDValue Chain, SDValue Dst,
04107                                SDValue Src, uint64_t Size,
04108                                unsigned Align, bool isVol,
04109                                MachinePointerInfo DstPtrInfo) {
04110   // Turn a memset of undef to nop.
04111   if (Src.getOpcode() == ISD::UNDEF)
04112     return Chain;
04113 
04114   // Expand memset to a series of load/store ops if the size operand
04115   // falls below a certain threshold.
04116   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04117   std::vector<EVT> MemOps;
04118   bool DstAlignCanChange = false;
04119   MachineFunction &MF = DAG.getMachineFunction();
04120   MachineFrameInfo *MFI = MF.getFrameInfo();
04121   bool OptSize = MF.getFunction()->getAttributes().
04122     hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
04123   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
04124   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
04125     DstAlignCanChange = true;
04126   bool IsZeroVal =
04127     isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
04128   if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize),
04129                                 Size, (DstAlignCanChange ? 0 : Align), 0,
04130                                 true, IsZeroVal, false, true, DAG, TLI))
04131     return SDValue();
04132 
04133   if (DstAlignCanChange) {
04134     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
04135     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
04136     if (NewAlign > Align) {
04137       // Give the stack frame object a larger alignment if needed.
04138       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
04139         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
04140       Align = NewAlign;
04141     }
04142   }
04143 
04144   SmallVector<SDValue, 8> OutChains;
04145   uint64_t DstOff = 0;
04146   unsigned NumMemOps = MemOps.size();
04147 
04148   // Find the largest store and generate the bit pattern for it.
04149   EVT LargestVT = MemOps[0];
04150   for (unsigned i = 1; i < NumMemOps; i++)
04151     if (MemOps[i].bitsGT(LargestVT))
04152       LargestVT = MemOps[i];
04153   SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
04154 
04155   for (unsigned i = 0; i < NumMemOps; i++) {
04156     EVT VT = MemOps[i];
04157     unsigned VTSize = VT.getSizeInBits() / 8;
04158     if (VTSize > Size) {
04159       // Issuing an unaligned load / store pair  that overlaps with the previous
04160       // pair. Adjust the offset accordingly.
04161       assert(i == NumMemOps-1 && i != 0);
04162       DstOff -= VTSize - Size;
04163     }
04164 
04165     // If this store is smaller than the largest store see whether we can get
04166     // the smaller value for free with a truncate.
04167     SDValue Value = MemSetValue;
04168     if (VT.bitsLT(LargestVT)) {
04169       if (!LargestVT.isVector() && !VT.isVector() &&
04170           TLI.isTruncateFree(LargestVT, VT))
04171         Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
04172       else
04173         Value = getMemsetValue(Src, VT, DAG, dl);
04174     }
04175     assert(Value.getValueType() == VT && "Value with wrong type.");
04176     SDValue Store = DAG.getStore(Chain, dl, Value,
04177                                  getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04178                                  DstPtrInfo.getWithOffset(DstOff),
04179                                  isVol, false, Align);
04180     OutChains.push_back(Store);
04181     DstOff += VT.getSizeInBits() / 8;
04182     Size -= VTSize;
04183   }
04184 
04185   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04186 }
04187 
04188 SDValue SelectionDAG::getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst,
04189                                 SDValue Src, SDValue Size,
04190                                 unsigned Align, bool isVol, bool AlwaysInline,
04191                                 MachinePointerInfo DstPtrInfo,
04192                                 MachinePointerInfo SrcPtrInfo) {
04193   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04194 
04195   // Check to see if we should lower the memcpy to loads and stores first.
04196   // For cases within the target-specified limits, this is the best choice.
04197   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04198   if (ConstantSize) {
04199     // Memcpy with size zero? Just return the original chain.
04200     if (ConstantSize->isNullValue())
04201       return Chain;
04202 
04203     SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
04204                                              ConstantSize->getZExtValue(),Align,
04205                                 isVol, false, DstPtrInfo, SrcPtrInfo);
04206     if (Result.getNode())
04207       return Result;
04208   }
04209 
04210   // Then check to see if we should lower the memcpy with target-specific
04211   // code. If the target chooses to do this, this is the next best.
04212   SDValue Result =
04213     TSI.EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align,
04214                                 isVol, AlwaysInline,
04215                                 DstPtrInfo, SrcPtrInfo);
04216   if (Result.getNode())
04217     return Result;
04218 
04219   // If we really need inline code and the target declined to provide it,
04220   // use a (potentially long) sequence of loads and stores.
04221   if (AlwaysInline) {
04222     assert(ConstantSize && "AlwaysInline requires a constant size!");
04223     return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
04224                                    ConstantSize->getZExtValue(), Align, isVol,
04225                                    true, DstPtrInfo, SrcPtrInfo);
04226   }
04227 
04228   // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
04229   // memcpy is not guaranteed to be safe. libc memcpys aren't required to
04230   // respect volatile, so they may do things like read or write memory
04231   // beyond the given memory regions. But fixing this isn't easy, and most
04232   // people don't care.
04233 
04234   const TargetLowering *TLI = TM.getTargetLowering();
04235 
04236   // Emit a library call.
04237   TargetLowering::ArgListTy Args;
04238   TargetLowering::ArgListEntry Entry;
04239   Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
04240   Entry.Node = Dst; Args.push_back(Entry);
04241   Entry.Node = Src; Args.push_back(Entry);
04242   Entry.Node = Size; Args.push_back(Entry);
04243   // FIXME: pass in SDLoc
04244   TargetLowering::CallLoweringInfo CLI(*this);
04245   CLI.setDebugLoc(dl).setChain(Chain)
04246     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
04247                Type::getVoidTy(*getContext()),
04248                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
04249                                  TLI->getPointerTy()), std::move(Args), 0)
04250     .setDiscardResult();
04251   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04252 
04253   return CallResult.second;
04254 }
04255 
04256 SDValue SelectionDAG::getMemmove(SDValue Chain, SDLoc dl, SDValue Dst,
04257                                  SDValue Src, SDValue Size,
04258                                  unsigned Align, bool isVol,
04259                                  MachinePointerInfo DstPtrInfo,
04260                                  MachinePointerInfo SrcPtrInfo) {
04261   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04262 
04263   // Check to see if we should lower the memmove to loads and stores first.
04264   // For cases within the target-specified limits, this is the best choice.
04265   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04266   if (ConstantSize) {
04267     // Memmove with size zero? Just return the original chain.
04268     if (ConstantSize->isNullValue())
04269       return Chain;
04270 
04271     SDValue Result =
04272       getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src,
04273                                ConstantSize->getZExtValue(), Align, isVol,
04274                                false, DstPtrInfo, SrcPtrInfo);
04275     if (Result.getNode())
04276       return Result;
04277   }
04278 
04279   // Then check to see if we should lower the memmove with target-specific
04280   // code. If the target chooses to do this, this is the next best.
04281   SDValue Result =
04282     TSI.EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size, Align, isVol,
04283                                  DstPtrInfo, SrcPtrInfo);
04284   if (Result.getNode())
04285     return Result;
04286 
04287   // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
04288   // not be safe.  See memcpy above for more details.
04289 
04290   const TargetLowering *TLI = TM.getTargetLowering();
04291 
04292   // Emit a library call.
04293   TargetLowering::ArgListTy Args;
04294   TargetLowering::ArgListEntry Entry;
04295   Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
04296   Entry.Node = Dst; Args.push_back(Entry);
04297   Entry.Node = Src; Args.push_back(Entry);
04298   Entry.Node = Size; Args.push_back(Entry);
04299   // FIXME:  pass in SDLoc
04300   TargetLowering::CallLoweringInfo CLI(*this);
04301   CLI.setDebugLoc(dl).setChain(Chain)
04302     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
04303                Type::getVoidTy(*getContext()),
04304                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
04305                                  TLI->getPointerTy()), std::move(Args), 0)
04306     .setDiscardResult();
04307   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04308 
04309   return CallResult.second;
04310 }
04311 
04312 SDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst,
04313                                 SDValue Src, SDValue Size,
04314                                 unsigned Align, bool isVol,
04315                                 MachinePointerInfo DstPtrInfo) {
04316   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04317 
04318   // Check to see if we should lower the memset to stores first.
04319   // For cases within the target-specified limits, this is the best choice.
04320   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04321   if (ConstantSize) {
04322     // Memset with size zero? Just return the original chain.
04323     if (ConstantSize->isNullValue())
04324       return Chain;
04325 
04326     SDValue Result =
04327       getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
04328                       Align, isVol, DstPtrInfo);
04329 
04330     if (Result.getNode())
04331       return Result;
04332   }
04333 
04334   // Then check to see if we should lower the memset with target-specific
04335   // code. If the target chooses to do this, this is the next best.
04336   SDValue Result =
04337     TSI.EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src, Size, Align, isVol,
04338                                 DstPtrInfo);
04339   if (Result.getNode())
04340     return Result;
04341 
04342   // Emit a library call.
04343   const TargetLowering *TLI = TM.getTargetLowering();
04344   Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(*getContext());
04345   TargetLowering::ArgListTy Args;
04346   TargetLowering::ArgListEntry Entry;
04347   Entry.Node = Dst; Entry.Ty = IntPtrTy;
04348   Args.push_back(Entry);
04349   // Extend or truncate the argument to be an i32 value for the call.
04350   if (Src.getValueType().bitsGT(MVT::i32))
04351     Src = getNode(ISD::TRUNCATE, dl, MVT::i32, Src);
04352   else
04353     Src = getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Src);
04354   Entry.Node = Src;
04355   Entry.Ty = Type::getInt32Ty(*getContext());
04356   Entry.isSExt = true;
04357   Args.push_back(Entry);
04358   Entry.Node = Size;
04359   Entry.Ty = IntPtrTy;
04360   Entry.isSExt = false;
04361   Args.push_back(Entry);
04362 
04363   // FIXME: pass in SDLoc
04364   TargetLowering::CallLoweringInfo CLI(*this);
04365   CLI.setDebugLoc(dl).setChain(Chain)
04366     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
04367                Type::getVoidTy(*getContext()),
04368                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
04369                                  TLI->getPointerTy()), std::move(Args), 0)
04370     .setDiscardResult();
04371 
04372   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04373   return CallResult.second;
04374 }
04375 
04376 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04377                                 SDVTList VTList, ArrayRef<SDValue> Ops,
04378                                 MachineMemOperand *MMO,
04379                                 AtomicOrdering SuccessOrdering,
04380                                 AtomicOrdering FailureOrdering,
04381                                 SynchronizationScope SynchScope) {
04382   FoldingSetNodeID ID;
04383   ID.AddInteger(MemVT.getRawBits());
04384   AddNodeIDNode(ID, Opcode, VTList, Ops);
04385   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04386   void* IP = nullptr;
04387   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04388     cast<AtomicSDNode>(E)->refineAlignment(MMO);
04389     return SDValue(E, 0);
04390   }
04391 
04392   // Allocate the operands array for the node out of the BumpPtrAllocator, since
04393   // SDNode doesn't have access to it.  This memory will be "leaked" when
04394   // the node is deallocated, but recovered when the allocator is released.
04395   // If the number of operands is less than 5 we use AtomicSDNode's internal
04396   // storage.
04397   unsigned NumOps = Ops.size();
04398   SDUse *DynOps = NumOps > 4 ? OperandAllocator.Allocate<SDUse>(NumOps)
04399                              : nullptr;
04400 
04401   SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(),
04402                                                dl.getDebugLoc(), VTList, MemVT,
04403                                                Ops.data(), DynOps, NumOps, MMO,
04404                                                SuccessOrdering, FailureOrdering,
04405                                                SynchScope);
04406   CSEMap.InsertNode(N, IP);
04407   InsertNode(N);
04408   return SDValue(N, 0);
04409 }
04410 
04411 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04412                                 SDVTList VTList, ArrayRef<SDValue> Ops,
04413                                 MachineMemOperand *MMO,
04414                                 AtomicOrdering Ordering,
04415                                 SynchronizationScope SynchScope) {
04416   return getAtomic(Opcode, dl, MemVT, VTList, Ops, MMO, Ordering,
04417                    Ordering, SynchScope);
04418 }
04419 
04420 SDValue SelectionDAG::getAtomicCmpSwap(
04421     unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs, SDValue Chain,
04422     SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo,
04423     unsigned Alignment, AtomicOrdering SuccessOrdering,
04424     AtomicOrdering FailureOrdering, SynchronizationScope SynchScope) {
04425   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
04426          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
04427   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
04428 
04429   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04430     Alignment = getEVTAlignment(MemVT);
04431 
04432   MachineFunction &MF = getMachineFunction();
04433 
04434   // FIXME: Volatile isn't really correct; we should keep track of atomic
04435   // orderings in the memoperand.
04436   unsigned Flags = MachineMemOperand::MOVolatile;
04437   Flags |= MachineMemOperand::MOLoad;
04438   Flags |= MachineMemOperand::MOStore;
04439 
04440   MachineMemOperand *MMO =
04441     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
04442 
04443   return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO,
04444                           SuccessOrdering, FailureOrdering, SynchScope);
04445 }
04446 
04447 SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, SDLoc dl, EVT MemVT,
04448                                        SDVTList VTs, SDValue Chain, SDValue Ptr,
04449                                        SDValue Cmp, SDValue Swp,
04450                                        MachineMemOperand *MMO,
04451                                        AtomicOrdering SuccessOrdering,
04452                                        AtomicOrdering FailureOrdering,
04453                                        SynchronizationScope SynchScope) {
04454   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
04455          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
04456   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
04457 
04458   SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
04459   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO,
04460                    SuccessOrdering, FailureOrdering, SynchScope);
04461 }
04462 
04463 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04464                                 SDValue Chain,
04465                                 SDValue Ptr, SDValue Val,
04466                                 const Value* PtrVal,
04467                                 unsigned Alignment,
04468                                 AtomicOrdering Ordering,
04469                                 SynchronizationScope SynchScope) {
04470   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04471     Alignment = getEVTAlignment(MemVT);
04472 
04473   MachineFunction &MF = getMachineFunction();
04474   // An atomic store does not load. An atomic load does not store.
04475   // (An atomicrmw obviously both loads and stores.)
04476   // For now, atomics are considered to be volatile always, and they are
04477   // chained as such.
04478   // FIXME: Volatile isn't really correct; we should keep track of atomic
04479   // orderings in the memoperand.
04480   unsigned Flags = MachineMemOperand::MOVolatile;
04481   if (Opcode != ISD::ATOMIC_STORE)
04482     Flags |= MachineMemOperand::MOLoad;
04483   if (Opcode != ISD::ATOMIC_LOAD)
04484     Flags |= MachineMemOperand::MOStore;
04485 
04486   MachineMemOperand *MMO =
04487     MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
04488                             MemVT.getStoreSize(), Alignment);
04489 
04490   return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO,
04491                    Ordering, SynchScope);
04492 }
04493 
04494 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04495                                 SDValue Chain,
04496                                 SDValue Ptr, SDValue Val,
04497                                 MachineMemOperand *MMO,
04498                                 AtomicOrdering Ordering,
04499                                 SynchronizationScope SynchScope) {
04500   assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
04501           Opcode == ISD::ATOMIC_LOAD_SUB ||
04502           Opcode == ISD::ATOMIC_LOAD_AND ||
04503           Opcode == ISD::ATOMIC_LOAD_OR ||
04504           Opcode == ISD::ATOMIC_LOAD_XOR ||
04505           Opcode == ISD::ATOMIC_LOAD_NAND ||
04506           Opcode == ISD::ATOMIC_LOAD_MIN ||
04507           Opcode == ISD::ATOMIC_LOAD_MAX ||
04508           Opcode == ISD::ATOMIC_LOAD_UMIN ||
04509           Opcode == ISD::ATOMIC_LOAD_UMAX ||
04510           Opcode == ISD::ATOMIC_SWAP ||
04511           Opcode == ISD::ATOMIC_STORE) &&
04512          "Invalid Atomic Op");
04513 
04514   EVT VT = Val.getValueType();
04515 
04516   SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
04517                                                getVTList(VT, MVT::Other);
04518   SDValue Ops[] = {Chain, Ptr, Val};
04519   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
04520 }
04521 
04522 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04523                                 EVT VT, SDValue Chain,
04524                                 SDValue Ptr,
04525                                 MachineMemOperand *MMO,
04526                                 AtomicOrdering Ordering,
04527                                 SynchronizationScope SynchScope) {
04528   assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
04529 
04530   SDVTList VTs = getVTList(VT, MVT::Other);
04531   SDValue Ops[] = {Chain, Ptr};
04532   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
04533 }
04534 
04535 /// getMergeValues - Create a MERGE_VALUES node from the given operands.
04536 SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, SDLoc dl) {
04537   if (Ops.size() == 1)
04538     return Ops[0];
04539 
04540   SmallVector<EVT, 4> VTs;
04541   VTs.reserve(Ops.size());
04542   for (unsigned i = 0; i < Ops.size(); ++i)
04543     VTs.push_back(Ops[i].getValueType());
04544   return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
04545 }
04546 
04547 SDValue
04548 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
04549                                   ArrayRef<SDValue> Ops,
04550                                   EVT MemVT, MachinePointerInfo PtrInfo,
04551                                   unsigned Align, bool Vol,
04552                                   bool ReadMem, bool WriteMem) {
04553   if (Align == 0)  // Ensure that codegen never sees alignment 0
04554     Align = getEVTAlignment(MemVT);
04555 
04556   MachineFunction &MF = getMachineFunction();
04557   unsigned Flags = 0;
04558   if (WriteMem)
04559     Flags |= MachineMemOperand::MOStore;
04560   if (ReadMem)
04561     Flags |= MachineMemOperand::MOLoad;
04562   if (Vol)
04563     Flags |= MachineMemOperand::MOVolatile;
04564   MachineMemOperand *MMO =
04565     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Align);
04566 
04567   return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
04568 }
04569 
04570 SDValue
04571 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
04572                                   ArrayRef<SDValue> Ops, EVT MemVT,
04573                                   MachineMemOperand *MMO) {
04574   assert((Opcode == ISD::INTRINSIC_VOID ||
04575           Opcode == ISD::INTRINSIC_W_CHAIN ||
04576           Opcode == ISD::PREFETCH ||
04577           Opcode == ISD::LIFETIME_START ||
04578           Opcode == ISD::LIFETIME_END ||
04579           (Opcode <= INT_MAX &&
04580            (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
04581          "Opcode is not a memory-accessing opcode!");
04582 
04583   // Memoize the node unless it returns a flag.
04584   MemIntrinsicSDNode *N;
04585   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
04586     FoldingSetNodeID ID;
04587     AddNodeIDNode(ID, Opcode, VTList, Ops);
04588     ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04589     void *IP = nullptr;
04590     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04591       cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
04592       return SDValue(E, 0);
04593     }
04594 
04595     N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
04596                                                dl.getDebugLoc(), VTList, Ops,
04597                                                MemVT, MMO);
04598     CSEMap.InsertNode(N, IP);
04599   } else {
04600     N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
04601                                                dl.getDebugLoc(), VTList, Ops,
04602                                                MemVT, MMO);
04603   }
04604   InsertNode(N);
04605   return SDValue(N, 0);
04606 }
04607 
04608 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
04609 /// MachinePointerInfo record from it.  This is particularly useful because the
04610 /// code generator has many cases where it doesn't bother passing in a
04611 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
04612 static MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) {
04613   // If this is FI+Offset, we can model it.
04614   if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
04615     return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset);
04616 
04617   // If this is (FI+Offset1)+Offset2, we can model it.
04618   if (Ptr.getOpcode() != ISD::ADD ||
04619       !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
04620       !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
04621     return MachinePointerInfo();
04622 
04623   int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
04624   return MachinePointerInfo::getFixedStack(FI, Offset+
04625                        cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
04626 }
04627 
04628 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
04629 /// MachinePointerInfo record from it.  This is particularly useful because the
04630 /// code generator has many cases where it doesn't bother passing in a
04631 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
04632 static MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) {
04633   // If the 'Offset' value isn't a constant, we can't handle this.
04634   if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
04635     return InferPointerInfo(Ptr, OffsetNode->getSExtValue());
04636   if (OffsetOp.getOpcode() == ISD::UNDEF)
04637     return InferPointerInfo(Ptr);
04638   return MachinePointerInfo();
04639 }
04640 
04641 
04642 SDValue
04643 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
04644                       EVT VT, SDLoc dl, SDValue Chain,
04645                       SDValue Ptr, SDValue Offset,
04646                       MachinePointerInfo PtrInfo, EVT MemVT,
04647                       bool isVolatile, bool isNonTemporal, bool isInvariant,
04648                       unsigned Alignment, const AAMDNodes &AAInfo,
04649                       const MDNode *Ranges) {
04650   assert(Chain.getValueType() == MVT::Other &&
04651         "Invalid chain type");
04652   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04653     Alignment = getEVTAlignment(VT);
04654 
04655   unsigned Flags = MachineMemOperand::MOLoad;
04656   if (isVolatile)
04657     Flags |= MachineMemOperand::MOVolatile;
04658   if (isNonTemporal)
04659     Flags |= MachineMemOperand::MONonTemporal;
04660   if (isInvariant)
04661     Flags |= MachineMemOperand::MOInvariant;
04662 
04663   // If we don't have a PtrInfo, infer the trivial frame index case to simplify
04664   // clients.
04665   if (PtrInfo.V.isNull())
04666     PtrInfo = InferPointerInfo(Ptr, Offset);
04667 
04668   MachineFunction &MF = getMachineFunction();
04669   MachineMemOperand *MMO =
04670     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment,
04671                             AAInfo, Ranges);
04672   return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
04673 }
04674 
04675 SDValue
04676 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
04677                       EVT VT, SDLoc dl, SDValue Chain,
04678                       SDValue Ptr, SDValue Offset, EVT MemVT,
04679                       MachineMemOperand *MMO) {
04680   if (VT == MemVT) {
04681     ExtType = ISD::NON_EXTLOAD;
04682   } else if (ExtType == ISD::NON_EXTLOAD) {
04683     assert(VT == MemVT && "Non-extending load from different memory type!");
04684   } else {
04685     // Extending load.
04686     assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
04687            "Should only be an extending load, not truncating!");
04688     assert(VT.isInteger() == MemVT.isInteger() &&
04689            "Cannot convert from FP to Int or Int -> FP!");
04690     assert(VT.isVector() == MemVT.isVector() &&
04691            "Cannot use trunc store to convert to or from a vector!");
04692     assert((!VT.isVector() ||
04693             VT.getVectorNumElements() == MemVT.getVectorNumElements()) &&
04694            "Cannot use trunc store to change the number of vector elements!");
04695   }
04696 
04697   bool Indexed = AM != ISD::UNINDEXED;
04698   assert((Indexed || Offset.getOpcode() == ISD::UNDEF) &&
04699          "Unindexed load with an offset!");
04700 
04701   SDVTList VTs = Indexed ?
04702     getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
04703   SDValue Ops[] = { Chain, Ptr, Offset };
04704   FoldingSetNodeID ID;
04705   AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
04706   ID.AddInteger(MemVT.getRawBits());
04707   ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(),
04708                                      MMO->isNonTemporal(),
04709                                      MMO->isInvariant()));
04710   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04711   void *IP = nullptr;
04712   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04713     cast<LoadSDNode>(E)->refineAlignment(MMO);
04714     return SDValue(E, 0);
04715   }
04716   SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl.getIROrder(),
04717                                              dl.getDebugLoc(), VTs, AM, ExtType,
04718                                              MemVT, MMO);
04719   CSEMap.InsertNode(N, IP);
04720   InsertNode(N);
04721   return SDValue(N, 0);
04722 }
04723 
04724 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
04725                               SDValue Chain, SDValue Ptr,
04726                               MachinePointerInfo PtrInfo,
04727                               bool isVolatile, bool isNonTemporal,
04728                               bool isInvariant, unsigned Alignment,
04729                               const AAMDNodes &AAInfo,
04730                               const MDNode *Ranges) {
04731   SDValue Undef = getUNDEF(Ptr.getValueType());
04732   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
04733                  PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment,
04734                  AAInfo, Ranges);
04735 }
04736 
04737 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
04738                               SDValue Chain, SDValue Ptr,
04739                               MachineMemOperand *MMO) {
04740   SDValue Undef = getUNDEF(Ptr.getValueType());
04741   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
04742                  VT, MMO);
04743 }
04744 
04745 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
04746                                  SDValue Chain, SDValue Ptr,
04747                                  MachinePointerInfo PtrInfo, EVT MemVT,
04748                                  bool isVolatile, bool isNonTemporal,
04749                                  unsigned Alignment, const AAMDNodes &AAInfo) {
04750   SDValue Undef = getUNDEF(Ptr.getValueType());
04751   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
04752                  PtrInfo, MemVT, isVolatile, isNonTemporal, false, Alignment,
04753                  AAInfo);
04754 }
04755 
04756 
04757 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
04758                                  SDValue Chain, SDValue Ptr, EVT MemVT,
04759                                  MachineMemOperand *MMO) {
04760   SDValue Undef = getUNDEF(Ptr.getValueType());
04761   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
04762                  MemVT, MMO);
04763 }
04764 
04765 SDValue
04766 SelectionDAG::getIndexedLoad(SDValue OrigLoad, SDLoc dl, SDValue Base,
04767                              SDValue Offset, ISD::MemIndexedMode AM) {
04768   LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
04769   assert(LD->getOffset().getOpcode() == ISD::UNDEF &&
04770          "Load is already a indexed load!");
04771   return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
04772                  LD->getChain(), Base, Offset, LD->getPointerInfo(),
04773                  LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(),
04774                  false, LD->getAlignment());
04775 }
04776 
04777 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
04778                                SDValue Ptr, MachinePointerInfo PtrInfo,
04779                                bool isVolatile, bool isNonTemporal,
04780                                unsigned Alignment, const AAMDNodes &AAInfo) {
04781   assert(Chain.getValueType() == MVT::Other &&
04782         "Invalid chain type");
04783   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04784     Alignment = getEVTAlignment(Val.getValueType());
04785 
04786   unsigned Flags = MachineMemOperand::MOStore;
04787   if (isVolatile)
04788     Flags |= MachineMemOperand::MOVolatile;
04789   if (isNonTemporal)
04790     Flags |= MachineMemOperand::MONonTemporal;
04791 
04792   if (PtrInfo.V.isNull())
04793     PtrInfo = InferPointerInfo(Ptr);
04794 
04795   MachineFunction &MF = getMachineFunction();
04796   MachineMemOperand *MMO =
04797     MF.getMachineMemOperand(PtrInfo, Flags,
04798                             Val.getValueType().getStoreSize(), Alignment,
04799                             AAInfo);
04800 
04801   return getStore(Chain, dl, Val, Ptr, MMO);
04802 }
04803 
04804 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
04805                                SDValue Ptr, MachineMemOperand *MMO) {
04806   assert(Chain.getValueType() == MVT::Other &&
04807         "Invalid chain type");
04808   EVT VT = Val.getValueType();
04809   SDVTList VTs = getVTList(MVT::Other);
04810   SDValue Undef = getUNDEF(Ptr.getValueType());
04811   SDValue Ops[] = { Chain, Val, Ptr, Undef };
04812   FoldingSetNodeID ID;
04813   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04814   ID.AddInteger(VT.getRawBits());
04815   ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
04816                                      MMO->isNonTemporal(), MMO->isInvariant()));
04817   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04818   void *IP = nullptr;
04819   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04820     cast<StoreSDNode>(E)->refineAlignment(MMO);
04821     return SDValue(E, 0);
04822   }
04823   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04824                                               dl.getDebugLoc(), VTs,
04825                                               ISD::UNINDEXED, false, VT, MMO);
04826   CSEMap.InsertNode(N, IP);
04827   InsertNode(N);
04828   return SDValue(N, 0);
04829 }
04830 
04831 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
04832                                     SDValue Ptr, MachinePointerInfo PtrInfo,
04833                                     EVT SVT,bool isVolatile, bool isNonTemporal,
04834                                     unsigned Alignment,
04835                                     const AAMDNodes &AAInfo) {
04836   assert(Chain.getValueType() == MVT::Other &&
04837         "Invalid chain type");
04838   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04839     Alignment = getEVTAlignment(SVT);
04840 
04841   unsigned Flags = MachineMemOperand::MOStore;
04842   if (isVolatile)
04843     Flags |= MachineMemOperand::MOVolatile;
04844   if (isNonTemporal)
04845     Flags |= MachineMemOperand::MONonTemporal;
04846 
04847   if (PtrInfo.V.isNull())
04848     PtrInfo = InferPointerInfo(Ptr);
04849 
04850   MachineFunction &MF = getMachineFunction();
04851   MachineMemOperand *MMO =
04852     MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment,
04853                             AAInfo);
04854 
04855   return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
04856 }
04857 
04858 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
04859                                     SDValue Ptr, EVT SVT,
04860                                     MachineMemOperand *MMO) {
04861   EVT VT = Val.getValueType();
04862 
04863   assert(Chain.getValueType() == MVT::Other &&
04864         "Invalid chain type");
04865   if (VT == SVT)
04866     return getStore(Chain, dl, Val, Ptr, MMO);
04867 
04868   assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
04869          "Should only be a truncating store, not extending!");
04870   assert(VT.isInteger() == SVT.isInteger() &&
04871          "Can't do FP-INT conversion!");
04872   assert(VT.isVector() == SVT.isVector() &&
04873          "Cannot use trunc store to convert to or from a vector!");
04874   assert((!VT.isVector() ||
04875           VT.getVectorNumElements() == SVT.getVectorNumElements()) &&
04876          "Cannot use trunc store to change the number of vector elements!");
04877 
04878   SDVTList VTs = getVTList(MVT::Other);
04879   SDValue Undef = getUNDEF(Ptr.getValueType());
04880   SDValue Ops[] = { Chain, Val, Ptr, Undef };
04881   FoldingSetNodeID ID;
04882   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04883   ID.AddInteger(SVT.getRawBits());
04884   ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(),
04885                                      MMO->isNonTemporal(), MMO->isInvariant()));
04886   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04887   void *IP = nullptr;
04888   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04889     cast<StoreSDNode>(E)->refineAlignment(MMO);
04890     return SDValue(E, 0);
04891   }
04892   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04893                                               dl.getDebugLoc(), VTs,
04894                                               ISD::UNINDEXED, true, SVT, MMO);
04895   CSEMap.InsertNode(N, IP);
04896   InsertNode(N);
04897   return SDValue(N, 0);
04898 }
04899 
04900 SDValue
04901 SelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base,
04902                               SDValue Offset, ISD::MemIndexedMode AM) {
04903   StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
04904   assert(ST->getOffset().getOpcode() == ISD::UNDEF &&
04905          "Store is already a indexed store!");
04906   SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
04907   SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
04908   FoldingSetNodeID ID;
04909   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04910   ID.AddInteger(ST->getMemoryVT().getRawBits());
04911   ID.AddInteger(ST->getRawSubclassData());
04912   ID.AddInteger(ST->getPointerInfo().getAddrSpace());
04913   void *IP = nullptr;
04914   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
04915     return SDValue(E, 0);
04916 
04917   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04918                                               dl.getDebugLoc(), VTs, AM,
04919                                               ST->isTruncatingStore(),
04920                                               ST->getMemoryVT(),
04921                                               ST->getMemOperand());
04922   CSEMap.InsertNode(N, IP);
04923   InsertNode(N);
04924   return SDValue(N, 0);
04925 }
04926 
04927 SDValue SelectionDAG::getVAArg(EVT VT, SDLoc dl,
04928                                SDValue Chain, SDValue Ptr,
04929                                SDValue SV,
04930                                unsigned Align) {
04931   SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) };
04932   return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
04933 }
04934 
04935 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
04936                               ArrayRef<SDUse> Ops) {
04937   switch (Ops.size()) {
04938   case 0: return getNode(Opcode, DL, VT);
04939   case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
04940   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
04941   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
04942   default: break;
04943   }
04944 
04945   // Copy from an SDUse array into an SDValue array for use with
04946   // the regular getNode logic.
04947   SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
04948   return getNode(Opcode, DL, VT, NewOps);
04949 }
04950 
04951 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
04952                               ArrayRef<SDValue> Ops) {
04953   unsigned NumOps = Ops.size();
04954   switch (NumOps) {
04955   case 0: return getNode(Opcode, DL, VT);
04956   case 1: return getNode(Opcode, DL, VT, Ops[0]);
04957   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
04958   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
04959   default: break;
04960   }
04961 
04962   switch (Opcode) {
04963   default: break;
04964   case ISD::SELECT_CC: {
04965     assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
04966     assert(Ops[0].getValueType() == Ops[1].getValueType() &&
04967            "LHS and RHS of condition must have same type!");
04968     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
04969            "True and False arms of SelectCC must have same type!");
04970     assert(Ops[2].getValueType() == VT &&
04971            "select_cc node must be of same type as true and false value!");
04972     break;
04973   }
04974   case ISD::BR_CC: {
04975     assert(NumOps == 5 && "BR_CC takes 5 operands!");
04976     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
04977            "LHS/RHS of comparison should match types!");
04978     break;
04979   }
04980   }
04981 
04982   // Memoize nodes.
04983   SDNode *N;
04984   SDVTList VTs = getVTList(VT);
04985 
04986   if (VT != MVT::Glue) {
04987     FoldingSetNodeID ID;
04988     AddNodeIDNode(ID, Opcode, VTs, Ops);
04989     void *IP = nullptr;
04990 
04991     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
04992       return SDValue(E, 0);
04993 
04994     N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
04995                                    VTs, Ops);
04996     CSEMap.InsertNode(N, IP);
04997   } else {
04998     N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
04999                                    VTs, Ops);
05000   }
05001 
05002   InsertNode(N);
05003   return SDValue(N, 0);
05004 }
05005 
05006 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
05007                               ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
05008   return getNode(Opcode, DL, getVTList(ResultTys), Ops);
05009 }
05010 
05011 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05012                               ArrayRef<SDValue> Ops) {
05013   if (VTList.NumVTs == 1)
05014     return getNode(Opcode, DL, VTList.VTs[0], Ops);
05015 
05016 #if 0
05017   switch (Opcode) {
05018   // FIXME: figure out how to safely handle things like
05019   // int foo(int x) { return 1 << (x & 255); }
05020   // int bar() { return foo(256); }
05021   case ISD::SRA_PARTS:
05022   case ISD::SRL_PARTS:
05023   case ISD::SHL_PARTS:
05024     if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
05025         cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
05026       return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
05027     else if (N3.getOpcode() == ISD::AND)
05028       if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
05029         // If the and is only masking out bits that cannot effect the shift,
05030         // eliminate the and.
05031         unsigned NumBits = VT.getScalarType().getSizeInBits()*2;
05032         if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
05033           return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
05034       }
05035     break;
05036   }
05037 #endif
05038 
05039   // Memoize the node unless it returns a flag.
05040   SDNode *N;
05041   unsigned NumOps = Ops.size();
05042   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
05043     FoldingSetNodeID ID;
05044     AddNodeIDNode(ID, Opcode, VTList, Ops);
05045     void *IP = nullptr;
05046     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05047       return SDValue(E, 0);
05048 
05049     if (NumOps == 1) {
05050       N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
05051                                           DL.getDebugLoc(), VTList, Ops[0]);
05052     } else if (NumOps == 2) {
05053       N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
05054                                            DL.getDebugLoc(), VTList, Ops[0],
05055                                            Ops[1]);
05056     } else if (NumOps == 3) {
05057       N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
05058                                             DL.getDebugLoc(), VTList, Ops[0],
05059                                             Ops[1], Ops[2]);
05060     } else {
05061       N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05062                                      VTList, Ops);
05063     }
05064     CSEMap.InsertNode(N, IP);
05065   } else {
05066     if (NumOps == 1) {
05067       N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
05068                                           DL.getDebugLoc(), VTList, Ops[0]);
05069     } else if (NumOps == 2) {
05070       N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
05071                                            DL.getDebugLoc(), VTList, Ops[0],
05072                                            Ops[1]);
05073     } else if (NumOps == 3) {
05074       N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
05075                                             DL.getDebugLoc(), VTList, Ops[0],
05076                                             Ops[1], Ops[2]);
05077     } else {
05078       N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05079                                      VTList, Ops);
05080     }
05081   }
05082   InsertNode(N);
05083   return SDValue(N, 0);
05084 }
05085 
05086 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList) {
05087   return getNode(Opcode, DL, VTList, ArrayRef<SDValue>());
05088 }
05089 
05090 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05091                               SDValue N1) {
05092   SDValue Ops[] = { N1 };
05093   return getNode(Opcode, DL, VTList, Ops);
05094 }
05095 
05096 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05097                               SDValue N1, SDValue N2) {
05098   SDValue Ops[] = { N1, N2 };
05099   return getNode(Opcode, DL, VTList, Ops);
05100 }
05101 
05102 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05103                               SDValue N1, SDValue N2, SDValue N3) {
05104   SDValue Ops[] = { N1, N2, N3 };
05105   return getNode(Opcode, DL, VTList, Ops);
05106 }
05107 
05108 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05109                               SDValue N1, SDValue N2, SDValue N3,
05110                               SDValue N4) {
05111   SDValue Ops[] = { N1, N2, N3, N4 };
05112   return getNode(Opcode, DL, VTList, Ops);
05113 }
05114 
05115 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05116                               SDValue N1, SDValue N2, SDValue N3,
05117                               SDValue N4, SDValue N5) {
05118   SDValue Ops[] = { N1, N2, N3, N4, N5 };
05119   return getNode(Opcode, DL, VTList, Ops);
05120 }
05121 
05122 SDVTList SelectionDAG::getVTList(EVT VT) {
05123   return makeVTList(SDNode::getValueTypeList(VT), 1);
05124 }
05125 
05126 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
05127   FoldingSetNodeID ID;
05128   ID.AddInteger(2U);
05129   ID.AddInteger(VT1.getRawBits());
05130   ID.AddInteger(VT2.getRawBits());
05131 
05132   void *IP = nullptr;
05133   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05134   if (!Result) {
05135     EVT *Array = Allocator.Allocate<EVT>(2);
05136     Array[0] = VT1;
05137     Array[1] = VT2;
05138     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
05139     VTListMap.InsertNode(Result, IP);
05140   }
05141   return Result->getSDVTList();
05142 }
05143 
05144 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
05145   FoldingSetNodeID ID;
05146   ID.AddInteger(3U);
05147   ID.AddInteger(VT1.getRawBits());
05148   ID.AddInteger(VT2.getRawBits());
05149   ID.AddInteger(VT3.getRawBits());
05150 
05151   void *IP = nullptr;
05152   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05153   if (!Result) {
05154     EVT *Array = Allocator.Allocate<EVT>(3);
05155     Array[0] = VT1;
05156     Array[1] = VT2;
05157     Array[2] = VT3;
05158     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
05159     VTListMap.InsertNode(Result, IP);
05160   }
05161   return Result->getSDVTList();
05162 }
05163 
05164 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
05165   FoldingSetNodeID ID;
05166   ID.AddInteger(4U);
05167   ID.AddInteger(VT1.getRawBits());
05168   ID.AddInteger(VT2.getRawBits());
05169   ID.AddInteger(VT3.getRawBits());
05170   ID.AddInteger(VT4.getRawBits());
05171 
05172   void *IP = nullptr;
05173   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05174   if (!Result) {
05175     EVT *Array = Allocator.Allocate<EVT>(4);
05176     Array[0] = VT1;
05177     Array[1] = VT2;
05178     Array[2] = VT3;
05179     Array[3] = VT4;
05180     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
05181     VTListMap.InsertNode(Result, IP);
05182   }
05183   return Result->getSDVTList();
05184 }
05185 
05186 SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
05187   unsigned NumVTs = VTs.size();
05188   FoldingSetNodeID ID;
05189   ID.AddInteger(NumVTs);
05190   for (unsigned index = 0; index < NumVTs; index++) {
05191     ID.AddInteger(VTs[index].getRawBits());
05192   }
05193 
05194   void *IP = nullptr;
05195   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05196   if (!Result) {
05197     EVT *Array = Allocator.Allocate<EVT>(NumVTs);
05198     std::copy(VTs.begin(), VTs.end(), Array);
05199     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
05200     VTListMap.InsertNode(Result, IP);
05201   }
05202   return Result->getSDVTList();
05203 }
05204 
05205 
05206 /// UpdateNodeOperands - *Mutate* the specified node in-place to have the
05207 /// specified operands.  If the resultant node already exists in the DAG,
05208 /// this does not modify the specified node, instead it returns the node that
05209 /// already exists.  If the resultant node does not exist in the DAG, the
05210 /// input node is returned.  As a degenerate case, if you specify the same
05211 /// input operands as the node already has, the input node is returned.
05212 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
05213   assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
05214 
05215   // Check to see if there is no change.
05216   if (Op == N->getOperand(0)) return N;
05217 
05218   // See if the modified node already exists.
05219   void *InsertPos = nullptr;
05220   if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
05221     return Existing;
05222 
05223   // Nope it doesn't.  Remove the node from its current place in the maps.
05224   if (InsertPos)
05225     if (!RemoveNodeFromCSEMaps(N))
05226       InsertPos = nullptr;
05227 
05228   // Now we update the operands.
05229   N->OperandList[0].set(Op);
05230 
05231   // If this gets put into a CSE map, add it.
05232   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05233   return N;
05234 }
05235 
05236 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
05237   assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
05238 
05239   // Check to see if there is no change.
05240   if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
05241     return N;   // No operands changed, just return the input node.
05242 
05243   // See if the modified node already exists.
05244   void *InsertPos = nullptr;
05245   if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
05246     return Existing;
05247 
05248   // Nope it doesn't.  Remove the node from its current place in the maps.
05249   if (InsertPos)
05250     if (!RemoveNodeFromCSEMaps(N))
05251       InsertPos = nullptr;
05252 
05253   // Now we update the operands.
05254   if (N->OperandList[0] != Op1)
05255     N->OperandList[0].set(Op1);
05256   if (N->OperandList[1] != Op2)
05257     N->OperandList[1].set(Op2);
05258 
05259   // If this gets put into a CSE map, add it.
05260   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05261   return N;
05262 }
05263 
05264 SDNode *SelectionDAG::
05265 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
05266   SDValue Ops[] = { Op1, Op2, Op3 };
05267   return UpdateNodeOperands(N, Ops);
05268 }
05269 
05270 SDNode *SelectionDAG::
05271 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
05272                    SDValue Op3, SDValue Op4) {
05273   SDValue Ops[] = { Op1, Op2, Op3, Op4 };
05274   return UpdateNodeOperands(N, Ops);
05275 }
05276 
05277 SDNode *SelectionDAG::
05278 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
05279                    SDValue Op3, SDValue Op4, SDValue Op5) {
05280   SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
05281   return UpdateNodeOperands(N, Ops);
05282 }
05283 
05284 SDNode *SelectionDAG::
05285 UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
05286   unsigned NumOps = Ops.size();
05287   assert(N->getNumOperands() == NumOps &&
05288          "Update with wrong number of operands");
05289 
05290   // Check to see if there is no change.
05291   bool AnyChange = false;
05292   for (unsigned i = 0; i != NumOps; ++i) {
05293     if (Ops[i] != N->getOperand(i)) {
05294       AnyChange = true;
05295       break;
05296     }
05297   }
05298 
05299   // No operands changed, just return the input node.
05300   if (!AnyChange) return N;
05301 
05302   // See if the modified node already exists.
05303   void *InsertPos = nullptr;
05304   if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
05305     return Existing;
05306 
05307   // Nope it doesn't.  Remove the node from its current place in the maps.
05308   if (InsertPos)
05309     if (!RemoveNodeFromCSEMaps(N))
05310       InsertPos = nullptr;
05311 
05312   // Now we update the operands.
05313   for (unsigned i = 0; i != NumOps; ++i)
05314     if (N->OperandList[i] != Ops[i])
05315       N->OperandList[i].set(Ops[i]);
05316 
05317   // If this gets put into a CSE map, add it.
05318   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05319   return N;
05320 }
05321 
05322 /// DropOperands - Release the operands and set this node to have
05323 /// zero operands.
05324 void SDNode::DropOperands() {
05325   // Unlike the code in MorphNodeTo that does this, we don't need to
05326   // watch for dead nodes here.
05327   for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
05328     SDUse &Use = *I++;
05329     Use.set(SDValue());
05330   }
05331 }
05332 
05333 /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
05334 /// machine opcode.
05335 ///
05336 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05337                                    EVT VT) {
05338   SDVTList VTs = getVTList(VT);
05339   return SelectNodeTo(N, MachineOpc, VTs, None);
05340 }
05341 
05342 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05343                                    EVT VT, SDValue Op1) {
05344   SDVTList VTs = getVTList(VT);
05345   SDValue Ops[] = { Op1 };
05346   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05347 }
05348 
05349 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05350                                    EVT VT, SDValue Op1,
05351                                    SDValue Op2) {
05352   SDVTList VTs = getVTList(VT);
05353   SDValue Ops[] = { Op1, Op2 };
05354   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05355 }
05356 
05357 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05358                                    EVT VT, SDValue Op1,
05359                                    SDValue Op2, SDValue Op3) {
05360   SDVTList VTs = getVTList(VT);
05361   SDValue Ops[] = { Op1, Op2, Op3 };
05362   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05363 }
05364 
05365 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05366                                    EVT VT, ArrayRef<SDValue> Ops) {
05367   SDVTList VTs = getVTList(VT);
05368   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05369 }
05370 
05371 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05372                                    EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
05373   SDVTList VTs = getVTList(VT1, VT2);
05374   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05375 }
05376 
05377 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05378                                    EVT VT1, EVT VT2) {
05379   SDVTList VTs = getVTList(VT1, VT2);
05380   return SelectNodeTo(N, MachineOpc, VTs, None);
05381 }
05382 
05383 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05384                                    EVT VT1, EVT VT2, EVT VT3,
05385                                    ArrayRef<SDValue> Ops) {
05386   SDVTList VTs = getVTList(VT1, VT2, VT3);
05387   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05388 }
05389 
05390 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05391                                    EVT VT1, EVT VT2, EVT VT3, EVT VT4,
05392                                    ArrayRef<SDValue> Ops) {
05393   SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
05394   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05395 }
05396 
05397 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05398                                    EVT VT1, EVT VT2,
05399                                    SDValue Op1) {
05400   SDVTList VTs = getVTList(VT1, VT2);
05401   SDValue Ops[] = { Op1 };
05402   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05403 }
05404 
05405 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05406                                    EVT VT1, EVT VT2,
05407                                    SDValue Op1, SDValue Op2) {
05408   SDVTList VTs = getVTList(VT1, VT2);
05409   SDValue Ops[] = { Op1, Op2 };
05410   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05411 }
05412 
05413 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05414                                    EVT VT1, EVT VT2,
05415                                    SDValue Op1, SDValue Op2,
05416                                    SDValue Op3) {
05417   SDVTList VTs = getVTList(VT1, VT2);
05418   SDValue Ops[] = { Op1, Op2, Op3 };
05419   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05420 }
05421 
05422 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05423                                    EVT VT1, EVT VT2, EVT VT3,
05424                                    SDValue Op1, SDValue Op2,
05425                                    SDValue Op3) {
05426   SDVTList VTs = getVTList(VT1, VT2, VT3);
05427   SDValue Ops[] = { Op1, Op2, Op3 };
05428   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05429 }
05430 
05431 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05432                                    SDVTList VTs,ArrayRef<SDValue> Ops) {
05433   N = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
05434   // Reset the NodeID to -1.
05435   N->setNodeId(-1);
05436   return N;
05437 }
05438 
05439 /// UpdadeSDLocOnMergedSDNode - If the opt level is -O0 then it throws away
05440 /// the line number information on the merged node since it is not possible to
05441 /// preserve the information that operation is associated with multiple lines.
05442 /// This will make the debugger working better at -O0, were there is a higher
05443 /// probability having other instructions associated with that line.
05444 ///
05445 /// For IROrder, we keep the smaller of the two
05446 SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) {
05447   DebugLoc NLoc = N->getDebugLoc();
05448   if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) &&
05449     (OLoc.getDebugLoc() != NLoc)) {
05450     N->setDebugLoc(DebugLoc());
05451   }
05452   unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
05453   N->setIROrder(Order);
05454   return N;
05455 }
05456 
05457 /// MorphNodeTo - This *mutates* the specified node to have the specified
05458 /// return type, opcode, and operands.
05459 ///
05460 /// Note that MorphNodeTo returns the resultant node.  If there is already a
05461 /// node of the specified opcode and operands, it returns that node instead of
05462 /// the current one.  Note that the SDLoc need not be the same.
05463 ///
05464 /// Using MorphNodeTo is faster than creating a new node and swapping it in
05465 /// with ReplaceAllUsesWith both because it often avoids allocating a new
05466 /// node, and because it doesn't require CSE recalculation for any of
05467 /// the node's users.
05468 ///
05469 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
05470                                   SDVTList VTs, ArrayRef<SDValue> Ops) {
05471   unsigned NumOps = Ops.size();
05472   // If an identical node already exists, use it.
05473   void *IP = nullptr;
05474   if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
05475     FoldingSetNodeID ID;
05476     AddNodeIDNode(ID, Opc, VTs, Ops);
05477     if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
05478       return UpdadeSDLocOnMergedSDNode(ON, SDLoc(N));
05479   }
05480 
05481   if (!RemoveNodeFromCSEMaps(N))
05482     IP = nullptr;
05483 
05484   // Start the morphing.
05485   N->NodeType = Opc;
05486   N->ValueList = VTs.VTs;
05487   N->NumValues = VTs.NumVTs;
05488 
05489   // Clear the operands list, updating used nodes to remove this from their
05490   // use list.  Keep track of any operands that become dead as a result.
05491   SmallPtrSet<SDNode*, 16> DeadNodeSet;
05492   for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
05493     SDUse &Use = *I++;
05494     SDNode *Used = Use.getNode();
05495     Use.set(SDValue());
05496     if (Used->use_empty())
05497       DeadNodeSet.insert(Used);
05498   }
05499 
05500   if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) {
05501     // Initialize the memory references information.
05502     MN->setMemRefs(nullptr, nullptr);
05503     // If NumOps is larger than the # of operands we can have in a
05504     // MachineSDNode, reallocate the operand list.
05505     if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) {
05506       if (MN->OperandsNeedDelete)
05507         delete[] MN->OperandList;
05508       if (NumOps > array_lengthof(MN->LocalOperands))
05509         // We're creating a final node that will live unmorphed for the
05510         // remainder of the current SelectionDAG iteration, so we can allocate
05511         // the operands directly out of a pool with no recycling metadata.
05512         MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
05513                          Ops.data(), NumOps);
05514       else
05515         MN->InitOperands(MN->LocalOperands, Ops.data(), NumOps);
05516       MN->OperandsNeedDelete = false;
05517     } else
05518       MN->InitOperands(MN->OperandList, Ops.data(), NumOps);
05519   } else {
05520     // If NumOps is larger than the # of operands we currently have, reallocate
05521     // the operand list.
05522     if (NumOps > N->NumOperands) {
05523       if (N->OperandsNeedDelete)
05524         delete[] N->OperandList;
05525       N->InitOperands(new SDUse[NumOps], Ops.data(), NumOps);
05526       N->OperandsNeedDelete = true;
05527     } else
05528       N->InitOperands(N->OperandList, Ops.data(), NumOps);
05529   }
05530 
05531   // Delete any nodes that are still dead after adding the uses for the
05532   // new operands.
05533   if (!DeadNodeSet.empty()) {
05534     SmallVector<SDNode *, 16> DeadNodes;
05535     for (SmallPtrSet<SDNode *, 16>::iterator I = DeadNodeSet.begin(),
05536          E = DeadNodeSet.end(); I != E; ++I)
05537       if ((*I)->use_empty())
05538         DeadNodes.push_back(*I);
05539     RemoveDeadNodes(DeadNodes);
05540   }
05541 
05542   if (IP)
05543     CSEMap.InsertNode(N, IP);   // Memoize the new node.
05544   return N;
05545 }
05546 
05547 
05548 /// getMachineNode - These are used for target selectors to create a new node
05549 /// with specified return type(s), MachineInstr opcode, and operands.
05550 ///
05551 /// Note that getMachineNode returns the resultant node.  If there is already a
05552 /// node of the specified opcode and operands, it returns that node instead of
05553 /// the current one.
05554 MachineSDNode *
05555 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT) {
05556   SDVTList VTs = getVTList(VT);
05557   return getMachineNode(Opcode, dl, VTs, None);
05558 }
05559 
05560 MachineSDNode *
05561 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, SDValue Op1) {
05562   SDVTList VTs = getVTList(VT);
05563   SDValue Ops[] = { Op1 };
05564   return getMachineNode(Opcode, dl, VTs, Ops);
05565 }
05566 
05567 MachineSDNode *
05568 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05569                              SDValue Op1, SDValue Op2) {
05570   SDVTList VTs = getVTList(VT);
05571   SDValue Ops[] = { Op1, Op2 };
05572   return getMachineNode(Opcode, dl, VTs, Ops);
05573 }
05574 
05575 MachineSDNode *
05576 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05577                              SDValue Op1, SDValue Op2, SDValue Op3) {
05578   SDVTList VTs = getVTList(VT);
05579   SDValue Ops[] = { Op1, Op2, Op3 };
05580   return getMachineNode(Opcode, dl, VTs, Ops);
05581 }
05582 
05583 MachineSDNode *
05584 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05585                              ArrayRef<SDValue> Ops) {
05586   SDVTList VTs = getVTList(VT);
05587   return getMachineNode(Opcode, dl, VTs, Ops);
05588 }
05589 
05590 MachineSDNode *
05591 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, EVT VT2) {
05592   SDVTList VTs = getVTList(VT1, VT2);
05593   return getMachineNode(Opcode, dl, VTs, None);
05594 }
05595 
05596 MachineSDNode *
05597 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05598                              EVT VT1, EVT VT2, SDValue Op1) {
05599   SDVTList VTs = getVTList(VT1, VT2);
05600   SDValue Ops[] = { Op1 };
05601   return getMachineNode(Opcode, dl, VTs, Ops);
05602 }
05603 
05604 MachineSDNode *
05605 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05606                              EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) {
05607   SDVTList VTs = getVTList(VT1, VT2);
05608   SDValue Ops[] = { Op1, Op2 };
05609   return getMachineNode(Opcode, dl, VTs, Ops);
05610 }
05611 
05612 MachineSDNode *
05613 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05614                              EVT VT1, EVT VT2, SDValue Op1,
05615                              SDValue Op2, SDValue Op3) {
05616   SDVTList VTs = getVTList(VT1, VT2);
05617   SDValue Ops[] = { Op1, Op2, Op3 };
05618   return getMachineNode(Opcode, dl, VTs, Ops);
05619 }
05620 
05621 MachineSDNode *
05622 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05623                              EVT VT1, EVT VT2,
05624                              ArrayRef<SDValue> Ops) {
05625   SDVTList VTs = getVTList(VT1, VT2);
05626   return getMachineNode(Opcode, dl, VTs, Ops);
05627 }
05628 
05629 MachineSDNode *
05630 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05631                              EVT VT1, EVT VT2, EVT VT3,
05632                              SDValue Op1, SDValue Op2) {
05633   SDVTList VTs = getVTList(VT1, VT2, VT3);
05634   SDValue Ops[] = { Op1, Op2 };
05635   return getMachineNode(Opcode, dl, VTs, Ops);
05636 }
05637 
05638 MachineSDNode *
05639 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05640                              EVT VT1, EVT VT2, EVT VT3,
05641                              SDValue Op1, SDValue Op2, SDValue Op3) {
05642   SDVTList VTs = getVTList(VT1, VT2, VT3);
05643   SDValue Ops[] = { Op1, Op2, Op3 };
05644   return getMachineNode(Opcode, dl, VTs, Ops);
05645 }
05646 
05647 MachineSDNode *
05648 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05649                              EVT VT1, EVT VT2, EVT VT3,
05650                              ArrayRef<SDValue> Ops) {
05651   SDVTList VTs = getVTList(VT1, VT2, VT3);
05652   return getMachineNode(Opcode, dl, VTs, Ops);
05653 }
05654 
05655 MachineSDNode *
05656 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1,
05657                              EVT VT2, EVT VT3, EVT VT4,
05658                              ArrayRef<SDValue> Ops) {
05659   SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
05660   return getMachineNode(Opcode, dl, VTs, Ops);
05661 }
05662 
05663 MachineSDNode *
05664 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05665                              ArrayRef<EVT> ResultTys,
05666                              ArrayRef<SDValue> Ops) {
05667   SDVTList VTs = getVTList(ResultTys);
05668   return getMachineNode(Opcode, dl, VTs, Ops);
05669 }
05670 
05671 MachineSDNode *
05672 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
05673                              ArrayRef<SDValue> OpsArray) {
05674   bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
05675   MachineSDNode *N;
05676   void *IP = nullptr;
05677   const SDValue *Ops = OpsArray.data();
05678   unsigned NumOps = OpsArray.size();
05679 
05680   if (DoCSE) {
05681     FoldingSetNodeID ID;
05682     AddNodeIDNode(ID, ~Opcode, VTs, OpsArray);
05683     IP = nullptr;
05684     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
05685       return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL));
05686     }
05687   }
05688 
05689   // Allocate a new MachineSDNode.
05690   N = new (NodeAllocator) MachineSDNode(~Opcode, DL.getIROrder(),
05691                                         DL.getDebugLoc(), VTs);
05692 
05693   // Initialize the operands list.
05694   if (NumOps > array_lengthof(N->LocalOperands))
05695     // We're creating a final node that will live unmorphed for the
05696     // remainder of the current SelectionDAG iteration, so we can allocate
05697     // the operands directly out of a pool with no recycling metadata.
05698     N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
05699                     Ops, NumOps);
05700   else
05701     N->InitOperands(N->LocalOperands, Ops, NumOps);
05702   N->OperandsNeedDelete = false;
05703 
05704   if (DoCSE)
05705     CSEMap.InsertNode(N, IP);
05706 
05707   InsertNode(N);
05708   return N;
05709 }
05710 
05711 /// getTargetExtractSubreg - A convenience function for creating
05712 /// TargetOpcode::EXTRACT_SUBREG nodes.
05713 SDValue
05714 SelectionDAG::getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT,
05715                                      SDValue Operand) {
05716   SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
05717   SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
05718                                   VT, Operand, SRIdxVal);
05719   return SDValue(Subreg, 0);
05720 }
05721 
05722 /// getTargetInsertSubreg - A convenience function for creating
05723 /// TargetOpcode::INSERT_SUBREG nodes.
05724 SDValue
05725 SelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT,
05726                                     SDValue Operand, SDValue Subreg) {
05727   SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
05728   SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
05729                                   VT, Operand, Subreg, SRIdxVal);
05730   return SDValue(Result, 0);
05731 }
05732 
05733 /// getNodeIfExists - Get the specified node if it's already available, or
05734 /// else return NULL.
05735 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
05736                                       ArrayRef<SDValue> Ops, bool nuw, bool nsw,
05737                                       bool exact) {
05738   if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
05739     FoldingSetNodeID ID;
05740     AddNodeIDNode(ID, Opcode, VTList, Ops);
05741     if (isBinOpWithFlags(Opcode))
05742       AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
05743     void *IP = nullptr;
05744     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05745       return E;
05746   }
05747   return nullptr;
05748 }
05749 
05750 /// getDbgValue - Creates a SDDbgValue node.
05751 ///
05752 /// SDNode
05753 SDDbgValue *
05754 SelectionDAG::getDbgValue(MDNode *MDPtr, SDNode *N, unsigned R,
05755         bool IsIndirect, uint64_t Off,
05756                           DebugLoc DL, unsigned O) {
05757   return new (Allocator) SDDbgValue(MDPtr, N, R, IsIndirect, Off, DL, O);
05758 }
05759 
05760 /// Constant
05761 SDDbgValue *
05762 SelectionDAG::getConstantDbgValue(MDNode *MDPtr, const Value *C,
05763           uint64_t Off,
05764           DebugLoc DL, unsigned O) {
05765   return new (Allocator) SDDbgValue(MDPtr, C, Off, DL, O);
05766 }
05767 
05768 /// FrameIndex
05769 SDDbgValue *
05770 SelectionDAG::getFrameIndexDbgValue(MDNode *MDPtr, unsigned FI, uint64_t Off,
05771             DebugLoc DL, unsigned O) {
05772   return new (Allocator) SDDbgValue(MDPtr, FI, Off, DL, O);
05773 }
05774 
05775 namespace {
05776 
05777 /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
05778 /// pointed to by a use iterator is deleted, increment the use iterator
05779 /// so that it doesn't dangle.
05780 ///
05781 class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
05782   SDNode::use_iterator &UI;
05783   SDNode::use_iterator &UE;
05784 
05785   void NodeDeleted(SDNode *N, SDNode *E) override {
05786     // Increment the iterator as needed.
05787     while (UI != UE && N == *UI)
05788       ++UI;
05789   }
05790 
05791 public:
05792   RAUWUpdateListener(SelectionDAG &d,
05793                      SDNode::use_iterator &ui,
05794                      SDNode::use_iterator &ue)
05795     : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
05796 };
05797 
05798 }
05799 
05800 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05801 /// This can cause recursive merging of nodes in the DAG.
05802 ///
05803 /// This version assumes From has a single result value.
05804 ///
05805 void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
05806   SDNode *From = FromN.getNode();
05807   assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
05808          "Cannot replace with this method!");
05809   assert(From != To.getNode() && "Cannot replace uses of with self");
05810 
05811   // Iterate over all the existing uses of From. New uses will be added
05812   // to the beginning of the use list, which we avoid visiting.
05813   // This specifically avoids visiting uses of From that arise while the
05814   // replacement is happening, because any such uses would be the result
05815   // of CSE: If an existing node looks like From after one of its operands
05816   // is replaced by To, we don't want to replace of all its users with To
05817   // too. See PR3018 for more info.
05818   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05819   RAUWUpdateListener Listener(*this, UI, UE);
05820   while (UI != UE) {
05821     SDNode *User = *UI;
05822 
05823     // This node is about to morph, remove its old self from the CSE maps.
05824     RemoveNodeFromCSEMaps(User);
05825 
05826     // A user can appear in a use list multiple times, and when this
05827     // happens the uses are usually next to each other in the list.
05828     // To help reduce the number of CSE recomputations, process all
05829     // the uses of this user that we can find this way.
05830     do {
05831       SDUse &Use = UI.getUse();
05832       ++UI;
05833       Use.set(To);
05834     } while (UI != UE && *UI == User);
05835 
05836     // Now that we have modified User, add it back to the CSE maps.  If it
05837     // already exists there, recursively merge the results together.
05838     AddModifiedNodeToCSEMaps(User);
05839   }
05840 
05841   // If we just RAUW'd the root, take note.
05842   if (FromN == getRoot())
05843     setRoot(To);
05844 }
05845 
05846 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05847 /// This can cause recursive merging of nodes in the DAG.
05848 ///
05849 /// This version assumes that for each value of From, there is a
05850 /// corresponding value in To in the same position with the same type.
05851 ///
05852 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
05853 #ifndef NDEBUG
05854   for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
05855     assert((!From->hasAnyUseOfValue(i) ||
05856             From->getValueType(i) == To->getValueType(i)) &&
05857            "Cannot use this version of ReplaceAllUsesWith!");
05858 #endif
05859 
05860   // Handle the trivial case.
05861   if (From == To)
05862     return;
05863 
05864   // Iterate over just the existing users of From. See the comments in
05865   // the ReplaceAllUsesWith above.
05866   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05867   RAUWUpdateListener Listener(*this, UI, UE);
05868   while (UI != UE) {
05869     SDNode *User = *UI;
05870 
05871     // This node is about to morph, remove its old self from the CSE maps.
05872     RemoveNodeFromCSEMaps(User);
05873 
05874     // A user can appear in a use list multiple times, and when this
05875     // happens the uses are usually next to each other in the list.
05876     // To help reduce the number of CSE recomputations, process all
05877     // the uses of this user that we can find this way.
05878     do {
05879       SDUse &Use = UI.getUse();
05880       ++UI;
05881       Use.setNode(To);
05882     } while (UI != UE && *UI == User);
05883 
05884     // Now that we have modified User, add it back to the CSE maps.  If it
05885     // already exists there, recursively merge the results together.
05886     AddModifiedNodeToCSEMaps(User);
05887   }
05888 
05889   // If we just RAUW'd the root, take note.
05890   if (From == getRoot().getNode())
05891     setRoot(SDValue(To, getRoot().getResNo()));
05892 }
05893 
05894 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05895 /// This can cause recursive merging of nodes in the DAG.
05896 ///
05897 /// This version can replace From with any result values.  To must match the
05898 /// number and types of values returned by From.
05899 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
05900   if (From->getNumValues() == 1)  // Handle the simple case efficiently.
05901     return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
05902 
05903   // Iterate over just the existing users of From. See the comments in
05904   // the ReplaceAllUsesWith above.
05905   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05906   RAUWUpdateListener Listener(*this, UI, UE);
05907   while (UI != UE) {
05908     SDNode *User = *UI;
05909 
05910     // This node is about to morph, remove its old self from the CSE maps.
05911     RemoveNodeFromCSEMaps(User);
05912 
05913     // A user can appear in a use list multiple times, and when this
05914     // happens the uses are usually next to each other in the list.
05915     // To help reduce the number of CSE recomputations, process all
05916     // the uses of this user that we can find this way.
05917     do {
05918       SDUse &Use = UI.getUse();
05919       const SDValue &ToOp = To[Use.getResNo()];
05920       ++UI;
05921       Use.set(ToOp);
05922     } while (UI != UE && *UI == User);
05923 
05924     // Now that we have modified User, add it back to the CSE maps.  If it
05925     // already exists there, recursively merge the results together.
05926     AddModifiedNodeToCSEMaps(User);
05927   }
05928 
05929   // If we just RAUW'd the root, take note.
05930   if (From == getRoot().getNode())
05931     setRoot(SDValue(To[getRoot().getResNo()]));
05932 }
05933 
05934 /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
05935 /// uses of other values produced by From.getNode() alone.  The Deleted
05936 /// vector is handled the same way as for ReplaceAllUsesWith.
05937 void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
05938   // Handle the really simple, really trivial case efficiently.
05939   if (From == To) return;
05940 
05941   // Handle the simple, trivial, case efficiently.
05942   if (From.getNode()->getNumValues() == 1) {
05943     ReplaceAllUsesWith(From, To);
05944     return;
05945   }
05946 
05947   // Iterate over just the existing users of From. See the comments in
05948   // the ReplaceAllUsesWith above.
05949   SDNode::use_iterator UI = From.getNode()->use_begin(),
05950                        UE = From.getNode()->use_end();
05951   RAUWUpdateListener Listener(*this, UI, UE);
05952   while (UI != UE) {
05953     SDNode *User = *UI;
05954     bool UserRemovedFromCSEMaps = false;
05955 
05956     // A user can appear in a use list multiple times, and when this
05957     // happens the uses are usually next to each other in the list.
05958     // To help reduce the number of CSE recomputations, process all
05959     // the uses of this user that we can find this way.
05960     do {
05961       SDUse &Use = UI.getUse();
05962 
05963       // Skip uses of different values from the same node.
05964       if (Use.getResNo() != From.getResNo()) {
05965         ++UI;
05966         continue;
05967       }
05968 
05969       // If this node hasn't been modified yet, it's still in the CSE maps,
05970       // so remove its old self from the CSE maps.
05971       if (!UserRemovedFromCSEMaps) {
05972         RemoveNodeFromCSEMaps(User);
05973         UserRemovedFromCSEMaps = true;
05974       }
05975 
05976       ++UI;
05977       Use.set(To);
05978     } while (UI != UE && *UI == User);
05979 
05980     // We are iterating over all uses of the From node, so if a use
05981     // doesn't use the specific value, no changes are made.
05982     if (!UserRemovedFromCSEMaps)
05983       continue;
05984 
05985     // Now that we have modified User, add it back to the CSE maps.  If it
05986     // already exists there, recursively merge the results together.
05987     AddModifiedNodeToCSEMaps(User);
05988   }
05989 
05990   // If we just RAUW'd the root, take note.
05991   if (From == getRoot())
05992     setRoot(To);
05993 }
05994 
05995 namespace {
05996   /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
05997   /// to record information about a use.
05998   struct UseMemo {
05999     SDNode *User;
06000     unsigned Index;
06001     SDUse *Use;
06002   };
06003 
06004   /// operator< - Sort Memos by User.
06005   bool operator<(const UseMemo &L, const UseMemo &R) {
06006     return (intptr_t)L.User < (intptr_t)R.User;
06007   }
06008 }
06009 
06010 /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
06011 /// uses of other values produced by From.getNode() alone.  The same value
06012 /// may appear in both the From and To list.  The Deleted vector is
06013 /// handled the same way as for ReplaceAllUsesWith.
06014 void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
06015                                               const SDValue *To,
06016                                               unsigned Num){
06017   // Handle the simple, trivial case efficiently.
06018   if (Num == 1)
06019     return ReplaceAllUsesOfValueWith(*From, *To);
06020 
06021   // Read up all the uses and make records of them. This helps
06022   // processing new uses that are introduced during the
06023   // replacement process.
06024   SmallVector<UseMemo, 4> Uses;
06025   for (unsigned i = 0; i != Num; ++i) {
06026     unsigned FromResNo = From[i].getResNo();
06027     SDNode *FromNode = From[i].getNode();
06028     for (SDNode::use_iterator UI = FromNode->use_begin(),
06029          E = FromNode->use_end(); UI != E; ++UI) {
06030       SDUse &Use = UI.getUse();
06031       if (Use.getResNo() == FromResNo) {
06032         UseMemo Memo = { *UI, i, &Use };
06033         Uses.push_back(Memo);
06034       }
06035     }
06036   }
06037 
06038   // Sort the uses, so that all the uses from a given User are together.
06039   std::sort(Uses.begin(), Uses.end());
06040 
06041   for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
06042        UseIndex != UseIndexEnd; ) {
06043     // We know that this user uses some value of From.  If it is the right
06044     // value, update it.
06045     SDNode *User = Uses[UseIndex].User;
06046 
06047     // This node is about to morph, remove its old self from the CSE maps.
06048     RemoveNodeFromCSEMaps(User);
06049 
06050     // The Uses array is sorted, so all the uses for a given User
06051     // are next to each other in the list.
06052     // To help reduce the number of CSE recomputations, process all
06053     // the uses of this user that we can find this way.
06054     do {
06055       unsigned i = Uses[UseIndex].Index;
06056       SDUse &Use = *Uses[UseIndex].Use;
06057       ++UseIndex;
06058 
06059       Use.set(To[i]);
06060     } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
06061 
06062     // Now that we have modified User, add it back to the CSE maps.  If it
06063     // already exists there, recursively merge the results together.
06064     AddModifiedNodeToCSEMaps(User);
06065   }
06066 }
06067 
06068 /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
06069 /// based on their topological order. It returns the maximum id and a vector
06070 /// of the SDNodes* in assigned order by reference.
06071 unsigned SelectionDAG::AssignTopologicalOrder() {
06072 
06073   unsigned DAGSize = 0;
06074 
06075   // SortedPos tracks the progress of the algorithm. Nodes before it are
06076   // sorted, nodes after it are unsorted. When the algorithm completes
06077   // it is at the end of the list.
06078   allnodes_iterator SortedPos = allnodes_begin();
06079 
06080   // Visit all the nodes. Move nodes with no operands to the front of
06081   // the list immediately. Annotate nodes that do have operands with their
06082   // operand count. Before we do this, the Node Id fields of the nodes
06083   // may contain arbitrary values. After, the Node Id fields for nodes
06084   // before SortedPos will contain the topological sort index, and the
06085   // Node Id fields for nodes At SortedPos and after will contain the
06086   // count of outstanding operands.
06087   for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
06088     SDNode *N = I++;
06089     checkForCycles(N, this);
06090     unsigned Degree = N->getNumOperands();
06091     if (Degree == 0) {
06092       // A node with no uses, add it to the result array immediately.
06093       N->setNodeId(DAGSize++);
06094       allnodes_iterator Q = N;
06095       if (Q != SortedPos)
06096         SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
06097       assert(SortedPos != AllNodes.end() && "Overran node list");
06098       ++SortedPos;
06099     } else {
06100       // Temporarily use the Node Id as scratch space for the degree count.
06101       N->setNodeId(Degree);
06102     }
06103   }
06104 
06105   // Visit all the nodes. As we iterate, move nodes into sorted order,
06106   // such that by the time the end is reached all nodes will be sorted.
06107   for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) {
06108     SDNode *N = I;
06109     checkForCycles(N, this);
06110     // N is in sorted position, so all its uses have one less operand
06111     // that needs to be sorted.
06112     for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
06113          UI != UE; ++UI) {
06114       SDNode *P = *UI;
06115       unsigned Degree = P->getNodeId();
06116       assert(Degree != 0 && "Invalid node degree");
06117       --Degree;
06118       if (Degree == 0) {
06119         // All of P's operands are sorted, so P may sorted now.
06120         P->setNodeId(DAGSize++);
06121         if (P != SortedPos)
06122           SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
06123         assert(SortedPos != AllNodes.end() && "Overran node list");
06124         ++SortedPos;
06125       } else {
06126         // Update P's outstanding operand count.
06127         P->setNodeId(Degree);
06128       }
06129     }
06130     if (I == SortedPos) {
06131 #ifndef NDEBUG
06132       SDNode *S = ++I;
06133       dbgs() << "Overran sorted position:\n";
06134       S->dumprFull(this); dbgs() << "\n";
06135       dbgs() << "Checking if this is due to cycles\n";
06136       checkForCycles(this, true);
06137 #endif
06138       llvm_unreachable(nullptr);
06139     }
06140   }
06141 
06142   assert(SortedPos == AllNodes.end() &&
06143          "Topological sort incomplete!");
06144   assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
06145          "First node in topological sort is not the entry token!");
06146   assert(AllNodes.front().getNodeId() == 0 &&
06147          "First node in topological sort has non-zero id!");
06148   assert(AllNodes.front().getNumOperands() == 0 &&
06149          "First node in topological sort has operands!");
06150   assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
06151          "Last node in topologic sort has unexpected id!");
06152   assert(AllNodes.back().use_empty() &&
06153          "Last node in topologic sort has users!");
06154   assert(DAGSize == allnodes_size() && "Node count mismatch!");
06155   return DAGSize;
06156 }
06157 
06158 /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
06159 /// value is produced by SD.
06160 void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) {
06161   DbgInfo->add(DB, SD, isParameter);
06162   if (SD)
06163     SD->setHasDebugValue(true);
06164 }
06165 
06166 /// TransferDbgValues - Transfer SDDbgValues.
06167 void SelectionDAG::TransferDbgValues(SDValue From, SDValue To) {
06168   if (From == To || !From.getNode()->getHasDebugValue())
06169     return;
06170   SDNode *FromNode = From.getNode();
06171   SDNode *ToNode = To.getNode();
06172   ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode);
06173   SmallVector<SDDbgValue *, 2> ClonedDVs;
06174   for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end();
06175        I != E; ++I) {
06176     SDDbgValue *Dbg = *I;
06177     if (Dbg->getKind() == SDDbgValue::SDNODE) {
06178       SDDbgValue *Clone = getDbgValue(Dbg->getMDPtr(), ToNode, To.getResNo(),
06179               Dbg->isIndirect(),
06180                                       Dbg->getOffset(), Dbg->getDebugLoc(),
06181                                       Dbg->getOrder());
06182       ClonedDVs.push_back(Clone);
06183     }
06184   }
06185   for (SmallVectorImpl<SDDbgValue *>::iterator I = ClonedDVs.begin(),
06186          E = ClonedDVs.end(); I != E; ++I)
06187     AddDbgValue(*I, ToNode, false);
06188 }
06189 
06190 //===----------------------------------------------------------------------===//
06191 //                              SDNode Class
06192 //===----------------------------------------------------------------------===//
06193 
06194 HandleSDNode::~HandleSDNode() {
06195   DropOperands();
06196 }
06197 
06198 GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
06199                                          DebugLoc DL, const GlobalValue *GA,
06200                                          EVT VT, int64_t o, unsigned char TF)
06201   : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
06202   TheGlobal = GA;
06203 }
06204 
06205 AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT,
06206                                          SDValue X, unsigned SrcAS,
06207                                          unsigned DestAS)
06208  : UnarySDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT), X),
06209    SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
06210 
06211 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
06212                      EVT memvt, MachineMemOperand *mmo)
06213  : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
06214   SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
06215                                       MMO->isNonTemporal(), MMO->isInvariant());
06216   assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
06217   assert(isNonTemporal() == MMO->isNonTemporal() &&
06218          "Non-temporal encoding error!");
06219   assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!");
06220 }
06221 
06222 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
06223                      ArrayRef<SDValue> Ops, EVT memvt, MachineMemOperand *mmo)
06224    : SDNode(Opc, Order, dl, VTs, Ops),
06225      MemoryVT(memvt), MMO(mmo) {
06226   SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
06227                                       MMO->isNonTemporal(), MMO->isInvariant());
06228   assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
06229   assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!");
06230 }
06231 
06232 /// Profile - Gather unique data for the node.
06233 ///
06234 void SDNode::Profile(FoldingSetNodeID &ID) const {
06235   AddNodeIDNode(ID, this);
06236 }
06237 
06238 namespace {
06239   struct EVTArray {
06240     std::vector<EVT> VTs;
06241 
06242     EVTArray() {
06243       VTs.reserve(MVT::LAST_VALUETYPE);
06244       for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i)
06245         VTs.push_back(MVT((MVT::SimpleValueType)i));
06246     }
06247   };
06248 }
06249 
06250 static ManagedStatic<std::set<EVT, EVT::compareRawBits> > EVTs;
06251 static ManagedStatic<EVTArray> SimpleVTArray;
06252 static ManagedStatic<sys::SmartMutex<true> > VTMutex;
06253 
06254 /// getValueTypeList - Return a pointer to the specified value type.
06255 ///
06256 const EVT *SDNode::getValueTypeList(EVT VT) {
06257   if (VT.isExtended()) {
06258     sys::SmartScopedLock<true> Lock(*VTMutex);
06259     return &(*EVTs->insert(VT).first);
06260   } else {
06261     assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE &&
06262            "Value type out of range!");
06263     return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy];
06264   }
06265 }
06266 
06267 /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
06268 /// indicated value.  This method ignores uses of other values defined by this
06269 /// operation.
06270 bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
06271   assert(Value < getNumValues() && "Bad value!");
06272 
06273   // TODO: Only iterate over uses of a given value of the node
06274   for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
06275     if (UI.getUse().getResNo() == Value) {
06276       if (NUses == 0)
06277         return false;
06278       --NUses;
06279     }
06280   }
06281 
06282   // Found exactly the right number of uses?
06283   return NUses == 0;
06284 }
06285 
06286 
06287 /// hasAnyUseOfValue - Return true if there are any use of the indicated
06288 /// value. This method ignores uses of other values defined by this operation.
06289 bool SDNode::hasAnyUseOfValue(unsigned Value) const {
06290   assert(Value < getNumValues() && "Bad value!");
06291 
06292   for