LLVM  mainline
SelectionDAG.cpp
Go to the documentation of this file.
00001 //===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This implements the SelectionDAG class.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "llvm/CodeGen/SelectionDAG.h"
00015 #include "SDNodeDbgValue.h"
00016 #include "llvm/ADT/SetVector.h"
00017 #include "llvm/ADT/SmallPtrSet.h"
00018 #include "llvm/ADT/SmallSet.h"
00019 #include "llvm/ADT/SmallVector.h"
00020 #include "llvm/ADT/StringExtras.h"
00021 #include "llvm/Analysis/ValueTracking.h"
00022 #include "llvm/CodeGen/MachineBasicBlock.h"
00023 #include "llvm/CodeGen/MachineConstantPool.h"
00024 #include "llvm/CodeGen/MachineFrameInfo.h"
00025 #include "llvm/CodeGen/MachineModuleInfo.h"
00026 #include "llvm/IR/CallingConv.h"
00027 #include "llvm/IR/Constants.h"
00028 #include "llvm/IR/DataLayout.h"
00029 #include "llvm/IR/DebugInfo.h"
00030 #include "llvm/IR/DerivedTypes.h"
00031 #include "llvm/IR/Function.h"
00032 #include "llvm/IR/GlobalAlias.h"
00033 #include "llvm/IR/GlobalVariable.h"
00034 #include "llvm/IR/Intrinsics.h"
00035 #include "llvm/Support/CommandLine.h"
00036 #include "llvm/Support/Debug.h"
00037 #include "llvm/Support/ErrorHandling.h"
00038 #include "llvm/Support/ManagedStatic.h"
00039 #include "llvm/Support/MathExtras.h"
00040 #include "llvm/Support/Mutex.h"
00041 #include "llvm/Support/raw_ostream.h"
00042 #include "llvm/Target/TargetInstrInfo.h"
00043 #include "llvm/Target/TargetIntrinsicInfo.h"
00044 #include "llvm/Target/TargetLowering.h"
00045 #include "llvm/Target/TargetMachine.h"
00046 #include "llvm/Target/TargetOptions.h"
00047 #include "llvm/Target/TargetRegisterInfo.h"
00048 #include "llvm/Target/TargetSelectionDAGInfo.h"
00049 #include "llvm/Target/TargetSubtargetInfo.h"
00050 #include <algorithm>
00051 #include <cmath>
00052 
00053 using namespace llvm;
00054 
00055 /// makeVTList - Return an instance of the SDVTList struct initialized with the
00056 /// specified members.
00057 static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
00058   SDVTList Res = {VTs, NumVTs};
00059   return Res;
00060 }
00061 
00062 // Default null implementations of the callbacks.
00063 void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
00064 void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
00065 
00066 //===----------------------------------------------------------------------===//
00067 //                              ConstantFPSDNode Class
00068 //===----------------------------------------------------------------------===//
00069 
00070 /// isExactlyValue - We don't rely on operator== working on double values, as
00071 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
00072 /// As such, this method can be used to do an exact bit-for-bit comparison of
00073 /// two floating point values.
00074 bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
00075   return getValueAPF().bitwiseIsEqual(V);
00076 }
00077 
00078 bool ConstantFPSDNode::isValueValidForType(EVT VT,
00079                                            const APFloat& Val) {
00080   assert(VT.isFloatingPoint() && "Can only convert between FP types");
00081 
00082   // convert modifies in place, so make a copy.
00083   APFloat Val2 = APFloat(Val);
00084   bool losesInfo;
00085   (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
00086                       APFloat::rmNearestTiesToEven,
00087                       &losesInfo);
00088   return !losesInfo;
00089 }
00090 
00091 //===----------------------------------------------------------------------===//
00092 //                              ISD Namespace
00093 //===----------------------------------------------------------------------===//
00094 
00095 /// isBuildVectorAllOnes - Return true if the specified node is a
00096 /// BUILD_VECTOR where all of the elements are ~0 or undef.
00097 bool ISD::isBuildVectorAllOnes(const SDNode *N) {
00098   // Look through a bit convert.
00099   while (N->getOpcode() == ISD::BITCAST)
00100     N = N->getOperand(0).getNode();
00101 
00102   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
00103 
00104   unsigned i = 0, e = N->getNumOperands();
00105 
00106   // Skip over all of the undef values.
00107   while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF)
00108     ++i;
00109 
00110   // Do not accept an all-undef vector.
00111   if (i == e) return false;
00112 
00113   // Do not accept build_vectors that aren't all constants or which have non-~0
00114   // elements. We have to be a bit careful here, as the type of the constant
00115   // may not be the same as the type of the vector elements due to type
00116   // legalization (the elements are promoted to a legal type for the target and
00117   // a vector of a type may be legal when the base element type is not).
00118   // We only want to check enough bits to cover the vector elements, because
00119   // we care if the resultant vector is all ones, not whether the individual
00120   // constants are.
00121   SDValue NotZero = N->getOperand(i);
00122   unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
00123   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
00124     if (CN->getAPIntValue().countTrailingOnes() < EltSize)
00125       return false;
00126   } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
00127     if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
00128       return false;
00129   } else
00130     return false;
00131 
00132   // Okay, we have at least one ~0 value, check to see if the rest match or are
00133   // undefs. Even with the above element type twiddling, this should be OK, as
00134   // the same type legalization should have applied to all the elements.
00135   for (++i; i != e; ++i)
00136     if (N->getOperand(i) != NotZero &&
00137         N->getOperand(i).getOpcode() != ISD::UNDEF)
00138       return false;
00139   return true;
00140 }
00141 
00142 
00143 /// isBuildVectorAllZeros - Return true if the specified node is a
00144 /// BUILD_VECTOR where all of the elements are 0 or undef.
00145 bool ISD::isBuildVectorAllZeros(const SDNode *N) {
00146   // Look through a bit convert.
00147   while (N->getOpcode() == ISD::BITCAST)
00148     N = N->getOperand(0).getNode();
00149 
00150   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
00151 
00152   bool IsAllUndef = true;
00153   for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) {
00154     if (N->getOperand(i).getOpcode() == ISD::UNDEF)
00155       continue;
00156     IsAllUndef = false;
00157     // Do not accept build_vectors that aren't all constants or which have non-0
00158     // elements. We have to be a bit careful here, as the type of the constant
00159     // may not be the same as the type of the vector elements due to type
00160     // legalization (the elements are promoted to a legal type for the target
00161     // and a vector of a type may be legal when the base element type is not).
00162     // We only want to check enough bits to cover the vector elements, because
00163     // we care if the resultant vector is all zeros, not whether the individual
00164     // constants are.
00165     SDValue Zero = N->getOperand(i);
00166     unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
00167     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Zero)) {
00168       if (CN->getAPIntValue().countTrailingZeros() < EltSize)
00169         return false;
00170     } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Zero)) {
00171       if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
00172         return false;
00173     } else
00174       return false;
00175   }
00176 
00177   // Do not accept an all-undef vector.
00178   if (IsAllUndef)
00179     return false;
00180   return true;
00181 }
00182 
00183 /// \brief Return true if the specified node is a BUILD_VECTOR node of
00184 /// all ConstantSDNode or undef.
00185 bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
00186   if (N->getOpcode() != ISD::BUILD_VECTOR)
00187     return false;
00188 
00189   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
00190     SDValue Op = N->getOperand(i);
00191     if (Op.getOpcode() == ISD::UNDEF)
00192       continue;
00193     if (!isa<ConstantSDNode>(Op))
00194       return false;
00195   }
00196   return true;
00197 }
00198 
00199 /// \brief Return true if the specified node is a BUILD_VECTOR node of
00200 /// all ConstantFPSDNode or undef.
00201 bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
00202   if (N->getOpcode() != ISD::BUILD_VECTOR)
00203     return false;
00204 
00205   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
00206     SDValue Op = N->getOperand(i);
00207     if (Op.getOpcode() == ISD::UNDEF)
00208       continue;
00209     if (!isa<ConstantFPSDNode>(Op))
00210       return false;
00211   }
00212   return true;
00213 }
00214 
00215 /// isScalarToVector - Return true if the specified node is a
00216 /// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low
00217 /// element is not an undef.
00218 bool ISD::isScalarToVector(const SDNode *N) {
00219   if (N->getOpcode() == ISD::SCALAR_TO_VECTOR)
00220     return true;
00221 
00222   if (N->getOpcode() != ISD::BUILD_VECTOR)
00223     return false;
00224   if (N->getOperand(0).getOpcode() == ISD::UNDEF)
00225     return false;
00226   unsigned NumElems = N->getNumOperands();
00227   if (NumElems == 1)
00228     return false;
00229   for (unsigned i = 1; i < NumElems; ++i) {
00230     SDValue V = N->getOperand(i);
00231     if (V.getOpcode() != ISD::UNDEF)
00232       return false;
00233   }
00234   return true;
00235 }
00236 
00237 /// allOperandsUndef - Return true if the node has at least one operand
00238 /// and all operands of the specified node are ISD::UNDEF.
00239 bool ISD::allOperandsUndef(const SDNode *N) {
00240   // Return false if the node has no operands.
00241   // This is "logically inconsistent" with the definition of "all" but
00242   // is probably the desired behavior.
00243   if (N->getNumOperands() == 0)
00244     return false;
00245 
00246   for (unsigned i = 0, e = N->getNumOperands(); i != e ; ++i)
00247     if (N->getOperand(i).getOpcode() != ISD::UNDEF)
00248       return false;
00249 
00250   return true;
00251 }
00252 
00253 ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
00254   switch (ExtType) {
00255   case ISD::EXTLOAD:
00256     return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
00257   case ISD::SEXTLOAD:
00258     return ISD::SIGN_EXTEND;
00259   case ISD::ZEXTLOAD:
00260     return ISD::ZERO_EXTEND;
00261   default:
00262     break;
00263   }
00264 
00265   llvm_unreachable("Invalid LoadExtType");
00266 }
00267 
00268 /// getSetCCSwappedOperands - Return the operation corresponding to (Y op X)
00269 /// when given the operation for (X op Y).
00270 ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
00271   // To perform this operation, we just need to swap the L and G bits of the
00272   // operation.
00273   unsigned OldL = (Operation >> 2) & 1;
00274   unsigned OldG = (Operation >> 1) & 1;
00275   return ISD::CondCode((Operation & ~6) |  // Keep the N, U, E bits
00276                        (OldL << 1) |       // New G bit
00277                        (OldG << 2));       // New L bit.
00278 }
00279 
00280 /// getSetCCInverse - Return the operation corresponding to !(X op Y), where
00281 /// 'op' is a valid SetCC operation.
00282 ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) {
00283   unsigned Operation = Op;
00284   if (isInteger)
00285     Operation ^= 7;   // Flip L, G, E bits, but not U.
00286   else
00287     Operation ^= 15;  // Flip all of the condition bits.
00288 
00289   if (Operation > ISD::SETTRUE2)
00290     Operation &= ~8;  // Don't let N and U bits get set.
00291 
00292   return ISD::CondCode(Operation);
00293 }
00294 
00295 
00296 /// isSignedOp - For an integer comparison, return 1 if the comparison is a
00297 /// signed operation and 2 if the result is an unsigned comparison.  Return zero
00298 /// if the operation does not depend on the sign of the input (setne and seteq).
00299 static int isSignedOp(ISD::CondCode Opcode) {
00300   switch (Opcode) {
00301   default: llvm_unreachable("Illegal integer setcc operation!");
00302   case ISD::SETEQ:
00303   case ISD::SETNE: return 0;
00304   case ISD::SETLT:
00305   case ISD::SETLE:
00306   case ISD::SETGT:
00307   case ISD::SETGE: return 1;
00308   case ISD::SETULT:
00309   case ISD::SETULE:
00310   case ISD::SETUGT:
00311   case ISD::SETUGE: return 2;
00312   }
00313 }
00314 
00315 /// getSetCCOrOperation - Return the result of a logical OR between different
00316 /// comparisons of identical values: ((X op1 Y) | (X op2 Y)).  This function
00317 /// returns SETCC_INVALID if it is not possible to represent the resultant
00318 /// comparison.
00319 ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
00320                                        bool isInteger) {
00321   if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
00322     // Cannot fold a signed integer setcc with an unsigned integer setcc.
00323     return ISD::SETCC_INVALID;
00324 
00325   unsigned Op = Op1 | Op2;  // Combine all of the condition bits.
00326 
00327   // If the N and U bits get set then the resultant comparison DOES suddenly
00328   // care about orderedness, and is true when ordered.
00329   if (Op > ISD::SETTRUE2)
00330     Op &= ~16;     // Clear the U bit if the N bit is set.
00331 
00332   // Canonicalize illegal integer setcc's.
00333   if (isInteger && Op == ISD::SETUNE)  // e.g. SETUGT | SETULT
00334     Op = ISD::SETNE;
00335 
00336   return ISD::CondCode(Op);
00337 }
00338 
00339 /// getSetCCAndOperation - Return the result of a logical AND between different
00340 /// comparisons of identical values: ((X op1 Y) & (X op2 Y)).  This
00341 /// function returns zero if it is not possible to represent the resultant
00342 /// comparison.
00343 ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
00344                                         bool isInteger) {
00345   if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
00346     // Cannot fold a signed setcc with an unsigned setcc.
00347     return ISD::SETCC_INVALID;
00348 
00349   // Combine all of the condition bits.
00350   ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
00351 
00352   // Canonicalize illegal integer setcc's.
00353   if (isInteger) {
00354     switch (Result) {
00355     default: break;
00356     case ISD::SETUO : Result = ISD::SETFALSE; break;  // SETUGT & SETULT
00357     case ISD::SETOEQ:                                 // SETEQ  & SETU[LG]E
00358     case ISD::SETUEQ: Result = ISD::SETEQ   ; break;  // SETUGE & SETULE
00359     case ISD::SETOLT: Result = ISD::SETULT  ; break;  // SETULT & SETNE
00360     case ISD::SETOGT: Result = ISD::SETUGT  ; break;  // SETUGT & SETNE
00361     }
00362   }
00363 
00364   return Result;
00365 }
00366 
00367 //===----------------------------------------------------------------------===//
00368 //                           SDNode Profile Support
00369 //===----------------------------------------------------------------------===//
00370 
00371 /// AddNodeIDOpcode - Add the node opcode to the NodeID data.
00372 ///
00373 static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC)  {
00374   ID.AddInteger(OpC);
00375 }
00376 
00377 /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
00378 /// solely with their pointer.
00379 static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
00380   ID.AddPointer(VTList.VTs);
00381 }
00382 
00383 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
00384 ///
00385 static void AddNodeIDOperands(FoldingSetNodeID &ID,
00386                               ArrayRef<SDValue> Ops) {
00387   for (auto& Op : Ops) {
00388     ID.AddPointer(Op.getNode());
00389     ID.AddInteger(Op.getResNo());
00390   }
00391 }
00392 
00393 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
00394 ///
00395 static void AddNodeIDOperands(FoldingSetNodeID &ID,
00396                               ArrayRef<SDUse> Ops) {
00397   for (auto& Op : Ops) {
00398     ID.AddPointer(Op.getNode());
00399     ID.AddInteger(Op.getResNo());
00400   }
00401 }
00402 
00403 static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, bool nuw, bool nsw,
00404                                   bool exact) {
00405   ID.AddBoolean(nuw);
00406   ID.AddBoolean(nsw);
00407   ID.AddBoolean(exact);
00408 }
00409 
00410 /// AddBinaryNodeIDCustom - Add BinarySDNodes special infos
00411 static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, unsigned Opcode,
00412                                   bool nuw, bool nsw, bool exact) {
00413   if (isBinOpWithFlags(Opcode))
00414     AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
00415 }
00416 
00417 static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
00418                           SDVTList VTList, ArrayRef<SDValue> OpList) {
00419   AddNodeIDOpcode(ID, OpC);
00420   AddNodeIDValueTypes(ID, VTList);
00421   AddNodeIDOperands(ID, OpList);
00422 }
00423 
00424 /// AddNodeIDCustom - If this is an SDNode with special info, add this info to
00425 /// the NodeID data.
00426 static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
00427   switch (N->getOpcode()) {
00428   case ISD::TargetExternalSymbol:
00429   case ISD::ExternalSymbol:
00430     llvm_unreachable("Should only be used on nodes with operands");
00431   default: break;  // Normal nodes don't need extra info.
00432   case ISD::TargetConstant:
00433   case ISD::Constant: {
00434     const ConstantSDNode *C = cast<ConstantSDNode>(N);
00435     ID.AddPointer(C->getConstantIntValue());
00436     ID.AddBoolean(C->isOpaque());
00437     break;
00438   }
00439   case ISD::TargetConstantFP:
00440   case ISD::ConstantFP: {
00441     ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
00442     break;
00443   }
00444   case ISD::TargetGlobalAddress:
00445   case ISD::GlobalAddress:
00446   case ISD::TargetGlobalTLSAddress:
00447   case ISD::GlobalTLSAddress: {
00448     const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
00449     ID.AddPointer(GA->getGlobal());
00450     ID.AddInteger(GA->getOffset());
00451     ID.AddInteger(GA->getTargetFlags());
00452     ID.AddInteger(GA->getAddressSpace());
00453     break;
00454   }
00455   case ISD::BasicBlock:
00456     ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
00457     break;
00458   case ISD::Register:
00459     ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
00460     break;
00461   case ISD::RegisterMask:
00462     ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
00463     break;
00464   case ISD::SRCVALUE:
00465     ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
00466     break;
00467   case ISD::FrameIndex:
00468   case ISD::TargetFrameIndex:
00469     ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
00470     break;
00471   case ISD::JumpTable:
00472   case ISD::TargetJumpTable:
00473     ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
00474     ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
00475     break;
00476   case ISD::ConstantPool:
00477   case ISD::TargetConstantPool: {
00478     const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
00479     ID.AddInteger(CP->getAlignment());
00480     ID.AddInteger(CP->getOffset());
00481     if (CP->isMachineConstantPoolEntry())
00482       CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
00483     else
00484       ID.AddPointer(CP->getConstVal());
00485     ID.AddInteger(CP->getTargetFlags());
00486     break;
00487   }
00488   case ISD::TargetIndex: {
00489     const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
00490     ID.AddInteger(TI->getIndex());
00491     ID.AddInteger(TI->getOffset());
00492     ID.AddInteger(TI->getTargetFlags());
00493     break;
00494   }
00495   case ISD::LOAD: {
00496     const LoadSDNode *LD = cast<LoadSDNode>(N);
00497     ID.AddInteger(LD->getMemoryVT().getRawBits());
00498     ID.AddInteger(LD->getRawSubclassData());
00499     ID.AddInteger(LD->getPointerInfo().getAddrSpace());
00500     break;
00501   }
00502   case ISD::STORE: {
00503     const StoreSDNode *ST = cast<StoreSDNode>(N);
00504     ID.AddInteger(ST->getMemoryVT().getRawBits());
00505     ID.AddInteger(ST->getRawSubclassData());
00506     ID.AddInteger(ST->getPointerInfo().getAddrSpace());
00507     break;
00508   }
00509   case ISD::SDIV:
00510   case ISD::UDIV:
00511   case ISD::SRA:
00512   case ISD::SRL:
00513   case ISD::MUL:
00514   case ISD::ADD:
00515   case ISD::SUB:
00516   case ISD::SHL: {
00517     const BinaryWithFlagsSDNode *BinNode = cast<BinaryWithFlagsSDNode>(N);
00518     AddBinaryNodeIDCustom(ID, N->getOpcode(), BinNode->hasNoUnsignedWrap(),
00519                           BinNode->hasNoSignedWrap(), BinNode->isExact());
00520     break;
00521   }
00522   case ISD::ATOMIC_CMP_SWAP:
00523   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
00524   case ISD::ATOMIC_SWAP:
00525   case ISD::ATOMIC_LOAD_ADD:
00526   case ISD::ATOMIC_LOAD_SUB:
00527   case ISD::ATOMIC_LOAD_AND:
00528   case ISD::ATOMIC_LOAD_OR:
00529   case ISD::ATOMIC_LOAD_XOR:
00530   case ISD::ATOMIC_LOAD_NAND:
00531   case ISD::ATOMIC_LOAD_MIN:
00532   case ISD::ATOMIC_LOAD_MAX:
00533   case ISD::ATOMIC_LOAD_UMIN:
00534   case ISD::ATOMIC_LOAD_UMAX:
00535   case ISD::ATOMIC_LOAD:
00536   case ISD::ATOMIC_STORE: {
00537     const AtomicSDNode *AT = cast<AtomicSDNode>(N);
00538     ID.AddInteger(AT->getMemoryVT().getRawBits());
00539     ID.AddInteger(AT->getRawSubclassData());
00540     ID.AddInteger(AT->getPointerInfo().getAddrSpace());
00541     break;
00542   }
00543   case ISD::PREFETCH: {
00544     const MemSDNode *PF = cast<MemSDNode>(N);
00545     ID.AddInteger(PF->getPointerInfo().getAddrSpace());
00546     break;
00547   }
00548   case ISD::VECTOR_SHUFFLE: {
00549     const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
00550     for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
00551          i != e; ++i)
00552       ID.AddInteger(SVN->getMaskElt(i));
00553     break;
00554   }
00555   case ISD::TargetBlockAddress:
00556   case ISD::BlockAddress: {
00557     const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
00558     ID.AddPointer(BA->getBlockAddress());
00559     ID.AddInteger(BA->getOffset());
00560     ID.AddInteger(BA->getTargetFlags());
00561     break;
00562   }
00563   } // end switch (N->getOpcode())
00564 
00565   // Target specific memory nodes could also have address spaces to check.
00566   if (N->isTargetMemoryOpcode())
00567     ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
00568 }
00569 
00570 /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
00571 /// data.
00572 static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
00573   AddNodeIDOpcode(ID, N->getOpcode());
00574   // Add the return value info.
00575   AddNodeIDValueTypes(ID, N->getVTList());
00576   // Add the operand info.
00577   AddNodeIDOperands(ID, N->ops());
00578 
00579   // Handle SDNode leafs with special info.
00580   AddNodeIDCustom(ID, N);
00581 }
00582 
00583 /// encodeMemSDNodeFlags - Generic routine for computing a value for use in
00584 /// the CSE map that carries volatility, temporalness, indexing mode, and
00585 /// extension/truncation information.
00586 ///
00587 static inline unsigned
00588 encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile,
00589                      bool isNonTemporal, bool isInvariant) {
00590   assert((ConvType & 3) == ConvType &&
00591          "ConvType may not require more than 2 bits!");
00592   assert((AM & 7) == AM &&
00593          "AM may not require more than 3 bits!");
00594   return ConvType |
00595          (AM << 2) |
00596          (isVolatile << 5) |
00597          (isNonTemporal << 6) |
00598          (isInvariant << 7);
00599 }
00600 
00601 //===----------------------------------------------------------------------===//
00602 //                              SelectionDAG Class
00603 //===----------------------------------------------------------------------===//
00604 
00605 /// doNotCSE - Return true if CSE should not be performed for this node.
00606 static bool doNotCSE(SDNode *N) {
00607   if (N->getValueType(0) == MVT::Glue)
00608     return true; // Never CSE anything that produces a flag.
00609 
00610   switch (N->getOpcode()) {
00611   default: break;
00612   case ISD::HANDLENODE:
00613   case ISD::EH_LABEL:
00614     return true;   // Never CSE these nodes.
00615   }
00616 
00617   // Check that remaining values produced are not flags.
00618   for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
00619     if (N->getValueType(i) == MVT::Glue)
00620       return true; // Never CSE anything that produces a flag.
00621 
00622   return false;
00623 }
00624 
00625 /// RemoveDeadNodes - This method deletes all unreachable nodes in the
00626 /// SelectionDAG.
00627 void SelectionDAG::RemoveDeadNodes() {
00628   // Create a dummy node (which is not added to allnodes), that adds a reference
00629   // to the root node, preventing it from being deleted.
00630   HandleSDNode Dummy(getRoot());
00631 
00632   SmallVector<SDNode*, 128> DeadNodes;
00633 
00634   // Add all obviously-dead nodes to the DeadNodes worklist.
00635   for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I)
00636     if (I->use_empty())
00637       DeadNodes.push_back(I);
00638 
00639   RemoveDeadNodes(DeadNodes);
00640 
00641   // If the root changed (e.g. it was a dead load, update the root).
00642   setRoot(Dummy.getValue());
00643 }
00644 
00645 /// RemoveDeadNodes - This method deletes the unreachable nodes in the
00646 /// given list, and any nodes that become unreachable as a result.
00647 void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
00648 
00649   // Process the worklist, deleting the nodes and adding their uses to the
00650   // worklist.
00651   while (!DeadNodes.empty()) {
00652     SDNode *N = DeadNodes.pop_back_val();
00653 
00654     for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00655       DUL->NodeDeleted(N, nullptr);
00656 
00657     // Take the node out of the appropriate CSE map.
00658     RemoveNodeFromCSEMaps(N);
00659 
00660     // Next, brutally remove the operand list.  This is safe to do, as there are
00661     // no cycles in the graph.
00662     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
00663       SDUse &Use = *I++;
00664       SDNode *Operand = Use.getNode();
00665       Use.set(SDValue());
00666 
00667       // Now that we removed this operand, see if there are no uses of it left.
00668       if (Operand->use_empty())
00669         DeadNodes.push_back(Operand);
00670     }
00671 
00672     DeallocateNode(N);
00673   }
00674 }
00675 
00676 void SelectionDAG::RemoveDeadNode(SDNode *N){
00677   SmallVector<SDNode*, 16> DeadNodes(1, N);
00678 
00679   // Create a dummy node that adds a reference to the root node, preventing
00680   // it from being deleted.  (This matters if the root is an operand of the
00681   // dead node.)
00682   HandleSDNode Dummy(getRoot());
00683 
00684   RemoveDeadNodes(DeadNodes);
00685 }
00686 
00687 void SelectionDAG::DeleteNode(SDNode *N) {
00688   // First take this out of the appropriate CSE map.
00689   RemoveNodeFromCSEMaps(N);
00690 
00691   // Finally, remove uses due to operands of this node, remove from the
00692   // AllNodes list, and delete the node.
00693   DeleteNodeNotInCSEMaps(N);
00694 }
00695 
00696 void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
00697   assert(N != AllNodes.begin() && "Cannot delete the entry node!");
00698   assert(N->use_empty() && "Cannot delete a node that is not dead!");
00699 
00700   // Drop all of the operands and decrement used node's use counts.
00701   N->DropOperands();
00702 
00703   DeallocateNode(N);
00704 }
00705 
00706 void SDDbgInfo::erase(const SDNode *Node) {
00707   DbgValMapType::iterator I = DbgValMap.find(Node);
00708   if (I == DbgValMap.end())
00709     return;
00710   for (auto &Val: I->second)
00711     Val->setIsInvalidated();
00712   DbgValMap.erase(I);
00713 }
00714 
00715 void SelectionDAG::DeallocateNode(SDNode *N) {
00716   if (N->OperandsNeedDelete)
00717     delete[] N->OperandList;
00718 
00719   // Set the opcode to DELETED_NODE to help catch bugs when node
00720   // memory is reallocated.
00721   N->NodeType = ISD::DELETED_NODE;
00722 
00723   NodeAllocator.Deallocate(AllNodes.remove(N));
00724 
00725   // If any of the SDDbgValue nodes refer to this SDNode, invalidate
00726   // them and forget about that node.
00727   DbgInfo->erase(N);
00728 }
00729 
00730 #ifndef NDEBUG
00731 /// VerifySDNode - Sanity check the given SDNode.  Aborts if it is invalid.
00732 static void VerifySDNode(SDNode *N) {
00733   switch (N->getOpcode()) {
00734   default:
00735     break;
00736   case ISD::BUILD_PAIR: {
00737     EVT VT = N->getValueType(0);
00738     assert(N->getNumValues() == 1 && "Too many results!");
00739     assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
00740            "Wrong return type!");
00741     assert(N->getNumOperands() == 2 && "Wrong number of operands!");
00742     assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
00743            "Mismatched operand types!");
00744     assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
00745            "Wrong operand type!");
00746     assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
00747            "Wrong return type size");
00748     break;
00749   }
00750   case ISD::BUILD_VECTOR: {
00751     assert(N->getNumValues() == 1 && "Too many results!");
00752     assert(N->getValueType(0).isVector() && "Wrong return type!");
00753     assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
00754            "Wrong number of operands!");
00755     EVT EltVT = N->getValueType(0).getVectorElementType();
00756     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
00757       assert((I->getValueType() == EltVT ||
00758              (EltVT.isInteger() && I->getValueType().isInteger() &&
00759               EltVT.bitsLE(I->getValueType()))) &&
00760             "Wrong operand type!");
00761       assert(I->getValueType() == N->getOperand(0).getValueType() &&
00762              "Operands must all have the same type");
00763     }
00764     break;
00765   }
00766   }
00767 }
00768 #endif // NDEBUG
00769 
00770 /// \brief Insert a newly allocated node into the DAG.
00771 ///
00772 /// Handles insertion into the all nodes list and CSE map, as well as
00773 /// verification and other common operations when a new node is allocated.
00774 void SelectionDAG::InsertNode(SDNode *N) {
00775   AllNodes.push_back(N);
00776 #ifndef NDEBUG
00777   VerifySDNode(N);
00778 #endif
00779 }
00780 
00781 /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
00782 /// correspond to it.  This is useful when we're about to delete or repurpose
00783 /// the node.  We don't want future request for structurally identical nodes
00784 /// to return N anymore.
00785 bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
00786   bool Erased = false;
00787   switch (N->getOpcode()) {
00788   case ISD::HANDLENODE: return false;  // noop.
00789   case ISD::CONDCODE:
00790     assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
00791            "Cond code doesn't exist!");
00792     Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
00793     CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
00794     break;
00795   case ISD::ExternalSymbol:
00796     Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
00797     break;
00798   case ISD::TargetExternalSymbol: {
00799     ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
00800     Erased = TargetExternalSymbols.erase(
00801                std::pair<std::string,unsigned char>(ESN->getSymbol(),
00802                                                     ESN->getTargetFlags()));
00803     break;
00804   }
00805   case ISD::VALUETYPE: {
00806     EVT VT = cast<VTSDNode>(N)->getVT();
00807     if (VT.isExtended()) {
00808       Erased = ExtendedValueTypeNodes.erase(VT);
00809     } else {
00810       Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
00811       ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
00812     }
00813     break;
00814   }
00815   default:
00816     // Remove it from the CSE Map.
00817     assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
00818     assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
00819     Erased = CSEMap.RemoveNode(N);
00820     break;
00821   }
00822 #ifndef NDEBUG
00823   // Verify that the node was actually in one of the CSE maps, unless it has a
00824   // flag result (which cannot be CSE'd) or is one of the special cases that are
00825   // not subject to CSE.
00826   if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
00827       !N->isMachineOpcode() && !doNotCSE(N)) {
00828     N->dump(this);
00829     dbgs() << "\n";
00830     llvm_unreachable("Node is not in map!");
00831   }
00832 #endif
00833   return Erased;
00834 }
00835 
00836 /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
00837 /// maps and modified in place. Add it back to the CSE maps, unless an identical
00838 /// node already exists, in which case transfer all its users to the existing
00839 /// node. This transfer can potentially trigger recursive merging.
00840 ///
00841 void
00842 SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
00843   // For node types that aren't CSE'd, just act as if no identical node
00844   // already exists.
00845   if (!doNotCSE(N)) {
00846     SDNode *Existing = CSEMap.GetOrInsertNode(N);
00847     if (Existing != N) {
00848       // If there was already an existing matching node, use ReplaceAllUsesWith
00849       // to replace the dead one with the existing one.  This can cause
00850       // recursive merging of other unrelated nodes down the line.
00851       ReplaceAllUsesWith(N, Existing);
00852 
00853       // N is now dead. Inform the listeners and delete it.
00854       for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00855         DUL->NodeDeleted(N, Existing);
00856       DeleteNodeNotInCSEMaps(N);
00857       return;
00858     }
00859   }
00860 
00861   // If the node doesn't already exist, we updated it.  Inform listeners.
00862   for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00863     DUL->NodeUpdated(N);
00864 }
00865 
00866 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00867 /// were replaced with those specified.  If this node is never memoized,
00868 /// return null, otherwise return a pointer to the slot it would take.  If a
00869 /// node already exists with these operands, the slot will be non-null.
00870 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
00871                                            void *&InsertPos) {
00872   if (doNotCSE(N))
00873     return nullptr;
00874 
00875   SDValue Ops[] = { Op };
00876   FoldingSetNodeID ID;
00877   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00878   AddNodeIDCustom(ID, N);
00879   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00880   return Node;
00881 }
00882 
00883 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00884 /// were replaced with those specified.  If this node is never memoized,
00885 /// return null, otherwise return a pointer to the slot it would take.  If a
00886 /// node already exists with these operands, the slot will be non-null.
00887 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
00888                                            SDValue Op1, SDValue Op2,
00889                                            void *&InsertPos) {
00890   if (doNotCSE(N))
00891     return nullptr;
00892 
00893   SDValue Ops[] = { Op1, Op2 };
00894   FoldingSetNodeID ID;
00895   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00896   AddNodeIDCustom(ID, N);
00897   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00898   return Node;
00899 }
00900 
00901 
00902 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00903 /// were replaced with those specified.  If this node is never memoized,
00904 /// return null, otherwise return a pointer to the slot it would take.  If a
00905 /// node already exists with these operands, the slot will be non-null.
00906 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
00907                                            void *&InsertPos) {
00908   if (doNotCSE(N))
00909     return nullptr;
00910 
00911   FoldingSetNodeID ID;
00912   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00913   AddNodeIDCustom(ID, N);
00914   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00915   return Node;
00916 }
00917 
00918 /// getEVTAlignment - Compute the default alignment value for the
00919 /// given type.
00920 ///
00921 unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
00922   Type *Ty = VT == MVT::iPTR ?
00923                    PointerType::get(Type::getInt8Ty(*getContext()), 0) :
00924                    VT.getTypeForEVT(*getContext());
00925 
00926   return TLI->getDataLayout()->getABITypeAlignment(Ty);
00927 }
00928 
00929 // EntryNode could meaningfully have debug info if we can find it...
00930 SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
00931     : TM(tm), TSI(nullptr), TLI(nullptr), OptLevel(OL),
00932       EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
00933       Root(getEntryNode()), NewNodesMustHaveLegalTypes(false),
00934       UpdateListeners(nullptr) {
00935   AllNodes.push_back(&EntryNode);
00936   DbgInfo = new SDDbgInfo();
00937 }
00938 
00939 void SelectionDAG::init(MachineFunction &mf) {
00940   MF = &mf;
00941   TLI = getSubtarget().getTargetLowering();
00942   TSI = getSubtarget().getSelectionDAGInfo();
00943   Context = &mf.getFunction()->getContext();
00944 }
00945 
00946 SelectionDAG::~SelectionDAG() {
00947   assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
00948   allnodes_clear();
00949   delete DbgInfo;
00950 }
00951 
00952 void SelectionDAG::allnodes_clear() {
00953   assert(&*AllNodes.begin() == &EntryNode);
00954   AllNodes.remove(AllNodes.begin());
00955   while (!AllNodes.empty())
00956     DeallocateNode(AllNodes.begin());
00957 }
00958 
00959 BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL,
00960                                             SDVTList VTs, SDValue N1,
00961                                             SDValue N2, bool nuw, bool nsw,
00962                                             bool exact) {
00963   if (isBinOpWithFlags(Opcode)) {
00964     BinaryWithFlagsSDNode *FN = new (NodeAllocator) BinaryWithFlagsSDNode(
00965         Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
00966     FN->setHasNoUnsignedWrap(nuw);
00967     FN->setHasNoSignedWrap(nsw);
00968     FN->setIsExact(exact);
00969 
00970     return FN;
00971   }
00972 
00973   BinarySDNode *N = new (NodeAllocator)
00974       BinarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
00975   return N;
00976 }
00977 
00978 void SelectionDAG::clear() {
00979   allnodes_clear();
00980   OperandAllocator.Reset();
00981   CSEMap.clear();
00982 
00983   ExtendedValueTypeNodes.clear();
00984   ExternalSymbols.clear();
00985   TargetExternalSymbols.clear();
00986   std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
00987             static_cast<CondCodeSDNode*>(nullptr));
00988   std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
00989             static_cast<SDNode*>(nullptr));
00990 
00991   EntryNode.UseList = nullptr;
00992   AllNodes.push_back(&EntryNode);
00993   Root = getEntryNode();
00994   DbgInfo->clear();
00995 }
00996 
00997 SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00998   return VT.bitsGT(Op.getValueType()) ?
00999     getNode(ISD::ANY_EXTEND, DL, VT, Op) :
01000     getNode(ISD::TRUNCATE, DL, VT, Op);
01001 }
01002 
01003 SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
01004   return VT.bitsGT(Op.getValueType()) ?
01005     getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
01006     getNode(ISD::TRUNCATE, DL, VT, Op);
01007 }
01008 
01009 SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
01010   return VT.bitsGT(Op.getValueType()) ?
01011     getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
01012     getNode(ISD::TRUNCATE, DL, VT, Op);
01013 }
01014 
01015 SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, SDLoc SL, EVT VT,
01016                                         EVT OpVT) {
01017   if (VT.bitsLE(Op.getValueType()))
01018     return getNode(ISD::TRUNCATE, SL, VT, Op);
01019 
01020   TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
01021   return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
01022 }
01023 
01024 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, SDLoc DL, EVT VT) {
01025   assert(!VT.isVector() &&
01026          "getZeroExtendInReg should use the vector element type instead of "
01027          "the vector type!");
01028   if (Op.getValueType() == VT) return Op;
01029   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01030   APInt Imm = APInt::getLowBitsSet(BitWidth,
01031                                    VT.getSizeInBits());
01032   return getNode(ISD::AND, DL, Op.getValueType(), Op,
01033                  getConstant(Imm, Op.getValueType()));
01034 }
01035 
01036 SDValue SelectionDAG::getAnyExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01037   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01038   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01039          "The sizes of the input and result must match in order to perform the "
01040          "extend in-register.");
01041   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01042          "The destination vector type must have fewer lanes than the input.");
01043   return getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Op);
01044 }
01045 
01046 SDValue SelectionDAG::getSignExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01047   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01048   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01049          "The sizes of the input and result must match in order to perform the "
01050          "extend in-register.");
01051   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01052          "The destination vector type must have fewer lanes than the input.");
01053   return getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Op);
01054 }
01055 
01056 SDValue SelectionDAG::getZeroExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01057   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01058   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01059          "The sizes of the input and result must match in order to perform the "
01060          "extend in-register.");
01061   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01062          "The destination vector type must have fewer lanes than the input.");
01063   return getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Op);
01064 }
01065 
01066 /// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
01067 ///
01068 SDValue SelectionDAG::getNOT(SDLoc DL, SDValue Val, EVT VT) {
01069   EVT EltVT = VT.getScalarType();
01070   SDValue NegOne =
01071     getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT);
01072   return getNode(ISD::XOR, DL, VT, Val, NegOne);
01073 }
01074 
01075 SDValue SelectionDAG::getLogicalNOT(SDLoc DL, SDValue Val, EVT VT) {
01076   EVT EltVT = VT.getScalarType();
01077   SDValue TrueValue;
01078   switch (TLI->getBooleanContents(VT)) {
01079     case TargetLowering::ZeroOrOneBooleanContent:
01080     case TargetLowering::UndefinedBooleanContent:
01081       TrueValue = getConstant(1, VT);
01082       break;
01083     case TargetLowering::ZeroOrNegativeOneBooleanContent:
01084       TrueValue = getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()),
01085                               VT);
01086       break;
01087   }
01088   return getNode(ISD::XOR, DL, VT, Val, TrueValue);
01089 }
01090 
01091 SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT, bool isO) {
01092   EVT EltVT = VT.getScalarType();
01093   assert((EltVT.getSizeInBits() >= 64 ||
01094          (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
01095          "getConstant with a uint64_t value that doesn't fit in the type!");
01096   return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT, isO);
01097 }
01098 
01099 SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT, bool isO)
01100 {
01101   return getConstant(*ConstantInt::get(*Context, Val), VT, isT, isO);
01102 }
01103 
01104 SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT,
01105                                   bool isO) {
01106   assert(VT.isInteger() && "Cannot create FP integer constant!");
01107 
01108   EVT EltVT = VT.getScalarType();
01109   const ConstantInt *Elt = &Val;
01110 
01111   // In some cases the vector type is legal but the element type is illegal and
01112   // needs to be promoted, for example v8i8 on ARM.  In this case, promote the
01113   // inserted value (the type does not need to match the vector element type).
01114   // Any extra bits introduced will be truncated away.
01115   if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
01116       TargetLowering::TypePromoteInteger) {
01117    EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
01118    APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits());
01119    Elt = ConstantInt::get(*getContext(), NewVal);
01120   }
01121   // In other cases the element type is illegal and needs to be expanded, for
01122   // example v2i64 on MIPS32. In this case, find the nearest legal type, split
01123   // the value into n parts and use a vector type with n-times the elements.
01124   // Then bitcast to the type requested.
01125   // Legalizing constants too early makes the DAGCombiner's job harder so we
01126   // only legalize if the DAG tells us we must produce legal types.
01127   else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
01128            TLI->getTypeAction(*getContext(), EltVT) ==
01129            TargetLowering::TypeExpandInteger) {
01130     APInt NewVal = Elt->getValue();
01131     EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
01132     unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
01133     unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
01134     EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
01135 
01136     // Check the temporary vector is the correct size. If this fails then
01137     // getTypeToTransformTo() probably returned a type whose size (in bits)
01138     // isn't a power-of-2 factor of the requested type size.
01139     assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
01140 
01141     SmallVector<SDValue, 2> EltParts;
01142     for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
01143       EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
01144                                            .trunc(ViaEltSizeInBits),
01145                                      ViaEltVT, isT, isO));
01146     }
01147 
01148     // EltParts is currently in little endian order. If we actually want
01149     // big-endian order then reverse it now.
01150     if (TLI->isBigEndian())
01151       std::reverse(EltParts.begin(), EltParts.end());
01152 
01153     // The elements must be reversed when the element order is different
01154     // to the endianness of the elements (because the BITCAST is itself a
01155     // vector shuffle in this situation). However, we do not need any code to
01156     // perform this reversal because getConstant() is producing a vector
01157     // splat.
01158     // This situation occurs in MIPS MSA.
01159 
01160     SmallVector<SDValue, 8> Ops;
01161     for (unsigned i = 0; i < VT.getVectorNumElements(); ++i)
01162       Ops.insert(Ops.end(), EltParts.begin(), EltParts.end());
01163 
01164     SDValue Result = getNode(ISD::BITCAST, SDLoc(), VT,
01165                              getNode(ISD::BUILD_VECTOR, SDLoc(), ViaVecVT,
01166                                      Ops));
01167     return Result;
01168   }
01169 
01170   assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
01171          "APInt size does not match type size!");
01172   unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
01173   FoldingSetNodeID ID;
01174   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
01175   ID.AddPointer(Elt);
01176   ID.AddBoolean(isO);
01177   void *IP = nullptr;
01178   SDNode *N = nullptr;
01179   if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
01180     if (!VT.isVector())
01181       return SDValue(N, 0);
01182 
01183   if (!N) {
01184     N = new (NodeAllocator) ConstantSDNode(isT, isO, Elt, EltVT);
01185     CSEMap.InsertNode(N, IP);
01186     InsertNode(N);
01187   }
01188 
01189   SDValue Result(N, 0);
01190   if (VT.isVector()) {
01191     SmallVector<SDValue, 8> Ops;
01192     Ops.assign(VT.getVectorNumElements(), Result);
01193     Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
01194   }
01195   return Result;
01196 }
01197 
01198 SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) {
01199   return getConstant(Val, TLI->getPointerTy(), isTarget);
01200 }
01201 
01202 
01203 SDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) {
01204   return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget);
01205 }
01206 
01207 SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){
01208   assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
01209 
01210   EVT EltVT = VT.getScalarType();
01211 
01212   // Do the map lookup using the actual bit pattern for the floating point
01213   // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
01214   // we don't have issues with SNANs.
01215   unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
01216   FoldingSetNodeID ID;
01217   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
01218   ID.AddPointer(&V);
01219   void *IP = nullptr;
01220   SDNode *N = nullptr;
01221   if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
01222     if (!VT.isVector())
01223       return SDValue(N, 0);
01224 
01225   if (!N) {
01226     N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT);
01227     CSEMap.InsertNode(N, IP);
01228     InsertNode(N);
01229   }
01230 
01231   SDValue Result(N, 0);
01232   if (VT.isVector()) {
01233     SmallVector<SDValue, 8> Ops;
01234     Ops.assign(VT.getVectorNumElements(), Result);
01235     // FIXME SDLoc info might be appropriate here
01236     Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
01237   }
01238   return Result;
01239 }
01240 
01241 SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) {
01242   EVT EltVT = VT.getScalarType();
01243   if (EltVT==MVT::f32)
01244     return getConstantFP(APFloat((float)Val), VT, isTarget);
01245   else if (EltVT==MVT::f64)
01246     return getConstantFP(APFloat(Val), VT, isTarget);
01247   else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 ||
01248            EltVT==MVT::f16) {
01249     bool ignored;
01250     APFloat apf = APFloat(Val);
01251     apf.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
01252                 &ignored);
01253     return getConstantFP(apf, VT, isTarget);
01254   } else
01255     llvm_unreachable("Unsupported type in getConstantFP");
01256 }
01257 
01258 SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, SDLoc DL,
01259                                        EVT VT, int64_t Offset,
01260                                        bool isTargetGA,
01261                                        unsigned char TargetFlags) {
01262   assert((TargetFlags == 0 || isTargetGA) &&
01263          "Cannot set target flags on target-independent globals");
01264 
01265   // Truncate (with sign-extension) the offset value to the pointer size.
01266   unsigned BitWidth = TLI->getPointerTypeSizeInBits(GV->getType());
01267   if (BitWidth < 64)
01268     Offset = SignExtend64(Offset, BitWidth);
01269 
01270   unsigned Opc;
01271   if (GV->isThreadLocal())
01272     Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
01273   else
01274     Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
01275 
01276   FoldingSetNodeID ID;
01277   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01278   ID.AddPointer(GV);
01279   ID.AddInteger(Offset);
01280   ID.AddInteger(TargetFlags);
01281   ID.AddInteger(GV->getType()->getAddressSpace());
01282   void *IP = nullptr;
01283   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01284     return SDValue(E, 0);
01285 
01286   SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL.getIROrder(),
01287                                                       DL.getDebugLoc(), GV, VT,
01288                                                       Offset, TargetFlags);
01289   CSEMap.InsertNode(N, IP);
01290     InsertNode(N);
01291   return SDValue(N, 0);
01292 }
01293 
01294 SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
01295   unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
01296   FoldingSetNodeID ID;
01297   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01298   ID.AddInteger(FI);
01299   void *IP = nullptr;
01300   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01301     return SDValue(E, 0);
01302 
01303   SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget);
01304   CSEMap.InsertNode(N, IP);
01305   InsertNode(N);
01306   return SDValue(N, 0);
01307 }
01308 
01309 SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
01310                                    unsigned char TargetFlags) {
01311   assert((TargetFlags == 0 || isTarget) &&
01312          "Cannot set target flags on target-independent jump tables");
01313   unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
01314   FoldingSetNodeID ID;
01315   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01316   ID.AddInteger(JTI);
01317   ID.AddInteger(TargetFlags);
01318   void *IP = nullptr;
01319   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01320     return SDValue(E, 0);
01321 
01322   SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget,
01323                                                   TargetFlags);
01324   CSEMap.InsertNode(N, IP);
01325   InsertNode(N);
01326   return SDValue(N, 0);
01327 }
01328 
01329 SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
01330                                       unsigned Alignment, int Offset,
01331                                       bool isTarget,
01332                                       unsigned char TargetFlags) {
01333   assert((TargetFlags == 0 || isTarget) &&
01334          "Cannot set target flags on target-independent globals");
01335   if (Alignment == 0)
01336     Alignment = TLI->getDataLayout()->getPrefTypeAlignment(C->getType());
01337   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
01338   FoldingSetNodeID ID;
01339   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01340   ID.AddInteger(Alignment);
01341   ID.AddInteger(Offset);
01342   ID.AddPointer(C);
01343   ID.AddInteger(TargetFlags);
01344   void *IP = nullptr;
01345   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01346     return SDValue(E, 0);
01347 
01348   SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
01349                                                      Alignment, TargetFlags);
01350   CSEMap.InsertNode(N, IP);
01351   InsertNode(N);
01352   return SDValue(N, 0);
01353 }
01354 
01355 
01356 SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
01357                                       unsigned Alignment, int Offset,
01358                                       bool isTarget,
01359                                       unsigned char TargetFlags) {
01360   assert((TargetFlags == 0 || isTarget) &&
01361          "Cannot set target flags on target-independent globals");
01362   if (Alignment == 0)
01363     Alignment = TLI->getDataLayout()->getPrefTypeAlignment(C->getType());
01364   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
01365   FoldingSetNodeID ID;
01366   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01367   ID.AddInteger(Alignment);
01368   ID.AddInteger(Offset);
01369   C->addSelectionDAGCSEId(ID);
01370   ID.AddInteger(TargetFlags);
01371   void *IP = nullptr;
01372   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01373     return SDValue(E, 0);
01374 
01375   SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
01376                                                      Alignment, TargetFlags);
01377   CSEMap.InsertNode(N, IP);
01378   InsertNode(N);
01379   return SDValue(N, 0);
01380 }
01381 
01382 SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
01383                                      unsigned char TargetFlags) {
01384   FoldingSetNodeID ID;
01385   AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None);
01386   ID.AddInteger(Index);
01387   ID.AddInteger(Offset);
01388   ID.AddInteger(TargetFlags);
01389   void *IP = nullptr;
01390   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01391     return SDValue(E, 0);
01392 
01393   SDNode *N = new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset,
01394                                                     TargetFlags);
01395   CSEMap.InsertNode(N, IP);
01396   InsertNode(N);
01397   return SDValue(N, 0);
01398 }
01399 
01400 SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
01401   FoldingSetNodeID ID;
01402   AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
01403   ID.AddPointer(MBB);
01404   void *IP = nullptr;
01405   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01406     return SDValue(E, 0);
01407 
01408   SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB);
01409   CSEMap.InsertNode(N, IP);
01410   InsertNode(N);
01411   return SDValue(N, 0);
01412 }
01413 
01414 SDValue SelectionDAG::getValueType(EVT VT) {
01415   if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
01416       ValueTypeNodes.size())
01417     ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
01418 
01419   SDNode *&N = VT.isExtended() ?
01420     ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
01421 
01422   if (N) return SDValue(N, 0);
01423   N = new (NodeAllocator) VTSDNode(VT);
01424   InsertNode(N);
01425   return SDValue(N, 0);
01426 }
01427 
01428 SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
01429   SDNode *&N = ExternalSymbols[Sym];
01430   if (N) return SDValue(N, 0);
01431   N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT);
01432   InsertNode(N);
01433   return SDValue(N, 0);
01434 }
01435 
01436 SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
01437                                               unsigned char TargetFlags) {
01438   SDNode *&N =
01439     TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym,
01440                                                                TargetFlags)];
01441   if (N) return SDValue(N, 0);
01442   N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT);
01443   InsertNode(N);
01444   return SDValue(N, 0);
01445 }
01446 
01447 SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
01448   if ((unsigned)Cond >= CondCodeNodes.size())
01449     CondCodeNodes.resize(Cond+1);
01450 
01451   if (!CondCodeNodes[Cond]) {
01452     CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond);
01453     CondCodeNodes[Cond] = N;
01454     InsertNode(N);
01455   }
01456 
01457   return SDValue(CondCodeNodes[Cond], 0);
01458 }
01459 
01460 // commuteShuffle - swaps the values of N1 and N2, and swaps all indices in
01461 // the shuffle mask M that point at N1 to point at N2, and indices that point
01462 // N2 to point at N1.
01463 static void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) {
01464   std::swap(N1, N2);
01465   ShuffleVectorSDNode::commuteMask(M);
01466 }
01467 
01468 SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1,
01469                                        SDValue N2, const int *Mask) {
01470   assert(VT == N1.getValueType() && VT == N2.getValueType() &&
01471          "Invalid VECTOR_SHUFFLE");
01472 
01473   // Canonicalize shuffle undef, undef -> undef
01474   if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF)
01475     return getUNDEF(VT);
01476 
01477   // Validate that all indices in Mask are within the range of the elements
01478   // input to the shuffle.
01479   unsigned NElts = VT.getVectorNumElements();
01480   SmallVector<int, 8> MaskVec;
01481   for (unsigned i = 0; i != NElts; ++i) {
01482     assert(Mask[i] < (int)(NElts * 2) && "Index out of range");
01483     MaskVec.push_back(Mask[i]);
01484   }
01485 
01486   // Canonicalize shuffle v, v -> v, undef
01487   if (N1 == N2) {
01488     N2 = getUNDEF(VT);
01489     for (unsigned i = 0; i != NElts; ++i)
01490       if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts;
01491   }
01492 
01493   // Canonicalize shuffle undef, v -> v, undef.  Commute the shuffle mask.
01494   if (N1.getOpcode() == ISD::UNDEF)
01495     commuteShuffle(N1, N2, MaskVec);
01496 
01497   // If shuffling a splat, try to blend the splat instead. We do this here so
01498   // that even when this arises during lowering we don't have to re-handle it.
01499   auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
01500     BitVector UndefElements;
01501     SDValue Splat = BV->getSplatValue(&UndefElements);
01502     if (!Splat)
01503       return;
01504 
01505     for (int i = 0; i < (int)NElts; ++i) {
01506       if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + (int)NElts))
01507         continue;
01508 
01509       // If this input comes from undef, mark it as such.
01510       if (UndefElements[MaskVec[i] - Offset]) {
01511         MaskVec[i] = -1;
01512         continue;
01513       }
01514 
01515       // If we can blend a non-undef lane, use that instead.
01516       if (!UndefElements[i])
01517         MaskVec[i] = i + Offset;
01518     }
01519   };
01520   if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
01521     BlendSplat(N1BV, 0);
01522   if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
01523     BlendSplat(N2BV, NElts);
01524 
01525   // Canonicalize all index into lhs, -> shuffle lhs, undef
01526   // Canonicalize all index into rhs, -> shuffle rhs, undef
01527   bool AllLHS = true, AllRHS = true;
01528   bool N2Undef = N2.getOpcode() == ISD::UNDEF;
01529   for (unsigned i = 0; i != NElts; ++i) {
01530     if (MaskVec[i] >= (int)NElts) {
01531       if (N2Undef)
01532         MaskVec[i] = -1;
01533       else
01534         AllLHS = false;
01535     } else if (MaskVec[i] >= 0) {
01536       AllRHS = false;
01537     }
01538   }
01539   if (AllLHS && AllRHS)
01540     return getUNDEF(VT);
01541   if (AllLHS && !N2Undef)
01542     N2 = getUNDEF(VT);
01543   if (AllRHS) {
01544     N1 = getUNDEF(VT);
01545     commuteShuffle(N1, N2, MaskVec);
01546   }
01547   // Reset our undef status after accounting for the mask.
01548   N2Undef = N2.getOpcode() == ISD::UNDEF;
01549   // Re-check whether both sides ended up undef.
01550   if (N1.getOpcode() == ISD::UNDEF && N2Undef)
01551     return getUNDEF(VT);
01552 
01553   // If Identity shuffle return that node.
01554   bool Identity = true, AllSame = true;
01555   for (unsigned i = 0; i != NElts; ++i) {
01556     if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false;
01557     if (MaskVec[i] != MaskVec[0]) AllSame = false;
01558   }
01559   if (Identity && NElts)
01560     return N1;
01561 
01562   // Shuffling a constant splat doesn't change the result.
01563   if (N2Undef) {
01564     SDValue V = N1;
01565 
01566     // Look through any bitcasts. We check that these don't change the number
01567     // (and size) of elements and just changes their types.
01568     while (V.getOpcode() == ISD::BITCAST)
01569       V = V->getOperand(0);
01570 
01571     // A splat should always show up as a build vector node.
01572     if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
01573       BitVector UndefElements;
01574       SDValue Splat = BV->getSplatValue(&UndefElements);
01575       // If this is a splat of an undef, shuffling it is also undef.
01576       if (Splat && Splat.getOpcode() == ISD::UNDEF)
01577         return getUNDEF(VT);
01578 
01579       bool SameNumElts =
01580           V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
01581 
01582       // We only have a splat which can skip shuffles if there is a splatted
01583       // value and no undef lanes rearranged by the shuffle.
01584       if (Splat && UndefElements.none()) {
01585         // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
01586         // number of elements match or the value splatted is a zero constant.
01587         if (SameNumElts)
01588           return N1;
01589         if (auto *C = dyn_cast<ConstantSDNode>(Splat))
01590           if (C->isNullValue())
01591             return N1;
01592       }
01593 
01594       // If the shuffle itself creates a splat, build the vector directly.
01595       if (AllSame && SameNumElts) {
01596         const SDValue &Splatted = BV->getOperand(MaskVec[0]);
01597         SmallVector<SDValue, 8> Ops(NElts, Splatted);
01598 
01599         EVT BuildVT = BV->getValueType(0);
01600         SDValue NewBV = getNode(ISD::BUILD_VECTOR, dl, BuildVT, Ops);
01601 
01602         // We may have jumped through bitcasts, so the type of the
01603         // BUILD_VECTOR may not match the type of the shuffle.
01604         if (BuildVT != VT)
01605           NewBV = getNode(ISD::BITCAST, dl, VT, NewBV);
01606         return NewBV;
01607       }
01608     }
01609   }
01610 
01611   FoldingSetNodeID ID;
01612   SDValue Ops[2] = { N1, N2 };
01613   AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
01614   for (unsigned i = 0; i != NElts; ++i)
01615     ID.AddInteger(MaskVec[i]);
01616 
01617   void* IP = nullptr;
01618   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01619     return SDValue(E, 0);
01620 
01621   // Allocate the mask array for the node out of the BumpPtrAllocator, since
01622   // SDNode doesn't have access to it.  This memory will be "leaked" when
01623   // the node is deallocated, but recovered when the NodeAllocator is released.
01624   int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
01625   memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int));
01626 
01627   ShuffleVectorSDNode *N =
01628     new (NodeAllocator) ShuffleVectorSDNode(VT, dl.getIROrder(),
01629                                             dl.getDebugLoc(), N1, N2,
01630                                             MaskAlloc);
01631   CSEMap.InsertNode(N, IP);
01632   InsertNode(N);
01633   return SDValue(N, 0);
01634 }
01635 
01636 SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
01637   MVT VT = SV.getSimpleValueType(0);
01638   SmallVector<int, 8> MaskVec(SV.getMask().begin(), SV.getMask().end());
01639   ShuffleVectorSDNode::commuteMask(MaskVec);
01640 
01641   SDValue Op0 = SV.getOperand(0);
01642   SDValue Op1 = SV.getOperand(1);
01643   return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, &MaskVec[0]);
01644 }
01645 
01646 SDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl,
01647                                        SDValue Val, SDValue DTy,
01648                                        SDValue STy, SDValue Rnd, SDValue Sat,
01649                                        ISD::CvtCode Code) {
01650   // If the src and dest types are the same and the conversion is between
01651   // integer types of the same sign or two floats, no conversion is necessary.
01652   if (DTy == STy &&
01653       (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF))
01654     return Val;
01655 
01656   FoldingSetNodeID ID;
01657   SDValue Ops[] = { Val, DTy, STy, Rnd, Sat };
01658   AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), Ops);
01659   void* IP = nullptr;
01660   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01661     return SDValue(E, 0);
01662 
01663   CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl.getIROrder(),
01664                                                            dl.getDebugLoc(),
01665                                                            Ops, Code);
01666   CSEMap.InsertNode(N, IP);
01667   InsertNode(N);
01668   return SDValue(N, 0);
01669 }
01670 
01671 SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
01672   FoldingSetNodeID ID;
01673   AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
01674   ID.AddInteger(RegNo);
01675   void *IP = nullptr;
01676   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01677     return SDValue(E, 0);
01678 
01679   SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT);
01680   CSEMap.InsertNode(N, IP);
01681   InsertNode(N);
01682   return SDValue(N, 0);
01683 }
01684 
01685 SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
01686   FoldingSetNodeID ID;
01687   AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
01688   ID.AddPointer(RegMask);
01689   void *IP = nullptr;
01690   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01691     return SDValue(E, 0);
01692 
01693   SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask);
01694   CSEMap.InsertNode(N, IP);
01695   InsertNode(N);
01696   return SDValue(N, 0);
01697 }
01698 
01699 SDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) {
01700   FoldingSetNodeID ID;
01701   SDValue Ops[] = { Root };
01702   AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), Ops);
01703   ID.AddPointer(Label);
01704   void *IP = nullptr;
01705   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01706     return SDValue(E, 0);
01707 
01708   SDNode *N = new (NodeAllocator) EHLabelSDNode(dl.getIROrder(),
01709                                                 dl.getDebugLoc(), Root, Label);
01710   CSEMap.InsertNode(N, IP);
01711   InsertNode(N);
01712   return SDValue(N, 0);
01713 }
01714 
01715 
01716 SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
01717                                       int64_t Offset,
01718                                       bool isTarget,
01719                                       unsigned char TargetFlags) {
01720   unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
01721 
01722   FoldingSetNodeID ID;
01723   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01724   ID.AddPointer(BA);
01725   ID.AddInteger(Offset);
01726   ID.AddInteger(TargetFlags);
01727   void *IP = nullptr;
01728   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01729     return SDValue(E, 0);
01730 
01731   SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset,
01732                                                      TargetFlags);
01733   CSEMap.InsertNode(N, IP);
01734   InsertNode(N);
01735   return SDValue(N, 0);
01736 }
01737 
01738 SDValue SelectionDAG::getSrcValue(const Value *V) {
01739   assert((!V || V->getType()->isPointerTy()) &&
01740          "SrcValue is not a pointer?");
01741 
01742   FoldingSetNodeID ID;
01743   AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None);
01744   ID.AddPointer(V);
01745 
01746   void *IP = nullptr;
01747   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01748     return SDValue(E, 0);
01749 
01750   SDNode *N = new (NodeAllocator) SrcValueSDNode(V);
01751   CSEMap.InsertNode(N, IP);
01752   InsertNode(N);
01753   return SDValue(N, 0);
01754 }
01755 
01756 /// getMDNode - Return an MDNodeSDNode which holds an MDNode.
01757 SDValue SelectionDAG::getMDNode(const MDNode *MD) {
01758   FoldingSetNodeID ID;
01759   AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None);
01760   ID.AddPointer(MD);
01761 
01762   void *IP = nullptr;
01763   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01764     return SDValue(E, 0);
01765 
01766   SDNode *N = new (NodeAllocator) MDNodeSDNode(MD);
01767   CSEMap.InsertNode(N, IP);
01768   InsertNode(N);
01769   return SDValue(N, 0);
01770 }
01771 
01772 /// getAddrSpaceCast - Return an AddrSpaceCastSDNode.
01773 SDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr,
01774                                        unsigned SrcAS, unsigned DestAS) {
01775   SDValue Ops[] = {Ptr};
01776   FoldingSetNodeID ID;
01777   AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
01778   ID.AddInteger(SrcAS);
01779   ID.AddInteger(DestAS);
01780 
01781   void *IP = nullptr;
01782   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01783     return SDValue(E, 0);
01784 
01785   SDNode *N = new (NodeAllocator) AddrSpaceCastSDNode(dl.getIROrder(),
01786                                                       dl.getDebugLoc(),
01787                                                       VT, Ptr, SrcAS, DestAS);
01788   CSEMap.InsertNode(N, IP);
01789   InsertNode(N);
01790   return SDValue(N, 0);
01791 }
01792 
01793 /// getShiftAmountOperand - Return the specified value casted to
01794 /// the target's desired shift amount type.
01795 SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
01796   EVT OpTy = Op.getValueType();
01797   EVT ShTy = TLI->getShiftAmountTy(LHSTy);
01798   if (OpTy == ShTy || OpTy.isVector()) return Op;
01799 
01800   ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ?  ISD::TRUNCATE : ISD::ZERO_EXTEND;
01801   return getNode(Opcode, SDLoc(Op), ShTy, Op);
01802 }
01803 
01804 /// CreateStackTemporary - Create a stack temporary, suitable for holding the
01805 /// specified value type.
01806 SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
01807   MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
01808   unsigned ByteSize = VT.getStoreSize();
01809   Type *Ty = VT.getTypeForEVT(*getContext());
01810   unsigned StackAlign =
01811   std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty), minAlign);
01812 
01813   int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false);
01814   return getFrameIndex(FrameIdx, TLI->getPointerTy());
01815 }
01816 
01817 /// CreateStackTemporary - Create a stack temporary suitable for holding
01818 /// either of the specified value types.
01819 SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
01820   unsigned Bytes = std::max(VT1.getStoreSizeInBits(),
01821                             VT2.getStoreSizeInBits())/8;
01822   Type *Ty1 = VT1.getTypeForEVT(*getContext());
01823   Type *Ty2 = VT2.getTypeForEVT(*getContext());
01824   const DataLayout *TD = TLI->getDataLayout();
01825   unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1),
01826                             TD->getPrefTypeAlignment(Ty2));
01827 
01828   MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
01829   int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false);
01830   return getFrameIndex(FrameIdx, TLI->getPointerTy());
01831 }
01832 
01833 SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1,
01834                                 SDValue N2, ISD::CondCode Cond, SDLoc dl) {
01835   // These setcc operations always fold.
01836   switch (Cond) {
01837   default: break;
01838   case ISD::SETFALSE:
01839   case ISD::SETFALSE2: return getConstant(0, VT);
01840   case ISD::SETTRUE:
01841   case ISD::SETTRUE2: {
01842     TargetLowering::BooleanContent Cnt =
01843         TLI->getBooleanContents(N1->getValueType(0));
01844     return getConstant(
01845         Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, VT);
01846   }
01847 
01848   case ISD::SETOEQ:
01849   case ISD::SETOGT:
01850   case ISD::SETOGE:
01851   case ISD::SETOLT:
01852   case ISD::SETOLE:
01853   case ISD::SETONE:
01854   case ISD::SETO:
01855   case ISD::SETUO:
01856   case ISD::SETUEQ:
01857   case ISD::SETUNE:
01858     assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!");
01859     break;
01860   }
01861 
01862   if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) {
01863     const APInt &C2 = N2C->getAPIntValue();
01864     if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
01865       const APInt &C1 = N1C->getAPIntValue();
01866 
01867       switch (Cond) {
01868       default: llvm_unreachable("Unknown integer setcc!");
01869       case ISD::SETEQ:  return getConstant(C1 == C2, VT);
01870       case ISD::SETNE:  return getConstant(C1 != C2, VT);
01871       case ISD::SETULT: return getConstant(C1.ult(C2), VT);
01872       case ISD::SETUGT: return getConstant(C1.ugt(C2), VT);
01873       case ISD::SETULE: return getConstant(C1.ule(C2), VT);
01874       case ISD::SETUGE: return getConstant(C1.uge(C2), VT);
01875       case ISD::SETLT:  return getConstant(C1.slt(C2), VT);
01876       case ISD::SETGT:  return getConstant(C1.sgt(C2), VT);
01877       case ISD::SETLE:  return getConstant(C1.sle(C2), VT);
01878       case ISD::SETGE:  return getConstant(C1.sge(C2), VT);
01879       }
01880     }
01881   }
01882   if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) {
01883     if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) {
01884       APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF());
01885       switch (Cond) {
01886       default: break;
01887       case ISD::SETEQ:  if (R==APFloat::cmpUnordered)
01888                           return getUNDEF(VT);
01889                         // fall through
01890       case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT);
01891       case ISD::SETNE:  if (R==APFloat::cmpUnordered)
01892                           return getUNDEF(VT);
01893                         // fall through
01894       case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan ||
01895                                            R==APFloat::cmpLessThan, VT);
01896       case ISD::SETLT:  if (R==APFloat::cmpUnordered)
01897                           return getUNDEF(VT);
01898                         // fall through
01899       case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT);
01900       case ISD::SETGT:  if (R==APFloat::cmpUnordered)
01901                           return getUNDEF(VT);
01902                         // fall through
01903       case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT);
01904       case ISD::SETLE:  if (R==APFloat::cmpUnordered)
01905                           return getUNDEF(VT);
01906                         // fall through
01907       case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan ||
01908                                            R==APFloat::cmpEqual, VT);
01909       case ISD::SETGE:  if (R==APFloat::cmpUnordered)
01910                           return getUNDEF(VT);
01911                         // fall through
01912       case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan ||
01913                                            R==APFloat::cmpEqual, VT);
01914       case ISD::SETO:   return getConstant(R!=APFloat::cmpUnordered, VT);
01915       case ISD::SETUO:  return getConstant(R==APFloat::cmpUnordered, VT);
01916       case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered ||
01917                                            R==APFloat::cmpEqual, VT);
01918       case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT);
01919       case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered ||
01920                                            R==APFloat::cmpLessThan, VT);
01921       case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan ||
01922                                            R==APFloat::cmpUnordered, VT);
01923       case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT);
01924       case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT);
01925       }
01926     } else {
01927       // Ensure that the constant occurs on the RHS.
01928       ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
01929       MVT CompVT = N1.getValueType().getSimpleVT();
01930       if (!TLI->isCondCodeLegal(SwappedCond, CompVT))
01931         return SDValue();
01932 
01933       return getSetCC(dl, VT, N2, N1, SwappedCond);
01934     }
01935   }
01936 
01937   // Could not fold it.
01938   return SDValue();
01939 }
01940 
01941 /// SignBitIsZero - Return true if the sign bit of Op is known to be zero.  We
01942 /// use this predicate to simplify operations downstream.
01943 bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
01944   // This predicate is not safe for vector operations.
01945   if (Op.getValueType().isVector())
01946     return false;
01947 
01948   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01949   return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth);
01950 }
01951 
01952 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero.  We use
01953 /// this predicate to simplify operations downstream.  Mask is known to be zero
01954 /// for bits that V cannot have.
01955 bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask,
01956                                      unsigned Depth) const {
01957   APInt KnownZero, KnownOne;
01958   computeKnownBits(Op, KnownZero, KnownOne, Depth);
01959   return (KnownZero & Mask) == Mask;
01960 }
01961 
01962 /// Determine which bits of Op are known to be either zero or one and return
01963 /// them in the KnownZero/KnownOne bitsets.
01964 void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
01965                                     APInt &KnownOne, unsigned Depth) const {
01966   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01967 
01968   KnownZero = KnownOne = APInt(BitWidth, 0);   // Don't know anything.
01969   if (Depth == 6)
01970     return;  // Limit search depth.
01971 
01972   APInt KnownZero2, KnownOne2;
01973 
01974   switch (Op.getOpcode()) {
01975   case ISD::Constant:
01976     // We know all of the bits for a constant!
01977     KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue();
01978     KnownZero = ~KnownOne;
01979     break;
01980   case ISD::AND:
01981     // If either the LHS or the RHS are Zero, the result is zero.
01982     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01983     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01984 
01985     // Output known-1 bits are only known if set in both the LHS & RHS.
01986     KnownOne &= KnownOne2;
01987     // Output known-0 are known to be clear if zero in either the LHS | RHS.
01988     KnownZero |= KnownZero2;
01989     break;
01990   case ISD::OR:
01991     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01992     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01993 
01994     // Output known-0 bits are only known if clear in both the LHS & RHS.
01995     KnownZero &= KnownZero2;
01996     // Output known-1 are known to be set if set in either the LHS | RHS.
01997     KnownOne |= KnownOne2;
01998     break;
01999   case ISD::XOR: {
02000     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
02001     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
02002 
02003     // Output known-0 bits are known if clear or set in both the LHS & RHS.
02004     APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2);
02005     // Output known-1 are known to be set if set in only one of the LHS, RHS.
02006     KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2);
02007     KnownZero = KnownZeroOut;
02008     break;
02009   }
02010   case ISD::MUL: {
02011     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
02012     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
02013 
02014     // If low bits are zero in either operand, output low known-0 bits.
02015     // Also compute a conserative estimate for high known-0 bits.
02016     // More trickiness is possible, but this is sufficient for the
02017     // interesting case of alignment computation.
02018     KnownOne.clearAllBits();
02019     unsigned TrailZ = KnownZero.countTrailingOnes() +
02020                       KnownZero2.countTrailingOnes();
02021     unsigned LeadZ =  std::max(KnownZero.countLeadingOnes() +
02022                                KnownZero2.countLeadingOnes(),
02023                                BitWidth) - BitWidth;
02024 
02025     TrailZ = std::min(TrailZ, BitWidth);
02026     LeadZ = std::min(LeadZ, BitWidth);
02027     KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) |
02028                 APInt::getHighBitsSet(BitWidth, LeadZ);
02029     break;
02030   }
02031   case ISD::UDIV: {
02032     // For the purposes of computing leading zeros we can conservatively
02033     // treat a udiv as a logical right shift by the power of 2 known to
02034     // be less than the denominator.
02035     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
02036     unsigned LeadZ = KnownZero2.countLeadingOnes();
02037 
02038     KnownOne2.clearAllBits();
02039     KnownZero2.clearAllBits();
02040     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02041     unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros();
02042     if (RHSUnknownLeadingOnes != BitWidth)
02043       LeadZ = std::min(BitWidth,
02044                        LeadZ + BitWidth - RHSUnknownLeadingOnes - 1);
02045 
02046     KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ);
02047     break;
02048   }
02049   case ISD::SELECT:
02050     computeKnownBits(Op.getOperand(2), KnownZero, KnownOne, Depth+1);
02051     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02052 
02053     // Only known if known in both the LHS and RHS.
02054     KnownOne &= KnownOne2;
02055     KnownZero &= KnownZero2;
02056     break;
02057   case ISD::SELECT_CC:
02058     computeKnownBits(Op.getOperand(3), KnownZero, KnownOne, Depth+1);
02059     computeKnownBits(Op.getOperand(2), KnownZero2, KnownOne2, Depth+1);
02060 
02061     // Only known if known in both the LHS and RHS.
02062     KnownOne &= KnownOne2;
02063     KnownZero &= KnownZero2;
02064     break;
02065   case ISD::SADDO:
02066   case ISD::UADDO:
02067   case ISD::SSUBO:
02068   case ISD::USUBO:
02069   case ISD::SMULO:
02070   case ISD::UMULO:
02071     if (Op.getResNo() != 1)
02072       break;
02073     // The boolean result conforms to getBooleanContents.
02074     // If we know the result of a setcc has the top bits zero, use this info.
02075     // We know that we have an integer-based boolean since these operations
02076     // are only available for integer.
02077     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
02078             TargetLowering::ZeroOrOneBooleanContent &&
02079         BitWidth > 1)
02080       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02081     break;
02082   case ISD::SETCC:
02083     // If we know the result of a setcc has the top bits zero, use this info.
02084     if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
02085             TargetLowering::ZeroOrOneBooleanContent &&
02086         BitWidth > 1)
02087       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02088     break;
02089   case ISD::SHL:
02090     // (shl X, C1) & C2 == 0   iff   (X & C2 >>u C1) == 0
02091     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02092       unsigned ShAmt = SA->getZExtValue();
02093 
02094       // If the shift count is an invalid immediate, don't do anything.
02095       if (ShAmt >= BitWidth)
02096         break;
02097 
02098       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02099       KnownZero <<= ShAmt;
02100       KnownOne  <<= ShAmt;
02101       // low bits known zero.
02102       KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt);
02103     }
02104     break;
02105   case ISD::SRL:
02106     // (ushr X, C1) & C2 == 0   iff  (-1 >> C1) & C2 == 0
02107     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02108       unsigned ShAmt = SA->getZExtValue();
02109 
02110       // If the shift count is an invalid immediate, don't do anything.
02111       if (ShAmt >= BitWidth)
02112         break;
02113 
02114       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02115       KnownZero = KnownZero.lshr(ShAmt);
02116       KnownOne  = KnownOne.lshr(ShAmt);
02117 
02118       APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
02119       KnownZero |= HighBits;  // High bits known zero.
02120     }
02121     break;
02122   case ISD::SRA:
02123     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02124       unsigned ShAmt = SA->getZExtValue();
02125 
02126       // If the shift count is an invalid immediate, don't do anything.
02127       if (ShAmt >= BitWidth)
02128         break;
02129 
02130       // If any of the demanded bits are produced by the sign extension, we also
02131       // demand the input sign bit.
02132       APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
02133 
02134       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02135       KnownZero = KnownZero.lshr(ShAmt);
02136       KnownOne  = KnownOne.lshr(ShAmt);
02137 
02138       // Handle the sign bits.
02139       APInt SignBit = APInt::getSignBit(BitWidth);
02140       SignBit = SignBit.lshr(ShAmt);  // Adjust to where it is now in the mask.
02141 
02142       if (KnownZero.intersects(SignBit)) {
02143         KnownZero |= HighBits;  // New bits are known zero.
02144       } else if (KnownOne.intersects(SignBit)) {
02145         KnownOne  |= HighBits;  // New bits are known one.
02146       }
02147     }
02148     break;
02149   case ISD::SIGN_EXTEND_INREG: {
02150     EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
02151     unsigned EBits = EVT.getScalarType().getSizeInBits();
02152 
02153     // Sign extension.  Compute the demanded bits in the result that are not
02154     // present in the input.
02155     APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits);
02156 
02157     APInt InSignBit = APInt::getSignBit(EBits);
02158     APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits);
02159 
02160     // If the sign extended bits are demanded, we know that the sign
02161     // bit is demanded.
02162     InSignBit = InSignBit.zext(BitWidth);
02163     if (NewBits.getBoolValue())
02164       InputDemandedBits |= InSignBit;
02165 
02166     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02167     KnownOne &= InputDemandedBits;
02168     KnownZero &= InputDemandedBits;
02169 
02170     // If the sign bit of the input is known set or clear, then we know the
02171     // top bits of the result.
02172     if (KnownZero.intersects(InSignBit)) {         // Input sign bit known clear
02173       KnownZero |= NewBits;
02174       KnownOne  &= ~NewBits;
02175     } else if (KnownOne.intersects(InSignBit)) {   // Input sign bit known set
02176       KnownOne  |= NewBits;
02177       KnownZero &= ~NewBits;
02178     } else {                              // Input sign bit unknown
02179       KnownZero &= ~NewBits;
02180       KnownOne  &= ~NewBits;
02181     }
02182     break;
02183   }
02184   case ISD::CTTZ:
02185   case ISD::CTTZ_ZERO_UNDEF:
02186   case ISD::CTLZ:
02187   case ISD::CTLZ_ZERO_UNDEF:
02188   case ISD::CTPOP: {
02189     unsigned LowBits = Log2_32(BitWidth)+1;
02190     KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits);
02191     KnownOne.clearAllBits();
02192     break;
02193   }
02194   case ISD::LOAD: {
02195     LoadSDNode *LD = cast<LoadSDNode>(Op);
02196     // If this is a ZEXTLoad and we are looking at the loaded value.
02197     if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
02198       EVT VT = LD->getMemoryVT();
02199       unsigned MemBits = VT.getScalarType().getSizeInBits();
02200       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
02201     } else if (const MDNode *Ranges = LD->getRanges()) {
02202       computeKnownBitsFromRangeMetadata(*Ranges, KnownZero);
02203     }
02204     break;
02205   }
02206   case ISD::ZERO_EXTEND: {
02207     EVT InVT = Op.getOperand(0).getValueType();
02208     unsigned InBits = InVT.getScalarType().getSizeInBits();
02209     APInt NewBits   = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
02210     KnownZero = KnownZero.trunc(InBits);
02211     KnownOne = KnownOne.trunc(InBits);
02212     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02213     KnownZero = KnownZero.zext(BitWidth);
02214     KnownOne = KnownOne.zext(BitWidth);
02215     KnownZero |= NewBits;
02216     break;
02217   }
02218   case ISD::SIGN_EXTEND: {
02219     EVT InVT = Op.getOperand(0).getValueType();
02220     unsigned InBits = InVT.getScalarType().getSizeInBits();
02221     APInt NewBits   = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
02222 
02223     KnownZero = KnownZero.trunc(InBits);
02224     KnownOne = KnownOne.trunc(InBits);
02225     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02226 
02227     // Note if the sign bit is known to be zero or one.
02228     bool SignBitKnownZero = KnownZero.isNegative();
02229     bool SignBitKnownOne  = KnownOne.isNegative();
02230 
02231     KnownZero = KnownZero.zext(BitWidth);
02232     KnownOne = KnownOne.zext(BitWidth);
02233 
02234     // If the sign bit is known zero or one, the top bits match.
02235     if (SignBitKnownZero)
02236       KnownZero |= NewBits;
02237     else if (SignBitKnownOne)
02238       KnownOne  |= NewBits;
02239     break;
02240   }
02241   case ISD::ANY_EXTEND: {
02242     EVT InVT = Op.getOperand(0).getValueType();
02243     unsigned InBits = InVT.getScalarType().getSizeInBits();
02244     KnownZero = KnownZero.trunc(InBits);
02245     KnownOne = KnownOne.trunc(InBits);
02246     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02247     KnownZero = KnownZero.zext(BitWidth);
02248     KnownOne = KnownOne.zext(BitWidth);
02249     break;
02250   }
02251   case ISD::TRUNCATE: {
02252     EVT InVT = Op.getOperand(0).getValueType();
02253     unsigned InBits = InVT.getScalarType().getSizeInBits();
02254     KnownZero = KnownZero.zext(InBits);
02255     KnownOne = KnownOne.zext(InBits);
02256     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02257     KnownZero = KnownZero.trunc(BitWidth);
02258     KnownOne = KnownOne.trunc(BitWidth);
02259     break;
02260   }
02261   case ISD::AssertZext: {
02262     EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
02263     APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
02264     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02265     KnownZero |= (~InMask);
02266     KnownOne  &= (~KnownZero);
02267     break;
02268   }
02269   case ISD::FGETSIGN:
02270     // All bits are zero except the low bit.
02271     KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02272     break;
02273 
02274   case ISD::SUB: {
02275     if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) {
02276       // We know that the top bits of C-X are clear if X contains less bits
02277       // than C (i.e. no wrap-around can happen).  For example, 20-X is
02278       // positive if we can prove that X is >= 0 and < 16.
02279       if (CLHS->getAPIntValue().isNonNegative()) {
02280         unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros();
02281         // NLZ can't be BitWidth with no sign bit
02282         APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1);
02283         computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02284 
02285         // If all of the MaskV bits are known to be zero, then we know the
02286         // output top bits are zero, because we now know that the output is
02287         // from [0-C].
02288         if ((KnownZero2 & MaskV) == MaskV) {
02289           unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros();
02290           // Top bits known zero.
02291           KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2);
02292         }
02293       }
02294     }
02295   }
02296   // fall through
02297   case ISD::ADD:
02298   case ISD::ADDE: {
02299     // Output known-0 bits are known if clear or set in both the low clear bits
02300     // common to both LHS & RHS.  For example, 8+(X<<3) is known to have the
02301     // low 3 bits clear.
02302     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
02303     unsigned KnownZeroOut = KnownZero2.countTrailingOnes();
02304 
02305     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02306     KnownZeroOut = std::min(KnownZeroOut,
02307                             KnownZero2.countTrailingOnes());
02308 
02309     if (Op.getOpcode() == ISD::ADD) {
02310       KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut);
02311       break;
02312     }
02313 
02314     // With ADDE, a carry bit may be added in, so we can only use this
02315     // information if we know (at least) that the low two bits are clear.  We
02316     // then return to the caller that the low bit is unknown but that other bits
02317     // are known zero.
02318     if (KnownZeroOut >= 2) // ADDE
02319       KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut);
02320     break;
02321   }
02322   case ISD::SREM:
02323     if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02324       const APInt &RA = Rem->getAPIntValue().abs();
02325       if (RA.isPowerOf2()) {
02326         APInt LowBits = RA - 1;
02327         computeKnownBits(Op.getOperand(0), KnownZero2,KnownOne2,Depth+1);
02328 
02329         // The low bits of the first operand are unchanged by the srem.
02330         KnownZero = KnownZero2 & LowBits;
02331         KnownOne = KnownOne2 & LowBits;
02332 
02333         // If the first operand is non-negative or has all low bits zero, then
02334         // the upper bits are all zero.
02335         if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits))
02336           KnownZero |= ~LowBits;
02337 
02338         // If the first operand is negative and not all low bits are zero, then
02339         // the upper bits are all one.
02340         if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0))
02341           KnownOne |= ~LowBits;
02342         assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?");
02343       }
02344     }
02345     break;
02346   case ISD::UREM: {
02347     if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02348       const APInt &RA = Rem->getAPIntValue();
02349       if (RA.isPowerOf2()) {
02350         APInt LowBits = (RA - 1);
02351         computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth + 1);
02352 
02353         // The upper bits are all zero, the lower ones are unchanged.
02354         KnownZero = KnownZero2 | ~LowBits;
02355         KnownOne = KnownOne2 & LowBits;
02356         break;
02357       }
02358     }
02359 
02360     // Since the result is less than or equal to either operand, any leading
02361     // zero bits in either operand must also exist in the result.
02362     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02363     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02364 
02365     uint32_t Leaders = std::max(KnownZero.countLeadingOnes(),
02366                                 KnownZero2.countLeadingOnes());
02367     KnownOne.clearAllBits();
02368     KnownZero = APInt::getHighBitsSet(BitWidth, Leaders);
02369     break;
02370   }
02371   case ISD::EXTRACT_ELEMENT: {
02372     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02373     const unsigned Index =
02374       cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
02375     const unsigned BitWidth = Op.getValueType().getSizeInBits();
02376 
02377     // Remove low part of known bits mask
02378     KnownZero = KnownZero.getHiBits(KnownZero.getBitWidth() - Index * BitWidth);
02379     KnownOne = KnownOne.getHiBits(KnownOne.getBitWidth() - Index * BitWidth);
02380 
02381     // Remove high part of known bit mask
02382     KnownZero = KnownZero.trunc(BitWidth);
02383     KnownOne = KnownOne.trunc(BitWidth);
02384     break;
02385   }
02386   case ISD::FrameIndex:
02387   case ISD::TargetFrameIndex:
02388     if (unsigned Align = InferPtrAlignment(Op)) {
02389       // The low bits are known zero if the pointer is aligned.
02390       KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align));
02391       break;
02392     }
02393     break;
02394 
02395   default:
02396     if (Op.getOpcode() < ISD::BUILTIN_OP_END)
02397       break;
02398     // Fallthrough
02399   case ISD::INTRINSIC_WO_CHAIN:
02400   case ISD::INTRINSIC_W_CHAIN:
02401   case ISD::INTRINSIC_VOID:
02402     // Allow the target to implement this method for its nodes.
02403     TLI->computeKnownBitsForTargetNode(Op, KnownZero, KnownOne, *this, Depth);
02404     break;
02405   }
02406 
02407   assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
02408 }
02409 
02410 /// ComputeNumSignBits - Return the number of times the sign bit of the
02411 /// register is replicated into the other bits.  We know that at least 1 bit
02412 /// is always equal to the sign bit (itself), but other cases can give us
02413 /// information.  For example, immediately after an "SRA X, 2", we know that
02414 /// the top 3 bits are all equal to each other, so we return 3.
02415 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{
02416   EVT VT = Op.getValueType();
02417   assert(VT.isInteger() && "Invalid VT!");
02418   unsigned VTBits = VT.getScalarType().getSizeInBits();
02419   unsigned Tmp, Tmp2;
02420   unsigned FirstAnswer = 1;
02421 
02422   if (Depth == 6)
02423     return 1;  // Limit search depth.
02424 
02425   switch (Op.getOpcode()) {
02426   default: break;
02427   case ISD::AssertSext:
02428     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
02429     return VTBits-Tmp+1;
02430   case ISD::AssertZext:
02431     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
02432     return VTBits-Tmp;
02433 
02434   case ISD::Constant: {
02435     const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue();
02436     return Val.getNumSignBits();
02437   }
02438 
02439   case ISD::SIGN_EXTEND:
02440     Tmp =
02441         VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits();
02442     return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp;
02443 
02444   case ISD::SIGN_EXTEND_INREG:
02445     // Max of the input and what this extends.
02446     Tmp =
02447       cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits();
02448     Tmp = VTBits-Tmp+1;
02449 
02450     Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02451     return std::max(Tmp, Tmp2);
02452 
02453   case ISD::SRA:
02454     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02455     // SRA X, C   -> adds C sign bits.
02456     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02457       Tmp += C->getZExtValue();
02458       if (Tmp > VTBits) Tmp = VTBits;
02459     }
02460     return Tmp;
02461   case ISD::SHL:
02462     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02463       // shl destroys sign bits.
02464       Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02465       if (C->getZExtValue() >= VTBits ||      // Bad shift.
02466           C->getZExtValue() >= Tmp) break;    // Shifted all sign bits out.
02467       return Tmp - C->getZExtValue();
02468     }
02469     break;
02470   case ISD::AND:
02471   case ISD::OR:
02472   case ISD::XOR:    // NOT is handled here.
02473     // Logical binary ops preserve the number of sign bits at the worst.
02474     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02475     if (Tmp != 1) {
02476       Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02477       FirstAnswer = std::min(Tmp, Tmp2);
02478       // We computed what we know about the sign bits as our first
02479       // answer. Now proceed to the generic code that uses
02480       // computeKnownBits, and pick whichever answer is better.
02481     }
02482     break;
02483 
02484   case ISD::SELECT:
02485     Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02486     if (Tmp == 1) return 1;  // Early out.
02487     Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1);
02488     return std::min(Tmp, Tmp2);
02489 
02490   case ISD::SADDO:
02491   case ISD::UADDO:
02492   case ISD::SSUBO:
02493   case ISD::USUBO:
02494   case ISD::SMULO:
02495   case ISD::UMULO:
02496     if (Op.getResNo() != 1)
02497       break;
02498     // The boolean result conforms to getBooleanContents.  Fall through.
02499     // If setcc returns 0/-1, all bits are sign bits.
02500     // We know that we have an integer-based boolean since these operations
02501     // are only available for integer.
02502     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
02503         TargetLowering::ZeroOrNegativeOneBooleanContent)
02504       return VTBits;
02505     break;
02506   case ISD::SETCC:
02507     // If setcc returns 0/-1, all bits are sign bits.
02508     if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
02509         TargetLowering::ZeroOrNegativeOneBooleanContent)
02510       return VTBits;
02511     break;
02512   case ISD::ROTL:
02513   case ISD::ROTR:
02514     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02515       unsigned RotAmt = C->getZExtValue() & (VTBits-1);
02516 
02517       // Handle rotate right by N like a rotate left by 32-N.
02518       if (Op.getOpcode() == ISD::ROTR)
02519         RotAmt = (VTBits-RotAmt) & (VTBits-1);
02520 
02521       // If we aren't rotating out all of the known-in sign bits, return the
02522       // number that are left.  This handles rotl(sext(x), 1) for example.
02523       Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02524       if (Tmp > RotAmt+1) return Tmp-RotAmt;
02525     }
02526     break;
02527   case ISD::ADD:
02528     // Add can have at most one carry bit.  Thus we know that the output
02529     // is, at worst, one more bit than the inputs.
02530     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02531     if (Tmp == 1) return 1;  // Early out.
02532 
02533     // Special case decrementing a value (ADD X, -1):
02534     if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
02535       if (CRHS->isAllOnesValue()) {
02536         APInt KnownZero, KnownOne;
02537         computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02538 
02539         // If the input is known to be 0 or 1, the output is 0/-1, which is all
02540         // sign bits set.
02541         if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
02542           return VTBits;
02543 
02544         // If we are subtracting one from a positive number, there is no carry
02545         // out of the result.
02546         if (KnownZero.isNegative())
02547           return Tmp;
02548       }
02549 
02550     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02551     if (Tmp2 == 1) return 1;
02552     return std::min(Tmp, Tmp2)-1;
02553 
02554   case ISD::SUB:
02555     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02556     if (Tmp2 == 1) return 1;
02557 
02558     // Handle NEG.
02559     if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0)))
02560       if (CLHS->isNullValue()) {
02561         APInt KnownZero, KnownOne;
02562         computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
02563         // If the input is known to be 0 or 1, the output is 0/-1, which is all
02564         // sign bits set.
02565         if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
02566           return VTBits;
02567 
02568         // If the input is known to be positive (the sign bit is known clear),
02569         // the output of the NEG has the same number of sign bits as the input.
02570         if (KnownZero.isNegative())
02571           return Tmp2;
02572 
02573         // Otherwise, we treat this like a SUB.
02574       }
02575 
02576     // Sub can have at most one carry bit.  Thus we know that the output
02577     // is, at worst, one more bit than the inputs.
02578     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02579     if (Tmp == 1) return 1;  // Early out.
02580     return std::min(Tmp, Tmp2)-1;
02581   case ISD::TRUNCATE:
02582     // FIXME: it's tricky to do anything useful for this, but it is an important
02583     // case for targets like X86.
02584     break;
02585   case ISD::EXTRACT_ELEMENT: {
02586     const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02587     const int BitWidth = Op.getValueType().getSizeInBits();
02588     const int Items =
02589       Op.getOperand(0).getValueType().getSizeInBits() / BitWidth;
02590 
02591     // Get reverse index (starting from 1), Op1 value indexes elements from
02592     // little end. Sign starts at big end.
02593     const int rIndex = Items - 1 -
02594       cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
02595 
02596     // If the sign portion ends in our element the substraction gives correct
02597     // result. Otherwise it gives either negative or > bitwidth result
02598     return std::max(std::min(KnownSign - rIndex * BitWidth, BitWidth), 0);
02599   }
02600   }
02601 
02602   // If we are looking at the loaded value of the SDNode.
02603   if (Op.getResNo() == 0) {
02604     // Handle LOADX separately here. EXTLOAD case will fallthrough.
02605     if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
02606       unsigned ExtType = LD->getExtensionType();
02607       switch (ExtType) {
02608         default: break;
02609         case ISD::SEXTLOAD:    // '17' bits known
02610           Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
02611           return VTBits-Tmp+1;
02612         case ISD::ZEXTLOAD:    // '16' bits known
02613           Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
02614           return VTBits-Tmp;
02615       }
02616     }
02617   }
02618 
02619   // Allow the target to implement this method for its nodes.
02620   if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
02621       Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
02622       Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
02623       Op.getOpcode() == ISD::INTRINSIC_VOID) {
02624     unsigned NumBits = TLI->ComputeNumSignBitsForTargetNode(Op, *this, Depth);
02625     if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits);
02626   }
02627 
02628   // Finally, if we can prove that the top bits of the result are 0's or 1's,
02629   // use this information.
02630   APInt KnownZero, KnownOne;
02631   computeKnownBits(Op, KnownZero, KnownOne, Depth);
02632 
02633   APInt Mask;
02634   if (KnownZero.isNegative()) {        // sign bit is 0
02635     Mask = KnownZero;
02636   } else if (KnownOne.isNegative()) {  // sign bit is 1;
02637     Mask = KnownOne;
02638   } else {
02639     // Nothing known.
02640     return FirstAnswer;
02641   }
02642 
02643   // Okay, we know that the sign bit in Mask is set.  Use CLZ to determine
02644   // the number of identical bits in the top of the input value.
02645   Mask = ~Mask;
02646   Mask <<= Mask.getBitWidth()-VTBits;
02647   // Return # leading zeros.  We use 'min' here in case Val was zero before
02648   // shifting.  We don't want to return '64' as for an i32 "0".
02649   return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros()));
02650 }
02651 
02652 /// isBaseWithConstantOffset - Return true if the specified operand is an
02653 /// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an
02654 /// ISD::OR with a ConstantSDNode that is guaranteed to have the same
02655 /// semantics as an ADD.  This handles the equivalence:
02656 ///     X|Cst == X+Cst iff X&Cst = 0.
02657 bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
02658   if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
02659       !isa<ConstantSDNode>(Op.getOperand(1)))
02660     return false;
02661 
02662   if (Op.getOpcode() == ISD::OR &&
02663       !MaskedValueIsZero(Op.getOperand(0),
02664                      cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue()))
02665     return false;
02666 
02667   return true;
02668 }
02669 
02670 
02671 bool SelectionDAG::isKnownNeverNaN(SDValue Op) const {
02672   // If we're told that NaNs won't happen, assume they won't.
02673   if (getTarget().Options.NoNaNsFPMath)
02674     return true;
02675 
02676   // If the value is a constant, we can obviously see if it is a NaN or not.
02677   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
02678     return !C->getValueAPF().isNaN();
02679 
02680   // TODO: Recognize more cases here.
02681 
02682   return false;
02683 }
02684 
02685 bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
02686   // If the value is a constant, we can obviously see if it is a zero or not.
02687   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
02688     return !C->isZero();
02689 
02690   // TODO: Recognize more cases here.
02691   switch (Op.getOpcode()) {
02692   default: break;
02693   case ISD::OR:
02694     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
02695       return !C->isNullValue();
02696     break;
02697   }
02698 
02699   return false;
02700 }
02701 
02702 bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
02703   // Check the obvious case.
02704   if (A == B) return true;
02705 
02706   // For for negative and positive zero.
02707   if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
02708     if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
02709       if (CA->isZero() && CB->isZero()) return true;
02710 
02711   // Otherwise they may not be equal.
02712   return false;
02713 }
02714 
02715 /// getNode - Gets or creates the specified node.
02716 ///
02717 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT) {
02718   FoldingSetNodeID ID;
02719   AddNodeIDNode(ID, Opcode, getVTList(VT), None);
02720   void *IP = nullptr;
02721   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
02722     return SDValue(E, 0);
02723 
02724   SDNode *N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(),
02725                                          DL.getDebugLoc(), getVTList(VT));
02726   CSEMap.InsertNode(N, IP);
02727 
02728   InsertNode(N);
02729   return SDValue(N, 0);
02730 }
02731 
02732 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
02733                               EVT VT, SDValue Operand) {
02734   // Constant fold unary operations with an integer constant operand. Even
02735   // opaque constant will be folded, because the folding of unary operations
02736   // doesn't create new constants with different values. Nevertheless, the
02737   // opaque flag is preserved during folding to prevent future folding with
02738   // other constants.
02739   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) {
02740     const APInt &Val = C->getAPIntValue();
02741     switch (Opcode) {
02742     default: break;
02743     case ISD::SIGN_EXTEND:
02744       return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT,
02745                          C->isTargetOpcode(), C->isOpaque());
02746     case ISD::ANY_EXTEND:
02747     case ISD::ZERO_EXTEND:
02748     case ISD::TRUNCATE:
02749       return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT,
02750                          C->isTargetOpcode(), C->isOpaque());
02751     case ISD::UINT_TO_FP:
02752     case ISD::SINT_TO_FP: {
02753       APFloat apf(EVTToAPFloatSemantics(VT),
02754                   APInt::getNullValue(VT.getSizeInBits()));
02755       (void)apf.convertFromAPInt(Val,
02756                                  Opcode==ISD::SINT_TO_FP,
02757                                  APFloat::rmNearestTiesToEven);
02758       return getConstantFP(apf, VT);
02759     }
02760     case ISD::BITCAST:
02761       if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
02762         return getConstantFP(APFloat(APFloat::IEEEhalf, Val), VT);
02763       if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
02764         return getConstantFP(APFloat(APFloat::IEEEsingle, Val), VT);
02765       else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
02766         return getConstantFP(APFloat(APFloat::IEEEdouble, Val), VT);
02767       break;
02768     case ISD::BSWAP:
02769       return getConstant(Val.byteSwap(), VT, C->isTargetOpcode(),
02770                          C->isOpaque());
02771     case ISD::CTPOP:
02772       return getConstant(Val.countPopulation(), VT, C->isTargetOpcode(),
02773                          C->isOpaque());
02774     case ISD::CTLZ:
02775     case ISD::CTLZ_ZERO_UNDEF:
02776       return getConstant(Val.countLeadingZeros(), VT, C->isTargetOpcode(),
02777                          C->isOpaque());
02778     case ISD::CTTZ:
02779     case ISD::CTTZ_ZERO_UNDEF:
02780       return getConstant(Val.countTrailingZeros(), VT, C->isTargetOpcode(),
02781                          C->isOpaque());
02782     }
02783   }
02784 
02785   // Constant fold unary operations with a floating point constant operand.
02786   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) {
02787     APFloat V = C->getValueAPF();    // make copy
02788     switch (Opcode) {
02789     case ISD::FNEG:
02790       V.changeSign();
02791       return getConstantFP(V, VT);
02792     case ISD::FABS:
02793       V.clearSign();
02794       return getConstantFP(V, VT);
02795     case ISD::FCEIL: {
02796       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
02797       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02798         return getConstantFP(V, VT);
02799       break;
02800     }
02801     case ISD::FTRUNC: {
02802       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
02803       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02804         return getConstantFP(V, VT);
02805       break;
02806     }
02807     case ISD::FFLOOR: {
02808       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
02809       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02810         return getConstantFP(V, VT);
02811       break;
02812     }
02813     case ISD::FP_EXTEND: {
02814       bool ignored;
02815       // This can return overflow, underflow, or inexact; we don't care.
02816       // FIXME need to be more flexible about rounding mode.
02817       (void)V.convert(EVTToAPFloatSemantics(VT),
02818                       APFloat::rmNearestTiesToEven, &ignored);
02819       return getConstantFP(V, VT);
02820     }
02821     case ISD::FP_TO_SINT:
02822     case ISD::FP_TO_UINT: {
02823       integerPart x[2];
02824       bool ignored;
02825       static_assert(integerPartWidth >= 64, "APFloat parts too small!");
02826       // FIXME need to be more flexible about rounding mode.
02827       APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(),
02828                             Opcode==ISD::FP_TO_SINT,
02829                             APFloat::rmTowardZero, &ignored);
02830       if (s==APFloat::opInvalidOp)     // inexact is OK, in fact usual
02831         break;
02832       APInt api(VT.getSizeInBits(), x);
02833       return getConstant(api, VT);
02834     }
02835     case ISD::BITCAST:
02836       if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
02837         return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), VT);
02838       else if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
02839         return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT);
02840       else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
02841         return getConstant(V.bitcastToAPInt().getZExtValue(), VT);
02842       break;
02843     }
02844   }
02845 
02846   // Constant fold unary operations with a vector integer or float operand.
02847   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand.getNode())) {
02848     if (BV->isConstant()) {
02849       switch (Opcode) {
02850       default:
02851         // FIXME: Entirely reasonable to perform folding of other unary
02852         // operations here as the need arises.
02853         break;
02854       case ISD::FNEG:
02855       case ISD::FABS:
02856       case ISD::FP_EXTEND:
02857       case ISD::TRUNCATE:
02858       case ISD::UINT_TO_FP:
02859       case ISD::SINT_TO_FP: {
02860         // Let the above scalar folding handle the folding of each element.
02861         SmallVector<SDValue, 8> Ops;
02862         for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
02863           SDValue OpN = BV->getOperand(i);
02864           OpN = getNode(Opcode, DL, VT.getVectorElementType(), OpN);
02865           if (OpN.getOpcode() != ISD::UNDEF &&
02866               OpN.getOpcode() != ISD::Constant &&
02867               OpN.getOpcode() != ISD::ConstantFP)
02868             break;
02869           Ops.push_back(OpN);
02870         }
02871         if (Ops.size() == VT.getVectorNumElements())
02872           return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
02873       }
02874       }
02875     }
02876   }
02877 
02878   unsigned OpOpcode = Operand.getNode()->getOpcode();
02879   switch (Opcode) {
02880   case ISD::TokenFactor:
02881   case ISD::MERGE_VALUES:
02882   case ISD::CONCAT_VECTORS:
02883     return Operand;         // Factor, merge or concat of one node?  No need.
02884   case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
02885   case ISD::FP_EXTEND:
02886     assert(VT.isFloatingPoint() &&
02887            Operand.getValueType().isFloatingPoint() && "Invalid FP cast!");
02888     if (Operand.getValueType() == VT) return Operand;  // noop conversion.
02889     assert((!VT.isVector() ||
02890             VT.getVectorNumElements() ==
02891             Operand.getValueType().getVectorNumElements()) &&
02892            "Vector element count mismatch!");
02893     if (Operand.getOpcode() == ISD::UNDEF)
02894       return getUNDEF(VT);
02895     break;
02896   case ISD::SIGN_EXTEND:
02897     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02898            "Invalid SIGN_EXTEND!");
02899     if (Operand.getValueType() == VT) return Operand;   // noop extension
02900     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02901            "Invalid sext node, dst < src!");
02902     assert((!VT.isVector() ||
02903             VT.getVectorNumElements() ==
02904             Operand.getValueType().getVectorNumElements()) &&
02905            "Vector element count mismatch!");
02906     if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
02907       return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02908     else if (OpOpcode == ISD::UNDEF)
02909       // sext(undef) = 0, because the top bits will all be the same.
02910       return getConstant(0, VT);
02911     break;
02912   case ISD::ZERO_EXTEND:
02913     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02914            "Invalid ZERO_EXTEND!");
02915     if (Operand.getValueType() == VT) return Operand;   // noop extension
02916     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02917            "Invalid zext node, dst < src!");
02918     assert((!VT.isVector() ||
02919             VT.getVectorNumElements() ==
02920             Operand.getValueType().getVectorNumElements()) &&
02921            "Vector element count mismatch!");
02922     if (OpOpcode == ISD::ZERO_EXTEND)   // (zext (zext x)) -> (zext x)
02923       return getNode(ISD::ZERO_EXTEND, DL, VT,
02924                      Operand.getNode()->getOperand(0));
02925     else if (OpOpcode == ISD::UNDEF)
02926       // zext(undef) = 0, because the top bits will be zero.
02927       return getConstant(0, VT);
02928     break;
02929   case ISD::ANY_EXTEND:
02930     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02931            "Invalid ANY_EXTEND!");
02932     if (Operand.getValueType() == VT) return Operand;   // noop extension
02933     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02934            "Invalid anyext node, dst < src!");
02935     assert((!VT.isVector() ||
02936             VT.getVectorNumElements() ==
02937             Operand.getValueType().getVectorNumElements()) &&
02938            "Vector element count mismatch!");
02939 
02940     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
02941         OpOpcode == ISD::ANY_EXTEND)
02942       // (ext (zext x)) -> (zext x)  and  (ext (sext x)) -> (sext x)
02943       return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02944     else if (OpOpcode == ISD::UNDEF)
02945       return getUNDEF(VT);
02946 
02947     // (ext (trunx x)) -> x
02948     if (OpOpcode == ISD::TRUNCATE) {
02949       SDValue OpOp = Operand.getNode()->getOperand(0);
02950       if (OpOp.getValueType() == VT)
02951         return OpOp;
02952     }
02953     break;
02954   case ISD::TRUNCATE:
02955     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02956            "Invalid TRUNCATE!");
02957     if (Operand.getValueType() == VT) return Operand;   // noop truncate
02958     assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) &&
02959            "Invalid truncate node, src < dst!");
02960     assert((!VT.isVector() ||
02961             VT.getVectorNumElements() ==
02962             Operand.getValueType().getVectorNumElements()) &&
02963            "Vector element count mismatch!");
02964     if (OpOpcode == ISD::TRUNCATE)
02965       return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
02966     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
02967         OpOpcode == ISD::ANY_EXTEND) {
02968       // If the source is smaller than the dest, we still need an extend.
02969       if (Operand.getNode()->getOperand(0).getValueType().getScalarType()
02970             .bitsLT(VT.getScalarType()))
02971         return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02972       if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT))
02973         return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
02974       return Operand.getNode()->getOperand(0);
02975     }
02976     if (OpOpcode == ISD::UNDEF)
02977       return getUNDEF(VT);
02978     break;
02979   case ISD::BITCAST:
02980     // Basic sanity checking.
02981     assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits()
02982            && "Cannot BITCAST between types of different sizes!");
02983     if (VT == Operand.getValueType()) return Operand;  // noop conversion.
02984     if (OpOpcode == ISD::BITCAST)  // bitconv(bitconv(x)) -> bitconv(x)
02985       return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
02986     if (OpOpcode == ISD::UNDEF)
02987       return getUNDEF(VT);
02988     break;
02989   case ISD::SCALAR_TO_VECTOR:
02990     assert(VT.isVector() && !Operand.getValueType().isVector() &&
02991            (VT.getVectorElementType() == Operand.getValueType() ||
02992             (VT.getVectorElementType().isInteger() &&
02993              Operand.getValueType().isInteger() &&
02994              VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&
02995            "Illegal SCALAR_TO_VECTOR node!");
02996     if (OpOpcode == ISD::UNDEF)
02997       return getUNDEF(VT);
02998     // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
02999     if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
03000         isa<ConstantSDNode>(Operand.getOperand(1)) &&
03001         Operand.getConstantOperandVal(1) == 0 &&
03002         Operand.getOperand(0).getValueType() == VT)
03003       return Operand.getOperand(0);
03004     break;
03005   case ISD::FNEG:
03006     // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0
03007     if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB)
03008       return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1),
03009                      Operand.getNode()->getOperand(0));
03010     if (OpOpcode == ISD::FNEG)  // --X -> X
03011       return Operand.getNode()->getOperand(0);
03012     break;
03013   case ISD::FABS:
03014     if (OpOpcode == ISD::FNEG)  // abs(-X) -> abs(X)
03015       return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0));
03016     break;
03017   }
03018 
03019   SDNode *N;
03020   SDVTList VTs = getVTList(VT);
03021   if (VT != MVT::Glue) { // Don't CSE flag producing nodes
03022     FoldingSetNodeID ID;
03023     SDValue Ops[1] = { Operand };
03024     AddNodeIDNode(ID, Opcode, VTs, Ops);
03025     void *IP = nullptr;
03026     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
03027       return SDValue(E, 0);
03028 
03029     N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
03030                                         DL.getDebugLoc(), VTs, Operand);
03031     CSEMap.InsertNode(N, IP);
03032   } else {
03033     N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
03034                                         DL.getDebugLoc(), VTs, Operand);
03035   }
03036 
03037   InsertNode(N);
03038   return SDValue(N, 0);
03039 }
03040 
03041 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT,
03042                                              SDNode *Cst1, SDNode *Cst2) {
03043   // If the opcode is a target-specific ISD node, there's nothing we can
03044   // do here and the operand rules may not line up with the below, so
03045   // bail early.
03046   if (Opcode >= ISD::BUILTIN_OP_END)
03047     return SDValue();
03048 
03049   SmallVector<std::pair<ConstantSDNode *, ConstantSDNode *>, 4> Inputs;
03050   SmallVector<SDValue, 4> Outputs;
03051   EVT SVT = VT.getScalarType();
03052 
03053   ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1);
03054   ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2);
03055   if (Scalar1 && Scalar2 && (Scalar1->isOpaque() || Scalar2->isOpaque()))
03056     return SDValue();
03057 
03058   if (Scalar1 && Scalar2)
03059     // Scalar instruction.
03060     Inputs.push_back(std::make_pair(Scalar1, Scalar2));
03061   else {
03062     // For vectors extract each constant element into Inputs so we can constant
03063     // fold them individually.
03064     BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1);
03065     BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2);
03066     if (!BV1 || !BV2)
03067       return SDValue();
03068 
03069     assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!");
03070 
03071     for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) {
03072       ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I));
03073       ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I));
03074       if (!V1 || !V2) // Not a constant, bail.
03075         return SDValue();
03076 
03077       if (V1->isOpaque() || V2->isOpaque())
03078         return SDValue();
03079 
03080       // Avoid BUILD_VECTOR nodes that perform implicit truncation.
03081       // FIXME: This is valid and could be handled by truncating the APInts.
03082       if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
03083         return SDValue();
03084 
03085       Inputs.push_back(std::make_pair(V1, V2));
03086     }
03087   }
03088 
03089   // We have a number of constant values, constant fold them element by element.
03090   for (unsigned I = 0, E = Inputs.size(); I != E; ++I) {
03091     const APInt &C1 = Inputs[I].first->getAPIntValue();
03092     const APInt &C2 = Inputs[I].second->getAPIntValue();
03093 
03094     switch (Opcode) {
03095     case ISD::ADD:
03096       Outputs.push_back(getConstant(C1 + C2, SVT));
03097       break;
03098     case ISD::SUB:
03099       Outputs.push_back(getConstant(C1 - C2, SVT));
03100       break;
03101     case ISD::MUL:
03102       Outputs.push_back(getConstant(C1 * C2, SVT));
03103       break;
03104     case ISD::UDIV:
03105       if (!C2.getBoolValue())
03106         return SDValue();
03107       Outputs.push_back(getConstant(C1.udiv(C2), SVT));
03108       break;
03109     case ISD::UREM:
03110       if (!C2.getBoolValue())
03111         return SDValue();
03112       Outputs.push_back(getConstant(C1.urem(C2), SVT));
03113       break;
03114     case ISD::SDIV:
03115       if (!C2.getBoolValue())
03116         return SDValue();
03117       Outputs.push_back(getConstant(C1.sdiv(C2), SVT));
03118       break;
03119     case ISD::SREM:
03120       if (!C2.getBoolValue())
03121         return SDValue();
03122       Outputs.push_back(getConstant(C1.srem(C2), SVT));
03123       break;
03124     case ISD::AND:
03125       Outputs.push_back(getConstant(C1 & C2, SVT));
03126       break;
03127     case ISD::OR:
03128       Outputs.push_back(getConstant(C1 | C2, SVT));
03129       break;
03130     case ISD::XOR:
03131       Outputs.push_back(getConstant(C1 ^ C2, SVT));
03132       break;
03133     case ISD::SHL:
03134       Outputs.push_back(getConstant(C1 << C2, SVT));
03135       break;
03136     case ISD::SRL:
03137       Outputs.push_back(getConstant(C1.lshr(C2), SVT));
03138       break;
03139     case ISD::SRA:
03140       Outputs.push_back(getConstant(C1.ashr(C2), SVT));
03141       break;
03142     case ISD::ROTL:
03143       Outputs.push_back(getConstant(C1.rotl(C2), SVT));
03144       break;
03145     case ISD::ROTR:
03146       Outputs.push_back(getConstant(C1.rotr(C2), SVT));
03147       break;
03148     default:
03149       return SDValue();
03150     }
03151   }
03152 
03153   assert((Scalar1 && Scalar2) || (VT.getVectorNumElements() == Outputs.size() &&
03154                                   "Expected a scalar or vector!"));
03155 
03156   // Handle the scalar case first.
03157   if (!VT.isVector())
03158     return Outputs.back();
03159 
03160   // We may have a vector type but a scalar result. Create a splat.
03161   Outputs.resize(VT.getVectorNumElements(), Outputs.back());
03162 
03163   // Build a big vector out of the scalar elements we generated.
03164   return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs);
03165 }
03166 
03167 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
03168                               SDValue N2, bool nuw, bool nsw, bool exact) {
03169   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
03170   ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode());
03171   switch (Opcode) {
03172   default: break;
03173   case ISD::TokenFactor:
03174     assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
03175            N2.getValueType() == MVT::Other && "Invalid token factor!");
03176     // Fold trivial token factors.
03177     if (N1.getOpcode() == ISD::EntryToken) return N2;
03178     if (N2.getOpcode() == ISD::EntryToken) return N1;
03179     if (N1 == N2) return N1;
03180     break;
03181   case ISD::CONCAT_VECTORS:
03182     // Concat of UNDEFs is UNDEF.
03183     if (N1.getOpcode() == ISD::UNDEF &&
03184         N2.getOpcode() == ISD::UNDEF)
03185       return getUNDEF(VT);
03186 
03187     // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
03188     // one big BUILD_VECTOR.
03189     if (N1.getOpcode() == ISD::BUILD_VECTOR &&
03190         N2.getOpcode() == ISD::BUILD_VECTOR) {
03191       SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
03192                                     N1.getNode()->op_end());
03193       Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
03194       return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
03195     }
03196     break;
03197   case ISD::AND:
03198     assert(VT.isInteger() && "This operator does not apply to FP types!");
03199     assert(N1.getValueType() == N2.getValueType() &&
03200            N1.getValueType() == VT && "Binary operator types must match!");
03201     // (X & 0) -> 0.  This commonly occurs when legalizing i64 values, so it's
03202     // worth handling here.
03203     if (N2C && N2C->isNullValue())
03204       return N2;
03205     if (N2C && N2C->isAllOnesValue())  // X & -1 -> X
03206       return N1;
03207     break;
03208   case ISD::OR:
03209   case ISD::XOR:
03210   case ISD::ADD:
03211   case ISD::SUB:
03212     assert(VT.isInteger() && "This operator does not apply to FP types!");
03213     assert(N1.getValueType() == N2.getValueType() &&
03214            N1.getValueType() == VT && "Binary operator types must match!");
03215     // (X ^|+- 0) -> X.  This commonly occurs when legalizing i64 values, so
03216     // it's worth handling here.
03217     if (N2C && N2C->isNullValue())
03218       return N1;
03219     break;
03220   case ISD::UDIV:
03221   case ISD::UREM:
03222   case ISD::MULHU:
03223   case ISD::MULHS:
03224   case ISD::MUL:
03225   case ISD::SDIV:
03226   case ISD::SREM:
03227     assert(VT.isInteger() && "This operator does not apply to FP types!");
03228     assert(N1.getValueType() == N2.getValueType() &&
03229            N1.getValueType() == VT && "Binary operator types must match!");
03230     break;
03231   case ISD::FADD:
03232   case ISD::FSUB:
03233   case ISD::FMUL:
03234   case ISD::FDIV:
03235   case ISD::FREM:
03236     if (getTarget().Options.UnsafeFPMath) {
03237       if (Opcode == ISD::FADD) {
03238         // 0+x --> x
03239         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1))
03240           if (CFP->getValueAPF().isZero())
03241             return N2;
03242         // x+0 --> x
03243         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
03244           if (CFP->getValueAPF().isZero())
03245             return N1;
03246       } else if (Opcode == ISD::FSUB) {
03247         // x-0 --> x
03248         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
03249           if (CFP->getValueAPF().isZero())
03250             return N1;
03251       } else if (Opcode == ISD::FMUL) {
03252         ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1);
03253         SDValue V = N2;
03254 
03255         // If the first operand isn't the constant, try the second
03256         if (!CFP) {
03257           CFP = dyn_cast<ConstantFPSDNode>(N2);
03258           V = N1;
03259         }
03260 
03261         if (CFP) {
03262           // 0*x --> 0
03263           if (CFP->isZero())
03264             return SDValue(CFP,0);
03265           // 1*x --> x
03266           if (CFP->isExactlyValue(1.0))
03267             return V;
03268         }
03269       }
03270     }
03271     assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
03272     assert(N1.getValueType() == N2.getValueType() &&
03273            N1.getValueType() == VT && "Binary operator types must match!");
03274     break;
03275   case ISD::FCOPYSIGN:   // N1 and result must match.  N1/N2 need not match.
03276     assert(N1.getValueType() == VT &&
03277            N1.getValueType().isFloatingPoint() &&
03278            N2.getValueType().isFloatingPoint() &&
03279            "Invalid FCOPYSIGN!");
03280     break;
03281   case ISD::SHL:
03282   case ISD::SRA:
03283   case ISD::SRL:
03284   case ISD::ROTL:
03285   case ISD::ROTR:
03286     assert(VT == N1.getValueType() &&
03287            "Shift operators return type must be the same as their first arg");
03288     assert(VT.isInteger() && N2.getValueType().isInteger() &&
03289            "Shifts only work on integers");
03290     assert((!VT.isVector() || VT == N2.getValueType()) &&
03291            "Vector shift amounts must be in the same as their first arg");
03292     // Verify that the shift amount VT is bit enough to hold valid shift
03293     // amounts.  This catches things like trying to shift an i1024 value by an
03294     // i8, which is easy to fall into in generic code that uses
03295     // TLI.getShiftAmount().
03296     assert(N2.getValueType().getSizeInBits() >=
03297                    Log2_32_Ceil(N1.getValueType().getSizeInBits()) &&
03298            "Invalid use of small shift amount with oversized value!");
03299 
03300     // Always fold shifts of i1 values so the code generator doesn't need to
03301     // handle them.  Since we know the size of the shift has to be less than the
03302     // size of the value, the shift/rotate count is guaranteed to be zero.
03303     if (VT == MVT::i1)
03304       return N1;
03305     if (N2C && N2C->isNullValue())
03306       return N1;
03307     break;
03308   case ISD::FP_ROUND_INREG: {
03309     EVT EVT = cast<VTSDNode>(N2)->getVT();
03310     assert(VT == N1.getValueType() && "Not an inreg round!");
03311     assert(VT.isFloatingPoint() && EVT.isFloatingPoint() &&
03312            "Cannot FP_ROUND_INREG integer types");
03313     assert(EVT.isVector() == VT.isVector() &&
03314            "FP_ROUND_INREG type should be vector iff the operand "
03315            "type is vector!");
03316     assert((!EVT.isVector() ||
03317             EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
03318            "Vector element counts must match in FP_ROUND_INREG");
03319     assert(EVT.bitsLE(VT) && "Not rounding down!");
03320     (void)EVT;
03321     if (cast<VTSDNode>(N2)->getVT() == VT) return N1;  // Not actually rounding.
03322     break;
03323   }
03324   case ISD::FP_ROUND:
03325     assert(VT.isFloatingPoint() &&
03326            N1.getValueType().isFloatingPoint() &&
03327            VT.bitsLE(N1.getValueType()) &&
03328            isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!");
03329     if (N1.getValueType() == VT) return N1;  // noop conversion.
03330     break;
03331   case ISD::AssertSext:
03332   case ISD::AssertZext: {
03333     EVT EVT = cast<VTSDNode>(N2)->getVT();
03334     assert(VT == N1.getValueType() && "Not an inreg extend!");
03335     assert(VT.isInteger() && EVT.isInteger() &&
03336            "Cannot *_EXTEND_INREG FP types");
03337     assert(!EVT.isVector() &&
03338            "AssertSExt/AssertZExt type should be the vector element type "
03339            "rather than the vector type!");
03340     assert(EVT.bitsLE(VT) && "Not extending!");
03341     if (VT == EVT) return N1; // noop assertion.
03342     break;
03343   }
03344   case ISD::SIGN_EXTEND_INREG: {
03345     EVT EVT = cast<VTSDNode>(N2)->getVT();
03346     assert(VT == N1.getValueType() && "Not an inreg extend!");
03347     assert(VT.isInteger() && EVT.isInteger() &&
03348            "Cannot *_EXTEND_INREG FP types");
03349     assert(EVT.isVector() == VT.isVector() &&
03350            "SIGN_EXTEND_INREG type should be vector iff the operand "
03351            "type is vector!");
03352     assert((!EVT.isVector() ||
03353             EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
03354            "Vector element counts must match in SIGN_EXTEND_INREG");
03355     assert(EVT.bitsLE(VT) && "Not extending!");
03356     if (EVT == VT) return N1;  // Not actually extending
03357 
03358     if (N1C) {
03359       APInt Val = N1C->getAPIntValue();
03360       unsigned FromBits = EVT.getScalarType().getSizeInBits();
03361       Val <<= Val.getBitWidth()-FromBits;
03362       Val = Val.ashr(Val.getBitWidth()-FromBits);
03363       return getConstant(Val, VT);
03364     }
03365     break;
03366   }
03367   case ISD::EXTRACT_VECTOR_ELT:
03368     // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF.
03369     if (N1.getOpcode() == ISD::UNDEF)
03370       return getUNDEF(VT);
03371 
03372     // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
03373     // expanding copies of large vectors from registers.
03374     if (N2C &&
03375         N1.getOpcode() == ISD::CONCAT_VECTORS &&
03376         N1.getNumOperands() > 0) {
03377       unsigned Factor =
03378         N1.getOperand(0).getValueType().getVectorNumElements();
03379       return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
03380                      N1.getOperand(N2C->getZExtValue() / Factor),
03381                      getConstant(N2C->getZExtValue() % Factor,
03382                                  N2.getValueType()));
03383     }
03384 
03385     // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
03386     // expanding large vector constants.
03387     if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) {
03388       SDValue Elt = N1.getOperand(N2C->getZExtValue());
03389 
03390       if (VT != Elt.getValueType())
03391         // If the vector element type is not legal, the BUILD_VECTOR operands
03392         // are promoted and implicitly truncated, and the result implicitly
03393         // extended. Make that explicit here.
03394         Elt = getAnyExtOrTrunc(Elt, DL, VT);
03395 
03396       return Elt;
03397     }
03398 
03399     // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
03400     // operations are lowered to scalars.
03401     if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
03402       // If the indices are the same, return the inserted element else
03403       // if the indices are known different, extract the element from
03404       // the original vector.
03405       SDValue N1Op2 = N1.getOperand(2);
03406       ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode());
03407 
03408       if (N1Op2C && N2C) {
03409         if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
03410           if (VT == N1.getOperand(1).getValueType())
03411             return N1.getOperand(1);
03412           else
03413             return getSExtOrTrunc(N1.getOperand(1), DL, VT);
03414         }
03415 
03416         return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
03417       }
03418     }
03419     break;
03420   case ISD::EXTRACT_ELEMENT:
03421     assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
03422     assert(!N1.getValueType().isVector() && !VT.isVector() &&
03423            (N1.getValueType().isInteger() == VT.isInteger()) &&
03424            N1.getValueType() != VT &&
03425            "Wrong types for EXTRACT_ELEMENT!");
03426 
03427     // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
03428     // 64-bit integers into 32-bit parts.  Instead of building the extract of
03429     // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
03430     if (N1.getOpcode() == ISD::BUILD_PAIR)
03431       return N1.getOperand(N2C->getZExtValue());
03432 
03433     // EXTRACT_ELEMENT of a constant int is also very common.
03434     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
03435       unsigned ElementSize = VT.getSizeInBits();
03436       unsigned Shift = ElementSize * N2C->getZExtValue();
03437       APInt ShiftedVal = C->getAPIntValue().lshr(Shift);
03438       return getConstant(ShiftedVal.trunc(ElementSize), VT);
03439     }
03440     break;
03441   case ISD::EXTRACT_SUBVECTOR: {
03442     SDValue Index = N2;
03443     if (VT.isSimple() && N1.getValueType().isSimple()) {
03444       assert(VT.isVector() && N1.getValueType().isVector() &&
03445              "Extract subvector VTs must be a vectors!");
03446       assert(VT.getVectorElementType() ==
03447              N1.getValueType().getVectorElementType() &&
03448              "Extract subvector VTs must have the same element type!");
03449       assert(VT.getSimpleVT() <= N1.getSimpleValueType() &&
03450              "Extract subvector must be from larger vector to smaller vector!");
03451 
03452       if (isa<ConstantSDNode>(Index.getNode())) {
03453         assert((VT.getVectorNumElements() +
03454                 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
03455                 <= N1.getValueType().getVectorNumElements())
03456                && "Extract subvector overflow!");
03457       }
03458 
03459       // Trivial extraction.
03460       if (VT.getSimpleVT() == N1.getSimpleValueType())
03461         return N1;
03462     }
03463     break;
03464   }
03465   }
03466 
03467   // Perform trivial constant folding.
03468   if (SDValue SV =
03469           FoldConstantArithmetic(Opcode, VT, N1.getNode(), N2.getNode()))
03470     return SV;
03471 
03472   // Canonicalize constant to RHS if commutative.
03473   if (N1C && !N2C && isCommutativeBinOp(Opcode)) {
03474     std::swap(N1C, N2C);
03475     std::swap(N1, N2);
03476   }
03477 
03478   // Constant fold FP operations.
03479   bool HasFPExceptions = TLI->hasFloatingPointExceptions();
03480   ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode());
03481   ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode());
03482   if (N1CFP) {
03483     if (!N2CFP && isCommutativeBinOp(Opcode)) {
03484       // Canonicalize constant to RHS if commutative.
03485       std::swap(N1CFP, N2CFP);
03486       std::swap(N1, N2);
03487     } else if (N2CFP) {
03488       APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF();
03489       APFloat::opStatus s;
03490       switch (Opcode) {
03491       case ISD::FADD:
03492         s = V1.add(V2, APFloat::rmNearestTiesToEven);
03493         if (!HasFPExceptions || s != APFloat::opInvalidOp)
03494           return getConstantFP(V1, VT);
03495         break;
03496       case ISD::FSUB:
03497         s = V1.subtract(V2, APFloat::rmNearestTiesToEven);
03498         if (!HasFPExceptions || s!=APFloat::opInvalidOp)
03499           return getConstantFP(V1, VT);
03500         break;
03501       case ISD::FMUL:
03502         s = V1.multiply(V2, APFloat::rmNearestTiesToEven);
03503         if (!HasFPExceptions || s!=APFloat::opInvalidOp)
03504           return getConstantFP(V1, VT);
03505         break;
03506       case ISD::FDIV:
03507         s = V1.divide(V2, APFloat::rmNearestTiesToEven);
03508         if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
03509                                  s!=APFloat::opDivByZero)) {
03510           return getConstantFP(V1, VT);
03511         }
03512         break;
03513       case ISD::FREM :
03514         s = V1.mod(V2, APFloat::rmNearestTiesToEven);
03515         if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
03516                                  s!=APFloat::opDivByZero)) {
03517           return getConstantFP(V1, VT);
03518         }
03519         break;
03520       case ISD::FCOPYSIGN:
03521         V1.copySign(V2);
03522         return getConstantFP(V1, VT);
03523       default: break;
03524       }
03525     }
03526 
03527     if (Opcode == ISD::FP_ROUND) {
03528       APFloat V = N1CFP->getValueAPF();    // make copy
03529       bool ignored;
03530       // This can return overflow, underflow, or inexact; we don't care.
03531       // FIXME need to be more flexible about rounding mode.
03532       (void)V.convert(EVTToAPFloatSemantics(VT),
03533                       APFloat::rmNearestTiesToEven, &ignored);
03534       return getConstantFP(V, VT);
03535     }
03536   }
03537 
03538   // Canonicalize an UNDEF to the RHS, even over a constant.
03539   if (N1.getOpcode() == ISD::UNDEF) {
03540     if (isCommutativeBinOp(Opcode)) {
03541       std::swap(N1, N2);
03542     } else {
03543       switch (Opcode) {
03544       case ISD::FP_ROUND_INREG:
03545       case ISD::SIGN_EXTEND_INREG:
03546       case ISD::SUB:
03547       case ISD::FSUB:
03548       case ISD::FDIV:
03549       case ISD::FREM:
03550       case ISD::SRA:
03551         return N1;     // fold op(undef, arg2) -> undef
03552       case ISD::UDIV:
03553       case ISD::SDIV:
03554       case ISD::UREM:
03555       case ISD::SREM:
03556       case ISD::SRL:
03557       case ISD::SHL:
03558         if (!VT.isVector())
03559           return getConstant(0, VT);    // fold op(undef, arg2) -> 0
03560         // For vectors, we can't easily build an all zero vector, just return
03561         // the LHS.
03562         return N2;
03563       }
03564     }
03565   }
03566 
03567   // Fold a bunch of operators when the RHS is undef.
03568   if (N2.getOpcode() == ISD::UNDEF) {
03569     switch (Opcode) {
03570     case ISD::XOR:
03571       if (N1.getOpcode() == ISD::UNDEF)
03572         // Handle undef ^ undef -> 0 special case. This is a common
03573         // idiom (misuse).
03574         return getConstant(0, VT);
03575       // fallthrough
03576     case ISD::ADD:
03577     case ISD::ADDC:
03578     case ISD::ADDE:
03579     case ISD::SUB:
03580     case ISD::UDIV:
03581     case ISD::SDIV:
03582     case ISD::UREM:
03583     case ISD::SREM:
03584       return N2;       // fold op(arg1, undef) -> undef
03585     case ISD::FADD:
03586     case ISD::FSUB:
03587     case ISD::FMUL:
03588     case ISD::FDIV:
03589     case ISD::FREM:
03590       if (getTarget().Options.UnsafeFPMath)
03591         return N2;
03592       break;
03593     case ISD::MUL:
03594     case ISD::AND:
03595     case ISD::SRL:
03596     case ISD::SHL:
03597       if (!VT.isVector())
03598         return getConstant(0, VT);  // fold op(arg1, undef) -> 0
03599       // For vectors, we can't easily build an all zero vector, just return
03600       // the LHS.
03601       return N1;
03602     case ISD::OR:
03603       if (!VT.isVector())
03604         return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT);
03605       // For vectors, we can't easily build an all one vector, just return
03606       // the LHS.
03607       return N1;
03608     case ISD::SRA:
03609       return N1;
03610     }
03611   }
03612 
03613   // Memoize this node if possible.
03614   BinarySDNode *N;
03615   SDVTList VTs = getVTList(VT);
03616   const bool BinOpHasFlags = isBinOpWithFlags(Opcode);
03617   if (VT != MVT::Glue) {
03618     SDValue Ops[] = {N1, N2};
03619     FoldingSetNodeID ID;
03620     AddNodeIDNode(ID, Opcode, VTs, Ops);
03621     if (BinOpHasFlags)
03622       AddBinaryNodeIDCustom(ID, Opcode, nuw, nsw, exact);
03623     void *IP = nullptr;
03624     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
03625       return SDValue(E, 0);
03626 
03627     N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
03628 
03629     CSEMap.InsertNode(N, IP);
03630   } else {
03631 
03632     N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
03633   }
03634 
03635   InsertNode(N);
03636   return SDValue(N, 0);
03637 }
03638 
03639 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03640                               SDValue N1, SDValue N2, SDValue N3) {
03641   // Perform various simplifications.
03642   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
03643   switch (Opcode) {
03644   case ISD::FMA: {
03645     ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
03646     ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
03647     ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
03648     if (N1CFP && N2CFP && N3CFP) {
03649       APFloat  V1 = N1CFP->getValueAPF();
03650       const APFloat &V2 = N2CFP->getValueAPF();
03651       const APFloat &V3 = N3CFP->getValueAPF();
03652       APFloat::opStatus s =
03653         V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
03654       if (!TLI->hasFloatingPointExceptions() || s != APFloat::opInvalidOp)
03655         return getConstantFP(V1, VT);
03656     }
03657     break;
03658   }
03659   case ISD::CONCAT_VECTORS:
03660     // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
03661     // one big BUILD_VECTOR.
03662     if (N1.getOpcode() == ISD::BUILD_VECTOR &&
03663         N2.getOpcode() == ISD::BUILD_VECTOR &&
03664         N3.getOpcode() == ISD::BUILD_VECTOR) {
03665       SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
03666                                     N1.getNode()->op_end());
03667       Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
03668       Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end());
03669       return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
03670     }
03671     break;
03672   case ISD::SETCC: {
03673     // Use FoldSetCC to simplify SETCC's.
03674     SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL);
03675     if (Simp.getNode()) return Simp;
03676     break;
03677   }
03678   case ISD::SELECT:
03679     if (N1C) {
03680      if (N1C->getZExtValue())
03681        return N2;             // select true, X, Y -> X
03682      return N3;             // select false, X, Y -> Y
03683     }
03684 
03685     if (N2 == N3) return N2;   // select C, X, X -> X
03686     break;
03687   case ISD::VECTOR_SHUFFLE:
03688     llvm_unreachable("should use getVectorShuffle constructor!");
03689   case ISD::INSERT_SUBVECTOR: {
03690     SDValue Index = N3;
03691     if (VT.isSimple() && N1.getValueType().isSimple()
03692         && N2.getValueType().isSimple()) {
03693       assert(VT.isVector() && N1.getValueType().isVector() &&
03694              N2.getValueType().isVector() &&
03695              "Insert subvector VTs must be a vectors");
03696       assert(VT == N1.getValueType() &&
03697              "Dest and insert subvector source types must match!");
03698       assert(N2.getSimpleValueType() <= N1.getSimpleValueType() &&
03699              "Insert subvector must be from smaller vector to larger vector!");
03700       if (isa<ConstantSDNode>(Index.getNode())) {
03701         assert((N2.getValueType().getVectorNumElements() +
03702                 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
03703                 <= VT.getVectorNumElements())
03704                && "Insert subvector overflow!");
03705       }
03706 
03707       // Trivial insertion.
03708       if (VT.getSimpleVT() == N2.getSimpleValueType())
03709         return N2;
03710     }
03711     break;
03712   }
03713   case ISD::BITCAST:
03714     // Fold bit_convert nodes from a type to themselves.
03715     if (N1.getValueType() == VT)
03716       return N1;
03717     break;
03718   }
03719 
03720   // Memoize node if it doesn't produce a flag.
03721   SDNode *N;
03722   SDVTList VTs = getVTList(VT);
03723   if (VT != MVT::Glue) {
03724     SDValue Ops[] = { N1, N2, N3 };
03725     FoldingSetNodeID ID;
03726     AddNodeIDNode(ID, Opcode, VTs, Ops);
03727     void *IP = nullptr;
03728     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
03729       return SDValue(E, 0);
03730 
03731     N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
03732                                           DL.getDebugLoc(), VTs, N1, N2, N3);
03733     CSEMap.InsertNode(N, IP);
03734   } else {
03735     N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
03736                                           DL.getDebugLoc(), VTs, N1, N2, N3);
03737   }
03738 
03739   InsertNode(N);
03740   return SDValue(N, 0);
03741 }
03742 
03743 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03744                               SDValue N1, SDValue N2, SDValue N3,
03745                               SDValue N4) {
03746   SDValue Ops[] = { N1, N2, N3, N4 };
03747   return getNode(Opcode, DL, VT, Ops);
03748 }
03749 
03750 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03751                               SDValue N1, SDValue N2, SDValue N3,
03752                               SDValue N4, SDValue N5) {
03753   SDValue Ops[] = { N1, N2, N3, N4, N5 };
03754   return getNode(Opcode, DL, VT, Ops);
03755 }
03756 
03757 /// getStackArgumentTokenFactor - Compute a TokenFactor to force all
03758 /// the incoming stack arguments to be loaded from the stack.
03759 SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
03760   SmallVector<SDValue, 8> ArgChains;
03761 
03762   // Include the original chain at the beginning of the list. When this is
03763   // used by target LowerCall hooks, this helps legalize find the
03764   // CALLSEQ_BEGIN node.
03765   ArgChains.push_back(Chain);
03766 
03767   // Add a chain value for each stack argument.
03768   for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(),
03769        UE = getEntryNode().getNode()->use_end(); U != UE; ++U)
03770     if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
03771       if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
03772         if (FI->getIndex() < 0)
03773           ArgChains.push_back(SDValue(L, 1));
03774 
03775   // Build a tokenfactor for all the chains.
03776   return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
03777 }
03778 
03779 /// getMemsetValue - Vectorized representation of the memset value
03780 /// operand.
03781 static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
03782                               SDLoc dl) {
03783   assert(Value.getOpcode() != ISD::UNDEF);
03784 
03785   unsigned NumBits = VT.getScalarType().getSizeInBits();
03786   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
03787     assert(C->getAPIntValue().getBitWidth() == 8);
03788     APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
03789     if (VT.isInteger())
03790       return DAG.getConstant(Val, VT);
03791     return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), VT);
03792   }
03793 
03794   Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value);
03795   if (NumBits > 8) {
03796     // Use a multiplication with 0x010101... to extend the input to the
03797     // required length.
03798     APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
03799     Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT));
03800   }
03801 
03802   return Value;
03803 }
03804 
03805 /// getMemsetStringVal - Similar to getMemsetValue. Except this is only
03806 /// used when a memcpy is turned into a memset when the source is a constant
03807 /// string ptr.
03808 static SDValue getMemsetStringVal(EVT VT, SDLoc dl, SelectionDAG &DAG,
03809                                   const TargetLowering &TLI, StringRef Str) {
03810   // Handle vector with all elements zero.
03811   if (Str.empty()) {
03812     if (VT.isInteger())
03813       return DAG.getConstant(0, VT);
03814     else if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
03815       return DAG.getConstantFP(0.0, VT);
03816     else if (VT.isVector()) {
03817       unsigned NumElts = VT.getVectorNumElements();
03818       MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
03819       return DAG.getNode(ISD::BITCAST, dl, VT,
03820                          DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(),
03821                                                              EltVT, NumElts)));
03822     } else
03823       llvm_unreachable("Expected type!");
03824   }
03825 
03826   assert(!VT.isVector() && "Can't handle vector type here!");
03827   unsigned NumVTBits = VT.getSizeInBits();
03828   unsigned NumVTBytes = NumVTBits / 8;
03829   unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size()));
03830 
03831   APInt Val(NumVTBits, 0);
03832   if (TLI.isLittleEndian()) {
03833     for (unsigned i = 0; i != NumBytes; ++i)
03834       Val |= (uint64_t)(unsigned char)Str[i] << i*8;
03835   } else {
03836     for (unsigned i = 0; i != NumBytes; ++i)
03837       Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8;
03838   }
03839 
03840   // If the "cost" of materializing the integer immediate is less than the cost
03841   // of a load, then it is cost effective to turn the load into the immediate.
03842   Type *Ty = VT.getTypeForEVT(*DAG.getContext());
03843   if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
03844     return DAG.getConstant(Val, VT);
03845   return SDValue(nullptr, 0);
03846 }
03847 
03848 /// getMemBasePlusOffset - Returns base and offset node for the
03849 ///
03850 static SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, SDLoc dl,
03851                                       SelectionDAG &DAG) {
03852   EVT VT = Base.getValueType();
03853   return DAG.getNode(ISD::ADD, dl,
03854                      VT, Base, DAG.getConstant(Offset, VT));
03855 }
03856 
03857 /// isMemSrcFromString - Returns true if memcpy source is a string constant.
03858 ///
03859 static bool isMemSrcFromString(SDValue Src, StringRef &Str) {
03860   unsigned SrcDelta = 0;
03861   GlobalAddressSDNode *G = nullptr;
03862   if (Src.getOpcode() == ISD::GlobalAddress)
03863     G = cast<GlobalAddressSDNode>(Src);
03864   else if (Src.getOpcode() == ISD::ADD &&
03865            Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
03866            Src.getOperand(1).getOpcode() == ISD::Constant) {
03867     G = cast<GlobalAddressSDNode>(Src.getOperand(0));
03868     SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue();
03869   }
03870   if (!G)
03871     return false;
03872 
03873   return getConstantStringInfo(G->getGlobal(), Str, SrcDelta, false);
03874 }
03875 
03876 /// FindOptimalMemOpLowering - Determines the optimial series memory ops
03877 /// to replace the memset / memcpy. Return true if the number of memory ops
03878 /// is below the threshold. It returns the types of the sequence of
03879 /// memory ops to perform memset / memcpy by reference.
03880 static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
03881                                      unsigned Limit, uint64_t Size,
03882                                      unsigned DstAlign, unsigned SrcAlign,
03883                                      bool IsMemset,
03884                                      bool ZeroMemset,
03885                                      bool MemcpyStrSrc,
03886                                      bool AllowOverlap,
03887                                      SelectionDAG &DAG,
03888                                      const TargetLowering &TLI) {
03889   assert((SrcAlign == 0 || SrcAlign >= DstAlign) &&
03890          "Expecting memcpy / memset source to meet alignment requirement!");
03891   // If 'SrcAlign' is zero, that means the memory operation does not need to
03892   // load the value, i.e. memset or memcpy from constant string. Otherwise,
03893   // it's the inferred alignment of the source. 'DstAlign', on the other hand,
03894   // is the specified alignment of the memory operation. If it is zero, that
03895   // means it's possible to change the alignment of the destination.
03896   // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does
03897   // not need to be loaded.
03898   EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign,
03899                                    IsMemset, ZeroMemset, MemcpyStrSrc,
03900                                    DAG.getMachineFunction());
03901 
03902   if (VT == MVT::Other) {
03903     unsigned AS = 0;
03904     if (DstAlign >= TLI.getDataLayout()->getPointerPrefAlignment(AS) ||
03905         TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign)) {
03906       VT = TLI.getPointerTy();
03907     } else {
03908       switch (DstAlign & 7) {
03909       case 0:  VT = MVT::i64; break;
03910       case 4:  VT = MVT::i32; break;
03911       case 2:  VT = MVT::i16; break;
03912       default: VT = MVT::i8;  break;
03913       }
03914     }
03915 
03916     MVT LVT = MVT::i64;
03917     while (!TLI.isTypeLegal(LVT))
03918       LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
03919     assert(LVT.isInteger());
03920 
03921     if (VT.bitsGT(LVT))
03922       VT = LVT;
03923   }
03924 
03925   unsigned NumMemOps = 0;
03926   while (Size != 0) {
03927     unsigned VTSize = VT.getSizeInBits() / 8;
03928     while (VTSize > Size) {
03929       // For now, only use non-vector load / store's for the left-over pieces.
03930       EVT NewVT = VT;
03931       unsigned NewVTSize;
03932 
03933       bool Found = false;
03934       if (VT.isVector() || VT.isFloatingPoint()) {
03935         NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
03936         if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) &&
03937             TLI.isSafeMemOpType(NewVT.getSimpleVT()))
03938           Found = true;
03939         else if (NewVT == MVT::i64 &&
03940                  TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
03941                  TLI.isSafeMemOpType(MVT::f64)) {
03942           // i64 is usually not legal on 32-bit targets, but f64 may be.
03943           NewVT = MVT::f64;
03944           Found = true;
03945         }
03946       }
03947 
03948       if (!Found) {
03949         do {
03950           NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
03951           if (NewVT == MVT::i8)
03952             break;
03953         } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT()));
03954       }
03955       NewVTSize = NewVT.getSizeInBits() / 8;
03956 
03957       // If the new VT cannot cover all of the remaining bits, then consider
03958       // issuing a (or a pair of) unaligned and overlapping load / store.
03959       // FIXME: Only does this for 64-bit or more since we don't have proper
03960       // cost model for unaligned load / store.
03961       bool Fast;
03962       unsigned AS = 0;
03963       if (NumMemOps && AllowOverlap &&
03964           VTSize >= 8 && NewVTSize < Size &&
03965           TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign, &Fast) && Fast)
03966         VTSize = Size;
03967       else {
03968         VT = NewVT;
03969         VTSize = NewVTSize;
03970       }
03971     }
03972 
03973     if (++NumMemOps > Limit)
03974       return false;
03975 
03976     MemOps.push_back(VT);
03977     Size -= VTSize;
03978   }
03979 
03980   return true;
03981 }
03982 
03983 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
03984                                        SDValue Chain, SDValue Dst,
03985                                        SDValue Src, uint64_t Size,
03986                                        unsigned Align, bool isVol,
03987                                        bool AlwaysInline,
03988                                        MachinePointerInfo DstPtrInfo,
03989                                        MachinePointerInfo SrcPtrInfo) {
03990   // Turn a memcpy of undef to nop.
03991   if (Src.getOpcode() == ISD::UNDEF)
03992     return Chain;
03993 
03994   // Expand memcpy to a series of load and store ops if the size operand falls
03995   // below a certain threshold.
03996   // TODO: In the AlwaysInline case, if the size is big then generate a loop
03997   // rather than maybe a humongous number of loads and stores.
03998   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03999   std::vector<EVT> MemOps;
04000   bool DstAlignCanChange = false;
04001   MachineFunction &MF = DAG.getMachineFunction();
04002   MachineFrameInfo *MFI = MF.getFrameInfo();
04003   bool OptSize = MF.getFunction()->hasFnAttribute(Attribute::OptimizeForSize);
04004   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
04005   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
04006     DstAlignCanChange = true;
04007   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
04008   if (Align > SrcAlign)
04009     SrcAlign = Align;
04010   StringRef Str;
04011   bool CopyFromStr = isMemSrcFromString(Src, Str);
04012   bool isZeroStr = CopyFromStr && Str.empty();
04013   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
04014 
04015   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
04016                                 (DstAlignCanChange ? 0 : Align),
04017                                 (isZeroStr ? 0 : SrcAlign),
04018                                 false, false, CopyFromStr, true, DAG, TLI))
04019     return SDValue();
04020 
04021   if (DstAlignCanChange) {
04022     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
04023     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
04024 
04025     // Don't promote to an alignment that would require dynamic stack
04026     // realignment.
04027     const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
04028     if (!TRI->needsStackRealignment(MF))
04029        while (NewAlign > Align &&
04030              TLI.getDataLayout()->exceedsNaturalStackAlignment(NewAlign))
04031           NewAlign /= 2;
04032 
04033     if (NewAlign > Align) {
04034       // Give the stack frame object a larger alignment if needed.
04035       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
04036         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
04037       Align = NewAlign;
04038     }
04039   }
04040 
04041   SmallVector<SDValue, 8> OutChains;
04042   unsigned NumMemOps = MemOps.size();
04043   uint64_t SrcOff = 0, DstOff = 0;
04044   for (unsigned i = 0; i != NumMemOps; ++i) {
04045     EVT VT = MemOps[i];
04046     unsigned VTSize = VT.getSizeInBits() / 8;
04047     SDValue Value, Store;
04048 
04049     if (VTSize > Size) {
04050       // Issuing an unaligned load / store pair  that overlaps with the previous
04051       // pair. Adjust the offset accordingly.
04052       assert(i == NumMemOps-1 && i != 0);
04053       SrcOff -= VTSize - Size;
04054       DstOff -= VTSize - Size;
04055     }
04056 
04057     if (CopyFromStr &&
04058         (isZeroStr || (VT.isInteger() && !VT.isVector()))) {
04059       // It's unlikely a store of a vector immediate can be done in a single
04060       // instruction. It would require a load from a constantpool first.
04061       // We only handle zero vectors here.
04062       // FIXME: Handle other cases where store of vector immediate is done in
04063       // a single instruction.
04064       Value = getMemsetStringVal(VT, dl, DAG, TLI, Str.substr(SrcOff));
04065       if (Value.getNode())
04066         Store = DAG.getStore(Chain, dl, Value,
04067                              getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04068                              DstPtrInfo.getWithOffset(DstOff), isVol,
04069                              false, Align);
04070     }
04071 
04072     if (!Store.getNode()) {
04073       // The type might not be legal for the target.  This should only happen
04074       // if the type is smaller than a legal type, as on PPC, so the right
04075       // thing to do is generate a LoadExt/StoreTrunc pair.  These simplify
04076       // to Load/Store if NVT==VT.
04077       // FIXME does the case above also need this?
04078       EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
04079       assert(NVT.bitsGE(VT));
04080       Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain,
04081                              getMemBasePlusOffset(Src, SrcOff, dl, DAG),
04082                              SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false,
04083                              false, MinAlign(SrcAlign, SrcOff));
04084       Store = DAG.getTruncStore(Chain, dl, Value,
04085                                 getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04086                                 DstPtrInfo.getWithOffset(DstOff), VT, isVol,
04087                                 false, Align);
04088     }
04089     OutChains.push_back(Store);
04090     SrcOff += VTSize;
04091     DstOff += VTSize;
04092     Size -= VTSize;
04093   }
04094 
04095   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04096 }
04097 
04098 static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
04099                                         SDValue Chain, SDValue Dst,
04100                                         SDValue Src, uint64_t Size,
04101                                         unsigned Align,  bool isVol,
04102                                         bool AlwaysInline,
04103                                         MachinePointerInfo DstPtrInfo,
04104                                         MachinePointerInfo SrcPtrInfo) {
04105   // Turn a memmove of undef to nop.
04106   if (Src.getOpcode() == ISD::UNDEF)
04107     return Chain;
04108 
04109   // Expand memmove to a series of load and store ops if the size operand falls
04110   // below a certain threshold.
04111   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04112   std::vector<EVT> MemOps;
04113   bool DstAlignCanChange = false;
04114   MachineFunction &MF = DAG.getMachineFunction();
04115   MachineFrameInfo *MFI = MF.getFrameInfo();
04116   bool OptSize = MF.getFunction()->hasFnAttribute(Attribute::OptimizeForSize);
04117   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
04118   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
04119     DstAlignCanChange = true;
04120   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
04121   if (Align > SrcAlign)
04122     SrcAlign = Align;
04123   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
04124 
04125   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
04126                                 (DstAlignCanChange ? 0 : Align), SrcAlign,
04127                                 false, false, false, false, DAG, TLI))
04128     return SDValue();
04129 
04130   if (DstAlignCanChange) {
04131     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
04132     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
04133     if (NewAlign > Align) {
04134       // Give the stack frame object a larger alignment if needed.
04135       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
04136         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
04137       Align = NewAlign;
04138     }
04139   }
04140 
04141   uint64_t SrcOff = 0, DstOff = 0;
04142   SmallVector<SDValue, 8> LoadValues;
04143   SmallVector<SDValue, 8> LoadChains;
04144   SmallVector<SDValue, 8> OutChains;
04145   unsigned NumMemOps = MemOps.size();
04146   for (unsigned i = 0; i < NumMemOps; i++) {
04147     EVT VT = MemOps[i];
04148     unsigned VTSize = VT.getSizeInBits() / 8;
04149     SDValue Value;
04150 
04151     Value = DAG.getLoad(VT, dl, Chain,
04152                         getMemBasePlusOffset(Src, SrcOff, dl, DAG),
04153                         SrcPtrInfo.getWithOffset(SrcOff), isVol,
04154                         false, false, SrcAlign);
04155     LoadValues.push_back(Value);
04156     LoadChains.push_back(Value.getValue(1));
04157     SrcOff += VTSize;
04158   }
04159   Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
04160   OutChains.clear();
04161   for (unsigned i = 0; i < NumMemOps; i++) {
04162     EVT VT = MemOps[i];
04163     unsigned VTSize = VT.getSizeInBits() / 8;
04164     SDValue Store;
04165 
04166     Store = DAG.getStore(Chain, dl, LoadValues[i],
04167                          getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04168                          DstPtrInfo.getWithOffset(DstOff), isVol, false, Align);
04169     OutChains.push_back(Store);
04170     DstOff += VTSize;
04171   }
04172 
04173   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04174 }
04175 
04176 /// \brief Lower the call to 'memset' intrinsic function into a series of store
04177 /// operations.
04178 ///
04179 /// \param DAG Selection DAG where lowered code is placed.
04180 /// \param dl Link to corresponding IR location.
04181 /// \param Chain Control flow dependency.
04182 /// \param Dst Pointer to destination memory location.
04183 /// \param Src Value of byte to write into the memory.
04184 /// \param Size Number of bytes to write.
04185 /// \param Align Alignment of the destination in bytes.
04186 /// \param isVol True if destination is volatile.
04187 /// \param DstPtrInfo IR information on the memory pointer.
04188 /// \returns New head in the control flow, if lowering was successful, empty
04189 /// SDValue otherwise.
04190 ///
04191 /// The function tries to replace 'llvm.memset' intrinsic with several store
04192 /// operations and value calculation code. This is usually profitable for small
04193 /// memory size.
04194 static SDValue getMemsetStores(SelectionDAG &DAG, SDLoc dl,
04195                                SDValue Chain, SDValue Dst,
04196                                SDValue Src, uint64_t Size,
04197                                unsigned Align, bool isVol,
04198                                MachinePointerInfo DstPtrInfo) {
04199   // Turn a memset of undef to nop.
04200   if (Src.getOpcode() == ISD::UNDEF)
04201     return Chain;
04202 
04203   // Expand memset to a series of load/store ops if the size operand
04204   // falls below a certain threshold.
04205   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04206   std::vector<EVT> MemOps;
04207   bool DstAlignCanChange = false;
04208   MachineFunction &MF = DAG.getMachineFunction();
04209   MachineFrameInfo *MFI = MF.getFrameInfo();
04210   bool OptSize = MF.getFunction()->hasFnAttribute(Attribute::OptimizeForSize);
04211   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
04212   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
04213     DstAlignCanChange = true;
04214   bool IsZeroVal =
04215     isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
04216   if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize),
04217                                 Size, (DstAlignCanChange ? 0 : Align), 0,
04218                                 true, IsZeroVal, false, true, DAG, TLI))
04219     return SDValue();
04220 
04221   if (DstAlignCanChange) {
04222     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
04223     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
04224     if (NewAlign > Align) {
04225       // Give the stack frame object a larger alignment if needed.
04226       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
04227         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
04228       Align = NewAlign;
04229     }
04230   }
04231 
04232   SmallVector<SDValue, 8> OutChains;
04233   uint64_t DstOff = 0;
04234   unsigned NumMemOps = MemOps.size();
04235 
04236   // Find the largest store and generate the bit pattern for it.
04237   EVT LargestVT = MemOps[0];
04238   for (unsigned i = 1; i < NumMemOps; i++)
04239     if (MemOps[i].bitsGT(LargestVT))
04240       LargestVT = MemOps[i];
04241   SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
04242 
04243   for (unsigned i = 0; i < NumMemOps; i++) {
04244     EVT VT = MemOps[i];
04245     unsigned VTSize = VT.getSizeInBits() / 8;
04246     if (VTSize > Size) {
04247       // Issuing an unaligned load / store pair  that overlaps with the previous
04248       // pair. Adjust the offset accordingly.
04249       assert(i == NumMemOps-1 && i != 0);
04250       DstOff -= VTSize - Size;
04251     }
04252 
04253     // If this store is smaller than the largest store see whether we can get
04254     // the smaller value for free with a truncate.
04255     SDValue Value = MemSetValue;
04256     if (VT.bitsLT(LargestVT)) {
04257       if (!LargestVT.isVector() && !VT.isVector() &&
04258           TLI.isTruncateFree(LargestVT, VT))
04259         Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
04260       else
04261         Value = getMemsetValue(Src, VT, DAG, dl);
04262     }
04263     assert(Value.getValueType() == VT && "Value with wrong type.");
04264     SDValue Store = DAG.getStore(Chain, dl, Value,
04265                                  getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04266                                  DstPtrInfo.getWithOffset(DstOff),
04267                                  isVol, false, Align);
04268     OutChains.push_back(Store);
04269     DstOff += VT.getSizeInBits() / 8;
04270     Size -= VTSize;
04271   }
04272 
04273   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04274 }
04275 
04276 SDValue SelectionDAG::getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst,
04277                                 SDValue Src, SDValue Size,
04278                                 unsigned Align, bool isVol, bool AlwaysInline,
04279                                 MachinePointerInfo DstPtrInfo,
04280                                 MachinePointerInfo SrcPtrInfo) {
04281   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04282 
04283   // Check to see if we should lower the memcpy to loads and stores first.
04284   // For cases within the target-specified limits, this is the best choice.
04285   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04286   if (ConstantSize) {
04287     // Memcpy with size zero? Just return the original chain.
04288     if (ConstantSize->isNullValue())
04289       return Chain;
04290 
04291     SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
04292                                              ConstantSize->getZExtValue(),Align,
04293                                 isVol, false, DstPtrInfo, SrcPtrInfo);
04294     if (Result.getNode())
04295       return Result;
04296   }
04297 
04298   // Then check to see if we should lower the memcpy with target-specific
04299   // code. If the target chooses to do this, this is the next best.
04300   if (TSI) {
04301     SDValue Result = TSI->EmitTargetCodeForMemcpy(
04302         *this, dl, Chain, Dst, Src, Size, Align, isVol, AlwaysInline,
04303         DstPtrInfo, SrcPtrInfo);
04304     if (Result.getNode())
04305       return Result;
04306   }
04307 
04308   // If we really need inline code and the target declined to provide it,
04309   // use a (potentially long) sequence of loads and stores.
04310   if (AlwaysInline) {
04311     assert(ConstantSize && "AlwaysInline requires a constant size!");
04312     return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
04313                                    ConstantSize->getZExtValue(), Align, isVol,
04314                                    true, DstPtrInfo, SrcPtrInfo);
04315   }
04316 
04317   // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
04318   // memcpy is not guaranteed to be safe. libc memcpys aren't required to
04319   // respect volatile, so they may do things like read or write memory
04320   // beyond the given memory regions. But fixing this isn't easy, and most
04321   // people don't care.
04322 
04323   // Emit a library call.
04324   TargetLowering::ArgListTy Args;
04325   TargetLowering::ArgListEntry Entry;
04326   Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
04327   Entry.Node = Dst; Args.push_back(Entry);
04328   Entry.Node = Src; Args.push_back(Entry);
04329   Entry.Node = Size; Args.push_back(Entry);
04330   // FIXME: pass in SDLoc
04331   TargetLowering::CallLoweringInfo CLI(*this);
04332   CLI.setDebugLoc(dl).setChain(Chain)
04333     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
04334                Type::getVoidTy(*getContext()),
04335                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
04336                                  TLI->getPointerTy()), std::move(Args), 0)
04337     .setDiscardResult();
04338   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04339 
04340   return CallResult.second;
04341 }
04342 
04343 SDValue SelectionDAG::getMemmove(SDValue Chain, SDLoc dl, SDValue Dst,
04344                                  SDValue Src, SDValue Size,
04345                                  unsigned Align, bool isVol,
04346                                  MachinePointerInfo DstPtrInfo,
04347                                  MachinePointerInfo SrcPtrInfo) {
04348   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04349 
04350   // Check to see if we should lower the memmove to loads and stores first.
04351   // For cases within the target-specified limits, this is the best choice.
04352   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04353   if (ConstantSize) {
04354     // Memmove with size zero? Just return the original chain.
04355     if (ConstantSize->isNullValue())
04356       return Chain;
04357 
04358     SDValue Result =
04359       getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src,
04360                                ConstantSize->getZExtValue(), Align, isVol,
04361                                false, DstPtrInfo, SrcPtrInfo);
04362     if (Result.getNode())
04363       return Result;
04364   }
04365 
04366   // Then check to see if we should lower the memmove with target-specific
04367   // code. If the target chooses to do this, this is the next best.
04368   if (TSI) {
04369     SDValue Result = TSI->EmitTargetCodeForMemmove(
04370         *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo, SrcPtrInfo);
04371     if (Result.getNode())
04372       return Result;
04373   }
04374 
04375   // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
04376   // not be safe.  See memcpy above for more details.
04377 
04378   // Emit a library call.
04379   TargetLowering::ArgListTy Args;
04380   TargetLowering::ArgListEntry Entry;
04381   Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
04382   Entry.Node = Dst; Args.push_back(Entry);
04383   Entry.Node = Src; Args.push_back(Entry);
04384   Entry.Node = Size; Args.push_back(Entry);
04385   // FIXME:  pass in SDLoc
04386   TargetLowering::CallLoweringInfo CLI(*this);
04387   CLI.setDebugLoc(dl).setChain(Chain)
04388     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
04389                Type::getVoidTy(*getContext()),
04390                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
04391                                  TLI->getPointerTy()), std::move(Args), 0)
04392     .setDiscardResult();
04393   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04394 
04395   return CallResult.second;
04396 }
04397 
04398 SDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst,
04399                                 SDValue Src, SDValue Size,
04400                                 unsigned Align, bool isVol,
04401                                 MachinePointerInfo DstPtrInfo) {
04402   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04403 
04404   // Check to see if we should lower the memset to stores first.
04405   // For cases within the target-specified limits, this is the best choice.
04406   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04407   if (ConstantSize) {
04408     // Memset with size zero? Just return the original chain.
04409     if (ConstantSize->isNullValue())
04410       return Chain;
04411 
04412     SDValue Result =
04413       getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
04414                       Align, isVol, DstPtrInfo);
04415 
04416     if (Result.getNode())
04417       return Result;
04418   }
04419 
04420   // Then check to see if we should lower the memset with target-specific
04421   // code. If the target chooses to do this, this is the next best.
04422   if (TSI) {
04423     SDValue Result = TSI->EmitTargetCodeForMemset(
04424         *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo);
04425     if (Result.getNode())
04426       return Result;
04427   }
04428 
04429   // Emit a library call.
04430   Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(*getContext());
04431   TargetLowering::ArgListTy Args;
04432   TargetLowering::ArgListEntry Entry;
04433   Entry.Node = Dst; Entry.Ty = IntPtrTy;
04434   Args.push_back(Entry);
04435   Entry.Node = Src;
04436   Entry.Ty = Src.getValueType().getTypeForEVT(*getContext());
04437   Args.push_back(Entry);
04438   Entry.Node = Size;
04439   Entry.Ty = IntPtrTy;
04440   Args.push_back(Entry);
04441 
04442   // FIXME: pass in SDLoc
04443   TargetLowering::CallLoweringInfo CLI(*this);
04444   CLI.setDebugLoc(dl).setChain(Chain)
04445     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
04446                Type::getVoidTy(*getContext()),
04447                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
04448                                  TLI->getPointerTy()), std::move(Args), 0)
04449     .setDiscardResult();
04450 
04451   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04452   return CallResult.second;
04453 }
04454 
04455 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04456                                 SDVTList VTList, ArrayRef<SDValue> Ops,
04457                                 MachineMemOperand *MMO,
04458                                 AtomicOrdering SuccessOrdering,
04459                                 AtomicOrdering FailureOrdering,
04460                                 SynchronizationScope SynchScope) {
04461   FoldingSetNodeID ID;
04462   ID.AddInteger(MemVT.getRawBits());
04463   AddNodeIDNode(ID, Opcode, VTList, Ops);
04464   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04465   void* IP = nullptr;
04466   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04467     cast<AtomicSDNode>(E)->refineAlignment(MMO);
04468     return SDValue(E, 0);
04469   }
04470 
04471   // Allocate the operands array for the node out of the BumpPtrAllocator, since
04472   // SDNode doesn't have access to it.  This memory will be "leaked" when
04473   // the node is deallocated, but recovered when the allocator is released.
04474   // If the number of operands is less than 5 we use AtomicSDNode's internal
04475   // storage.
04476   unsigned NumOps = Ops.size();
04477   SDUse *DynOps = NumOps > 4 ? OperandAllocator.Allocate<SDUse>(NumOps)
04478                              : nullptr;
04479 
04480   SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(),
04481                                                dl.getDebugLoc(), VTList, MemVT,
04482                                                Ops.data(), DynOps, NumOps, MMO,
04483                                                SuccessOrdering, FailureOrdering,
04484                                                SynchScope);
04485   CSEMap.InsertNode(N, IP);
04486   InsertNode(N);
04487   return SDValue(N, 0);
04488 }
04489 
04490 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04491                                 SDVTList VTList, ArrayRef<SDValue> Ops,
04492                                 MachineMemOperand *MMO,
04493                                 AtomicOrdering Ordering,
04494                                 SynchronizationScope SynchScope) {
04495   return getAtomic(Opcode, dl, MemVT, VTList, Ops, MMO, Ordering,
04496                    Ordering, SynchScope);
04497 }
04498 
04499 SDValue SelectionDAG::getAtomicCmpSwap(
04500     unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs, SDValue Chain,
04501     SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo,
04502     unsigned Alignment, AtomicOrdering SuccessOrdering,
04503     AtomicOrdering FailureOrdering, SynchronizationScope SynchScope) {
04504   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
04505          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
04506   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
04507 
04508   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04509     Alignment = getEVTAlignment(MemVT);
04510 
04511   MachineFunction &MF = getMachineFunction();
04512 
04513   // FIXME: Volatile isn't really correct; we should keep track of atomic
04514   // orderings in the memoperand.
04515   unsigned Flags = MachineMemOperand::MOVolatile;
04516   Flags |= MachineMemOperand::MOLoad;
04517   Flags |= MachineMemOperand::MOStore;
04518 
04519   MachineMemOperand *MMO =
04520     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
04521 
04522   return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO,
04523                           SuccessOrdering, FailureOrdering, SynchScope);
04524 }
04525 
04526 SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, SDLoc dl, EVT MemVT,
04527                                        SDVTList VTs, SDValue Chain, SDValue Ptr,
04528                                        SDValue Cmp, SDValue Swp,
04529                                        MachineMemOperand *MMO,
04530                                        AtomicOrdering SuccessOrdering,
04531                                        AtomicOrdering FailureOrdering,
04532                                        SynchronizationScope SynchScope) {
04533   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
04534          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
04535   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
04536 
04537   SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
04538   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO,
04539                    SuccessOrdering, FailureOrdering, SynchScope);
04540 }
04541 
04542 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04543                                 SDValue Chain,
04544                                 SDValue Ptr, SDValue Val,
04545                                 const Value* PtrVal,
04546                                 unsigned Alignment,
04547                                 AtomicOrdering Ordering,
04548                                 SynchronizationScope SynchScope) {
04549   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04550     Alignment = getEVTAlignment(MemVT);
04551 
04552   MachineFunction &MF = getMachineFunction();
04553   // An atomic store does not load. An atomic load does not store.
04554   // (An atomicrmw obviously both loads and stores.)
04555   // For now, atomics are considered to be volatile always, and they are
04556   // chained as such.
04557   // FIXME: Volatile isn't really correct; we should keep track of atomic
04558   // orderings in the memoperand.
04559   unsigned Flags = MachineMemOperand::MOVolatile;
04560   if (Opcode != ISD::ATOMIC_STORE)
04561     Flags |= MachineMemOperand::MOLoad;
04562   if (Opcode != ISD::ATOMIC_LOAD)
04563     Flags |= MachineMemOperand::MOStore;
04564 
04565   MachineMemOperand *MMO =
04566     MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
04567                             MemVT.getStoreSize(), Alignment);
04568 
04569   return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO,
04570                    Ordering, SynchScope);
04571 }
04572 
04573 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04574                                 SDValue Chain,
04575                                 SDValue Ptr, SDValue Val,
04576                                 MachineMemOperand *MMO,
04577                                 AtomicOrdering Ordering,
04578                                 SynchronizationScope SynchScope) {
04579   assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
04580           Opcode == ISD::ATOMIC_LOAD_SUB ||
04581           Opcode == ISD::ATOMIC_LOAD_AND ||
04582           Opcode == ISD::ATOMIC_LOAD_OR ||
04583           Opcode == ISD::ATOMIC_LOAD_XOR ||
04584           Opcode == ISD::ATOMIC_LOAD_NAND ||
04585           Opcode == ISD::ATOMIC_LOAD_MIN ||
04586           Opcode == ISD::ATOMIC_LOAD_MAX ||
04587           Opcode == ISD::ATOMIC_LOAD_UMIN ||
04588           Opcode == ISD::ATOMIC_LOAD_UMAX ||
04589           Opcode == ISD::ATOMIC_SWAP ||
04590           Opcode == ISD::ATOMIC_STORE) &&
04591          "Invalid Atomic Op");
04592 
04593   EVT VT = Val.getValueType();
04594 
04595   SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
04596                                                getVTList(VT, MVT::Other);
04597   SDValue Ops[] = {Chain, Ptr, Val};
04598   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
04599 }
04600 
04601 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04602                                 EVT VT, SDValue Chain,
04603                                 SDValue Ptr,
04604                                 MachineMemOperand *MMO,
04605                                 AtomicOrdering Ordering,
04606                                 SynchronizationScope SynchScope) {
04607   assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
04608 
04609   SDVTList VTs = getVTList(VT, MVT::Other);
04610   SDValue Ops[] = {Chain, Ptr};
04611   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
04612 }
04613 
04614 /// getMergeValues - Create a MERGE_VALUES node from the given operands.
04615 SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, SDLoc dl) {
04616   if (Ops.size() == 1)
04617     return Ops[0];
04618 
04619   SmallVector<EVT, 4> VTs;
04620   VTs.reserve(Ops.size());
04621   for (unsigned i = 0; i < Ops.size(); ++i)
04622     VTs.push_back(Ops[i].getValueType());
04623   return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
04624 }
04625 
04626 SDValue
04627 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
04628                                   ArrayRef<SDValue> Ops,
04629                                   EVT MemVT, MachinePointerInfo PtrInfo,
04630                                   unsigned Align, bool Vol,
04631                                   bool ReadMem, bool WriteMem, unsigned Size) {
04632   if (Align == 0)  // Ensure that codegen never sees alignment 0
04633     Align = getEVTAlignment(MemVT);
04634 
04635   MachineFunction &MF = getMachineFunction();
04636   unsigned Flags = 0;
04637   if (WriteMem)
04638     Flags |= MachineMemOperand::MOStore;
04639   if (ReadMem)
04640     Flags |= MachineMemOperand::MOLoad;
04641   if (Vol)
04642     Flags |= MachineMemOperand::MOVolatile;
04643   if (!Size)
04644     Size = MemVT.getStoreSize();
04645   MachineMemOperand *MMO =
04646     MF.getMachineMemOperand(PtrInfo, Flags, Size, Align);
04647 
04648   return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
04649 }
04650 
04651 SDValue
04652 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
04653                                   ArrayRef<SDValue> Ops, EVT MemVT,
04654                                   MachineMemOperand *MMO) {
04655   assert((Opcode == ISD::INTRINSIC_VOID ||
04656           Opcode == ISD::INTRINSIC_W_CHAIN ||
04657           Opcode == ISD::PREFETCH ||
04658           Opcode == ISD::LIFETIME_START ||
04659           Opcode == ISD::LIFETIME_END ||
04660           (Opcode <= INT_MAX &&
04661            (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
04662          "Opcode is not a memory-accessing opcode!");
04663 
04664   // Memoize the node unless it returns a flag.
04665   MemIntrinsicSDNode *N;
04666   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
04667     FoldingSetNodeID ID;
04668     AddNodeIDNode(ID, Opcode, VTList, Ops);
04669     ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04670     void *IP = nullptr;
04671     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04672       cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
04673       return SDValue(E, 0);
04674     }
04675 
04676     N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
04677                                                dl.getDebugLoc(), VTList, Ops,
04678                                                MemVT, MMO);
04679     CSEMap.InsertNode(N, IP);
04680   } else {
04681     N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
04682                                                dl.getDebugLoc(), VTList, Ops,
04683                                                MemVT, MMO);
04684   }
04685   InsertNode(N);
04686   return SDValue(N, 0);
04687 }
04688 
04689 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
04690 /// MachinePointerInfo record from it.  This is particularly useful because the
04691 /// code generator has many cases where it doesn't bother passing in a
04692 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
04693 static MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) {
04694   // If this is FI+Offset, we can model it.
04695   if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
04696     return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset);
04697 
04698   // If this is (FI+Offset1)+Offset2, we can model it.
04699   if (Ptr.getOpcode() != ISD::ADD ||
04700       !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
04701       !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
04702     return MachinePointerInfo();
04703 
04704   int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
04705   return MachinePointerInfo::getFixedStack(FI, Offset+
04706                        cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
04707 }
04708 
04709 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
04710 /// MachinePointerInfo record from it.  This is particularly useful because the
04711 /// code generator has many cases where it doesn't bother passing in a
04712 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
04713 static MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) {
04714   // If the 'Offset' value isn't a constant, we can't handle this.
04715   if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
04716     return InferPointerInfo(Ptr, OffsetNode->getSExtValue());
04717   if (OffsetOp.getOpcode() == ISD::UNDEF)
04718     return InferPointerInfo(Ptr);
04719   return MachinePointerInfo();
04720 }
04721 
04722 
04723 SDValue
04724 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
04725                       EVT VT, SDLoc dl, SDValue Chain,
04726                       SDValue Ptr, SDValue Offset,
04727                       MachinePointerInfo PtrInfo, EVT MemVT,
04728                       bool isVolatile, bool isNonTemporal, bool isInvariant,
04729                       unsigned Alignment, const AAMDNodes &AAInfo,
04730                       const MDNode *Ranges) {
04731   assert(Chain.getValueType() == MVT::Other &&
04732         "Invalid chain type");
04733   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04734     Alignment = getEVTAlignment(VT);
04735 
04736   unsigned Flags = MachineMemOperand::MOLoad;
04737   if (isVolatile)
04738     Flags |= MachineMemOperand::MOVolatile;
04739   if (isNonTemporal)
04740     Flags |= MachineMemOperand::MONonTemporal;
04741   if (isInvariant)
04742     Flags |= MachineMemOperand::MOInvariant;
04743 
04744   // If we don't have a PtrInfo, infer the trivial frame index case to simplify
04745   // clients.
04746   if (PtrInfo.V.isNull())
04747     PtrInfo = InferPointerInfo(Ptr, Offset);
04748 
04749   MachineFunction &MF = getMachineFunction();
04750   MachineMemOperand *MMO =
04751     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment,
04752                             AAInfo, Ranges);
04753   return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
04754 }
04755 
04756 SDValue
04757 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
04758                       EVT VT, SDLoc dl, SDValue Chain,
04759                       SDValue Ptr, SDValue Offset, EVT MemVT,
04760                       MachineMemOperand *MMO) {
04761   if (VT == MemVT) {
04762     ExtType = ISD::NON_EXTLOAD;
04763   } else if (ExtType == ISD::NON_EXTLOAD) {
04764     assert(VT == MemVT && "Non-extending load from different memory type!");
04765   } else {
04766     // Extending load.
04767     assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
04768            "Should only be an extending load, not truncating!");
04769     assert(VT.isInteger() == MemVT.isInteger() &&
04770            "Cannot convert from FP to Int or Int -> FP!");
04771     assert(VT.isVector() == MemVT.isVector() &&
04772            "Cannot use an ext load to convert to or from a vector!");
04773     assert((!VT.isVector() ||
04774             VT.getVectorNumElements() == MemVT.getVectorNumElements()) &&
04775            "Cannot use an ext load to change the number of vector elements!");
04776   }
04777 
04778   bool Indexed = AM != ISD::UNINDEXED;
04779   assert((Indexed || Offset.getOpcode() == ISD::UNDEF) &&
04780          "Unindexed load with an offset!");
04781 
04782   SDVTList VTs = Indexed ?
04783     getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
04784   SDValue Ops[] = { Chain, Ptr, Offset };
04785   FoldingSetNodeID ID;
04786   AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
04787   ID.AddInteger(MemVT.getRawBits());
04788   ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(),
04789                                      MMO->isNonTemporal(),
04790                                      MMO->isInvariant()));
04791   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04792   void *IP = nullptr;
04793   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04794     cast<LoadSDNode>(E)->refineAlignment(MMO);
04795     return SDValue(E, 0);
04796   }
04797   SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl.getIROrder(),
04798                                              dl.getDebugLoc(), VTs, AM, ExtType,
04799                                              MemVT, MMO);
04800   CSEMap.InsertNode(N, IP);
04801   InsertNode(N);
04802   return SDValue(N, 0);
04803 }
04804 
04805 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
04806                               SDValue Chain, SDValue Ptr,
04807                               MachinePointerInfo PtrInfo,
04808                               bool isVolatile, bool isNonTemporal,
04809                               bool isInvariant, unsigned Alignment,
04810                               const AAMDNodes &AAInfo,
04811                               const MDNode *Ranges) {
04812   SDValue Undef = getUNDEF(Ptr.getValueType());
04813   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
04814                  PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment,
04815                  AAInfo, Ranges);
04816 }
04817 
04818 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
04819                               SDValue Chain, SDValue Ptr,
04820                               MachineMemOperand *MMO) {
04821   SDValue Undef = getUNDEF(Ptr.getValueType());
04822   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
04823                  VT, MMO);
04824 }
04825 
04826 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
04827                                  SDValue Chain, SDValue Ptr,
04828                                  MachinePointerInfo PtrInfo, EVT MemVT,
04829                                  bool isVolatile, bool isNonTemporal,
04830                                  bool isInvariant, unsigned Alignment,
04831                                  const AAMDNodes &AAInfo) {
04832   SDValue Undef = getUNDEF(Ptr.getValueType());
04833   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
04834                  PtrInfo, MemVT, isVolatile, isNonTemporal, isInvariant,
04835                  Alignment, AAInfo);
04836 }
04837 
04838 
04839 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
04840                                  SDValue Chain, SDValue Ptr, EVT MemVT,
04841                                  MachineMemOperand *MMO) {
04842   SDValue Undef = getUNDEF(Ptr.getValueType());
04843   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
04844                  MemVT, MMO);
04845 }
04846 
04847 SDValue
04848 SelectionDAG::getIndexedLoad(SDValue OrigLoad, SDLoc dl, SDValue Base,
04849                              SDValue Offset, ISD::MemIndexedMode AM) {
04850   LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
04851   assert(LD->getOffset().getOpcode() == ISD::UNDEF &&
04852          "Load is already a indexed load!");
04853   return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
04854                  LD->getChain(), Base, Offset, LD->getPointerInfo(),
04855                  LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(),
04856                  false, LD->getAlignment());
04857 }
04858 
04859 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
04860                                SDValue Ptr, MachinePointerInfo PtrInfo,
04861                                bool isVolatile, bool isNonTemporal,
04862                                unsigned Alignment, const AAMDNodes &AAInfo) {
04863   assert(Chain.getValueType() == MVT::Other &&
04864         "Invalid chain type");
04865   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04866     Alignment = getEVTAlignment(Val.getValueType());
04867 
04868   unsigned Flags = MachineMemOperand::MOStore;
04869   if (isVolatile)
04870     Flags |= MachineMemOperand::MOVolatile;
04871   if (isNonTemporal)
04872     Flags |= MachineMemOperand::MONonTemporal;
04873 
04874   if (PtrInfo.V.isNull())
04875     PtrInfo = InferPointerInfo(Ptr);
04876 
04877   MachineFunction &MF = getMachineFunction();
04878   MachineMemOperand *MMO =
04879     MF.getMachineMemOperand(PtrInfo, Flags,
04880                             Val.getValueType().getStoreSize(), Alignment,
04881                             AAInfo);
04882 
04883   return getStore(Chain, dl, Val, Ptr, MMO);
04884 }
04885 
04886 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
04887                                SDValue Ptr, MachineMemOperand *MMO) {
04888   assert(Chain.getValueType() == MVT::Other &&
04889         "Invalid chain type");
04890   EVT VT = Val.getValueType();
04891   SDVTList VTs = getVTList(MVT::Other);
04892   SDValue Undef = getUNDEF(Ptr.getValueType());
04893   SDValue Ops[] = { Chain, Val, Ptr, Undef };
04894   FoldingSetNodeID ID;
04895   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04896   ID.AddInteger(VT.getRawBits());
04897   ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
04898                                      MMO->isNonTemporal(), MMO->isInvariant()));
04899   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04900   void *IP = nullptr;
04901   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04902     cast<StoreSDNode>(E)->refineAlignment(MMO);
04903     return SDValue(E, 0);
04904   }
04905   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04906                                               dl.getDebugLoc(), VTs,
04907                                               ISD::UNINDEXED, false, VT, MMO);
04908   CSEMap.InsertNode(N, IP);
04909   InsertNode(N);
04910   return SDValue(N, 0);
04911 }
04912 
04913 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
04914                                     SDValue Ptr, MachinePointerInfo PtrInfo,
04915                                     EVT SVT,bool isVolatile, bool isNonTemporal,
04916                                     unsigned Alignment,
04917                                     const AAMDNodes &AAInfo) {
04918   assert(Chain.getValueType() == MVT::Other &&
04919         "Invalid chain type");
04920   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04921     Alignment = getEVTAlignment(SVT);
04922 
04923   unsigned Flags = MachineMemOperand::MOStore;
04924   if (isVolatile)
04925     Flags |= MachineMemOperand::MOVolatile;
04926   if (isNonTemporal)
04927     Flags |= MachineMemOperand::MONonTemporal;
04928 
04929   if (PtrInfo.V.isNull())
04930     PtrInfo = InferPointerInfo(Ptr);
04931 
04932   MachineFunction &MF = getMachineFunction();
04933   MachineMemOperand *MMO =
04934     MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment,
04935                             AAInfo);
04936 
04937   return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
04938 }
04939 
04940 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
04941                                     SDValue Ptr, EVT SVT,
04942                                     MachineMemOperand *MMO) {
04943   EVT VT = Val.getValueType();
04944 
04945   assert(Chain.getValueType() == MVT::Other &&
04946         "Invalid chain type");
04947   if (VT == SVT)
04948     return getStore(Chain, dl, Val, Ptr, MMO);
04949 
04950   assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
04951          "Should only be a truncating store, not extending!");
04952   assert(VT.isInteger() == SVT.isInteger() &&
04953          "Can't do FP-INT conversion!");
04954   assert(VT.isVector() == SVT.isVector() &&
04955          "Cannot use trunc store to convert to or from a vector!");
04956   assert((!VT.isVector() ||
04957           VT.getVectorNumElements() == SVT.getVectorNumElements()) &&
04958          "Cannot use trunc store to change the number of vector elements!");
04959 
04960   SDVTList VTs = getVTList(MVT::Other);
04961   SDValue Undef = getUNDEF(Ptr.getValueType());
04962   SDValue Ops[] = { Chain, Val, Ptr, Undef };
04963   FoldingSetNodeID ID;
04964   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04965   ID.AddInteger(SVT.getRawBits());
04966   ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(),
04967                                      MMO->isNonTemporal(), MMO->isInvariant()));
04968   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04969   void *IP = nullptr;
04970   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04971     cast<StoreSDNode>(E)->refineAlignment(MMO);
04972     return SDValue(E, 0);
04973   }
04974   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04975                                               dl.getDebugLoc(), VTs,
04976                                               ISD::UNINDEXED, true, SVT, MMO);
04977   CSEMap.InsertNode(N, IP);
04978   InsertNode(N);
04979   return SDValue(N, 0);
04980 }
04981 
04982 SDValue
04983 SelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base,
04984                               SDValue Offset, ISD::MemIndexedMode AM) {
04985   StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
04986   assert(ST->getOffset().getOpcode() == ISD::UNDEF &&
04987          "Store is already a indexed store!");
04988   SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
04989   SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
04990   FoldingSetNodeID ID;
04991   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04992   ID.AddInteger(ST->getMemoryVT().getRawBits());
04993   ID.AddInteger(ST->getRawSubclassData());
04994   ID.AddInteger(ST->getPointerInfo().getAddrSpace());
04995   void *IP = nullptr;
04996   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
04997     return SDValue(E, 0);
04998 
04999   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
05000                                               dl.getDebugLoc(), VTs, AM,
05001                                               ST->isTruncatingStore(),
05002                                               ST->getMemoryVT(),
05003                                               ST->getMemOperand());
05004   CSEMap.InsertNode(N, IP);
05005   InsertNode(N);
05006   return SDValue(N, 0);
05007 }
05008 
05009 SDValue
05010 SelectionDAG::getMaskedLoad(EVT VT, SDLoc dl, SDValue Chain,
05011                             SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT,
05012                             MachineMemOperand *MMO, ISD::LoadExtType ExtTy) {
05013 
05014   SDVTList VTs = getVTList(VT, MVT::Other);
05015   SDValue Ops[] = { Chain, Ptr, Mask, Src0 };
05016   FoldingSetNodeID ID;
05017   AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops);
05018   ID.AddInteger(VT.getRawBits());
05019   ID.AddInteger(encodeMemSDNodeFlags(ExtTy, ISD::UNINDEXED,
05020                                      MMO->isVolatile(),
05021                                      MMO->isNonTemporal(),
05022                                      MMO->isInvariant()));
05023   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
05024   void *IP = nullptr;
05025   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
05026     cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
05027     return SDValue(E, 0);
05028   }
05029   SDNode *N = new (NodeAllocator) MaskedLoadSDNode(dl.getIROrder(),
05030                                              dl.getDebugLoc(), Ops, 4, VTs,
05031                                              ExtTy, MemVT, MMO);
05032   CSEMap.InsertNode(N, IP);
05033   InsertNode(N);
05034   return SDValue(N, 0);
05035 }
05036 
05037 SDValue SelectionDAG::getMaskedStore(SDValue Chain, SDLoc dl, SDValue Val,
05038                                      SDValue Ptr, SDValue Mask, EVT MemVT,
05039                                      MachineMemOperand *MMO, bool isTrunc) {
05040   assert(Chain.getValueType() == MVT::Other &&
05041         "Invalid chain type");
05042   EVT VT = Val.getValueType();
05043   SDVTList VTs = getVTList(MVT::Other);
05044   SDValue Ops[] = { Chain, Ptr, Mask, Val };
05045   FoldingSetNodeID ID;
05046   AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops);
05047   ID.AddInteger(VT.getRawBits());
05048   ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
05049                                      MMO->isNonTemporal(), MMO->isInvariant()));
05050   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
05051   void *IP = nullptr;
05052   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
05053     cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
05054     return SDValue(E, 0);
05055   }
05056   SDNode *N = new (NodeAllocator) MaskedStoreSDNode(dl.getIROrder(),
05057                                                     dl.getDebugLoc(), Ops, 4,
05058                                                     VTs, isTrunc, MemVT, MMO);
05059   CSEMap.InsertNode(N, IP);
05060   InsertNode(N);
05061   return SDValue(N, 0);
05062 }
05063 
05064 SDValue SelectionDAG::getVAArg(EVT VT, SDLoc dl,
05065                                SDValue Chain, SDValue Ptr,
05066                                SDValue SV,
05067                                unsigned Align) {
05068   SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) };
05069   return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
05070 }
05071 
05072 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
05073                               ArrayRef<SDUse> Ops) {
05074   switch (Ops.size()) {
05075   case 0: return getNode(Opcode, DL, VT);
05076   case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
05077   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
05078   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
05079   default: break;
05080   }
05081 
05082   // Copy from an SDUse array into an SDValue array for use with
05083   // the regular getNode logic.
05084   SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
05085   return getNode(Opcode, DL, VT, NewOps);
05086 }
05087 
05088 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
05089                               ArrayRef<SDValue> Ops) {
05090   unsigned NumOps = Ops.size();
05091   switch (NumOps) {
05092   case 0: return getNode(Opcode, DL, VT);
05093   case 1: return getNode(Opcode, DL, VT, Ops[0]);
05094   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
05095   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
05096   default: break;
05097   }
05098 
05099   switch (Opcode) {
05100   default: break;
05101   case ISD::SELECT_CC: {
05102     assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
05103     assert(Ops[0].getValueType() == Ops[1].getValueType() &&
05104            "LHS and RHS of condition must have same type!");
05105     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
05106            "True and False arms of SelectCC must have same type!");
05107     assert(Ops[2].getValueType() == VT &&
05108            "select_cc node must be of same type as true and false value!");
05109     break;
05110   }
05111   case ISD::BR_CC: {
05112     assert(NumOps == 5 && "BR_CC takes 5 operands!");
05113     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
05114            "LHS/RHS of comparison should match types!");
05115     break;
05116   }
05117   }
05118 
05119   // Memoize nodes.
05120   SDNode *N;
05121   SDVTList VTs = getVTList(VT);
05122 
05123   if (VT != MVT::Glue) {
05124     FoldingSetNodeID ID;
05125     AddNodeIDNode(ID, Opcode, VTs, Ops);
05126     void *IP = nullptr;
05127 
05128     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05129       return SDValue(E, 0);
05130 
05131     N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05132                                    VTs, Ops);
05133     CSEMap.InsertNode(N, IP);
05134   } else {
05135     N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05136                                    VTs, Ops);
05137   }
05138 
05139   InsertNode(N);
05140   return SDValue(N, 0);
05141 }
05142 
05143 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
05144                               ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
05145   return getNode(Opcode, DL, getVTList(ResultTys), Ops);
05146 }
05147 
05148 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05149                               ArrayRef<SDValue> Ops) {
05150   if (VTList.NumVTs == 1)
05151     return getNode(Opcode, DL, VTList.VTs[0], Ops);
05152 
05153 #if 0
05154   switch (Opcode) {
05155   // FIXME: figure out how to safely handle things like
05156   // int foo(int x) { return 1 << (x & 255); }
05157   // int bar() { return foo(256); }
05158   case ISD::SRA_PARTS:
05159   case ISD::SRL_PARTS:
05160   case ISD::SHL_PARTS:
05161     if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
05162         cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
05163       return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
05164     else if (N3.getOpcode() == ISD::AND)
05165       if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
05166         // If the and is only masking out bits that cannot effect the shift,
05167         // eliminate the and.
05168         unsigned NumBits = VT.getScalarType().getSizeInBits()*2;
05169         if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
05170           return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
05171       }
05172     break;
05173   }
05174 #endif
05175 
05176   // Memoize the node unless it returns a flag.
05177   SDNode *N;
05178   unsigned NumOps = Ops.size();
05179   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
05180     FoldingSetNodeID ID;
05181     AddNodeIDNode(ID, Opcode, VTList, Ops);
05182     void *IP = nullptr;
05183     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05184       return SDValue(E, 0);
05185 
05186     if (NumOps == 1) {
05187       N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
05188                                           DL.getDebugLoc(), VTList, Ops[0]);
05189     } else if (NumOps == 2) {
05190       N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
05191                                            DL.getDebugLoc(), VTList, Ops[0],
05192                                            Ops[1]);
05193     } else if (NumOps == 3) {
05194       N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
05195                                             DL.getDebugLoc(), VTList, Ops[0],
05196                                             Ops[1], Ops[2]);
05197     } else {
05198       N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05199                                      VTList, Ops);
05200     }
05201     CSEMap.InsertNode(N, IP);
05202   } else {
05203     if (NumOps == 1) {
05204       N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
05205                                           DL.getDebugLoc(), VTList, Ops[0]);
05206     } else if (NumOps == 2) {
05207       N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
05208                                            DL.getDebugLoc(), VTList, Ops[0],
05209                                            Ops[1]);
05210     } else if (NumOps == 3) {
05211       N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
05212                                             DL.getDebugLoc(), VTList, Ops[0],
05213                                             Ops[1], Ops[2]);
05214     } else {
05215       N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05216                                      VTList, Ops);
05217     }
05218   }
05219   InsertNode(N);
05220   return SDValue(N, 0);
05221 }
05222 
05223 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList) {
05224   return getNode(Opcode, DL, VTList, None);
05225 }
05226 
05227 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05228                               SDValue N1) {
05229   SDValue Ops[] = { N1 };
05230   return getNode(Opcode, DL, VTList, Ops);
05231 }
05232 
05233 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05234                               SDValue N1, SDValue N2) {
05235   SDValue Ops[] = { N1, N2 };
05236   return getNode(Opcode, DL, VTList, Ops);
05237 }
05238 
05239 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05240                               SDValue N1, SDValue N2, SDValue N3) {
05241   SDValue Ops[] = { N1, N2, N3 };
05242   return getNode(Opcode, DL, VTList, Ops);
05243 }
05244 
05245 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05246                               SDValue N1, SDValue N2, SDValue N3,
05247                               SDValue N4) {
05248   SDValue Ops[] = { N1, N2, N3, N4 };
05249   return getNode(Opcode, DL, VTList, Ops);
05250 }
05251 
05252 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05253                               SDValue N1, SDValue N2, SDValue N3,
05254                               SDValue N4, SDValue N5) {
05255   SDValue Ops[] = { N1, N2, N3, N4, N5 };
05256   return getNode(Opcode, DL, VTList, Ops);
05257 }
05258 
05259 SDVTList SelectionDAG::getVTList(EVT VT) {
05260   return makeVTList(SDNode::getValueTypeList(VT), 1);
05261 }
05262 
05263 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
05264   FoldingSetNodeID ID;
05265   ID.AddInteger(2U);
05266   ID.AddInteger(VT1.getRawBits());
05267   ID.AddInteger(VT2.getRawBits());
05268 
05269   void *IP = nullptr;
05270   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05271   if (!Result) {
05272     EVT *Array = Allocator.Allocate<EVT>(2);
05273     Array[0] = VT1;
05274     Array[1] = VT2;
05275     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
05276     VTListMap.InsertNode(Result, IP);
05277   }
05278   return Result->getSDVTList();
05279 }
05280 
05281 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
05282   FoldingSetNodeID ID;
05283   ID.AddInteger(3U);
05284   ID.AddInteger(VT1.getRawBits());
05285   ID.AddInteger(VT2.getRawBits());
05286   ID.AddInteger(VT3.getRawBits());
05287 
05288   void *IP = nullptr;
05289   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05290   if (!Result) {
05291     EVT *Array = Allocator.Allocate<EVT>(3);
05292     Array[0] = VT1;
05293     Array[1] = VT2;
05294     Array[2] = VT3;
05295     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
05296     VTListMap.InsertNode(Result, IP);
05297   }
05298   return Result->getSDVTList();
05299 }
05300 
05301 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
05302   FoldingSetNodeID ID;
05303   ID.AddInteger(4U);
05304   ID.AddInteger(VT1.getRawBits());
05305   ID.AddInteger(VT2.getRawBits());
05306   ID.AddInteger(VT3.getRawBits());
05307   ID.AddInteger(VT4.getRawBits());
05308 
05309   void *IP = nullptr;
05310   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05311   if (!Result) {
05312     EVT *Array = Allocator.Allocate<EVT>(4);
05313     Array[0] = VT1;
05314     Array[1] = VT2;
05315     Array[2] = VT3;
05316     Array[3] = VT4;
05317     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
05318     VTListMap.InsertNode(Result, IP);
05319   }
05320   return Result->getSDVTList();
05321 }
05322 
05323 SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
05324   unsigned NumVTs = VTs.size();
05325   FoldingSetNodeID ID;
05326   ID.AddInteger(NumVTs);
05327   for (unsigned index = 0; index < NumVTs; index++) {
05328     ID.AddInteger(VTs[index].getRawBits());
05329   }
05330 
05331   void *IP = nullptr;
05332   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05333   if (!Result) {
05334     EVT *Array = Allocator.Allocate<EVT>(NumVTs);
05335     std::copy(VTs.begin(), VTs.end(), Array);
05336     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
05337     VTListMap.InsertNode(Result, IP);
05338   }
05339   return Result->getSDVTList();
05340 }
05341 
05342 
05343 /// UpdateNodeOperands - *Mutate* the specified node in-place to have the
05344 /// specified operands.  If the resultant node already exists in the DAG,
05345 /// this does not modify the specified node, instead it returns the node that
05346 /// already exists.  If the resultant node does not exist in the DAG, the
05347 /// input node is returned.  As a degenerate case, if you specify the same
05348 /// input operands as the node already has, the input node is returned.
05349 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
05350   assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
05351 
05352   // Check to see if there is no change.
05353   if (Op == N->getOperand(0)) return N;
05354 
05355   // See if the modified node already exists.
05356   void *InsertPos = nullptr;
05357   if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
05358     return Existing;
05359 
05360   // Nope it doesn't.  Remove the node from its current place in the maps.
05361   if (InsertPos)
05362     if (!RemoveNodeFromCSEMaps(N))
05363       InsertPos = nullptr;
05364 
05365   // Now we update the operands.
05366   N->OperandList[0].set(Op);
05367 
05368   // If this gets put into a CSE map, add it.
05369   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05370   return N;
05371 }
05372 
05373 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
05374   assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
05375 
05376   // Check to see if there is no change.
05377   if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
05378     return N;   // No operands changed, just return the input node.
05379 
05380   // See if the modified node already exists.
05381   void *InsertPos = nullptr;
05382   if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
05383     return Existing;
05384 
05385   // Nope it doesn't.  Remove the node from its current place in the maps.
05386   if (InsertPos)
05387     if (!RemoveNodeFromCSEMaps(N))
05388       InsertPos = nullptr;
05389 
05390   // Now we update the operands.
05391   if (N->OperandList[0] != Op1)
05392     N->OperandList[0].set(Op1);
05393   if (N->OperandList[1] != Op2)
05394     N->OperandList[1].set(Op2);
05395 
05396   // If this gets put into a CSE map, add it.
05397   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05398   return N;
05399 }
05400 
05401 SDNode *SelectionDAG::
05402 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
05403   SDValue Ops[] = { Op1, Op2, Op3 };
05404   return UpdateNodeOperands(N, Ops);
05405 }
05406 
05407 SDNode *SelectionDAG::
05408 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
05409                    SDValue Op3, SDValue Op4) {
05410   SDValue Ops[] = { Op1, Op2, Op3, Op4 };
05411   return UpdateNodeOperands(N, Ops);
05412 }
05413 
05414 SDNode *SelectionDAG::
05415 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
05416                    SDValue Op3, SDValue Op4, SDValue Op5) {
05417   SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
05418   return UpdateNodeOperands(N, Ops);
05419 }
05420 
05421 SDNode *SelectionDAG::
05422 UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
05423   unsigned NumOps = Ops.size();
05424   assert(N->getNumOperands() == NumOps &&
05425          "Update with wrong number of operands");
05426 
05427   // If no operands changed just return the input node.
05428   if (Ops.empty() || std::equal(Ops.begin(), Ops.end(), N->op_begin()))
05429     return N;
05430 
05431   // See if the modified node already exists.
05432   void *InsertPos = nullptr;
05433   if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
05434     return Existing;
05435 
05436   // Nope it doesn't.  Remove the node from its current place in the maps.
05437   if (InsertPos)
05438     if (!RemoveNodeFromCSEMaps(N))
05439       InsertPos = nullptr;
05440 
05441   // Now we update the operands.
05442   for (unsigned i = 0; i != NumOps; ++i)
05443     if (N->OperandList[i] != Ops[i])
05444       N->OperandList[i].set(Ops[i]);
05445 
05446   // If this gets put into a CSE map, add it.
05447   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05448   return N;
05449 }
05450 
05451 /// DropOperands - Release the operands and set this node to have
05452 /// zero operands.
05453 void SDNode::DropOperands() {
05454   // Unlike the code in MorphNodeTo that does this, we don't need to
05455   // watch for dead nodes here.
05456   for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
05457     SDUse &Use = *I++;
05458     Use.set(SDValue());
05459   }
05460 }
05461 
05462 /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
05463 /// machine opcode.
05464 ///
05465 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05466                                    EVT VT) {
05467   SDVTList VTs = getVTList(VT);
05468   return SelectNodeTo(N, MachineOpc, VTs, None);
05469 }
05470 
05471 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05472                                    EVT VT, SDValue Op1) {
05473   SDVTList VTs = getVTList(VT);
05474   SDValue Ops[] = { Op1 };
05475   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05476 }
05477 
05478 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05479                                    EVT VT, SDValue Op1,
05480                                    SDValue Op2) {
05481   SDVTList VTs = getVTList(VT);
05482   SDValue Ops[] = { Op1, Op2 };
05483   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05484 }
05485 
05486 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05487                                    EVT VT, SDValue Op1,
05488                                    SDValue Op2, SDValue Op3) {
05489   SDVTList VTs = getVTList(VT);
05490   SDValue Ops[] = { Op1, Op2, Op3 };
05491   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05492 }
05493 
05494 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05495                                    EVT VT, ArrayRef<SDValue> Ops) {
05496   SDVTList VTs = getVTList(VT);
05497   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05498 }
05499 
05500 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05501                                    EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
05502   SDVTList VTs = getVTList(VT1, VT2);
05503   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05504 }
05505 
05506 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05507                                    EVT VT1, EVT VT2) {
05508   SDVTList VTs = getVTList(VT1, VT2);
05509   return SelectNodeTo(N, MachineOpc, VTs, None);
05510 }
05511 
05512 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05513                                    EVT VT1, EVT VT2, EVT VT3,
05514                                    ArrayRef<SDValue> Ops) {
05515   SDVTList VTs = getVTList(VT1, VT2, VT3);
05516   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05517 }
05518 
05519 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05520                                    EVT VT1, EVT VT2, EVT VT3, EVT VT4,
05521                                    ArrayRef<SDValue> Ops) {
05522   SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
05523   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05524 }
05525 
05526 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05527                                    EVT VT1, EVT VT2,
05528                                    SDValue Op1) {
05529   SDVTList VTs = getVTList(VT1, VT2);
05530   SDValue Ops[] = { Op1 };
05531   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05532 }
05533 
05534 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05535                                    EVT VT1, EVT VT2,
05536                                    SDValue Op1, SDValue Op2) {
05537   SDVTList VTs = getVTList(VT1, VT2);
05538   SDValue Ops[] = { Op1, Op2 };
05539   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05540 }
05541 
05542 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05543                                    EVT VT1, EVT VT2,
05544                                    SDValue Op1, SDValue Op2,
05545                                    SDValue Op3) {
05546   SDVTList VTs = getVTList(VT1, VT2);
05547   SDValue Ops[] = { Op1, Op2, Op3 };
05548   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05549 }
05550 
05551 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05552                                    EVT VT1, EVT VT2, EVT VT3,
05553                                    SDValue Op1, SDValue Op2,
05554                                    SDValue Op3) {
05555   SDVTList VTs = getVTList(VT1, VT2, VT3);
05556   SDValue Ops[] = { Op1, Op2, Op3 };
05557   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05558 }
05559 
05560 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05561                                    SDVTList VTs,ArrayRef<SDValue> Ops) {
05562   N = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
05563   // Reset the NodeID to -1.
05564   N->setNodeId(-1);
05565   return N;
05566 }
05567 
05568 /// UpdadeSDLocOnMergedSDNode - If the opt level is -O0 then it throws away
05569 /// the line number information on the merged node since it is not possible to
05570 /// preserve the information that operation is associated with multiple lines.
05571 /// This will make the debugger working better at -O0, were there is a higher
05572 /// probability having other instructions associated with that line.
05573 ///
05574 /// For IROrder, we keep the smaller of the two
05575 SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) {
05576   DebugLoc NLoc = N->getDebugLoc();
05577   if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) &&
05578     (OLoc.getDebugLoc() != NLoc)) {
05579     N->setDebugLoc(DebugLoc());
05580   }
05581   unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
05582   N->setIROrder(Order);
05583   return N;
05584 }
05585 
05586 /// MorphNodeTo - This *mutates* the specified node to have the specified
05587 /// return type, opcode, and operands.
05588 ///
05589 /// Note that MorphNodeTo returns the resultant node.  If there is already a
05590 /// node of the specified opcode and operands, it returns that node instead of
05591 /// the current one.  Note that the SDLoc need not be the same.
05592 ///
05593 /// Using MorphNodeTo is faster than creating a new node and swapping it in
05594 /// with ReplaceAllUsesWith both because it often avoids allocating a new
05595 /// node, and because it doesn't require CSE recalculation for any of
05596 /// the node's users.
05597 ///
05598 /// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
05599 /// As a consequence it isn't appropriate to use from within the DAG combiner or
05600 /// the legalizer which maintain worklists that would need to be updated when
05601 /// deleting things.
05602 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
05603                                   SDVTList VTs, ArrayRef<SDValue> Ops) {
05604   unsigned NumOps = Ops.size();
05605   // If an identical node already exists, use it.
05606   void *IP = nullptr;
05607   if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
05608     FoldingSetNodeID ID;
05609     AddNodeIDNode(ID, Opc, VTs, Ops);
05610     if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
05611       return UpdadeSDLocOnMergedSDNode(ON, SDLoc(N));
05612   }
05613 
05614   if (!RemoveNodeFromCSEMaps(N))
05615     IP = nullptr;
05616 
05617   // Start the morphing.
05618   N->NodeType = Opc;
05619   N->ValueList = VTs.VTs;
05620   N->NumValues = VTs.NumVTs;
05621 
05622   // Clear the operands list, updating used nodes to remove this from their
05623   // use list.  Keep track of any operands that become dead as a result.
05624   SmallPtrSet<SDNode*, 16> DeadNodeSet;
05625   for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
05626     SDUse &Use = *I++;
05627     SDNode *Used = Use.getNode();
05628     Use.set(SDValue());
05629     if (Used->use_empty())
05630       DeadNodeSet.insert(Used);
05631   }
05632 
05633   if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) {
05634     // Initialize the memory references information.
05635     MN->setMemRefs(nullptr, nullptr);
05636     // If NumOps is larger than the # of operands we can have in a
05637     // MachineSDNode, reallocate the operand list.
05638     if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) {
05639       if (MN->OperandsNeedDelete)
05640         delete[] MN->OperandList;
05641       if (NumOps > array_lengthof(MN->LocalOperands))
05642         // We're creating a final node that will live unmorphed for the
05643         // remainder of the current SelectionDAG iteration, so we can allocate
05644         // the operands directly out of a pool with no recycling metadata.
05645         MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
05646                          Ops.data(), NumOps);
05647       else
05648         MN->InitOperands(MN->LocalOperands, Ops.data(), NumOps);
05649       MN->OperandsNeedDelete = false;
05650     } else
05651       MN->InitOperands(MN->OperandList, Ops.data(), NumOps);
05652   } else {
05653     // If NumOps is larger than the # of operands we currently have, reallocate
05654     // the operand list.
05655     if (NumOps > N->NumOperands) {
05656       if (N->OperandsNeedDelete)
05657         delete[] N->OperandList;
05658       N->InitOperands(new SDUse[NumOps], Ops.data(), NumOps);
05659       N->OperandsNeedDelete = true;
05660     } else
05661       N->InitOperands(N->OperandList, Ops.data(), NumOps);
05662   }
05663 
05664   // Delete any nodes that are still dead after adding the uses for the
05665   // new operands.
05666   if (!DeadNodeSet.empty()) {
05667     SmallVector<SDNode *, 16> DeadNodes;
05668     for (SDNode *N : DeadNodeSet)
05669       if (N->use_empty())
05670         DeadNodes.push_back(N);
05671     RemoveDeadNodes(DeadNodes);
05672   }
05673 
05674   if (IP)
05675     CSEMap.InsertNode(N, IP);   // Memoize the new node.
05676   return N;
05677 }
05678 
05679 
05680 /// getMachineNode - These are used for target selectors to create a new node
05681 /// with specified return type(s), MachineInstr opcode, and operands.
05682 ///
05683 /// Note that getMachineNode returns the resultant node.  If there is already a
05684 /// node of the specified opcode and operands, it returns that node instead of
05685 /// the current one.
05686 MachineSDNode *
05687 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT) {
05688   SDVTList VTs = getVTList(VT);
05689   return getMachineNode(Opcode, dl, VTs, None);
05690 }
05691 
05692 MachineSDNode *
05693 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, SDValue Op1) {
05694   SDVTList VTs = getVTList(VT);
05695   SDValue Ops[] = { Op1 };
05696   return getMachineNode(Opcode, dl, VTs, Ops);
05697 }
05698 
05699 MachineSDNode *
05700 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05701                              SDValue Op1, SDValue Op2) {
05702   SDVTList VTs = getVTList(VT);
05703   SDValue Ops[] = { Op1, Op2 };
05704   return getMachineNode(Opcode, dl, VTs, Ops);
05705 }
05706 
05707 MachineSDNode *
05708 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05709                              SDValue Op1, SDValue Op2, SDValue Op3) {
05710   SDVTList VTs = getVTList(VT);
05711   SDValue Ops[] = { Op1, Op2, Op3 };
05712   return getMachineNode(Opcode, dl, VTs, Ops);
05713 }
05714 
05715 MachineSDNode *
05716 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05717                              ArrayRef<SDValue> Ops) {
05718   SDVTList VTs = getVTList(VT);
05719   return getMachineNode(Opcode, dl, VTs, Ops);
05720 }
05721 
05722 MachineSDNode *
05723 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, EVT VT2) {
05724   SDVTList VTs = getVTList(VT1, VT2);
05725   return getMachineNode(Opcode, dl, VTs, None);
05726 }
05727 
05728 MachineSDNode *
05729 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05730                              EVT VT1, EVT VT2, SDValue Op1) {
05731   SDVTList VTs = getVTList(VT1, VT2);
05732   SDValue Ops[] = { Op1 };
05733   return getMachineNode(Opcode, dl, VTs, Ops);
05734 }
05735 
05736 MachineSDNode *
05737 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05738                              EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) {
05739   SDVTList VTs = getVTList(VT1, VT2);
05740   SDValue Ops[] = { Op1, Op2 };
05741   return getMachineNode(Opcode, dl, VTs, Ops);
05742 }
05743 
05744 MachineSDNode *
05745 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05746                              EVT VT1, EVT VT2, SDValue Op1,
05747                              SDValue Op2, SDValue Op3) {
05748   SDVTList VTs = getVTList(VT1, VT2);
05749   SDValue Ops[] = { Op1, Op2, Op3 };
05750   return getMachineNode(Opcode, dl, VTs, Ops);
05751 }
05752 
05753 MachineSDNode *
05754 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05755                              EVT VT1, EVT VT2,
05756                              ArrayRef<SDValue> Ops) {
05757   SDVTList VTs = getVTList(VT1, VT2);
05758   return getMachineNode(Opcode, dl, VTs, Ops);
05759 }
05760 
05761 MachineSDNode *
05762 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05763                              EVT VT1, EVT VT2, EVT VT3,
05764                              SDValue Op1, SDValue Op2) {
05765   SDVTList VTs = getVTList(VT1, VT2, VT3);
05766   SDValue Ops[] = { Op1, Op2 };
05767   return getMachineNode(Opcode, dl, VTs, Ops);
05768 }
05769 
05770 MachineSDNode *
05771 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05772                              EVT VT1, EVT VT2, EVT VT3,
05773                              SDValue Op1, SDValue Op2, SDValue Op3) {
05774   SDVTList VTs = getVTList(VT1, VT2, VT3);
05775   SDValue Ops[] = { Op1, Op2, Op3 };
05776   return getMachineNode(Opcode, dl, VTs, Ops);
05777 }
05778 
05779 MachineSDNode *
05780 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05781                              EVT VT1, EVT VT2, EVT VT3,
05782                              ArrayRef<SDValue> Ops) {
05783   SDVTList VTs = getVTList(VT1, VT2, VT3);
05784   return getMachineNode(Opcode, dl, VTs, Ops);
05785 }
05786 
05787 MachineSDNode *
05788 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1,
05789                              EVT VT2, EVT VT3, EVT VT4,
05790                              ArrayRef<SDValue> Ops) {
05791   SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
05792   return getMachineNode(Opcode, dl, VTs, Ops);
05793 }
05794 
05795 MachineSDNode *
05796 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05797                              ArrayRef<EVT> ResultTys,
05798                              ArrayRef<SDValue> Ops) {
05799   SDVTList VTs = getVTList(ResultTys);
05800   return getMachineNode(Opcode, dl, VTs, Ops);
05801 }
05802 
05803 MachineSDNode *
05804 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
05805                              ArrayRef<SDValue> OpsArray) {
05806   bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
05807   MachineSDNode *N;
05808   void *IP = nullptr;
05809   const SDValue *Ops = OpsArray.data();
05810   unsigned NumOps = OpsArray.size();
05811 
05812   if (DoCSE) {
05813     FoldingSetNodeID ID;
05814     AddNodeIDNode(ID, ~Opcode, VTs, OpsArray);
05815     IP = nullptr;
05816     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
05817       return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL));
05818     }
05819   }
05820 
05821   // Allocate a new MachineSDNode.
05822   N = new (NodeAllocator) MachineSDNode(~Opcode, DL.getIROrder(),
05823                                         DL.getDebugLoc(), VTs);
05824 
05825   // Initialize the operands list.
05826   if (NumOps > array_lengthof(N->LocalOperands))
05827     // We're creating a final node that will live unmorphed for the
05828     // remainder of the current SelectionDAG iteration, so we can allocate
05829     // the operands directly out of a pool with no recycling metadata.
05830     N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
05831                     Ops, NumOps);
05832   else
05833     N->InitOperands(N->LocalOperands, Ops, NumOps);
05834   N->OperandsNeedDelete = false;
05835 
05836   if (DoCSE)
05837     CSEMap.InsertNode(N, IP);
05838 
05839   InsertNode(N);
05840   return N;
05841 }
05842 
05843 /// getTargetExtractSubreg - A convenience function for creating
05844 /// TargetOpcode::EXTRACT_SUBREG nodes.
05845 SDValue
05846 SelectionDAG::getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT,
05847                                      SDValue Operand) {
05848   SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
05849   SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
05850                                   VT, Operand, SRIdxVal);
05851   return SDValue(Subreg, 0);
05852 }
05853 
05854 /// getTargetInsertSubreg - A convenience function for creating
05855 /// TargetOpcode::INSERT_SUBREG nodes.
05856 SDValue
05857 SelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT,
05858                                     SDValue Operand, SDValue Subreg) {
05859   SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
05860   SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
05861                                   VT, Operand, Subreg, SRIdxVal);
05862   return SDValue(Result, 0);
05863 }
05864 
05865 /// getNodeIfExists - Get the specified node if it's already available, or
05866 /// else return NULL.
05867 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
05868                                       ArrayRef<SDValue> Ops, bool nuw, bool nsw,
05869                                       bool exact) {
05870   if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
05871     FoldingSetNodeID ID;
05872     AddNodeIDNode(ID, Opcode, VTList, Ops);
05873     if (isBinOpWithFlags(Opcode))
05874       AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
05875     void *IP = nullptr;
05876     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05877       return E;
05878   }
05879   return nullptr;
05880 }
05881 
05882 /// getDbgValue - Creates a SDDbgValue node.
05883 ///
05884 /// SDNode
05885 SDDbgValue *SelectionDAG::getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N,
05886                                       unsigned R, bool IsIndirect, uint64_t Off,
05887                                       DebugLoc DL, unsigned O) {
05888   return new (Allocator) SDDbgValue(Var, Expr, N, R, IsIndirect, Off, DL, O);
05889 }
05890 
05891 /// Constant
05892 SDDbgValue *SelectionDAG::getConstantDbgValue(MDNode *Var, MDNode *Expr,
05893                                               const Value *C, uint64_t Off,
05894                                               DebugLoc DL, unsigned O) {
05895   return new (Allocator) SDDbgValue(Var, Expr, C, Off, DL, O);
05896 }
05897 
05898 /// FrameIndex
05899 SDDbgValue *SelectionDAG::getFrameIndexDbgValue(MDNode *Var, MDNode *Expr,
05900                                                 unsigned FI, uint64_t Off,
05901                                                 DebugLoc DL, unsigned O) {
05902   return new (Allocator) SDDbgValue(Var, Expr, FI, Off, DL, O);
05903 }
05904 
05905 namespace {
05906 
05907 /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
05908 /// pointed to by a use iterator is deleted, increment the use iterator
05909 /// so that it doesn't dangle.
05910 ///
05911 class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
05912   SDNode::use_iterator &UI;
05913   SDNode::use_iterator &UE;
05914 
05915   void NodeDeleted(SDNode *N, SDNode *E) override {
05916     // Increment the iterator as needed.
05917     while (UI != UE && N == *UI)
05918       ++UI;
05919   }
05920 
05921 public:
05922   RAUWUpdateListener(SelectionDAG &d,
05923                      SDNode::use_iterator &ui,
05924                      SDNode::use_iterator &ue)
05925     : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
05926 };
05927 
05928 }
05929 
05930 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05931 /// This can cause recursive merging of nodes in the DAG.
05932 ///
05933 /// This version assumes From has a single result value.
05934 ///
05935 void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
05936   SDNode *From = FromN.getNode();
05937   assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
05938          "Cannot replace with this method!");
05939   assert(From != To.getNode() && "Cannot replace uses of with self");
05940 
05941   // Iterate over all the existing uses of From. New uses will be added
05942   // to the beginning of the use list, which we avoid visiting.
05943   // This specifically avoids visiting uses of From that arise while the
05944   // replacement is happening, because any such uses would be the result
05945   // of CSE: If an existing node looks like From after one of its operands
05946   // is replaced by To, we don't want to replace of all its users with To
05947   // too. See PR3018 for more info.
05948   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05949   RAUWUpdateListener Listener(*this, UI, UE);
05950   while (UI != UE) {
05951     SDNode *User = *UI;
05952 
05953     // This node is about to morph, remove its old self from the CSE maps.
05954     RemoveNodeFromCSEMaps(User);
05955 
05956     // A user can appear in a use list multiple times, and when this
05957     // happens the uses are usually next to each other in the list.
05958     // To help reduce the number of CSE recomputations, process all
05959     // the uses of this user that we can find this way.
05960     do {
05961       SDUse &Use = UI.getUse();
05962       ++UI;
05963       Use.set(To);
05964     } while (UI != UE && *UI == User);
05965 
05966     // Now that we have modified User, add it back to the CSE maps.  If it
05967     // already exists there, recursively merge the results together.
05968     AddModifiedNodeToCSEMaps(User);
05969   }
05970 
05971   // If we just RAUW'd the root, take note.
05972   if (FromN == getRoot())
05973     setRoot(To);
05974 }
05975 
05976 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05977 /// This can cause recursive merging of nodes in the DAG.
05978 ///
05979 /// This version assumes that for each value of From, there is a
05980 /// corresponding value in To in the same position with the same type.
05981 ///
05982 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
05983 #ifndef NDEBUG
05984   for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
05985     assert((!From->hasAnyUseOfValue(i) ||
05986             From->getValueType(i) == To->getValueType(i)) &&
05987            "Cannot use this version of ReplaceAllUsesWith!");
05988 #endif
05989 
05990   // Handle the trivial case.
05991   if (From == To)
05992     return;
05993 
05994   // Iterate over just the existing users of From. See the comments in
05995   // the ReplaceAllUsesWith above.
05996   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05997   RAUWUpdateListener Listener(*this, UI, UE);
05998   while (UI != UE) {
05999     SDNode *User = *UI;
06000 
06001     // This node is about to morph, remove its old self from the CSE maps.
06002     RemoveNodeFromCSEMaps(User);
06003 
06004     // A user can appear in a use list multiple times, and when this
06005     // happens the uses are usually next to each other in the list.
06006     // To help reduce the number of CSE recomputations, process all
06007     // the uses of this user that we can find this way.
06008     do {
06009       SDUse &Use = UI.getUse();
06010       ++UI;
06011       Use.setNode(To);
06012     } while (UI != UE && *UI == User);
06013 
06014     // Now that we have modified User, add it back to the CSE maps.  If it
06015     // already exists there, recursively merge the results together.
06016     AddModifiedNodeToCSEMaps(User);
06017   }
06018 
06019   // If we just RAUW'd the root, take note.
06020   if (From == getRoot().getNode())
06021     setRoot(SDValue(To, getRoot().getResNo()));
06022 }
06023 
06024 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
06025 /// This can cause recursive merging of nodes in the DAG.
06026 ///
06027 /// This version can replace From with any result values.  To must match the
06028 /// number and types of values returned by From.
06029 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
06030   if (From->getNumValues() == 1)  // Handle the simple case efficiently.
06031     return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
06032 
06033   // Iterate over just the existing users of From. See the comments in
06034   // the ReplaceAllUsesWith above.
06035   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
06036   RAUWUpdateListener Listener(*this, UI, UE);
06037   while (UI != UE) {
06038     SDNode *User = *UI;
06039 
06040     // This node is about to morph, remove its old self from the CSE maps.
06041     RemoveNodeFromCSEMaps(User);
06042 
06043     // A user can appear in a use list multiple times, and when this
06044     // happens the uses are usually next to each other in the list.
06045     // To help reduce the number of CSE recomputations, process all
06046     // the uses of this user that we can find this way.
06047     do {
06048       SDUse &Use = UI.getUse();
06049       const SDValue &ToOp = To[Use.getResNo()];
06050       ++UI;
06051       Use.set(ToOp);
06052     } while (UI != UE && *UI == User);
06053 
06054     // Now that we have modified User, add it back to the CSE maps.  If it
06055     // already exists there, recursively merge the results together.
06056     AddModifiedNodeToCSEMaps(User);
06057   }
06058 
06059   // If we just RAUW'd the root, take note.
06060   if (From == getRoot().getNode())
06061     setRoot(SDValue(To[getRoot().getResNo()]));
06062 }
06063 
06064 /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
06065 /// uses of other values produced by From.getNode() alone.  The Deleted
06066 /// vector is handled the same way as for ReplaceAllUsesWith.
06067 void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
06068   // Handle the really simple, really trivial case efficiently.
06069   if (From == To) return;
06070 
06071   // Handle the simple, trivial, case efficiently.
06072   if (From.getNode()->getNumValues() == 1) {
06073     ReplaceAllUsesWith(From, To);
06074     return;
06075   }
06076 
06077   // Iterate over just the existing users of From. See the comments in
06078   // the ReplaceAllUsesWith above.
06079   SDNode::use_iterator UI = From.getNode()->use_begin(),
06080                        UE = From.getNode()->use_end();
06081   RAUWUpdateListener Listener(*this, UI, UE);
06082   while (UI != UE) {
06083     SDNode *User = *UI;
06084     bool UserRemovedFromCSEMaps = false;
06085 
06086     // A user can appear in a use list multiple times, and when this
06087     // happens the uses are usually next to each other in the list.
06088     // To help reduce the number of CSE recomputations, process all
06089     // the uses of this user that we can find this way.
06090     do {
06091       SDUse &Use = UI.getUse();
06092 
06093       // Skip uses of different values from the same node.
06094       if (Use.getResNo() != From.getResNo()) {
06095         ++UI;
06096         continue;
06097       }
06098 
06099       // If this node hasn't been modified yet, it's still in the CSE maps,
06100       // so remove its old self from the CSE maps.
06101       if (!UserRemovedFromCSEMaps) {
06102         RemoveNodeFromCSEMaps(User);
06103         UserRemovedFromCSEMaps = true;
06104       }
06105 
06106       ++UI;
06107       Use.set(To);
06108     } while (UI != UE && *UI == User);
06109 
06110     // We are iterating over all uses of the From node, so if a use
06111     // doesn't use the specific value, no changes are made.
06112     if (!UserRemovedFromCSEMaps)
06113       continue;
06114 
06115     // Now that we have modified User, add it back to the CSE maps.  If it
06116     // already exists there, recursively merge the results together.
06117     AddModifiedNodeToCSEMaps(User);
06118   }
06119 
06120   // If we just RAUW'd the root, take note.
06121   if (From == getRoot())
06122     setRoot(To);
06123 }
06124 
06125 namespace {
06126   /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
06127   /// to record information about a use.
06128   struct UseMemo {
06129     SDNode *User;
06130     unsigned Index;
06131     SDUse *Use;
06132   };
06133 
06134   /// operator< - Sort Memos by User.
06135   bool operator<(const UseMemo &L, const UseMemo &R) {
06136     return (intptr_t)L.User < (intptr_t)R.User;
06137   }
06138 }
06139 
06140 /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
06141 /// uses of other values produced by From.getNode() alone.  The same value
06142 /// may appear in both the From and To list.  The Deleted vector is
06143 /// handled the same way as for ReplaceAllUsesWith.
06144 void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
06145                                               const SDValue *To,
06146                                               unsigned Num){
06147   // Handle the simple, trivial case efficiently.
06148   if (Num == 1)
06149     return ReplaceAllUsesOfValueWith(*From, *To);
06150 
06151   // Read up all the uses and make records of them. This helps
06152   // processing new uses that are introduced during the
06153   // replacement process.
06154   SmallVector<UseMemo, 4> Uses;
06155   for (unsigned i = 0; i != Num; ++i) {
06156     unsigned FromResNo = From[i].getResNo();
06157     SDNode *FromNode = From[i].getNode();
06158     for (SDNode::use_iterator UI = FromNode->use_begin(),
06159          E = FromNode->use_end(); UI != E; ++UI) {
06160       SDUse &Use = UI.getUse();
06161       if (Use.getResNo() == FromResNo) {
06162         UseMemo Memo = { *UI, i, &Use };
06163         Uses.push_back(Memo);
06164       }
06165     }
06166   }
06167 
06168   // Sort the uses, so that all the uses from a given User are together.
06169   std::sort(Uses.begin(), Uses.end());
06170 
06171   for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
06172        UseIndex != UseIndexEnd; ) {
06173     // We know that this user uses some value of From.  If it is the right
06174     // value, update it.
06175     SDNode *User = Uses[UseIndex].User;
06176 
06177     // This node is about to morph, remove its old self from the CSE maps.
06178     RemoveNodeFromCSEMaps(User);
06179 
06180     // The Uses array is sorted, so all the uses for a given User
06181     // are next to each other in the list.
06182     // To help reduce the number of CSE recomputations, process all
06183     // the uses of this user that we can find this way.
06184     do {
06185       unsigned i = Uses[UseIndex].Index;
06186       SDUse &Use = *Uses[UseIndex].Use;
06187       ++UseIndex;
06188 
06189       Use.set(To[i]);
06190     } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
06191 
06192     // Now that we have modified User, add it back to the CSE maps.  If it
06193     // already exists there, recursively merge the results together.
06194     AddModifiedNodeToCSEMaps(User);
06195   }
06196 }
06197 
06198 /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
06199 /// based on their topological order. It returns the maximum id and a vector
06200 /// of the SDNodes* in assigned order by reference.
06201 unsigned SelectionDAG::AssignTopologicalOrder() {
06202 
06203   unsigned DAGSize = 0;
06204 
06205   // SortedPos tracks the progress of the algorithm. Nodes before it are
06206   // sorted, nodes after it are unsorted. When the algorithm completes
06207   // it is at the end of the list.
06208   allnodes_iterator SortedPos = allnodes_begin();
06209 
06210   // Visit all the nodes. Move nodes with no operands to the front of
06211   // the list immediately. Annotate nodes that do have operands with their
06212   // operand count. Before we do this, the Node Id fields of the nodes
06213   // may contain arbitrary values. After, the Node Id fields for nodes
06214   // before SortedPos will contain the topological sort index, and the
06215   // Node Id fields for nodes At SortedPos and after will contain the
06216   // count of outstanding operands.
06217   for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
06218     SDNode *N = I++;
06219     checkForCycles(N, this);
06220     unsigned Degree = N->getNumOperands();
06221     if (Degree == 0) {
06222       // A node with no uses, add it to the result array immediately.
06223       N->setNodeId(DAGSize++);
06224       allnodes_iterator Q = N;
06225       if (Q != SortedPos)
06226         SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
06227       assert(SortedPos != AllNodes.end() && "Overran node list");
06228       ++SortedPos;
06229     } else {
06230       // Temporarily use the Node Id as scratch space for the degree count.
06231       N->setNodeId(Degree);
06232     }
06233   }
06234 
06235   // Visit all the nodes. As we iterate, move nodes into sorted order,
06236   // such that by the time the end is reached all nodes will be sorted.
06237   for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) {
06238     SDNode *N = I;
06239     checkForCycles(N, this);
06240     // N is in sorted position, so all its uses have one less operand
06241     // that needs to be sorted.
06242     for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
06243          UI != UE; ++UI) {
06244       SDNode *P = *UI;
06245       unsigned Degree = P->getNodeId();
06246       assert(Degree != 0 && "Invalid node degree");
06247       --Degree;
06248       if (Degree == 0) {
06249         // All of P's operands are sorted, so P may sorted now.
06250         P->setNodeId(DAGSize++);
06251         if (P != SortedPos)
06252           SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
06253         assert(SortedPos != AllNodes.end() && "Overran node list