LLVM API Documentation

SelectionDAG.cpp
Go to the documentation of this file.
00001 //===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This implements the SelectionDAG class.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "llvm/CodeGen/SelectionDAG.h"
00015 #include "SDNodeDbgValue.h"
00016 #include "llvm/ADT/SetVector.h"
00017 #include "llvm/ADT/SmallPtrSet.h"
00018 #include "llvm/ADT/SmallSet.h"
00019 #include "llvm/ADT/SmallVector.h"
00020 #include "llvm/ADT/StringExtras.h"
00021 #include "llvm/Analysis/ValueTracking.h"
00022 #include "llvm/CodeGen/MachineBasicBlock.h"
00023 #include "llvm/CodeGen/MachineConstantPool.h"
00024 #include "llvm/CodeGen/MachineFrameInfo.h"
00025 #include "llvm/CodeGen/MachineModuleInfo.h"
00026 #include "llvm/IR/CallingConv.h"
00027 #include "llvm/IR/Constants.h"
00028 #include "llvm/IR/DataLayout.h"
00029 #include "llvm/IR/DebugInfo.h"
00030 #include "llvm/IR/DerivedTypes.h"
00031 #include "llvm/IR/Function.h"
00032 #include "llvm/IR/GlobalAlias.h"
00033 #include "llvm/IR/GlobalVariable.h"
00034 #include "llvm/IR/Intrinsics.h"
00035 #include "llvm/Support/CommandLine.h"
00036 #include "llvm/Support/Debug.h"
00037 #include "llvm/Support/ErrorHandling.h"
00038 #include "llvm/Support/ManagedStatic.h"
00039 #include "llvm/Support/MathExtras.h"
00040 #include "llvm/Support/Mutex.h"
00041 #include "llvm/Support/raw_ostream.h"
00042 #include "llvm/Target/TargetInstrInfo.h"
00043 #include "llvm/Target/TargetIntrinsicInfo.h"
00044 #include "llvm/Target/TargetLowering.h"
00045 #include "llvm/Target/TargetMachine.h"
00046 #include "llvm/Target/TargetOptions.h"
00047 #include "llvm/Target/TargetRegisterInfo.h"
00048 #include "llvm/Target/TargetSelectionDAGInfo.h"
00049 #include "llvm/Target/TargetSubtargetInfo.h"
00050 #include <algorithm>
00051 #include <cmath>
00052 
00053 using namespace llvm;
00054 
00055 /// makeVTList - Return an instance of the SDVTList struct initialized with the
00056 /// specified members.
00057 static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
00058   SDVTList Res = {VTs, NumVTs};
00059   return Res;
00060 }
00061 
00062 // Default null implementations of the callbacks.
00063 void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
00064 void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
00065 
00066 //===----------------------------------------------------------------------===//
00067 //                              ConstantFPSDNode Class
00068 //===----------------------------------------------------------------------===//
00069 
00070 /// isExactlyValue - We don't rely on operator== working on double values, as
00071 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
00072 /// As such, this method can be used to do an exact bit-for-bit comparison of
00073 /// two floating point values.
00074 bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
00075   return getValueAPF().bitwiseIsEqual(V);
00076 }
00077 
00078 bool ConstantFPSDNode::isValueValidForType(EVT VT,
00079                                            const APFloat& Val) {
00080   assert(VT.isFloatingPoint() && "Can only convert between FP types");
00081 
00082   // convert modifies in place, so make a copy.
00083   APFloat Val2 = APFloat(Val);
00084   bool losesInfo;
00085   (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
00086                       APFloat::rmNearestTiesToEven,
00087                       &losesInfo);
00088   return !losesInfo;
00089 }
00090 
00091 //===----------------------------------------------------------------------===//
00092 //                              ISD Namespace
00093 //===----------------------------------------------------------------------===//
00094 
00095 /// isBuildVectorAllOnes - Return true if the specified node is a
00096 /// BUILD_VECTOR where all of the elements are ~0 or undef.
00097 bool ISD::isBuildVectorAllOnes(const SDNode *N) {
00098   // Look through a bit convert.
00099   if (N->getOpcode() == ISD::BITCAST)
00100     N = N->getOperand(0).getNode();
00101 
00102   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
00103 
00104   unsigned i = 0, e = N->getNumOperands();
00105 
00106   // Skip over all of the undef values.
00107   while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF)
00108     ++i;
00109 
00110   // Do not accept an all-undef vector.
00111   if (i == e) return false;
00112 
00113   // Do not accept build_vectors that aren't all constants or which have non-~0
00114   // elements. We have to be a bit careful here, as the type of the constant
00115   // may not be the same as the type of the vector elements due to type
00116   // legalization (the elements are promoted to a legal type for the target and
00117   // a vector of a type may be legal when the base element type is not).
00118   // We only want to check enough bits to cover the vector elements, because
00119   // we care if the resultant vector is all ones, not whether the individual
00120   // constants are.
00121   SDValue NotZero = N->getOperand(i);
00122   unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
00123   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
00124     if (CN->getAPIntValue().countTrailingOnes() < EltSize)
00125       return false;
00126   } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
00127     if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
00128       return false;
00129   } else
00130     return false;
00131 
00132   // Okay, we have at least one ~0 value, check to see if the rest match or are
00133   // undefs. Even with the above element type twiddling, this should be OK, as
00134   // the same type legalization should have applied to all the elements.
00135   for (++i; i != e; ++i)
00136     if (N->getOperand(i) != NotZero &&
00137         N->getOperand(i).getOpcode() != ISD::UNDEF)
00138       return false;
00139   return true;
00140 }
00141 
00142 
00143 /// isBuildVectorAllZeros - Return true if the specified node is a
00144 /// BUILD_VECTOR where all of the elements are 0 or undef.
00145 bool ISD::isBuildVectorAllZeros(const SDNode *N) {
00146   // Look through a bit convert.
00147   if (N->getOpcode() == ISD::BITCAST)
00148     N = N->getOperand(0).getNode();
00149 
00150   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
00151 
00152   bool IsAllUndef = true;
00153   for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) {
00154     if (N->getOperand(i).getOpcode() == ISD::UNDEF)
00155       continue;
00156     IsAllUndef = false;
00157     // Do not accept build_vectors that aren't all constants or which have non-0
00158     // elements. We have to be a bit careful here, as the type of the constant
00159     // may not be the same as the type of the vector elements due to type
00160     // legalization (the elements are promoted to a legal type for the target
00161     // and a vector of a type may be legal when the base element type is not).
00162     // We only want to check enough bits to cover the vector elements, because
00163     // we care if the resultant vector is all zeros, not whether the individual
00164     // constants are.
00165     SDValue Zero = N->getOperand(i);
00166     unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
00167     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Zero)) {
00168       if (CN->getAPIntValue().countTrailingZeros() < EltSize)
00169         return false;
00170     } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Zero)) {
00171       if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
00172         return false;
00173     } else
00174       return false;
00175   }
00176 
00177   // Do not accept an all-undef vector.
00178   if (IsAllUndef)
00179     return false;
00180   return true;
00181 }
00182 
00183 /// \brief Return true if the specified node is a BUILD_VECTOR node of
00184 /// all ConstantSDNode or undef.
00185 bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
00186   if (N->getOpcode() != ISD::BUILD_VECTOR)
00187     return false;
00188 
00189   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
00190     SDValue Op = N->getOperand(i);
00191     if (Op.getOpcode() == ISD::UNDEF)
00192       continue;
00193     if (!isa<ConstantSDNode>(Op))
00194       return false;
00195   }
00196   return true;
00197 }
00198 
00199 /// isScalarToVector - Return true if the specified node is a
00200 /// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low
00201 /// element is not an undef.
00202 bool ISD::isScalarToVector(const SDNode *N) {
00203   if (N->getOpcode() == ISD::SCALAR_TO_VECTOR)
00204     return true;
00205 
00206   if (N->getOpcode() != ISD::BUILD_VECTOR)
00207     return false;
00208   if (N->getOperand(0).getOpcode() == ISD::UNDEF)
00209     return false;
00210   unsigned NumElems = N->getNumOperands();
00211   if (NumElems == 1)
00212     return false;
00213   for (unsigned i = 1; i < NumElems; ++i) {
00214     SDValue V = N->getOperand(i);
00215     if (V.getOpcode() != ISD::UNDEF)
00216       return false;
00217   }
00218   return true;
00219 }
00220 
00221 /// allOperandsUndef - Return true if the node has at least one operand
00222 /// and all operands of the specified node are ISD::UNDEF.
00223 bool ISD::allOperandsUndef(const SDNode *N) {
00224   // Return false if the node has no operands.
00225   // This is "logically inconsistent" with the definition of "all" but
00226   // is probably the desired behavior.
00227   if (N->getNumOperands() == 0)
00228     return false;
00229 
00230   for (unsigned i = 0, e = N->getNumOperands(); i != e ; ++i)
00231     if (N->getOperand(i).getOpcode() != ISD::UNDEF)
00232       return false;
00233 
00234   return true;
00235 }
00236 
00237 ISD::NodeType ISD::getExtForLoadExtType(ISD::LoadExtType ExtType) {
00238   switch (ExtType) {
00239   case ISD::EXTLOAD:
00240     return ISD::ANY_EXTEND;
00241   case ISD::SEXTLOAD:
00242     return ISD::SIGN_EXTEND;
00243   case ISD::ZEXTLOAD:
00244     return ISD::ZERO_EXTEND;
00245   default:
00246     break;
00247   }
00248 
00249   llvm_unreachable("Invalid LoadExtType");
00250 }
00251 
00252 /// getSetCCSwappedOperands - Return the operation corresponding to (Y op X)
00253 /// when given the operation for (X op Y).
00254 ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
00255   // To perform this operation, we just need to swap the L and G bits of the
00256   // operation.
00257   unsigned OldL = (Operation >> 2) & 1;
00258   unsigned OldG = (Operation >> 1) & 1;
00259   return ISD::CondCode((Operation & ~6) |  // Keep the N, U, E bits
00260                        (OldL << 1) |       // New G bit
00261                        (OldG << 2));       // New L bit.
00262 }
00263 
00264 /// getSetCCInverse - Return the operation corresponding to !(X op Y), where
00265 /// 'op' is a valid SetCC operation.
00266 ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) {
00267   unsigned Operation = Op;
00268   if (isInteger)
00269     Operation ^= 7;   // Flip L, G, E bits, but not U.
00270   else
00271     Operation ^= 15;  // Flip all of the condition bits.
00272 
00273   if (Operation > ISD::SETTRUE2)
00274     Operation &= ~8;  // Don't let N and U bits get set.
00275 
00276   return ISD::CondCode(Operation);
00277 }
00278 
00279 
00280 /// isSignedOp - For an integer comparison, return 1 if the comparison is a
00281 /// signed operation and 2 if the result is an unsigned comparison.  Return zero
00282 /// if the operation does not depend on the sign of the input (setne and seteq).
00283 static int isSignedOp(ISD::CondCode Opcode) {
00284   switch (Opcode) {
00285   default: llvm_unreachable("Illegal integer setcc operation!");
00286   case ISD::SETEQ:
00287   case ISD::SETNE: return 0;
00288   case ISD::SETLT:
00289   case ISD::SETLE:
00290   case ISD::SETGT:
00291   case ISD::SETGE: return 1;
00292   case ISD::SETULT:
00293   case ISD::SETULE:
00294   case ISD::SETUGT:
00295   case ISD::SETUGE: return 2;
00296   }
00297 }
00298 
00299 /// getSetCCOrOperation - Return the result of a logical OR between different
00300 /// comparisons of identical values: ((X op1 Y) | (X op2 Y)).  This function
00301 /// returns SETCC_INVALID if it is not possible to represent the resultant
00302 /// comparison.
00303 ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
00304                                        bool isInteger) {
00305   if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
00306     // Cannot fold a signed integer setcc with an unsigned integer setcc.
00307     return ISD::SETCC_INVALID;
00308 
00309   unsigned Op = Op1 | Op2;  // Combine all of the condition bits.
00310 
00311   // If the N and U bits get set then the resultant comparison DOES suddenly
00312   // care about orderedness, and is true when ordered.
00313   if (Op > ISD::SETTRUE2)
00314     Op &= ~16;     // Clear the U bit if the N bit is set.
00315 
00316   // Canonicalize illegal integer setcc's.
00317   if (isInteger && Op == ISD::SETUNE)  // e.g. SETUGT | SETULT
00318     Op = ISD::SETNE;
00319 
00320   return ISD::CondCode(Op);
00321 }
00322 
00323 /// getSetCCAndOperation - Return the result of a logical AND between different
00324 /// comparisons of identical values: ((X op1 Y) & (X op2 Y)).  This
00325 /// function returns zero if it is not possible to represent the resultant
00326 /// comparison.
00327 ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
00328                                         bool isInteger) {
00329   if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
00330     // Cannot fold a signed setcc with an unsigned setcc.
00331     return ISD::SETCC_INVALID;
00332 
00333   // Combine all of the condition bits.
00334   ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
00335 
00336   // Canonicalize illegal integer setcc's.
00337   if (isInteger) {
00338     switch (Result) {
00339     default: break;
00340     case ISD::SETUO : Result = ISD::SETFALSE; break;  // SETUGT & SETULT
00341     case ISD::SETOEQ:                                 // SETEQ  & SETU[LG]E
00342     case ISD::SETUEQ: Result = ISD::SETEQ   ; break;  // SETUGE & SETULE
00343     case ISD::SETOLT: Result = ISD::SETULT  ; break;  // SETULT & SETNE
00344     case ISD::SETOGT: Result = ISD::SETUGT  ; break;  // SETUGT & SETNE
00345     }
00346   }
00347 
00348   return Result;
00349 }
00350 
00351 //===----------------------------------------------------------------------===//
00352 //                           SDNode Profile Support
00353 //===----------------------------------------------------------------------===//
00354 
00355 /// AddNodeIDOpcode - Add the node opcode to the NodeID data.
00356 ///
00357 static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC)  {
00358   ID.AddInteger(OpC);
00359 }
00360 
00361 /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
00362 /// solely with their pointer.
00363 static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
00364   ID.AddPointer(VTList.VTs);
00365 }
00366 
00367 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
00368 ///
00369 static void AddNodeIDOperands(FoldingSetNodeID &ID,
00370                               ArrayRef<SDValue> Ops) {
00371   for (auto& Op : Ops) {
00372     ID.AddPointer(Op.getNode());
00373     ID.AddInteger(Op.getResNo());
00374   }
00375 }
00376 
00377 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
00378 ///
00379 static void AddNodeIDOperands(FoldingSetNodeID &ID,
00380                               ArrayRef<SDUse> Ops) {
00381   for (auto& Op : Ops) {
00382     ID.AddPointer(Op.getNode());
00383     ID.AddInteger(Op.getResNo());
00384   }
00385 }
00386 
00387 static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, bool nuw, bool nsw,
00388                                   bool exact) {
00389   ID.AddBoolean(nuw);
00390   ID.AddBoolean(nsw);
00391   ID.AddBoolean(exact);
00392 }
00393 
00394 /// AddBinaryNodeIDCustom - Add BinarySDNodes special infos
00395 static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, unsigned Opcode,
00396                                   bool nuw, bool nsw, bool exact) {
00397   if (isBinOpWithFlags(Opcode))
00398     AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
00399 }
00400 
00401 static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
00402                           SDVTList VTList, ArrayRef<SDValue> OpList) {
00403   AddNodeIDOpcode(ID, OpC);
00404   AddNodeIDValueTypes(ID, VTList);
00405   AddNodeIDOperands(ID, OpList);
00406 }
00407 
00408 /// AddNodeIDCustom - If this is an SDNode with special info, add this info to
00409 /// the NodeID data.
00410 static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
00411   switch (N->getOpcode()) {
00412   case ISD::TargetExternalSymbol:
00413   case ISD::ExternalSymbol:
00414     llvm_unreachable("Should only be used on nodes with operands");
00415   default: break;  // Normal nodes don't need extra info.
00416   case ISD::TargetConstant:
00417   case ISD::Constant: {
00418     const ConstantSDNode *C = cast<ConstantSDNode>(N);
00419     ID.AddPointer(C->getConstantIntValue());
00420     ID.AddBoolean(C->isOpaque());
00421     break;
00422   }
00423   case ISD::TargetConstantFP:
00424   case ISD::ConstantFP: {
00425     ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
00426     break;
00427   }
00428   case ISD::TargetGlobalAddress:
00429   case ISD::GlobalAddress:
00430   case ISD::TargetGlobalTLSAddress:
00431   case ISD::GlobalTLSAddress: {
00432     const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
00433     ID.AddPointer(GA->getGlobal());
00434     ID.AddInteger(GA->getOffset());
00435     ID.AddInteger(GA->getTargetFlags());
00436     ID.AddInteger(GA->getAddressSpace());
00437     break;
00438   }
00439   case ISD::BasicBlock:
00440     ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
00441     break;
00442   case ISD::Register:
00443     ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
00444     break;
00445   case ISD::RegisterMask:
00446     ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
00447     break;
00448   case ISD::SRCVALUE:
00449     ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
00450     break;
00451   case ISD::FrameIndex:
00452   case ISD::TargetFrameIndex:
00453     ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
00454     break;
00455   case ISD::JumpTable:
00456   case ISD::TargetJumpTable:
00457     ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
00458     ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
00459     break;
00460   case ISD::ConstantPool:
00461   case ISD::TargetConstantPool: {
00462     const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
00463     ID.AddInteger(CP->getAlignment());
00464     ID.AddInteger(CP->getOffset());
00465     if (CP->isMachineConstantPoolEntry())
00466       CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
00467     else
00468       ID.AddPointer(CP->getConstVal());
00469     ID.AddInteger(CP->getTargetFlags());
00470     break;
00471   }
00472   case ISD::TargetIndex: {
00473     const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
00474     ID.AddInteger(TI->getIndex());
00475     ID.AddInteger(TI->getOffset());
00476     ID.AddInteger(TI->getTargetFlags());
00477     break;
00478   }
00479   case ISD::LOAD: {
00480     const LoadSDNode *LD = cast<LoadSDNode>(N);
00481     ID.AddInteger(LD->getMemoryVT().getRawBits());
00482     ID.AddInteger(LD->getRawSubclassData());
00483     ID.AddInteger(LD->getPointerInfo().getAddrSpace());
00484     break;
00485   }
00486   case ISD::STORE: {
00487     const StoreSDNode *ST = cast<StoreSDNode>(N);
00488     ID.AddInteger(ST->getMemoryVT().getRawBits());
00489     ID.AddInteger(ST->getRawSubclassData());
00490     ID.AddInteger(ST->getPointerInfo().getAddrSpace());
00491     break;
00492   }
00493   case ISD::SDIV:
00494   case ISD::UDIV:
00495   case ISD::SRA:
00496   case ISD::SRL:
00497   case ISD::MUL:
00498   case ISD::ADD:
00499   case ISD::SUB:
00500   case ISD::SHL: {
00501     const BinaryWithFlagsSDNode *BinNode = cast<BinaryWithFlagsSDNode>(N);
00502     AddBinaryNodeIDCustom(ID, N->getOpcode(), BinNode->hasNoUnsignedWrap(),
00503                           BinNode->hasNoSignedWrap(), BinNode->isExact());
00504     break;
00505   }
00506   case ISD::ATOMIC_CMP_SWAP:
00507   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
00508   case ISD::ATOMIC_SWAP:
00509   case ISD::ATOMIC_LOAD_ADD:
00510   case ISD::ATOMIC_LOAD_SUB:
00511   case ISD::ATOMIC_LOAD_AND:
00512   case ISD::ATOMIC_LOAD_OR:
00513   case ISD::ATOMIC_LOAD_XOR:
00514   case ISD::ATOMIC_LOAD_NAND:
00515   case ISD::ATOMIC_LOAD_MIN:
00516   case ISD::ATOMIC_LOAD_MAX:
00517   case ISD::ATOMIC_LOAD_UMIN:
00518   case ISD::ATOMIC_LOAD_UMAX:
00519   case ISD::ATOMIC_LOAD:
00520   case ISD::ATOMIC_STORE: {
00521     const AtomicSDNode *AT = cast<AtomicSDNode>(N);
00522     ID.AddInteger(AT->getMemoryVT().getRawBits());
00523     ID.AddInteger(AT->getRawSubclassData());
00524     ID.AddInteger(AT->getPointerInfo().getAddrSpace());
00525     break;
00526   }
00527   case ISD::PREFETCH: {
00528     const MemSDNode *PF = cast<MemSDNode>(N);
00529     ID.AddInteger(PF->getPointerInfo().getAddrSpace());
00530     break;
00531   }
00532   case ISD::VECTOR_SHUFFLE: {
00533     const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
00534     for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
00535          i != e; ++i)
00536       ID.AddInteger(SVN->getMaskElt(i));
00537     break;
00538   }
00539   case ISD::TargetBlockAddress:
00540   case ISD::BlockAddress: {
00541     const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
00542     ID.AddPointer(BA->getBlockAddress());
00543     ID.AddInteger(BA->getOffset());
00544     ID.AddInteger(BA->getTargetFlags());
00545     break;
00546   }
00547   } // end switch (N->getOpcode())
00548 
00549   // Target specific memory nodes could also have address spaces to check.
00550   if (N->isTargetMemoryOpcode())
00551     ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
00552 }
00553 
00554 /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
00555 /// data.
00556 static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
00557   AddNodeIDOpcode(ID, N->getOpcode());
00558   // Add the return value info.
00559   AddNodeIDValueTypes(ID, N->getVTList());
00560   // Add the operand info.
00561   AddNodeIDOperands(ID, N->ops());
00562 
00563   // Handle SDNode leafs with special info.
00564   AddNodeIDCustom(ID, N);
00565 }
00566 
00567 /// encodeMemSDNodeFlags - Generic routine for computing a value for use in
00568 /// the CSE map that carries volatility, temporalness, indexing mode, and
00569 /// extension/truncation information.
00570 ///
00571 static inline unsigned
00572 encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile,
00573                      bool isNonTemporal, bool isInvariant) {
00574   assert((ConvType & 3) == ConvType &&
00575          "ConvType may not require more than 2 bits!");
00576   assert((AM & 7) == AM &&
00577          "AM may not require more than 3 bits!");
00578   return ConvType |
00579          (AM << 2) |
00580          (isVolatile << 5) |
00581          (isNonTemporal << 6) |
00582          (isInvariant << 7);
00583 }
00584 
00585 //===----------------------------------------------------------------------===//
00586 //                              SelectionDAG Class
00587 //===----------------------------------------------------------------------===//
00588 
00589 /// doNotCSE - Return true if CSE should not be performed for this node.
00590 static bool doNotCSE(SDNode *N) {
00591   if (N->getValueType(0) == MVT::Glue)
00592     return true; // Never CSE anything that produces a flag.
00593 
00594   switch (N->getOpcode()) {
00595   default: break;
00596   case ISD::HANDLENODE:
00597   case ISD::EH_LABEL:
00598     return true;   // Never CSE these nodes.
00599   }
00600 
00601   // Check that remaining values produced are not flags.
00602   for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
00603     if (N->getValueType(i) == MVT::Glue)
00604       return true; // Never CSE anything that produces a flag.
00605 
00606   return false;
00607 }
00608 
00609 /// RemoveDeadNodes - This method deletes all unreachable nodes in the
00610 /// SelectionDAG.
00611 void SelectionDAG::RemoveDeadNodes() {
00612   // Create a dummy node (which is not added to allnodes), that adds a reference
00613   // to the root node, preventing it from being deleted.
00614   HandleSDNode Dummy(getRoot());
00615 
00616   SmallVector<SDNode*, 128> DeadNodes;
00617 
00618   // Add all obviously-dead nodes to the DeadNodes worklist.
00619   for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I)
00620     if (I->use_empty())
00621       DeadNodes.push_back(I);
00622 
00623   RemoveDeadNodes(DeadNodes);
00624 
00625   // If the root changed (e.g. it was a dead load, update the root).
00626   setRoot(Dummy.getValue());
00627 }
00628 
00629 /// RemoveDeadNodes - This method deletes the unreachable nodes in the
00630 /// given list, and any nodes that become unreachable as a result.
00631 void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
00632 
00633   // Process the worklist, deleting the nodes and adding their uses to the
00634   // worklist.
00635   while (!DeadNodes.empty()) {
00636     SDNode *N = DeadNodes.pop_back_val();
00637 
00638     for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00639       DUL->NodeDeleted(N, nullptr);
00640 
00641     // Take the node out of the appropriate CSE map.
00642     RemoveNodeFromCSEMaps(N);
00643 
00644     // Next, brutally remove the operand list.  This is safe to do, as there are
00645     // no cycles in the graph.
00646     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
00647       SDUse &Use = *I++;
00648       SDNode *Operand = Use.getNode();
00649       Use.set(SDValue());
00650 
00651       // Now that we removed this operand, see if there are no uses of it left.
00652       if (Operand->use_empty())
00653         DeadNodes.push_back(Operand);
00654     }
00655 
00656     DeallocateNode(N);
00657   }
00658 }
00659 
00660 void SelectionDAG::RemoveDeadNode(SDNode *N){
00661   SmallVector<SDNode*, 16> DeadNodes(1, N);
00662 
00663   // Create a dummy node that adds a reference to the root node, preventing
00664   // it from being deleted.  (This matters if the root is an operand of the
00665   // dead node.)
00666   HandleSDNode Dummy(getRoot());
00667 
00668   RemoveDeadNodes(DeadNodes);
00669 }
00670 
00671 void SelectionDAG::DeleteNode(SDNode *N) {
00672   // First take this out of the appropriate CSE map.
00673   RemoveNodeFromCSEMaps(N);
00674 
00675   // Finally, remove uses due to operands of this node, remove from the
00676   // AllNodes list, and delete the node.
00677   DeleteNodeNotInCSEMaps(N);
00678 }
00679 
00680 void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
00681   assert(N != AllNodes.begin() && "Cannot delete the entry node!");
00682   assert(N->use_empty() && "Cannot delete a node that is not dead!");
00683 
00684   // Drop all of the operands and decrement used node's use counts.
00685   N->DropOperands();
00686 
00687   DeallocateNode(N);
00688 }
00689 
00690 void SelectionDAG::DeallocateNode(SDNode *N) {
00691   if (N->OperandsNeedDelete)
00692     delete[] N->OperandList;
00693 
00694   // Set the opcode to DELETED_NODE to help catch bugs when node
00695   // memory is reallocated.
00696   N->NodeType = ISD::DELETED_NODE;
00697 
00698   NodeAllocator.Deallocate(AllNodes.remove(N));
00699 
00700   // If any of the SDDbgValue nodes refer to this SDNode, invalidate them.
00701   ArrayRef<SDDbgValue*> DbgVals = DbgInfo->getSDDbgValues(N);
00702   for (unsigned i = 0, e = DbgVals.size(); i != e; ++i)
00703     DbgVals[i]->setIsInvalidated();
00704 }
00705 
00706 #ifndef NDEBUG
00707 /// VerifySDNode - Sanity check the given SDNode.  Aborts if it is invalid.
00708 static void VerifySDNode(SDNode *N) {
00709   switch (N->getOpcode()) {
00710   default:
00711     break;
00712   case ISD::BUILD_PAIR: {
00713     EVT VT = N->getValueType(0);
00714     assert(N->getNumValues() == 1 && "Too many results!");
00715     assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
00716            "Wrong return type!");
00717     assert(N->getNumOperands() == 2 && "Wrong number of operands!");
00718     assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
00719            "Mismatched operand types!");
00720     assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
00721            "Wrong operand type!");
00722     assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
00723            "Wrong return type size");
00724     break;
00725   }
00726   case ISD::BUILD_VECTOR: {
00727     assert(N->getNumValues() == 1 && "Too many results!");
00728     assert(N->getValueType(0).isVector() && "Wrong return type!");
00729     assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
00730            "Wrong number of operands!");
00731     EVT EltVT = N->getValueType(0).getVectorElementType();
00732     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
00733       assert((I->getValueType() == EltVT ||
00734              (EltVT.isInteger() && I->getValueType().isInteger() &&
00735               EltVT.bitsLE(I->getValueType()))) &&
00736             "Wrong operand type!");
00737       assert(I->getValueType() == N->getOperand(0).getValueType() &&
00738              "Operands must all have the same type");
00739     }
00740     break;
00741   }
00742   }
00743 }
00744 #endif // NDEBUG
00745 
00746 /// \brief Insert a newly allocated node into the DAG.
00747 ///
00748 /// Handles insertion into the all nodes list and CSE map, as well as
00749 /// verification and other common operations when a new node is allocated.
00750 void SelectionDAG::InsertNode(SDNode *N) {
00751   AllNodes.push_back(N);
00752 #ifndef NDEBUG
00753   VerifySDNode(N);
00754 #endif
00755 }
00756 
00757 /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
00758 /// correspond to it.  This is useful when we're about to delete or repurpose
00759 /// the node.  We don't want future request for structurally identical nodes
00760 /// to return N anymore.
00761 bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
00762   bool Erased = false;
00763   switch (N->getOpcode()) {
00764   case ISD::HANDLENODE: return false;  // noop.
00765   case ISD::CONDCODE:
00766     assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
00767            "Cond code doesn't exist!");
00768     Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
00769     CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
00770     break;
00771   case ISD::ExternalSymbol:
00772     Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
00773     break;
00774   case ISD::TargetExternalSymbol: {
00775     ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
00776     Erased = TargetExternalSymbols.erase(
00777                std::pair<std::string,unsigned char>(ESN->getSymbol(),
00778                                                     ESN->getTargetFlags()));
00779     break;
00780   }
00781   case ISD::VALUETYPE: {
00782     EVT VT = cast<VTSDNode>(N)->getVT();
00783     if (VT.isExtended()) {
00784       Erased = ExtendedValueTypeNodes.erase(VT);
00785     } else {
00786       Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
00787       ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
00788     }
00789     break;
00790   }
00791   default:
00792     // Remove it from the CSE Map.
00793     assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
00794     assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
00795     Erased = CSEMap.RemoveNode(N);
00796     break;
00797   }
00798 #ifndef NDEBUG
00799   // Verify that the node was actually in one of the CSE maps, unless it has a
00800   // flag result (which cannot be CSE'd) or is one of the special cases that are
00801   // not subject to CSE.
00802   if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
00803       !N->isMachineOpcode() && !doNotCSE(N)) {
00804     N->dump(this);
00805     dbgs() << "\n";
00806     llvm_unreachable("Node is not in map!");
00807   }
00808 #endif
00809   return Erased;
00810 }
00811 
00812 /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
00813 /// maps and modified in place. Add it back to the CSE maps, unless an identical
00814 /// node already exists, in which case transfer all its users to the existing
00815 /// node. This transfer can potentially trigger recursive merging.
00816 ///
00817 void
00818 SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
00819   // For node types that aren't CSE'd, just act as if no identical node
00820   // already exists.
00821   if (!doNotCSE(N)) {
00822     SDNode *Existing = CSEMap.GetOrInsertNode(N);
00823     if (Existing != N) {
00824       // If there was already an existing matching node, use ReplaceAllUsesWith
00825       // to replace the dead one with the existing one.  This can cause
00826       // recursive merging of other unrelated nodes down the line.
00827       ReplaceAllUsesWith(N, Existing);
00828 
00829       // N is now dead. Inform the listeners and delete it.
00830       for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00831         DUL->NodeDeleted(N, Existing);
00832       DeleteNodeNotInCSEMaps(N);
00833       return;
00834     }
00835   }
00836 
00837   // If the node doesn't already exist, we updated it.  Inform listeners.
00838   for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00839     DUL->NodeUpdated(N);
00840 }
00841 
00842 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00843 /// were replaced with those specified.  If this node is never memoized,
00844 /// return null, otherwise return a pointer to the slot it would take.  If a
00845 /// node already exists with these operands, the slot will be non-null.
00846 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
00847                                            void *&InsertPos) {
00848   if (doNotCSE(N))
00849     return nullptr;
00850 
00851   SDValue Ops[] = { Op };
00852   FoldingSetNodeID ID;
00853   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00854   AddNodeIDCustom(ID, N);
00855   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00856   return Node;
00857 }
00858 
00859 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00860 /// were replaced with those specified.  If this node is never memoized,
00861 /// return null, otherwise return a pointer to the slot it would take.  If a
00862 /// node already exists with these operands, the slot will be non-null.
00863 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
00864                                            SDValue Op1, SDValue Op2,
00865                                            void *&InsertPos) {
00866   if (doNotCSE(N))
00867     return nullptr;
00868 
00869   SDValue Ops[] = { Op1, Op2 };
00870   FoldingSetNodeID ID;
00871   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00872   AddNodeIDCustom(ID, N);
00873   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00874   return Node;
00875 }
00876 
00877 
00878 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00879 /// were replaced with those specified.  If this node is never memoized,
00880 /// return null, otherwise return a pointer to the slot it would take.  If a
00881 /// node already exists with these operands, the slot will be non-null.
00882 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
00883                                            void *&InsertPos) {
00884   if (doNotCSE(N))
00885     return nullptr;
00886 
00887   FoldingSetNodeID ID;
00888   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00889   AddNodeIDCustom(ID, N);
00890   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00891   return Node;
00892 }
00893 
00894 /// getEVTAlignment - Compute the default alignment value for the
00895 /// given type.
00896 ///
00897 unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
00898   Type *Ty = VT == MVT::iPTR ?
00899                    PointerType::get(Type::getInt8Ty(*getContext()), 0) :
00900                    VT.getTypeForEVT(*getContext());
00901 
00902   return TLI->getDataLayout()->getABITypeAlignment(Ty);
00903 }
00904 
00905 // EntryNode could meaningfully have debug info if we can find it...
00906 SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
00907     : TM(tm), TSI(nullptr), TLI(nullptr), OptLevel(OL),
00908       EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
00909       Root(getEntryNode()), NewNodesMustHaveLegalTypes(false),
00910       UpdateListeners(nullptr) {
00911   AllNodes.push_back(&EntryNode);
00912   DbgInfo = new SDDbgInfo();
00913 }
00914 
00915 void SelectionDAG::init(MachineFunction &mf) {
00916   MF = &mf;
00917   TLI = getSubtarget().getTargetLowering();
00918   TSI = getSubtarget().getSelectionDAGInfo();
00919   Context = &mf.getFunction()->getContext();
00920 }
00921 
00922 SelectionDAG::~SelectionDAG() {
00923   assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
00924   allnodes_clear();
00925   delete DbgInfo;
00926 }
00927 
00928 void SelectionDAG::allnodes_clear() {
00929   assert(&*AllNodes.begin() == &EntryNode);
00930   AllNodes.remove(AllNodes.begin());
00931   while (!AllNodes.empty())
00932     DeallocateNode(AllNodes.begin());
00933 }
00934 
00935 BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL,
00936                                             SDVTList VTs, SDValue N1,
00937                                             SDValue N2, bool nuw, bool nsw,
00938                                             bool exact) {
00939   if (isBinOpWithFlags(Opcode)) {
00940     BinaryWithFlagsSDNode *FN = new (NodeAllocator) BinaryWithFlagsSDNode(
00941         Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
00942     FN->setHasNoUnsignedWrap(nuw);
00943     FN->setHasNoSignedWrap(nsw);
00944     FN->setIsExact(exact);
00945 
00946     return FN;
00947   }
00948 
00949   BinarySDNode *N = new (NodeAllocator)
00950       BinarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
00951   return N;
00952 }
00953 
00954 void SelectionDAG::clear() {
00955   allnodes_clear();
00956   OperandAllocator.Reset();
00957   CSEMap.clear();
00958 
00959   ExtendedValueTypeNodes.clear();
00960   ExternalSymbols.clear();
00961   TargetExternalSymbols.clear();
00962   std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
00963             static_cast<CondCodeSDNode*>(nullptr));
00964   std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
00965             static_cast<SDNode*>(nullptr));
00966 
00967   EntryNode.UseList = nullptr;
00968   AllNodes.push_back(&EntryNode);
00969   Root = getEntryNode();
00970   DbgInfo->clear();
00971 }
00972 
00973 SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00974   return VT.bitsGT(Op.getValueType()) ?
00975     getNode(ISD::ANY_EXTEND, DL, VT, Op) :
00976     getNode(ISD::TRUNCATE, DL, VT, Op);
00977 }
00978 
00979 SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00980   return VT.bitsGT(Op.getValueType()) ?
00981     getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
00982     getNode(ISD::TRUNCATE, DL, VT, Op);
00983 }
00984 
00985 SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00986   return VT.bitsGT(Op.getValueType()) ?
00987     getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
00988     getNode(ISD::TRUNCATE, DL, VT, Op);
00989 }
00990 
00991 SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, SDLoc SL, EVT VT,
00992                                         EVT OpVT) {
00993   if (VT.bitsLE(Op.getValueType()))
00994     return getNode(ISD::TRUNCATE, SL, VT, Op);
00995 
00996   TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
00997   return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
00998 }
00999 
01000 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, SDLoc DL, EVT VT) {
01001   assert(!VT.isVector() &&
01002          "getZeroExtendInReg should use the vector element type instead of "
01003          "the vector type!");
01004   if (Op.getValueType() == VT) return Op;
01005   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01006   APInt Imm = APInt::getLowBitsSet(BitWidth,
01007                                    VT.getSizeInBits());
01008   return getNode(ISD::AND, DL, Op.getValueType(), Op,
01009                  getConstant(Imm, Op.getValueType()));
01010 }
01011 
01012 SDValue SelectionDAG::getAnyExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01013   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01014   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01015          "The sizes of the input and result must match in order to perform the "
01016          "extend in-register.");
01017   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01018          "The destination vector type must have fewer lanes than the input.");
01019   return getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Op);
01020 }
01021 
01022 SDValue SelectionDAG::getSignExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01023   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01024   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01025          "The sizes of the input and result must match in order to perform the "
01026          "extend in-register.");
01027   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01028          "The destination vector type must have fewer lanes than the input.");
01029   return getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Op);
01030 }
01031 
01032 SDValue SelectionDAG::getZeroExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01033   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01034   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01035          "The sizes of the input and result must match in order to perform the "
01036          "extend in-register.");
01037   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01038          "The destination vector type must have fewer lanes than the input.");
01039   return getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Op);
01040 }
01041 
01042 /// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
01043 ///
01044 SDValue SelectionDAG::getNOT(SDLoc DL, SDValue Val, EVT VT) {
01045   EVT EltVT = VT.getScalarType();
01046   SDValue NegOne =
01047     getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT);
01048   return getNode(ISD::XOR, DL, VT, Val, NegOne);
01049 }
01050 
01051 SDValue SelectionDAG::getLogicalNOT(SDLoc DL, SDValue Val, EVT VT) {
01052   EVT EltVT = VT.getScalarType();
01053   SDValue TrueValue;
01054   switch (TLI->getBooleanContents(VT)) {
01055     case TargetLowering::ZeroOrOneBooleanContent:
01056     case TargetLowering::UndefinedBooleanContent:
01057       TrueValue = getConstant(1, VT);
01058       break;
01059     case TargetLowering::ZeroOrNegativeOneBooleanContent:
01060       TrueValue = getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()),
01061                               VT);
01062       break;
01063   }
01064   return getNode(ISD::XOR, DL, VT, Val, TrueValue);
01065 }
01066 
01067 SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT, bool isO) {
01068   EVT EltVT = VT.getScalarType();
01069   assert((EltVT.getSizeInBits() >= 64 ||
01070          (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
01071          "getConstant with a uint64_t value that doesn't fit in the type!");
01072   return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT, isO);
01073 }
01074 
01075 SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT, bool isO)
01076 {
01077   return getConstant(*ConstantInt::get(*Context, Val), VT, isT, isO);
01078 }
01079 
01080 SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT,
01081                                   bool isO) {
01082   assert(VT.isInteger() && "Cannot create FP integer constant!");
01083 
01084   EVT EltVT = VT.getScalarType();
01085   const ConstantInt *Elt = &Val;
01086 
01087   // In some cases the vector type is legal but the element type is illegal and
01088   // needs to be promoted, for example v8i8 on ARM.  In this case, promote the
01089   // inserted value (the type does not need to match the vector element type).
01090   // Any extra bits introduced will be truncated away.
01091   if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
01092       TargetLowering::TypePromoteInteger) {
01093    EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
01094    APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits());
01095    Elt = ConstantInt::get(*getContext(), NewVal);
01096   }
01097   // In other cases the element type is illegal and needs to be expanded, for
01098   // example v2i64 on MIPS32. In this case, find the nearest legal type, split
01099   // the value into n parts and use a vector type with n-times the elements.
01100   // Then bitcast to the type requested.
01101   // Legalizing constants too early makes the DAGCombiner's job harder so we
01102   // only legalize if the DAG tells us we must produce legal types.
01103   else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
01104            TLI->getTypeAction(*getContext(), EltVT) ==
01105            TargetLowering::TypeExpandInteger) {
01106     APInt NewVal = Elt->getValue();
01107     EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
01108     unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
01109     unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
01110     EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
01111 
01112     // Check the temporary vector is the correct size. If this fails then
01113     // getTypeToTransformTo() probably returned a type whose size (in bits)
01114     // isn't a power-of-2 factor of the requested type size.
01115     assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
01116 
01117     SmallVector<SDValue, 2> EltParts;
01118     for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
01119       EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
01120                                            .trunc(ViaEltSizeInBits),
01121                                      ViaEltVT, isT, isO));
01122     }
01123 
01124     // EltParts is currently in little endian order. If we actually want
01125     // big-endian order then reverse it now.
01126     if (TLI->isBigEndian())
01127       std::reverse(EltParts.begin(), EltParts.end());
01128 
01129     // The elements must be reversed when the element order is different
01130     // to the endianness of the elements (because the BITCAST is itself a
01131     // vector shuffle in this situation). However, we do not need any code to
01132     // perform this reversal because getConstant() is producing a vector
01133     // splat.
01134     // This situation occurs in MIPS MSA.
01135 
01136     SmallVector<SDValue, 8> Ops;
01137     for (unsigned i = 0; i < VT.getVectorNumElements(); ++i)
01138       Ops.insert(Ops.end(), EltParts.begin(), EltParts.end());
01139 
01140     SDValue Result = getNode(ISD::BITCAST, SDLoc(), VT,
01141                              getNode(ISD::BUILD_VECTOR, SDLoc(), ViaVecVT,
01142                                      Ops));
01143     return Result;
01144   }
01145 
01146   assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
01147          "APInt size does not match type size!");
01148   unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
01149   FoldingSetNodeID ID;
01150   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
01151   ID.AddPointer(Elt);
01152   ID.AddBoolean(isO);
01153   void *IP = nullptr;
01154   SDNode *N = nullptr;
01155   if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
01156     if (!VT.isVector())
01157       return SDValue(N, 0);
01158 
01159   if (!N) {
01160     N = new (NodeAllocator) ConstantSDNode(isT, isO, Elt, EltVT);
01161     CSEMap.InsertNode(N, IP);
01162     InsertNode(N);
01163   }
01164 
01165   SDValue Result(N, 0);
01166   if (VT.isVector()) {
01167     SmallVector<SDValue, 8> Ops;
01168     Ops.assign(VT.getVectorNumElements(), Result);
01169     Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
01170   }
01171   return Result;
01172 }
01173 
01174 SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) {
01175   return getConstant(Val, TLI->getPointerTy(), isTarget);
01176 }
01177 
01178 
01179 SDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) {
01180   return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget);
01181 }
01182 
01183 SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){
01184   assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
01185 
01186   EVT EltVT = VT.getScalarType();
01187 
01188   // Do the map lookup using the actual bit pattern for the floating point
01189   // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
01190   // we don't have issues with SNANs.
01191   unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
01192   FoldingSetNodeID ID;
01193   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
01194   ID.AddPointer(&V);
01195   void *IP = nullptr;
01196   SDNode *N = nullptr;
01197   if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
01198     if (!VT.isVector())
01199       return SDValue(N, 0);
01200 
01201   if (!N) {
01202     N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT);
01203     CSEMap.InsertNode(N, IP);
01204     InsertNode(N);
01205   }
01206 
01207   SDValue Result(N, 0);
01208   if (VT.isVector()) {
01209     SmallVector<SDValue, 8> Ops;
01210     Ops.assign(VT.getVectorNumElements(), Result);
01211     // FIXME SDLoc info might be appropriate here
01212     Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
01213   }
01214   return Result;
01215 }
01216 
01217 SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) {
01218   EVT EltVT = VT.getScalarType();
01219   if (EltVT==MVT::f32)
01220     return getConstantFP(APFloat((float)Val), VT, isTarget);
01221   else if (EltVT==MVT::f64)
01222     return getConstantFP(APFloat(Val), VT, isTarget);
01223   else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 ||
01224            EltVT==MVT::f16) {
01225     bool ignored;
01226     APFloat apf = APFloat(Val);
01227     apf.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
01228                 &ignored);
01229     return getConstantFP(apf, VT, isTarget);
01230   } else
01231     llvm_unreachable("Unsupported type in getConstantFP");
01232 }
01233 
01234 SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, SDLoc DL,
01235                                        EVT VT, int64_t Offset,
01236                                        bool isTargetGA,
01237                                        unsigned char TargetFlags) {
01238   assert((TargetFlags == 0 || isTargetGA) &&
01239          "Cannot set target flags on target-independent globals");
01240 
01241   // Truncate (with sign-extension) the offset value to the pointer size.
01242   unsigned BitWidth = TLI->getPointerTypeSizeInBits(GV->getType());
01243   if (BitWidth < 64)
01244     Offset = SignExtend64(Offset, BitWidth);
01245 
01246   unsigned Opc;
01247   if (GV->isThreadLocal())
01248     Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
01249   else
01250     Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
01251 
01252   FoldingSetNodeID ID;
01253   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01254   ID.AddPointer(GV);
01255   ID.AddInteger(Offset);
01256   ID.AddInteger(TargetFlags);
01257   ID.AddInteger(GV->getType()->getAddressSpace());
01258   void *IP = nullptr;
01259   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01260     return SDValue(E, 0);
01261 
01262   SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL.getIROrder(),
01263                                                       DL.getDebugLoc(), GV, VT,
01264                                                       Offset, TargetFlags);
01265   CSEMap.InsertNode(N, IP);
01266     InsertNode(N);
01267   return SDValue(N, 0);
01268 }
01269 
01270 SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
01271   unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
01272   FoldingSetNodeID ID;
01273   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01274   ID.AddInteger(FI);
01275   void *IP = nullptr;
01276   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01277     return SDValue(E, 0);
01278 
01279   SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget);
01280   CSEMap.InsertNode(N, IP);
01281   InsertNode(N);
01282   return SDValue(N, 0);
01283 }
01284 
01285 SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
01286                                    unsigned char TargetFlags) {
01287   assert((TargetFlags == 0 || isTarget) &&
01288          "Cannot set target flags on target-independent jump tables");
01289   unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
01290   FoldingSetNodeID ID;
01291   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01292   ID.AddInteger(JTI);
01293   ID.AddInteger(TargetFlags);
01294   void *IP = nullptr;
01295   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01296     return SDValue(E, 0);
01297 
01298   SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget,
01299                                                   TargetFlags);
01300   CSEMap.InsertNode(N, IP);
01301   InsertNode(N);
01302   return SDValue(N, 0);
01303 }
01304 
01305 SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
01306                                       unsigned Alignment, int Offset,
01307                                       bool isTarget,
01308                                       unsigned char TargetFlags) {
01309   assert((TargetFlags == 0 || isTarget) &&
01310          "Cannot set target flags on target-independent globals");
01311   if (Alignment == 0)
01312     Alignment = TLI->getDataLayout()->getPrefTypeAlignment(C->getType());
01313   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
01314   FoldingSetNodeID ID;
01315   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01316   ID.AddInteger(Alignment);
01317   ID.AddInteger(Offset);
01318   ID.AddPointer(C);
01319   ID.AddInteger(TargetFlags);
01320   void *IP = nullptr;
01321   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01322     return SDValue(E, 0);
01323 
01324   SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
01325                                                      Alignment, TargetFlags);
01326   CSEMap.InsertNode(N, IP);
01327   InsertNode(N);
01328   return SDValue(N, 0);
01329 }
01330 
01331 
01332 SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
01333                                       unsigned Alignment, int Offset,
01334                                       bool isTarget,
01335                                       unsigned char TargetFlags) {
01336   assert((TargetFlags == 0 || isTarget) &&
01337          "Cannot set target flags on target-independent globals");
01338   if (Alignment == 0)
01339     Alignment = TLI->getDataLayout()->getPrefTypeAlignment(C->getType());
01340   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
01341   FoldingSetNodeID ID;
01342   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01343   ID.AddInteger(Alignment);
01344   ID.AddInteger(Offset);
01345   C->addSelectionDAGCSEId(ID);
01346   ID.AddInteger(TargetFlags);
01347   void *IP = nullptr;
01348   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01349     return SDValue(E, 0);
01350 
01351   SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
01352                                                      Alignment, TargetFlags);
01353   CSEMap.InsertNode(N, IP);
01354   InsertNode(N);
01355   return SDValue(N, 0);
01356 }
01357 
01358 SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
01359                                      unsigned char TargetFlags) {
01360   FoldingSetNodeID ID;
01361   AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None);
01362   ID.AddInteger(Index);
01363   ID.AddInteger(Offset);
01364   ID.AddInteger(TargetFlags);
01365   void *IP = nullptr;
01366   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01367     return SDValue(E, 0);
01368 
01369   SDNode *N = new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset,
01370                                                     TargetFlags);
01371   CSEMap.InsertNode(N, IP);
01372   InsertNode(N);
01373   return SDValue(N, 0);
01374 }
01375 
01376 SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
01377   FoldingSetNodeID ID;
01378   AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
01379   ID.AddPointer(MBB);
01380   void *IP = nullptr;
01381   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01382     return SDValue(E, 0);
01383 
01384   SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB);
01385   CSEMap.InsertNode(N, IP);
01386   InsertNode(N);
01387   return SDValue(N, 0);
01388 }
01389 
01390 SDValue SelectionDAG::getValueType(EVT VT) {
01391   if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
01392       ValueTypeNodes.size())
01393     ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
01394 
01395   SDNode *&N = VT.isExtended() ?
01396     ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
01397 
01398   if (N) return SDValue(N, 0);
01399   N = new (NodeAllocator) VTSDNode(VT);
01400   InsertNode(N);
01401   return SDValue(N, 0);
01402 }
01403 
01404 SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
01405   SDNode *&N = ExternalSymbols[Sym];
01406   if (N) return SDValue(N, 0);
01407   N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT);
01408   InsertNode(N);
01409   return SDValue(N, 0);
01410 }
01411 
01412 SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
01413                                               unsigned char TargetFlags) {
01414   SDNode *&N =
01415     TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym,
01416                                                                TargetFlags)];
01417   if (N) return SDValue(N, 0);
01418   N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT);
01419   InsertNode(N);
01420   return SDValue(N, 0);
01421 }
01422 
01423 SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
01424   if ((unsigned)Cond >= CondCodeNodes.size())
01425     CondCodeNodes.resize(Cond+1);
01426 
01427   if (!CondCodeNodes[Cond]) {
01428     CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond);
01429     CondCodeNodes[Cond] = N;
01430     InsertNode(N);
01431   }
01432 
01433   return SDValue(CondCodeNodes[Cond], 0);
01434 }
01435 
01436 // commuteShuffle - swaps the values of N1 and N2, and swaps all indices in
01437 // the shuffle mask M that point at N1 to point at N2, and indices that point
01438 // N2 to point at N1.
01439 static void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) {
01440   std::swap(N1, N2);
01441   int NElts = M.size();
01442   for (int i = 0; i != NElts; ++i) {
01443     if (M[i] >= NElts)
01444       M[i] -= NElts;
01445     else if (M[i] >= 0)
01446       M[i] += NElts;
01447   }
01448 }
01449 
01450 SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1,
01451                                        SDValue N2, const int *Mask) {
01452   assert(VT == N1.getValueType() && VT == N2.getValueType() &&
01453          "Invalid VECTOR_SHUFFLE");
01454 
01455   // Canonicalize shuffle undef, undef -> undef
01456   if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF)
01457     return getUNDEF(VT);
01458 
01459   // Validate that all indices in Mask are within the range of the elements
01460   // input to the shuffle.
01461   unsigned NElts = VT.getVectorNumElements();
01462   SmallVector<int, 8> MaskVec;
01463   for (unsigned i = 0; i != NElts; ++i) {
01464     assert(Mask[i] < (int)(NElts * 2) && "Index out of range");
01465     MaskVec.push_back(Mask[i]);
01466   }
01467 
01468   // Canonicalize shuffle v, v -> v, undef
01469   if (N1 == N2) {
01470     N2 = getUNDEF(VT);
01471     for (unsigned i = 0; i != NElts; ++i)
01472       if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts;
01473   }
01474 
01475   // Canonicalize shuffle undef, v -> v, undef.  Commute the shuffle mask.
01476   if (N1.getOpcode() == ISD::UNDEF)
01477     commuteShuffle(N1, N2, MaskVec);
01478 
01479   // Canonicalize all index into lhs, -> shuffle lhs, undef
01480   // Canonicalize all index into rhs, -> shuffle rhs, undef
01481   bool AllLHS = true, AllRHS = true;
01482   bool N2Undef = N2.getOpcode() == ISD::UNDEF;
01483   for (unsigned i = 0; i != NElts; ++i) {
01484     if (MaskVec[i] >= (int)NElts) {
01485       if (N2Undef)
01486         MaskVec[i] = -1;
01487       else
01488         AllLHS = false;
01489     } else if (MaskVec[i] >= 0) {
01490       AllRHS = false;
01491     }
01492   }
01493   if (AllLHS && AllRHS)
01494     return getUNDEF(VT);
01495   if (AllLHS && !N2Undef)
01496     N2 = getUNDEF(VT);
01497   if (AllRHS) {
01498     N1 = getUNDEF(VT);
01499     commuteShuffle(N1, N2, MaskVec);
01500   }
01501   // Reset our undef status after accounting for the mask.
01502   N2Undef = N2.getOpcode() == ISD::UNDEF;
01503   // Re-check whether both sides ended up undef.
01504   if (N1.getOpcode() == ISD::UNDEF && N2Undef)
01505     return getUNDEF(VT);
01506 
01507   // If Identity shuffle return that node.
01508   bool Identity = true;
01509   for (unsigned i = 0; i != NElts; ++i) {
01510     if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false;
01511   }
01512   if (Identity && NElts)
01513     return N1;
01514 
01515   // Shuffling a constant splat doesn't change the result.
01516   if (N2Undef) {
01517     SDValue V = N1;
01518 
01519     // Look through any bitcasts. We check that these don't change the number
01520     // (and size) of elements and just changes their types.
01521     while (V.getOpcode() == ISD::BITCAST)
01522       V = V->getOperand(0);
01523 
01524     // A splat should always show up as a build vector node.
01525     if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
01526       BitVector UndefElements;
01527       SDValue Splat = BV->getSplatValue(&UndefElements);
01528       // If this is a splat of an undef, shuffling it is also undef.
01529       if (Splat && Splat.getOpcode() == ISD::UNDEF)
01530         return getUNDEF(VT);
01531 
01532       // We only have a splat which can skip shuffles if there is a splatted
01533       // value and no undef lanes rearranged by the shuffle.
01534       if (Splat && UndefElements.none()) {
01535         // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
01536         // number of elements match or the value splatted is a zero constant.
01537         if (V.getValueType().getVectorNumElements() ==
01538             VT.getVectorNumElements())
01539           return N1;
01540         if (auto *C = dyn_cast<ConstantSDNode>(Splat))
01541           if (C->isNullValue())
01542             return N1;
01543       }
01544     }
01545   }
01546 
01547   FoldingSetNodeID ID;
01548   SDValue Ops[2] = { N1, N2 };
01549   AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
01550   for (unsigned i = 0; i != NElts; ++i)
01551     ID.AddInteger(MaskVec[i]);
01552 
01553   void* IP = nullptr;
01554   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01555     return SDValue(E, 0);
01556 
01557   // Allocate the mask array for the node out of the BumpPtrAllocator, since
01558   // SDNode doesn't have access to it.  This memory will be "leaked" when
01559   // the node is deallocated, but recovered when the NodeAllocator is released.
01560   int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
01561   memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int));
01562 
01563   ShuffleVectorSDNode *N =
01564     new (NodeAllocator) ShuffleVectorSDNode(VT, dl.getIROrder(),
01565                                             dl.getDebugLoc(), N1, N2,
01566                                             MaskAlloc);
01567   CSEMap.InsertNode(N, IP);
01568   InsertNode(N);
01569   return SDValue(N, 0);
01570 }
01571 
01572 SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
01573   MVT VT = SV.getSimpleValueType(0);
01574   unsigned NumElems = VT.getVectorNumElements();
01575   SmallVector<int, 8> MaskVec;
01576 
01577   for (unsigned i = 0; i != NumElems; ++i) {
01578     int Idx = SV.getMaskElt(i);
01579     if (Idx >= 0) {
01580       if (Idx < (int)NumElems)
01581         Idx += NumElems;
01582       else
01583         Idx -= NumElems;
01584     }
01585     MaskVec.push_back(Idx);
01586   }
01587 
01588   SDValue Op0 = SV.getOperand(0);
01589   SDValue Op1 = SV.getOperand(1);
01590   return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, &MaskVec[0]);
01591 }
01592 
01593 SDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl,
01594                                        SDValue Val, SDValue DTy,
01595                                        SDValue STy, SDValue Rnd, SDValue Sat,
01596                                        ISD::CvtCode Code) {
01597   // If the src and dest types are the same and the conversion is between
01598   // integer types of the same sign or two floats, no conversion is necessary.
01599   if (DTy == STy &&
01600       (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF))
01601     return Val;
01602 
01603   FoldingSetNodeID ID;
01604   SDValue Ops[] = { Val, DTy, STy, Rnd, Sat };
01605   AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), Ops);
01606   void* IP = nullptr;
01607   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01608     return SDValue(E, 0);
01609 
01610   CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl.getIROrder(),
01611                                                            dl.getDebugLoc(),
01612                                                            Ops, Code);
01613   CSEMap.InsertNode(N, IP);
01614   InsertNode(N);
01615   return SDValue(N, 0);
01616 }
01617 
01618 SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
01619   FoldingSetNodeID ID;
01620   AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
01621   ID.AddInteger(RegNo);
01622   void *IP = nullptr;
01623   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01624     return SDValue(E, 0);
01625 
01626   SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT);
01627   CSEMap.InsertNode(N, IP);
01628   InsertNode(N);
01629   return SDValue(N, 0);
01630 }
01631 
01632 SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
01633   FoldingSetNodeID ID;
01634   AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
01635   ID.AddPointer(RegMask);
01636   void *IP = nullptr;
01637   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01638     return SDValue(E, 0);
01639 
01640   SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask);
01641   CSEMap.InsertNode(N, IP);
01642   InsertNode(N);
01643   return SDValue(N, 0);
01644 }
01645 
01646 SDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) {
01647   FoldingSetNodeID ID;
01648   SDValue Ops[] = { Root };
01649   AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), Ops);
01650   ID.AddPointer(Label);
01651   void *IP = nullptr;
01652   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01653     return SDValue(E, 0);
01654 
01655   SDNode *N = new (NodeAllocator) EHLabelSDNode(dl.getIROrder(),
01656                                                 dl.getDebugLoc(), Root, Label);
01657   CSEMap.InsertNode(N, IP);
01658   InsertNode(N);
01659   return SDValue(N, 0);
01660 }
01661 
01662 
01663 SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
01664                                       int64_t Offset,
01665                                       bool isTarget,
01666                                       unsigned char TargetFlags) {
01667   unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
01668 
01669   FoldingSetNodeID ID;
01670   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01671   ID.AddPointer(BA);
01672   ID.AddInteger(Offset);
01673   ID.AddInteger(TargetFlags);
01674   void *IP = nullptr;
01675   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01676     return SDValue(E, 0);
01677 
01678   SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset,
01679                                                      TargetFlags);
01680   CSEMap.InsertNode(N, IP);
01681   InsertNode(N);
01682   return SDValue(N, 0);
01683 }
01684 
01685 SDValue SelectionDAG::getSrcValue(const Value *V) {
01686   assert((!V || V->getType()->isPointerTy()) &&
01687          "SrcValue is not a pointer?");
01688 
01689   FoldingSetNodeID ID;
01690   AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None);
01691   ID.AddPointer(V);
01692 
01693   void *IP = nullptr;
01694   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01695     return SDValue(E, 0);
01696 
01697   SDNode *N = new (NodeAllocator) SrcValueSDNode(V);
01698   CSEMap.InsertNode(N, IP);
01699   InsertNode(N);
01700   return SDValue(N, 0);
01701 }
01702 
01703 /// getMDNode - Return an MDNodeSDNode which holds an MDNode.
01704 SDValue SelectionDAG::getMDNode(const MDNode *MD) {
01705   FoldingSetNodeID ID;
01706   AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None);
01707   ID.AddPointer(MD);
01708 
01709   void *IP = nullptr;
01710   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01711     return SDValue(E, 0);
01712 
01713   SDNode *N = new (NodeAllocator) MDNodeSDNode(MD);
01714   CSEMap.InsertNode(N, IP);
01715   InsertNode(N);
01716   return SDValue(N, 0);
01717 }
01718 
01719 /// getAddrSpaceCast - Return an AddrSpaceCastSDNode.
01720 SDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr,
01721                                        unsigned SrcAS, unsigned DestAS) {
01722   SDValue Ops[] = {Ptr};
01723   FoldingSetNodeID ID;
01724   AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
01725   ID.AddInteger(SrcAS);
01726   ID.AddInteger(DestAS);
01727 
01728   void *IP = nullptr;
01729   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01730     return SDValue(E, 0);
01731 
01732   SDNode *N = new (NodeAllocator) AddrSpaceCastSDNode(dl.getIROrder(),
01733                                                       dl.getDebugLoc(),
01734                                                       VT, Ptr, SrcAS, DestAS);
01735   CSEMap.InsertNode(N, IP);
01736   InsertNode(N);
01737   return SDValue(N, 0);
01738 }
01739 
01740 /// getShiftAmountOperand - Return the specified value casted to
01741 /// the target's desired shift amount type.
01742 SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
01743   EVT OpTy = Op.getValueType();
01744   EVT ShTy = TLI->getShiftAmountTy(LHSTy);
01745   if (OpTy == ShTy || OpTy.isVector()) return Op;
01746 
01747   ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ?  ISD::TRUNCATE : ISD::ZERO_EXTEND;
01748   return getNode(Opcode, SDLoc(Op), ShTy, Op);
01749 }
01750 
01751 /// CreateStackTemporary - Create a stack temporary, suitable for holding the
01752 /// specified value type.
01753 SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
01754   MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
01755   unsigned ByteSize = VT.getStoreSize();
01756   Type *Ty = VT.getTypeForEVT(*getContext());
01757   unsigned StackAlign =
01758   std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty), minAlign);
01759 
01760   int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false);
01761   return getFrameIndex(FrameIdx, TLI->getPointerTy());
01762 }
01763 
01764 /// CreateStackTemporary - Create a stack temporary suitable for holding
01765 /// either of the specified value types.
01766 SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
01767   unsigned Bytes = std::max(VT1.getStoreSizeInBits(),
01768                             VT2.getStoreSizeInBits())/8;
01769   Type *Ty1 = VT1.getTypeForEVT(*getContext());
01770   Type *Ty2 = VT2.getTypeForEVT(*getContext());
01771   const DataLayout *TD = TLI->getDataLayout();
01772   unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1),
01773                             TD->getPrefTypeAlignment(Ty2));
01774 
01775   MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
01776   int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false);
01777   return getFrameIndex(FrameIdx, TLI->getPointerTy());
01778 }
01779 
01780 SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1,
01781                                 SDValue N2, ISD::CondCode Cond, SDLoc dl) {
01782   // These setcc operations always fold.
01783   switch (Cond) {
01784   default: break;
01785   case ISD::SETFALSE:
01786   case ISD::SETFALSE2: return getConstant(0, VT);
01787   case ISD::SETTRUE:
01788   case ISD::SETTRUE2: {
01789     TargetLowering::BooleanContent Cnt =
01790         TLI->getBooleanContents(N1->getValueType(0));
01791     return getConstant(
01792         Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, VT);
01793   }
01794 
01795   case ISD::SETOEQ:
01796   case ISD::SETOGT:
01797   case ISD::SETOGE:
01798   case ISD::SETOLT:
01799   case ISD::SETOLE:
01800   case ISD::SETONE:
01801   case ISD::SETO:
01802   case ISD::SETUO:
01803   case ISD::SETUEQ:
01804   case ISD::SETUNE:
01805     assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!");
01806     break;
01807   }
01808 
01809   if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) {
01810     const APInt &C2 = N2C->getAPIntValue();
01811     if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
01812       const APInt &C1 = N1C->getAPIntValue();
01813 
01814       switch (Cond) {
01815       default: llvm_unreachable("Unknown integer setcc!");
01816       case ISD::SETEQ:  return getConstant(C1 == C2, VT);
01817       case ISD::SETNE:  return getConstant(C1 != C2, VT);
01818       case ISD::SETULT: return getConstant(C1.ult(C2), VT);
01819       case ISD::SETUGT: return getConstant(C1.ugt(C2), VT);
01820       case ISD::SETULE: return getConstant(C1.ule(C2), VT);
01821       case ISD::SETUGE: return getConstant(C1.uge(C2), VT);
01822       case ISD::SETLT:  return getConstant(C1.slt(C2), VT);
01823       case ISD::SETGT:  return getConstant(C1.sgt(C2), VT);
01824       case ISD::SETLE:  return getConstant(C1.sle(C2), VT);
01825       case ISD::SETGE:  return getConstant(C1.sge(C2), VT);
01826       }
01827     }
01828   }
01829   if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) {
01830     if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) {
01831       APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF());
01832       switch (Cond) {
01833       default: break;
01834       case ISD::SETEQ:  if (R==APFloat::cmpUnordered)
01835                           return getUNDEF(VT);
01836                         // fall through
01837       case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT);
01838       case ISD::SETNE:  if (R==APFloat::cmpUnordered)
01839                           return getUNDEF(VT);
01840                         // fall through
01841       case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan ||
01842                                            R==APFloat::cmpLessThan, VT);
01843       case ISD::SETLT:  if (R==APFloat::cmpUnordered)
01844                           return getUNDEF(VT);
01845                         // fall through
01846       case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT);
01847       case ISD::SETGT:  if (R==APFloat::cmpUnordered)
01848                           return getUNDEF(VT);
01849                         // fall through
01850       case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT);
01851       case ISD::SETLE:  if (R==APFloat::cmpUnordered)
01852                           return getUNDEF(VT);
01853                         // fall through
01854       case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan ||
01855                                            R==APFloat::cmpEqual, VT);
01856       case ISD::SETGE:  if (R==APFloat::cmpUnordered)
01857                           return getUNDEF(VT);
01858                         // fall through
01859       case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan ||
01860                                            R==APFloat::cmpEqual, VT);
01861       case ISD::SETO:   return getConstant(R!=APFloat::cmpUnordered, VT);
01862       case ISD::SETUO:  return getConstant(R==APFloat::cmpUnordered, VT);
01863       case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered ||
01864                                            R==APFloat::cmpEqual, VT);
01865       case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT);
01866       case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered ||
01867                                            R==APFloat::cmpLessThan, VT);
01868       case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan ||
01869                                            R==APFloat::cmpUnordered, VT);
01870       case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT);
01871       case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT);
01872       }
01873     } else {
01874       // Ensure that the constant occurs on the RHS.
01875       ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
01876       MVT CompVT = N1.getValueType().getSimpleVT();
01877       if (!TLI->isCondCodeLegal(SwappedCond, CompVT))
01878         return SDValue();
01879 
01880       return getSetCC(dl, VT, N2, N1, SwappedCond);
01881     }
01882   }
01883 
01884   // Could not fold it.
01885   return SDValue();
01886 }
01887 
01888 /// SignBitIsZero - Return true if the sign bit of Op is known to be zero.  We
01889 /// use this predicate to simplify operations downstream.
01890 bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
01891   // This predicate is not safe for vector operations.
01892   if (Op.getValueType().isVector())
01893     return false;
01894 
01895   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01896   return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth);
01897 }
01898 
01899 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero.  We use
01900 /// this predicate to simplify operations downstream.  Mask is known to be zero
01901 /// for bits that V cannot have.
01902 bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask,
01903                                      unsigned Depth) const {
01904   APInt KnownZero, KnownOne;
01905   computeKnownBits(Op, KnownZero, KnownOne, Depth);
01906   return (KnownZero & Mask) == Mask;
01907 }
01908 
01909 /// Determine which bits of Op are known to be either zero or one and return
01910 /// them in the KnownZero/KnownOne bitsets.
01911 void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
01912                                     APInt &KnownOne, unsigned Depth) const {
01913   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01914 
01915   KnownZero = KnownOne = APInt(BitWidth, 0);   // Don't know anything.
01916   if (Depth == 6)
01917     return;  // Limit search depth.
01918 
01919   APInt KnownZero2, KnownOne2;
01920 
01921   switch (Op.getOpcode()) {
01922   case ISD::Constant:
01923     // We know all of the bits for a constant!
01924     KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue();
01925     KnownZero = ~KnownOne;
01926     break;
01927   case ISD::AND:
01928     // If either the LHS or the RHS are Zero, the result is zero.
01929     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01930     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01931 
01932     // Output known-1 bits are only known if set in both the LHS & RHS.
01933     KnownOne &= KnownOne2;
01934     // Output known-0 are known to be clear if zero in either the LHS | RHS.
01935     KnownZero |= KnownZero2;
01936     break;
01937   case ISD::OR:
01938     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01939     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01940 
01941     // Output known-0 bits are only known if clear in both the LHS & RHS.
01942     KnownZero &= KnownZero2;
01943     // Output known-1 are known to be set if set in either the LHS | RHS.
01944     KnownOne |= KnownOne2;
01945     break;
01946   case ISD::XOR: {
01947     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01948     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01949 
01950     // Output known-0 bits are known if clear or set in both the LHS & RHS.
01951     APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2);
01952     // Output known-1 are known to be set if set in only one of the LHS, RHS.
01953     KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2);
01954     KnownZero = KnownZeroOut;
01955     break;
01956   }
01957   case ISD::MUL: {
01958     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01959     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01960 
01961     // If low bits are zero in either operand, output low known-0 bits.
01962     // Also compute a conserative estimate for high known-0 bits.
01963     // More trickiness is possible, but this is sufficient for the
01964     // interesting case of alignment computation.
01965     KnownOne.clearAllBits();
01966     unsigned TrailZ = KnownZero.countTrailingOnes() +
01967                       KnownZero2.countTrailingOnes();
01968     unsigned LeadZ =  std::max(KnownZero.countLeadingOnes() +
01969                                KnownZero2.countLeadingOnes(),
01970                                BitWidth) - BitWidth;
01971 
01972     TrailZ = std::min(TrailZ, BitWidth);
01973     LeadZ = std::min(LeadZ, BitWidth);
01974     KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) |
01975                 APInt::getHighBitsSet(BitWidth, LeadZ);
01976     break;
01977   }
01978   case ISD::UDIV: {
01979     // For the purposes of computing leading zeros we can conservatively
01980     // treat a udiv as a logical right shift by the power of 2 known to
01981     // be less than the denominator.
01982     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01983     unsigned LeadZ = KnownZero2.countLeadingOnes();
01984 
01985     KnownOne2.clearAllBits();
01986     KnownZero2.clearAllBits();
01987     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
01988     unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros();
01989     if (RHSUnknownLeadingOnes != BitWidth)
01990       LeadZ = std::min(BitWidth,
01991                        LeadZ + BitWidth - RHSUnknownLeadingOnes - 1);
01992 
01993     KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ);
01994     break;
01995   }
01996   case ISD::SELECT:
01997     computeKnownBits(Op.getOperand(2), KnownZero, KnownOne, Depth+1);
01998     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
01999 
02000     // Only known if known in both the LHS and RHS.
02001     KnownOne &= KnownOne2;
02002     KnownZero &= KnownZero2;
02003     break;
02004   case ISD::SELECT_CC:
02005     computeKnownBits(Op.getOperand(3), KnownZero, KnownOne, Depth+1);
02006     computeKnownBits(Op.getOperand(2), KnownZero2, KnownOne2, Depth+1);
02007 
02008     // Only known if known in both the LHS and RHS.
02009     KnownOne &= KnownOne2;
02010     KnownZero &= KnownZero2;
02011     break;
02012   case ISD::SADDO:
02013   case ISD::UADDO:
02014   case ISD::SSUBO:
02015   case ISD::USUBO:
02016   case ISD::SMULO:
02017   case ISD::UMULO:
02018     if (Op.getResNo() != 1)
02019       break;
02020     // The boolean result conforms to getBooleanContents.
02021     // If we know the result of a setcc has the top bits zero, use this info.
02022     // We know that we have an integer-based boolean since these operations
02023     // are only available for integer.
02024     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
02025             TargetLowering::ZeroOrOneBooleanContent &&
02026         BitWidth > 1)
02027       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02028     break;
02029   case ISD::SETCC:
02030     // If we know the result of a setcc has the top bits zero, use this info.
02031     if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
02032             TargetLowering::ZeroOrOneBooleanContent &&
02033         BitWidth > 1)
02034       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02035     break;
02036   case ISD::SHL:
02037     // (shl X, C1) & C2 == 0   iff   (X & C2 >>u C1) == 0
02038     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02039       unsigned ShAmt = SA->getZExtValue();
02040 
02041       // If the shift count is an invalid immediate, don't do anything.
02042       if (ShAmt >= BitWidth)
02043         break;
02044 
02045       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02046       KnownZero <<= ShAmt;
02047       KnownOne  <<= ShAmt;
02048       // low bits known zero.
02049       KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt);
02050     }
02051     break;
02052   case ISD::SRL:
02053     // (ushr X, C1) & C2 == 0   iff  (-1 >> C1) & C2 == 0
02054     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02055       unsigned ShAmt = SA->getZExtValue();
02056 
02057       // If the shift count is an invalid immediate, don't do anything.
02058       if (ShAmt >= BitWidth)
02059         break;
02060 
02061       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02062       KnownZero = KnownZero.lshr(ShAmt);
02063       KnownOne  = KnownOne.lshr(ShAmt);
02064 
02065       APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
02066       KnownZero |= HighBits;  // High bits known zero.
02067     }
02068     break;
02069   case ISD::SRA:
02070     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02071       unsigned ShAmt = SA->getZExtValue();
02072 
02073       // If the shift count is an invalid immediate, don't do anything.
02074       if (ShAmt >= BitWidth)
02075         break;
02076 
02077       // If any of the demanded bits are produced by the sign extension, we also
02078       // demand the input sign bit.
02079       APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
02080 
02081       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02082       KnownZero = KnownZero.lshr(ShAmt);
02083       KnownOne  = KnownOne.lshr(ShAmt);
02084 
02085       // Handle the sign bits.
02086       APInt SignBit = APInt::getSignBit(BitWidth);
02087       SignBit = SignBit.lshr(ShAmt);  // Adjust to where it is now in the mask.
02088 
02089       if (KnownZero.intersects(SignBit)) {
02090         KnownZero |= HighBits;  // New bits are known zero.
02091       } else if (KnownOne.intersects(SignBit)) {
02092         KnownOne  |= HighBits;  // New bits are known one.
02093       }
02094     }
02095     break;
02096   case ISD::SIGN_EXTEND_INREG: {
02097     EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
02098     unsigned EBits = EVT.getScalarType().getSizeInBits();
02099 
02100     // Sign extension.  Compute the demanded bits in the result that are not
02101     // present in the input.
02102     APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits);
02103 
02104     APInt InSignBit = APInt::getSignBit(EBits);
02105     APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits);
02106 
02107     // If the sign extended bits are demanded, we know that the sign
02108     // bit is demanded.
02109     InSignBit = InSignBit.zext(BitWidth);
02110     if (NewBits.getBoolValue())
02111       InputDemandedBits |= InSignBit;
02112 
02113     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02114     KnownOne &= InputDemandedBits;
02115     KnownZero &= InputDemandedBits;
02116 
02117     // If the sign bit of the input is known set or clear, then we know the
02118     // top bits of the result.
02119     if (KnownZero.intersects(InSignBit)) {         // Input sign bit known clear
02120       KnownZero |= NewBits;
02121       KnownOne  &= ~NewBits;
02122     } else if (KnownOne.intersects(InSignBit)) {   // Input sign bit known set
02123       KnownOne  |= NewBits;
02124       KnownZero &= ~NewBits;
02125     } else {                              // Input sign bit unknown
02126       KnownZero &= ~NewBits;
02127       KnownOne  &= ~NewBits;
02128     }
02129     break;
02130   }
02131   case ISD::CTTZ:
02132   case ISD::CTTZ_ZERO_UNDEF:
02133   case ISD::CTLZ:
02134   case ISD::CTLZ_ZERO_UNDEF:
02135   case ISD::CTPOP: {
02136     unsigned LowBits = Log2_32(BitWidth)+1;
02137     KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits);
02138     KnownOne.clearAllBits();
02139     break;
02140   }
02141   case ISD::LOAD: {
02142     LoadSDNode *LD = cast<LoadSDNode>(Op);
02143     // If this is a ZEXTLoad and we are looking at the loaded value.
02144     if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
02145       EVT VT = LD->getMemoryVT();
02146       unsigned MemBits = VT.getScalarType().getSizeInBits();
02147       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
02148     } else if (const MDNode *Ranges = LD->getRanges()) {
02149       computeKnownBitsFromRangeMetadata(*Ranges, KnownZero);
02150     }
02151     break;
02152   }
02153   case ISD::ZERO_EXTEND: {
02154     EVT InVT = Op.getOperand(0).getValueType();
02155     unsigned InBits = InVT.getScalarType().getSizeInBits();
02156     APInt NewBits   = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
02157     KnownZero = KnownZero.trunc(InBits);
02158     KnownOne = KnownOne.trunc(InBits);
02159     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02160     KnownZero = KnownZero.zext(BitWidth);
02161     KnownOne = KnownOne.zext(BitWidth);
02162     KnownZero |= NewBits;
02163     break;
02164   }
02165   case ISD::SIGN_EXTEND: {
02166     EVT InVT = Op.getOperand(0).getValueType();
02167     unsigned InBits = InVT.getScalarType().getSizeInBits();
02168     APInt NewBits   = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
02169 
02170     KnownZero = KnownZero.trunc(InBits);
02171     KnownOne = KnownOne.trunc(InBits);
02172     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02173 
02174     // Note if the sign bit is known to be zero or one.
02175     bool SignBitKnownZero = KnownZero.isNegative();
02176     bool SignBitKnownOne  = KnownOne.isNegative();
02177 
02178     KnownZero = KnownZero.zext(BitWidth);
02179     KnownOne = KnownOne.zext(BitWidth);
02180 
02181     // If the sign bit is known zero or one, the top bits match.
02182     if (SignBitKnownZero)
02183       KnownZero |= NewBits;
02184     else if (SignBitKnownOne)
02185       KnownOne  |= NewBits;
02186     break;
02187   }
02188   case ISD::ANY_EXTEND: {
02189     EVT InVT = Op.getOperand(0).getValueType();
02190     unsigned InBits = InVT.getScalarType().getSizeInBits();
02191     KnownZero = KnownZero.trunc(InBits);
02192     KnownOne = KnownOne.trunc(InBits);
02193     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02194     KnownZero = KnownZero.zext(BitWidth);
02195     KnownOne = KnownOne.zext(BitWidth);
02196     break;
02197   }
02198   case ISD::TRUNCATE: {
02199     EVT InVT = Op.getOperand(0).getValueType();
02200     unsigned InBits = InVT.getScalarType().getSizeInBits();
02201     KnownZero = KnownZero.zext(InBits);
02202     KnownOne = KnownOne.zext(InBits);
02203     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02204     KnownZero = KnownZero.trunc(BitWidth);
02205     KnownOne = KnownOne.trunc(BitWidth);
02206     break;
02207   }
02208   case ISD::AssertZext: {
02209     EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
02210     APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
02211     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02212     KnownZero |= (~InMask);
02213     KnownOne  &= (~KnownZero);
02214     break;
02215   }
02216   case ISD::FGETSIGN:
02217     // All bits are zero except the low bit.
02218     KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02219     break;
02220 
02221   case ISD::SUB: {
02222     if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) {
02223       // We know that the top bits of C-X are clear if X contains less bits
02224       // than C (i.e. no wrap-around can happen).  For example, 20-X is
02225       // positive if we can prove that X is >= 0 and < 16.
02226       if (CLHS->getAPIntValue().isNonNegative()) {
02227         unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros();
02228         // NLZ can't be BitWidth with no sign bit
02229         APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1);
02230         computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02231 
02232         // If all of the MaskV bits are known to be zero, then we know the
02233         // output top bits are zero, because we now know that the output is
02234         // from [0-C].
02235         if ((KnownZero2 & MaskV) == MaskV) {
02236           unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros();
02237           // Top bits known zero.
02238           KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2);
02239         }
02240       }
02241     }
02242   }
02243   // fall through
02244   case ISD::ADD:
02245   case ISD::ADDE: {
02246     // Output known-0 bits are known if clear or set in both the low clear bits
02247     // common to both LHS & RHS.  For example, 8+(X<<3) is known to have the
02248     // low 3 bits clear.
02249     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
02250     unsigned KnownZeroOut = KnownZero2.countTrailingOnes();
02251 
02252     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02253     KnownZeroOut = std::min(KnownZeroOut,
02254                             KnownZero2.countTrailingOnes());
02255 
02256     if (Op.getOpcode() == ISD::ADD) {
02257       KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut);
02258       break;
02259     }
02260 
02261     // With ADDE, a carry bit may be added in, so we can only use this
02262     // information if we know (at least) that the low two bits are clear.  We
02263     // then return to the caller that the low bit is unknown but that other bits
02264     // are known zero.
02265     if (KnownZeroOut >= 2) // ADDE
02266       KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut);
02267     break;
02268   }
02269   case ISD::SREM:
02270     if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02271       const APInt &RA = Rem->getAPIntValue().abs();
02272       if (RA.isPowerOf2()) {
02273         APInt LowBits = RA - 1;
02274         computeKnownBits(Op.getOperand(0), KnownZero2,KnownOne2,Depth+1);
02275 
02276         // The low bits of the first operand are unchanged by the srem.
02277         KnownZero = KnownZero2 & LowBits;
02278         KnownOne = KnownOne2 & LowBits;
02279 
02280         // If the first operand is non-negative or has all low bits zero, then
02281         // the upper bits are all zero.
02282         if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits))
02283           KnownZero |= ~LowBits;
02284 
02285         // If the first operand is negative and not all low bits are zero, then
02286         // the upper bits are all one.
02287         if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0))
02288           KnownOne |= ~LowBits;
02289         assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?");
02290       }
02291     }
02292     break;
02293   case ISD::UREM: {
02294     if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02295       const APInt &RA = Rem->getAPIntValue();
02296       if (RA.isPowerOf2()) {
02297         APInt LowBits = (RA - 1);
02298         computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth + 1);
02299 
02300         // The upper bits are all zero, the lower ones are unchanged.
02301         KnownZero = KnownZero2 | ~LowBits;
02302         KnownOne = KnownOne2 & LowBits;
02303         break;
02304       }
02305     }
02306 
02307     // Since the result is less than or equal to either operand, any leading
02308     // zero bits in either operand must also exist in the result.
02309     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02310     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02311 
02312     uint32_t Leaders = std::max(KnownZero.countLeadingOnes(),
02313                                 KnownZero2.countLeadingOnes());
02314     KnownOne.clearAllBits();
02315     KnownZero = APInt::getHighBitsSet(BitWidth, Leaders);
02316     break;
02317   }
02318   case ISD::FrameIndex:
02319   case ISD::TargetFrameIndex:
02320     if (unsigned Align = InferPtrAlignment(Op)) {
02321       // The low bits are known zero if the pointer is aligned.
02322       KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align));
02323       break;
02324     }
02325     break;
02326 
02327   default:
02328     if (Op.getOpcode() < ISD::BUILTIN_OP_END)
02329       break;
02330     // Fallthrough
02331   case ISD::INTRINSIC_WO_CHAIN:
02332   case ISD::INTRINSIC_W_CHAIN:
02333   case ISD::INTRINSIC_VOID:
02334     // Allow the target to implement this method for its nodes.
02335     TLI->computeKnownBitsForTargetNode(Op, KnownZero, KnownOne, *this, Depth);
02336     break;
02337   }
02338 
02339   assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
02340 }
02341 
02342 /// ComputeNumSignBits - Return the number of times the sign bit of the
02343 /// register is replicated into the other bits.  We know that at least 1 bit
02344 /// is always equal to the sign bit (itself), but other cases can give us
02345 /// information.  For example, immediately after an "SRA X, 2", we know that
02346 /// the top 3 bits are all equal to each other, so we return 3.
02347 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{
02348   EVT VT = Op.getValueType();
02349   assert(VT.isInteger() && "Invalid VT!");
02350   unsigned VTBits = VT.getScalarType().getSizeInBits();
02351   unsigned Tmp, Tmp2;
02352   unsigned FirstAnswer = 1;
02353 
02354   if (Depth == 6)
02355     return 1;  // Limit search depth.
02356 
02357   switch (Op.getOpcode()) {
02358   default: break;
02359   case ISD::AssertSext:
02360     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
02361     return VTBits-Tmp+1;
02362   case ISD::AssertZext:
02363     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
02364     return VTBits-Tmp;
02365 
02366   case ISD::Constant: {
02367     const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue();
02368     return Val.getNumSignBits();
02369   }
02370 
02371   case ISD::SIGN_EXTEND:
02372     Tmp =
02373         VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits();
02374     return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp;
02375 
02376   case ISD::SIGN_EXTEND_INREG:
02377     // Max of the input and what this extends.
02378     Tmp =
02379       cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits();
02380     Tmp = VTBits-Tmp+1;
02381 
02382     Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02383     return std::max(Tmp, Tmp2);
02384 
02385   case ISD::SRA:
02386     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02387     // SRA X, C   -> adds C sign bits.
02388     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02389       Tmp += C->getZExtValue();
02390       if (Tmp > VTBits) Tmp = VTBits;
02391     }
02392     return Tmp;
02393   case ISD::SHL:
02394     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02395       // shl destroys sign bits.
02396       Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02397       if (C->getZExtValue() >= VTBits ||      // Bad shift.
02398           C->getZExtValue() >= Tmp) break;    // Shifted all sign bits out.
02399       return Tmp - C->getZExtValue();
02400     }
02401     break;
02402   case ISD::AND:
02403   case ISD::OR:
02404   case ISD::XOR:    // NOT is handled here.
02405     // Logical binary ops preserve the number of sign bits at the worst.
02406     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02407     if (Tmp != 1) {
02408       Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02409       FirstAnswer = std::min(Tmp, Tmp2);
02410       // We computed what we know about the sign bits as our first
02411       // answer. Now proceed to the generic code that uses
02412       // computeKnownBits, and pick whichever answer is better.
02413     }
02414     break;
02415 
02416   case ISD::SELECT:
02417     Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02418     if (Tmp == 1) return 1;  // Early out.
02419     Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1);
02420     return std::min(Tmp, Tmp2);
02421 
02422   case ISD::SADDO:
02423   case ISD::UADDO:
02424   case ISD::SSUBO:
02425   case ISD::USUBO:
02426   case ISD::SMULO:
02427   case ISD::UMULO:
02428     if (Op.getResNo() != 1)
02429       break;
02430     // The boolean result conforms to getBooleanContents.  Fall through.
02431     // If setcc returns 0/-1, all bits are sign bits.
02432     // We know that we have an integer-based boolean since these operations
02433     // are only available for integer.
02434     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
02435         TargetLowering::ZeroOrNegativeOneBooleanContent)
02436       return VTBits;
02437     break;
02438   case ISD::SETCC:
02439     // If setcc returns 0/-1, all bits are sign bits.
02440     if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
02441         TargetLowering::ZeroOrNegativeOneBooleanContent)
02442       return VTBits;
02443     break;
02444   case ISD::ROTL:
02445   case ISD::ROTR:
02446     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02447       unsigned RotAmt = C->getZExtValue() & (VTBits-1);
02448 
02449       // Handle rotate right by N like a rotate left by 32-N.
02450       if (Op.getOpcode() == ISD::ROTR)
02451         RotAmt = (VTBits-RotAmt) & (VTBits-1);
02452 
02453       // If we aren't rotating out all of the known-in sign bits, return the
02454       // number that are left.  This handles rotl(sext(x), 1) for example.
02455       Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02456       if (Tmp > RotAmt+1) return Tmp-RotAmt;
02457     }
02458     break;
02459   case ISD::ADD:
02460     // Add can have at most one carry bit.  Thus we know that the output
02461     // is, at worst, one more bit than the inputs.
02462     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02463     if (Tmp == 1) return 1;  // Early out.
02464 
02465     // Special case decrementing a value (ADD X, -1):
02466     if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
02467       if (CRHS->isAllOnesValue()) {
02468         APInt KnownZero, KnownOne;
02469         computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02470 
02471         // If the input is known to be 0 or 1, the output is 0/-1, which is all
02472         // sign bits set.
02473         if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
02474           return VTBits;
02475 
02476         // If we are subtracting one from a positive number, there is no carry
02477         // out of the result.
02478         if (KnownZero.isNegative())
02479           return Tmp;
02480       }
02481 
02482     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02483     if (Tmp2 == 1) return 1;
02484     return std::min(Tmp, Tmp2)-1;
02485 
02486   case ISD::SUB:
02487     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02488     if (Tmp2 == 1) return 1;
02489 
02490     // Handle NEG.
02491     if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0)))
02492       if (CLHS->isNullValue()) {
02493         APInt KnownZero, KnownOne;
02494         computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
02495         // If the input is known to be 0 or 1, the output is 0/-1, which is all
02496         // sign bits set.
02497         if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
02498           return VTBits;
02499 
02500         // If the input is known to be positive (the sign bit is known clear),
02501         // the output of the NEG has the same number of sign bits as the input.
02502         if (KnownZero.isNegative())
02503           return Tmp2;
02504 
02505         // Otherwise, we treat this like a SUB.
02506       }
02507 
02508     // Sub can have at most one carry bit.  Thus we know that the output
02509     // is, at worst, one more bit than the inputs.
02510     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02511     if (Tmp == 1) return 1;  // Early out.
02512     return std::min(Tmp, Tmp2)-1;
02513   case ISD::TRUNCATE:
02514     // FIXME: it's tricky to do anything useful for this, but it is an important
02515     // case for targets like X86.
02516     break;
02517   }
02518 
02519   // If we are looking at the loaded value of the SDNode.
02520   if (Op.getResNo() == 0) {
02521     // Handle LOADX separately here. EXTLOAD case will fallthrough.
02522     if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
02523       unsigned ExtType = LD->getExtensionType();
02524       switch (ExtType) {
02525         default: break;
02526         case ISD::SEXTLOAD:    // '17' bits known
02527           Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
02528           return VTBits-Tmp+1;
02529         case ISD::ZEXTLOAD:    // '16' bits known
02530           Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
02531           return VTBits-Tmp;
02532       }
02533     }
02534   }
02535 
02536   // Allow the target to implement this method for its nodes.
02537   if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
02538       Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
02539       Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
02540       Op.getOpcode() == ISD::INTRINSIC_VOID) {
02541     unsigned NumBits = TLI->ComputeNumSignBitsForTargetNode(Op, *this, Depth);
02542     if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits);
02543   }
02544 
02545   // Finally, if we can prove that the top bits of the result are 0's or 1's,
02546   // use this information.
02547   APInt KnownZero, KnownOne;
02548   computeKnownBits(Op, KnownZero, KnownOne, Depth);
02549 
02550   APInt Mask;
02551   if (KnownZero.isNegative()) {        // sign bit is 0
02552     Mask = KnownZero;
02553   } else if (KnownOne.isNegative()) {  // sign bit is 1;
02554     Mask = KnownOne;
02555   } else {
02556     // Nothing known.
02557     return FirstAnswer;
02558   }
02559 
02560   // Okay, we know that the sign bit in Mask is set.  Use CLZ to determine
02561   // the number of identical bits in the top of the input value.
02562   Mask = ~Mask;
02563   Mask <<= Mask.getBitWidth()-VTBits;
02564   // Return # leading zeros.  We use 'min' here in case Val was zero before
02565   // shifting.  We don't want to return '64' as for an i32 "0".
02566   return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros()));
02567 }
02568 
02569 /// isBaseWithConstantOffset - Return true if the specified operand is an
02570 /// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an
02571 /// ISD::OR with a ConstantSDNode that is guaranteed to have the same
02572 /// semantics as an ADD.  This handles the equivalence:
02573 ///     X|Cst == X+Cst iff X&Cst = 0.
02574 bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
02575   if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
02576       !isa<ConstantSDNode>(Op.getOperand(1)))
02577     return false;
02578 
02579   if (Op.getOpcode() == ISD::OR &&
02580       !MaskedValueIsZero(Op.getOperand(0),
02581                      cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue()))
02582     return false;
02583 
02584   return true;
02585 }
02586 
02587 
02588 bool SelectionDAG::isKnownNeverNaN(SDValue Op) const {
02589   // If we're told that NaNs won't happen, assume they won't.
02590   if (getTarget().Options.NoNaNsFPMath)
02591     return true;
02592 
02593   // If the value is a constant, we can obviously see if it is a NaN or not.
02594   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
02595     return !C->getValueAPF().isNaN();
02596 
02597   // TODO: Recognize more cases here.
02598 
02599   return false;
02600 }
02601 
02602 bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
02603   // If the value is a constant, we can obviously see if it is a zero or not.
02604   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
02605     return !C->isZero();
02606 
02607   // TODO: Recognize more cases here.
02608   switch (Op.getOpcode()) {
02609   default: break;
02610   case ISD::OR:
02611     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
02612       return !C->isNullValue();
02613     break;
02614   }
02615 
02616   return false;
02617 }
02618 
02619 bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
02620   // Check the obvious case.
02621   if (A == B) return true;
02622 
02623   // For for negative and positive zero.
02624   if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
02625     if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
02626       if (CA->isZero() && CB->isZero()) return true;
02627 
02628   // Otherwise they may not be equal.
02629   return false;
02630 }
02631 
02632 /// getNode - Gets or creates the specified node.
02633 ///
02634 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT) {
02635   FoldingSetNodeID ID;
02636   AddNodeIDNode(ID, Opcode, getVTList(VT), None);
02637   void *IP = nullptr;
02638   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
02639     return SDValue(E, 0);
02640 
02641   SDNode *N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(),
02642                                          DL.getDebugLoc(), getVTList(VT));
02643   CSEMap.InsertNode(N, IP);
02644 
02645   InsertNode(N);
02646   return SDValue(N, 0);
02647 }
02648 
02649 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
02650                               EVT VT, SDValue Operand) {
02651   // Constant fold unary operations with an integer constant operand. Even
02652   // opaque constant will be folded, because the folding of unary operations
02653   // doesn't create new constants with different values. Nevertheless, the
02654   // opaque flag is preserved during folding to prevent future folding with
02655   // other constants.
02656   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) {
02657     const APInt &Val = C->getAPIntValue();
02658     switch (Opcode) {
02659     default: break;
02660     case ISD::SIGN_EXTEND:
02661       return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT,
02662                          C->isTargetOpcode(), C->isOpaque());
02663     case ISD::ANY_EXTEND:
02664     case ISD::ZERO_EXTEND:
02665     case ISD::TRUNCATE:
02666       return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT,
02667                          C->isTargetOpcode(), C->isOpaque());
02668     case ISD::UINT_TO_FP:
02669     case ISD::SINT_TO_FP: {
02670       APFloat apf(EVTToAPFloatSemantics(VT),
02671                   APInt::getNullValue(VT.getSizeInBits()));
02672       (void)apf.convertFromAPInt(Val,
02673                                  Opcode==ISD::SINT_TO_FP,
02674                                  APFloat::rmNearestTiesToEven);
02675       return getConstantFP(apf, VT);
02676     }
02677     case ISD::BITCAST:
02678       if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
02679         return getConstantFP(APFloat(APFloat::IEEEsingle, Val), VT);
02680       else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
02681         return getConstantFP(APFloat(APFloat::IEEEdouble, Val), VT);
02682       break;
02683     case ISD::BSWAP:
02684       return getConstant(Val.byteSwap(), VT, C->isTargetOpcode(),
02685                          C->isOpaque());
02686     case ISD::CTPOP:
02687       return getConstant(Val.countPopulation(), VT, C->isTargetOpcode(),
02688                          C->isOpaque());
02689     case ISD::CTLZ:
02690     case ISD::CTLZ_ZERO_UNDEF:
02691       return getConstant(Val.countLeadingZeros(), VT, C->isTargetOpcode(),
02692                          C->isOpaque());
02693     case ISD::CTTZ:
02694     case ISD::CTTZ_ZERO_UNDEF:
02695       return getConstant(Val.countTrailingZeros(), VT, C->isTargetOpcode(),
02696                          C->isOpaque());
02697     }
02698   }
02699 
02700   // Constant fold unary operations with a floating point constant operand.
02701   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) {
02702     APFloat V = C->getValueAPF();    // make copy
02703     switch (Opcode) {
02704     case ISD::FNEG:
02705       V.changeSign();
02706       return getConstantFP(V, VT);
02707     case ISD::FABS:
02708       V.clearSign();
02709       return getConstantFP(V, VT);
02710     case ISD::FCEIL: {
02711       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
02712       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02713         return getConstantFP(V, VT);
02714       break;
02715     }
02716     case ISD::FTRUNC: {
02717       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
02718       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02719         return getConstantFP(V, VT);
02720       break;
02721     }
02722     case ISD::FFLOOR: {
02723       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
02724       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02725         return getConstantFP(V, VT);
02726       break;
02727     }
02728     case ISD::FP_EXTEND: {
02729       bool ignored;
02730       // This can return overflow, underflow, or inexact; we don't care.
02731       // FIXME need to be more flexible about rounding mode.
02732       (void)V.convert(EVTToAPFloatSemantics(VT),
02733                       APFloat::rmNearestTiesToEven, &ignored);
02734       return getConstantFP(V, VT);
02735     }
02736     case ISD::FP_TO_SINT:
02737     case ISD::FP_TO_UINT: {
02738       integerPart x[2];
02739       bool ignored;
02740       static_assert(integerPartWidth >= 64, "APFloat parts too small!");
02741       // FIXME need to be more flexible about rounding mode.
02742       APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(),
02743                             Opcode==ISD::FP_TO_SINT,
02744                             APFloat::rmTowardZero, &ignored);
02745       if (s==APFloat::opInvalidOp)     // inexact is OK, in fact usual
02746         break;
02747       APInt api(VT.getSizeInBits(), x);
02748       return getConstant(api, VT);
02749     }
02750     case ISD::BITCAST:
02751       if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
02752         return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT);
02753       else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
02754         return getConstant(V.bitcastToAPInt().getZExtValue(), VT);
02755       break;
02756     }
02757   }
02758 
02759   // Constant fold unary operations with a vector integer operand.
02760   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand.getNode())) {
02761     if (BV->isConstant()) {
02762       switch (Opcode) {
02763       default:
02764         // FIXME: Entirely reasonable to perform folding of other unary
02765         // operations here as the need arises.
02766         break;
02767       case ISD::UINT_TO_FP:
02768       case ISD::SINT_TO_FP: {
02769         SmallVector<SDValue, 8> Ops;
02770         for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
02771           SDValue OpN = BV->getOperand(i);
02772           // Let the above scalar folding handle the conversion of each
02773           // element.
02774           OpN = getNode(ISD::SINT_TO_FP, DL, VT.getVectorElementType(),
02775                         OpN);
02776           Ops.push_back(OpN);
02777         }
02778         return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
02779       }
02780       }
02781     }
02782   }
02783 
02784   unsigned OpOpcode = Operand.getNode()->getOpcode();
02785   switch (Opcode) {
02786   case ISD::TokenFactor:
02787   case ISD::MERGE_VALUES:
02788   case ISD::CONCAT_VECTORS:
02789     return Operand;         // Factor, merge or concat of one node?  No need.
02790   case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
02791   case ISD::FP_EXTEND:
02792     assert(VT.isFloatingPoint() &&
02793            Operand.getValueType().isFloatingPoint() && "Invalid FP cast!");
02794     if (Operand.getValueType() == VT) return Operand;  // noop conversion.
02795     assert((!VT.isVector() ||
02796             VT.getVectorNumElements() ==
02797             Operand.getValueType().getVectorNumElements()) &&
02798            "Vector element count mismatch!");
02799     if (Operand.getOpcode() == ISD::UNDEF)
02800       return getUNDEF(VT);
02801     break;
02802   case ISD::SIGN_EXTEND:
02803     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02804            "Invalid SIGN_EXTEND!");
02805     if (Operand.getValueType() == VT) return Operand;   // noop extension
02806     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02807            "Invalid sext node, dst < src!");
02808     assert((!VT.isVector() ||
02809             VT.getVectorNumElements() ==
02810             Operand.getValueType().getVectorNumElements()) &&
02811            "Vector element count mismatch!");
02812     if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
02813       return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02814     else if (OpOpcode == ISD::UNDEF)
02815       // sext(undef) = 0, because the top bits will all be the same.
02816       return getConstant(0, VT);
02817     break;
02818   case ISD::ZERO_EXTEND:
02819     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02820            "Invalid ZERO_EXTEND!");
02821     if (Operand.getValueType() == VT) return Operand;   // noop extension
02822     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02823            "Invalid zext node, dst < src!");
02824     assert((!VT.isVector() ||
02825             VT.getVectorNumElements() ==
02826             Operand.getValueType().getVectorNumElements()) &&
02827            "Vector element count mismatch!");
02828     if (OpOpcode == ISD::ZERO_EXTEND)   // (zext (zext x)) -> (zext x)
02829       return getNode(ISD::ZERO_EXTEND, DL, VT,
02830                      Operand.getNode()->getOperand(0));
02831     else if (OpOpcode == ISD::UNDEF)
02832       // zext(undef) = 0, because the top bits will be zero.
02833       return getConstant(0, VT);
02834     break;
02835   case ISD::ANY_EXTEND:
02836     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02837            "Invalid ANY_EXTEND!");
02838     if (Operand.getValueType() == VT) return Operand;   // noop extension
02839     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02840            "Invalid anyext node, dst < src!");
02841     assert((!VT.isVector() ||
02842             VT.getVectorNumElements() ==
02843             Operand.getValueType().getVectorNumElements()) &&
02844            "Vector element count mismatch!");
02845 
02846     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
02847         OpOpcode == ISD::ANY_EXTEND)
02848       // (ext (zext x)) -> (zext x)  and  (ext (sext x)) -> (sext x)
02849       return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02850     else if (OpOpcode == ISD::UNDEF)
02851       return getUNDEF(VT);
02852 
02853     // (ext (trunx x)) -> x
02854     if (OpOpcode == ISD::TRUNCATE) {
02855       SDValue OpOp = Operand.getNode()->getOperand(0);
02856       if (OpOp.getValueType() == VT)
02857         return OpOp;
02858     }
02859     break;
02860   case ISD::TRUNCATE:
02861     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02862            "Invalid TRUNCATE!");
02863     if (Operand.getValueType() == VT) return Operand;   // noop truncate
02864     assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) &&
02865            "Invalid truncate node, src < dst!");
02866     assert((!VT.isVector() ||
02867             VT.getVectorNumElements() ==
02868             Operand.getValueType().getVectorNumElements()) &&
02869            "Vector element count mismatch!");
02870     if (OpOpcode == ISD::TRUNCATE)
02871       return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
02872     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
02873         OpOpcode == ISD::ANY_EXTEND) {
02874       // If the source is smaller than the dest, we still need an extend.
02875       if (Operand.getNode()->getOperand(0).getValueType().getScalarType()
02876             .bitsLT(VT.getScalarType()))
02877         return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02878       if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT))
02879         return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
02880       return Operand.getNode()->getOperand(0);
02881     }
02882     if (OpOpcode == ISD::UNDEF)
02883       return getUNDEF(VT);
02884     break;
02885   case ISD::BITCAST:
02886     // Basic sanity checking.
02887     assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits()
02888            && "Cannot BITCAST between types of different sizes!");
02889     if (VT == Operand.getValueType()) return Operand;  // noop conversion.
02890     if (OpOpcode == ISD::BITCAST)  // bitconv(bitconv(x)) -> bitconv(x)
02891       return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
02892     if (OpOpcode == ISD::UNDEF)
02893       return getUNDEF(VT);
02894     break;
02895   case ISD::SCALAR_TO_VECTOR:
02896     assert(VT.isVector() && !Operand.getValueType().isVector() &&
02897            (VT.getVectorElementType() == Operand.getValueType() ||
02898             (VT.getVectorElementType().isInteger() &&
02899              Operand.getValueType().isInteger() &&
02900              VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&
02901            "Illegal SCALAR_TO_VECTOR node!");
02902     if (OpOpcode == ISD::UNDEF)
02903       return getUNDEF(VT);
02904     // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
02905     if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
02906         isa<ConstantSDNode>(Operand.getOperand(1)) &&
02907         Operand.getConstantOperandVal(1) == 0 &&
02908         Operand.getOperand(0).getValueType() == VT)
02909       return Operand.getOperand(0);
02910     break;
02911   case ISD::FNEG:
02912     // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0
02913     if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB)
02914       return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1),
02915                      Operand.getNode()->getOperand(0));
02916     if (OpOpcode == ISD::FNEG)  // --X -> X
02917       return Operand.getNode()->getOperand(0);
02918     break;
02919   case ISD::FABS:
02920     if (OpOpcode == ISD::FNEG)  // abs(-X) -> abs(X)
02921       return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0));
02922     break;
02923   }
02924 
02925   SDNode *N;
02926   SDVTList VTs = getVTList(VT);
02927   if (VT != MVT::Glue) { // Don't CSE flag producing nodes
02928     FoldingSetNodeID ID;
02929     SDValue Ops[1] = { Operand };
02930     AddNodeIDNode(ID, Opcode, VTs, Ops);
02931     void *IP = nullptr;
02932     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
02933       return SDValue(E, 0);
02934 
02935     N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
02936                                         DL.getDebugLoc(), VTs, Operand);
02937     CSEMap.InsertNode(N, IP);
02938   } else {
02939     N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
02940                                         DL.getDebugLoc(), VTs, Operand);
02941   }
02942 
02943   InsertNode(N);
02944   return SDValue(N, 0);
02945 }
02946 
02947 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT,
02948                                              SDNode *Cst1, SDNode *Cst2) {
02949   // If the opcode is a target-specific ISD node, there's nothing we can
02950   // do here and the operand rules may not line up with the below, so
02951   // bail early.
02952   if (Opcode >= ISD::BUILTIN_OP_END)
02953     return SDValue();
02954 
02955   SmallVector<std::pair<ConstantSDNode *, ConstantSDNode *>, 4> Inputs;
02956   SmallVector<SDValue, 4> Outputs;
02957   EVT SVT = VT.getScalarType();
02958 
02959   ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1);
02960   ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2);
02961   if (Scalar1 && Scalar2 && (Scalar1->isOpaque() || Scalar2->isOpaque()))
02962     return SDValue();
02963 
02964   if (Scalar1 && Scalar2)
02965     // Scalar instruction.
02966     Inputs.push_back(std::make_pair(Scalar1, Scalar2));
02967   else {
02968     // For vectors extract each constant element into Inputs so we can constant
02969     // fold them individually.
02970     BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1);
02971     BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2);
02972     if (!BV1 || !BV2)
02973       return SDValue();
02974 
02975     assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!");
02976 
02977     for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) {
02978       ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I));
02979       ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I));
02980       if (!V1 || !V2) // Not a constant, bail.
02981         return SDValue();
02982 
02983       if (V1->isOpaque() || V2->isOpaque())
02984         return SDValue();
02985 
02986       // Avoid BUILD_VECTOR nodes that perform implicit truncation.
02987       // FIXME: This is valid and could be handled by truncating the APInts.
02988       if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
02989         return SDValue();
02990 
02991       Inputs.push_back(std::make_pair(V1, V2));
02992     }
02993   }
02994 
02995   // We have a number of constant values, constant fold them element by element.
02996   for (unsigned I = 0, E = Inputs.size(); I != E; ++I) {
02997     const APInt &C1 = Inputs[I].first->getAPIntValue();
02998     const APInt &C2 = Inputs[I].second->getAPIntValue();
02999 
03000     switch (Opcode) {
03001     case ISD::ADD:
03002       Outputs.push_back(getConstant(C1 + C2, SVT));
03003       break;
03004     case ISD::SUB:
03005       Outputs.push_back(getConstant(C1 - C2, SVT));
03006       break;
03007     case ISD::MUL:
03008       Outputs.push_back(getConstant(C1 * C2, SVT));
03009       break;
03010     case ISD::UDIV:
03011       if (!C2.getBoolValue())
03012         return SDValue();
03013       Outputs.push_back(getConstant(C1.udiv(C2), SVT));
03014       break;
03015     case ISD::UREM:
03016       if (!C2.getBoolValue())
03017         return SDValue();
03018       Outputs.push_back(getConstant(C1.urem(C2), SVT));
03019       break;
03020     case ISD::SDIV:
03021       if (!C2.getBoolValue())
03022         return SDValue();
03023       Outputs.push_back(getConstant(C1.sdiv(C2), SVT));
03024       break;
03025     case ISD::SREM:
03026       if (!C2.getBoolValue())
03027         return SDValue();
03028       Outputs.push_back(getConstant(C1.srem(C2), SVT));
03029       break;
03030     case ISD::AND:
03031       Outputs.push_back(getConstant(C1 & C2, SVT));
03032       break;
03033     case ISD::OR:
03034       Outputs.push_back(getConstant(C1 | C2, SVT));
03035       break;
03036     case ISD::XOR:
03037       Outputs.push_back(getConstant(C1 ^ C2, SVT));
03038       break;
03039     case ISD::SHL:
03040       Outputs.push_back(getConstant(C1 << C2, SVT));
03041       break;
03042     case ISD::SRL:
03043       Outputs.push_back(getConstant(C1.lshr(C2), SVT));
03044       break;
03045     case ISD::SRA:
03046       Outputs.push_back(getConstant(C1.ashr(C2), SVT));
03047       break;
03048     case ISD::ROTL:
03049       Outputs.push_back(getConstant(C1.rotl(C2), SVT));
03050       break;
03051     case ISD::ROTR:
03052       Outputs.push_back(getConstant(C1.rotr(C2), SVT));
03053       break;
03054     default:
03055       return SDValue();
03056     }
03057   }
03058 
03059   assert((Scalar1 && Scalar2) || (VT.getVectorNumElements() == Outputs.size() &&
03060                                   "Expected a scalar or vector!"));
03061 
03062   // Handle the scalar case first.
03063   if (!VT.isVector())
03064     return Outputs.back();
03065 
03066   // We may have a vector type but a scalar result. Create a splat.
03067   Outputs.resize(VT.getVectorNumElements(), Outputs.back());
03068 
03069   // Build a big vector out of the scalar elements we generated.
03070   return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs);
03071 }
03072 
03073 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
03074                               SDValue N2, bool nuw, bool nsw, bool exact) {
03075   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
03076   ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode());
03077   switch (Opcode) {
03078   default: break;
03079   case ISD::TokenFactor:
03080     assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
03081            N2.getValueType() == MVT::Other && "Invalid token factor!");
03082     // Fold trivial token factors.
03083     if (N1.getOpcode() == ISD::EntryToken) return N2;
03084     if (N2.getOpcode() == ISD::EntryToken) return N1;
03085     if (N1 == N2) return N1;
03086     break;
03087   case ISD::CONCAT_VECTORS:
03088     // Concat of UNDEFs is UNDEF.
03089     if (N1.getOpcode() == ISD::UNDEF &&
03090         N2.getOpcode() == ISD::UNDEF)
03091       return getUNDEF(VT);
03092 
03093     // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
03094     // one big BUILD_VECTOR.
03095     if (N1.getOpcode() == ISD::BUILD_VECTOR &&
03096         N2.getOpcode() == ISD::BUILD_VECTOR) {
03097       SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
03098                                     N1.getNode()->op_end());
03099       Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
03100       return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
03101     }
03102     break;
03103   case ISD::AND:
03104     assert(VT.isInteger() && "This operator does not apply to FP types!");
03105     assert(N1.getValueType() == N2.getValueType() &&
03106            N1.getValueType() == VT && "Binary operator types must match!");
03107     // (X & 0) -> 0.  This commonly occurs when legalizing i64 values, so it's
03108     // worth handling here.
03109     if (N2C && N2C->isNullValue())
03110       return N2;
03111     if (N2C && N2C->isAllOnesValue())  // X & -1 -> X
03112       return N1;
03113     break;
03114   case ISD::OR:
03115   case ISD::XOR:
03116   case ISD::ADD:
03117   case ISD::SUB:
03118     assert(VT.isInteger() && "This operator does not apply to FP types!");
03119     assert(N1.getValueType() == N2.getValueType() &&
03120            N1.getValueType() == VT && "Binary operator types must match!");
03121     // (X ^|+- 0) -> X.  This commonly occurs when legalizing i64 values, so
03122     // it's worth handling here.
03123     if (N2C && N2C->isNullValue())
03124       return N1;
03125     break;
03126   case ISD::UDIV:
03127   case ISD::UREM:
03128   case ISD::MULHU:
03129   case ISD::MULHS:
03130   case ISD::MUL:
03131   case ISD::SDIV:
03132   case ISD::SREM:
03133     assert(VT.isInteger() && "This operator does not apply to FP types!");
03134     assert(N1.getValueType() == N2.getValueType() &&
03135            N1.getValueType() == VT && "Binary operator types must match!");
03136     break;
03137   case ISD::FADD:
03138   case ISD::FSUB:
03139   case ISD::FMUL:
03140   case ISD::FDIV:
03141   case ISD::FREM:
03142     if (getTarget().Options.UnsafeFPMath) {
03143       if (Opcode == ISD::FADD) {
03144         // 0+x --> x
03145         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1))
03146           if (CFP->getValueAPF().isZero())
03147             return N2;
03148         // x+0 --> x
03149         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
03150           if (CFP->getValueAPF().isZero())
03151             return N1;
03152       } else if (Opcode == ISD::FSUB) {
03153         // x-0 --> x
03154         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
03155           if (CFP->getValueAPF().isZero())
03156             return N1;
03157       } else if (Opcode == ISD::FMUL) {
03158         ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1);
03159         SDValue V = N2;
03160 
03161         // If the first operand isn't the constant, try the second
03162         if (!CFP) {
03163           CFP = dyn_cast<ConstantFPSDNode>(N2);
03164           V = N1;
03165         }
03166 
03167         if (CFP) {
03168           // 0*x --> 0
03169           if (CFP->isZero())
03170             return SDValue(CFP,0);
03171           // 1*x --> x
03172           if (CFP->isExactlyValue(1.0))
03173             return V;
03174         }
03175       }
03176     }
03177     assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
03178     assert(N1.getValueType() == N2.getValueType() &&
03179            N1.getValueType() == VT && "Binary operator types must match!");
03180     break;
03181   case ISD::FCOPYSIGN:   // N1 and result must match.  N1/N2 need not match.
03182     assert(N1.getValueType() == VT &&
03183            N1.getValueType().isFloatingPoint() &&
03184            N2.getValueType().isFloatingPoint() &&
03185            "Invalid FCOPYSIGN!");
03186     break;
03187   case ISD::SHL:
03188   case ISD::SRA:
03189   case ISD::SRL:
03190   case ISD::ROTL:
03191   case ISD::ROTR:
03192     assert(VT == N1.getValueType() &&
03193            "Shift operators return type must be the same as their first arg");
03194     assert(VT.isInteger() && N2.getValueType().isInteger() &&
03195            "Shifts only work on integers");
03196     assert((!VT.isVector() || VT == N2.getValueType()) &&
03197            "Vector shift amounts must be in the same as their first arg");
03198     // Verify that the shift amount VT is bit enough to hold valid shift
03199     // amounts.  This catches things like trying to shift an i1024 value by an
03200     // i8, which is easy to fall into in generic code that uses
03201     // TLI.getShiftAmount().
03202     assert(N2.getValueType().getSizeInBits() >=
03203                    Log2_32_Ceil(N1.getValueType().getSizeInBits()) &&
03204            "Invalid use of small shift amount with oversized value!");
03205 
03206     // Always fold shifts of i1 values so the code generator doesn't need to
03207     // handle them.  Since we know the size of the shift has to be less than the
03208     // size of the value, the shift/rotate count is guaranteed to be zero.
03209     if (VT == MVT::i1)
03210       return N1;
03211     if (N2C && N2C->isNullValue())
03212       return N1;
03213     break;
03214   case ISD::FP_ROUND_INREG: {
03215     EVT EVT = cast<VTSDNode>(N2)->getVT();
03216     assert(VT == N1.getValueType() && "Not an inreg round!");
03217     assert(VT.isFloatingPoint() && EVT.isFloatingPoint() &&
03218            "Cannot FP_ROUND_INREG integer types");
03219     assert(EVT.isVector() == VT.isVector() &&
03220            "FP_ROUND_INREG type should be vector iff the operand "
03221            "type is vector!");
03222     assert((!EVT.isVector() ||
03223             EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
03224            "Vector element counts must match in FP_ROUND_INREG");
03225     assert(EVT.bitsLE(VT) && "Not rounding down!");
03226     (void)EVT;
03227     if (cast<VTSDNode>(N2)->getVT() == VT) return N1;  // Not actually rounding.
03228     break;
03229   }
03230   case ISD::FP_ROUND:
03231     assert(VT.isFloatingPoint() &&
03232            N1.getValueType().isFloatingPoint() &&
03233            VT.bitsLE(N1.getValueType()) &&
03234            isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!");
03235     if (N1.getValueType() == VT) return N1;  // noop conversion.
03236     break;
03237   case ISD::AssertSext:
03238   case ISD::AssertZext: {
03239     EVT EVT = cast<VTSDNode>(N2)->getVT();
03240     assert(VT == N1.getValueType() && "Not an inreg extend!");
03241     assert(VT.isInteger() && EVT.isInteger() &&
03242            "Cannot *_EXTEND_INREG FP types");
03243     assert(!EVT.isVector() &&
03244            "AssertSExt/AssertZExt type should be the vector element type "
03245            "rather than the vector type!");
03246     assert(EVT.bitsLE(VT) && "Not extending!");
03247     if (VT == EVT) return N1; // noop assertion.
03248     break;
03249   }
03250   case ISD::SIGN_EXTEND_INREG: {
03251     EVT EVT = cast<VTSDNode>(N2)->getVT();
03252     assert(VT == N1.getValueType() && "Not an inreg extend!");
03253     assert(VT.isInteger() && EVT.isInteger() &&
03254            "Cannot *_EXTEND_INREG FP types");
03255     assert(EVT.isVector() == VT.isVector() &&
03256            "SIGN_EXTEND_INREG type should be vector iff the operand "
03257            "type is vector!");
03258     assert((!EVT.isVector() ||
03259             EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
03260            "Vector element counts must match in SIGN_EXTEND_INREG");
03261     assert(EVT.bitsLE(VT) && "Not extending!");
03262     if (EVT == VT) return N1;  // Not actually extending
03263 
03264     if (N1C) {
03265       APInt Val = N1C->getAPIntValue();
03266       unsigned FromBits = EVT.getScalarType().getSizeInBits();
03267       Val <<= Val.getBitWidth()-FromBits;
03268       Val = Val.ashr(Val.getBitWidth()-FromBits);
03269       return getConstant(Val, VT);
03270     }
03271     break;
03272   }
03273   case ISD::EXTRACT_VECTOR_ELT:
03274     // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF.
03275     if (N1.getOpcode() == ISD::UNDEF)
03276       return getUNDEF(VT);
03277 
03278     // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
03279     // expanding copies of large vectors from registers.
03280     if (N2C &&
03281         N1.getOpcode() == ISD::CONCAT_VECTORS &&
03282         N1.getNumOperands() > 0) {
03283       unsigned Factor =
03284         N1.getOperand(0).getValueType().getVectorNumElements();
03285       return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
03286                      N1.getOperand(N2C->getZExtValue() / Factor),
03287                      getConstant(N2C->getZExtValue() % Factor,
03288                                  N2.getValueType()));
03289     }
03290 
03291     // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
03292     // expanding large vector constants.
03293     if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) {
03294       SDValue Elt = N1.getOperand(N2C->getZExtValue());
03295 
03296       if (VT != Elt.getValueType())
03297         // If the vector element type is not legal, the BUILD_VECTOR operands
03298         // are promoted and implicitly truncated, and the result implicitly
03299         // extended. Make that explicit here.
03300         Elt = getAnyExtOrTrunc(Elt, DL, VT);
03301 
03302       return Elt;
03303     }
03304 
03305     // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
03306     // operations are lowered to scalars.
03307     if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
03308       // If the indices are the same, return the inserted element else
03309       // if the indices are known different, extract the element from
03310       // the original vector.
03311       SDValue N1Op2 = N1.getOperand(2);
03312       ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode());
03313 
03314       if (N1Op2C && N2C) {
03315         if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
03316           if (VT == N1.getOperand(1).getValueType())
03317             return N1.getOperand(1);
03318           else
03319             return getSExtOrTrunc(N1.getOperand(1), DL, VT);
03320         }
03321 
03322         return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
03323       }
03324     }
03325     break;
03326   case ISD::EXTRACT_ELEMENT:
03327     assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
03328     assert(!N1.getValueType().isVector() && !VT.isVector() &&
03329            (N1.getValueType().isInteger() == VT.isInteger()) &&
03330            N1.getValueType() != VT &&
03331            "Wrong types for EXTRACT_ELEMENT!");
03332 
03333     // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
03334     // 64-bit integers into 32-bit parts.  Instead of building the extract of
03335     // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
03336     if (N1.getOpcode() == ISD::BUILD_PAIR)
03337       return N1.getOperand(N2C->getZExtValue());
03338 
03339     // EXTRACT_ELEMENT of a constant int is also very common.
03340     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
03341       unsigned ElementSize = VT.getSizeInBits();
03342       unsigned Shift = ElementSize * N2C->getZExtValue();
03343       APInt ShiftedVal = C->getAPIntValue().lshr(Shift);
03344       return getConstant(ShiftedVal.trunc(ElementSize), VT);
03345     }
03346     break;
03347   case ISD::EXTRACT_SUBVECTOR: {
03348     SDValue Index = N2;
03349     if (VT.isSimple() && N1.getValueType().isSimple()) {
03350       assert(VT.isVector() && N1.getValueType().isVector() &&
03351              "Extract subvector VTs must be a vectors!");
03352       assert(VT.getVectorElementType() ==
03353              N1.getValueType().getVectorElementType() &&
03354              "Extract subvector VTs must have the same element type!");
03355       assert(VT.getSimpleVT() <= N1.getSimpleValueType() &&
03356              "Extract subvector must be from larger vector to smaller vector!");
03357 
03358       if (isa<ConstantSDNode>(Index.getNode())) {
03359         assert((VT.getVectorNumElements() +
03360                 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
03361                 <= N1.getValueType().getVectorNumElements())
03362                && "Extract subvector overflow!");
03363       }
03364 
03365       // Trivial extraction.
03366       if (VT.getSimpleVT() == N1.getSimpleValueType())
03367         return N1;
03368     }
03369     break;
03370   }
03371   }
03372 
03373   // Perform trivial constant folding.
03374   SDValue SV = FoldConstantArithmetic(Opcode, VT, N1.getNode(), N2.getNode());
03375   if (SV.getNode()) return SV;
03376 
03377   // Canonicalize constant to RHS if commutative.
03378   if (N1C && !N2C && isCommutativeBinOp(Opcode)) {
03379     std::swap(N1C, N2C);
03380     std::swap(N1, N2);
03381   }
03382 
03383   // Constant fold FP operations.
03384   bool HasFPExceptions = TLI->hasFloatingPointExceptions();
03385   ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode());
03386   ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode());
03387   if (N1CFP) {
03388     if (!N2CFP && isCommutativeBinOp(Opcode)) {
03389       // Canonicalize constant to RHS if commutative.
03390       std::swap(N1CFP, N2CFP);
03391       std::swap(N1, N2);
03392     } else if (N2CFP) {
03393       APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF();
03394       APFloat::opStatus s;
03395       switch (Opcode) {
03396       case ISD::FADD:
03397         s = V1.add(V2, APFloat::rmNearestTiesToEven);
03398         if (!HasFPExceptions || s != APFloat::opInvalidOp)
03399           return getConstantFP(V1, VT);
03400         break;
03401       case ISD::FSUB:
03402         s = V1.subtract(V2, APFloat::rmNearestTiesToEven);
03403         if (!HasFPExceptions || s!=APFloat::opInvalidOp)
03404           return getConstantFP(V1, VT);
03405         break;
03406       case ISD::FMUL:
03407         s = V1.multiply(V2, APFloat::rmNearestTiesToEven);
03408         if (!HasFPExceptions || s!=APFloat::opInvalidOp)
03409           return getConstantFP(V1, VT);
03410         break;
03411       case ISD::FDIV:
03412         s = V1.divide(V2, APFloat::rmNearestTiesToEven);
03413         if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
03414                                  s!=APFloat::opDivByZero)) {
03415           return getConstantFP(V1, VT);
03416         }
03417         break;
03418       case ISD::FREM :
03419         s = V1.mod(V2, APFloat::rmNearestTiesToEven);
03420         if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
03421                                  s!=APFloat::opDivByZero)) {
03422           return getConstantFP(V1, VT);
03423         }
03424         break;
03425       case ISD::FCOPYSIGN:
03426         V1.copySign(V2);
03427         return getConstantFP(V1, VT);
03428       default: break;
03429       }
03430     }
03431 
03432     if (Opcode == ISD::FP_ROUND) {
03433       APFloat V = N1CFP->getValueAPF();    // make copy
03434       bool ignored;
03435       // This can return overflow, underflow, or inexact; we don't care.
03436       // FIXME need to be more flexible about rounding mode.
03437       (void)V.convert(EVTToAPFloatSemantics(VT),
03438                       APFloat::rmNearestTiesToEven, &ignored);
03439       return getConstantFP(V, VT);
03440     }
03441   }
03442 
03443   // Canonicalize an UNDEF to the RHS, even over a constant.
03444   if (N1.getOpcode() == ISD::UNDEF) {
03445     if (isCommutativeBinOp(Opcode)) {
03446       std::swap(N1, N2);
03447     } else {
03448       switch (Opcode) {
03449       case ISD::FP_ROUND_INREG:
03450       case ISD::SIGN_EXTEND_INREG:
03451       case ISD::SUB:
03452       case ISD::FSUB:
03453       case ISD::FDIV:
03454       case ISD::FREM:
03455       case ISD::SRA:
03456         return N1;     // fold op(undef, arg2) -> undef
03457       case ISD::UDIV:
03458       case ISD::SDIV:
03459       case ISD::UREM:
03460       case ISD::SREM:
03461       case ISD::SRL:
03462       case ISD::SHL:
03463         if (!VT.isVector())
03464           return getConstant(0, VT);    // fold op(undef, arg2) -> 0
03465         // For vectors, we can't easily build an all zero vector, just return
03466         // the LHS.
03467         return N2;
03468       }
03469     }
03470   }
03471 
03472   // Fold a bunch of operators when the RHS is undef.
03473   if (N2.getOpcode() == ISD::UNDEF) {
03474     switch (Opcode) {
03475     case ISD::XOR:
03476       if (N1.getOpcode() == ISD::UNDEF)
03477         // Handle undef ^ undef -> 0 special case. This is a common
03478         // idiom (misuse).
03479         return getConstant(0, VT);
03480       // fallthrough
03481     case ISD::ADD:
03482     case ISD::ADDC:
03483     case ISD::ADDE:
03484     case ISD::SUB:
03485     case ISD::UDIV:
03486     case ISD::SDIV:
03487     case ISD::UREM:
03488     case ISD::SREM:
03489       return N2;       // fold op(arg1, undef) -> undef
03490     case ISD::FADD:
03491     case ISD::FSUB:
03492     case ISD::FMUL:
03493     case ISD::FDIV:
03494     case ISD::FREM:
03495       if (getTarget().Options.UnsafeFPMath)
03496         return N2;
03497       break;
03498     case ISD::MUL:
03499     case ISD::AND:
03500     case ISD::SRL:
03501     case ISD::SHL:
03502       if (!VT.isVector())
03503         return getConstant(0, VT);  // fold op(arg1, undef) -> 0
03504       // For vectors, we can't easily build an all zero vector, just return
03505       // the LHS.
03506       return N1;
03507     case ISD::OR:
03508       if (!VT.isVector())
03509         return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT);
03510       // For vectors, we can't easily build an all one vector, just return
03511       // the LHS.
03512       return N1;
03513     case ISD::SRA:
03514       return N1;
03515     }
03516   }
03517 
03518   // Memoize this node if possible.
03519   BinarySDNode *N;
03520   SDVTList VTs = getVTList(VT);
03521   const bool BinOpHasFlags = isBinOpWithFlags(Opcode);
03522   if (VT != MVT::Glue) {
03523     SDValue Ops[] = {N1, N2};
03524     FoldingSetNodeID ID;
03525     AddNodeIDNode(ID, Opcode, VTs, Ops);
03526     if (BinOpHasFlags)
03527       AddBinaryNodeIDCustom(ID, Opcode, nuw, nsw, exact);
03528     void *IP = nullptr;
03529     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
03530       return SDValue(E, 0);
03531 
03532     N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
03533 
03534     CSEMap.InsertNode(N, IP);
03535   } else {
03536 
03537     N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
03538   }
03539 
03540   InsertNode(N);
03541   return SDValue(N, 0);
03542 }
03543 
03544 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03545                               SDValue N1, SDValue N2, SDValue N3) {
03546   // Perform various simplifications.
03547   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
03548   switch (Opcode) {
03549   case ISD::FMA: {
03550     ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
03551     ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
03552     ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
03553     if (N1CFP && N2CFP && N3CFP) {
03554       APFloat  V1 = N1CFP->getValueAPF();
03555       const APFloat &V2 = N2CFP->getValueAPF();
03556       const APFloat &V3 = N3CFP->getValueAPF();
03557       APFloat::opStatus s =
03558         V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
03559       if (s != APFloat::opInvalidOp)
03560         return getConstantFP(V1, VT);
03561     }
03562     break;
03563   }
03564   case ISD::CONCAT_VECTORS:
03565     // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
03566     // one big BUILD_VECTOR.
03567     if (N1.getOpcode() == ISD::BUILD_VECTOR &&
03568         N2.getOpcode() == ISD::BUILD_VECTOR &&
03569         N3.getOpcode() == ISD::BUILD_VECTOR) {
03570       SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
03571                                     N1.getNode()->op_end());
03572       Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
03573       Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end());
03574       return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
03575     }
03576     break;
03577   case ISD::SETCC: {
03578     // Use FoldSetCC to simplify SETCC's.
03579     SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL);
03580     if (Simp.getNode()) return Simp;
03581     break;
03582   }
03583   case ISD::SELECT:
03584     if (N1C) {
03585      if (N1C->getZExtValue())
03586        return N2;             // select true, X, Y -> X
03587      return N3;             // select false, X, Y -> Y
03588     }
03589 
03590     if (N2 == N3) return N2;   // select C, X, X -> X
03591     break;
03592   case ISD::VECTOR_SHUFFLE:
03593     llvm_unreachable("should use getVectorShuffle constructor!");
03594   case ISD::INSERT_SUBVECTOR: {
03595     SDValue Index = N3;
03596     if (VT.isSimple() && N1.getValueType().isSimple()
03597         && N2.getValueType().isSimple()) {
03598       assert(VT.isVector() && N1.getValueType().isVector() &&
03599              N2.getValueType().isVector() &&
03600              "Insert subvector VTs must be a vectors");
03601       assert(VT == N1.getValueType() &&
03602              "Dest and insert subvector source types must match!");
03603       assert(N2.getSimpleValueType() <= N1.getSimpleValueType() &&
03604              "Insert subvector must be from smaller vector to larger vector!");
03605       if (isa<ConstantSDNode>(Index.getNode())) {
03606         assert((N2.getValueType().getVectorNumElements() +
03607                 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
03608                 <= VT.getVectorNumElements())
03609                && "Insert subvector overflow!");
03610       }
03611 
03612       // Trivial insertion.
03613       if (VT.getSimpleVT() == N2.getSimpleValueType())
03614         return N2;
03615     }
03616     break;
03617   }
03618   case ISD::BITCAST:
03619     // Fold bit_convert nodes from a type to themselves.
03620     if (N1.getValueType() == VT)
03621       return N1;
03622     break;
03623   }
03624 
03625   // Memoize node if it doesn't produce a flag.
03626   SDNode *N;
03627   SDVTList VTs = getVTList(VT);
03628   if (VT != MVT::Glue) {
03629     SDValue Ops[] = { N1, N2, N3 };
03630     FoldingSetNodeID ID;
03631     AddNodeIDNode(ID, Opcode, VTs, Ops);
03632     void *IP = nullptr;
03633     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
03634       return SDValue(E, 0);
03635 
03636     N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
03637                                           DL.getDebugLoc(), VTs, N1, N2, N3);
03638     CSEMap.InsertNode(N, IP);
03639   } else {
03640     N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
03641                                           DL.getDebugLoc(), VTs, N1, N2, N3);
03642   }
03643 
03644   InsertNode(N);
03645   return SDValue(N, 0);
03646 }
03647 
03648 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03649                               SDValue N1, SDValue N2, SDValue N3,
03650                               SDValue N4) {
03651   SDValue Ops[] = { N1, N2, N3, N4 };
03652   return getNode(Opcode, DL, VT, Ops);
03653 }
03654 
03655 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03656                               SDValue N1, SDValue N2, SDValue N3,
03657                               SDValue N4, SDValue N5) {
03658   SDValue Ops[] = { N1, N2, N3, N4, N5 };
03659   return getNode(Opcode, DL, VT, Ops);
03660 }
03661 
03662 /// getStackArgumentTokenFactor - Compute a TokenFactor to force all
03663 /// the incoming stack arguments to be loaded from the stack.
03664 SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
03665   SmallVector<SDValue, 8> ArgChains;
03666 
03667   // Include the original chain at the beginning of the list. When this is
03668   // used by target LowerCall hooks, this helps legalize find the
03669   // CALLSEQ_BEGIN node.
03670   ArgChains.push_back(Chain);
03671 
03672   // Add a chain value for each stack argument.
03673   for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(),
03674        UE = getEntryNode().getNode()->use_end(); U != UE; ++U)
03675     if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
03676       if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
03677         if (FI->getIndex() < 0)
03678           ArgChains.push_back(SDValue(L, 1));
03679 
03680   // Build a tokenfactor for all the chains.
03681   return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
03682 }
03683 
03684 /// getMemsetValue - Vectorized representation of the memset value
03685 /// operand.
03686 static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
03687                               SDLoc dl) {
03688   assert(Value.getOpcode() != ISD::UNDEF);
03689 
03690   unsigned NumBits = VT.getScalarType().getSizeInBits();
03691   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
03692     assert(C->getAPIntValue().getBitWidth() == 8);
03693     APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
03694     if (VT.isInteger())
03695       return DAG.getConstant(Val, VT);
03696     return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), VT);
03697   }
03698 
03699   Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value);
03700   if (NumBits > 8) {
03701     // Use a multiplication with 0x010101... to extend the input to the
03702     // required length.
03703     APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
03704     Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT));
03705   }
03706 
03707   return Value;
03708 }
03709 
03710 /// getMemsetStringVal - Similar to getMemsetValue. Except this is only
03711 /// used when a memcpy is turned into a memset when the source is a constant
03712 /// string ptr.
03713 static SDValue getMemsetStringVal(EVT VT, SDLoc dl, SelectionDAG &DAG,
03714                                   const TargetLowering &TLI, StringRef Str) {
03715   // Handle vector with all elements zero.
03716   if (Str.empty()) {
03717     if (VT.isInteger())
03718       return DAG.getConstant(0, VT);
03719     else if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
03720       return DAG.getConstantFP(0.0, VT);
03721     else if (VT.isVector()) {
03722       unsigned NumElts = VT.getVectorNumElements();
03723       MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
03724       return DAG.getNode(ISD::BITCAST, dl, VT,
03725                          DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(),
03726                                                              EltVT, NumElts)));
03727     } else
03728       llvm_unreachable("Expected type!");
03729   }
03730 
03731   assert(!VT.isVector() && "Can't handle vector type here!");
03732   unsigned NumVTBits = VT.getSizeInBits();
03733   unsigned NumVTBytes = NumVTBits / 8;
03734   unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size()));
03735 
03736   APInt Val(NumVTBits, 0);
03737   if (TLI.isLittleEndian()) {
03738     for (unsigned i = 0; i != NumBytes; ++i)
03739       Val |= (uint64_t)(unsigned char)Str[i] << i*8;
03740   } else {
03741     for (unsigned i = 0; i != NumBytes; ++i)
03742       Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8;
03743   }
03744 
03745   // If the "cost" of materializing the integer immediate is less than the cost
03746   // of a load, then it is cost effective to turn the load into the immediate.
03747   Type *Ty = VT.getTypeForEVT(*DAG.getContext());
03748   if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
03749     return DAG.getConstant(Val, VT);
03750   return SDValue(nullptr, 0);
03751 }
03752 
03753 /// getMemBasePlusOffset - Returns base and offset node for the
03754 ///
03755 static SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, SDLoc dl,
03756                                       SelectionDAG &DAG) {
03757   EVT VT = Base.getValueType();
03758   return DAG.getNode(ISD::ADD, dl,
03759                      VT, Base, DAG.getConstant(Offset, VT));
03760 }
03761 
03762 /// isMemSrcFromString - Returns true if memcpy source is a string constant.
03763 ///
03764 static bool isMemSrcFromString(SDValue Src, StringRef &Str) {
03765   unsigned SrcDelta = 0;
03766   GlobalAddressSDNode *G = nullptr;
03767   if (Src.getOpcode() == ISD::GlobalAddress)
03768     G = cast<GlobalAddressSDNode>(Src);
03769   else if (Src.getOpcode() == ISD::ADD &&
03770            Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
03771            Src.getOperand(1).getOpcode() == ISD::Constant) {
03772     G = cast<GlobalAddressSDNode>(Src.getOperand(0));
03773     SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue();
03774   }
03775   if (!G)
03776     return false;
03777 
03778   return getConstantStringInfo(G->getGlobal(), Str, SrcDelta, false);
03779 }
03780 
03781 /// FindOptimalMemOpLowering - Determines the optimial series memory ops
03782 /// to replace the memset / memcpy. Return true if the number of memory ops
03783 /// is below the threshold. It returns the types of the sequence of
03784 /// memory ops to perform memset / memcpy by reference.
03785 static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
03786                                      unsigned Limit, uint64_t Size,
03787                                      unsigned DstAlign, unsigned SrcAlign,
03788                                      bool IsMemset,
03789                                      bool ZeroMemset,
03790                                      bool MemcpyStrSrc,
03791                                      bool AllowOverlap,
03792                                      SelectionDAG &DAG,
03793                                      const TargetLowering &TLI) {
03794   assert((SrcAlign == 0 || SrcAlign >= DstAlign) &&
03795          "Expecting memcpy / memset source to meet alignment requirement!");
03796   // If 'SrcAlign' is zero, that means the memory operation does not need to
03797   // load the value, i.e. memset or memcpy from constant string. Otherwise,
03798   // it's the inferred alignment of the source. 'DstAlign', on the other hand,
03799   // is the specified alignment of the memory operation. If it is zero, that
03800   // means it's possible to change the alignment of the destination.
03801   // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does
03802   // not need to be loaded.
03803   EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign,
03804                                    IsMemset, ZeroMemset, MemcpyStrSrc,
03805                                    DAG.getMachineFunction());
03806 
03807   if (VT == MVT::Other) {
03808     unsigned AS = 0;
03809     if (DstAlign >= TLI.getDataLayout()->getPointerPrefAlignment(AS) ||
03810         TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign)) {
03811       VT = TLI.getPointerTy();
03812     } else {
03813       switch (DstAlign & 7) {
03814       case 0:  VT = MVT::i64; break;
03815       case 4:  VT = MVT::i32; break;
03816       case 2:  VT = MVT::i16; break;
03817       default: VT = MVT::i8;  break;
03818       }
03819     }
03820 
03821     MVT LVT = MVT::i64;
03822     while (!TLI.isTypeLegal(LVT))
03823       LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
03824     assert(LVT.isInteger());
03825 
03826     if (VT.bitsGT(LVT))
03827       VT = LVT;
03828   }
03829 
03830   unsigned NumMemOps = 0;
03831   while (Size != 0) {
03832     unsigned VTSize = VT.getSizeInBits() / 8;
03833     while (VTSize > Size) {
03834       // For now, only use non-vector load / store's for the left-over pieces.
03835       EVT NewVT = VT;
03836       unsigned NewVTSize;
03837 
03838       bool Found = false;
03839       if (VT.isVector() || VT.isFloatingPoint()) {
03840         NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
03841         if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) &&
03842             TLI.isSafeMemOpType(NewVT.getSimpleVT()))
03843           Found = true;
03844         else if (NewVT == MVT::i64 &&
03845                  TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
03846                  TLI.isSafeMemOpType(MVT::f64)) {
03847           // i64 is usually not legal on 32-bit targets, but f64 may be.
03848           NewVT = MVT::f64;
03849           Found = true;
03850         }
03851       }
03852 
03853       if (!Found) {
03854         do {
03855           NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
03856           if (NewVT == MVT::i8)
03857             break;
03858         } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT()));
03859       }
03860       NewVTSize = NewVT.getSizeInBits() / 8;
03861 
03862       // If the new VT cannot cover all of the remaining bits, then consider
03863       // issuing a (or a pair of) unaligned and overlapping load / store.
03864       // FIXME: Only does this for 64-bit or more since we don't have proper
03865       // cost model for unaligned load / store.
03866       bool Fast;
03867       unsigned AS = 0;
03868       if (NumMemOps && AllowOverlap &&
03869           VTSize >= 8 && NewVTSize < Size &&
03870           TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign, &Fast) && Fast)
03871         VTSize = Size;
03872       else {
03873         VT = NewVT;
03874         VTSize = NewVTSize;
03875       }
03876     }
03877 
03878     if (++NumMemOps > Limit)
03879       return false;
03880 
03881     MemOps.push_back(VT);
03882     Size -= VTSize;
03883   }
03884 
03885   return true;
03886 }
03887 
03888 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
03889                                        SDValue Chain, SDValue Dst,
03890                                        SDValue Src, uint64_t Size,
03891                                        unsigned Align, bool isVol,
03892                                        bool AlwaysInline,
03893                                        MachinePointerInfo DstPtrInfo,
03894                                        MachinePointerInfo SrcPtrInfo) {
03895   // Turn a memcpy of undef to nop.
03896   if (Src.getOpcode() == ISD::UNDEF)
03897     return Chain;
03898 
03899   // Expand memcpy to a series of load and store ops if the size operand falls
03900   // below a certain threshold.
03901   // TODO: In the AlwaysInline case, if the size is big then generate a loop
03902   // rather than maybe a humongous number of loads and stores.
03903   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03904   std::vector<EVT> MemOps;
03905   bool DstAlignCanChange = false;
03906   MachineFunction &MF = DAG.getMachineFunction();
03907   MachineFrameInfo *MFI = MF.getFrameInfo();
03908   bool OptSize =
03909     MF.getFunction()->getAttributes().
03910       hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
03911   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
03912   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
03913     DstAlignCanChange = true;
03914   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
03915   if (Align > SrcAlign)
03916     SrcAlign = Align;
03917   StringRef Str;
03918   bool CopyFromStr = isMemSrcFromString(Src, Str);
03919   bool isZeroStr = CopyFromStr && Str.empty();
03920   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
03921 
03922   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
03923                                 (DstAlignCanChange ? 0 : Align),
03924                                 (isZeroStr ? 0 : SrcAlign),
03925                                 false, false, CopyFromStr, true, DAG, TLI))
03926     return SDValue();
03927 
03928   if (DstAlignCanChange) {
03929     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
03930     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
03931 
03932     // Don't promote to an alignment that would require dynamic stack
03933     // realignment.
03934     const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
03935     if (!TRI->needsStackRealignment(MF))
03936        while (NewAlign > Align &&
03937              TLI.getDataLayout()->exceedsNaturalStackAlignment(NewAlign))
03938           NewAlign /= 2;
03939 
03940     if (NewAlign > Align) {
03941       // Give the stack frame object a larger alignment if needed.
03942       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
03943         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
03944       Align = NewAlign;
03945     }
03946   }
03947 
03948   SmallVector<SDValue, 8> OutChains;
03949   unsigned NumMemOps = MemOps.size();
03950   uint64_t SrcOff = 0, DstOff = 0;
03951   for (unsigned i = 0; i != NumMemOps; ++i) {
03952     EVT VT = MemOps[i];
03953     unsigned VTSize = VT.getSizeInBits() / 8;
03954     SDValue Value, Store;
03955 
03956     if (VTSize > Size) {
03957       // Issuing an unaligned load / store pair  that overlaps with the previous
03958       // pair. Adjust the offset accordingly.
03959       assert(i == NumMemOps-1 && i != 0);
03960       SrcOff -= VTSize - Size;
03961       DstOff -= VTSize - Size;
03962     }
03963 
03964     if (CopyFromStr &&
03965         (isZeroStr || (VT.isInteger() && !VT.isVector()))) {
03966       // It's unlikely a store of a vector immediate can be done in a single
03967       // instruction. It would require a load from a constantpool first.
03968       // We only handle zero vectors here.
03969       // FIXME: Handle other cases where store of vector immediate is done in
03970       // a single instruction.
03971       Value = getMemsetStringVal(VT, dl, DAG, TLI, Str.substr(SrcOff));
03972       if (Value.getNode())
03973         Store = DAG.getStore(Chain, dl, Value,
03974                              getMemBasePlusOffset(Dst, DstOff, dl, DAG),
03975                              DstPtrInfo.getWithOffset(DstOff), isVol,
03976                              false, Align);
03977     }
03978 
03979     if (!Store.getNode()) {
03980       // The type might not be legal for the target.  This should only happen
03981       // if the type is smaller than a legal type, as on PPC, so the right
03982       // thing to do is generate a LoadExt/StoreTrunc pair.  These simplify
03983       // to Load/Store if NVT==VT.
03984       // FIXME does the case above also need this?
03985       EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
03986       assert(NVT.bitsGE(VT));
03987       Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain,
03988                              getMemBasePlusOffset(Src, SrcOff, dl, DAG),
03989                              SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false,
03990                              false, MinAlign(SrcAlign, SrcOff));
03991       Store = DAG.getTruncStore(Chain, dl, Value,
03992                                 getMemBasePlusOffset(Dst, DstOff, dl, DAG),
03993                                 DstPtrInfo.getWithOffset(DstOff), VT, isVol,
03994                                 false, Align);
03995     }
03996     OutChains.push_back(Store);
03997     SrcOff += VTSize;
03998     DstOff += VTSize;
03999     Size -= VTSize;
04000   }
04001 
04002   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04003 }
04004 
04005 static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
04006                                         SDValue Chain, SDValue Dst,
04007                                         SDValue Src, uint64_t Size,
04008                                         unsigned Align,  bool isVol,
04009                                         bool AlwaysInline,
04010                                         MachinePointerInfo DstPtrInfo,
04011                                         MachinePointerInfo SrcPtrInfo) {
04012   // Turn a memmove of undef to nop.
04013   if (Src.getOpcode() == ISD::UNDEF)
04014     return Chain;
04015 
04016   // Expand memmove to a series of load and store ops if the size operand falls
04017   // below a certain threshold.
04018   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04019   std::vector<EVT> MemOps;
04020   bool DstAlignCanChange = false;
04021   MachineFunction &MF = DAG.getMachineFunction();
04022   MachineFrameInfo *MFI = MF.getFrameInfo();
04023   bool OptSize = MF.getFunction()->getAttributes().
04024     hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
04025   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
04026   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
04027     DstAlignCanChange = true;
04028   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
04029   if (Align > SrcAlign)
04030     SrcAlign = Align;
04031   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
04032 
04033   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
04034                                 (DstAlignCanChange ? 0 : Align), SrcAlign,
04035                                 false, false, false, false, DAG, TLI))
04036     return SDValue();
04037 
04038   if (DstAlignCanChange) {
04039     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
04040     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
04041     if (NewAlign > Align) {
04042       // Give the stack frame object a larger alignment if needed.
04043       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
04044         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
04045       Align = NewAlign;
04046     }
04047   }
04048 
04049   uint64_t SrcOff = 0, DstOff = 0;
04050   SmallVector<SDValue, 8> LoadValues;
04051   SmallVector<SDValue, 8> LoadChains;
04052   SmallVector<SDValue, 8> OutChains;
04053   unsigned NumMemOps = MemOps.size();
04054   for (unsigned i = 0; i < NumMemOps; i++) {
04055     EVT VT = MemOps[i];
04056     unsigned VTSize = VT.getSizeInBits() / 8;
04057     SDValue Value;
04058 
04059     Value = DAG.getLoad(VT, dl, Chain,
04060                         getMemBasePlusOffset(Src, SrcOff, dl, DAG),
04061                         SrcPtrInfo.getWithOffset(SrcOff), isVol,
04062                         false, false, SrcAlign);
04063     LoadValues.push_back(Value);
04064     LoadChains.push_back(Value.getValue(1));
04065     SrcOff += VTSize;
04066   }
04067   Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
04068   OutChains.clear();
04069   for (unsigned i = 0; i < NumMemOps; i++) {
04070     EVT VT = MemOps[i];
04071     unsigned VTSize = VT.getSizeInBits() / 8;
04072     SDValue Store;
04073 
04074     Store = DAG.getStore(Chain, dl, LoadValues[i],
04075                          getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04076                          DstPtrInfo.getWithOffset(DstOff), isVol, false, Align);
04077     OutChains.push_back(Store);
04078     DstOff += VTSize;
04079   }
04080 
04081   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04082 }
04083 
04084 /// \brief Lower the call to 'memset' intrinsic function into a series of store
04085 /// operations.
04086 ///
04087 /// \param DAG Selection DAG where lowered code is placed.
04088 /// \param dl Link to corresponding IR location.
04089 /// \param Chain Control flow dependency.
04090 /// \param Dst Pointer to destination memory location.
04091 /// \param Src Value of byte to write into the memory.
04092 /// \param Size Number of bytes to write.
04093 /// \param Align Alignment of the destination in bytes.
04094 /// \param isVol True if destination is volatile.
04095 /// \param DstPtrInfo IR information on the memory pointer.
04096 /// \returns New head in the control flow, if lowering was successful, empty
04097 /// SDValue otherwise.
04098 ///
04099 /// The function tries to replace 'llvm.memset' intrinsic with several store
04100 /// operations and value calculation code. This is usually profitable for small
04101 /// memory size.
04102 static SDValue getMemsetStores(SelectionDAG &DAG, SDLoc dl,
04103                                SDValue Chain, SDValue Dst,
04104                                SDValue Src, uint64_t Size,
04105                                unsigned Align, bool isVol,
04106                                MachinePointerInfo DstPtrInfo) {
04107   // Turn a memset of undef to nop.
04108   if (Src.getOpcode() == ISD::UNDEF)
04109     return Chain;
04110 
04111   // Expand memset to a series of load/store ops if the size operand
04112   // falls below a certain threshold.
04113   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04114   std::vector<EVT> MemOps;
04115   bool DstAlignCanChange = false;
04116   MachineFunction &MF = DAG.getMachineFunction();
04117   MachineFrameInfo *MFI = MF.getFrameInfo();
04118   bool OptSize = MF.getFunction()->getAttributes().
04119     hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
04120   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
04121   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
04122     DstAlignCanChange = true;
04123   bool IsZeroVal =
04124     isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
04125   if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize),
04126                                 Size, (DstAlignCanChange ? 0 : Align), 0,
04127                                 true, IsZeroVal, false, true, DAG, TLI))
04128     return SDValue();
04129 
04130   if (DstAlignCanChange) {
04131     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
04132     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
04133     if (NewAlign > Align) {
04134       // Give the stack frame object a larger alignment if needed.
04135       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
04136         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
04137       Align = NewAlign;
04138     }
04139   }
04140 
04141   SmallVector<SDValue, 8> OutChains;
04142   uint64_t DstOff = 0;
04143   unsigned NumMemOps = MemOps.size();
04144 
04145   // Find the largest store and generate the bit pattern for it.
04146   EVT LargestVT = MemOps[0];
04147   for (unsigned i = 1; i < NumMemOps; i++)
04148     if (MemOps[i].bitsGT(LargestVT))
04149       LargestVT = MemOps[i];
04150   SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
04151 
04152   for (unsigned i = 0; i < NumMemOps; i++) {
04153     EVT VT = MemOps[i];
04154     unsigned VTSize = VT.getSizeInBits() / 8;
04155     if (VTSize > Size) {
04156       // Issuing an unaligned load / store pair  that overlaps with the previous
04157       // pair. Adjust the offset accordingly.
04158       assert(i == NumMemOps-1 && i != 0);
04159       DstOff -= VTSize - Size;
04160     }
04161 
04162     // If this store is smaller than the largest store see whether we can get
04163     // the smaller value for free with a truncate.
04164     SDValue Value = MemSetValue;
04165     if (VT.bitsLT(LargestVT)) {
04166       if (!LargestVT.isVector() && !VT.isVector() &&
04167           TLI.isTruncateFree(LargestVT, VT))
04168         Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
04169       else
04170         Value = getMemsetValue(Src, VT, DAG, dl);
04171     }
04172     assert(Value.getValueType() == VT && "Value with wrong type.");
04173     SDValue Store = DAG.getStore(Chain, dl, Value,
04174                                  getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04175                                  DstPtrInfo.getWithOffset(DstOff),
04176                                  isVol, false, Align);
04177     OutChains.push_back(Store);
04178     DstOff += VT.getSizeInBits() / 8;
04179     Size -= VTSize;
04180   }
04181 
04182   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04183 }
04184 
04185 SDValue SelectionDAG::getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst,
04186                                 SDValue Src, SDValue Size,
04187                                 unsigned Align, bool isVol, bool AlwaysInline,
04188                                 MachinePointerInfo DstPtrInfo,
04189                                 MachinePointerInfo SrcPtrInfo) {
04190   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04191 
04192   // Check to see if we should lower the memcpy to loads and stores first.
04193   // For cases within the target-specified limits, this is the best choice.
04194   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04195   if (ConstantSize) {
04196     // Memcpy with size zero? Just return the original chain.
04197     if (ConstantSize->isNullValue())
04198       return Chain;
04199 
04200     SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
04201                                              ConstantSize->getZExtValue(),Align,
04202                                 isVol, false, DstPtrInfo, SrcPtrInfo);
04203     if (Result.getNode())
04204       return Result;
04205   }
04206 
04207   // Then check to see if we should lower the memcpy with target-specific
04208   // code. If the target chooses to do this, this is the next best.
04209   SDValue Result =
04210       TSI->EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align,
04211                                    isVol, AlwaysInline, DstPtrInfo, SrcPtrInfo);
04212   if (Result.getNode())
04213     return Result;
04214 
04215   // If we really need inline code and the target declined to provide it,
04216   // use a (potentially long) sequence of loads and stores.
04217   if (AlwaysInline) {
04218     assert(ConstantSize && "AlwaysInline requires a constant size!");
04219     return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
04220                                    ConstantSize->getZExtValue(), Align, isVol,
04221                                    true, DstPtrInfo, SrcPtrInfo);
04222   }
04223 
04224   // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
04225   // memcpy is not guaranteed to be safe. libc memcpys aren't required to
04226   // respect volatile, so they may do things like read or write memory
04227   // beyond the given memory regions. But fixing this isn't easy, and most
04228   // people don't care.
04229 
04230   // Emit a library call.
04231   TargetLowering::ArgListTy Args;
04232   TargetLowering::ArgListEntry Entry;
04233   Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
04234   Entry.Node = Dst; Args.push_back(Entry);
04235   Entry.Node = Src; Args.push_back(Entry);
04236   Entry.Node = Size; Args.push_back(Entry);
04237   // FIXME: pass in SDLoc
04238   TargetLowering::CallLoweringInfo CLI(*this);
04239   CLI.setDebugLoc(dl).setChain(Chain)
04240     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
04241                Type::getVoidTy(*getContext()),
04242                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
04243                                  TLI->getPointerTy()), std::move(Args), 0)
04244     .setDiscardResult();
04245   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04246 
04247   return CallResult.second;
04248 }
04249 
04250 SDValue SelectionDAG::getMemmove(SDValue Chain, SDLoc dl, SDValue Dst,
04251                                  SDValue Src, SDValue Size,
04252                                  unsigned Align, bool isVol,
04253                                  MachinePointerInfo DstPtrInfo,
04254                                  MachinePointerInfo SrcPtrInfo) {
04255   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04256 
04257   // Check to see if we should lower the memmove to loads and stores first.
04258   // For cases within the target-specified limits, this is the best choice.
04259   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04260   if (ConstantSize) {
04261     // Memmove with size zero? Just return the original chain.
04262     if (ConstantSize->isNullValue())
04263       return Chain;
04264 
04265     SDValue Result =
04266       getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src,
04267                                ConstantSize->getZExtValue(), Align, isVol,
04268                                false, DstPtrInfo, SrcPtrInfo);
04269     if (Result.getNode())
04270       return Result;
04271   }
04272 
04273   // Then check to see if we should lower the memmove with target-specific
04274   // code. If the target chooses to do this, this is the next best.
04275   SDValue Result = TSI->EmitTargetCodeForMemmove(
04276       *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo, SrcPtrInfo);
04277   if (Result.getNode())
04278     return Result;
04279 
04280   // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
04281   // not be safe.  See memcpy above for more details.
04282 
04283   // Emit a library call.
04284   TargetLowering::ArgListTy Args;
04285   TargetLowering::ArgListEntry Entry;
04286   Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
04287   Entry.Node = Dst; Args.push_back(Entry);
04288   Entry.Node = Src; Args.push_back(Entry);
04289   Entry.Node = Size; Args.push_back(Entry);
04290   // FIXME:  pass in SDLoc
04291   TargetLowering::CallLoweringInfo CLI(*this);
04292   CLI.setDebugLoc(dl).setChain(Chain)
04293     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
04294                Type::getVoidTy(*getContext()),
04295                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
04296                                  TLI->getPointerTy()), std::move(Args), 0)
04297     .setDiscardResult();
04298   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04299 
04300   return CallResult.second;
04301 }
04302 
04303 SDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst,
04304                                 SDValue Src, SDValue Size,
04305                                 unsigned Align, bool isVol,
04306                                 MachinePointerInfo DstPtrInfo) {
04307   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04308 
04309   // Check to see if we should lower the memset to stores first.
04310   // For cases within the target-specified limits, this is the best choice.
04311   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04312   if (ConstantSize) {
04313     // Memset with size zero? Just return the original chain.
04314     if (ConstantSize->isNullValue())
04315       return Chain;
04316 
04317     SDValue Result =
04318       getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
04319                       Align, isVol, DstPtrInfo);
04320 
04321     if (Result.getNode())
04322       return Result;
04323   }
04324 
04325   // Then check to see if we should lower the memset with target-specific
04326   // code. If the target chooses to do this, this is the next best.
04327   SDValue Result = TSI->EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src,
04328                                                 Size, Align, isVol, DstPtrInfo);
04329   if (Result.getNode())
04330     return Result;
04331 
04332   // Emit a library call.
04333   Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(*getContext());
04334   TargetLowering::ArgListTy Args;
04335   TargetLowering::ArgListEntry Entry;
04336   Entry.Node = Dst; Entry.Ty = IntPtrTy;
04337   Args.push_back(Entry);
04338   Entry.Node = Src;
04339   Entry.Ty = Src.getValueType().getTypeForEVT(*getContext());
04340   Args.push_back(Entry);
04341   Entry.Node = Size;
04342   Entry.Ty = IntPtrTy;
04343   Args.push_back(Entry);
04344 
04345   // FIXME: pass in SDLoc
04346   TargetLowering::CallLoweringInfo CLI(*this);
04347   CLI.setDebugLoc(dl).setChain(Chain)
04348     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
04349                Type::getVoidTy(*getContext()),
04350                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
04351                                  TLI->getPointerTy()), std::move(Args), 0)
04352     .setDiscardResult();
04353 
04354   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04355   return CallResult.second;
04356 }
04357 
04358 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04359                                 SDVTList VTList, ArrayRef<SDValue> Ops,
04360                                 MachineMemOperand *MMO,
04361                                 AtomicOrdering SuccessOrdering,
04362                                 AtomicOrdering FailureOrdering,
04363                                 SynchronizationScope SynchScope) {
04364   FoldingSetNodeID ID;
04365   ID.AddInteger(MemVT.getRawBits());
04366   AddNodeIDNode(ID, Opcode, VTList, Ops);
04367   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04368   void* IP = nullptr;
04369   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04370     cast<AtomicSDNode>(E)->refineAlignment(MMO);
04371     return SDValue(E, 0);
04372   }
04373 
04374   // Allocate the operands array for the node out of the BumpPtrAllocator, since
04375   // SDNode doesn't have access to it.  This memory will be "leaked" when
04376   // the node is deallocated, but recovered when the allocator is released.
04377   // If the number of operands is less than 5 we use AtomicSDNode's internal
04378   // storage.
04379   unsigned NumOps = Ops.size();
04380   SDUse *DynOps = NumOps > 4 ? OperandAllocator.Allocate<SDUse>(NumOps)
04381                              : nullptr;
04382 
04383   SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(),
04384                                                dl.getDebugLoc(), VTList, MemVT,
04385                                                Ops.data(), DynOps, NumOps, MMO,
04386                                                SuccessOrdering, FailureOrdering,
04387                                                SynchScope);
04388   CSEMap.InsertNode(N, IP);
04389   InsertNode(N);
04390   return SDValue(N, 0);
04391 }
04392 
04393 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04394                                 SDVTList VTList, ArrayRef<SDValue> Ops,
04395                                 MachineMemOperand *MMO,
04396                                 AtomicOrdering Ordering,
04397                                 SynchronizationScope SynchScope) {
04398   return getAtomic(Opcode, dl, MemVT, VTList, Ops, MMO, Ordering,
04399                    Ordering, SynchScope);
04400 }
04401 
04402 SDValue SelectionDAG::getAtomicCmpSwap(
04403     unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs, SDValue Chain,
04404     SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo,
04405     unsigned Alignment, AtomicOrdering SuccessOrdering,
04406     AtomicOrdering FailureOrdering, SynchronizationScope SynchScope) {
04407   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
04408          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
04409   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
04410 
04411   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04412     Alignment = getEVTAlignment(MemVT);
04413 
04414   MachineFunction &MF = getMachineFunction();
04415 
04416   // FIXME: Volatile isn't really correct; we should keep track of atomic
04417   // orderings in the memoperand.
04418   unsigned Flags = MachineMemOperand::MOVolatile;
04419   Flags |= MachineMemOperand::MOLoad;
04420   Flags |= MachineMemOperand::MOStore;
04421 
04422   MachineMemOperand *MMO =
04423     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
04424 
04425   return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO,
04426                           SuccessOrdering, FailureOrdering, SynchScope);
04427 }
04428 
04429 SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, SDLoc dl, EVT MemVT,
04430                                        SDVTList VTs, SDValue Chain, SDValue Ptr,
04431                                        SDValue Cmp, SDValue Swp,
04432                                        MachineMemOperand *MMO,
04433                                        AtomicOrdering SuccessOrdering,
04434                                        AtomicOrdering FailureOrdering,
04435                                        SynchronizationScope SynchScope) {
04436   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
04437          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
04438   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
04439 
04440   SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
04441   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO,
04442                    SuccessOrdering, FailureOrdering, SynchScope);
04443 }
04444 
04445 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04446                                 SDValue Chain,
04447                                 SDValue Ptr, SDValue Val,
04448                                 const Value* PtrVal,
04449                                 unsigned Alignment,
04450                                 AtomicOrdering Ordering,
04451                                 SynchronizationScope SynchScope) {
04452   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04453     Alignment = getEVTAlignment(MemVT);
04454 
04455   MachineFunction &MF = getMachineFunction();
04456   // An atomic store does not load. An atomic load does not store.
04457   // (An atomicrmw obviously both loads and stores.)
04458   // For now, atomics are considered to be volatile always, and they are
04459   // chained as such.
04460   // FIXME: Volatile isn't really correct; we should keep track of atomic
04461   // orderings in the memoperand.
04462   unsigned Flags = MachineMemOperand::MOVolatile;
04463   if (Opcode != ISD::ATOMIC_STORE)
04464     Flags |= MachineMemOperand::MOLoad;
04465   if (Opcode != ISD::ATOMIC_LOAD)
04466     Flags |= MachineMemOperand::MOStore;
04467 
04468   MachineMemOperand *MMO =
04469     MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
04470                             MemVT.getStoreSize(), Alignment);
04471 
04472   return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO,
04473                    Ordering, SynchScope);
04474 }
04475 
04476 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04477                                 SDValue Chain,
04478                                 SDValue Ptr, SDValue Val,
04479                                 MachineMemOperand *MMO,
04480                                 AtomicOrdering Ordering,
04481                                 SynchronizationScope SynchScope) {
04482   assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
04483           Opcode == ISD::ATOMIC_LOAD_SUB ||
04484           Opcode == ISD::ATOMIC_LOAD_AND ||
04485           Opcode == ISD::ATOMIC_LOAD_OR ||
04486           Opcode == ISD::ATOMIC_LOAD_XOR ||
04487           Opcode == ISD::ATOMIC_LOAD_NAND ||
04488           Opcode == ISD::ATOMIC_LOAD_MIN ||
04489           Opcode == ISD::ATOMIC_LOAD_MAX ||
04490           Opcode == ISD::ATOMIC_LOAD_UMIN ||
04491           Opcode == ISD::ATOMIC_LOAD_UMAX ||
04492           Opcode == ISD::ATOMIC_SWAP ||
04493           Opcode == ISD::ATOMIC_STORE) &&
04494          "Invalid Atomic Op");
04495 
04496   EVT VT = Val.getValueType();
04497 
04498   SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
04499                                                getVTList(VT, MVT::Other);
04500   SDValue Ops[] = {Chain, Ptr, Val};
04501   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
04502 }
04503 
04504 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04505                                 EVT VT, SDValue Chain,
04506                                 SDValue Ptr,
04507                                 MachineMemOperand *MMO,
04508                                 AtomicOrdering Ordering,
04509                                 SynchronizationScope SynchScope) {
04510   assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
04511 
04512   SDVTList VTs = getVTList(VT, MVT::Other);
04513   SDValue Ops[] = {Chain, Ptr};
04514   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
04515 }
04516 
04517 /// getMergeValues - Create a MERGE_VALUES node from the given operands.
04518 SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, SDLoc dl) {
04519   if (Ops.size() == 1)
04520     return Ops[0];
04521 
04522   SmallVector<EVT, 4> VTs;
04523   VTs.reserve(Ops.size());
04524   for (unsigned i = 0; i < Ops.size(); ++i)
04525     VTs.push_back(Ops[i].getValueType());
04526   return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
04527 }
04528 
04529 SDValue
04530 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
04531                                   ArrayRef<SDValue> Ops,
04532                                   EVT MemVT, MachinePointerInfo PtrInfo,
04533                                   unsigned Align, bool Vol,
04534                                   bool ReadMem, bool WriteMem, unsigned Size) {
04535   if (Align == 0)  // Ensure that codegen never sees alignment 0
04536     Align = getEVTAlignment(MemVT);
04537 
04538   MachineFunction &MF = getMachineFunction();
04539   unsigned Flags = 0;
04540   if (WriteMem)
04541     Flags |= MachineMemOperand::MOStore;
04542   if (ReadMem)
04543     Flags |= MachineMemOperand::MOLoad;
04544   if (Vol)
04545     Flags |= MachineMemOperand::MOVolatile;
04546   if (!Size)
04547     Size = MemVT.getStoreSize();
04548   MachineMemOperand *MMO =
04549     MF.getMachineMemOperand(PtrInfo, Flags, Size, Align);
04550 
04551   return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
04552 }
04553 
04554 SDValue
04555 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
04556                                   ArrayRef<SDValue> Ops, EVT MemVT,
04557                                   MachineMemOperand *MMO) {
04558   assert((Opcode == ISD::INTRINSIC_VOID ||
04559           Opcode == ISD::INTRINSIC_W_CHAIN ||
04560           Opcode == ISD::PREFETCH ||
04561           Opcode == ISD::LIFETIME_START ||
04562           Opcode == ISD::LIFETIME_END ||
04563           (Opcode <= INT_MAX &&
04564            (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
04565          "Opcode is not a memory-accessing opcode!");
04566 
04567   // Memoize the node unless it returns a flag.
04568   MemIntrinsicSDNode *N;
04569   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
04570     FoldingSetNodeID ID;
04571     AddNodeIDNode(ID, Opcode, VTList, Ops);
04572     ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04573     void *IP = nullptr;
04574     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04575       cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
04576       return SDValue(E, 0);
04577     }
04578 
04579     N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
04580                                                dl.getDebugLoc(), VTList, Ops,
04581                                                MemVT, MMO);
04582     CSEMap.InsertNode(N, IP);
04583   } else {
04584     N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
04585                                                dl.getDebugLoc(), VTList, Ops,
04586                                                MemVT, MMO);
04587   }
04588   InsertNode(N);
04589   return SDValue(N, 0);
04590 }
04591 
04592 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
04593 /// MachinePointerInfo record from it.  This is particularly useful because the
04594 /// code generator has many cases where it doesn't bother passing in a
04595 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
04596 static MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) {
04597   // If this is FI+Offset, we can model it.
04598   if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
04599     return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset);
04600 
04601   // If this is (FI+Offset1)+Offset2, we can model it.
04602   if (Ptr.getOpcode() != ISD::ADD ||
04603       !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
04604       !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
04605     return MachinePointerInfo();
04606 
04607   int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
04608   return MachinePointerInfo::getFixedStack(FI, Offset+
04609                        cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
04610 }
04611 
04612 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
04613 /// MachinePointerInfo record from it.  This is particularly useful because the
04614 /// code generator has many cases where it doesn't bother passing in a
04615 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
04616 static MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) {
04617   // If the 'Offset' value isn't a constant, we can't handle this.
04618   if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
04619     return InferPointerInfo(Ptr, OffsetNode->getSExtValue());
04620   if (OffsetOp.getOpcode() == ISD::UNDEF)
04621     return InferPointerInfo(Ptr);
04622   return MachinePointerInfo();
04623 }
04624 
04625 
04626 SDValue
04627 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
04628                       EVT VT, SDLoc dl, SDValue Chain,
04629                       SDValue Ptr, SDValue Offset,
04630                       MachinePointerInfo PtrInfo, EVT MemVT,
04631                       bool isVolatile, bool isNonTemporal, bool isInvariant,
04632                       unsigned Alignment, const AAMDNodes &AAInfo,
04633                       const MDNode *Ranges) {
04634   assert(Chain.getValueType() == MVT::Other &&
04635         "Invalid chain type");
04636   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04637     Alignment = getEVTAlignment(VT);
04638 
04639   unsigned Flags = MachineMemOperand::MOLoad;
04640   if (isVolatile)
04641     Flags |= MachineMemOperand::MOVolatile;
04642   if (isNonTemporal)
04643     Flags |= MachineMemOperand::MONonTemporal;
04644   if (isInvariant)
04645     Flags |= MachineMemOperand::MOInvariant;
04646 
04647   // If we don't have a PtrInfo, infer the trivial frame index case to simplify
04648   // clients.
04649   if (PtrInfo.V.isNull())
04650     PtrInfo = InferPointerInfo(Ptr, Offset);
04651 
04652   MachineFunction &MF = getMachineFunction();
04653   MachineMemOperand *MMO =
04654     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment,
04655                             AAInfo, Ranges);
04656   return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
04657 }
04658 
04659 SDValue
04660 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
04661                       EVT VT, SDLoc dl, SDValue Chain,
04662                       SDValue Ptr, SDValue Offset, EVT MemVT,
04663                       MachineMemOperand *MMO) {
04664   if (VT == MemVT) {
04665     ExtType = ISD::NON_EXTLOAD;
04666   } else if (ExtType == ISD::NON_EXTLOAD) {
04667     assert(VT == MemVT && "Non-extending load from different memory type!");
04668   } else {
04669     // Extending load.
04670     assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
04671            "Should only be an extending load, not truncating!");
04672     assert(VT.isInteger() == MemVT.isInteger() &&
04673            "Cannot convert from FP to Int or Int -> FP!");
04674     assert(VT.isVector() == MemVT.isVector() &&
04675            "Cannot use trunc store to convert to or from a vector!");
04676     assert((!VT.isVector() ||
04677             VT.getVectorNumElements() == MemVT.getVectorNumElements()) &&
04678            "Cannot use trunc store to change the number of vector elements!");
04679   }
04680 
04681   bool Indexed = AM != ISD::UNINDEXED;
04682   assert((Indexed || Offset.getOpcode() == ISD::UNDEF) &&
04683          "Unindexed load with an offset!");
04684 
04685   SDVTList VTs = Indexed ?
04686     getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
04687   SDValue Ops[] = { Chain, Ptr, Offset };
04688   FoldingSetNodeID ID;
04689   AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
04690   ID.AddInteger(MemVT.getRawBits());
04691   ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(),
04692                                      MMO->isNonTemporal(),
04693                                      MMO->isInvariant()));
04694   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04695   void *IP = nullptr;
04696   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04697     cast<LoadSDNode>(E)->refineAlignment(MMO);
04698     return SDValue(E, 0);
04699   }
04700   SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl.getIROrder(),
04701                                              dl.getDebugLoc(), VTs, AM, ExtType,
04702                                              MemVT, MMO);
04703   CSEMap.InsertNode(N, IP);
04704   InsertNode(N);
04705   return SDValue(N, 0);
04706 }
04707 
04708 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
04709                               SDValue Chain, SDValue Ptr,
04710                               MachinePointerInfo PtrInfo,
04711                               bool isVolatile, bool isNonTemporal,
04712                               bool isInvariant, unsigned Alignment,
04713                               const AAMDNodes &AAInfo,
04714                               const MDNode *Ranges) {
04715   SDValue Undef = getUNDEF(Ptr.getValueType());
04716   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
04717                  PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment,
04718                  AAInfo, Ranges);
04719 }
04720 
04721 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
04722                               SDValue Chain, SDValue Ptr,
04723                               MachineMemOperand *MMO) {
04724   SDValue Undef = getUNDEF(Ptr.getValueType());
04725   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
04726                  VT, MMO);
04727 }
04728 
04729 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
04730                                  SDValue Chain, SDValue Ptr,
04731                                  MachinePointerInfo PtrInfo, EVT MemVT,
04732                                  bool isVolatile, bool isNonTemporal,
04733                                  bool isInvariant, unsigned Alignment,
04734                                  const AAMDNodes &AAInfo) {
04735   SDValue Undef = getUNDEF(Ptr.getValueType());
04736   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
04737                  PtrInfo, MemVT, isVolatile, isNonTemporal, isInvariant,
04738                  Alignment, AAInfo);
04739 }
04740 
04741 
04742 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
04743                                  SDValue Chain, SDValue Ptr, EVT MemVT,
04744                                  MachineMemOperand *MMO) {
04745   SDValue Undef = getUNDEF(Ptr.getValueType());
04746   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
04747                  MemVT, MMO);
04748 }
04749 
04750 SDValue
04751 SelectionDAG::getIndexedLoad(SDValue OrigLoad, SDLoc dl, SDValue Base,
04752                              SDValue Offset, ISD::MemIndexedMode AM) {
04753   LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
04754   assert(LD->getOffset().getOpcode() == ISD::UNDEF &&
04755          "Load is already a indexed load!");
04756   return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
04757                  LD->getChain(), Base, Offset, LD->getPointerInfo(),
04758                  LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(),
04759                  false, LD->getAlignment());
04760 }
04761 
04762 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
04763                                SDValue Ptr, MachinePointerInfo PtrInfo,
04764                                bool isVolatile, bool isNonTemporal,
04765                                unsigned Alignment, const AAMDNodes &AAInfo) {
04766   assert(Chain.getValueType() == MVT::Other &&
04767         "Invalid chain type");
04768   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04769     Alignment = getEVTAlignment(Val.getValueType());
04770 
04771   unsigned Flags = MachineMemOperand::MOStore;
04772   if (isVolatile)
04773     Flags |= MachineMemOperand::MOVolatile;
04774   if (isNonTemporal)
04775     Flags |= MachineMemOperand::MONonTemporal;
04776 
04777   if (PtrInfo.V.isNull())
04778     PtrInfo = InferPointerInfo(Ptr);
04779 
04780   MachineFunction &MF = getMachineFunction();
04781   MachineMemOperand *MMO =
04782     MF.getMachineMemOperand(PtrInfo, Flags,
04783                             Val.getValueType().getStoreSize(), Alignment,
04784                             AAInfo);
04785 
04786   return getStore(Chain, dl, Val, Ptr, MMO);
04787 }
04788 
04789 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
04790                                SDValue Ptr, MachineMemOperand *MMO) {
04791   assert(Chain.getValueType() == MVT::Other &&
04792         "Invalid chain type");
04793   EVT VT = Val.getValueType();
04794   SDVTList VTs = getVTList(MVT::Other);
04795   SDValue Undef = getUNDEF(Ptr.getValueType());
04796   SDValue Ops[] = { Chain, Val, Ptr, Undef };
04797   FoldingSetNodeID ID;
04798   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04799   ID.AddInteger(VT.getRawBits());
04800   ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
04801                                      MMO->isNonTemporal(), MMO->isInvariant()));
04802   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04803   void *IP = nullptr;
04804   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04805     cast<StoreSDNode>(E)->refineAlignment(MMO);
04806     return SDValue(E, 0);
04807   }
04808   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04809                                               dl.getDebugLoc(), VTs,
04810                                               ISD::UNINDEXED, false, VT, MMO);
04811   CSEMap.InsertNode(N, IP);
04812   InsertNode(N);
04813   return SDValue(N, 0);
04814 }
04815 
04816 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
04817                                     SDValue Ptr, MachinePointerInfo PtrInfo,
04818                                     EVT SVT,bool isVolatile, bool isNonTemporal,
04819                                     unsigned Alignment,
04820                                     const AAMDNodes &AAInfo) {
04821   assert(Chain.getValueType() == MVT::Other &&
04822         "Invalid chain type");
04823   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04824     Alignment = getEVTAlignment(SVT);
04825 
04826   unsigned Flags = MachineMemOperand::MOStore;
04827   if (isVolatile)
04828     Flags |= MachineMemOperand::MOVolatile;
04829   if (isNonTemporal)
04830     Flags |= MachineMemOperand::MONonTemporal;
04831 
04832   if (PtrInfo.V.isNull())
04833     PtrInfo = InferPointerInfo(Ptr);
04834 
04835   MachineFunction &MF = getMachineFunction();
04836   MachineMemOperand *MMO =
04837     MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment,
04838                             AAInfo);
04839 
04840   return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
04841 }
04842 
04843 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
04844                                     SDValue Ptr, EVT SVT,
04845                                     MachineMemOperand *MMO) {
04846   EVT VT = Val.getValueType();
04847 
04848   assert(Chain.getValueType() == MVT::Other &&
04849         "Invalid chain type");
04850   if (VT == SVT)
04851     return getStore(Chain, dl, Val, Ptr, MMO);
04852 
04853   assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
04854          "Should only be a truncating store, not extending!");
04855   assert(VT.isInteger() == SVT.isInteger() &&
04856          "Can't do FP-INT conversion!");
04857   assert(VT.isVector() == SVT.isVector() &&
04858          "Cannot use trunc store to convert to or from a vector!");
04859   assert((!VT.isVector() ||
04860           VT.getVectorNumElements() == SVT.getVectorNumElements()) &&
04861          "Cannot use trunc store to change the number of vector elements!");
04862 
04863   SDVTList VTs = getVTList(MVT::Other);
04864   SDValue Undef = getUNDEF(Ptr.getValueType());
04865   SDValue Ops[] = { Chain, Val, Ptr, Undef };
04866   FoldingSetNodeID ID;
04867   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04868   ID.AddInteger(SVT.getRawBits());
04869   ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(),
04870                                      MMO->isNonTemporal(), MMO->isInvariant()));
04871   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04872   void *IP = nullptr;
04873   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04874     cast<StoreSDNode>(E)->refineAlignment(MMO);
04875     return SDValue(E, 0);
04876   }
04877   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04878                                               dl.getDebugLoc(), VTs,
04879                                               ISD::UNINDEXED, true, SVT, MMO);
04880   CSEMap.InsertNode(N, IP);
04881   InsertNode(N);
04882   return SDValue(N, 0);
04883 }
04884 
04885 SDValue
04886 SelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base,
04887                               SDValue Offset, ISD::MemIndexedMode AM) {
04888   StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
04889   assert(ST->getOffset().getOpcode() == ISD::UNDEF &&
04890          "Store is already a indexed store!");
04891   SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
04892   SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
04893   FoldingSetNodeID ID;
04894   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04895   ID.AddInteger(ST->getMemoryVT().getRawBits());
04896   ID.AddInteger(ST->getRawSubclassData());
04897   ID.AddInteger(ST->getPointerInfo().getAddrSpace());
04898   void *IP = nullptr;
04899   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
04900     return SDValue(E, 0);
04901 
04902   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04903                                               dl.getDebugLoc(), VTs, AM,
04904                                               ST->isTruncatingStore(),
04905                                               ST->getMemoryVT(),
04906                                               ST->getMemOperand());
04907   CSEMap.InsertNode(N, IP);
04908   InsertNode(N);
04909   return SDValue(N, 0);
04910 }
04911 
04912 SDValue SelectionDAG::getVAArg(EVT VT, SDLoc dl,
04913                                SDValue Chain, SDValue Ptr,
04914                                SDValue SV,
04915                                unsigned Align) {
04916   SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) };
04917   return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
04918 }
04919 
04920 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
04921                               ArrayRef<SDUse> Ops) {
04922   switch (Ops.size()) {
04923   case 0: return getNode(Opcode, DL, VT);
04924   case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
04925   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
04926   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
04927   default: break;
04928   }
04929 
04930   // Copy from an SDUse array into an SDValue array for use with
04931   // the regular getNode logic.
04932   SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
04933   return getNode(Opcode, DL, VT, NewOps);
04934 }
04935 
04936 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
04937                               ArrayRef<SDValue> Ops) {
04938   unsigned NumOps = Ops.size();
04939   switch (NumOps) {
04940   case 0: return getNode(Opcode, DL, VT);
04941   case 1: return getNode(Opcode, DL, VT, Ops[0]);
04942   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
04943   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
04944   default: break;
04945   }
04946 
04947   switch (Opcode) {
04948   default: break;
04949   case ISD::SELECT_CC: {
04950     assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
04951     assert(Ops[0].getValueType() == Ops[1].getValueType() &&
04952            "LHS and RHS of condition must have same type!");
04953     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
04954            "True and False arms of SelectCC must have same type!");
04955     assert(Ops[2].getValueType() == VT &&
04956            "select_cc node must be of same type as true and false value!");
04957     break;
04958   }
04959   case ISD::BR_CC: {
04960     assert(NumOps == 5 && "BR_CC takes 5 operands!");
04961     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
04962            "LHS/RHS of comparison should match types!");
04963     break;
04964   }
04965   }
04966 
04967   // Memoize nodes.
04968   SDNode *N;
04969   SDVTList VTs = getVTList(VT);
04970 
04971   if (VT != MVT::Glue) {
04972     FoldingSetNodeID ID;
04973     AddNodeIDNode(ID, Opcode, VTs, Ops);
04974     void *IP = nullptr;
04975 
04976     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
04977       return SDValue(E, 0);
04978 
04979     N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
04980                                    VTs, Ops);
04981     CSEMap.InsertNode(N, IP);
04982   } else {
04983     N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
04984                                    VTs, Ops);
04985   }
04986 
04987   InsertNode(N);
04988   return SDValue(N, 0);
04989 }
04990 
04991 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
04992                               ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
04993   return getNode(Opcode, DL, getVTList(ResultTys), Ops);
04994 }
04995 
04996 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
04997                               ArrayRef<SDValue> Ops) {
04998   if (VTList.NumVTs == 1)
04999     return getNode(Opcode, DL, VTList.VTs[0], Ops);
05000 
05001 #if 0
05002   switch (Opcode) {
05003   // FIXME: figure out how to safely handle things like
05004   // int foo(int x) { return 1 << (x & 255); }
05005   // int bar() { return foo(256); }
05006   case ISD::SRA_PARTS:
05007   case ISD::SRL_PARTS:
05008   case ISD::SHL_PARTS:
05009     if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
05010         cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
05011       return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
05012     else if (N3.getOpcode() == ISD::AND)
05013       if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
05014         // If the and is only masking out bits that cannot effect the shift,
05015         // eliminate the and.
05016         unsigned NumBits = VT.getScalarType().getSizeInBits()*2;
05017         if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
05018           return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
05019       }
05020     break;
05021   }
05022 #endif
05023 
05024   // Memoize the node unless it returns a flag.
05025   SDNode *N;
05026   unsigned NumOps = Ops.size();
05027   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
05028     FoldingSetNodeID ID;
05029     AddNodeIDNode(ID, Opcode, VTList, Ops);
05030     void *IP = nullptr;
05031     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05032       return SDValue(E, 0);
05033 
05034     if (NumOps == 1) {
05035       N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
05036                                           DL.getDebugLoc(), VTList, Ops[0]);
05037     } else if (NumOps == 2) {
05038       N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
05039                                            DL.getDebugLoc(), VTList, Ops[0],
05040                                            Ops[1]);
05041     } else if (NumOps == 3) {
05042       N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
05043                                             DL.getDebugLoc(), VTList, Ops[0],
05044                                             Ops[1], Ops[2]);
05045     } else {
05046       N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05047                                      VTList, Ops);
05048     }
05049     CSEMap.InsertNode(N, IP);
05050   } else {
05051     if (NumOps == 1) {
05052       N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
05053                                           DL.getDebugLoc(), VTList, Ops[0]);
05054     } else if (NumOps == 2) {
05055       N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
05056                                            DL.getDebugLoc(), VTList, Ops[0],
05057                                            Ops[1]);
05058     } else if (NumOps == 3) {
05059       N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
05060                                             DL.getDebugLoc(), VTList, Ops[0],
05061                                             Ops[1], Ops[2]);
05062     } else {
05063       N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05064                                      VTList, Ops);
05065     }
05066   }
05067   InsertNode(N);
05068   return SDValue(N, 0);
05069 }
05070 
05071 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList) {
05072   return getNode(Opcode, DL, VTList, None);
05073 }
05074 
05075 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05076                               SDValue N1) {
05077   SDValue Ops[] = { N1 };
05078   return getNode(Opcode, DL, VTList, Ops);
05079 }
05080 
05081 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05082                               SDValue N1, SDValue N2) {
05083   SDValue Ops[] = { N1, N2 };
05084   return getNode(Opcode, DL, VTList, Ops);
05085 }
05086 
05087 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05088                               SDValue N1, SDValue N2, SDValue N3) {
05089   SDValue Ops[] = { N1, N2, N3 };
05090   return getNode(Opcode, DL, VTList, Ops);
05091 }
05092 
05093 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05094                               SDValue N1, SDValue N2, SDValue N3,
05095                               SDValue N4) {
05096   SDValue Ops[] = { N1, N2, N3, N4 };
05097   return getNode(Opcode, DL, VTList, Ops);
05098 }
05099 
05100 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05101                               SDValue N1, SDValue N2, SDValue N3,
05102                               SDValue N4, SDValue N5) {
05103   SDValue Ops[] = { N1, N2, N3, N4, N5 };
05104   return getNode(Opcode, DL, VTList, Ops);
05105 }
05106 
05107 SDVTList SelectionDAG::getVTList(EVT VT) {
05108   return makeVTList(SDNode::getValueTypeList(VT), 1);
05109 }
05110 
05111 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
05112   FoldingSetNodeID ID;
05113   ID.AddInteger(2U);
05114   ID.AddInteger(VT1.getRawBits());
05115   ID.AddInteger(VT2.getRawBits());
05116 
05117   void *IP = nullptr;
05118   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05119   if (!Result) {
05120     EVT *Array = Allocator.Allocate<EVT>(2);
05121     Array[0] = VT1;
05122     Array[1] = VT2;
05123     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
05124     VTListMap.InsertNode(Result, IP);
05125   }
05126   return Result->getSDVTList();
05127 }
05128 
05129 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
05130   FoldingSetNodeID ID;
05131   ID.AddInteger(3U);
05132   ID.AddInteger(VT1.getRawBits());
05133   ID.AddInteger(VT2.getRawBits());
05134   ID.AddInteger(VT3.getRawBits());
05135 
05136   void *IP = nullptr;
05137   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05138   if (!Result) {
05139     EVT *Array = Allocator.Allocate<EVT>(3);
05140     Array[0] = VT1;
05141     Array[1] = VT2;
05142     Array[2] = VT3;
05143     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
05144     VTListMap.InsertNode(Result, IP);
05145   }
05146   return Result->getSDVTList();
05147 }
05148 
05149 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
05150   FoldingSetNodeID ID;
05151   ID.AddInteger(4U);
05152   ID.AddInteger(VT1.getRawBits());
05153   ID.AddInteger(VT2.getRawBits());
05154   ID.AddInteger(VT3.getRawBits());
05155   ID.AddInteger(VT4.getRawBits());
05156 
05157   void *IP = nullptr;
05158   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05159   if (!Result) {
05160     EVT *Array = Allocator.Allocate<EVT>(4);
05161     Array[0] = VT1;
05162     Array[1] = VT2;
05163     Array[2] = VT3;
05164     Array[3] = VT4;
05165     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
05166     VTListMap.InsertNode(Result, IP);
05167   }
05168   return Result->getSDVTList();
05169 }
05170 
05171 SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
05172   unsigned NumVTs = VTs.size();
05173   FoldingSetNodeID ID;
05174   ID.AddInteger(NumVTs);
05175   for (unsigned index = 0; index < NumVTs; index++) {
05176     ID.AddInteger(VTs[index].getRawBits());
05177   }
05178 
05179   void *IP = nullptr;
05180   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05181   if (!Result) {
05182     EVT *Array = Allocator.Allocate<EVT>(NumVTs);
05183     std::copy(VTs.begin(), VTs.end(), Array);
05184     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
05185     VTListMap.InsertNode(Result, IP);
05186   }
05187   return Result->getSDVTList();
05188 }
05189 
05190 
05191 /// UpdateNodeOperands - *Mutate* the specified node in-place to have the
05192 /// specified operands.  If the resultant node already exists in the DAG,
05193 /// this does not modify the specified node, instead it returns the node that
05194 /// already exists.  If the resultant node does not exist in the DAG, the
05195 /// input node is returned.  As a degenerate case, if you specify the same
05196 /// input operands as the node already has, the input node is returned.
05197 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
05198   assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
05199 
05200   // Check to see if there is no change.
05201   if (Op == N->getOperand(0)) return N;
05202 
05203   // See if the modified node already exists.
05204   void *InsertPos = nullptr;
05205   if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
05206     return Existing;
05207 
05208   // Nope it doesn't.  Remove the node from its current place in the maps.
05209   if (InsertPos)
05210     if (!RemoveNodeFromCSEMaps(N))
05211       InsertPos = nullptr;
05212 
05213   // Now we update the operands.
05214   N->OperandList[0].set(Op);
05215 
05216   // If this gets put into a CSE map, add it.
05217   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05218   return N;
05219 }
05220 
05221 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
05222   assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
05223 
05224   // Check to see if there is no change.
05225   if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
05226     return N;   // No operands changed, just return the input node.
05227 
05228   // See if the modified node already exists.
05229   void *InsertPos = nullptr;
05230   if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
05231     return Existing;
05232 
05233   // Nope it doesn't.  Remove the node from its current place in the maps.
05234   if (InsertPos)
05235     if (!RemoveNodeFromCSEMaps(N))
05236       InsertPos = nullptr;
05237 
05238   // Now we update the operands.
05239   if (N->OperandList[0] != Op1)
05240     N->OperandList[0].set(Op1);
05241   if (N->OperandList[1] != Op2)
05242     N->OperandList[1].set(Op2);
05243 
05244   // If this gets put into a CSE map, add it.
05245   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05246   return N;
05247 }
05248 
05249 SDNode *SelectionDAG::
05250 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
05251   SDValue Ops[] = { Op1, Op2, Op3 };
05252   return UpdateNodeOperands(N, Ops);
05253 }
05254 
05255 SDNode *SelectionDAG::
05256 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
05257                    SDValue Op3, SDValue Op4) {
05258   SDValue Ops[] = { Op1, Op2, Op3, Op4 };
05259   return UpdateNodeOperands(N, Ops);
05260 }
05261 
05262 SDNode *SelectionDAG::
05263 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
05264                    SDValue Op3, SDValue Op4, SDValue Op5) {
05265   SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
05266   return UpdateNodeOperands(N, Ops);
05267 }
05268 
05269 SDNode *SelectionDAG::
05270 UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
05271   unsigned NumOps = Ops.size();
05272   assert(N->getNumOperands() == NumOps &&
05273          "Update with wrong number of operands");
05274 
05275   // Check to see if there is no change.
05276   bool AnyChange = false;
05277   for (unsigned i = 0; i != NumOps; ++i) {
05278     if (Ops[i] != N->getOperand(i)) {
05279       AnyChange = true;
05280       break;
05281     }
05282   }
05283 
05284   // No operands changed, just return the input node.
05285   if (!AnyChange) return N;
05286 
05287   // See if the modified node already exists.
05288   void *InsertPos = nullptr;
05289   if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
05290     return Existing;
05291 
05292   // Nope it doesn't.  Remove the node from its current place in the maps.
05293   if (InsertPos)
05294     if (!RemoveNodeFromCSEMaps(N))
05295       InsertPos = nullptr;
05296 
05297   // Now we update the operands.
05298   for (unsigned i = 0; i != NumOps; ++i)
05299     if (N->OperandList[i] != Ops[i])
05300       N->OperandList[i].set(Ops[i]);
05301 
05302   // If this gets put into a CSE map, add it.
05303   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05304   return N;
05305 }
05306 
05307 /// DropOperands - Release the operands and set this node to have
05308 /// zero operands.
05309 void SDNode::DropOperands() {
05310   // Unlike the code in MorphNodeTo that does this, we don't need to
05311   // watch for dead nodes here.
05312   for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
05313     SDUse &Use = *I++;
05314     Use.set(SDValue());
05315   }
05316 }
05317 
05318 /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
05319 /// machine opcode.
05320 ///
05321 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05322                                    EVT VT) {
05323   SDVTList VTs = getVTList(VT);
05324   return SelectNodeTo(N, MachineOpc, VTs, None);
05325 }
05326 
05327 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05328                                    EVT VT, SDValue Op1) {
05329   SDVTList VTs = getVTList(VT);
05330   SDValue Ops[] = { Op1 };
05331   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05332 }
05333 
05334 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05335                                    EVT VT, SDValue Op1,
05336                                    SDValue Op2) {
05337   SDVTList VTs = getVTList(VT);
05338   SDValue Ops[] = { Op1, Op2 };
05339   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05340 }
05341 
05342 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05343                                    EVT VT, SDValue Op1,
05344                                    SDValue Op2, SDValue Op3) {
05345   SDVTList VTs = getVTList(VT);
05346   SDValue Ops[] = { Op1, Op2, Op3 };
05347   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05348 }
05349 
05350 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05351                                    EVT VT, ArrayRef<SDValue> Ops) {
05352   SDVTList VTs = getVTList(VT);
05353   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05354 }
05355 
05356 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05357                                    EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
05358   SDVTList VTs = getVTList(VT1, VT2);
05359   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05360 }
05361 
05362 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05363                                    EVT VT1, EVT VT2) {
05364   SDVTList VTs = getVTList(VT1, VT2);
05365   return SelectNodeTo(N, MachineOpc, VTs, None);
05366 }
05367 
05368 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05369                                    EVT VT1, EVT VT2, EVT VT3,
05370                                    ArrayRef<SDValue> Ops) {
05371   SDVTList VTs = getVTList(VT1, VT2, VT3);
05372   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05373 }
05374 
05375 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05376                                    EVT VT1, EVT VT2, EVT VT3, EVT VT4,
05377                                    ArrayRef<SDValue> Ops) {
05378   SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
05379   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05380 }
05381 
05382 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05383                                    EVT VT1, EVT VT2,
05384                                    SDValue Op1) {
05385   SDVTList VTs = getVTList(VT1, VT2);
05386   SDValue Ops[] = { Op1 };
05387   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05388 }
05389 
05390 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05391                                    EVT VT1, EVT VT2,
05392                                    SDValue Op1, SDValue Op2) {
05393   SDVTList VTs = getVTList(VT1, VT2);
05394   SDValue Ops[] = { Op1, Op2 };
05395   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05396 }
05397 
05398 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05399                                    EVT VT1, EVT VT2,
05400                                    SDValue Op1, SDValue Op2,
05401                                    SDValue Op3) {
05402   SDVTList VTs = getVTList(VT1, VT2);
05403   SDValue Ops[] = { Op1, Op2, Op3 };
05404   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05405 }
05406 
05407 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05408                                    EVT VT1, EVT VT2, EVT VT3,
05409                                    SDValue Op1, SDValue Op2,
05410                                    SDValue Op3) {
05411   SDVTList VTs = getVTList(VT1, VT2, VT3);
05412   SDValue Ops[] = { Op1, Op2, Op3 };
05413   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05414 }
05415 
05416 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05417                                    SDVTList VTs,ArrayRef<SDValue> Ops) {
05418   N = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
05419   // Reset the NodeID to -1.
05420   N->setNodeId(-1);
05421   return N;
05422 }
05423 
05424 /// UpdadeSDLocOnMergedSDNode - If the opt level is -O0 then it throws away
05425 /// the line number information on the merged node since it is not possible to
05426 /// preserve the information that operation is associated with multiple lines.
05427 /// This will make the debugger working better at -O0, were there is a higher
05428 /// probability having other instructions associated with that line.
05429 ///
05430 /// For IROrder, we keep the smaller of the two
05431 SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) {
05432   DebugLoc NLoc = N->getDebugLoc();
05433   if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) &&
05434     (OLoc.getDebugLoc() != NLoc)) {
05435     N->setDebugLoc(DebugLoc());
05436   }
05437   unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
05438   N->setIROrder(Order);
05439   return N;
05440 }
05441 
05442 /// MorphNodeTo - This *mutates* the specified node to have the specified
05443 /// return type, opcode, and operands.
05444 ///
05445 /// Note that MorphNodeTo returns the resultant node.  If there is already a
05446 /// node of the specified opcode and operands, it returns that node instead of
05447 /// the current one.  Note that the SDLoc need not be the same.
05448 ///
05449 /// Using MorphNodeTo is faster than creating a new node and swapping it in
05450 /// with ReplaceAllUsesWith both because it often avoids allocating a new
05451 /// node, and because it doesn't require CSE recalculation for any of
05452 /// the node's users.
05453 ///
05454 /// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
05455 /// As a consequence it isn't appropriate to use from within the DAG combiner or
05456 /// the legalizer which maintain worklists that would need to be updated when
05457 /// deleting things.
05458 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
05459                                   SDVTList VTs, ArrayRef<SDValue> Ops) {
05460   unsigned NumOps = Ops.size();
05461   // If an identical node already exists, use it.
05462   void *IP = nullptr;
05463   if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
05464     FoldingSetNodeID ID;
05465     AddNodeIDNode(ID, Opc, VTs, Ops);
05466     if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
05467       return UpdadeSDLocOnMergedSDNode(ON, SDLoc(N));
05468   }
05469 
05470   if (!RemoveNodeFromCSEMaps(N))
05471     IP = nullptr;
05472 
05473   // Start the morphing.
05474   N->NodeType = Opc;
05475   N->ValueList = VTs.VTs;
05476   N->NumValues = VTs.NumVTs;
05477 
05478   // Clear the operands list, updating used nodes to remove this from their
05479   // use list.  Keep track of any operands that become dead as a result.
05480   SmallPtrSet<SDNode*, 16> DeadNodeSet;
05481   for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
05482     SDUse &Use = *I++;
05483     SDNode *Used = Use.getNode();
05484     Use.set(SDValue());
05485     if (Used->use_empty())
05486       DeadNodeSet.insert(Used);
05487   }
05488 
05489   if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) {
05490     // Initialize the memory references information.
05491     MN->setMemRefs(nullptr, nullptr);
05492     // If NumOps is larger than the # of operands we can have in a
05493     // MachineSDNode, reallocate the operand list.
05494     if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) {
05495       if (MN->OperandsNeedDelete)
05496         delete[] MN->OperandList;
05497       if (NumOps > array_lengthof(MN->LocalOperands))
05498         // We're creating a final node that will live unmorphed for the
05499         // remainder of the current SelectionDAG iteration, so we can allocate
05500         // the operands directly out of a pool with no recycling metadata.
05501         MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
05502                          Ops.data(), NumOps);
05503       else
05504         MN->InitOperands(MN->LocalOperands, Ops.data(), NumOps);
05505       MN->OperandsNeedDelete = false;
05506     } else
05507       MN->InitOperands(MN->OperandList, Ops.data(), NumOps);
05508   } else {
05509     // If NumOps is larger than the # of operands we currently have, reallocate
05510     // the operand list.
05511     if (NumOps > N->NumOperands) {
05512       if (N->OperandsNeedDelete)
05513         delete[] N->OperandList;
05514       N->InitOperands(new SDUse[NumOps], Ops.data(), NumOps);
05515       N->OperandsNeedDelete = true;
05516     } else
05517       N->InitOperands(N->OperandList, Ops.data(), NumOps);
05518   }
05519 
05520   // Delete any nodes that are still dead after adding the uses for the
05521   // new operands.
05522   if (!DeadNodeSet.empty()) {
05523     SmallVector<SDNode *, 16> DeadNodes;
05524     for (SDNode *N : DeadNodeSet)
05525       if (N->use_empty())
05526         DeadNodes.push_back(N);
05527     RemoveDeadNodes(DeadNodes);
05528   }
05529 
05530   if (IP)
05531     CSEMap.InsertNode(N, IP);   // Memoize the new node.
05532   return N;
05533 }
05534 
05535 
05536 /// getMachineNode - These are used for target selectors to create a new node
05537 /// with specified return type(s), MachineInstr opcode, and operands.
05538 ///
05539 /// Note that getMachineNode returns the resultant node.  If there is already a
05540 /// node of the specified opcode and operands, it returns that node instead of
05541 /// the current one.
05542 MachineSDNode *
05543 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT) {
05544   SDVTList VTs = getVTList(VT);
05545   return getMachineNode(Opcode, dl, VTs, None);
05546 }
05547 
05548 MachineSDNode *
05549 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, SDValue Op1) {
05550   SDVTList VTs = getVTList(VT);
05551   SDValue Ops[] = { Op1 };
05552   return getMachineNode(Opcode, dl, VTs, Ops);
05553 }
05554 
05555 MachineSDNode *
05556 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05557                              SDValue Op1, SDValue Op2) {
05558   SDVTList VTs = getVTList(VT);
05559   SDValue Ops[] = { Op1, Op2 };
05560   return getMachineNode(Opcode, dl, VTs, Ops);
05561 }
05562 
05563 MachineSDNode *
05564 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05565                              SDValue Op1, SDValue Op2, SDValue Op3) {
05566   SDVTList VTs = getVTList(VT);
05567   SDValue Ops[] = { Op1, Op2, Op3 };
05568   return getMachineNode(Opcode, dl, VTs, Ops);
05569 }
05570 
05571 MachineSDNode *
05572 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05573                              ArrayRef<SDValue> Ops) {
05574   SDVTList VTs = getVTList(VT);
05575   return getMachineNode(Opcode, dl, VTs, Ops);
05576 }
05577 
05578 MachineSDNode *
05579 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, EVT VT2) {
05580   SDVTList VTs = getVTList(VT1, VT2);
05581   return getMachineNode(Opcode, dl, VTs, None);
05582 }
05583 
05584 MachineSDNode *
05585 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05586                              EVT VT1, EVT VT2, SDValue Op1) {
05587   SDVTList VTs = getVTList(VT1, VT2);
05588   SDValue Ops[] = { Op1 };
05589   return getMachineNode(Opcode, dl, VTs, Ops);
05590 }
05591 
05592 MachineSDNode *
05593 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05594                              EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) {
05595   SDVTList VTs = getVTList(VT1, VT2);
05596   SDValue Ops[] = { Op1, Op2 };
05597   return getMachineNode(Opcode, dl, VTs, Ops);
05598 }
05599 
05600 MachineSDNode *
05601 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05602                              EVT VT1, EVT VT2, SDValue Op1,
05603                              SDValue Op2, SDValue Op3) {
05604   SDVTList VTs = getVTList(VT1, VT2);
05605   SDValue Ops[] = { Op1, Op2, Op3 };
05606   return getMachineNode(Opcode, dl, VTs, Ops);
05607 }
05608 
05609 MachineSDNode *
05610 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05611                              EVT VT1, EVT VT2,
05612                              ArrayRef<SDValue> Ops) {
05613   SDVTList VTs = getVTList(VT1, VT2);
05614   return getMachineNode(Opcode, dl, VTs, Ops);
05615 }
05616 
05617 MachineSDNode *
05618 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05619                              EVT VT1, EVT VT2, EVT VT3,
05620                              SDValue Op1, SDValue Op2) {
05621   SDVTList VTs = getVTList(VT1, VT2, VT3);
05622   SDValue Ops[] = { Op1, Op2 };
05623   return getMachineNode(Opcode, dl, VTs, Ops);
05624 }
05625 
05626 MachineSDNode *
05627 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05628                              EVT VT1, EVT VT2, EVT VT3,
05629                              SDValue Op1, SDValue Op2, SDValue Op3) {
05630   SDVTList VTs = getVTList(VT1, VT2, VT3);
05631   SDValue Ops[] = { Op1, Op2, Op3 };
05632   return getMachineNode(Opcode, dl, VTs, Ops);
05633 }
05634 
05635 MachineSDNode *
05636 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05637                              EVT VT1, EVT VT2, EVT VT3,
05638                              ArrayRef<SDValue> Ops) {
05639   SDVTList VTs = getVTList(VT1, VT2, VT3);
05640   return getMachineNode(Opcode, dl, VTs, Ops);
05641 }
05642 
05643 MachineSDNode *
05644 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1,
05645                              EVT VT2, EVT VT3, EVT VT4,
05646                              ArrayRef<SDValue> Ops) {
05647   SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
05648   return getMachineNode(Opcode, dl, VTs, Ops);
05649 }
05650 
05651 MachineSDNode *
05652 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05653                              ArrayRef<EVT> ResultTys,
05654                              ArrayRef<SDValue> Ops) {
05655   SDVTList VTs = getVTList(ResultTys);
05656   return getMachineNode(Opcode, dl, VTs, Ops);
05657 }
05658 
05659 MachineSDNode *
05660 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
05661                              ArrayRef<SDValue> OpsArray) {
05662   bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
05663   MachineSDNode *N;
05664   void *IP = nullptr;
05665   const SDValue *Ops = OpsArray.data();
05666   unsigned NumOps = OpsArray.size();
05667 
05668   if (DoCSE) {
05669     FoldingSetNodeID ID;
05670     AddNodeIDNode(ID, ~Opcode, VTs, OpsArray);
05671     IP = nullptr;
05672     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
05673       return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL));
05674     }
05675   }
05676 
05677   // Allocate a new MachineSDNode.
05678   N = new (NodeAllocator) MachineSDNode(~Opcode, DL.getIROrder(),
05679                                         DL.getDebugLoc(), VTs);
05680 
05681   // Initialize the operands list.
05682   if (NumOps > array_lengthof(N->LocalOperands))
05683     // We're creating a final node that will live unmorphed for the
05684     // remainder of the current SelectionDAG iteration, so we can allocate
05685     // the operands directly out of a pool with no recycling metadata.
05686     N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
05687                     Ops, NumOps);
05688   else
05689     N->InitOperands(N->LocalOperands, Ops, NumOps);
05690   N->OperandsNeedDelete = false;
05691 
05692   if (DoCSE)
05693     CSEMap.InsertNode(N, IP);
05694 
05695   InsertNode(N);
05696   return N;
05697 }
05698 
05699 /// getTargetExtractSubreg - A convenience function for creating
05700 /// TargetOpcode::EXTRACT_SUBREG nodes.
05701 SDValue
05702 SelectionDAG::getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT,
05703                                      SDValue Operand) {
05704   SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
05705   SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
05706                                   VT, Operand, SRIdxVal);
05707   return SDValue(Subreg, 0);
05708 }
05709 
05710 /// getTargetInsertSubreg - A convenience function for creating
05711 /// TargetOpcode::INSERT_SUBREG nodes.
05712 SDValue
05713 SelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT,
05714                                     SDValue Operand, SDValue Subreg) {
05715   SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
05716   SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
05717                                   VT, Operand, Subreg, SRIdxVal);
05718   return SDValue(Result, 0);
05719 }
05720 
05721 /// getNodeIfExists - Get the specified node if it's already available, or
05722 /// else return NULL.
05723 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
05724                                       ArrayRef<SDValue> Ops, bool nuw, bool nsw,
05725                                       bool exact) {
05726   if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
05727     FoldingSetNodeID ID;
05728     AddNodeIDNode(ID, Opcode, VTList, Ops);
05729     if (isBinOpWithFlags(Opcode))
05730       AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
05731     void *IP = nullptr;
05732     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05733       return E;
05734   }
05735   return nullptr;
05736 }
05737 
05738 /// getDbgValue - Creates a SDDbgValue node.
05739 ///
05740 /// SDNode
05741 SDDbgValue *SelectionDAG::getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N,
05742                                       unsigned R, bool IsIndirect, uint64_t Off,
05743                                       DebugLoc DL, unsigned O) {
05744   return new (Allocator) SDDbgValue(Var, Expr, N, R, IsIndirect, Off, DL, O);
05745 }
05746 
05747 /// Constant
05748 SDDbgValue *SelectionDAG::getConstantDbgValue(MDNode *Var, MDNode *Expr,
05749                                               const Value *C, uint64_t Off,
05750                                               DebugLoc DL, unsigned O) {
05751   return new (Allocator) SDDbgValue(Var, Expr, C, Off, DL, O);
05752 }
05753 
05754 /// FrameIndex
05755 SDDbgValue *SelectionDAG::getFrameIndexDbgValue(MDNode *Var, MDNode *Expr,
05756                                                 unsigned FI, uint64_t Off,
05757                                                 DebugLoc DL, unsigned O) {
05758   return new (Allocator) SDDbgValue(Var, Expr, FI, Off, DL, O);
05759 }
05760 
05761 namespace {
05762 
05763 /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
05764 /// pointed to by a use iterator is deleted, increment the use iterator
05765 /// so that it doesn't dangle.
05766 ///
05767 class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
05768   SDNode::use_iterator &UI;
05769   SDNode::use_iterator &UE;
05770 
05771   void NodeDeleted(SDNode *N, SDNode *E) override {
05772     // Increment the iterator as needed.
05773     while (UI != UE && N == *UI)
05774       ++UI;
05775   }
05776 
05777 public:
05778   RAUWUpdateListener(SelectionDAG &d,
05779                      SDNode::use_iterator &ui,
05780                      SDNode::use_iterator &ue)
05781     : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
05782 };
05783 
05784 }
05785 
05786 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05787 /// This can cause recursive merging of nodes in the DAG.
05788 ///
05789 /// This version assumes From has a single result value.
05790 ///
05791 void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
05792   SDNode *From = FromN.getNode();
05793   assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
05794          "Cannot replace with this method!");
05795   assert(From != To.getNode() && "Cannot replace uses of with self");
05796 
05797   // Iterate over all the existing uses of From. New uses will be added
05798   // to the beginning of the use list, which we avoid visiting.
05799   // This specifically avoids visiting uses of From that arise while the
05800   // replacement is happening, because any such uses would be the result
05801   // of CSE: If an existing node looks like From after one of its operands
05802   // is replaced by To, we don't want to replace of all its users with To
05803   // too. See PR3018 for more info.
05804   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05805   RAUWUpdateListener Listener(*this, UI, UE);
05806   while (UI != UE) {
05807     SDNode *User = *UI;
05808 
05809     // This node is about to morph, remove its old self from the CSE maps.
05810     RemoveNodeFromCSEMaps(User);
05811 
05812     // A user can appear in a use list multiple times, and when this
05813     // happens the uses are usually next to each other in the list.
05814     // To help reduce the number of CSE recomputations, process all
05815     // the uses of this user that we can find this way.
05816     do {
05817       SDUse &Use = UI.getUse();
05818       ++UI;
05819       Use.set(To);
05820     } while (UI != UE && *UI == User);
05821 
05822     // Now that we have modified User, add it back to the CSE maps.  If it
05823     // already exists there, recursively merge the results together.
05824     AddModifiedNodeToCSEMaps(User);
05825   }
05826 
05827   // If we just RAUW'd the root, take note.
05828   if (FromN == getRoot())
05829     setRoot(To);
05830 }
05831 
05832 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05833 /// This can cause recursive merging of nodes in the DAG.
05834 ///
05835 /// This version assumes that for each value of From, there is a
05836 /// corresponding value in To in the same position with the same type.
05837 ///
05838 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
05839 #ifndef NDEBUG
05840   for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
05841     assert((!From->hasAnyUseOfValue(i) ||
05842             From->getValueType(i) == To->getValueType(i)) &&
05843            "Cannot use this version of ReplaceAllUsesWith!");
05844 #endif
05845 
05846   // Handle the trivial case.
05847   if (From == To)
05848     return;
05849 
05850   // Iterate over just the existing users of From. See the comments in
05851   // the ReplaceAllUsesWith above.
05852   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05853   RAUWUpdateListener Listener(*this, UI, UE);
05854   while (UI != UE) {
05855     SDNode *User = *UI;
05856 
05857     // This node is about to morph, remove its old self from the CSE maps.
05858     RemoveNodeFromCSEMaps(User);
05859 
05860     // A user can appear in a use list multiple times, and when this
05861     // happens the uses are usually next to each other in the list.
05862     // To help reduce the number of CSE recomputations, process all
05863     // the uses of this user that we can find this way.
05864     do {
05865       SDUse &Use = UI.getUse();
05866       ++UI;
05867       Use.setNode(To);
05868     } while (UI != UE && *UI == User);
05869 
05870     // Now that we have modified User, add it back to the CSE maps.  If it
05871     // already exists there, recursively merge the results together.
05872     AddModifiedNodeToCSEMaps(User);
05873   }
05874 
05875   // If we just RAUW'd the root, take note.
05876   if (From == getRoot().getNode())
05877     setRoot(SDValue(To, getRoot().getResNo()));
05878 }
05879 
05880 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05881 /// This can cause recursive merging of nodes in the DAG.
05882 ///
05883 /// This version can replace From with any result values.  To must match the
05884 /// number and types of values returned by From.
05885 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
05886   if (From->getNumValues() == 1)  // Handle the simple case efficiently.
05887     return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
05888 
05889   // Iterate over just the existing users of From. See the comments in
05890   // the ReplaceAllUsesWith above.
05891   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05892   RAUWUpdateListener Listener(*this, UI, UE);
05893   while (UI != UE) {
05894     SDNode *User = *UI;
05895 
05896     // This node is about to morph, remove its old self from the CSE maps.
05897     RemoveNodeFromCSEMaps(User);
05898 
05899     // A user can appear in a use list multiple times, and when this
05900     // happens the uses are usually next to each other in the list.
05901     // To help reduce the number of CSE recomputations, process all
05902     // the uses of this user that we can find this way.
05903     do {
05904       SDUse &Use = UI.getUse();
05905       const SDValue &ToOp = To[Use.getResNo()];
05906       ++UI;
05907       Use.set(ToOp);
05908     } while (UI != UE && *UI == User);
05909 
05910     // Now that we have modified User, add it back to the CSE maps.  If it
05911     // already exists there, recursively merge the results together.
05912     AddModifiedNodeToCSEMaps(User);
05913   }
05914 
05915   // If we just RAUW'd the root, take note.
05916   if (From == getRoot().getNode())
05917     setRoot(SDValue(To[getRoot().getResNo()]));
05918 }
05919 
05920 /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
05921 /// uses of other values produced by From.getNode() alone.  The Deleted
05922 /// vector is handled the same way as for ReplaceAllUsesWith.
05923 void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
05924   // Handle the really simple, really trivial case efficiently.
05925   if (From == To) return;
05926 
05927   // Handle the simple, trivial, case efficiently.
05928   if (From.getNode()->getNumValues() == 1) {
05929     ReplaceAllUsesWith(From, To);
05930     return;
05931   }
05932 
05933   // Iterate over just the existing users of From. See the comments in
05934   // the ReplaceAllUsesWith above.
05935   SDNode::use_iterator UI = From.getNode()->use_begin(),
05936                        UE = From.getNode()->use_end();
05937   RAUWUpdateListener Listener(*this, UI, UE);
05938   while (UI != UE) {
05939     SDNode *User = *UI;
05940     bool UserRemovedFromCSEMaps = false;
05941 
05942     // A user can appear in a use list multiple times, and when this
05943     // happens the uses are usually next to each other in the list.
05944     // To help reduce the number of CSE recomputations, process all
05945     // the uses of this user that we can find this way.
05946     do {
05947       SDUse &Use = UI.getUse();
05948 
05949       // Skip uses of different values from the same node.
05950       if (Use.getResNo() != From.getResNo()) {
05951         ++UI;
05952         continue;
05953       }
05954 
05955       // If this node hasn't been modified yet, it's still in the CSE maps,
05956       // so remove its old self from the CSE maps.
05957       if (!UserRemovedFromCSEMaps) {
05958         RemoveNodeFromCSEMaps(User);
05959         UserRemovedFromCSEMaps = true;
05960       }
05961 
05962       ++UI;
05963       Use.set(To);
05964     } while (UI != UE && *UI == User);
05965 
05966     // We are iterating over all uses of the From node, so if a use
05967     // doesn't use the specific value, no changes are made.
05968     if (!UserRemovedFromCSEMaps)
05969       continue;
05970 
05971     // Now that we have modified User, add it back to the CSE maps.  If it
05972     // already exists there, recursively merge the results together.
05973     AddModifiedNodeToCSEMaps(User);
05974   }
05975 
05976   // If we just RAUW'd the root, take note.
05977   if (From == getRoot())
05978     setRoot(To);
05979 }
05980 
05981 namespace {
05982   /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
05983   /// to record information about a use.
05984   struct UseMemo {
05985     SDNode *User;
05986     unsigned Index;
05987     SDUse *Use;
05988   };
05989 
05990   /// operator< - Sort Memos by User.
05991   bool operator<(const UseMemo &L, const UseMemo &R) {
05992     return (intptr_t)L.User < (intptr_t)R.User;
05993   }
05994 }
05995 
05996 /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
05997 /// uses of other values produced by From.getNode() alone.  The same value
05998 /// may appear in both the From and To list.  The Deleted vector is
05999 /// handled the same way as for ReplaceAllUsesWith.
06000 void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
06001                                               const SDValue *To,
06002                                               unsigned Num){
06003   // Handle the simple, trivial case efficiently.
06004   if (Num == 1)
06005     return ReplaceAllUsesOfValueWith(*From, *To);
06006 
06007   // Read up all the uses and make records of them. This helps
06008   // processing new uses that are introduced during the
06009   // replacement process.
06010   SmallVector<UseMemo, 4> Uses;
06011   for (unsigned i = 0; i != Num; ++i) {
06012     unsigned FromResNo = From[i].getResNo();
06013     SDNode *FromNode = From[i].getNode();
06014     for (SDNode::use_iterator UI = FromNode->use_begin(),
06015          E = FromNode->use_end(); UI != E; ++UI) {
06016       SDUse &Use = UI.getUse();
06017       if (Use.getResNo() == FromResNo) {
06018         UseMemo Memo = { *UI, i, &Use };
06019         Uses.push_back(Memo);
06020       }
06021     }
06022   }
06023 
06024   // Sort the uses, so that all the uses from a given User are together.
06025   std::sort(Uses.begin(), Uses.end());
06026 
06027   for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
06028        UseIndex != UseIndexEnd; ) {
06029     // We know that this user uses some value of From.  If it is the right
06030     // value, update it.
06031     SDNode *User = Uses[UseIndex].User;
06032 
06033     // This node is about to morph, remove its old self from the CSE maps.
06034     RemoveNodeFromCSEMaps(User);
06035 
06036     // The Uses array is sorted, so all the uses for a given User
06037     // are next to each other in the list.
06038     // To help reduce the number of CSE recomputations, process all
06039     // the uses of this user that we can find this way.
06040     do {
06041       unsigned i = Uses[UseIndex].Index;
06042       SDUse &Use = *Uses[UseIndex].Use;
06043       ++UseIndex;
06044 
06045       Use.set(To[i]);
06046     } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
06047 
06048     // Now that we have modified User, add it back to the CSE maps.  If it
06049     // already exists there, recursively merge the results together.
06050     AddModifiedNodeToCSEMaps(User);
06051   }
06052 }
06053 
06054 /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
06055 /// based on their topological order. It returns the maximum id and a vector
06056 /// of the SDNodes* in assigned order by reference.
06057 unsigned SelectionDAG::AssignTopologicalOrder() {
06058 
06059   unsigned DAGSize = 0;
06060 
06061   // SortedPos tracks the progress of the algorithm. Nodes before it are
06062   // sorted, nodes after it are unsorted. When the algorithm completes
06063   // it is at the end of the list.
06064   allnodes_iterator SortedPos = allnodes_begin();
06065 
06066   // Visit all the nodes. Move nodes with no operands to the front of
06067   // the list immediately. Annotate nodes that do have operands with their
06068   // operand count. Before we do this, the Node Id fields of the nodes
06069   // may contain arbitrary values. After, the Node Id fields for nodes
06070   // before SortedPos will contain the topological sort index, and the
06071   // Node Id fields for nodes At SortedPos and after will contain the
06072   // count of outstanding operands.
06073   for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
06074     SDNode *N = I++;
06075     checkForCycles(N, this);
06076     unsigned Degree = N->getNumOperands();
06077     if (Degree == 0) {
06078       // A node with no uses, add it to the result array immediately.
06079       N->setNodeId(DAGSize++);
06080       allnodes_iterator Q = N;
06081       if (Q != SortedPos)
06082         SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
06083       assert(SortedPos != AllNodes.end() && "Overran node list");
06084       ++SortedPos;
06085     } else {
06086       // Temporarily use the Node Id as scratch space for the degree count.
06087       N->setNodeId(Degree);
06088     }
06089   }
06090 
06091   // Visit all the nodes. As we iterate, move nodes into sorted order,
06092   // such that by the time the end is reached all nodes will be sorted.
06093   for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) {
06094     SDNode *N = I;
06095     checkForCycles(N, this);
06096     // N is in sorted position, so all its uses have one less operand
06097     // that needs to be sorted.
06098     for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
06099          UI != UE; ++UI) {
06100       SDNode *P = *UI;
06101       unsigned Degree = P->getNodeId();
06102       assert(Degree != 0 && "Invalid node degree");
06103       --Degree;
06104       if (Degree == 0) {
06105         // All of P's operands are sorted, so P may sorted now.
06106         P->setNodeId(DAGSize++);
06107         if (P != SortedPos)
06108           SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
06109         assert(SortedPos != AllNodes.end() && "Overran node list");
06110         ++SortedPos;
06111       } else {
06112         // Update P's outstanding operand count.
06113         P->setNodeId(Degree);
06114       }
06115     }
06116     if (I == SortedPos) {
06117 #ifndef NDEBUG
06118       SDNode *S = ++I;
06119       dbgs() << "Overran sorted position:\n";
06120       S->dumprFull(this); dbgs() << "\n";
06121       dbgs() << "Checking if this is due to cycles\n";
06122       checkForCycles(this, true);
06123 #endif
06124       llvm_unreachable(nullptr);
06125     }
06126   }
06127 
06128   assert(SortedPos == AllNodes.end() &&
06129          "Topological sort incomplete!");
06130   assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
06131          "First node in topological sort is not the entry token!");
06132   assert(AllNodes.front().getNodeId() == 0 &&
06133          "First node in topological sort has non-zero id!");
06134   assert(AllNodes.front().getNumOperands() == 0 &&
06135          "First node in topological sort has operands!");
06136   assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
06137          "Last node in topologic sort has unexpected id!");
06138   assert(AllNodes.back().use_empty() &&
06139          "Last node in topologic sort has users!");
06140   assert(DAGSize == allnodes_size() && "Node count mismatch!");
06141   return DAGSize;
06142 }
06143 
06144 /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
06145 /// value is produced by SD.
06146 void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) {
06147   DbgInfo->add(DB, SD, isParameter);
06148   if (SD)
06149     SD->setHasDebugValue(true);
06150 }
06151 
06152 /// TransferDbgValues - Transfer SDDbgValues.
06153 void SelectionDAG::TransferDbgValues(SDValue From, SDValue To) {
06154   if (From == To || !From.getNode()->getHasDebugValue())
06155     return;
06156   SDNode *FromNode = From.getNode();
06157   SDNode *ToNode = To.getNode();
06158   ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode);
06159   SmallVector<SDDbgValue *, 2> ClonedDVs;
06160   for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end();
06161        I != E; ++I) {
06162     SDDbgValue *Dbg = *I;
06163     if (Dbg->getKind() == SDDbgValue::SDNODE) {
06164       SDDbgValue *Clone =
06165           getDbgValue(Dbg->getVariable(), Dbg->getExpression(), ToNode,
06166                       To.getResNo(), Dbg->isIndirect(), Dbg->getOffset(),
06167                       Dbg->getDebugLoc(), Dbg->getOrder());
06168       ClonedDVs.push_back(Clone);
06169     }
06170   }
06171   for (SmallVectorImpl<SDDbgValue *>::iterator I = ClonedDVs.begin(),
06172          E = ClonedDVs.end(); I != E; ++I)
06173     AddDbgValue(*I, ToNode, false);
06174 }
06175 
06176 //===----------------------------------------------------------------------===//
06177 //                              SDNode Class
06178 //===----------------------------------------------------------------------===//
06179 
06180 HandleSDNode::~HandleSDNode() {
06181   DropOperands();
06182 }
06183 
06184 GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
06185                                          DebugLoc DL, const GlobalValue *GA,
06186                                          EVT VT, int64_t o, unsigned char TF)
06187   : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
06188   TheGlobal = GA;
06189 }
06190 
06191 AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT,
06192                                          SDValue X, unsigned SrcAS,
06193                                          unsigned DestAS)
06194  : UnarySDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT), X),
06195    SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
06196 
06197 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
06198                      EVT memvt, MachineMemOperand *mmo)
06199  : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
06200   SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
06201                                       MMO->isNonTemporal(), MMO->isInvariant());
06202   assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
06203   assert(isNonTemporal() == MMO->isNonTemporal() &&
06204          "Non-temporal encoding error!");
06205   // We check here that the size of the memory operand fits within the size of
06206   // the MMO. This is because the MMO might indicate only a possible address
06207   // range instead of specifying the affected memory addresses precisely.
06208   assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
06209 }
06210 
06211 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
06212                      ArrayRef<SDValue> Ops, EVT memvt, MachineMemOperand *mmo)
06213    : SDNode(Opc, Order, dl, VTs, Ops),
06214      MemoryVT(memvt), MMO(mmo) {
06215   SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
06216                                       MMO->isNonTemporal(), MMO->isInvariant());
06217   assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
06218   assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
06219 }
06220 
06221 /// Profile - Gather unique data for the node.
06222 ///
06223 void SDNode::Profile(FoldingSetNodeID &ID) const {
06224   AddNodeIDNode(ID, this);
06225 }
06226 
06227 namespace {
06228   struct EVTArray {
06229     std::vector<EVT> VTs;
06230 
06231     EVTArray() {
06232       VTs.reserve(MVT::LAST_VALUETYPE);
06233       for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i)
06234         VTs.push_back(MVT((MVT::SimpleValueType)i));
06235     }
06236   };
06237 }
06238 
06239 static ManagedStatic<std::set<EVT, EVT::compareRawBits> > EVTs;
06240 static ManagedStatic<EVTArray> SimpleVTArray;
06241 static ManagedStatic<sys::SmartMutex<true> > VTMutex;
06242 
06243 /// getValueTypeList - Return a pointer to the specified value type.
06244 ///
06245 const EVT *SDNode::getValueTypeList(EVT VT) {
06246   if (VT.isExtended()) {
06247     sys::SmartScopedLock<true> Lock(*VTMutex);
06248     return &(*EVTs->insert(VT).first);
06249   } else {
06250     assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE &&
06251            "Value type out of range!");
06252     return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy];
06253   }
06254 }
06255 
06256 /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
06257 /// indicated value.  This method ignores uses of other values defined by this
06258 /// operation.
06259 bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
06260   assert(Value < getNumValues() && "Bad value!");
06261 
06262   // TODO: Only iterate over uses of a given value of the node
06263   for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
06264     if (UI.getUse().getResNo() == Value) {
06265       if (NUses == 0)
06266         return false;
06267       --NUses;
06268     }
06269   }
06270 
06271   // Found exactly the right number of uses?
06272   return NUses == 0;
06273 }
06274 
06275 
06276 /// hasAnyUseOfValue - Return true if there are any use of the indicated
06277 /// value. This method ignores uses of other values defined by this operation.
06278 bool SDNode::hasAnyUseOfValue(unsigned Value) const {
06279   assert(Value < getNumValues() && "Bad value!");
06280 
06281   for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI)
06282     if (UI.getUse().getResNo() == Value)
06283       return true;
06284 
06285   return false;
06286 }
06287 
06288 
06289 /// isOnlyUserOf - Return true if this node is the only use of N.
06290 ///
06291 bool SDNode::isOnlyUserOf(SDNode *N) const {
06292   bool Seen = false;
06293   for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
06294     SDNode *User = *I;
06295     if (User == this)
06296       Seen = true;
06297     else
06298       return false;
06299   }
06300 
06301   return Seen;
06302 }
06303 
06304 /// isOperand - Return true if this node is an operand of N.
06305 ///
06306 bool SDValue::isOperandOf(SDNode *N) const {
06307   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)