LLVM API Documentation

SelectionDAG.cpp
Go to the documentation of this file.
00001 //===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This implements the SelectionDAG class.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "llvm/CodeGen/SelectionDAG.h"
00015 #include "SDNodeDbgValue.h"
00016 #include "llvm/ADT/SetVector.h"
00017 #include "llvm/ADT/SmallPtrSet.h"
00018 #include "llvm/ADT/SmallSet.h"
00019 #include "llvm/ADT/SmallVector.h"
00020 #include "llvm/ADT/StringExtras.h"
00021 #include "llvm/Analysis/ValueTracking.h"
00022 #include "llvm/CodeGen/MachineBasicBlock.h"
00023 #include "llvm/CodeGen/MachineConstantPool.h"
00024 #include "llvm/CodeGen/MachineFrameInfo.h"
00025 #include "llvm/CodeGen/MachineModuleInfo.h"
00026 #include "llvm/IR/CallingConv.h"
00027 #include "llvm/IR/Constants.h"
00028 #include "llvm/IR/DataLayout.h"
00029 #include "llvm/IR/DebugInfo.h"
00030 #include "llvm/IR/DerivedTypes.h"
00031 #include "llvm/IR/Function.h"
00032 #include "llvm/IR/GlobalAlias.h"
00033 #include "llvm/IR/GlobalVariable.h"
00034 #include "llvm/IR/Intrinsics.h"
00035 #include "llvm/Support/CommandLine.h"
00036 #include "llvm/Support/Debug.h"
00037 #include "llvm/Support/ErrorHandling.h"
00038 #include "llvm/Support/ManagedStatic.h"
00039 #include "llvm/Support/MathExtras.h"
00040 #include "llvm/Support/Mutex.h"
00041 #include "llvm/Support/raw_ostream.h"
00042 #include "llvm/Target/TargetInstrInfo.h"
00043 #include "llvm/Target/TargetIntrinsicInfo.h"
00044 #include "llvm/Target/TargetLowering.h"
00045 #include "llvm/Target/TargetMachine.h"
00046 #include "llvm/Target/TargetOptions.h"
00047 #include "llvm/Target/TargetRegisterInfo.h"
00048 #include "llvm/Target/TargetSelectionDAGInfo.h"
00049 #include "llvm/Target/TargetSubtargetInfo.h"
00050 #include <algorithm>
00051 #include <cmath>
00052 
00053 using namespace llvm;
00054 
00055 /// makeVTList - Return an instance of the SDVTList struct initialized with the
00056 /// specified members.
00057 static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
00058   SDVTList Res = {VTs, NumVTs};
00059   return Res;
00060 }
00061 
00062 // Default null implementations of the callbacks.
00063 void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
00064 void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
00065 
00066 //===----------------------------------------------------------------------===//
00067 //                              ConstantFPSDNode Class
00068 //===----------------------------------------------------------------------===//
00069 
00070 /// isExactlyValue - We don't rely on operator== working on double values, as
00071 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
00072 /// As such, this method can be used to do an exact bit-for-bit comparison of
00073 /// two floating point values.
00074 bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
00075   return getValueAPF().bitwiseIsEqual(V);
00076 }
00077 
00078 bool ConstantFPSDNode::isValueValidForType(EVT VT,
00079                                            const APFloat& Val) {
00080   assert(VT.isFloatingPoint() && "Can only convert between FP types");
00081 
00082   // convert modifies in place, so make a copy.
00083   APFloat Val2 = APFloat(Val);
00084   bool losesInfo;
00085   (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
00086                       APFloat::rmNearestTiesToEven,
00087                       &losesInfo);
00088   return !losesInfo;
00089 }
00090 
00091 //===----------------------------------------------------------------------===//
00092 //                              ISD Namespace
00093 //===----------------------------------------------------------------------===//
00094 
00095 /// isBuildVectorAllOnes - Return true if the specified node is a
00096 /// BUILD_VECTOR where all of the elements are ~0 or undef.
00097 bool ISD::isBuildVectorAllOnes(const SDNode *N) {
00098   // Look through a bit convert.
00099   while (N->getOpcode() == ISD::BITCAST)
00100     N = N->getOperand(0).getNode();
00101 
00102   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
00103 
00104   unsigned i = 0, e = N->getNumOperands();
00105 
00106   // Skip over all of the undef values.
00107   while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF)
00108     ++i;
00109 
00110   // Do not accept an all-undef vector.
00111   if (i == e) return false;
00112 
00113   // Do not accept build_vectors that aren't all constants or which have non-~0
00114   // elements. We have to be a bit careful here, as the type of the constant
00115   // may not be the same as the type of the vector elements due to type
00116   // legalization (the elements are promoted to a legal type for the target and
00117   // a vector of a type may be legal when the base element type is not).
00118   // We only want to check enough bits to cover the vector elements, because
00119   // we care if the resultant vector is all ones, not whether the individual
00120   // constants are.
00121   SDValue NotZero = N->getOperand(i);
00122   unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
00123   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
00124     if (CN->getAPIntValue().countTrailingOnes() < EltSize)
00125       return false;
00126   } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
00127     if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
00128       return false;
00129   } else
00130     return false;
00131 
00132   // Okay, we have at least one ~0 value, check to see if the rest match or are
00133   // undefs. Even with the above element type twiddling, this should be OK, as
00134   // the same type legalization should have applied to all the elements.
00135   for (++i; i != e; ++i)
00136     if (N->getOperand(i) != NotZero &&
00137         N->getOperand(i).getOpcode() != ISD::UNDEF)
00138       return false;
00139   return true;
00140 }
00141 
00142 
00143 /// isBuildVectorAllZeros - Return true if the specified node is a
00144 /// BUILD_VECTOR where all of the elements are 0 or undef.
00145 bool ISD::isBuildVectorAllZeros(const SDNode *N) {
00146   // Look through a bit convert.
00147   while (N->getOpcode() == ISD::BITCAST)
00148     N = N->getOperand(0).getNode();
00149 
00150   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
00151 
00152   bool IsAllUndef = true;
00153   for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) {
00154     if (N->getOperand(i).getOpcode() == ISD::UNDEF)
00155       continue;
00156     IsAllUndef = false;
00157     // Do not accept build_vectors that aren't all constants or which have non-0
00158     // elements. We have to be a bit careful here, as the type of the constant
00159     // may not be the same as the type of the vector elements due to type
00160     // legalization (the elements are promoted to a legal type for the target
00161     // and a vector of a type may be legal when the base element type is not).
00162     // We only want to check enough bits to cover the vector elements, because
00163     // we care if the resultant vector is all zeros, not whether the individual
00164     // constants are.
00165     SDValue Zero = N->getOperand(i);
00166     unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
00167     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Zero)) {
00168       if (CN->getAPIntValue().countTrailingZeros() < EltSize)
00169         return false;
00170     } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Zero)) {
00171       if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
00172         return false;
00173     } else
00174       return false;
00175   }
00176 
00177   // Do not accept an all-undef vector.
00178   if (IsAllUndef)
00179     return false;
00180   return true;
00181 }
00182 
00183 /// \brief Return true if the specified node is a BUILD_VECTOR node of
00184 /// all ConstantSDNode or undef.
00185 bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
00186   if (N->getOpcode() != ISD::BUILD_VECTOR)
00187     return false;
00188 
00189   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
00190     SDValue Op = N->getOperand(i);
00191     if (Op.getOpcode() == ISD::UNDEF)
00192       continue;
00193     if (!isa<ConstantSDNode>(Op))
00194       return false;
00195   }
00196   return true;
00197 }
00198 
00199 /// isScalarToVector - Return true if the specified node is a
00200 /// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low
00201 /// element is not an undef.
00202 bool ISD::isScalarToVector(const SDNode *N) {
00203   if (N->getOpcode() == ISD::SCALAR_TO_VECTOR)
00204     return true;
00205 
00206   if (N->getOpcode() != ISD::BUILD_VECTOR)
00207     return false;
00208   if (N->getOperand(0).getOpcode() == ISD::UNDEF)
00209     return false;
00210   unsigned NumElems = N->getNumOperands();
00211   if (NumElems == 1)
00212     return false;
00213   for (unsigned i = 1; i < NumElems; ++i) {
00214     SDValue V = N->getOperand(i);
00215     if (V.getOpcode() != ISD::UNDEF)
00216       return false;
00217   }
00218   return true;
00219 }
00220 
00221 /// allOperandsUndef - Return true if the node has at least one operand
00222 /// and all operands of the specified node are ISD::UNDEF.
00223 bool ISD::allOperandsUndef(const SDNode *N) {
00224   // Return false if the node has no operands.
00225   // This is "logically inconsistent" with the definition of "all" but
00226   // is probably the desired behavior.
00227   if (N->getNumOperands() == 0)
00228     return false;
00229 
00230   for (unsigned i = 0, e = N->getNumOperands(); i != e ; ++i)
00231     if (N->getOperand(i).getOpcode() != ISD::UNDEF)
00232       return false;
00233 
00234   return true;
00235 }
00236 
00237 ISD::NodeType ISD::getExtForLoadExtType(ISD::LoadExtType ExtType) {
00238   switch (ExtType) {
00239   case ISD::EXTLOAD:
00240     return ISD::ANY_EXTEND;
00241   case ISD::SEXTLOAD:
00242     return ISD::SIGN_EXTEND;
00243   case ISD::ZEXTLOAD:
00244     return ISD::ZERO_EXTEND;
00245   default:
00246     break;
00247   }
00248 
00249   llvm_unreachable("Invalid LoadExtType");
00250 }
00251 
00252 /// getSetCCSwappedOperands - Return the operation corresponding to (Y op X)
00253 /// when given the operation for (X op Y).
00254 ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
00255   // To perform this operation, we just need to swap the L and G bits of the
00256   // operation.
00257   unsigned OldL = (Operation >> 2) & 1;
00258   unsigned OldG = (Operation >> 1) & 1;
00259   return ISD::CondCode((Operation & ~6) |  // Keep the N, U, E bits
00260                        (OldL << 1) |       // New G bit
00261                        (OldG << 2));       // New L bit.
00262 }
00263 
00264 /// getSetCCInverse - Return the operation corresponding to !(X op Y), where
00265 /// 'op' is a valid SetCC operation.
00266 ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) {
00267   unsigned Operation = Op;
00268   if (isInteger)
00269     Operation ^= 7;   // Flip L, G, E bits, but not U.
00270   else
00271     Operation ^= 15;  // Flip all of the condition bits.
00272 
00273   if (Operation > ISD::SETTRUE2)
00274     Operation &= ~8;  // Don't let N and U bits get set.
00275 
00276   return ISD::CondCode(Operation);
00277 }
00278 
00279 
00280 /// isSignedOp - For an integer comparison, return 1 if the comparison is a
00281 /// signed operation and 2 if the result is an unsigned comparison.  Return zero
00282 /// if the operation does not depend on the sign of the input (setne and seteq).
00283 static int isSignedOp(ISD::CondCode Opcode) {
00284   switch (Opcode) {
00285   default: llvm_unreachable("Illegal integer setcc operation!");
00286   case ISD::SETEQ:
00287   case ISD::SETNE: return 0;
00288   case ISD::SETLT:
00289   case ISD::SETLE:
00290   case ISD::SETGT:
00291   case ISD::SETGE: return 1;
00292   case ISD::SETULT:
00293   case ISD::SETULE:
00294   case ISD::SETUGT:
00295   case ISD::SETUGE: return 2;
00296   }
00297 }
00298 
00299 /// getSetCCOrOperation - Return the result of a logical OR between different
00300 /// comparisons of identical values: ((X op1 Y) | (X op2 Y)).  This function
00301 /// returns SETCC_INVALID if it is not possible to represent the resultant
00302 /// comparison.
00303 ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
00304                                        bool isInteger) {
00305   if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
00306     // Cannot fold a signed integer setcc with an unsigned integer setcc.
00307     return ISD::SETCC_INVALID;
00308 
00309   unsigned Op = Op1 | Op2;  // Combine all of the condition bits.
00310 
00311   // If the N and U bits get set then the resultant comparison DOES suddenly
00312   // care about orderedness, and is true when ordered.
00313   if (Op > ISD::SETTRUE2)
00314     Op &= ~16;     // Clear the U bit if the N bit is set.
00315 
00316   // Canonicalize illegal integer setcc's.
00317   if (isInteger && Op == ISD::SETUNE)  // e.g. SETUGT | SETULT
00318     Op = ISD::SETNE;
00319 
00320   return ISD::CondCode(Op);
00321 }
00322 
00323 /// getSetCCAndOperation - Return the result of a logical AND between different
00324 /// comparisons of identical values: ((X op1 Y) & (X op2 Y)).  This
00325 /// function returns zero if it is not possible to represent the resultant
00326 /// comparison.
00327 ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
00328                                         bool isInteger) {
00329   if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
00330     // Cannot fold a signed setcc with an unsigned setcc.
00331     return ISD::SETCC_INVALID;
00332 
00333   // Combine all of the condition bits.
00334   ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
00335 
00336   // Canonicalize illegal integer setcc's.
00337   if (isInteger) {
00338     switch (Result) {
00339     default: break;
00340     case ISD::SETUO : Result = ISD::SETFALSE; break;  // SETUGT & SETULT
00341     case ISD::SETOEQ:                                 // SETEQ  & SETU[LG]E
00342     case ISD::SETUEQ: Result = ISD::SETEQ   ; break;  // SETUGE & SETULE
00343     case ISD::SETOLT: Result = ISD::SETULT  ; break;  // SETULT & SETNE
00344     case ISD::SETOGT: Result = ISD::SETUGT  ; break;  // SETUGT & SETNE
00345     }
00346   }
00347 
00348   return Result;
00349 }
00350 
00351 //===----------------------------------------------------------------------===//
00352 //                           SDNode Profile Support
00353 //===----------------------------------------------------------------------===//
00354 
00355 /// AddNodeIDOpcode - Add the node opcode to the NodeID data.
00356 ///
00357 static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC)  {
00358   ID.AddInteger(OpC);
00359 }
00360 
00361 /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
00362 /// solely with their pointer.
00363 static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
00364   ID.AddPointer(VTList.VTs);
00365 }
00366 
00367 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
00368 ///
00369 static void AddNodeIDOperands(FoldingSetNodeID &ID,
00370                               ArrayRef<SDValue> Ops) {
00371   for (auto& Op : Ops) {
00372     ID.AddPointer(Op.getNode());
00373     ID.AddInteger(Op.getResNo());
00374   }
00375 }
00376 
00377 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
00378 ///
00379 static void AddNodeIDOperands(FoldingSetNodeID &ID,
00380                               ArrayRef<SDUse> Ops) {
00381   for (auto& Op : Ops) {
00382     ID.AddPointer(Op.getNode());
00383     ID.AddInteger(Op.getResNo());
00384   }
00385 }
00386 
00387 static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, bool nuw, bool nsw,
00388                                   bool exact) {
00389   ID.AddBoolean(nuw);
00390   ID.AddBoolean(nsw);
00391   ID.AddBoolean(exact);
00392 }
00393 
00394 /// AddBinaryNodeIDCustom - Add BinarySDNodes special infos
00395 static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, unsigned Opcode,
00396                                   bool nuw, bool nsw, bool exact) {
00397   if (isBinOpWithFlags(Opcode))
00398     AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
00399 }
00400 
00401 static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
00402                           SDVTList VTList, ArrayRef<SDValue> OpList) {
00403   AddNodeIDOpcode(ID, OpC);
00404   AddNodeIDValueTypes(ID, VTList);
00405   AddNodeIDOperands(ID, OpList);
00406 }
00407 
00408 /// AddNodeIDCustom - If this is an SDNode with special info, add this info to
00409 /// the NodeID data.
00410 static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
00411   switch (N->getOpcode()) {
00412   case ISD::TargetExternalSymbol:
00413   case ISD::ExternalSymbol:
00414     llvm_unreachable("Should only be used on nodes with operands");
00415   default: break;  // Normal nodes don't need extra info.
00416   case ISD::TargetConstant:
00417   case ISD::Constant: {
00418     const ConstantSDNode *C = cast<ConstantSDNode>(N);
00419     ID.AddPointer(C->getConstantIntValue());
00420     ID.AddBoolean(C->isOpaque());
00421     break;
00422   }
00423   case ISD::TargetConstantFP:
00424   case ISD::ConstantFP: {
00425     ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
00426     break;
00427   }
00428   case ISD::TargetGlobalAddress:
00429   case ISD::GlobalAddress:
00430   case ISD::TargetGlobalTLSAddress:
00431   case ISD::GlobalTLSAddress: {
00432     const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
00433     ID.AddPointer(GA->getGlobal());
00434     ID.AddInteger(GA->getOffset());
00435     ID.AddInteger(GA->getTargetFlags());
00436     ID.AddInteger(GA->getAddressSpace());
00437     break;
00438   }
00439   case ISD::BasicBlock:
00440     ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
00441     break;
00442   case ISD::Register:
00443     ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
00444     break;
00445   case ISD::RegisterMask:
00446     ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
00447     break;
00448   case ISD::SRCVALUE:
00449     ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
00450     break;
00451   case ISD::FrameIndex:
00452   case ISD::TargetFrameIndex:
00453     ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
00454     break;
00455   case ISD::JumpTable:
00456   case ISD::TargetJumpTable:
00457     ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
00458     ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
00459     break;
00460   case ISD::ConstantPool:
00461   case ISD::TargetConstantPool: {
00462     const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
00463     ID.AddInteger(CP->getAlignment());
00464     ID.AddInteger(CP->getOffset());
00465     if (CP->isMachineConstantPoolEntry())
00466       CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
00467     else
00468       ID.AddPointer(CP->getConstVal());
00469     ID.AddInteger(CP->getTargetFlags());
00470     break;
00471   }
00472   case ISD::TargetIndex: {
00473     const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
00474     ID.AddInteger(TI->getIndex());
00475     ID.AddInteger(TI->getOffset());
00476     ID.AddInteger(TI->getTargetFlags());
00477     break;
00478   }
00479   case ISD::LOAD: {
00480     const LoadSDNode *LD = cast<LoadSDNode>(N);
00481     ID.AddInteger(LD->getMemoryVT().getRawBits());
00482     ID.AddInteger(LD->getRawSubclassData());
00483     ID.AddInteger(LD->getPointerInfo().getAddrSpace());
00484     break;
00485   }
00486   case ISD::STORE: {
00487     const StoreSDNode *ST = cast<StoreSDNode>(N);
00488     ID.AddInteger(ST->getMemoryVT().getRawBits());
00489     ID.AddInteger(ST->getRawSubclassData());
00490     ID.AddInteger(ST->getPointerInfo().getAddrSpace());
00491     break;
00492   }
00493   case ISD::SDIV:
00494   case ISD::UDIV:
00495   case ISD::SRA:
00496   case ISD::SRL:
00497   case ISD::MUL:
00498   case ISD::ADD:
00499   case ISD::SUB:
00500   case ISD::SHL: {
00501     const BinaryWithFlagsSDNode *BinNode = cast<BinaryWithFlagsSDNode>(N);
00502     AddBinaryNodeIDCustom(ID, N->getOpcode(), BinNode->hasNoUnsignedWrap(),
00503                           BinNode->hasNoSignedWrap(), BinNode->isExact());
00504     break;
00505   }
00506   case ISD::ATOMIC_CMP_SWAP:
00507   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
00508   case ISD::ATOMIC_SWAP:
00509   case ISD::ATOMIC_LOAD_ADD:
00510   case ISD::ATOMIC_LOAD_SUB:
00511   case ISD::ATOMIC_LOAD_AND:
00512   case ISD::ATOMIC_LOAD_OR:
00513   case ISD::ATOMIC_LOAD_XOR:
00514   case ISD::ATOMIC_LOAD_NAND:
00515   case ISD::ATOMIC_LOAD_MIN:
00516   case ISD::ATOMIC_LOAD_MAX:
00517   case ISD::ATOMIC_LOAD_UMIN:
00518   case ISD::ATOMIC_LOAD_UMAX:
00519   case ISD::ATOMIC_LOAD:
00520   case ISD::ATOMIC_STORE: {
00521     const AtomicSDNode *AT = cast<AtomicSDNode>(N);
00522     ID.AddInteger(AT->getMemoryVT().getRawBits());
00523     ID.AddInteger(AT->getRawSubclassData());
00524     ID.AddInteger(AT->getPointerInfo().getAddrSpace());
00525     break;
00526   }
00527   case ISD::PREFETCH: {
00528     const MemSDNode *PF = cast<MemSDNode>(N);
00529     ID.AddInteger(PF->getPointerInfo().getAddrSpace());
00530     break;
00531   }
00532   case ISD::VECTOR_SHUFFLE: {
00533     const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
00534     for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
00535          i != e; ++i)
00536       ID.AddInteger(SVN->getMaskElt(i));
00537     break;
00538   }
00539   case ISD::TargetBlockAddress:
00540   case ISD::BlockAddress: {
00541     const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
00542     ID.AddPointer(BA->getBlockAddress());
00543     ID.AddInteger(BA->getOffset());
00544     ID.AddInteger(BA->getTargetFlags());
00545     break;
00546   }
00547   } // end switch (N->getOpcode())
00548 
00549   // Target specific memory nodes could also have address spaces to check.
00550   if (N->isTargetMemoryOpcode())
00551     ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
00552 }
00553 
00554 /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
00555 /// data.
00556 static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
00557   AddNodeIDOpcode(ID, N->getOpcode());
00558   // Add the return value info.
00559   AddNodeIDValueTypes(ID, N->getVTList());
00560   // Add the operand info.
00561   AddNodeIDOperands(ID, N->ops());
00562 
00563   // Handle SDNode leafs with special info.
00564   AddNodeIDCustom(ID, N);
00565 }
00566 
00567 /// encodeMemSDNodeFlags - Generic routine for computing a value for use in
00568 /// the CSE map that carries volatility, temporalness, indexing mode, and
00569 /// extension/truncation information.
00570 ///
00571 static inline unsigned
00572 encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile,
00573                      bool isNonTemporal, bool isInvariant) {
00574   assert((ConvType & 3) == ConvType &&
00575          "ConvType may not require more than 2 bits!");
00576   assert((AM & 7) == AM &&
00577          "AM may not require more than 3 bits!");
00578   return ConvType |
00579          (AM << 2) |
00580          (isVolatile << 5) |
00581          (isNonTemporal << 6) |
00582          (isInvariant << 7);
00583 }
00584 
00585 //===----------------------------------------------------------------------===//
00586 //                              SelectionDAG Class
00587 //===----------------------------------------------------------------------===//
00588 
00589 /// doNotCSE - Return true if CSE should not be performed for this node.
00590 static bool doNotCSE(SDNode *N) {
00591   if (N->getValueType(0) == MVT::Glue)
00592     return true; // Never CSE anything that produces a flag.
00593 
00594   switch (N->getOpcode()) {
00595   default: break;
00596   case ISD::HANDLENODE:
00597   case ISD::EH_LABEL:
00598     return true;   // Never CSE these nodes.
00599   }
00600 
00601   // Check that remaining values produced are not flags.
00602   for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
00603     if (N->getValueType(i) == MVT::Glue)
00604       return true; // Never CSE anything that produces a flag.
00605 
00606   return false;
00607 }
00608 
00609 /// RemoveDeadNodes - This method deletes all unreachable nodes in the
00610 /// SelectionDAG.
00611 void SelectionDAG::RemoveDeadNodes() {
00612   // Create a dummy node (which is not added to allnodes), that adds a reference
00613   // to the root node, preventing it from being deleted.
00614   HandleSDNode Dummy(getRoot());
00615 
00616   SmallVector<SDNode*, 128> DeadNodes;
00617 
00618   // Add all obviously-dead nodes to the DeadNodes worklist.
00619   for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I)
00620     if (I->use_empty())
00621       DeadNodes.push_back(I);
00622 
00623   RemoveDeadNodes(DeadNodes);
00624 
00625   // If the root changed (e.g. it was a dead load, update the root).
00626   setRoot(Dummy.getValue());
00627 }
00628 
00629 /// RemoveDeadNodes - This method deletes the unreachable nodes in the
00630 /// given list, and any nodes that become unreachable as a result.
00631 void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
00632 
00633   // Process the worklist, deleting the nodes and adding their uses to the
00634   // worklist.
00635   while (!DeadNodes.empty()) {
00636     SDNode *N = DeadNodes.pop_back_val();
00637 
00638     for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00639       DUL->NodeDeleted(N, nullptr);
00640 
00641     // Take the node out of the appropriate CSE map.
00642     RemoveNodeFromCSEMaps(N);
00643 
00644     // Next, brutally remove the operand list.  This is safe to do, as there are
00645     // no cycles in the graph.
00646     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
00647       SDUse &Use = *I++;
00648       SDNode *Operand = Use.getNode();
00649       Use.set(SDValue());
00650 
00651       // Now that we removed this operand, see if there are no uses of it left.
00652       if (Operand->use_empty())
00653         DeadNodes.push_back(Operand);
00654     }
00655 
00656     DeallocateNode(N);
00657   }
00658 }
00659 
00660 void SelectionDAG::RemoveDeadNode(SDNode *N){
00661   SmallVector<SDNode*, 16> DeadNodes(1, N);
00662 
00663   // Create a dummy node that adds a reference to the root node, preventing
00664   // it from being deleted.  (This matters if the root is an operand of the
00665   // dead node.)
00666   HandleSDNode Dummy(getRoot());
00667 
00668   RemoveDeadNodes(DeadNodes);
00669 }
00670 
00671 void SelectionDAG::DeleteNode(SDNode *N) {
00672   // First take this out of the appropriate CSE map.
00673   RemoveNodeFromCSEMaps(N);
00674 
00675   // Finally, remove uses due to operands of this node, remove from the
00676   // AllNodes list, and delete the node.
00677   DeleteNodeNotInCSEMaps(N);
00678 }
00679 
00680 void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
00681   assert(N != AllNodes.begin() && "Cannot delete the entry node!");
00682   assert(N->use_empty() && "Cannot delete a node that is not dead!");
00683 
00684   // Drop all of the operands and decrement used node's use counts.
00685   N->DropOperands();
00686 
00687   DeallocateNode(N);
00688 }
00689 
00690 void SDDbgInfo::erase(const SDNode *Node) {
00691   DbgValMapType::iterator I = DbgValMap.find(Node);
00692   if (I == DbgValMap.end())
00693     return;
00694   for (auto &Val: I->second)
00695     Val->setIsInvalidated();
00696   DbgValMap.erase(I);
00697 }
00698 
00699 void SelectionDAG::DeallocateNode(SDNode *N) {
00700   if (N->OperandsNeedDelete)
00701     delete[] N->OperandList;
00702 
00703   // Set the opcode to DELETED_NODE to help catch bugs when node
00704   // memory is reallocated.
00705   N->NodeType = ISD::DELETED_NODE;
00706 
00707   NodeAllocator.Deallocate(AllNodes.remove(N));
00708 
00709   // If any of the SDDbgValue nodes refer to this SDNode, invalidate
00710   // them and forget about that node.
00711   DbgInfo->erase(N);
00712 }
00713 
00714 #ifndef NDEBUG
00715 /// VerifySDNode - Sanity check the given SDNode.  Aborts if it is invalid.
00716 static void VerifySDNode(SDNode *N) {
00717   switch (N->getOpcode()) {
00718   default:
00719     break;
00720   case ISD::BUILD_PAIR: {
00721     EVT VT = N->getValueType(0);
00722     assert(N->getNumValues() == 1 && "Too many results!");
00723     assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
00724            "Wrong return type!");
00725     assert(N->getNumOperands() == 2 && "Wrong number of operands!");
00726     assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
00727            "Mismatched operand types!");
00728     assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
00729            "Wrong operand type!");
00730     assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
00731            "Wrong return type size");
00732     break;
00733   }
00734   case ISD::BUILD_VECTOR: {
00735     assert(N->getNumValues() == 1 && "Too many results!");
00736     assert(N->getValueType(0).isVector() && "Wrong return type!");
00737     assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
00738            "Wrong number of operands!");
00739     EVT EltVT = N->getValueType(0).getVectorElementType();
00740     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
00741       assert((I->getValueType() == EltVT ||
00742              (EltVT.isInteger() && I->getValueType().isInteger() &&
00743               EltVT.bitsLE(I->getValueType()))) &&
00744             "Wrong operand type!");
00745       assert(I->getValueType() == N->getOperand(0).getValueType() &&
00746              "Operands must all have the same type");
00747     }
00748     break;
00749   }
00750   }
00751 }
00752 #endif // NDEBUG
00753 
00754 /// \brief Insert a newly allocated node into the DAG.
00755 ///
00756 /// Handles insertion into the all nodes list and CSE map, as well as
00757 /// verification and other common operations when a new node is allocated.
00758 void SelectionDAG::InsertNode(SDNode *N) {
00759   AllNodes.push_back(N);
00760 #ifndef NDEBUG
00761   VerifySDNode(N);
00762 #endif
00763 }
00764 
00765 /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
00766 /// correspond to it.  This is useful when we're about to delete or repurpose
00767 /// the node.  We don't want future request for structurally identical nodes
00768 /// to return N anymore.
00769 bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
00770   bool Erased = false;
00771   switch (N->getOpcode()) {
00772   case ISD::HANDLENODE: return false;  // noop.
00773   case ISD::CONDCODE:
00774     assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
00775            "Cond code doesn't exist!");
00776     Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
00777     CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
00778     break;
00779   case ISD::ExternalSymbol:
00780     Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
00781     break;
00782   case ISD::TargetExternalSymbol: {
00783     ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
00784     Erased = TargetExternalSymbols.erase(
00785                std::pair<std::string,unsigned char>(ESN->getSymbol(),
00786                                                     ESN->getTargetFlags()));
00787     break;
00788   }
00789   case ISD::VALUETYPE: {
00790     EVT VT = cast<VTSDNode>(N)->getVT();
00791     if (VT.isExtended()) {
00792       Erased = ExtendedValueTypeNodes.erase(VT);
00793     } else {
00794       Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
00795       ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
00796     }
00797     break;
00798   }
00799   default:
00800     // Remove it from the CSE Map.
00801     assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
00802     assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
00803     Erased = CSEMap.RemoveNode(N);
00804     break;
00805   }
00806 #ifndef NDEBUG
00807   // Verify that the node was actually in one of the CSE maps, unless it has a
00808   // flag result (which cannot be CSE'd) or is one of the special cases that are
00809   // not subject to CSE.
00810   if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
00811       !N->isMachineOpcode() && !doNotCSE(N)) {
00812     N->dump(this);
00813     dbgs() << "\n";
00814     llvm_unreachable("Node is not in map!");
00815   }
00816 #endif
00817   return Erased;
00818 }
00819 
00820 /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
00821 /// maps and modified in place. Add it back to the CSE maps, unless an identical
00822 /// node already exists, in which case transfer all its users to the existing
00823 /// node. This transfer can potentially trigger recursive merging.
00824 ///
00825 void
00826 SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
00827   // For node types that aren't CSE'd, just act as if no identical node
00828   // already exists.
00829   if (!doNotCSE(N)) {
00830     SDNode *Existing = CSEMap.GetOrInsertNode(N);
00831     if (Existing != N) {
00832       // If there was already an existing matching node, use ReplaceAllUsesWith
00833       // to replace the dead one with the existing one.  This can cause
00834       // recursive merging of other unrelated nodes down the line.
00835       ReplaceAllUsesWith(N, Existing);
00836 
00837       // N is now dead. Inform the listeners and delete it.
00838       for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00839         DUL->NodeDeleted(N, Existing);
00840       DeleteNodeNotInCSEMaps(N);
00841       return;
00842     }
00843   }
00844 
00845   // If the node doesn't already exist, we updated it.  Inform listeners.
00846   for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00847     DUL->NodeUpdated(N);
00848 }
00849 
00850 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00851 /// were replaced with those specified.  If this node is never memoized,
00852 /// return null, otherwise return a pointer to the slot it would take.  If a
00853 /// node already exists with these operands, the slot will be non-null.
00854 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
00855                                            void *&InsertPos) {
00856   if (doNotCSE(N))
00857     return nullptr;
00858 
00859   SDValue Ops[] = { Op };
00860   FoldingSetNodeID ID;
00861   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00862   AddNodeIDCustom(ID, N);
00863   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00864   return Node;
00865 }
00866 
00867 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00868 /// were replaced with those specified.  If this node is never memoized,
00869 /// return null, otherwise return a pointer to the slot it would take.  If a
00870 /// node already exists with these operands, the slot will be non-null.
00871 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
00872                                            SDValue Op1, SDValue Op2,
00873                                            void *&InsertPos) {
00874   if (doNotCSE(N))
00875     return nullptr;
00876 
00877   SDValue Ops[] = { Op1, Op2 };
00878   FoldingSetNodeID ID;
00879   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00880   AddNodeIDCustom(ID, N);
00881   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00882   return Node;
00883 }
00884 
00885 
00886 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00887 /// were replaced with those specified.  If this node is never memoized,
00888 /// return null, otherwise return a pointer to the slot it would take.  If a
00889 /// node already exists with these operands, the slot will be non-null.
00890 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
00891                                            void *&InsertPos) {
00892   if (doNotCSE(N))
00893     return nullptr;
00894 
00895   FoldingSetNodeID ID;
00896   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00897   AddNodeIDCustom(ID, N);
00898   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00899   return Node;
00900 }
00901 
00902 /// getEVTAlignment - Compute the default alignment value for the
00903 /// given type.
00904 ///
00905 unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
00906   Type *Ty = VT == MVT::iPTR ?
00907                    PointerType::get(Type::getInt8Ty(*getContext()), 0) :
00908                    VT.getTypeForEVT(*getContext());
00909 
00910   return TLI->getDataLayout()->getABITypeAlignment(Ty);
00911 }
00912 
00913 // EntryNode could meaningfully have debug info if we can find it...
00914 SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
00915     : TM(tm), TSI(nullptr), TLI(nullptr), OptLevel(OL),
00916       EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
00917       Root(getEntryNode()), NewNodesMustHaveLegalTypes(false),
00918       UpdateListeners(nullptr) {
00919   AllNodes.push_back(&EntryNode);
00920   DbgInfo = new SDDbgInfo();
00921 }
00922 
00923 void SelectionDAG::init(MachineFunction &mf) {
00924   MF = &mf;
00925   TLI = getSubtarget().getTargetLowering();
00926   TSI = getSubtarget().getSelectionDAGInfo();
00927   Context = &mf.getFunction()->getContext();
00928 }
00929 
00930 SelectionDAG::~SelectionDAG() {
00931   assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
00932   allnodes_clear();
00933   delete DbgInfo;
00934 }
00935 
00936 void SelectionDAG::allnodes_clear() {
00937   assert(&*AllNodes.begin() == &EntryNode);
00938   AllNodes.remove(AllNodes.begin());
00939   while (!AllNodes.empty())
00940     DeallocateNode(AllNodes.begin());
00941 }
00942 
00943 BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL,
00944                                             SDVTList VTs, SDValue N1,
00945                                             SDValue N2, bool nuw, bool nsw,
00946                                             bool exact) {
00947   if (isBinOpWithFlags(Opcode)) {
00948     BinaryWithFlagsSDNode *FN = new (NodeAllocator) BinaryWithFlagsSDNode(
00949         Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
00950     FN->setHasNoUnsignedWrap(nuw);
00951     FN->setHasNoSignedWrap(nsw);
00952     FN->setIsExact(exact);
00953 
00954     return FN;
00955   }
00956 
00957   BinarySDNode *N = new (NodeAllocator)
00958       BinarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
00959   return N;
00960 }
00961 
00962 void SelectionDAG::clear() {
00963   allnodes_clear();
00964   OperandAllocator.Reset();
00965   CSEMap.clear();
00966 
00967   ExtendedValueTypeNodes.clear();
00968   ExternalSymbols.clear();
00969   TargetExternalSymbols.clear();
00970   std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
00971             static_cast<CondCodeSDNode*>(nullptr));
00972   std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
00973             static_cast<SDNode*>(nullptr));
00974 
00975   EntryNode.UseList = nullptr;
00976   AllNodes.push_back(&EntryNode);
00977   Root = getEntryNode();
00978   DbgInfo->clear();
00979 }
00980 
00981 SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00982   return VT.bitsGT(Op.getValueType()) ?
00983     getNode(ISD::ANY_EXTEND, DL, VT, Op) :
00984     getNode(ISD::TRUNCATE, DL, VT, Op);
00985 }
00986 
00987 SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00988   return VT.bitsGT(Op.getValueType()) ?
00989     getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
00990     getNode(ISD::TRUNCATE, DL, VT, Op);
00991 }
00992 
00993 SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00994   return VT.bitsGT(Op.getValueType()) ?
00995     getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
00996     getNode(ISD::TRUNCATE, DL, VT, Op);
00997 }
00998 
00999 SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, SDLoc SL, EVT VT,
01000                                         EVT OpVT) {
01001   if (VT.bitsLE(Op.getValueType()))
01002     return getNode(ISD::TRUNCATE, SL, VT, Op);
01003 
01004   TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
01005   return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
01006 }
01007 
01008 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, SDLoc DL, EVT VT) {
01009   assert(!VT.isVector() &&
01010          "getZeroExtendInReg should use the vector element type instead of "
01011          "the vector type!");
01012   if (Op.getValueType() == VT) return Op;
01013   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01014   APInt Imm = APInt::getLowBitsSet(BitWidth,
01015                                    VT.getSizeInBits());
01016   return getNode(ISD::AND, DL, Op.getValueType(), Op,
01017                  getConstant(Imm, Op.getValueType()));
01018 }
01019 
01020 SDValue SelectionDAG::getAnyExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01021   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01022   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01023          "The sizes of the input and result must match in order to perform the "
01024          "extend in-register.");
01025   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01026          "The destination vector type must have fewer lanes than the input.");
01027   return getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Op);
01028 }
01029 
01030 SDValue SelectionDAG::getSignExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01031   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01032   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01033          "The sizes of the input and result must match in order to perform the "
01034          "extend in-register.");
01035   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01036          "The destination vector type must have fewer lanes than the input.");
01037   return getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Op);
01038 }
01039 
01040 SDValue SelectionDAG::getZeroExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01041   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01042   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01043          "The sizes of the input and result must match in order to perform the "
01044          "extend in-register.");
01045   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01046          "The destination vector type must have fewer lanes than the input.");
01047   return getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Op);
01048 }
01049 
01050 /// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
01051 ///
01052 SDValue SelectionDAG::getNOT(SDLoc DL, SDValue Val, EVT VT) {
01053   EVT EltVT = VT.getScalarType();
01054   SDValue NegOne =
01055     getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT);
01056   return getNode(ISD::XOR, DL, VT, Val, NegOne);
01057 }
01058 
01059 SDValue SelectionDAG::getLogicalNOT(SDLoc DL, SDValue Val, EVT VT) {
01060   EVT EltVT = VT.getScalarType();
01061   SDValue TrueValue;
01062   switch (TLI->getBooleanContents(VT)) {
01063     case TargetLowering::ZeroOrOneBooleanContent:
01064     case TargetLowering::UndefinedBooleanContent:
01065       TrueValue = getConstant(1, VT);
01066       break;
01067     case TargetLowering::ZeroOrNegativeOneBooleanContent:
01068       TrueValue = getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()),
01069                               VT);
01070       break;
01071   }
01072   return getNode(ISD::XOR, DL, VT, Val, TrueValue);
01073 }
01074 
01075 SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT, bool isO) {
01076   EVT EltVT = VT.getScalarType();
01077   assert((EltVT.getSizeInBits() >= 64 ||
01078          (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
01079          "getConstant with a uint64_t value that doesn't fit in the type!");
01080   return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT, isO);
01081 }
01082 
01083 SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT, bool isO)
01084 {
01085   return getConstant(*ConstantInt::get(*Context, Val), VT, isT, isO);
01086 }
01087 
01088 SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT,
01089                                   bool isO) {
01090   assert(VT.isInteger() && "Cannot create FP integer constant!");
01091 
01092   EVT EltVT = VT.getScalarType();
01093   const ConstantInt *Elt = &Val;
01094 
01095   // In some cases the vector type is legal but the element type is illegal and
01096   // needs to be promoted, for example v8i8 on ARM.  In this case, promote the
01097   // inserted value (the type does not need to match the vector element type).
01098   // Any extra bits introduced will be truncated away.
01099   if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
01100       TargetLowering::TypePromoteInteger) {
01101    EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
01102    APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits());
01103    Elt = ConstantInt::get(*getContext(), NewVal);
01104   }
01105   // In other cases the element type is illegal and needs to be expanded, for
01106   // example v2i64 on MIPS32. In this case, find the nearest legal type, split
01107   // the value into n parts and use a vector type with n-times the elements.
01108   // Then bitcast to the type requested.
01109   // Legalizing constants too early makes the DAGCombiner's job harder so we
01110   // only legalize if the DAG tells us we must produce legal types.
01111   else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
01112            TLI->getTypeAction(*getContext(), EltVT) ==
01113            TargetLowering::TypeExpandInteger) {
01114     APInt NewVal = Elt->getValue();
01115     EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
01116     unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
01117     unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
01118     EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
01119 
01120     // Check the temporary vector is the correct size. If this fails then
01121     // getTypeToTransformTo() probably returned a type whose size (in bits)
01122     // isn't a power-of-2 factor of the requested type size.
01123     assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
01124 
01125     SmallVector<SDValue, 2> EltParts;
01126     for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
01127       EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
01128                                            .trunc(ViaEltSizeInBits),
01129                                      ViaEltVT, isT, isO));
01130     }
01131 
01132     // EltParts is currently in little endian order. If we actually want
01133     // big-endian order then reverse it now.
01134     if (TLI->isBigEndian())
01135       std::reverse(EltParts.begin(), EltParts.end());
01136 
01137     // The elements must be reversed when the element order is different
01138     // to the endianness of the elements (because the BITCAST is itself a
01139     // vector shuffle in this situation). However, we do not need any code to
01140     // perform this reversal because getConstant() is producing a vector
01141     // splat.
01142     // This situation occurs in MIPS MSA.
01143 
01144     SmallVector<SDValue, 8> Ops;
01145     for (unsigned i = 0; i < VT.getVectorNumElements(); ++i)
01146       Ops.insert(Ops.end(), EltParts.begin(), EltParts.end());
01147 
01148     SDValue Result = getNode(ISD::BITCAST, SDLoc(), VT,
01149                              getNode(ISD::BUILD_VECTOR, SDLoc(), ViaVecVT,
01150                                      Ops));
01151     return Result;
01152   }
01153 
01154   assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
01155          "APInt size does not match type size!");
01156   unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
01157   FoldingSetNodeID ID;
01158   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
01159   ID.AddPointer(Elt);
01160   ID.AddBoolean(isO);
01161   void *IP = nullptr;
01162   SDNode *N = nullptr;
01163   if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
01164     if (!VT.isVector())
01165       return SDValue(N, 0);
01166 
01167   if (!N) {
01168     N = new (NodeAllocator) ConstantSDNode(isT, isO, Elt, EltVT);
01169     CSEMap.InsertNode(N, IP);
01170     InsertNode(N);
01171   }
01172 
01173   SDValue Result(N, 0);
01174   if (VT.isVector()) {
01175     SmallVector<SDValue, 8> Ops;
01176     Ops.assign(VT.getVectorNumElements(), Result);
01177     Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
01178   }
01179   return Result;
01180 }
01181 
01182 SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) {
01183   return getConstant(Val, TLI->getPointerTy(), isTarget);
01184 }
01185 
01186 
01187 SDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) {
01188   return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget);
01189 }
01190 
01191 SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){
01192   assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
01193 
01194   EVT EltVT = VT.getScalarType();
01195 
01196   // Do the map lookup using the actual bit pattern for the floating point
01197   // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
01198   // we don't have issues with SNANs.
01199   unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
01200   FoldingSetNodeID ID;
01201   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
01202   ID.AddPointer(&V);
01203   void *IP = nullptr;
01204   SDNode *N = nullptr;
01205   if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
01206     if (!VT.isVector())
01207       return SDValue(N, 0);
01208 
01209   if (!N) {
01210     N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT);
01211     CSEMap.InsertNode(N, IP);
01212     InsertNode(N);
01213   }
01214 
01215   SDValue Result(N, 0);
01216   if (VT.isVector()) {
01217     SmallVector<SDValue, 8> Ops;
01218     Ops.assign(VT.getVectorNumElements(), Result);
01219     // FIXME SDLoc info might be appropriate here
01220     Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
01221   }
01222   return Result;
01223 }
01224 
01225 SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) {
01226   EVT EltVT = VT.getScalarType();
01227   if (EltVT==MVT::f32)
01228     return getConstantFP(APFloat((float)Val), VT, isTarget);
01229   else if (EltVT==MVT::f64)
01230     return getConstantFP(APFloat(Val), VT, isTarget);
01231   else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 ||
01232            EltVT==MVT::f16) {
01233     bool ignored;
01234     APFloat apf = APFloat(Val);
01235     apf.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
01236                 &ignored);
01237     return getConstantFP(apf, VT, isTarget);
01238   } else
01239     llvm_unreachable("Unsupported type in getConstantFP");
01240 }
01241 
01242 SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, SDLoc DL,
01243                                        EVT VT, int64_t Offset,
01244                                        bool isTargetGA,
01245                                        unsigned char TargetFlags) {
01246   assert((TargetFlags == 0 || isTargetGA) &&
01247          "Cannot set target flags on target-independent globals");
01248 
01249   // Truncate (with sign-extension) the offset value to the pointer size.
01250   unsigned BitWidth = TLI->getPointerTypeSizeInBits(GV->getType());
01251   if (BitWidth < 64)
01252     Offset = SignExtend64(Offset, BitWidth);
01253 
01254   unsigned Opc;
01255   if (GV->isThreadLocal())
01256     Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
01257   else
01258     Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
01259 
01260   FoldingSetNodeID ID;
01261   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01262   ID.AddPointer(GV);
01263   ID.AddInteger(Offset);
01264   ID.AddInteger(TargetFlags);
01265   ID.AddInteger(GV->getType()->getAddressSpace());
01266   void *IP = nullptr;
01267   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01268     return SDValue(E, 0);
01269 
01270   SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL.getIROrder(),
01271                                                       DL.getDebugLoc(), GV, VT,
01272                                                       Offset, TargetFlags);
01273   CSEMap.InsertNode(N, IP);
01274     InsertNode(N);
01275   return SDValue(N, 0);
01276 }
01277 
01278 SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
01279   unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
01280   FoldingSetNodeID ID;
01281   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01282   ID.AddInteger(FI);
01283   void *IP = nullptr;
01284   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01285     return SDValue(E, 0);
01286 
01287   SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget);
01288   CSEMap.InsertNode(N, IP);
01289   InsertNode(N);
01290   return SDValue(N, 0);
01291 }
01292 
01293 SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
01294                                    unsigned char TargetFlags) {
01295   assert((TargetFlags == 0 || isTarget) &&
01296          "Cannot set target flags on target-independent jump tables");
01297   unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
01298   FoldingSetNodeID ID;
01299   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01300   ID.AddInteger(JTI);
01301   ID.AddInteger(TargetFlags);
01302   void *IP = nullptr;
01303   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01304     return SDValue(E, 0);
01305 
01306   SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget,
01307                                                   TargetFlags);
01308   CSEMap.InsertNode(N, IP);
01309   InsertNode(N);
01310   return SDValue(N, 0);
01311 }
01312 
01313 SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
01314                                       unsigned Alignment, int Offset,
01315                                       bool isTarget,
01316                                       unsigned char TargetFlags) {
01317   assert((TargetFlags == 0 || isTarget) &&
01318          "Cannot set target flags on target-independent globals");
01319   if (Alignment == 0)
01320     Alignment = TLI->getDataLayout()->getPrefTypeAlignment(C->getType());
01321   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
01322   FoldingSetNodeID ID;
01323   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01324   ID.AddInteger(Alignment);
01325   ID.AddInteger(Offset);
01326   ID.AddPointer(C);
01327   ID.AddInteger(TargetFlags);
01328   void *IP = nullptr;
01329   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01330     return SDValue(E, 0);
01331 
01332   SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
01333                                                      Alignment, TargetFlags);
01334   CSEMap.InsertNode(N, IP);
01335   InsertNode(N);
01336   return SDValue(N, 0);
01337 }
01338 
01339 
01340 SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
01341                                       unsigned Alignment, int Offset,
01342                                       bool isTarget,
01343                                       unsigned char TargetFlags) {
01344   assert((TargetFlags == 0 || isTarget) &&
01345          "Cannot set target flags on target-independent globals");
01346   if (Alignment == 0)
01347     Alignment = TLI->getDataLayout()->getPrefTypeAlignment(C->getType());
01348   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
01349   FoldingSetNodeID ID;
01350   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01351   ID.AddInteger(Alignment);
01352   ID.AddInteger(Offset);
01353   C->addSelectionDAGCSEId(ID);
01354   ID.AddInteger(TargetFlags);
01355   void *IP = nullptr;
01356   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01357     return SDValue(E, 0);
01358 
01359   SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
01360                                                      Alignment, TargetFlags);
01361   CSEMap.InsertNode(N, IP);
01362   InsertNode(N);
01363   return SDValue(N, 0);
01364 }
01365 
01366 SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
01367                                      unsigned char TargetFlags) {
01368   FoldingSetNodeID ID;
01369   AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None);
01370   ID.AddInteger(Index);
01371   ID.AddInteger(Offset);
01372   ID.AddInteger(TargetFlags);
01373   void *IP = nullptr;
01374   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01375     return SDValue(E, 0);
01376 
01377   SDNode *N = new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset,
01378                                                     TargetFlags);
01379   CSEMap.InsertNode(N, IP);
01380   InsertNode(N);
01381   return SDValue(N, 0);
01382 }
01383 
01384 SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
01385   FoldingSetNodeID ID;
01386   AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
01387   ID.AddPointer(MBB);
01388   void *IP = nullptr;
01389   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01390     return SDValue(E, 0);
01391 
01392   SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB);
01393   CSEMap.InsertNode(N, IP);
01394   InsertNode(N);
01395   return SDValue(N, 0);
01396 }
01397 
01398 SDValue SelectionDAG::getValueType(EVT VT) {
01399   if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
01400       ValueTypeNodes.size())
01401     ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
01402 
01403   SDNode *&N = VT.isExtended() ?
01404     ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
01405 
01406   if (N) return SDValue(N, 0);
01407   N = new (NodeAllocator) VTSDNode(VT);
01408   InsertNode(N);
01409   return SDValue(N, 0);
01410 }
01411 
01412 SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
01413   SDNode *&N = ExternalSymbols[Sym];
01414   if (N) return SDValue(N, 0);
01415   N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT);
01416   InsertNode(N);
01417   return SDValue(N, 0);
01418 }
01419 
01420 SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
01421                                               unsigned char TargetFlags) {
01422   SDNode *&N =
01423     TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym,
01424                                                                TargetFlags)];
01425   if (N) return SDValue(N, 0);
01426   N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT);
01427   InsertNode(N);
01428   return SDValue(N, 0);
01429 }
01430 
01431 SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
01432   if ((unsigned)Cond >= CondCodeNodes.size())
01433     CondCodeNodes.resize(Cond+1);
01434 
01435   if (!CondCodeNodes[Cond]) {
01436     CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond);
01437     CondCodeNodes[Cond] = N;
01438     InsertNode(N);
01439   }
01440 
01441   return SDValue(CondCodeNodes[Cond], 0);
01442 }
01443 
01444 // commuteShuffle - swaps the values of N1 and N2, and swaps all indices in
01445 // the shuffle mask M that point at N1 to point at N2, and indices that point
01446 // N2 to point at N1.
01447 static void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) {
01448   std::swap(N1, N2);
01449   int NElts = M.size();
01450   for (int i = 0; i != NElts; ++i) {
01451     if (M[i] >= NElts)
01452       M[i] -= NElts;
01453     else if (M[i] >= 0)
01454       M[i] += NElts;
01455   }
01456 }
01457 
01458 SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1,
01459                                        SDValue N2, const int *Mask) {
01460   assert(VT == N1.getValueType() && VT == N2.getValueType() &&
01461          "Invalid VECTOR_SHUFFLE");
01462 
01463   // Canonicalize shuffle undef, undef -> undef
01464   if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF)
01465     return getUNDEF(VT);
01466 
01467   // Validate that all indices in Mask are within the range of the elements
01468   // input to the shuffle.
01469   unsigned NElts = VT.getVectorNumElements();
01470   SmallVector<int, 8> MaskVec;
01471   for (unsigned i = 0; i != NElts; ++i) {
01472     assert(Mask[i] < (int)(NElts * 2) && "Index out of range");
01473     MaskVec.push_back(Mask[i]);
01474   }
01475 
01476   // Canonicalize shuffle v, v -> v, undef
01477   if (N1 == N2) {
01478     N2 = getUNDEF(VT);
01479     for (unsigned i = 0; i != NElts; ++i)
01480       if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts;
01481   }
01482 
01483   // Canonicalize shuffle undef, v -> v, undef.  Commute the shuffle mask.
01484   if (N1.getOpcode() == ISD::UNDEF)
01485     commuteShuffle(N1, N2, MaskVec);
01486 
01487   // Canonicalize all index into lhs, -> shuffle lhs, undef
01488   // Canonicalize all index into rhs, -> shuffle rhs, undef
01489   bool AllLHS = true, AllRHS = true;
01490   bool N2Undef = N2.getOpcode() == ISD::UNDEF;
01491   for (unsigned i = 0; i != NElts; ++i) {
01492     if (MaskVec[i] >= (int)NElts) {
01493       if (N2Undef)
01494         MaskVec[i] = -1;
01495       else
01496         AllLHS = false;
01497     } else if (MaskVec[i] >= 0) {
01498       AllRHS = false;
01499     }
01500   }
01501   if (AllLHS && AllRHS)
01502     return getUNDEF(VT);
01503   if (AllLHS && !N2Undef)
01504     N2 = getUNDEF(VT);
01505   if (AllRHS) {
01506     N1 = getUNDEF(VT);
01507     commuteShuffle(N1, N2, MaskVec);
01508   }
01509   // Reset our undef status after accounting for the mask.
01510   N2Undef = N2.getOpcode() == ISD::UNDEF;
01511   // Re-check whether both sides ended up undef.
01512   if (N1.getOpcode() == ISD::UNDEF && N2Undef)
01513     return getUNDEF(VT);
01514 
01515   // If Identity shuffle return that node.
01516   bool Identity = true;
01517   for (unsigned i = 0; i != NElts; ++i) {
01518     if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false;
01519   }
01520   if (Identity && NElts)
01521     return N1;
01522 
01523   // Shuffling a constant splat doesn't change the result.
01524   if (N2Undef) {
01525     SDValue V = N1;
01526 
01527     // Look through any bitcasts. We check that these don't change the number
01528     // (and size) of elements and just changes their types.
01529     while (V.getOpcode() == ISD::BITCAST)
01530       V = V->getOperand(0);
01531 
01532     // A splat should always show up as a build vector node.
01533     if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
01534       BitVector UndefElements;
01535       SDValue Splat = BV->getSplatValue(&UndefElements);
01536       // If this is a splat of an undef, shuffling it is also undef.
01537       if (Splat && Splat.getOpcode() == ISD::UNDEF)
01538         return getUNDEF(VT);
01539 
01540       // We only have a splat which can skip shuffles if there is a splatted
01541       // value and no undef lanes rearranged by the shuffle.
01542       if (Splat && UndefElements.none()) {
01543         // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
01544         // number of elements match or the value splatted is a zero constant.
01545         if (V.getValueType().getVectorNumElements() ==
01546             VT.getVectorNumElements())
01547           return N1;
01548         if (auto *C = dyn_cast<ConstantSDNode>(Splat))
01549           if (C->isNullValue())
01550             return N1;
01551       }
01552     }
01553   }
01554 
01555   FoldingSetNodeID ID;
01556   SDValue Ops[2] = { N1, N2 };
01557   AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
01558   for (unsigned i = 0; i != NElts; ++i)
01559     ID.AddInteger(MaskVec[i]);
01560 
01561   void* IP = nullptr;
01562   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01563     return SDValue(E, 0);
01564 
01565   // Allocate the mask array for the node out of the BumpPtrAllocator, since
01566   // SDNode doesn't have access to it.  This memory will be "leaked" when
01567   // the node is deallocated, but recovered when the NodeAllocator is released.
01568   int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
01569   memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int));
01570 
01571   ShuffleVectorSDNode *N =
01572     new (NodeAllocator) ShuffleVectorSDNode(VT, dl.getIROrder(),
01573                                             dl.getDebugLoc(), N1, N2,
01574                                             MaskAlloc);
01575   CSEMap.InsertNode(N, IP);
01576   InsertNode(N);
01577   return SDValue(N, 0);
01578 }
01579 
01580 SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
01581   MVT VT = SV.getSimpleValueType(0);
01582   unsigned NumElems = VT.getVectorNumElements();
01583   SmallVector<int, 8> MaskVec;
01584 
01585   for (unsigned i = 0; i != NumElems; ++i) {
01586     int Idx = SV.getMaskElt(i);
01587     if (Idx >= 0) {
01588       if (Idx < (int)NumElems)
01589         Idx += NumElems;
01590       else
01591         Idx -= NumElems;
01592     }
01593     MaskVec.push_back(Idx);
01594   }
01595 
01596   SDValue Op0 = SV.getOperand(0);
01597   SDValue Op1 = SV.getOperand(1);
01598   return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, &MaskVec[0]);
01599 }
01600 
01601 SDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl,
01602                                        SDValue Val, SDValue DTy,
01603                                        SDValue STy, SDValue Rnd, SDValue Sat,
01604                                        ISD::CvtCode Code) {
01605   // If the src and dest types are the same and the conversion is between
01606   // integer types of the same sign or two floats, no conversion is necessary.
01607   if (DTy == STy &&
01608       (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF))
01609     return Val;
01610 
01611   FoldingSetNodeID ID;
01612   SDValue Ops[] = { Val, DTy, STy, Rnd, Sat };
01613   AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), Ops);
01614   void* IP = nullptr;
01615   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01616     return SDValue(E, 0);
01617 
01618   CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl.getIROrder(),
01619                                                            dl.getDebugLoc(),
01620                                                            Ops, Code);
01621   CSEMap.InsertNode(N, IP);
01622   InsertNode(N);
01623   return SDValue(N, 0);
01624 }
01625 
01626 SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
01627   FoldingSetNodeID ID;
01628   AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
01629   ID.AddInteger(RegNo);
01630   void *IP = nullptr;
01631   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01632     return SDValue(E, 0);
01633 
01634   SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT);
01635   CSEMap.InsertNode(N, IP);
01636   InsertNode(N);
01637   return SDValue(N, 0);
01638 }
01639 
01640 SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
01641   FoldingSetNodeID ID;
01642   AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
01643   ID.AddPointer(RegMask);
01644   void *IP = nullptr;
01645   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01646     return SDValue(E, 0);
01647 
01648   SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask);
01649   CSEMap.InsertNode(N, IP);
01650   InsertNode(N);
01651   return SDValue(N, 0);
01652 }
01653 
01654 SDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) {
01655   FoldingSetNodeID ID;
01656   SDValue Ops[] = { Root };
01657   AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), Ops);
01658   ID.AddPointer(Label);
01659   void *IP = nullptr;
01660   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01661     return SDValue(E, 0);
01662 
01663   SDNode *N = new (NodeAllocator) EHLabelSDNode(dl.getIROrder(),
01664                                                 dl.getDebugLoc(), Root, Label);
01665   CSEMap.InsertNode(N, IP);
01666   InsertNode(N);
01667   return SDValue(N, 0);
01668 }
01669 
01670 
01671 SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
01672                                       int64_t Offset,
01673                                       bool isTarget,
01674                                       unsigned char TargetFlags) {
01675   unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
01676 
01677   FoldingSetNodeID ID;
01678   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01679   ID.AddPointer(BA);
01680   ID.AddInteger(Offset);
01681   ID.AddInteger(TargetFlags);
01682   void *IP = nullptr;
01683   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01684     return SDValue(E, 0);
01685 
01686   SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset,
01687                                                      TargetFlags);
01688   CSEMap.InsertNode(N, IP);
01689   InsertNode(N);
01690   return SDValue(N, 0);
01691 }
01692 
01693 SDValue SelectionDAG::getSrcValue(const Value *V) {
01694   assert((!V || V->getType()->isPointerTy()) &&
01695          "SrcValue is not a pointer?");
01696 
01697   FoldingSetNodeID ID;
01698   AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None);
01699   ID.AddPointer(V);
01700 
01701   void *IP = nullptr;
01702   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01703     return SDValue(E, 0);
01704 
01705   SDNode *N = new (NodeAllocator) SrcValueSDNode(V);
01706   CSEMap.InsertNode(N, IP);
01707   InsertNode(N);
01708   return SDValue(N, 0);
01709 }
01710 
01711 /// getMDNode - Return an MDNodeSDNode which holds an MDNode.
01712 SDValue SelectionDAG::getMDNode(const MDNode *MD) {
01713   FoldingSetNodeID ID;
01714   AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None);
01715   ID.AddPointer(MD);
01716 
01717   void *IP = nullptr;
01718   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01719     return SDValue(E, 0);
01720 
01721   SDNode *N = new (NodeAllocator) MDNodeSDNode(MD);
01722   CSEMap.InsertNode(N, IP);
01723   InsertNode(N);
01724   return SDValue(N, 0);
01725 }
01726 
01727 /// getAddrSpaceCast - Return an AddrSpaceCastSDNode.
01728 SDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr,
01729                                        unsigned SrcAS, unsigned DestAS) {
01730   SDValue Ops[] = {Ptr};
01731   FoldingSetNodeID ID;
01732   AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
01733   ID.AddInteger(SrcAS);
01734   ID.AddInteger(DestAS);
01735 
01736   void *IP = nullptr;
01737   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01738     return SDValue(E, 0);
01739 
01740   SDNode *N = new (NodeAllocator) AddrSpaceCastSDNode(dl.getIROrder(),
01741                                                       dl.getDebugLoc(),
01742                                                       VT, Ptr, SrcAS, DestAS);
01743   CSEMap.InsertNode(N, IP);
01744   InsertNode(N);
01745   return SDValue(N, 0);
01746 }
01747 
01748 /// getShiftAmountOperand - Return the specified value casted to
01749 /// the target's desired shift amount type.
01750 SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
01751   EVT OpTy = Op.getValueType();
01752   EVT ShTy = TLI->getShiftAmountTy(LHSTy);
01753   if (OpTy == ShTy || OpTy.isVector()) return Op;
01754 
01755   ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ?  ISD::TRUNCATE : ISD::ZERO_EXTEND;
01756   return getNode(Opcode, SDLoc(Op), ShTy, Op);
01757 }
01758 
01759 /// CreateStackTemporary - Create a stack temporary, suitable for holding the
01760 /// specified value type.
01761 SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
01762   MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
01763   unsigned ByteSize = VT.getStoreSize();
01764   Type *Ty = VT.getTypeForEVT(*getContext());
01765   unsigned StackAlign =
01766   std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty), minAlign);
01767 
01768   int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false);
01769   return getFrameIndex(FrameIdx, TLI->getPointerTy());
01770 }
01771 
01772 /// CreateStackTemporary - Create a stack temporary suitable for holding
01773 /// either of the specified value types.
01774 SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
01775   unsigned Bytes = std::max(VT1.getStoreSizeInBits(),
01776                             VT2.getStoreSizeInBits())/8;
01777   Type *Ty1 = VT1.getTypeForEVT(*getContext());
01778   Type *Ty2 = VT2.getTypeForEVT(*getContext());
01779   const DataLayout *TD = TLI->getDataLayout();
01780   unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1),
01781                             TD->getPrefTypeAlignment(Ty2));
01782 
01783   MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
01784   int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false);
01785   return getFrameIndex(FrameIdx, TLI->getPointerTy());
01786 }
01787 
01788 SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1,
01789                                 SDValue N2, ISD::CondCode Cond, SDLoc dl) {
01790   // These setcc operations always fold.
01791   switch (Cond) {
01792   default: break;
01793   case ISD::SETFALSE:
01794   case ISD::SETFALSE2: return getConstant(0, VT);
01795   case ISD::SETTRUE:
01796   case ISD::SETTRUE2: {
01797     TargetLowering::BooleanContent Cnt =
01798         TLI->getBooleanContents(N1->getValueType(0));
01799     return getConstant(
01800         Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, VT);
01801   }
01802 
01803   case ISD::SETOEQ:
01804   case ISD::SETOGT:
01805   case ISD::SETOGE:
01806   case ISD::SETOLT:
01807   case ISD::SETOLE:
01808   case ISD::SETONE:
01809   case ISD::SETO:
01810   case ISD::SETUO:
01811   case ISD::SETUEQ:
01812   case ISD::SETUNE:
01813     assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!");
01814     break;
01815   }
01816 
01817   if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) {
01818     const APInt &C2 = N2C->getAPIntValue();
01819     if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
01820       const APInt &C1 = N1C->getAPIntValue();
01821 
01822       switch (Cond) {
01823       default: llvm_unreachable("Unknown integer setcc!");
01824       case ISD::SETEQ:  return getConstant(C1 == C2, VT);
01825       case ISD::SETNE:  return getConstant(C1 != C2, VT);
01826       case ISD::SETULT: return getConstant(C1.ult(C2), VT);
01827       case ISD::SETUGT: return getConstant(C1.ugt(C2), VT);
01828       case ISD::SETULE: return getConstant(C1.ule(C2), VT);
01829       case ISD::SETUGE: return getConstant(C1.uge(C2), VT);
01830       case ISD::SETLT:  return getConstant(C1.slt(C2), VT);
01831       case ISD::SETGT:  return getConstant(C1.sgt(C2), VT);
01832       case ISD::SETLE:  return getConstant(C1.sle(C2), VT);
01833       case ISD::SETGE:  return getConstant(C1.sge(C2), VT);
01834       }
01835     }
01836   }
01837   if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) {
01838     if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) {
01839       APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF());
01840       switch (Cond) {
01841       default: break;
01842       case ISD::SETEQ:  if (R==APFloat::cmpUnordered)
01843                           return getUNDEF(VT);
01844                         // fall through
01845       case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT);
01846       case ISD::SETNE:  if (R==APFloat::cmpUnordered)
01847                           return getUNDEF(VT);
01848                         // fall through
01849       case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan ||
01850                                            R==APFloat::cmpLessThan, VT);
01851       case ISD::SETLT:  if (R==APFloat::cmpUnordered)
01852                           return getUNDEF(VT);
01853                         // fall through
01854       case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT);
01855       case ISD::SETGT:  if (R==APFloat::cmpUnordered)
01856                           return getUNDEF(VT);
01857                         // fall through
01858       case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT);
01859       case ISD::SETLE:  if (R==APFloat::cmpUnordered)
01860                           return getUNDEF(VT);
01861                         // fall through
01862       case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan ||
01863                                            R==APFloat::cmpEqual, VT);
01864       case ISD::SETGE:  if (R==APFloat::cmpUnordered)
01865                           return getUNDEF(VT);
01866                         // fall through
01867       case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan ||
01868                                            R==APFloat::cmpEqual, VT);
01869       case ISD::SETO:   return getConstant(R!=APFloat::cmpUnordered, VT);
01870       case ISD::SETUO:  return getConstant(R==APFloat::cmpUnordered, VT);
01871       case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered ||
01872                                            R==APFloat::cmpEqual, VT);
01873       case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT);
01874       case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered ||
01875                                            R==APFloat::cmpLessThan, VT);
01876       case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan ||
01877                                            R==APFloat::cmpUnordered, VT);
01878       case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT);
01879       case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT);
01880       }
01881     } else {
01882       // Ensure that the constant occurs on the RHS.
01883       ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
01884       MVT CompVT = N1.getValueType().getSimpleVT();
01885       if (!TLI->isCondCodeLegal(SwappedCond, CompVT))
01886         return SDValue();
01887 
01888       return getSetCC(dl, VT, N2, N1, SwappedCond);
01889     }
01890   }
01891 
01892   // Could not fold it.
01893   return SDValue();
01894 }
01895 
01896 /// SignBitIsZero - Return true if the sign bit of Op is known to be zero.  We
01897 /// use this predicate to simplify operations downstream.
01898 bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
01899   // This predicate is not safe for vector operations.
01900   if (Op.getValueType().isVector())
01901     return false;
01902 
01903   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01904   return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth);
01905 }
01906 
01907 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero.  We use
01908 /// this predicate to simplify operations downstream.  Mask is known to be zero
01909 /// for bits that V cannot have.
01910 bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask,
01911                                      unsigned Depth) const {
01912   APInt KnownZero, KnownOne;
01913   computeKnownBits(Op, KnownZero, KnownOne, Depth);
01914   return (KnownZero & Mask) == Mask;
01915 }
01916 
01917 /// Determine which bits of Op are known to be either zero or one and return
01918 /// them in the KnownZero/KnownOne bitsets.
01919 void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
01920                                     APInt &KnownOne, unsigned Depth) const {
01921   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01922 
01923   KnownZero = KnownOne = APInt(BitWidth, 0);   // Don't know anything.
01924   if (Depth == 6)
01925     return;  // Limit search depth.
01926 
01927   APInt KnownZero2, KnownOne2;
01928 
01929   switch (Op.getOpcode()) {
01930   case ISD::Constant:
01931     // We know all of the bits for a constant!
01932     KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue();
01933     KnownZero = ~KnownOne;
01934     break;
01935   case ISD::AND:
01936     // If either the LHS or the RHS are Zero, the result is zero.
01937     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01938     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01939 
01940     // Output known-1 bits are only known if set in both the LHS & RHS.
01941     KnownOne &= KnownOne2;
01942     // Output known-0 are known to be clear if zero in either the LHS | RHS.
01943     KnownZero |= KnownZero2;
01944     break;
01945   case ISD::OR:
01946     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01947     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01948 
01949     // Output known-0 bits are only known if clear in both the LHS & RHS.
01950     KnownZero &= KnownZero2;
01951     // Output known-1 are known to be set if set in either the LHS | RHS.
01952     KnownOne |= KnownOne2;
01953     break;
01954   case ISD::XOR: {
01955     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01956     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01957 
01958     // Output known-0 bits are known if clear or set in both the LHS & RHS.
01959     APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2);
01960     // Output known-1 are known to be set if set in only one of the LHS, RHS.
01961     KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2);
01962     KnownZero = KnownZeroOut;
01963     break;
01964   }
01965   case ISD::MUL: {
01966     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01967     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01968 
01969     // If low bits are zero in either operand, output low known-0 bits.
01970     // Also compute a conserative estimate for high known-0 bits.
01971     // More trickiness is possible, but this is sufficient for the
01972     // interesting case of alignment computation.
01973     KnownOne.clearAllBits();
01974     unsigned TrailZ = KnownZero.countTrailingOnes() +
01975                       KnownZero2.countTrailingOnes();
01976     unsigned LeadZ =  std::max(KnownZero.countLeadingOnes() +
01977                                KnownZero2.countLeadingOnes(),
01978                                BitWidth) - BitWidth;
01979 
01980     TrailZ = std::min(TrailZ, BitWidth);
01981     LeadZ = std::min(LeadZ, BitWidth);
01982     KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) |
01983                 APInt::getHighBitsSet(BitWidth, LeadZ);
01984     break;
01985   }
01986   case ISD::UDIV: {
01987     // For the purposes of computing leading zeros we can conservatively
01988     // treat a udiv as a logical right shift by the power of 2 known to
01989     // be less than the denominator.
01990     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01991     unsigned LeadZ = KnownZero2.countLeadingOnes();
01992 
01993     KnownOne2.clearAllBits();
01994     KnownZero2.clearAllBits();
01995     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
01996     unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros();
01997     if (RHSUnknownLeadingOnes != BitWidth)
01998       LeadZ = std::min(BitWidth,
01999                        LeadZ + BitWidth - RHSUnknownLeadingOnes - 1);
02000 
02001     KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ);
02002     break;
02003   }
02004   case ISD::SELECT:
02005     computeKnownBits(Op.getOperand(2), KnownZero, KnownOne, Depth+1);
02006     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02007 
02008     // Only known if known in both the LHS and RHS.
02009     KnownOne &= KnownOne2;
02010     KnownZero &= KnownZero2;
02011     break;
02012   case ISD::SELECT_CC:
02013     computeKnownBits(Op.getOperand(3), KnownZero, KnownOne, Depth+1);
02014     computeKnownBits(Op.getOperand(2), KnownZero2, KnownOne2, Depth+1);
02015 
02016     // Only known if known in both the LHS and RHS.
02017     KnownOne &= KnownOne2;
02018     KnownZero &= KnownZero2;
02019     break;
02020   case ISD::SADDO:
02021   case ISD::UADDO:
02022   case ISD::SSUBO:
02023   case ISD::USUBO:
02024   case ISD::SMULO:
02025   case ISD::UMULO:
02026     if (Op.getResNo() != 1)
02027       break;
02028     // The boolean result conforms to getBooleanContents.
02029     // If we know the result of a setcc has the top bits zero, use this info.
02030     // We know that we have an integer-based boolean since these operations
02031     // are only available for integer.
02032     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
02033             TargetLowering::ZeroOrOneBooleanContent &&
02034         BitWidth > 1)
02035       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02036     break;
02037   case ISD::SETCC:
02038     // If we know the result of a setcc has the top bits zero, use this info.
02039     if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
02040             TargetLowering::ZeroOrOneBooleanContent &&
02041         BitWidth > 1)
02042       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02043     break;
02044   case ISD::SHL:
02045     // (shl X, C1) & C2 == 0   iff   (X & C2 >>u C1) == 0
02046     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02047       unsigned ShAmt = SA->getZExtValue();
02048 
02049       // If the shift count is an invalid immediate, don't do anything.
02050       if (ShAmt >= BitWidth)
02051         break;
02052 
02053       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02054       KnownZero <<= ShAmt;
02055       KnownOne  <<= ShAmt;
02056       // low bits known zero.
02057       KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt);
02058     }
02059     break;
02060   case ISD::SRL:
02061     // (ushr X, C1) & C2 == 0   iff  (-1 >> C1) & C2 == 0
02062     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02063       unsigned ShAmt = SA->getZExtValue();
02064 
02065       // If the shift count is an invalid immediate, don't do anything.
02066       if (ShAmt >= BitWidth)
02067         break;
02068 
02069       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02070       KnownZero = KnownZero.lshr(ShAmt);
02071       KnownOne  = KnownOne.lshr(ShAmt);
02072 
02073       APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
02074       KnownZero |= HighBits;  // High bits known zero.
02075     }
02076     break;
02077   case ISD::SRA:
02078     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02079       unsigned ShAmt = SA->getZExtValue();
02080 
02081       // If the shift count is an invalid immediate, don't do anything.
02082       if (ShAmt >= BitWidth)
02083         break;
02084 
02085       // If any of the demanded bits are produced by the sign extension, we also
02086       // demand the input sign bit.
02087       APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
02088 
02089       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02090       KnownZero = KnownZero.lshr(ShAmt);
02091       KnownOne  = KnownOne.lshr(ShAmt);
02092 
02093       // Handle the sign bits.
02094       APInt SignBit = APInt::getSignBit(BitWidth);
02095       SignBit = SignBit.lshr(ShAmt);  // Adjust to where it is now in the mask.
02096 
02097       if (KnownZero.intersects(SignBit)) {
02098         KnownZero |= HighBits;  // New bits are known zero.
02099       } else if (KnownOne.intersects(SignBit)) {
02100         KnownOne  |= HighBits;  // New bits are known one.
02101       }
02102     }
02103     break;
02104   case ISD::SIGN_EXTEND_INREG: {
02105     EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
02106     unsigned EBits = EVT.getScalarType().getSizeInBits();
02107 
02108     // Sign extension.  Compute the demanded bits in the result that are not
02109     // present in the input.
02110     APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits);
02111 
02112     APInt InSignBit = APInt::getSignBit(EBits);
02113     APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits);
02114 
02115     // If the sign extended bits are demanded, we know that the sign
02116     // bit is demanded.
02117     InSignBit = InSignBit.zext(BitWidth);
02118     if (NewBits.getBoolValue())
02119       InputDemandedBits |= InSignBit;
02120 
02121     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02122     KnownOne &= InputDemandedBits;
02123     KnownZero &= InputDemandedBits;
02124 
02125     // If the sign bit of the input is known set or clear, then we know the
02126     // top bits of the result.
02127     if (KnownZero.intersects(InSignBit)) {         // Input sign bit known clear
02128       KnownZero |= NewBits;
02129       KnownOne  &= ~NewBits;
02130     } else if (KnownOne.intersects(InSignBit)) {   // Input sign bit known set
02131       KnownOne  |= NewBits;
02132       KnownZero &= ~NewBits;
02133     } else {                              // Input sign bit unknown
02134       KnownZero &= ~NewBits;
02135       KnownOne  &= ~NewBits;
02136     }
02137     break;
02138   }
02139   case ISD::CTTZ:
02140   case ISD::CTTZ_ZERO_UNDEF:
02141   case ISD::CTLZ:
02142   case ISD::CTLZ_ZERO_UNDEF:
02143   case ISD::CTPOP: {
02144     unsigned LowBits = Log2_32(BitWidth)+1;
02145     KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits);
02146     KnownOne.clearAllBits();
02147     break;
02148   }
02149   case ISD::LOAD: {
02150     LoadSDNode *LD = cast<LoadSDNode>(Op);
02151     // If this is a ZEXTLoad and we are looking at the loaded value.
02152     if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
02153       EVT VT = LD->getMemoryVT();
02154       unsigned MemBits = VT.getScalarType().getSizeInBits();
02155       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
02156     } else if (const MDNode *Ranges = LD->getRanges()) {
02157       computeKnownBitsFromRangeMetadata(*Ranges, KnownZero);
02158     }
02159     break;
02160   }
02161   case ISD::ZERO_EXTEND: {
02162     EVT InVT = Op.getOperand(0).getValueType();
02163     unsigned InBits = InVT.getScalarType().getSizeInBits();
02164     APInt NewBits   = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
02165     KnownZero = KnownZero.trunc(InBits);
02166     KnownOne = KnownOne.trunc(InBits);
02167     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02168     KnownZero = KnownZero.zext(BitWidth);
02169     KnownOne = KnownOne.zext(BitWidth);
02170     KnownZero |= NewBits;
02171     break;
02172   }
02173   case ISD::SIGN_EXTEND: {
02174     EVT InVT = Op.getOperand(0).getValueType();
02175     unsigned InBits = InVT.getScalarType().getSizeInBits();
02176     APInt NewBits   = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
02177 
02178     KnownZero = KnownZero.trunc(InBits);
02179     KnownOne = KnownOne.trunc(InBits);
02180     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02181 
02182     // Note if the sign bit is known to be zero or one.
02183     bool SignBitKnownZero = KnownZero.isNegative();
02184     bool SignBitKnownOne  = KnownOne.isNegative();
02185 
02186     KnownZero = KnownZero.zext(BitWidth);
02187     KnownOne = KnownOne.zext(BitWidth);
02188 
02189     // If the sign bit is known zero or one, the top bits match.
02190     if (SignBitKnownZero)
02191       KnownZero |= NewBits;
02192     else if (SignBitKnownOne)
02193       KnownOne  |= NewBits;
02194     break;
02195   }
02196   case ISD::ANY_EXTEND: {
02197     EVT InVT = Op.getOperand(0).getValueType();
02198     unsigned InBits = InVT.getScalarType().getSizeInBits();
02199     KnownZero = KnownZero.trunc(InBits);
02200     KnownOne = KnownOne.trunc(InBits);
02201     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02202     KnownZero = KnownZero.zext(BitWidth);
02203     KnownOne = KnownOne.zext(BitWidth);
02204     break;
02205   }
02206   case ISD::TRUNCATE: {
02207     EVT InVT = Op.getOperand(0).getValueType();
02208     unsigned InBits = InVT.getScalarType().getSizeInBits();
02209     KnownZero = KnownZero.zext(InBits);
02210     KnownOne = KnownOne.zext(InBits);
02211     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02212     KnownZero = KnownZero.trunc(BitWidth);
02213     KnownOne = KnownOne.trunc(BitWidth);
02214     break;
02215   }
02216   case ISD::AssertZext: {
02217     EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
02218     APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
02219     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02220     KnownZero |= (~InMask);
02221     KnownOne  &= (~KnownZero);
02222     break;
02223   }
02224   case ISD::FGETSIGN:
02225     // All bits are zero except the low bit.
02226     KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02227     break;
02228 
02229   case ISD::SUB: {
02230     if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) {
02231       // We know that the top bits of C-X are clear if X contains less bits
02232       // than C (i.e. no wrap-around can happen).  For example, 20-X is
02233       // positive if we can prove that X is >= 0 and < 16.
02234       if (CLHS->getAPIntValue().isNonNegative()) {
02235         unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros();
02236         // NLZ can't be BitWidth with no sign bit
02237         APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1);
02238         computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02239 
02240         // If all of the MaskV bits are known to be zero, then we know the
02241         // output top bits are zero, because we now know that the output is
02242         // from [0-C].
02243         if ((KnownZero2 & MaskV) == MaskV) {
02244           unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros();
02245           // Top bits known zero.
02246           KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2);
02247         }
02248       }
02249     }
02250   }
02251   // fall through
02252   case ISD::ADD:
02253   case ISD::ADDE: {
02254     // Output known-0 bits are known if clear or set in both the low clear bits
02255     // common to both LHS & RHS.  For example, 8+(X<<3) is known to have the
02256     // low 3 bits clear.
02257     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
02258     unsigned KnownZeroOut = KnownZero2.countTrailingOnes();
02259 
02260     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02261     KnownZeroOut = std::min(KnownZeroOut,
02262                             KnownZero2.countTrailingOnes());
02263 
02264     if (Op.getOpcode() == ISD::ADD) {
02265       KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut);
02266       break;
02267     }
02268 
02269     // With ADDE, a carry bit may be added in, so we can only use this
02270     // information if we know (at least) that the low two bits are clear.  We
02271     // then return to the caller that the low bit is unknown but that other bits
02272     // are known zero.
02273     if (KnownZeroOut >= 2) // ADDE
02274       KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut);
02275     break;
02276   }
02277   case ISD::SREM:
02278     if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02279       const APInt &RA = Rem->getAPIntValue().abs();
02280       if (RA.isPowerOf2()) {
02281         APInt LowBits = RA - 1;
02282         computeKnownBits(Op.getOperand(0), KnownZero2,KnownOne2,Depth+1);
02283 
02284         // The low bits of the first operand are unchanged by the srem.
02285         KnownZero = KnownZero2 & LowBits;
02286         KnownOne = KnownOne2 & LowBits;
02287 
02288         // If the first operand is non-negative or has all low bits zero, then
02289         // the upper bits are all zero.
02290         if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits))
02291           KnownZero |= ~LowBits;
02292 
02293         // If the first operand is negative and not all low bits are zero, then
02294         // the upper bits are all one.
02295         if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0))
02296           KnownOne |= ~LowBits;
02297         assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?");
02298       }
02299     }
02300     break;
02301   case ISD::UREM: {
02302     if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02303       const APInt &RA = Rem->getAPIntValue();
02304       if (RA.isPowerOf2()) {
02305         APInt LowBits = (RA - 1);
02306         computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth + 1);
02307 
02308         // The upper bits are all zero, the lower ones are unchanged.
02309         KnownZero = KnownZero2 | ~LowBits;
02310         KnownOne = KnownOne2 & LowBits;
02311         break;
02312       }
02313     }
02314 
02315     // Since the result is less than or equal to either operand, any leading
02316     // zero bits in either operand must also exist in the result.
02317     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02318     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02319 
02320     uint32_t Leaders = std::max(KnownZero.countLeadingOnes(),
02321                                 KnownZero2.countLeadingOnes());
02322     KnownOne.clearAllBits();
02323     KnownZero = APInt::getHighBitsSet(BitWidth, Leaders);
02324     break;
02325   }
02326   case ISD::FrameIndex:
02327   case ISD::TargetFrameIndex:
02328     if (unsigned Align = InferPtrAlignment(Op)) {
02329       // The low bits are known zero if the pointer is aligned.
02330       KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align));
02331       break;
02332     }
02333     break;
02334 
02335   default:
02336     if (Op.getOpcode() < ISD::BUILTIN_OP_END)
02337       break;
02338     // Fallthrough
02339   case ISD::INTRINSIC_WO_CHAIN:
02340   case ISD::INTRINSIC_W_CHAIN:
02341   case ISD::INTRINSIC_VOID:
02342     // Allow the target to implement this method for its nodes.
02343     TLI->computeKnownBitsForTargetNode(Op, KnownZero, KnownOne, *this, Depth);
02344     break;
02345   }
02346 
02347   assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
02348 }
02349 
02350 /// ComputeNumSignBits - Return the number of times the sign bit of the
02351 /// register is replicated into the other bits.  We know that at least 1 bit
02352 /// is always equal to the sign bit (itself), but other cases can give us
02353 /// information.  For example, immediately after an "SRA X, 2", we know that
02354 /// the top 3 bits are all equal to each other, so we return 3.
02355 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{
02356   EVT VT = Op.getValueType();
02357   assert(VT.isInteger() && "Invalid VT!");
02358   unsigned VTBits = VT.getScalarType().getSizeInBits();
02359   unsigned Tmp, Tmp2;
02360   unsigned FirstAnswer = 1;
02361 
02362   if (Depth == 6)
02363     return 1;  // Limit search depth.
02364 
02365   switch (Op.getOpcode()) {
02366   default: break;
02367   case ISD::AssertSext:
02368     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
02369     return VTBits-Tmp+1;
02370   case ISD::AssertZext:
02371     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
02372     return VTBits-Tmp;
02373 
02374   case ISD::Constant: {
02375     const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue();
02376     return Val.getNumSignBits();
02377   }
02378 
02379   case ISD::SIGN_EXTEND:
02380     Tmp =
02381         VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits();
02382     return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp;
02383 
02384   case ISD::SIGN_EXTEND_INREG:
02385     // Max of the input and what this extends.
02386     Tmp =
02387       cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits();
02388     Tmp = VTBits-Tmp+1;
02389 
02390     Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02391     return std::max(Tmp, Tmp2);
02392 
02393   case ISD::SRA:
02394     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02395     // SRA X, C   -> adds C sign bits.
02396     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02397       Tmp += C->getZExtValue();
02398       if (Tmp > VTBits) Tmp = VTBits;
02399     }
02400     return Tmp;
02401   case ISD::SHL:
02402     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02403       // shl destroys sign bits.
02404       Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02405       if (C->getZExtValue() >= VTBits ||      // Bad shift.
02406           C->getZExtValue() >= Tmp) break;    // Shifted all sign bits out.
02407       return Tmp - C->getZExtValue();
02408     }
02409     break;
02410   case ISD::AND:
02411   case ISD::OR:
02412   case ISD::XOR:    // NOT is handled here.
02413     // Logical binary ops preserve the number of sign bits at the worst.
02414     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02415     if (Tmp != 1) {
02416       Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02417       FirstAnswer = std::min(Tmp, Tmp2);
02418       // We computed what we know about the sign bits as our first
02419       // answer. Now proceed to the generic code that uses
02420       // computeKnownBits, and pick whichever answer is better.
02421     }
02422     break;
02423 
02424   case ISD::SELECT:
02425     Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02426     if (Tmp == 1) return 1;  // Early out.
02427     Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1);
02428     return std::min(Tmp, Tmp2);
02429 
02430   case ISD::SADDO:
02431   case ISD::UADDO:
02432   case ISD::SSUBO:
02433   case ISD::USUBO:
02434   case ISD::SMULO:
02435   case ISD::UMULO:
02436     if (Op.getResNo() != 1)
02437       break;
02438     // The boolean result conforms to getBooleanContents.  Fall through.
02439     // If setcc returns 0/-1, all bits are sign bits.
02440     // We know that we have an integer-based boolean since these operations
02441     // are only available for integer.
02442     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
02443         TargetLowering::ZeroOrNegativeOneBooleanContent)
02444       return VTBits;
02445     break;
02446   case ISD::SETCC:
02447     // If setcc returns 0/-1, all bits are sign bits.
02448     if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
02449         TargetLowering::ZeroOrNegativeOneBooleanContent)
02450       return VTBits;
02451     break;
02452   case ISD::ROTL:
02453   case ISD::ROTR:
02454     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02455       unsigned RotAmt = C->getZExtValue() & (VTBits-1);
02456 
02457       // Handle rotate right by N like a rotate left by 32-N.
02458       if (Op.getOpcode() == ISD::ROTR)
02459         RotAmt = (VTBits-RotAmt) & (VTBits-1);
02460 
02461       // If we aren't rotating out all of the known-in sign bits, return the
02462       // number that are left.  This handles rotl(sext(x), 1) for example.
02463       Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02464       if (Tmp > RotAmt+1) return Tmp-RotAmt;
02465     }
02466     break;
02467   case ISD::ADD:
02468     // Add can have at most one carry bit.  Thus we know that the output
02469     // is, at worst, one more bit than the inputs.
02470     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02471     if (Tmp == 1) return 1;  // Early out.
02472 
02473     // Special case decrementing a value (ADD X, -1):
02474     if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
02475       if (CRHS->isAllOnesValue()) {
02476         APInt KnownZero, KnownOne;
02477         computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02478 
02479         // If the input is known to be 0 or 1, the output is 0/-1, which is all
02480         // sign bits set.
02481         if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
02482           return VTBits;
02483 
02484         // If we are subtracting one from a positive number, there is no carry
02485         // out of the result.
02486         if (KnownZero.isNegative())
02487           return Tmp;
02488       }
02489 
02490     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02491     if (Tmp2 == 1) return 1;
02492     return std::min(Tmp, Tmp2)-1;
02493 
02494   case ISD::SUB:
02495     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02496     if (Tmp2 == 1) return 1;
02497 
02498     // Handle NEG.
02499     if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0)))
02500       if (CLHS->isNullValue()) {
02501         APInt KnownZero, KnownOne;
02502         computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
02503         // If the input is known to be 0 or 1, the output is 0/-1, which is all
02504         // sign bits set.
02505         if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
02506           return VTBits;
02507 
02508         // If the input is known to be positive (the sign bit is known clear),
02509         // the output of the NEG has the same number of sign bits as the input.
02510         if (KnownZero.isNegative())
02511           return Tmp2;
02512 
02513         // Otherwise, we treat this like a SUB.
02514       }
02515 
02516     // Sub can have at most one carry bit.  Thus we know that the output
02517     // is, at worst, one more bit than the inputs.
02518     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02519     if (Tmp == 1) return 1;  // Early out.
02520     return std::min(Tmp, Tmp2)-1;
02521   case ISD::TRUNCATE:
02522     // FIXME: it's tricky to do anything useful for this, but it is an important
02523     // case for targets like X86.
02524     break;
02525   }
02526 
02527   // If we are looking at the loaded value of the SDNode.
02528   if (Op.getResNo() == 0) {
02529     // Handle LOADX separately here. EXTLOAD case will fallthrough.
02530     if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
02531       unsigned ExtType = LD->getExtensionType();
02532       switch (ExtType) {
02533         default: break;
02534         case ISD::SEXTLOAD:    // '17' bits known
02535           Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
02536           return VTBits-Tmp+1;
02537         case ISD::ZEXTLOAD:    // '16' bits known
02538           Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
02539           return VTBits-Tmp;
02540       }
02541     }
02542   }
02543 
02544   // Allow the target to implement this method for its nodes.
02545   if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
02546       Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
02547       Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
02548       Op.getOpcode() == ISD::INTRINSIC_VOID) {
02549     unsigned NumBits = TLI->ComputeNumSignBitsForTargetNode(Op, *this, Depth);
02550     if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits);
02551   }
02552 
02553   // Finally, if we can prove that the top bits of the result are 0's or 1's,
02554   // use this information.
02555   APInt KnownZero, KnownOne;
02556   computeKnownBits(Op, KnownZero, KnownOne, Depth);
02557 
02558   APInt Mask;
02559   if (KnownZero.isNegative()) {        // sign bit is 0
02560     Mask = KnownZero;
02561   } else if (KnownOne.isNegative()) {  // sign bit is 1;
02562     Mask = KnownOne;
02563   } else {
02564     // Nothing known.
02565     return FirstAnswer;
02566   }
02567 
02568   // Okay, we know that the sign bit in Mask is set.  Use CLZ to determine
02569   // the number of identical bits in the top of the input value.
02570   Mask = ~Mask;
02571   Mask <<= Mask.getBitWidth()-VTBits;
02572   // Return # leading zeros.  We use 'min' here in case Val was zero before
02573   // shifting.  We don't want to return '64' as for an i32 "0".
02574   return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros()));
02575 }
02576 
02577 /// isBaseWithConstantOffset - Return true if the specified operand is an
02578 /// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an
02579 /// ISD::OR with a ConstantSDNode that is guaranteed to have the same
02580 /// semantics as an ADD.  This handles the equivalence:
02581 ///     X|Cst == X+Cst iff X&Cst = 0.
02582 bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
02583   if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
02584       !isa<ConstantSDNode>(Op.getOperand(1)))
02585     return false;
02586 
02587   if (Op.getOpcode() == ISD::OR &&
02588       !MaskedValueIsZero(Op.getOperand(0),
02589                      cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue()))
02590     return false;
02591 
02592   return true;
02593 }
02594 
02595 
02596 bool SelectionDAG::isKnownNeverNaN(SDValue Op) const {
02597   // If we're told that NaNs won't happen, assume they won't.
02598   if (getTarget().Options.NoNaNsFPMath)
02599     return true;
02600 
02601   // If the value is a constant, we can obviously see if it is a NaN or not.
02602   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
02603     return !C->getValueAPF().isNaN();
02604 
02605   // TODO: Recognize more cases here.
02606 
02607   return false;
02608 }
02609 
02610 bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
02611   // If the value is a constant, we can obviously see if it is a zero or not.
02612   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
02613     return !C->isZero();
02614 
02615   // TODO: Recognize more cases here.
02616   switch (Op.getOpcode()) {
02617   default: break;
02618   case ISD::OR:
02619     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
02620       return !C->isNullValue();
02621     break;
02622   }
02623 
02624   return false;
02625 }
02626 
02627 bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
02628   // Check the obvious case.
02629   if (A == B) return true;
02630 
02631   // For for negative and positive zero.
02632   if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
02633     if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
02634       if (CA->isZero() && CB->isZero()) return true;
02635 
02636   // Otherwise they may not be equal.
02637   return false;
02638 }
02639 
02640 /// getNode - Gets or creates the specified node.
02641 ///
02642 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT) {
02643   FoldingSetNodeID ID;
02644   AddNodeIDNode(ID, Opcode, getVTList(VT), None);
02645   void *IP = nullptr;
02646   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
02647     return SDValue(E, 0);
02648 
02649   SDNode *N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(),
02650                                          DL.getDebugLoc(), getVTList(VT));
02651   CSEMap.InsertNode(N, IP);
02652 
02653   InsertNode(N);
02654   return SDValue(N, 0);
02655 }
02656 
02657 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
02658                               EVT VT, SDValue Operand) {
02659   // Constant fold unary operations with an integer constant operand. Even
02660   // opaque constant will be folded, because the folding of unary operations
02661   // doesn't create new constants with different values. Nevertheless, the
02662   // opaque flag is preserved during folding to prevent future folding with
02663   // other constants.
02664   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) {
02665     const APInt &Val = C->getAPIntValue();
02666     switch (Opcode) {
02667     default: break;
02668     case ISD::SIGN_EXTEND:
02669       return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT,
02670                          C->isTargetOpcode(), C->isOpaque());
02671     case ISD::ANY_EXTEND:
02672     case ISD::ZERO_EXTEND:
02673     case ISD::TRUNCATE:
02674       return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT,
02675                          C->isTargetOpcode(), C->isOpaque());
02676     case ISD::UINT_TO_FP:
02677     case ISD::SINT_TO_FP: {
02678       APFloat apf(EVTToAPFloatSemantics(VT),
02679                   APInt::getNullValue(VT.getSizeInBits()));
02680       (void)apf.convertFromAPInt(Val,
02681                                  Opcode==ISD::SINT_TO_FP,
02682                                  APFloat::rmNearestTiesToEven);
02683       return getConstantFP(apf, VT);
02684     }
02685     case ISD::BITCAST:
02686       if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
02687         return getConstantFP(APFloat(APFloat::IEEEhalf, Val), VT);
02688       if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
02689         return getConstantFP(APFloat(APFloat::IEEEsingle, Val), VT);
02690       else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
02691         return getConstantFP(APFloat(APFloat::IEEEdouble, Val), VT);
02692       break;
02693     case ISD::BSWAP:
02694       return getConstant(Val.byteSwap(), VT, C->isTargetOpcode(),
02695                          C->isOpaque());
02696     case ISD::CTPOP:
02697       return getConstant(Val.countPopulation(), VT, C->isTargetOpcode(),
02698                          C->isOpaque());
02699     case ISD::CTLZ:
02700     case ISD::CTLZ_ZERO_UNDEF:
02701       return getConstant(Val.countLeadingZeros(), VT, C->isTargetOpcode(),
02702                          C->isOpaque());
02703     case ISD::CTTZ:
02704     case ISD::CTTZ_ZERO_UNDEF:
02705       return getConstant(Val.countTrailingZeros(), VT, C->isTargetOpcode(),
02706                          C->isOpaque());
02707     }
02708   }
02709 
02710   // Constant fold unary operations with a floating point constant operand.
02711   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) {
02712     APFloat V = C->getValueAPF();    // make copy
02713     switch (Opcode) {
02714     case ISD::FNEG:
02715       V.changeSign();
02716       return getConstantFP(V, VT);
02717     case ISD::FABS:
02718       V.clearSign();
02719       return getConstantFP(V, VT);
02720     case ISD::FCEIL: {
02721       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
02722       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02723         return getConstantFP(V, VT);
02724       break;
02725     }
02726     case ISD::FTRUNC: {
02727       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
02728       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02729         return getConstantFP(V, VT);
02730       break;
02731     }
02732     case ISD::FFLOOR: {
02733       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
02734       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02735         return getConstantFP(V, VT);
02736       break;
02737     }
02738     case ISD::FP_EXTEND: {
02739       bool ignored;
02740       // This can return overflow, underflow, or inexact; we don't care.
02741       // FIXME need to be more flexible about rounding mode.
02742       (void)V.convert(EVTToAPFloatSemantics(VT),
02743                       APFloat::rmNearestTiesToEven, &ignored);
02744       return getConstantFP(V, VT);
02745     }
02746     case ISD::FP_TO_SINT:
02747     case ISD::FP_TO_UINT: {
02748       integerPart x[2];
02749       bool ignored;
02750       static_assert(integerPartWidth >= 64, "APFloat parts too small!");
02751       // FIXME need to be more flexible about rounding mode.
02752       APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(),
02753                             Opcode==ISD::FP_TO_SINT,
02754                             APFloat::rmTowardZero, &ignored);
02755       if (s==APFloat::opInvalidOp)     // inexact is OK, in fact usual
02756         break;
02757       APInt api(VT.getSizeInBits(), x);
02758       return getConstant(api, VT);
02759     }
02760     case ISD::BITCAST:
02761       if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
02762         return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), VT);
02763       else if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
02764         return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT);
02765       else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
02766         return getConstant(V.bitcastToAPInt().getZExtValue(), VT);
02767       break;
02768     }
02769   }
02770 
02771   // Constant fold unary operations with a vector integer operand.
02772   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand.getNode())) {
02773     if (BV->isConstant()) {
02774       switch (Opcode) {
02775       default:
02776         // FIXME: Entirely reasonable to perform folding of other unary
02777         // operations here as the need arises.
02778         break;
02779       case ISD::UINT_TO_FP:
02780       case ISD::SINT_TO_FP: {
02781         SmallVector<SDValue, 8> Ops;
02782         for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
02783           SDValue OpN = BV->getOperand(i);
02784           // Let the above scalar folding handle the conversion of each
02785           // element.
02786           OpN = getNode(ISD::SINT_TO_FP, DL, VT.getVectorElementType(),
02787                         OpN);
02788           Ops.push_back(OpN);
02789         }
02790         return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
02791       }
02792       }
02793     }
02794   }
02795 
02796   unsigned OpOpcode = Operand.getNode()->getOpcode();
02797   switch (Opcode) {
02798   case ISD::TokenFactor:
02799   case ISD::MERGE_VALUES:
02800   case ISD::CONCAT_VECTORS:
02801     return Operand;         // Factor, merge or concat of one node?  No need.
02802   case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
02803   case ISD::FP_EXTEND:
02804     assert(VT.isFloatingPoint() &&
02805            Operand.getValueType().isFloatingPoint() && "Invalid FP cast!");
02806     if (Operand.getValueType() == VT) return Operand;  // noop conversion.
02807     assert((!VT.isVector() ||
02808             VT.getVectorNumElements() ==
02809             Operand.getValueType().getVectorNumElements()) &&
02810            "Vector element count mismatch!");
02811     if (Operand.getOpcode() == ISD::UNDEF)
02812       return getUNDEF(VT);
02813     break;
02814   case ISD::SIGN_EXTEND:
02815     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02816            "Invalid SIGN_EXTEND!");
02817     if (Operand.getValueType() == VT) return Operand;   // noop extension
02818     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02819            "Invalid sext node, dst < src!");
02820     assert((!VT.isVector() ||
02821             VT.getVectorNumElements() ==
02822             Operand.getValueType().getVectorNumElements()) &&
02823            "Vector element count mismatch!");
02824     if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
02825       return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02826     else if (OpOpcode == ISD::UNDEF)
02827       // sext(undef) = 0, because the top bits will all be the same.
02828       return getConstant(0, VT);
02829     break;
02830   case ISD::ZERO_EXTEND:
02831     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02832            "Invalid ZERO_EXTEND!");
02833     if (Operand.getValueType() == VT) return Operand;   // noop extension
02834     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02835            "Invalid zext node, dst < src!");
02836     assert((!VT.isVector() ||
02837             VT.getVectorNumElements() ==
02838             Operand.getValueType().getVectorNumElements()) &&
02839            "Vector element count mismatch!");
02840     if (OpOpcode == ISD::ZERO_EXTEND)   // (zext (zext x)) -> (zext x)
02841       return getNode(ISD::ZERO_EXTEND, DL, VT,
02842                      Operand.getNode()->getOperand(0));
02843     else if (OpOpcode == ISD::UNDEF)
02844       // zext(undef) = 0, because the top bits will be zero.
02845       return getConstant(0, VT);
02846     break;
02847   case ISD::ANY_EXTEND:
02848     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02849            "Invalid ANY_EXTEND!");
02850     if (Operand.getValueType() == VT) return Operand;   // noop extension
02851     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02852            "Invalid anyext node, dst < src!");
02853     assert((!VT.isVector() ||
02854             VT.getVectorNumElements() ==
02855             Operand.getValueType().getVectorNumElements()) &&
02856            "Vector element count mismatch!");
02857 
02858     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
02859         OpOpcode == ISD::ANY_EXTEND)
02860       // (ext (zext x)) -> (zext x)  and  (ext (sext x)) -> (sext x)
02861       return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02862     else if (OpOpcode == ISD::UNDEF)
02863       return getUNDEF(VT);
02864 
02865     // (ext (trunx x)) -> x
02866     if (OpOpcode == ISD::TRUNCATE) {
02867       SDValue OpOp = Operand.getNode()->getOperand(0);
02868       if (OpOp.getValueType() == VT)
02869         return OpOp;
02870     }
02871     break;
02872   case ISD::TRUNCATE:
02873     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02874            "Invalid TRUNCATE!");
02875     if (Operand.getValueType() == VT) return Operand;   // noop truncate
02876     assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) &&
02877            "Invalid truncate node, src < dst!");
02878     assert((!VT.isVector() ||
02879             VT.getVectorNumElements() ==
02880             Operand.getValueType().getVectorNumElements()) &&
02881            "Vector element count mismatch!");
02882     if (OpOpcode == ISD::TRUNCATE)
02883       return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
02884     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
02885         OpOpcode == ISD::ANY_EXTEND) {
02886       // If the source is smaller than the dest, we still need an extend.
02887       if (Operand.getNode()->getOperand(0).getValueType().getScalarType()
02888             .bitsLT(VT.getScalarType()))
02889         return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02890       if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT))
02891         return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
02892       return Operand.getNode()->getOperand(0);
02893     }
02894     if (OpOpcode == ISD::UNDEF)
02895       return getUNDEF(VT);
02896     break;
02897   case ISD::BITCAST:
02898     // Basic sanity checking.
02899     assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits()
02900            && "Cannot BITCAST between types of different sizes!");
02901     if (VT == Operand.getValueType()) return Operand;  // noop conversion.
02902     if (OpOpcode == ISD::BITCAST)  // bitconv(bitconv(x)) -> bitconv(x)
02903       return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
02904     if (OpOpcode == ISD::UNDEF)
02905       return getUNDEF(VT);
02906     break;
02907   case ISD::SCALAR_TO_VECTOR:
02908     assert(VT.isVector() && !Operand.getValueType().isVector() &&
02909            (VT.getVectorElementType() == Operand.getValueType() ||
02910             (VT.getVectorElementType().isInteger() &&
02911              Operand.getValueType().isInteger() &&
02912              VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&
02913            "Illegal SCALAR_TO_VECTOR node!");
02914     if (OpOpcode == ISD::UNDEF)
02915       return getUNDEF(VT);
02916     // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
02917     if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
02918         isa<ConstantSDNode>(Operand.getOperand(1)) &&
02919         Operand.getConstantOperandVal(1) == 0 &&
02920         Operand.getOperand(0).getValueType() == VT)
02921       return Operand.getOperand(0);
02922     break;
02923   case ISD::FNEG:
02924     // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0
02925     if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB)
02926       return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1),
02927                      Operand.getNode()->getOperand(0));
02928     if (OpOpcode == ISD::FNEG)  // --X -> X
02929       return Operand.getNode()->getOperand(0);
02930     break;
02931   case ISD::FABS:
02932     if (OpOpcode == ISD::FNEG)  // abs(-X) -> abs(X)
02933       return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0));
02934     break;
02935   }
02936 
02937   SDNode *N;
02938   SDVTList VTs = getVTList(VT);
02939   if (VT != MVT::Glue) { // Don't CSE flag producing nodes
02940     FoldingSetNodeID ID;
02941     SDValue Ops[1] = { Operand };
02942     AddNodeIDNode(ID, Opcode, VTs, Ops);
02943     void *IP = nullptr;
02944     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
02945       return SDValue(E, 0);
02946 
02947     N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
02948                                         DL.getDebugLoc(), VTs, Operand);
02949     CSEMap.InsertNode(N, IP);
02950   } else {
02951     N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
02952                                         DL.getDebugLoc(), VTs, Operand);
02953   }
02954 
02955   InsertNode(N);
02956   return SDValue(N, 0);
02957 }
02958 
02959 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT,
02960                                              SDNode *Cst1, SDNode *Cst2) {
02961   // If the opcode is a target-specific ISD node, there's nothing we can
02962   // do here and the operand rules may not line up with the below, so
02963   // bail early.
02964   if (Opcode >= ISD::BUILTIN_OP_END)
02965     return SDValue();
02966 
02967   SmallVector<std::pair<ConstantSDNode *, ConstantSDNode *>, 4> Inputs;
02968   SmallVector<SDValue, 4> Outputs;
02969   EVT SVT = VT.getScalarType();
02970 
02971   ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1);
02972   ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2);
02973   if (Scalar1 && Scalar2 && (Scalar1->isOpaque() || Scalar2->isOpaque()))
02974     return SDValue();
02975 
02976   if (Scalar1 && Scalar2)
02977     // Scalar instruction.
02978     Inputs.push_back(std::make_pair(Scalar1, Scalar2));
02979   else {
02980     // For vectors extract each constant element into Inputs so we can constant
02981     // fold them individually.
02982     BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1);
02983     BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2);
02984     if (!BV1 || !BV2)
02985       return SDValue();
02986 
02987     assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!");
02988 
02989     for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) {
02990       ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I));
02991       ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I));
02992       if (!V1 || !V2) // Not a constant, bail.
02993         return SDValue();
02994 
02995       if (V1->isOpaque() || V2->isOpaque())
02996         return SDValue();
02997 
02998       // Avoid BUILD_VECTOR nodes that perform implicit truncation.
02999       // FIXME: This is valid and could be handled by truncating the APInts.
03000       if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
03001         return SDValue();
03002 
03003       Inputs.push_back(std::make_pair(V1, V2));
03004     }
03005   }
03006 
03007   // We have a number of constant values, constant fold them element by element.
03008   for (unsigned I = 0, E = Inputs.size(); I != E; ++I) {
03009     const APInt &C1 = Inputs[I].first->getAPIntValue();
03010     const APInt &C2 = Inputs[I].second->getAPIntValue();
03011 
03012     switch (Opcode) {
03013     case ISD::ADD:
03014       Outputs.push_back(getConstant(C1 + C2, SVT));
03015       break;
03016     case ISD::SUB:
03017       Outputs.push_back(getConstant(C1 - C2, SVT));
03018       break;
03019     case ISD::MUL:
03020       Outputs.push_back(getConstant(C1 * C2, SVT));
03021       break;
03022     case ISD::UDIV:
03023       if (!C2.getBoolValue())
03024         return SDValue();
03025       Outputs.push_back(getConstant(C1.udiv(C2), SVT));
03026       break;
03027     case ISD::UREM:
03028       if (!C2.getBoolValue())
03029         return SDValue();
03030       Outputs.push_back(getConstant(C1.urem(C2), SVT));
03031       break;
03032     case ISD::SDIV:
03033       if (!C2.getBoolValue())
03034         return SDValue();
03035       Outputs.push_back(getConstant(C1.sdiv(C2), SVT));
03036       break;
03037     case ISD::SREM:
03038       if (!C2.getBoolValue())
03039         return SDValue();
03040       Outputs.push_back(getConstant(C1.srem(C2), SVT));
03041       break;
03042     case ISD::AND:
03043       Outputs.push_back(getConstant(C1 & C2, SVT));
03044       break;
03045     case ISD::OR:
03046       Outputs.push_back(getConstant(C1 | C2, SVT));
03047       break;
03048     case ISD::XOR:
03049       Outputs.push_back(getConstant(C1 ^ C2, SVT));
03050       break;
03051     case ISD::SHL:
03052       Outputs.push_back(getConstant(C1 << C2, SVT));
03053       break;
03054     case ISD::SRL:
03055       Outputs.push_back(getConstant(C1.lshr(C2), SVT));
03056       break;
03057     case ISD::SRA:
03058       Outputs.push_back(getConstant(C1.ashr(C2), SVT));
03059       break;
03060     case ISD::ROTL:
03061       Outputs.push_back(getConstant(C1.rotl(C2), SVT));
03062       break;
03063     case ISD::ROTR:
03064       Outputs.push_back(getConstant(C1.rotr(C2), SVT));
03065       break;
03066     default:
03067       return SDValue();
03068     }
03069   }
03070 
03071   assert((Scalar1 && Scalar2) || (VT.getVectorNumElements() == Outputs.size() &&
03072                                   "Expected a scalar or vector!"));
03073 
03074   // Handle the scalar case first.
03075   if (!VT.isVector())
03076     return Outputs.back();
03077 
03078   // We may have a vector type but a scalar result. Create a splat.
03079   Outputs.resize(VT.getVectorNumElements(), Outputs.back());
03080 
03081   // Build a big vector out of the scalar elements we generated.
03082   return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs);
03083 }
03084 
03085 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
03086                               SDValue N2, bool nuw, bool nsw, bool exact) {
03087   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
03088   ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode());
03089   switch (Opcode) {
03090   default: break;
03091   case ISD::TokenFactor:
03092     assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
03093            N2.getValueType() == MVT::Other && "Invalid token factor!");
03094     // Fold trivial token factors.
03095     if (N1.getOpcode() == ISD::EntryToken) return N2;
03096     if (N2.getOpcode() == ISD::EntryToken) return N1;
03097     if (N1 == N2) return N1;
03098     break;
03099   case ISD::CONCAT_VECTORS:
03100     // Concat of UNDEFs is UNDEF.
03101     if (N1.getOpcode() == ISD::UNDEF &&
03102         N2.getOpcode() == ISD::UNDEF)
03103       return getUNDEF(VT);
03104 
03105     // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
03106     // one big BUILD_VECTOR.
03107     if (N1.getOpcode() == ISD::BUILD_VECTOR &&
03108         N2.getOpcode() == ISD::BUILD_VECTOR) {
03109       SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
03110                                     N1.getNode()->op_end());
03111       Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
03112       return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
03113     }
03114     break;
03115   case ISD::AND:
03116     assert(VT.isInteger() && "This operator does not apply to FP types!");
03117     assert(N1.getValueType() == N2.getValueType() &&
03118            N1.getValueType() == VT && "Binary operator types must match!");
03119     // (X & 0) -> 0.  This commonly occurs when legalizing i64 values, so it's
03120     // worth handling here.
03121     if (N2C && N2C->isNullValue())
03122       return N2;
03123     if (N2C && N2C->isAllOnesValue())  // X & -1 -> X
03124       return N1;
03125     break;
03126   case ISD::OR:
03127   case ISD::XOR:
03128   case ISD::ADD:
03129   case ISD::SUB:
03130     assert(VT.isInteger() && "This operator does not apply to FP types!");
03131     assert(N1.getValueType() == N2.getValueType() &&
03132            N1.getValueType() == VT && "Binary operator types must match!");
03133     // (X ^|+- 0) -> X.  This commonly occurs when legalizing i64 values, so
03134     // it's worth handling here.
03135     if (N2C && N2C->isNullValue())
03136       return N1;
03137     break;
03138   case ISD::UDIV:
03139   case ISD::UREM:
03140   case ISD::MULHU:
03141   case ISD::MULHS:
03142   case ISD::MUL:
03143   case ISD::SDIV:
03144   case ISD::SREM:
03145     assert(VT.isInteger() && "This operator does not apply to FP types!");
03146     assert(N1.getValueType() == N2.getValueType() &&
03147            N1.getValueType() == VT && "Binary operator types must match!");
03148     break;
03149   case ISD::FADD:
03150   case ISD::FSUB:
03151   case ISD::FMUL:
03152   case ISD::FDIV:
03153   case ISD::FREM:
03154     if (getTarget().Options.UnsafeFPMath) {
03155       if (Opcode == ISD::FADD) {
03156         // 0+x --> x
03157         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1))
03158           if (CFP->getValueAPF().isZero())
03159             return N2;
03160         // x+0 --> x
03161         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
03162           if (CFP->getValueAPF().isZero())
03163             return N1;
03164       } else if (Opcode == ISD::FSUB) {
03165         // x-0 --> x
03166         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
03167           if (CFP->getValueAPF().isZero())
03168             return N1;
03169       } else if (Opcode == ISD::FMUL) {
03170         ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1);
03171         SDValue V = N2;
03172 
03173         // If the first operand isn't the constant, try the second
03174         if (!CFP) {
03175           CFP = dyn_cast<ConstantFPSDNode>(N2);
03176           V = N1;
03177         }
03178 
03179         if (CFP) {
03180           // 0*x --> 0
03181           if (CFP->isZero())
03182             return SDValue(CFP,0);
03183           // 1*x --> x
03184           if (CFP->isExactlyValue(1.0))
03185             return V;
03186         }
03187       }
03188     }
03189     assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
03190     assert(N1.getValueType() == N2.getValueType() &&
03191            N1.getValueType() == VT && "Binary operator types must match!");
03192     break;
03193   case ISD::FCOPYSIGN:   // N1 and result must match.  N1/N2 need not match.
03194     assert(N1.getValueType() == VT &&
03195            N1.getValueType().isFloatingPoint() &&
03196            N2.getValueType().isFloatingPoint() &&
03197            "Invalid FCOPYSIGN!");
03198     break;
03199   case ISD::SHL:
03200   case ISD::SRA:
03201   case ISD::SRL:
03202   case ISD::ROTL:
03203   case ISD::ROTR:
03204     assert(VT == N1.getValueType() &&
03205            "Shift operators return type must be the same as their first arg");
03206     assert(VT.isInteger() && N2.getValueType().isInteger() &&
03207            "Shifts only work on integers");
03208     assert((!VT.isVector() || VT == N2.getValueType()) &&
03209            "Vector shift amounts must be in the same as their first arg");
03210     // Verify that the shift amount VT is bit enough to hold valid shift
03211     // amounts.  This catches things like trying to shift an i1024 value by an
03212     // i8, which is easy to fall into in generic code that uses
03213     // TLI.getShiftAmount().
03214     assert(N2.getValueType().getSizeInBits() >=
03215                    Log2_32_Ceil(N1.getValueType().getSizeInBits()) &&
03216            "Invalid use of small shift amount with oversized value!");
03217 
03218     // Always fold shifts of i1 values so the code generator doesn't need to
03219     // handle them.  Since we know the size of the shift has to be less than the
03220     // size of the value, the shift/rotate count is guaranteed to be zero.
03221     if (VT == MVT::i1)
03222       return N1;
03223     if (N2C && N2C->isNullValue())
03224       return N1;
03225     break;
03226   case ISD::FP_ROUND_INREG: {
03227     EVT EVT = cast<VTSDNode>(N2)->getVT();
03228     assert(VT == N1.getValueType() && "Not an inreg round!");
03229     assert(VT.isFloatingPoint() && EVT.isFloatingPoint() &&
03230            "Cannot FP_ROUND_INREG integer types");
03231     assert(EVT.isVector() == VT.isVector() &&
03232            "FP_ROUND_INREG type should be vector iff the operand "
03233            "type is vector!");
03234     assert((!EVT.isVector() ||
03235             EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
03236            "Vector element counts must match in FP_ROUND_INREG");
03237     assert(EVT.bitsLE(VT) && "Not rounding down!");
03238     (void)EVT;
03239     if (cast<VTSDNode>(N2)->getVT() == VT) return N1;  // Not actually rounding.
03240     break;
03241   }
03242   case ISD::FP_ROUND:
03243     assert(VT.isFloatingPoint() &&
03244            N1.getValueType().isFloatingPoint() &&
03245            VT.bitsLE(N1.getValueType()) &&
03246            isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!");
03247     if (N1.getValueType() == VT) return N1;  // noop conversion.
03248     break;
03249   case ISD::AssertSext:
03250   case ISD::AssertZext: {
03251     EVT EVT = cast<VTSDNode>(N2)->getVT();
03252     assert(VT == N1.getValueType() && "Not an inreg extend!");
03253     assert(VT.isInteger() && EVT.isInteger() &&
03254            "Cannot *_EXTEND_INREG FP types");
03255     assert(!EVT.isVector() &&
03256            "AssertSExt/AssertZExt type should be the vector element type "
03257            "rather than the vector type!");
03258     assert(EVT.bitsLE(VT) && "Not extending!");
03259     if (VT == EVT) return N1; // noop assertion.
03260     break;
03261   }
03262   case ISD::SIGN_EXTEND_INREG: {
03263     EVT EVT = cast<VTSDNode>(N2)->getVT();
03264     assert(VT == N1.getValueType() && "Not an inreg extend!");
03265     assert(VT.isInteger() && EVT.isInteger() &&
03266            "Cannot *_EXTEND_INREG FP types");
03267     assert(EVT.isVector() == VT.isVector() &&
03268            "SIGN_EXTEND_INREG type should be vector iff the operand "
03269            "type is vector!");
03270     assert((!EVT.isVector() ||
03271             EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
03272            "Vector element counts must match in SIGN_EXTEND_INREG");
03273     assert(EVT.bitsLE(VT) && "Not extending!");
03274     if (EVT == VT) return N1;  // Not actually extending
03275 
03276     if (N1C) {
03277       APInt Val = N1C->getAPIntValue();
03278       unsigned FromBits = EVT.getScalarType().getSizeInBits();
03279       Val <<= Val.getBitWidth()-FromBits;
03280       Val = Val.ashr(Val.getBitWidth()-FromBits);
03281       return getConstant(Val, VT);
03282     }
03283     break;
03284   }
03285   case ISD::EXTRACT_VECTOR_ELT:
03286     // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF.
03287     if (N1.getOpcode() == ISD::UNDEF)
03288       return getUNDEF(VT);
03289 
03290     // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
03291     // expanding copies of large vectors from registers.
03292     if (N2C &&
03293         N1.getOpcode() == ISD::CONCAT_VECTORS &&
03294         N1.getNumOperands() > 0) {
03295       unsigned Factor =
03296         N1.getOperand(0).getValueType().getVectorNumElements();
03297       return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
03298                      N1.getOperand(N2C->getZExtValue() / Factor),
03299                      getConstant(N2C->getZExtValue() % Factor,
03300                                  N2.getValueType()));
03301     }
03302 
03303     // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
03304     // expanding large vector constants.
03305     if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) {
03306       SDValue Elt = N1.getOperand(N2C->getZExtValue());
03307 
03308       if (VT != Elt.getValueType())
03309         // If the vector element type is not legal, the BUILD_VECTOR operands
03310         // are promoted and implicitly truncated, and the result implicitly
03311         // extended. Make that explicit here.
03312         Elt = getAnyExtOrTrunc(Elt, DL, VT);
03313 
03314       return Elt;
03315     }
03316 
03317     // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
03318     // operations are lowered to scalars.
03319     if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
03320       // If the indices are the same, return the inserted element else
03321       // if the indices are known different, extract the element from
03322       // the original vector.
03323       SDValue N1Op2 = N1.getOperand(2);
03324       ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode());
03325 
03326       if (N1Op2C && N2C) {
03327         if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
03328           if (VT == N1.getOperand(1).getValueType())
03329             return N1.getOperand(1);
03330           else
03331             return getSExtOrTrunc(N1.getOperand(1), DL, VT);
03332         }
03333 
03334         return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
03335       }
03336     }
03337     break;
03338   case ISD::EXTRACT_ELEMENT:
03339     assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
03340     assert(!N1.getValueType().isVector() && !VT.isVector() &&
03341            (N1.getValueType().isInteger() == VT.isInteger()) &&
03342            N1.getValueType() != VT &&
03343            "Wrong types for EXTRACT_ELEMENT!");
03344 
03345     // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
03346     // 64-bit integers into 32-bit parts.  Instead of building the extract of
03347     // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
03348     if (N1.getOpcode() == ISD::BUILD_PAIR)
03349       return N1.getOperand(N2C->getZExtValue());
03350 
03351     // EXTRACT_ELEMENT of a constant int is also very common.
03352     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
03353       unsigned ElementSize = VT.getSizeInBits();
03354       unsigned Shift = ElementSize * N2C->getZExtValue();
03355       APInt ShiftedVal = C->getAPIntValue().lshr(Shift);
03356       return getConstant(ShiftedVal.trunc(ElementSize), VT);
03357     }
03358     break;
03359   case ISD::EXTRACT_SUBVECTOR: {
03360     SDValue Index = N2;
03361     if (VT.isSimple() && N1.getValueType().isSimple()) {
03362       assert(VT.isVector() && N1.getValueType().isVector() &&
03363              "Extract subvector VTs must be a vectors!");
03364       assert(VT.getVectorElementType() ==
03365              N1.getValueType().getVectorElementType() &&
03366              "Extract subvector VTs must have the same element type!");
03367       assert(VT.getSimpleVT() <= N1.getSimpleValueType() &&
03368              "Extract subvector must be from larger vector to smaller vector!");
03369 
03370       if (isa<ConstantSDNode>(Index.getNode())) {
03371         assert((VT.getVectorNumElements() +
03372                 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
03373                 <= N1.getValueType().getVectorNumElements())
03374                && "Extract subvector overflow!");
03375       }
03376 
03377       // Trivial extraction.
03378       if (VT.getSimpleVT() == N1.getSimpleValueType())
03379         return N1;
03380     }
03381     break;
03382   }
03383   }
03384 
03385   // Perform trivial constant folding.
03386   SDValue SV = FoldConstantArithmetic(Opcode, VT, N1.getNode(), N2.getNode());
03387   if (SV.getNode()) return SV;
03388 
03389   // Canonicalize constant to RHS if commutative.
03390   if (N1C && !N2C && isCommutativeBinOp(Opcode)) {
03391     std::swap(N1C, N2C);
03392     std::swap(N1, N2);
03393   }
03394 
03395   // Constant fold FP operations.
03396   bool HasFPExceptions = TLI->hasFloatingPointExceptions();
03397   ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode());
03398   ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode());
03399   if (N1CFP) {
03400     if (!N2CFP && isCommutativeBinOp(Opcode)) {
03401       // Canonicalize constant to RHS if commutative.
03402       std::swap(N1CFP, N2CFP);
03403       std::swap(N1, N2);
03404     } else if (N2CFP) {
03405       APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF();
03406       APFloat::opStatus s;
03407       switch (Opcode) {
03408       case ISD::FADD:
03409         s = V1.add(V2, APFloat::rmNearestTiesToEven);
03410         if (!HasFPExceptions || s != APFloat::opInvalidOp)
03411           return getConstantFP(V1, VT);
03412         break;
03413       case ISD::FSUB:
03414         s = V1.subtract(V2, APFloat::rmNearestTiesToEven);
03415         if (!HasFPExceptions || s!=APFloat::opInvalidOp)
03416           return getConstantFP(V1, VT);
03417         break;
03418       case ISD::FMUL:
03419         s = V1.multiply(V2, APFloat::rmNearestTiesToEven);
03420         if (!HasFPExceptions || s!=APFloat::opInvalidOp)
03421           return getConstantFP(V1, VT);
03422         break;
03423       case ISD::FDIV:
03424         s = V1.divide(V2, APFloat::rmNearestTiesToEven);
03425         if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
03426                                  s!=APFloat::opDivByZero)) {
03427           return getConstantFP(V1, VT);
03428         }
03429         break;
03430       case ISD::FREM :
03431         s = V1.mod(V2, APFloat::rmNearestTiesToEven);
03432         if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
03433                                  s!=APFloat::opDivByZero)) {
03434           return getConstantFP(V1, VT);
03435         }
03436         break;
03437       case ISD::FCOPYSIGN:
03438         V1.copySign(V2);
03439         return getConstantFP(V1, VT);
03440       default: break;
03441       }
03442     }
03443 
03444     if (Opcode == ISD::FP_ROUND) {
03445       APFloat V = N1CFP->getValueAPF();    // make copy
03446       bool ignored;
03447       // This can return overflow, underflow, or inexact; we don't care.
03448       // FIXME need to be more flexible about rounding mode.
03449       (void)V.convert(EVTToAPFloatSemantics(VT),
03450                       APFloat::rmNearestTiesToEven, &ignored);
03451       return getConstantFP(V, VT);
03452     }
03453   }
03454 
03455   // Canonicalize an UNDEF to the RHS, even over a constant.
03456   if (N1.getOpcode() == ISD::UNDEF) {
03457     if (isCommutativeBinOp(Opcode)) {
03458       std::swap(N1, N2);
03459     } else {
03460       switch (Opcode) {
03461       case ISD::FP_ROUND_INREG:
03462       case ISD::SIGN_EXTEND_INREG:
03463       case ISD::SUB:
03464       case ISD::FSUB:
03465       case ISD::FDIV:
03466       case ISD::FREM:
03467       case ISD::SRA:
03468         return N1;     // fold op(undef, arg2) -> undef
03469       case ISD::UDIV:
03470       case ISD::SDIV:
03471       case ISD::UREM:
03472       case ISD::SREM:
03473       case ISD::SRL:
03474       case ISD::SHL:
03475         if (!VT.isVector())
03476           return getConstant(0, VT);    // fold op(undef, arg2) -> 0
03477         // For vectors, we can't easily build an all zero vector, just return
03478         // the LHS.
03479         return N2;
03480       }
03481     }
03482   }
03483 
03484   // Fold a bunch of operators when the RHS is undef.
03485   if (N2.getOpcode() == ISD::UNDEF) {
03486     switch (Opcode) {
03487     case ISD::XOR:
03488       if (N1.getOpcode() == ISD::UNDEF)
03489         // Handle undef ^ undef -> 0 special case. This is a common
03490         // idiom (misuse).
03491         return getConstant(0, VT);
03492       // fallthrough
03493     case ISD::ADD:
03494     case ISD::ADDC:
03495     case ISD::ADDE:
03496     case ISD::SUB:
03497     case ISD::UDIV:
03498     case ISD::SDIV:
03499     case ISD::UREM:
03500     case ISD::SREM:
03501       return N2;       // fold op(arg1, undef) -> undef
03502     case ISD::FADD:
03503     case ISD::FSUB:
03504     case ISD::FMUL:
03505     case ISD::FDIV:
03506     case ISD::FREM:
03507       if (getTarget().Options.UnsafeFPMath)
03508         return N2;
03509       break;
03510     case ISD::MUL:
03511     case ISD::AND:
03512     case ISD::SRL:
03513     case ISD::SHL:
03514       if (!VT.isVector())
03515         return getConstant(0, VT);  // fold op(arg1, undef) -> 0
03516       // For vectors, we can't easily build an all zero vector, just return
03517       // the LHS.
03518       return N1;
03519     case ISD::OR:
03520       if (!VT.isVector())
03521         return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT);
03522       // For vectors, we can't easily build an all one vector, just return
03523       // the LHS.
03524       return N1;
03525     case ISD::SRA:
03526       return N1;
03527     }
03528   }
03529 
03530   // Memoize this node if possible.
03531   BinarySDNode *N;
03532   SDVTList VTs = getVTList(VT);
03533   const bool BinOpHasFlags = isBinOpWithFlags(Opcode);
03534   if (VT != MVT::Glue) {
03535     SDValue Ops[] = {N1, N2};
03536     FoldingSetNodeID ID;
03537     AddNodeIDNode(ID, Opcode, VTs, Ops);
03538     if (BinOpHasFlags)
03539       AddBinaryNodeIDCustom(ID, Opcode, nuw, nsw, exact);
03540     void *IP = nullptr;
03541     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
03542       return SDValue(E, 0);
03543 
03544     N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
03545 
03546     CSEMap.InsertNode(N, IP);
03547   } else {
03548 
03549     N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
03550   }
03551 
03552   InsertNode(N);
03553   return SDValue(N, 0);
03554 }
03555 
03556 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03557                               SDValue N1, SDValue N2, SDValue N3) {
03558   // Perform various simplifications.
03559   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
03560   switch (Opcode) {
03561   case ISD::FMA: {
03562     ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
03563     ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
03564     ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
03565     if (N1CFP && N2CFP && N3CFP) {
03566       APFloat  V1 = N1CFP->getValueAPF();
03567       const APFloat &V2 = N2CFP->getValueAPF();
03568       const APFloat &V3 = N3CFP->getValueAPF();
03569       APFloat::opStatus s =
03570         V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
03571       if (s != APFloat::opInvalidOp)
03572         return getConstantFP(V1, VT);
03573     }
03574     break;
03575   }
03576   case ISD::CONCAT_VECTORS:
03577     // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
03578     // one big BUILD_VECTOR.
03579     if (N1.getOpcode() == ISD::BUILD_VECTOR &&
03580         N2.getOpcode() == ISD::BUILD_VECTOR &&
03581         N3.getOpcode() == ISD::BUILD_VECTOR) {
03582       SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
03583                                     N1.getNode()->op_end());
03584       Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
03585       Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end());
03586       return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
03587     }
03588     break;
03589   case ISD::SETCC: {
03590     // Use FoldSetCC to simplify SETCC's.
03591     SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL);
03592     if (Simp.getNode()) return Simp;
03593     break;
03594   }
03595   case ISD::SELECT:
03596     if (N1C) {
03597      if (N1C->getZExtValue())
03598        return N2;             // select true, X, Y -> X
03599      return N3;             // select false, X, Y -> Y
03600     }
03601 
03602     if (N2 == N3) return N2;   // select C, X, X -> X
03603     break;
03604   case ISD::VECTOR_SHUFFLE:
03605     llvm_unreachable("should use getVectorShuffle constructor!");
03606   case ISD::INSERT_SUBVECTOR: {
03607     SDValue Index = N3;
03608     if (VT.isSimple() && N1.getValueType().isSimple()
03609         && N2.getValueType().isSimple()) {
03610       assert(VT.isVector() && N1.getValueType().isVector() &&
03611              N2.getValueType().isVector() &&
03612              "Insert subvector VTs must be a vectors");
03613       assert(VT == N1.getValueType() &&
03614              "Dest and insert subvector source types must match!");
03615       assert(N2.getSimpleValueType() <= N1.getSimpleValueType() &&
03616              "Insert subvector must be from smaller vector to larger vector!");
03617       if (isa<ConstantSDNode>(Index.getNode())) {
03618         assert((N2.getValueType().getVectorNumElements() +
03619                 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
03620                 <= VT.getVectorNumElements())
03621                && "Insert subvector overflow!");
03622       }
03623 
03624       // Trivial insertion.
03625       if (VT.getSimpleVT() == N2.getSimpleValueType())
03626         return N2;
03627     }
03628     break;
03629   }
03630   case ISD::BITCAST:
03631     // Fold bit_convert nodes from a type to themselves.
03632     if (N1.getValueType() == VT)
03633       return N1;
03634     break;
03635   }
03636 
03637   // Memoize node if it doesn't produce a flag.
03638   SDNode *N;
03639   SDVTList VTs = getVTList(VT);
03640   if (VT != MVT::Glue) {
03641     SDValue Ops[] = { N1, N2, N3 };
03642     FoldingSetNodeID ID;
03643     AddNodeIDNode(ID, Opcode, VTs, Ops);
03644     void *IP = nullptr;
03645     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
03646       return SDValue(E, 0);
03647 
03648     N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
03649                                           DL.getDebugLoc(), VTs, N1, N2, N3);
03650     CSEMap.InsertNode(N, IP);
03651   } else {
03652     N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
03653                                           DL.getDebugLoc(), VTs, N1, N2, N3);
03654   }
03655 
03656   InsertNode(N);
03657   return SDValue(N, 0);
03658 }
03659 
03660 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03661                               SDValue N1, SDValue N2, SDValue N3,
03662                               SDValue N4) {
03663   SDValue Ops[] = { N1, N2, N3, N4 };
03664   return getNode(Opcode, DL, VT, Ops);
03665 }
03666 
03667 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03668                               SDValue N1, SDValue N2, SDValue N3,
03669                               SDValue N4, SDValue N5) {
03670   SDValue Ops[] = { N1, N2, N3, N4, N5 };
03671   return getNode(Opcode, DL, VT, Ops);
03672 }
03673 
03674 /// getStackArgumentTokenFactor - Compute a TokenFactor to force all
03675 /// the incoming stack arguments to be loaded from the stack.
03676 SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
03677   SmallVector<SDValue, 8> ArgChains;
03678 
03679   // Include the original chain at the beginning of the list. When this is
03680   // used by target LowerCall hooks, this helps legalize find the
03681   // CALLSEQ_BEGIN node.
03682   ArgChains.push_back(Chain);
03683 
03684   // Add a chain value for each stack argument.
03685   for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(),
03686        UE = getEntryNode().getNode()->use_end(); U != UE; ++U)
03687     if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
03688       if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
03689         if (FI->getIndex() < 0)
03690           ArgChains.push_back(SDValue(L, 1));
03691 
03692   // Build a tokenfactor for all the chains.
03693   return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
03694 }
03695 
03696 /// getMemsetValue - Vectorized representation of the memset value
03697 /// operand.
03698 static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
03699                               SDLoc dl) {
03700   assert(Value.getOpcode() != ISD::UNDEF);
03701 
03702   unsigned NumBits = VT.getScalarType().getSizeInBits();
03703   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
03704     assert(C->getAPIntValue().getBitWidth() == 8);
03705     APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
03706     if (VT.isInteger())
03707       return DAG.getConstant(Val, VT);
03708     return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), VT);
03709   }
03710 
03711   Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value);
03712   if (NumBits > 8) {
03713     // Use a multiplication with 0x010101... to extend the input to the
03714     // required length.
03715     APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
03716     Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT));
03717   }
03718 
03719   return Value;
03720 }
03721 
03722 /// getMemsetStringVal - Similar to getMemsetValue. Except this is only
03723 /// used when a memcpy is turned into a memset when the source is a constant
03724 /// string ptr.
03725 static SDValue getMemsetStringVal(EVT VT, SDLoc dl, SelectionDAG &DAG,
03726                                   const TargetLowering &TLI, StringRef Str) {
03727   // Handle vector with all elements zero.
03728   if (Str.empty()) {
03729     if (VT.isInteger())
03730       return DAG.getConstant(0, VT);
03731     else if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
03732       return DAG.getConstantFP(0.0, VT);
03733     else if (VT.isVector()) {
03734       unsigned NumElts = VT.getVectorNumElements();
03735       MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
03736       return DAG.getNode(ISD::BITCAST, dl, VT,
03737                          DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(),
03738                                                              EltVT, NumElts)));
03739     } else
03740       llvm_unreachable("Expected type!");
03741   }
03742 
03743   assert(!VT.isVector() && "Can't handle vector type here!");
03744   unsigned NumVTBits = VT.getSizeInBits();
03745   unsigned NumVTBytes = NumVTBits / 8;
03746   unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size()));
03747 
03748   APInt Val(NumVTBits, 0);
03749   if (TLI.isLittleEndian()) {
03750     for (unsigned i = 0; i != NumBytes; ++i)
03751       Val |= (uint64_t)(unsigned char)Str[i] << i*8;
03752   } else {
03753     for (unsigned i = 0; i != NumBytes; ++i)
03754       Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8;
03755   }
03756 
03757   // If the "cost" of materializing the integer immediate is less than the cost
03758   // of a load, then it is cost effective to turn the load into the immediate.
03759   Type *Ty = VT.getTypeForEVT(*DAG.getContext());
03760   if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
03761     return DAG.getConstant(Val, VT);
03762   return SDValue(nullptr, 0);
03763 }
03764 
03765 /// getMemBasePlusOffset - Returns base and offset node for the
03766 ///
03767 static SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, SDLoc dl,
03768                                       SelectionDAG &DAG) {
03769   EVT VT = Base.getValueType();
03770   return DAG.getNode(ISD::ADD, dl,
03771                      VT, Base, DAG.getConstant(Offset, VT));
03772 }
03773 
03774 /// isMemSrcFromString - Returns true if memcpy source is a string constant.
03775 ///
03776 static bool isMemSrcFromString(SDValue Src, StringRef &Str) {
03777   unsigned SrcDelta = 0;
03778   GlobalAddressSDNode *G = nullptr;
03779   if (Src.getOpcode() == ISD::GlobalAddress)
03780     G = cast<GlobalAddressSDNode>(Src);
03781   else if (Src.getOpcode() == ISD::ADD &&
03782            Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
03783            Src.getOperand(1).getOpcode() == ISD::Constant) {
03784     G = cast<GlobalAddressSDNode>(Src.getOperand(0));
03785     SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue();
03786   }
03787   if (!G)
03788     return false;
03789 
03790   return getConstantStringInfo(G->getGlobal(), Str, SrcDelta, false);
03791 }
03792 
03793 /// FindOptimalMemOpLowering - Determines the optimial series memory ops
03794 /// to replace the memset / memcpy. Return true if the number of memory ops
03795 /// is below the threshold. It returns the types of the sequence of
03796 /// memory ops to perform memset / memcpy by reference.
03797 static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
03798                                      unsigned Limit, uint64_t Size,
03799                                      unsigned DstAlign, unsigned SrcAlign,
03800                                      bool IsMemset,
03801                                      bool ZeroMemset,
03802                                      bool MemcpyStrSrc,
03803                                      bool AllowOverlap,
03804                                      SelectionDAG &DAG,
03805                                      const TargetLowering &TLI) {
03806   assert((SrcAlign == 0 || SrcAlign >= DstAlign) &&
03807          "Expecting memcpy / memset source to meet alignment requirement!");
03808   // If 'SrcAlign' is zero, that means the memory operation does not need to
03809   // load the value, i.e. memset or memcpy from constant string. Otherwise,
03810   // it's the inferred alignment of the source. 'DstAlign', on the other hand,
03811   // is the specified alignment of the memory operation. If it is zero, that
03812   // means it's possible to change the alignment of the destination.
03813   // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does
03814   // not need to be loaded.
03815   EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign,
03816                                    IsMemset, ZeroMemset, MemcpyStrSrc,
03817                                    DAG.getMachineFunction());
03818 
03819   if (VT == MVT::Other) {
03820     unsigned AS = 0;
03821     if (DstAlign >= TLI.getDataLayout()->getPointerPrefAlignment(AS) ||
03822         TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign)) {
03823       VT = TLI.getPointerTy();
03824     } else {
03825       switch (DstAlign & 7) {
03826       case 0:  VT = MVT::i64; break;
03827       case 4:  VT = MVT::i32; break;
03828       case 2:  VT = MVT::i16; break;
03829       default: VT = MVT::i8;  break;
03830       }
03831     }
03832 
03833     MVT LVT = MVT::i64;
03834     while (!TLI.isTypeLegal(LVT))
03835       LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
03836     assert(LVT.isInteger());
03837 
03838     if (VT.bitsGT(LVT))
03839       VT = LVT;
03840   }
03841 
03842   unsigned NumMemOps = 0;
03843   while (Size != 0) {
03844     unsigned VTSize = VT.getSizeInBits() / 8;
03845     while (VTSize > Size) {
03846       // For now, only use non-vector load / store's for the left-over pieces.
03847       EVT NewVT = VT;
03848       unsigned NewVTSize;
03849 
03850       bool Found = false;
03851       if (VT.isVector() || VT.isFloatingPoint()) {
03852         NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
03853         if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) &&
03854             TLI.isSafeMemOpType(NewVT.getSimpleVT()))
03855           Found = true;
03856         else if (NewVT == MVT::i64 &&
03857                  TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
03858                  TLI.isSafeMemOpType(MVT::f64)) {
03859           // i64 is usually not legal on 32-bit targets, but f64 may be.
03860           NewVT = MVT::f64;
03861           Found = true;
03862         }
03863       }
03864 
03865       if (!Found) {
03866         do {
03867           NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
03868           if (NewVT == MVT::i8)
03869             break;
03870         } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT()));
03871       }
03872       NewVTSize = NewVT.getSizeInBits() / 8;
03873 
03874       // If the new VT cannot cover all of the remaining bits, then consider
03875       // issuing a (or a pair of) unaligned and overlapping load / store.
03876       // FIXME: Only does this for 64-bit or more since we don't have proper
03877       // cost model for unaligned load / store.
03878       bool Fast;
03879       unsigned AS = 0;
03880       if (NumMemOps && AllowOverlap &&
03881           VTSize >= 8 && NewVTSize < Size &&
03882           TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign, &Fast) && Fast)
03883         VTSize = Size;
03884       else {
03885         VT = NewVT;
03886         VTSize = NewVTSize;
03887       }
03888     }
03889 
03890     if (++NumMemOps > Limit)
03891       return false;
03892 
03893     MemOps.push_back(VT);
03894     Size -= VTSize;
03895   }
03896 
03897   return true;
03898 }
03899 
03900 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
03901                                        SDValue Chain, SDValue Dst,
03902                                        SDValue Src, uint64_t Size,
03903                                        unsigned Align, bool isVol,
03904                                        bool AlwaysInline,
03905                                        MachinePointerInfo DstPtrInfo,
03906                                        MachinePointerInfo SrcPtrInfo) {
03907   // Turn a memcpy of undef to nop.
03908   if (Src.getOpcode() == ISD::UNDEF)
03909     return Chain;
03910 
03911   // Expand memcpy to a series of load and store ops if the size operand falls
03912   // below a certain threshold.
03913   // TODO: In the AlwaysInline case, if the size is big then generate a loop
03914   // rather than maybe a humongous number of loads and stores.
03915   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03916   std::vector<EVT> MemOps;
03917   bool DstAlignCanChange = false;
03918   MachineFunction &MF = DAG.getMachineFunction();
03919   MachineFrameInfo *MFI = MF.getFrameInfo();
03920   bool OptSize =
03921     MF.getFunction()->getAttributes().
03922       hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
03923   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
03924   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
03925     DstAlignCanChange = true;
03926   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
03927   if (Align > SrcAlign)
03928     SrcAlign = Align;
03929   StringRef Str;
03930   bool CopyFromStr = isMemSrcFromString(Src, Str);
03931   bool isZeroStr = CopyFromStr && Str.empty();
03932   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
03933 
03934   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
03935                                 (DstAlignCanChange ? 0 : Align),
03936                                 (isZeroStr ? 0 : SrcAlign),
03937                                 false, false, CopyFromStr, true, DAG, TLI))
03938     return SDValue();
03939 
03940   if (DstAlignCanChange) {
03941     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
03942     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
03943 
03944     // Don't promote to an alignment that would require dynamic stack
03945     // realignment.
03946     const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
03947     if (!TRI->needsStackRealignment(MF))
03948        while (NewAlign > Align &&
03949              TLI.getDataLayout()->exceedsNaturalStackAlignment(NewAlign))
03950           NewAlign /= 2;
03951 
03952     if (NewAlign > Align) {
03953       // Give the stack frame object a larger alignment if needed.
03954       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
03955         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
03956       Align = NewAlign;
03957     }
03958   }
03959 
03960   SmallVector<SDValue, 8> OutChains;
03961   unsigned NumMemOps = MemOps.size();
03962   uint64_t SrcOff = 0, DstOff = 0;
03963   for (unsigned i = 0; i != NumMemOps; ++i) {
03964     EVT VT = MemOps[i];
03965     unsigned VTSize = VT.getSizeInBits() / 8;
03966     SDValue Value, Store;
03967 
03968     if (VTSize > Size) {
03969       // Issuing an unaligned load / store pair  that overlaps with the previous
03970       // pair. Adjust the offset accordingly.
03971       assert(i == NumMemOps-1 && i != 0);
03972       SrcOff -= VTSize - Size;
03973       DstOff -= VTSize - Size;
03974     }
03975 
03976     if (CopyFromStr &&
03977         (isZeroStr || (VT.isInteger() && !VT.isVector()))) {
03978       // It's unlikely a store of a vector immediate can be done in a single
03979       // instruction. It would require a load from a constantpool first.
03980       // We only handle zero vectors here.
03981       // FIXME: Handle other cases where store of vector immediate is done in
03982       // a single instruction.
03983       Value = getMemsetStringVal(VT, dl, DAG, TLI, Str.substr(SrcOff));
03984       if (Value.getNode())
03985         Store = DAG.getStore(Chain, dl, Value,
03986                              getMemBasePlusOffset(Dst, DstOff, dl, DAG),
03987                              DstPtrInfo.getWithOffset(DstOff), isVol,
03988                              false, Align);
03989     }
03990 
03991     if (!Store.getNode()) {
03992       // The type might not be legal for the target.  This should only happen
03993       // if the type is smaller than a legal type, as on PPC, so the right
03994       // thing to do is generate a LoadExt/StoreTrunc pair.  These simplify
03995       // to Load/Store if NVT==VT.
03996       // FIXME does the case above also need this?
03997       EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
03998       assert(NVT.bitsGE(VT));
03999       Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain,
04000                              getMemBasePlusOffset(Src, SrcOff, dl, DAG),
04001                              SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false,
04002                              false, MinAlign(SrcAlign, SrcOff));
04003       Store = DAG.getTruncStore(Chain, dl, Value,
04004                                 getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04005                                 DstPtrInfo.getWithOffset(DstOff), VT, isVol,
04006                                 false, Align);
04007     }
04008     OutChains.push_back(Store);
04009     SrcOff += VTSize;
04010     DstOff += VTSize;
04011     Size -= VTSize;
04012   }
04013 
04014   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04015 }
04016 
04017 static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
04018                                         SDValue Chain, SDValue Dst,
04019                                         SDValue Src, uint64_t Size,
04020                                         unsigned Align,  bool isVol,
04021                                         bool AlwaysInline,
04022                                         MachinePointerInfo DstPtrInfo,
04023                                         MachinePointerInfo SrcPtrInfo) {
04024   // Turn a memmove of undef to nop.
04025   if (Src.getOpcode() == ISD::UNDEF)
04026     return Chain;
04027 
04028   // Expand memmove to a series of load and store ops if the size operand falls
04029   // below a certain threshold.
04030   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04031   std::vector<EVT> MemOps;
04032   bool DstAlignCanChange = false;
04033   MachineFunction &MF = DAG.getMachineFunction();
04034   MachineFrameInfo *MFI = MF.getFrameInfo();
04035   bool OptSize = MF.getFunction()->getAttributes().
04036     hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
04037   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
04038   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
04039     DstAlignCanChange = true;
04040   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
04041   if (Align > SrcAlign)
04042     SrcAlign = Align;
04043   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
04044 
04045   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
04046                                 (DstAlignCanChange ? 0 : Align), SrcAlign,
04047                                 false, false, false, false, DAG, TLI))
04048     return SDValue();
04049 
04050   if (DstAlignCanChange) {
04051     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
04052     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
04053     if (NewAlign > Align) {
04054       // Give the stack frame object a larger alignment if needed.
04055       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
04056         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
04057       Align = NewAlign;
04058     }
04059   }
04060 
04061   uint64_t SrcOff = 0, DstOff = 0;
04062   SmallVector<SDValue, 8> LoadValues;
04063   SmallVector<SDValue, 8> LoadChains;
04064   SmallVector<SDValue, 8> OutChains;
04065   unsigned NumMemOps = MemOps.size();
04066   for (unsigned i = 0; i < NumMemOps; i++) {
04067     EVT VT = MemOps[i];
04068     unsigned VTSize = VT.getSizeInBits() / 8;
04069     SDValue Value;
04070 
04071     Value = DAG.getLoad(VT, dl, Chain,
04072                         getMemBasePlusOffset(Src, SrcOff, dl, DAG),
04073                         SrcPtrInfo.getWithOffset(SrcOff), isVol,
04074                         false, false, SrcAlign);
04075     LoadValues.push_back(Value);
04076     LoadChains.push_back(Value.getValue(1));
04077     SrcOff += VTSize;
04078   }
04079   Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
04080   OutChains.clear();
04081   for (unsigned i = 0; i < NumMemOps; i++) {
04082     EVT VT = MemOps[i];
04083     unsigned VTSize = VT.getSizeInBits() / 8;
04084     SDValue Store;
04085 
04086     Store = DAG.getStore(Chain, dl, LoadValues[i],
04087                          getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04088                          DstPtrInfo.getWithOffset(DstOff), isVol, false, Align);
04089     OutChains.push_back(Store);
04090     DstOff += VTSize;
04091   }
04092 
04093   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04094 }
04095 
04096 /// \brief Lower the call to 'memset' intrinsic function into a series of store
04097 /// operations.
04098 ///
04099 /// \param DAG Selection DAG where lowered code is placed.
04100 /// \param dl Link to corresponding IR location.
04101 /// \param Chain Control flow dependency.
04102 /// \param Dst Pointer to destination memory location.
04103 /// \param Src Value of byte to write into the memory.
04104 /// \param Size Number of bytes to write.
04105 /// \param Align Alignment of the destination in bytes.
04106 /// \param isVol True if destination is volatile.
04107 /// \param DstPtrInfo IR information on the memory pointer.
04108 /// \returns New head in the control flow, if lowering was successful, empty
04109 /// SDValue otherwise.
04110 ///
04111 /// The function tries to replace 'llvm.memset' intrinsic with several store
04112 /// operations and value calculation code. This is usually profitable for small
04113 /// memory size.
04114 static SDValue getMemsetStores(SelectionDAG &DAG, SDLoc dl,
04115                                SDValue Chain, SDValue Dst,
04116                                SDValue Src, uint64_t Size,
04117                                unsigned Align, bool isVol,
04118                                MachinePointerInfo DstPtrInfo) {
04119   // Turn a memset of undef to nop.
04120   if (Src.getOpcode() == ISD::UNDEF)
04121     return Chain;
04122 
04123   // Expand memset to a series of load/store ops if the size operand
04124   // falls below a certain threshold.
04125   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04126   std::vector<EVT> MemOps;
04127   bool DstAlignCanChange = false;
04128   MachineFunction &MF = DAG.getMachineFunction();
04129   MachineFrameInfo *MFI = MF.getFrameInfo();
04130   bool OptSize = MF.getFunction()->getAttributes().
04131     hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
04132   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
04133   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
04134     DstAlignCanChange = true;
04135   bool IsZeroVal =
04136     isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
04137   if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize),
04138                                 Size, (DstAlignCanChange ? 0 : Align), 0,
04139                                 true, IsZeroVal, false, true, DAG, TLI))
04140     return SDValue();
04141 
04142   if (DstAlignCanChange) {
04143     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
04144     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
04145     if (NewAlign > Align) {
04146       // Give the stack frame object a larger alignment if needed.
04147       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
04148         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
04149       Align = NewAlign;
04150     }
04151   }
04152 
04153   SmallVector<SDValue, 8> OutChains;
04154   uint64_t DstOff = 0;
04155   unsigned NumMemOps = MemOps.size();
04156 
04157   // Find the largest store and generate the bit pattern for it.
04158   EVT LargestVT = MemOps[0];
04159   for (unsigned i = 1; i < NumMemOps; i++)
04160     if (MemOps[i].bitsGT(LargestVT))
04161       LargestVT = MemOps[i];
04162   SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
04163 
04164   for (unsigned i = 0; i < NumMemOps; i++) {
04165     EVT VT = MemOps[i];
04166     unsigned VTSize = VT.getSizeInBits() / 8;
04167     if (VTSize > Size) {
04168       // Issuing an unaligned load / store pair  that overlaps with the previous
04169       // pair. Adjust the offset accordingly.
04170       assert(i == NumMemOps-1 && i != 0);
04171       DstOff -= VTSize - Size;
04172     }
04173 
04174     // If this store is smaller than the largest store see whether we can get
04175     // the smaller value for free with a truncate.
04176     SDValue Value = MemSetValue;
04177     if (VT.bitsLT(LargestVT)) {
04178       if (!LargestVT.isVector() && !VT.isVector() &&
04179           TLI.isTruncateFree(LargestVT, VT))
04180         Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
04181       else
04182         Value = getMemsetValue(Src, VT, DAG, dl);
04183     }
04184     assert(Value.getValueType() == VT && "Value with wrong type.");
04185     SDValue Store = DAG.getStore(Chain, dl, Value,
04186                                  getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04187                                  DstPtrInfo.getWithOffset(DstOff),
04188                                  isVol, false, Align);
04189     OutChains.push_back(Store);
04190     DstOff += VT.getSizeInBits() / 8;
04191     Size -= VTSize;
04192   }
04193 
04194   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04195 }
04196 
04197 SDValue SelectionDAG::getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst,
04198                                 SDValue Src, SDValue Size,
04199                                 unsigned Align, bool isVol, bool AlwaysInline,
04200                                 MachinePointerInfo DstPtrInfo,
04201                                 MachinePointerInfo SrcPtrInfo) {
04202   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04203 
04204   // Check to see if we should lower the memcpy to loads and stores first.
04205   // For cases within the target-specified limits, this is the best choice.
04206   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04207   if (ConstantSize) {
04208     // Memcpy with size zero? Just return the original chain.
04209     if (ConstantSize->isNullValue())
04210       return Chain;
04211 
04212     SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
04213                                              ConstantSize->getZExtValue(),Align,
04214                                 isVol, false, DstPtrInfo, SrcPtrInfo);
04215     if (Result.getNode())
04216       return Result;
04217   }
04218 
04219   // Then check to see if we should lower the memcpy with target-specific
04220   // code. If the target chooses to do this, this is the next best.
04221   SDValue Result =
04222       TSI->EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align,
04223                                    isVol, AlwaysInline, DstPtrInfo, SrcPtrInfo);
04224   if (Result.getNode())
04225     return Result;
04226 
04227   // If we really need inline code and the target declined to provide it,
04228   // use a (potentially long) sequence of loads and stores.
04229   if (AlwaysInline) {
04230     assert(ConstantSize && "AlwaysInline requires a constant size!");
04231     return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
04232                                    ConstantSize->getZExtValue(), Align, isVol,
04233                                    true, DstPtrInfo, SrcPtrInfo);
04234   }
04235 
04236   // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
04237   // memcpy is not guaranteed to be safe. libc memcpys aren't required to
04238   // respect volatile, so they may do things like read or write memory
04239   // beyond the given memory regions. But fixing this isn't easy, and most
04240   // people don't care.
04241 
04242   // Emit a library call.
04243   TargetLowering::ArgListTy Args;
04244   TargetLowering::ArgListEntry Entry;
04245   Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
04246   Entry.Node = Dst; Args.push_back(Entry);
04247   Entry.Node = Src; Args.push_back(Entry);
04248   Entry.Node = Size; Args.push_back(Entry);
04249   // FIXME: pass in SDLoc
04250   TargetLowering::CallLoweringInfo CLI(*this);
04251   CLI.setDebugLoc(dl).setChain(Chain)
04252     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
04253                Type::getVoidTy(*getContext()),
04254                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
04255                                  TLI->getPointerTy()), std::move(Args), 0)
04256     .setDiscardResult();
04257   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04258 
04259   return CallResult.second;
04260 }
04261 
04262 SDValue SelectionDAG::getMemmove(SDValue Chain, SDLoc dl, SDValue Dst,
04263                                  SDValue Src, SDValue Size,
04264                                  unsigned Align, bool isVol,
04265                                  MachinePointerInfo DstPtrInfo,
04266                                  MachinePointerInfo SrcPtrInfo) {
04267   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04268 
04269   // Check to see if we should lower the memmove to loads and stores first.
04270   // For cases within the target-specified limits, this is the best choice.
04271   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04272   if (ConstantSize) {
04273     // Memmove with size zero? Just return the original chain.
04274     if (ConstantSize->isNullValue())
04275       return Chain;
04276 
04277     SDValue Result =
04278       getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src,
04279                                ConstantSize->getZExtValue(), Align, isVol,
04280                                false, DstPtrInfo, SrcPtrInfo);
04281     if (Result.getNode())
04282       return Result;
04283   }
04284 
04285   // Then check to see if we should lower the memmove with target-specific
04286   // code. If the target chooses to do this, this is the next best.
04287   SDValue Result = TSI->EmitTargetCodeForMemmove(
04288       *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo, SrcPtrInfo);
04289   if (Result.getNode())
04290     return Result;
04291 
04292   // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
04293   // not be safe.  See memcpy above for more details.
04294 
04295   // Emit a library call.
04296   TargetLowering::ArgListTy Args;
04297   TargetLowering::ArgListEntry Entry;
04298   Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
04299   Entry.Node = Dst; Args.push_back(Entry);
04300   Entry.Node = Src; Args.push_back(Entry);
04301   Entry.Node = Size; Args.push_back(Entry);
04302   // FIXME:  pass in SDLoc
04303   TargetLowering::CallLoweringInfo CLI(*this);
04304   CLI.setDebugLoc(dl).setChain(Chain)
04305     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
04306                Type::getVoidTy(*getContext()),
04307                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
04308                                  TLI->getPointerTy()), std::move(Args), 0)
04309     .setDiscardResult();
04310   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04311 
04312   return CallResult.second;
04313 }
04314 
04315 SDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst,
04316                                 SDValue Src, SDValue Size,
04317                                 unsigned Align, bool isVol,
04318                                 MachinePointerInfo DstPtrInfo) {
04319   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04320 
04321   // Check to see if we should lower the memset to stores first.
04322   // For cases within the target-specified limits, this is the best choice.
04323   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04324   if (ConstantSize) {
04325     // Memset with size zero? Just return the original chain.
04326     if (ConstantSize->isNullValue())
04327       return Chain;
04328 
04329     SDValue Result =
04330       getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
04331                       Align, isVol, DstPtrInfo);
04332 
04333     if (Result.getNode())
04334       return Result;
04335   }
04336 
04337   // Then check to see if we should lower the memset with target-specific
04338   // code. If the target chooses to do this, this is the next best.
04339   SDValue Result = TSI->EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src,
04340                                                 Size, Align, isVol, DstPtrInfo);
04341   if (Result.getNode())
04342     return Result;
04343 
04344   // Emit a library call.
04345   Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(*getContext());
04346   TargetLowering::ArgListTy Args;
04347   TargetLowering::ArgListEntry Entry;
04348   Entry.Node = Dst; Entry.Ty = IntPtrTy;
04349   Args.push_back(Entry);
04350   Entry.Node = Src;
04351   Entry.Ty = Src.getValueType().getTypeForEVT(*getContext());
04352   Args.push_back(Entry);
04353   Entry.Node = Size;
04354   Entry.Ty = IntPtrTy;
04355   Args.push_back(Entry);
04356 
04357   // FIXME: pass in SDLoc
04358   TargetLowering::CallLoweringInfo CLI(*this);
04359   CLI.setDebugLoc(dl).setChain(Chain)
04360     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
04361                Type::getVoidTy(*getContext()),
04362                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
04363                                  TLI->getPointerTy()), std::move(Args), 0)
04364     .setDiscardResult();
04365 
04366   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04367   return CallResult.second;
04368 }
04369 
04370 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04371                                 SDVTList VTList, ArrayRef<SDValue> Ops,
04372                                 MachineMemOperand *MMO,
04373                                 AtomicOrdering SuccessOrdering,
04374                                 AtomicOrdering FailureOrdering,
04375                                 SynchronizationScope SynchScope) {
04376   FoldingSetNodeID ID;
04377   ID.AddInteger(MemVT.getRawBits());
04378   AddNodeIDNode(ID, Opcode, VTList, Ops);
04379   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04380   void* IP = nullptr;
04381   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04382     cast<AtomicSDNode>(E)->refineAlignment(MMO);
04383     return SDValue(E, 0);
04384   }
04385 
04386   // Allocate the operands array for the node out of the BumpPtrAllocator, since
04387   // SDNode doesn't have access to it.  This memory will be "leaked" when
04388   // the node is deallocated, but recovered when the allocator is released.
04389   // If the number of operands is less than 5 we use AtomicSDNode's internal
04390   // storage.
04391   unsigned NumOps = Ops.size();
04392   SDUse *DynOps = NumOps > 4 ? OperandAllocator.Allocate<SDUse>(NumOps)
04393                              : nullptr;
04394 
04395   SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(),
04396                                                dl.getDebugLoc(), VTList, MemVT,
04397                                                Ops.data(), DynOps, NumOps, MMO,
04398                                                SuccessOrdering, FailureOrdering,
04399                                                SynchScope);
04400   CSEMap.InsertNode(N, IP);
04401   InsertNode(N);
04402   return SDValue(N, 0);
04403 }
04404 
04405 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04406                                 SDVTList VTList, ArrayRef<SDValue> Ops,
04407                                 MachineMemOperand *MMO,
04408                                 AtomicOrdering Ordering,
04409                                 SynchronizationScope SynchScope) {
04410   return getAtomic(Opcode, dl, MemVT, VTList, Ops, MMO, Ordering,
04411                    Ordering, SynchScope);
04412 }
04413 
04414 SDValue SelectionDAG::getAtomicCmpSwap(
04415     unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs, SDValue Chain,
04416     SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo,
04417     unsigned Alignment, AtomicOrdering SuccessOrdering,
04418     AtomicOrdering FailureOrdering, SynchronizationScope SynchScope) {
04419   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
04420          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
04421   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
04422 
04423   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04424     Alignment = getEVTAlignment(MemVT);
04425 
04426   MachineFunction &MF = getMachineFunction();
04427 
04428   // FIXME: Volatile isn't really correct; we should keep track of atomic
04429   // orderings in the memoperand.
04430   unsigned Flags = MachineMemOperand::MOVolatile;
04431   Flags |= MachineMemOperand::MOLoad;
04432   Flags |= MachineMemOperand::MOStore;
04433 
04434   MachineMemOperand *MMO =
04435     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
04436 
04437   return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO,
04438                           SuccessOrdering, FailureOrdering, SynchScope);
04439 }
04440 
04441 SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, SDLoc dl, EVT MemVT,
04442                                        SDVTList VTs, SDValue Chain, SDValue Ptr,
04443                                        SDValue Cmp, SDValue Swp,
04444                                        MachineMemOperand *MMO,
04445                                        AtomicOrdering SuccessOrdering,
04446                                        AtomicOrdering FailureOrdering,
04447                                        SynchronizationScope SynchScope) {
04448   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
04449          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
04450   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
04451 
04452   SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
04453   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO,
04454                    SuccessOrdering, FailureOrdering, SynchScope);
04455 }
04456 
04457 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04458                                 SDValue Chain,
04459                                 SDValue Ptr, SDValue Val,
04460                                 const Value* PtrVal,
04461                                 unsigned Alignment,
04462                                 AtomicOrdering Ordering,
04463                                 SynchronizationScope SynchScope) {
04464   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04465     Alignment = getEVTAlignment(MemVT);
04466 
04467   MachineFunction &MF = getMachineFunction();
04468   // An atomic store does not load. An atomic load does not store.
04469   // (An atomicrmw obviously both loads and stores.)
04470   // For now, atomics are considered to be volatile always, and they are
04471   // chained as such.
04472   // FIXME: Volatile isn't really correct; we should keep track of atomic
04473   // orderings in the memoperand.
04474   unsigned Flags = MachineMemOperand::MOVolatile;
04475   if (Opcode != ISD::ATOMIC_STORE)
04476     Flags |= MachineMemOperand::MOLoad;
04477   if (Opcode != ISD::ATOMIC_LOAD)
04478     Flags |= MachineMemOperand::MOStore;
04479 
04480   MachineMemOperand *MMO =
04481     MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
04482                             MemVT.getStoreSize(), Alignment);
04483 
04484   return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO,
04485                    Ordering, SynchScope);
04486 }
04487 
04488 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04489                                 SDValue Chain,
04490                                 SDValue Ptr, SDValue Val,
04491                                 MachineMemOperand *MMO,
04492                                 AtomicOrdering Ordering,
04493                                 SynchronizationScope SynchScope) {
04494   assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
04495           Opcode == ISD::ATOMIC_LOAD_SUB ||
04496           Opcode == ISD::ATOMIC_LOAD_AND ||
04497           Opcode == ISD::ATOMIC_LOAD_OR ||
04498           Opcode == ISD::ATOMIC_LOAD_XOR ||
04499           Opcode == ISD::ATOMIC_LOAD_NAND ||
04500           Opcode == ISD::ATOMIC_LOAD_MIN ||
04501           Opcode == ISD::ATOMIC_LOAD_MAX ||
04502           Opcode == ISD::ATOMIC_LOAD_UMIN ||
04503           Opcode == ISD::ATOMIC_LOAD_UMAX ||
04504           Opcode == ISD::ATOMIC_SWAP ||
04505           Opcode == ISD::ATOMIC_STORE) &&
04506          "Invalid Atomic Op");
04507 
04508   EVT VT = Val.getValueType();
04509 
04510   SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
04511                                                getVTList(VT, MVT::Other);
04512   SDValue Ops[] = {Chain, Ptr, Val};
04513   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
04514 }
04515 
04516 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04517                                 EVT VT, SDValue Chain,
04518                                 SDValue Ptr,
04519                                 MachineMemOperand *MMO,
04520                                 AtomicOrdering Ordering,
04521                                 SynchronizationScope SynchScope) {
04522   assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
04523 
04524   SDVTList VTs = getVTList(VT, MVT::Other);
04525   SDValue Ops[] = {Chain, Ptr};
04526   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
04527 }
04528 
04529 /// getMergeValues - Create a MERGE_VALUES node from the given operands.
04530 SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, SDLoc dl) {
04531   if (Ops.size() == 1)
04532     return Ops[0];
04533 
04534   SmallVector<EVT, 4> VTs;
04535   VTs.reserve(Ops.size());
04536   for (unsigned i = 0; i < Ops.size(); ++i)
04537     VTs.push_back(Ops[i].getValueType());
04538   return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
04539 }
04540 
04541 SDValue
04542 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
04543                                   ArrayRef<SDValue> Ops,
04544                                   EVT MemVT, MachinePointerInfo PtrInfo,
04545                                   unsigned Align, bool Vol,
04546                                   bool ReadMem, bool WriteMem, unsigned Size) {
04547   if (Align == 0)  // Ensure that codegen never sees alignment 0
04548     Align = getEVTAlignment(MemVT);
04549 
04550   MachineFunction &MF = getMachineFunction();
04551   unsigned Flags = 0;
04552   if (WriteMem)
04553     Flags |= MachineMemOperand::MOStore;
04554   if (ReadMem)
04555     Flags |= MachineMemOperand::MOLoad;
04556   if (Vol)
04557     Flags |= MachineMemOperand::MOVolatile;
04558   if (!Size)
04559     Size = MemVT.getStoreSize();
04560   MachineMemOperand *MMO =
04561     MF.getMachineMemOperand(PtrInfo, Flags, Size, Align);
04562 
04563   return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
04564 }
04565 
04566 SDValue
04567 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
04568                                   ArrayRef<SDValue> Ops, EVT MemVT,
04569                                   MachineMemOperand *MMO) {
04570   assert((Opcode == ISD::INTRINSIC_VOID ||
04571           Opcode == ISD::INTRINSIC_W_CHAIN ||
04572           Opcode == ISD::PREFETCH ||
04573           Opcode == ISD::LIFETIME_START ||
04574           Opcode == ISD::LIFETIME_END ||
04575           (Opcode <= INT_MAX &&
04576            (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
04577          "Opcode is not a memory-accessing opcode!");
04578 
04579   // Memoize the node unless it returns a flag.
04580   MemIntrinsicSDNode *N;
04581   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
04582     FoldingSetNodeID ID;
04583     AddNodeIDNode(ID, Opcode, VTList, Ops);
04584     ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04585     void *IP = nullptr;
04586     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04587       cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
04588       return SDValue(E, 0);
04589     }
04590 
04591     N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
04592                                                dl.getDebugLoc(), VTList, Ops,
04593                                                MemVT, MMO);
04594     CSEMap.InsertNode(N, IP);
04595   } else {
04596     N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
04597                                                dl.getDebugLoc(), VTList, Ops,
04598                                                MemVT, MMO);
04599   }
04600   InsertNode(N);
04601   return SDValue(N, 0);
04602 }
04603 
04604 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
04605 /// MachinePointerInfo record from it.  This is particularly useful because the
04606 /// code generator has many cases where it doesn't bother passing in a
04607 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
04608 static MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) {
04609   // If this is FI+Offset, we can model it.
04610   if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
04611     return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset);
04612 
04613   // If this is (FI+Offset1)+Offset2, we can model it.
04614   if (Ptr.getOpcode() != ISD::ADD ||
04615       !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
04616       !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
04617     return MachinePointerInfo();
04618 
04619   int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
04620   return MachinePointerInfo::getFixedStack(FI, Offset+
04621                        cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
04622 }
04623 
04624 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
04625 /// MachinePointerInfo record from it.  This is particularly useful because the
04626 /// code generator has many cases where it doesn't bother passing in a
04627 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
04628 static MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) {
04629   // If the 'Offset' value isn't a constant, we can't handle this.
04630   if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
04631     return InferPointerInfo(Ptr, OffsetNode->getSExtValue());
04632   if (OffsetOp.getOpcode() == ISD::UNDEF)
04633     return InferPointerInfo(Ptr);
04634   return MachinePointerInfo();
04635 }
04636 
04637 
04638 SDValue
04639 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
04640                       EVT VT, SDLoc dl, SDValue Chain,
04641                       SDValue Ptr, SDValue Offset,
04642                       MachinePointerInfo PtrInfo, EVT MemVT,
04643                       bool isVolatile, bool isNonTemporal, bool isInvariant,
04644                       unsigned Alignment, const AAMDNodes &AAInfo,
04645                       const MDNode *Ranges) {
04646   assert(Chain.getValueType() == MVT::Other &&
04647         "Invalid chain type");
04648   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04649     Alignment = getEVTAlignment(VT);
04650 
04651   unsigned Flags = MachineMemOperand::MOLoad;
04652   if (isVolatile)
04653     Flags |= MachineMemOperand::MOVolatile;
04654   if (isNonTemporal)
04655     Flags |= MachineMemOperand::MONonTemporal;
04656   if (isInvariant)
04657     Flags |= MachineMemOperand::MOInvariant;
04658 
04659   // If we don't have a PtrInfo, infer the trivial frame index case to simplify
04660   // clients.
04661   if (PtrInfo.V.isNull())
04662     PtrInfo = InferPointerInfo(Ptr, Offset);
04663 
04664   MachineFunction &MF = getMachineFunction();
04665   MachineMemOperand *MMO =
04666     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment,
04667                             AAInfo, Ranges);
04668   return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
04669 }
04670 
04671 SDValue
04672 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
04673                       EVT VT, SDLoc dl, SDValue Chain,
04674                       SDValue Ptr, SDValue Offset, EVT MemVT,
04675                       MachineMemOperand *MMO) {
04676   if (VT == MemVT) {
04677     ExtType = ISD::NON_EXTLOAD;
04678   } else if (ExtType == ISD::NON_EXTLOAD) {
04679     assert(VT == MemVT && "Non-extending load from different memory type!");
04680   } else {
04681     // Extending load.
04682     assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
04683            "Should only be an extending load, not truncating!");
04684     assert(VT.isInteger() == MemVT.isInteger() &&
04685            "Cannot convert from FP to Int or Int -> FP!");
04686     assert(VT.isVector() == MemVT.isVector() &&
04687            "Cannot use trunc store to convert to or from a vector!");
04688     assert((!VT.isVector() ||
04689             VT.getVectorNumElements() == MemVT.getVectorNumElements()) &&
04690            "Cannot use trunc store to change the number of vector elements!");
04691   }
04692 
04693   bool Indexed = AM != ISD::UNINDEXED;
04694   assert((Indexed || Offset.getOpcode() == ISD::UNDEF) &&
04695          "Unindexed load with an offset!");
04696 
04697   SDVTList VTs = Indexed ?
04698     getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
04699   SDValue Ops[] = { Chain, Ptr, Offset };
04700   FoldingSetNodeID ID;
04701   AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
04702   ID.AddInteger(MemVT.getRawBits());
04703   ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(),
04704                                      MMO->isNonTemporal(),
04705                                      MMO->isInvariant()));
04706   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04707   void *IP = nullptr;
04708   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04709     cast<LoadSDNode>(E)->refineAlignment(MMO);
04710     return SDValue(E, 0);
04711   }
04712   SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl.getIROrder(),
04713                                              dl.getDebugLoc(), VTs, AM, ExtType,
04714                                              MemVT, MMO);
04715   CSEMap.InsertNode(N, IP);
04716   InsertNode(N);
04717   return SDValue(N, 0);
04718 }
04719 
04720 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
04721                               SDValue Chain, SDValue Ptr,
04722                               MachinePointerInfo PtrInfo,
04723                               bool isVolatile, bool isNonTemporal,
04724                               bool isInvariant, unsigned Alignment,
04725                               const AAMDNodes &AAInfo,
04726                               const MDNode *Ranges) {
04727   SDValue Undef = getUNDEF(Ptr.getValueType());
04728   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
04729                  PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment,
04730                  AAInfo, Ranges);
04731 }
04732 
04733 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
04734                               SDValue Chain, SDValue Ptr,
04735                               MachineMemOperand *MMO) {
04736   SDValue Undef = getUNDEF(Ptr.getValueType());
04737   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
04738                  VT, MMO);
04739 }
04740 
04741 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
04742                                  SDValue Chain, SDValue Ptr,
04743                                  MachinePointerInfo PtrInfo, EVT MemVT,
04744                                  bool isVolatile, bool isNonTemporal,
04745                                  bool isInvariant, unsigned Alignment,
04746                                  const AAMDNodes &AAInfo) {
04747   SDValue Undef = getUNDEF(Ptr.getValueType());
04748   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
04749                  PtrInfo, MemVT, isVolatile, isNonTemporal, isInvariant,
04750                  Alignment, AAInfo);
04751 }
04752 
04753 
04754 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
04755                                  SDValue Chain, SDValue Ptr, EVT MemVT,
04756                                  MachineMemOperand *MMO) {
04757   SDValue Undef = getUNDEF(Ptr.getValueType());
04758   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
04759                  MemVT, MMO);
04760 }
04761 
04762 SDValue
04763 SelectionDAG::getIndexedLoad(SDValue OrigLoad, SDLoc dl, SDValue Base,
04764                              SDValue Offset, ISD::MemIndexedMode AM) {
04765   LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
04766   assert(LD->getOffset().getOpcode() == ISD::UNDEF &&
04767          "Load is already a indexed load!");
04768   return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
04769                  LD->getChain(), Base, Offset, LD->getPointerInfo(),
04770                  LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(),
04771                  false, LD->getAlignment());
04772 }
04773 
04774 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
04775                                SDValue Ptr, MachinePointerInfo PtrInfo,
04776                                bool isVolatile, bool isNonTemporal,
04777                                unsigned Alignment, const AAMDNodes &AAInfo) {
04778   assert(Chain.getValueType() == MVT::Other &&
04779         "Invalid chain type");
04780   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04781     Alignment = getEVTAlignment(Val.getValueType());
04782 
04783   unsigned Flags = MachineMemOperand::MOStore;
04784   if (isVolatile)
04785     Flags |= MachineMemOperand::MOVolatile;
04786   if (isNonTemporal)
04787     Flags |= MachineMemOperand::MONonTemporal;
04788 
04789   if (PtrInfo.V.isNull())
04790     PtrInfo = InferPointerInfo(Ptr);
04791 
04792   MachineFunction &MF = getMachineFunction();
04793   MachineMemOperand *MMO =
04794     MF.getMachineMemOperand(PtrInfo, Flags,
04795                             Val.getValueType().getStoreSize(), Alignment,
04796                             AAInfo);
04797 
04798   return getStore(Chain, dl, Val, Ptr, MMO);
04799 }
04800 
04801 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
04802                                SDValue Ptr, MachineMemOperand *MMO) {
04803   assert(Chain.getValueType() == MVT::Other &&
04804         "Invalid chain type");
04805   EVT VT = Val.getValueType();
04806   SDVTList VTs = getVTList(MVT::Other);
04807   SDValue Undef = getUNDEF(Ptr.getValueType());
04808   SDValue Ops[] = { Chain, Val, Ptr, Undef };
04809   FoldingSetNodeID ID;
04810   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04811   ID.AddInteger(VT.getRawBits());
04812   ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
04813                                      MMO->isNonTemporal(), MMO->isInvariant()));
04814   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04815   void *IP = nullptr;
04816   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04817     cast<StoreSDNode>(E)->refineAlignment(MMO);
04818     return SDValue(E, 0);
04819   }
04820   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04821                                               dl.getDebugLoc(), VTs,
04822                                               ISD::UNINDEXED, false, VT, MMO);
04823   CSEMap.InsertNode(N, IP);
04824   InsertNode(N);
04825   return SDValue(N, 0);
04826 }
04827 
04828 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
04829                                     SDValue Ptr, MachinePointerInfo PtrInfo,
04830                                     EVT SVT,bool isVolatile, bool isNonTemporal,
04831                                     unsigned Alignment,
04832                                     const AAMDNodes &AAInfo) {
04833   assert(Chain.getValueType() == MVT::Other &&
04834         "Invalid chain type");
04835   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04836     Alignment = getEVTAlignment(SVT);
04837 
04838   unsigned Flags = MachineMemOperand::MOStore;
04839   if (isVolatile)
04840     Flags |= MachineMemOperand::MOVolatile;
04841   if (isNonTemporal)
04842     Flags |= MachineMemOperand::MONonTemporal;
04843 
04844   if (PtrInfo.V.isNull())
04845     PtrInfo = InferPointerInfo(Ptr);
04846 
04847   MachineFunction &MF = getMachineFunction();
04848   MachineMemOperand *MMO =
04849     MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment,
04850                             AAInfo);
04851 
04852   return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
04853 }
04854 
04855 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
04856                                     SDValue Ptr, EVT SVT,
04857                                     MachineMemOperand *MMO) {
04858   EVT VT = Val.getValueType();
04859 
04860   assert(Chain.getValueType() == MVT::Other &&
04861         "Invalid chain type");
04862   if (VT == SVT)
04863     return getStore(Chain, dl, Val, Ptr, MMO);
04864 
04865   assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
04866          "Should only be a truncating store, not extending!");
04867   assert(VT.isInteger() == SVT.isInteger() &&
04868          "Can't do FP-INT conversion!");
04869   assert(VT.isVector() == SVT.isVector() &&
04870          "Cannot use trunc store to convert to or from a vector!");
04871   assert((!VT.isVector() ||
04872           VT.getVectorNumElements() == SVT.getVectorNumElements()) &&
04873          "Cannot use trunc store to change the number of vector elements!");
04874 
04875   SDVTList VTs = getVTList(MVT::Other);
04876   SDValue Undef = getUNDEF(Ptr.getValueType());
04877   SDValue Ops[] = { Chain, Val, Ptr, Undef };
04878   FoldingSetNodeID ID;
04879   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04880   ID.AddInteger(SVT.getRawBits());
04881   ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(),
04882                                      MMO->isNonTemporal(), MMO->isInvariant()));
04883   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04884   void *IP = nullptr;
04885   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04886     cast<StoreSDNode>(E)->refineAlignment(MMO);
04887     return SDValue(E, 0);
04888   }
04889   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04890                                               dl.getDebugLoc(), VTs,
04891                                               ISD::UNINDEXED, true, SVT, MMO);
04892   CSEMap.InsertNode(N, IP);
04893   InsertNode(N);
04894   return SDValue(N, 0);
04895 }
04896 
04897 SDValue
04898 SelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base,
04899                               SDValue Offset, ISD::MemIndexedMode AM) {
04900   StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
04901   assert(ST->getOffset().getOpcode() == ISD::UNDEF &&
04902          "Store is already a indexed store!");
04903   SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
04904   SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
04905   FoldingSetNodeID ID;
04906   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04907   ID.AddInteger(ST->getMemoryVT().getRawBits());
04908   ID.AddInteger(ST->getRawSubclassData());
04909   ID.AddInteger(ST->getPointerInfo().getAddrSpace());
04910   void *IP = nullptr;
04911   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
04912     return SDValue(E, 0);
04913 
04914   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04915                                               dl.getDebugLoc(), VTs, AM,
04916                                               ST->isTruncatingStore(),
04917                                               ST->getMemoryVT(),
04918                                               ST->getMemOperand());
04919   CSEMap.InsertNode(N, IP);
04920   InsertNode(N);
04921   return SDValue(N, 0);
04922 }
04923 
04924 SDValue
04925 SelectionDAG::getMaskedLoad(EVT VT, SDLoc dl, SDValue Chain,
04926                             SDValue Ptr, SDValue Mask, SDValue Src0,
04927                             MachineMemOperand *MMO) {
04928 
04929   SDVTList VTs = getVTList(VT, MVT::Other);
04930   SDValue Ops[] = { Chain, Ptr, Mask, Src0 };
04931   FoldingSetNodeID ID;
04932   AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops);
04933   ID.AddInteger(VT.getRawBits());
04934   ID.AddInteger(encodeMemSDNodeFlags(ISD::NON_EXTLOAD, ISD::UNINDEXED,
04935                                      MMO->isVolatile(),
04936                                      MMO->isNonTemporal(),
04937                                      MMO->isInvariant()));
04938   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04939   void *IP = nullptr;
04940   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04941     cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
04942     return SDValue(E, 0);
04943   }
04944   SDNode *N = new (NodeAllocator) MaskedLoadSDNode(dl.getIROrder(),
04945                                              dl.getDebugLoc(), Ops, 4, VTs,
04946                                              VT, MMO);
04947   CSEMap.InsertNode(N, IP);
04948   InsertNode(N);
04949   return SDValue(N, 0);
04950 }
04951 
04952 SDValue SelectionDAG::getMaskedStore(SDValue Chain, SDLoc dl, SDValue Val,
04953                                SDValue Ptr, SDValue Mask, MachineMemOperand *MMO) {
04954   assert(Chain.getValueType() == MVT::Other &&
04955         "Invalid chain type");
04956   EVT VT = Val.getValueType();
04957   SDVTList VTs = getVTList(MVT::Other);
04958   SDValue Ops[] = { Chain, Ptr, Mask, Val };
04959   FoldingSetNodeID ID;
04960   AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops);
04961   ID.AddInteger(VT.getRawBits());
04962   ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
04963                                      MMO->isNonTemporal(), MMO->isInvariant()));
04964   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04965   void *IP = nullptr;
04966   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04967     cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
04968     return SDValue(E, 0);
04969   }
04970   SDNode *N = new (NodeAllocator) MaskedStoreSDNode(dl.getIROrder(),
04971                                                     dl.getDebugLoc(), Ops, 4,
04972                                                     VTs, VT, MMO);
04973   CSEMap.InsertNode(N, IP);
04974   InsertNode(N);
04975   return SDValue(N, 0);
04976 }
04977 
04978 SDValue SelectionDAG::getVAArg(EVT VT, SDLoc dl,
04979                                SDValue Chain, SDValue Ptr,
04980                                SDValue SV,
04981                                unsigned Align) {
04982   SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) };
04983   return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
04984 }
04985 
04986 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
04987                               ArrayRef<SDUse> Ops) {
04988   switch (Ops.size()) {
04989   case 0: return getNode(Opcode, DL, VT);
04990   case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
04991   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
04992   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
04993   default: break;
04994   }
04995 
04996   // Copy from an SDUse array into an SDValue array for use with
04997   // the regular getNode logic.
04998   SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
04999   return getNode(Opcode, DL, VT, NewOps);
05000 }
05001 
05002 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
05003                               ArrayRef<SDValue> Ops) {
05004   unsigned NumOps = Ops.size();
05005   switch (NumOps) {
05006   case 0: return getNode(Opcode, DL, VT);
05007   case 1: return getNode(Opcode, DL, VT, Ops[0]);
05008   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
05009   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
05010   default: break;
05011   }
05012 
05013   switch (Opcode) {
05014   default: break;
05015   case ISD::SELECT_CC: {
05016     assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
05017     assert(Ops[0].getValueType() == Ops[1].getValueType() &&
05018            "LHS and RHS of condition must have same type!");
05019     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
05020            "True and False arms of SelectCC must have same type!");
05021     assert(Ops[2].getValueType() == VT &&
05022            "select_cc node must be of same type as true and false value!");
05023     break;
05024   }
05025   case ISD::BR_CC: {
05026     assert(NumOps == 5 && "BR_CC takes 5 operands!");
05027     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
05028            "LHS/RHS of comparison should match types!");
05029     break;
05030   }
05031   }
05032 
05033   // Memoize nodes.
05034   SDNode *N;
05035   SDVTList VTs = getVTList(VT);
05036 
05037   if (VT != MVT::Glue) {
05038     FoldingSetNodeID ID;
05039     AddNodeIDNode(ID, Opcode, VTs, Ops);
05040     void *IP = nullptr;
05041 
05042     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05043       return SDValue(E, 0);
05044 
05045     N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05046                                    VTs, Ops);
05047     CSEMap.InsertNode(N, IP);
05048   } else {
05049     N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05050                                    VTs, Ops);
05051   }
05052 
05053   InsertNode(N);
05054   return SDValue(N, 0);
05055 }
05056 
05057 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
05058                               ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
05059   return getNode(Opcode, DL, getVTList(ResultTys), Ops);
05060 }
05061 
05062 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05063                               ArrayRef<SDValue> Ops) {
05064   if (VTList.NumVTs == 1)
05065     return getNode(Opcode, DL, VTList.VTs[0], Ops);
05066 
05067 #if 0
05068   switch (Opcode) {
05069   // FIXME: figure out how to safely handle things like
05070   // int foo(int x) { return 1 << (x & 255); }
05071   // int bar() { return foo(256); }
05072   case ISD::SRA_PARTS:
05073   case ISD::SRL_PARTS:
05074   case ISD::SHL_PARTS:
05075     if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
05076         cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
05077       return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
05078     else if (N3.getOpcode() == ISD::AND)
05079       if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
05080         // If the and is only masking out bits that cannot effect the shift,
05081         // eliminate the and.
05082         unsigned NumBits = VT.getScalarType().getSizeInBits()*2;
05083         if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
05084           return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
05085       }
05086     break;
05087   }
05088 #endif
05089 
05090   // Memoize the node unless it returns a flag.
05091   SDNode *N;
05092   unsigned NumOps = Ops.size();
05093   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
05094     FoldingSetNodeID ID;
05095     AddNodeIDNode(ID, Opcode, VTList, Ops);
05096     void *IP = nullptr;
05097     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05098       return SDValue(E, 0);
05099 
05100     if (NumOps == 1) {
05101       N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
05102                                           DL.getDebugLoc(), VTList, Ops[0]);
05103     } else if (NumOps == 2) {
05104       N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
05105                                            DL.getDebugLoc(), VTList, Ops[0],
05106                                            Ops[1]);
05107     } else if (NumOps == 3) {
05108       N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
05109                                             DL.getDebugLoc(), VTList, Ops[0],
05110                                             Ops[1], Ops[2]);
05111     } else {
05112       N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05113                                      VTList, Ops);
05114     }
05115     CSEMap.InsertNode(N, IP);
05116   } else {
05117     if (NumOps == 1) {
05118       N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
05119                                           DL.getDebugLoc(), VTList, Ops[0]);
05120     } else if (NumOps == 2) {
05121       N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
05122                                            DL.getDebugLoc(), VTList, Ops[0],
05123                                            Ops[1]);
05124     } else if (NumOps == 3) {
05125       N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
05126                                             DL.getDebugLoc(), VTList, Ops[0],
05127                                             Ops[1], Ops[2]);
05128     } else {
05129       N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05130                                      VTList, Ops);
05131     }
05132   }
05133   InsertNode(N);
05134   return SDValue(N, 0);
05135 }
05136 
05137 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList) {
05138   return getNode(Opcode, DL, VTList, None);
05139 }
05140 
05141 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05142                               SDValue N1) {
05143   SDValue Ops[] = { N1 };
05144   return getNode(Opcode, DL, VTList, Ops);
05145 }
05146 
05147 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05148                               SDValue N1, SDValue N2) {
05149   SDValue Ops[] = { N1, N2 };
05150   return getNode(Opcode, DL, VTList, Ops);
05151 }
05152 
05153 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05154                               SDValue N1, SDValue N2, SDValue N3) {
05155   SDValue Ops[] = { N1, N2, N3 };
05156   return getNode(Opcode, DL, VTList, Ops);
05157 }
05158 
05159 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05160                               SDValue N1, SDValue N2, SDValue N3,
05161                               SDValue N4) {
05162   SDValue Ops[] = { N1, N2, N3, N4 };
05163   return getNode(Opcode, DL, VTList, Ops);
05164 }
05165 
05166 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05167                               SDValue N1, SDValue N2, SDValue N3,
05168                               SDValue N4, SDValue N5) {
05169   SDValue Ops[] = { N1, N2, N3, N4, N5 };
05170   return getNode(Opcode, DL, VTList, Ops);
05171 }
05172 
05173 SDVTList SelectionDAG::getVTList(EVT VT) {
05174   return makeVTList(SDNode::getValueTypeList(VT), 1);
05175 }
05176 
05177 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
05178   FoldingSetNodeID ID;
05179   ID.AddInteger(2U);
05180   ID.AddInteger(VT1.getRawBits());
05181   ID.AddInteger(VT2.getRawBits());
05182 
05183   void *IP = nullptr;
05184   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05185   if (!Result) {
05186     EVT *Array = Allocator.Allocate<EVT>(2);
05187     Array[0] = VT1;
05188     Array[1] = VT2;
05189     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
05190     VTListMap.InsertNode(Result, IP);
05191   }
05192   return Result->getSDVTList();
05193 }
05194 
05195 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
05196   FoldingSetNodeID ID;
05197   ID.AddInteger(3U);
05198   ID.AddInteger(VT1.getRawBits());
05199   ID.AddInteger(VT2.getRawBits());
05200   ID.AddInteger(VT3.getRawBits());
05201 
05202   void *IP = nullptr;
05203   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05204   if (!Result) {
05205     EVT *Array = Allocator.Allocate<EVT>(3);
05206     Array[0] = VT1;
05207     Array[1] = VT2;
05208     Array[2] = VT3;
05209     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
05210     VTListMap.InsertNode(Result, IP);
05211   }
05212   return Result->getSDVTList();
05213 }
05214 
05215 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
05216   FoldingSetNodeID ID;
05217   ID.AddInteger(4U);
05218   ID.AddInteger(VT1.getRawBits());
05219   ID.AddInteger(VT2.getRawBits());
05220   ID.AddInteger(VT3.getRawBits());
05221   ID.AddInteger(VT4.getRawBits());
05222 
05223   void *IP = nullptr;
05224   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05225   if (!Result) {
05226     EVT *Array = Allocator.Allocate<EVT>(4);
05227     Array[0] = VT1;
05228     Array[1] = VT2;
05229     Array[2] = VT3;
05230     Array[3] = VT4;
05231     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
05232     VTListMap.InsertNode(Result, IP);
05233   }
05234   return Result->getSDVTList();
05235 }
05236 
05237 SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
05238   unsigned NumVTs = VTs.size();
05239   FoldingSetNodeID ID;
05240   ID.AddInteger(NumVTs);
05241   for (unsigned index = 0; index < NumVTs; index++) {
05242     ID.AddInteger(VTs[index].getRawBits());
05243   }
05244 
05245   void *IP = nullptr;
05246   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05247   if (!Result) {
05248     EVT *Array = Allocator.Allocate<EVT>(NumVTs);
05249     std::copy(VTs.begin(), VTs.end(), Array);
05250     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
05251     VTListMap.InsertNode(Result, IP);
05252   }
05253   return Result->getSDVTList();
05254 }
05255 
05256 
05257 /// UpdateNodeOperands - *Mutate* the specified node in-place to have the
05258 /// specified operands.  If the resultant node already exists in the DAG,
05259 /// this does not modify the specified node, instead it returns the node that
05260 /// already exists.  If the resultant node does not exist in the DAG, the
05261 /// input node is returned.  As a degenerate case, if you specify the same
05262 /// input operands as the node already has, the input node is returned.
05263 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
05264   assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
05265 
05266   // Check to see if there is no change.
05267   if (Op == N->getOperand(0)) return N;
05268 
05269   // See if the modified node already exists.
05270   void *InsertPos = nullptr;
05271   if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
05272     return Existing;
05273 
05274   // Nope it doesn't.  Remove the node from its current place in the maps.
05275   if (InsertPos)
05276     if (!RemoveNodeFromCSEMaps(N))
05277       InsertPos = nullptr;
05278 
05279   // Now we update the operands.
05280   N->OperandList[0].set(Op);
05281 
05282   // If this gets put into a CSE map, add it.
05283   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05284   return N;
05285 }
05286 
05287 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
05288   assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
05289 
05290   // Check to see if there is no change.
05291   if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
05292     return N;   // No operands changed, just return the input node.
05293 
05294   // See if the modified node already exists.
05295   void *InsertPos = nullptr;
05296   if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
05297     return Existing;
05298 
05299   // Nope it doesn't.  Remove the node from its current place in the maps.
05300   if (InsertPos)
05301     if (!RemoveNodeFromCSEMaps(N))
05302       InsertPos = nullptr;
05303 
05304   // Now we update the operands.
05305   if (N->OperandList[0] != Op1)
05306     N->OperandList[0].set(Op1);
05307   if (N->OperandList[1] != Op2)
05308     N->OperandList[1].set(Op2);
05309 
05310   // If this gets put into a CSE map, add it.
05311   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05312   return N;
05313 }
05314 
05315 SDNode *SelectionDAG::
05316 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
05317   SDValue Ops[] = { Op1, Op2, Op3 };
05318   return UpdateNodeOperands(N, Ops);
05319 }
05320 
05321 SDNode *SelectionDAG::
05322 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
05323                    SDValue Op3, SDValue Op4) {
05324   SDValue Ops[] = { Op1, Op2, Op3, Op4 };
05325   return UpdateNodeOperands(N, Ops);
05326 }
05327 
05328 SDNode *SelectionDAG::
05329 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
05330                    SDValue Op3, SDValue Op4, SDValue Op5) {
05331   SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
05332   return UpdateNodeOperands(N, Ops);
05333 }
05334 
05335 SDNode *SelectionDAG::
05336 UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
05337   unsigned NumOps = Ops.size();
05338   assert(N->getNumOperands() == NumOps &&
05339          "Update with wrong number of operands");
05340 
05341   // Check to see if there is no change.
05342   bool AnyChange = false;
05343   for (unsigned i = 0; i != NumOps; ++i) {
05344     if (Ops[i] != N->getOperand(i)) {
05345       AnyChange = true;
05346       break;
05347     }
05348   }
05349 
05350   // No operands changed, just return the input node.
05351   if (!AnyChange) return N;
05352 
05353   // See if the modified node already exists.
05354   void *InsertPos = nullptr;
05355   if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
05356     return Existing;
05357 
05358   // Nope it doesn't.  Remove the node from its current place in the maps.
05359   if (InsertPos)
05360     if (!RemoveNodeFromCSEMaps(N))
05361       InsertPos = nullptr;
05362 
05363   // Now we update the operands.
05364   for (unsigned i = 0; i != NumOps; ++i)
05365     if (N->OperandList[i] != Ops[i])
05366       N->OperandList[i].set(Ops[i]);
05367 
05368   // If this gets put into a CSE map, add it.
05369   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05370   return N;
05371 }
05372 
05373 /// DropOperands - Release the operands and set this node to have
05374 /// zero operands.
05375 void SDNode::DropOperands() {
05376   // Unlike the code in MorphNodeTo that does this, we don't need to
05377   // watch for dead nodes here.
05378   for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
05379     SDUse &Use = *I++;
05380     Use.set(SDValue());
05381   }
05382 }
05383 
05384 /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
05385 /// machine opcode.
05386 ///
05387 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05388                                    EVT VT) {
05389   SDVTList VTs = getVTList(VT);
05390   return SelectNodeTo(N, MachineOpc, VTs, None);
05391 }
05392 
05393 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05394                                    EVT VT, SDValue Op1) {
05395   SDVTList VTs = getVTList(VT);
05396   SDValue Ops[] = { Op1 };
05397   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05398 }
05399 
05400 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05401                                    EVT VT, SDValue Op1,
05402                                    SDValue Op2) {
05403   SDVTList VTs = getVTList(VT);
05404   SDValue Ops[] = { Op1, Op2 };
05405   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05406 }
05407 
05408 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05409                                    EVT VT, SDValue Op1,
05410                                    SDValue Op2, SDValue Op3) {
05411   SDVTList VTs = getVTList(VT);
05412   SDValue Ops[] = { Op1, Op2, Op3 };
05413   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05414 }
05415 
05416 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05417                                    EVT VT, ArrayRef<SDValue> Ops) {
05418   SDVTList VTs = getVTList(VT);
05419   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05420 }
05421 
05422 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05423                                    EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
05424   SDVTList VTs = getVTList(VT1, VT2);
05425   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05426 }
05427 
05428 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05429                                    EVT VT1, EVT VT2) {
05430   SDVTList VTs = getVTList(VT1, VT2);
05431   return SelectNodeTo(N, MachineOpc, VTs, None);
05432 }
05433 
05434 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05435                                    EVT VT1, EVT VT2, EVT VT3,
05436                                    ArrayRef<SDValue> Ops) {
05437   SDVTList VTs = getVTList(VT1, VT2, VT3);
05438   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05439 }
05440 
05441 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05442                                    EVT VT1, EVT VT2, EVT VT3, EVT VT4,
05443                                    ArrayRef<SDValue> Ops) {
05444   SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
05445   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05446 }
05447 
05448 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05449                                    EVT VT1, EVT VT2,
05450                                    SDValue Op1) {
05451   SDVTList VTs = getVTList(VT1, VT2);
05452   SDValue Ops[] = { Op1 };
05453   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05454 }
05455 
05456 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05457                                    EVT VT1, EVT VT2,
05458                                    SDValue Op1, SDValue Op2) {
05459   SDVTList VTs = getVTList(VT1, VT2);
05460   SDValue Ops[] = { Op1, Op2 };
05461   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05462 }
05463 
05464 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05465                                    EVT VT1, EVT VT2,
05466                                    SDValue Op1, SDValue Op2,
05467                                    SDValue Op3) {
05468   SDVTList VTs = getVTList(VT1, VT2);
05469   SDValue Ops[] = { Op1, Op2, Op3 };
05470   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05471 }
05472 
05473 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05474                                    EVT VT1, EVT VT2, EVT VT3,
05475                                    SDValue Op1, SDValue Op2,
05476                                    SDValue Op3) {
05477   SDVTList VTs = getVTList(VT1, VT2, VT3);
05478   SDValue Ops[] = { Op1, Op2, Op3 };
05479   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05480 }
05481 
05482 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05483                                    SDVTList VTs,ArrayRef<SDValue> Ops) {
05484   N = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
05485   // Reset the NodeID to -1.
05486   N->setNodeId(-1);
05487   return N;
05488 }
05489 
05490 /// UpdadeSDLocOnMergedSDNode - If the opt level is -O0 then it throws away
05491 /// the line number information on the merged node since it is not possible to
05492 /// preserve the information that operation is associated with multiple lines.
05493 /// This will make the debugger working better at -O0, were there is a higher
05494 /// probability having other instructions associated with that line.
05495 ///
05496 /// For IROrder, we keep the smaller of the two
05497 SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) {
05498   DebugLoc NLoc = N->getDebugLoc();
05499   if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) &&
05500     (OLoc.getDebugLoc() != NLoc)) {
05501     N->setDebugLoc(DebugLoc());
05502   }
05503   unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
05504   N->setIROrder(Order);
05505   return N;
05506 }
05507 
05508 /// MorphNodeTo - This *mutates* the specified node to have the specified
05509 /// return type, opcode, and operands.
05510 ///
05511 /// Note that MorphNodeTo returns the resultant node.  If there is already a
05512 /// node of the specified opcode and operands, it returns that node instead of
05513 /// the current one.  Note that the SDLoc need not be the same.
05514 ///
05515 /// Using MorphNodeTo is faster than creating a new node and swapping it in
05516 /// with ReplaceAllUsesWith both because it often avoids allocating a new
05517 /// node, and because it doesn't require CSE recalculation for any of
05518 /// the node's users.
05519 ///
05520 /// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
05521 /// As a consequence it isn't appropriate to use from within the DAG combiner or
05522 /// the legalizer which maintain worklists that would need to be updated when
05523 /// deleting things.
05524 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
05525                                   SDVTList VTs, ArrayRef<SDValue> Ops) {
05526   unsigned NumOps = Ops.size();
05527   // If an identical node already exists, use it.
05528   void *IP = nullptr;
05529   if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
05530     FoldingSetNodeID ID;
05531     AddNodeIDNode(ID, Opc, VTs, Ops);
05532     if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
05533       return UpdadeSDLocOnMergedSDNode(ON, SDLoc(N));
05534   }
05535 
05536   if (!RemoveNodeFromCSEMaps(N))
05537     IP = nullptr;
05538 
05539   // Start the morphing.
05540   N->NodeType = Opc;
05541   N->ValueList = VTs.VTs;
05542   N->NumValues = VTs.NumVTs;
05543 
05544   // Clear the operands list, updating used nodes to remove this from their
05545   // use list.  Keep track of any operands that become dead as a result.
05546   SmallPtrSet<SDNode*, 16> DeadNodeSet;
05547   for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
05548     SDUse &Use = *I++;
05549     SDNode *Used = Use.getNode();
05550     Use.set(SDValue());
05551     if (Used->use_empty())
05552       DeadNodeSet.insert(Used);
05553   }
05554 
05555   if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) {
05556     // Initialize the memory references information.
05557     MN->setMemRefs(nullptr, nullptr);
05558     // If NumOps is larger than the # of operands we can have in a
05559     // MachineSDNode, reallocate the operand list.
05560     if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) {
05561       if (MN->OperandsNeedDelete)
05562         delete[] MN->OperandList;
05563       if (NumOps > array_lengthof(MN->LocalOperands))
05564         // We're creating a final node that will live unmorphed for the
05565         // remainder of the current SelectionDAG iteration, so we can allocate
05566         // the operands directly out of a pool with no recycling metadata.
05567         MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
05568                          Ops.data(), NumOps);
05569       else
05570         MN->InitOperands(MN->LocalOperands, Ops.data(), NumOps);
05571       MN->OperandsNeedDelete = false;
05572     } else
05573       MN->InitOperands(MN->OperandList, Ops.data(), NumOps);
05574   } else {
05575     // If NumOps is larger than the # of operands we currently have, reallocate
05576     // the operand list.
05577     if (NumOps > N->NumOperands) {
05578       if (N->OperandsNeedDelete)
05579         delete[] N->OperandList;
05580       N->InitOperands(new SDUse[NumOps], Ops.data(), NumOps);
05581       N->OperandsNeedDelete = true;
05582     } else
05583       N->InitOperands(N->OperandList, Ops.data(), NumOps);
05584   }
05585 
05586   // Delete any nodes that are still dead after adding the uses for the
05587   // new operands.
05588   if (!DeadNodeSet.empty()) {
05589     SmallVector<SDNode *, 16> DeadNodes;
05590     for (SDNode *N : DeadNodeSet)
05591       if (N->use_empty())
05592         DeadNodes.push_back(N);
05593     RemoveDeadNodes(DeadNodes);
05594   }
05595 
05596   if (IP)
05597     CSEMap.InsertNode(N, IP);   // Memoize the new node.
05598   return N;
05599 }
05600 
05601 
05602 /// getMachineNode - These are used for target selectors to create a new node
05603 /// with specified return type(s), MachineInstr opcode, and operands.
05604 ///
05605 /// Note that getMachineNode returns the resultant node.  If there is already a
05606 /// node of the specified opcode and operands, it returns that node instead of
05607 /// the current one.
05608 MachineSDNode *
05609 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT) {
05610   SDVTList VTs = getVTList(VT);
05611   return getMachineNode(Opcode, dl, VTs, None);
05612 }
05613 
05614 MachineSDNode *
05615 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, SDValue Op1) {
05616   SDVTList VTs = getVTList(VT);
05617   SDValue Ops[] = { Op1 };
05618   return getMachineNode(Opcode, dl, VTs, Ops);
05619 }
05620 
05621 MachineSDNode *
05622 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05623                              SDValue Op1, SDValue Op2) {
05624   SDVTList VTs = getVTList(VT);
05625   SDValue Ops[] = { Op1, Op2 };
05626   return getMachineNode(Opcode, dl, VTs, Ops);
05627 }
05628 
05629 MachineSDNode *
05630 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05631                              SDValue Op1, SDValue Op2, SDValue Op3) {
05632   SDVTList VTs = getVTList(VT);
05633   SDValue Ops[] = { Op1, Op2, Op3 };
05634   return getMachineNode(Opcode, dl, VTs, Ops);
05635 }
05636 
05637 MachineSDNode *
05638 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05639                              ArrayRef<SDValue> Ops) {
05640   SDVTList VTs = getVTList(VT);
05641   return getMachineNode(Opcode, dl, VTs, Ops);
05642 }
05643 
05644 MachineSDNode *
05645 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, EVT VT2) {
05646   SDVTList VTs = getVTList(VT1, VT2);
05647   return getMachineNode(Opcode, dl, VTs, None);
05648 }
05649 
05650 MachineSDNode *
05651 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05652                              EVT VT1, EVT VT2, SDValue Op1) {
05653   SDVTList VTs = getVTList(VT1, VT2);
05654   SDValue Ops[] = { Op1 };
05655   return getMachineNode(Opcode, dl, VTs, Ops);
05656 }
05657 
05658 MachineSDNode *
05659 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05660                              EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) {
05661   SDVTList VTs = getVTList(VT1, VT2);
05662   SDValue Ops[] = { Op1, Op2 };
05663   return getMachineNode(Opcode, dl, VTs, Ops);
05664 }
05665 
05666 MachineSDNode *
05667 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05668                              EVT VT1, EVT VT2, SDValue Op1,
05669                              SDValue Op2, SDValue Op3) {
05670   SDVTList VTs = getVTList(VT1, VT2);
05671   SDValue Ops[] = { Op1, Op2, Op3 };
05672   return getMachineNode(Opcode, dl, VTs, Ops);
05673 }
05674 
05675 MachineSDNode *
05676 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05677                              EVT VT1, EVT VT2,
05678                              ArrayRef<SDValue> Ops) {
05679   SDVTList VTs = getVTList(VT1, VT2);
05680   return getMachineNode(Opcode, dl, VTs, Ops);
05681 }
05682 
05683 MachineSDNode *
05684 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05685                              EVT VT1, EVT VT2, EVT VT3,
05686                              SDValue Op1, SDValue Op2) {
05687   SDVTList VTs = getVTList(VT1, VT2, VT3);
05688   SDValue Ops[] = { Op1, Op2 };
05689   return getMachineNode(Opcode, dl, VTs, Ops);
05690 }
05691 
05692 MachineSDNode *
05693 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05694                              EVT VT1, EVT VT2, EVT VT3,
05695                              SDValue Op1, SDValue Op2, SDValue Op3) {
05696   SDVTList VTs = getVTList(VT1, VT2, VT3);
05697   SDValue Ops[] = { Op1, Op2, Op3 };
05698   return getMachineNode(Opcode, dl, VTs, Ops);
05699 }
05700 
05701 MachineSDNode *
05702 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05703                              EVT VT1, EVT VT2, EVT VT3,
05704                              ArrayRef<SDValue> Ops) {
05705   SDVTList VTs = getVTList(VT1, VT2, VT3);
05706   return getMachineNode(Opcode, dl, VTs, Ops);
05707 }
05708 
05709 MachineSDNode *
05710 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1,
05711                              EVT VT2, EVT VT3, EVT VT4,
05712                              ArrayRef<SDValue> Ops) {
05713   SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
05714   return getMachineNode(Opcode, dl, VTs, Ops);
05715 }
05716 
05717 MachineSDNode *
05718 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05719                              ArrayRef<EVT> ResultTys,
05720                              ArrayRef<SDValue> Ops) {
05721   SDVTList VTs = getVTList(ResultTys);
05722   return getMachineNode(Opcode, dl, VTs, Ops);
05723 }
05724 
05725 MachineSDNode *
05726 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
05727                              ArrayRef<SDValue> OpsArray) {
05728   bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
05729   MachineSDNode *N;
05730   void *IP = nullptr;
05731   const SDValue *Ops = OpsArray.data();
05732   unsigned NumOps = OpsArray.size();
05733 
05734   if (DoCSE) {
05735     FoldingSetNodeID ID;
05736     AddNodeIDNode(ID, ~Opcode, VTs, OpsArray);
05737     IP = nullptr;
05738     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
05739       return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL));
05740     }
05741   }
05742 
05743   // Allocate a new MachineSDNode.
05744   N = new (NodeAllocator) MachineSDNode(~Opcode, DL.getIROrder(),
05745                                         DL.getDebugLoc(), VTs);
05746 
05747   // Initialize the operands list.
05748   if (NumOps > array_lengthof(N->LocalOperands))
05749     // We're creating a final node that will live unmorphed for the
05750     // remainder of the current SelectionDAG iteration, so we can allocate
05751     // the operands directly out of a pool with no recycling metadata.
05752     N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
05753                     Ops, NumOps);
05754   else
05755     N->InitOperands(N->LocalOperands, Ops, NumOps);
05756   N->OperandsNeedDelete = false;
05757 
05758   if (DoCSE)
05759     CSEMap.InsertNode(N, IP);
05760 
05761   InsertNode(N);
05762   return N;
05763 }
05764 
05765 /// getTargetExtractSubreg - A convenience function for creating
05766 /// TargetOpcode::EXTRACT_SUBREG nodes.
05767 SDValue
05768 SelectionDAG::getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT,
05769                                      SDValue Operand) {
05770   SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
05771   SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
05772                                   VT, Operand, SRIdxVal);
05773   return SDValue(Subreg, 0);
05774 }
05775 
05776 /// getTargetInsertSubreg - A convenience function for creating
05777 /// TargetOpcode::INSERT_SUBREG nodes.
05778 SDValue
05779 SelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT,
05780                                     SDValue Operand, SDValue Subreg) {
05781   SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
05782   SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
05783                                   VT, Operand, Subreg, SRIdxVal);
05784   return SDValue(Result, 0);
05785 }
05786 
05787 /// getNodeIfExists - Get the specified node if it's already available, or
05788 /// else return NULL.
05789 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
05790                                       ArrayRef<SDValue> Ops, bool nuw, bool nsw,
05791                                       bool exact) {
05792   if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
05793     FoldingSetNodeID ID;
05794     AddNodeIDNode(ID, Opcode, VTList, Ops);
05795     if (isBinOpWithFlags(Opcode))
05796       AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
05797     void *IP = nullptr;
05798     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05799       return E;
05800   }
05801   return nullptr;
05802 }
05803 
05804 /// getDbgValue - Creates a SDDbgValue node.
05805 ///
05806 /// SDNode
05807 SDDbgValue *SelectionDAG::getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N,
05808                                       unsigned R, bool IsIndirect, uint64_t Off,
05809                                       DebugLoc DL, unsigned O) {
05810   return new (Allocator) SDDbgValue(Var, Expr, N, R, IsIndirect, Off, DL, O);
05811 }
05812 
05813 /// Constant
05814 SDDbgValue *SelectionDAG::getConstantDbgValue(MDNode *Var, MDNode *Expr,
05815                                               const Value *C, uint64_t Off,
05816                                               DebugLoc DL, unsigned O) {
05817   return new (Allocator) SDDbgValue(Var, Expr, C, Off, DL, O);
05818 }
05819 
05820 /// FrameIndex
05821 SDDbgValue *SelectionDAG::getFrameIndexDbgValue(MDNode *Var, MDNode *Expr,
05822                                                 unsigned FI, uint64_t Off,
05823                                                 DebugLoc DL, unsigned O) {
05824   return new (Allocator) SDDbgValue(Var, Expr, FI, Off, DL, O);
05825 }
05826 
05827 namespace {
05828 
05829 /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
05830 /// pointed to by a use iterator is deleted, increment the use iterator
05831 /// so that it doesn't dangle.
05832 ///
05833 class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
05834   SDNode::use_iterator &UI;
05835   SDNode::use_iterator &UE;
05836 
05837   void NodeDeleted(SDNode *N, SDNode *E) override {
05838     // Increment the iterator as needed.
05839     while (UI != UE && N == *UI)
05840       ++UI;
05841   }
05842 
05843 public:
05844   RAUWUpdateListener(SelectionDAG &d,
05845                      SDNode::use_iterator &ui,
05846                      SDNode::use_iterator &ue)
05847     : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
05848 };
05849 
05850 }
05851 
05852 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05853 /// This can cause recursive merging of nodes in the DAG.
05854 ///
05855 /// This version assumes From has a single result value.
05856 ///
05857 void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
05858   SDNode *From = FromN.getNode();
05859   assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
05860          "Cannot replace with this method!");
05861   assert(From != To.getNode() && "Cannot replace uses of with self");
05862 
05863   // Iterate over all the existing uses of From. New uses will be added
05864   // to the beginning of the use list, which we avoid visiting.
05865   // This specifically avoids visiting uses of From that arise while the
05866   // replacement is happening, because any such uses would be the result
05867   // of CSE: If an existing node looks like From after one of its operands
05868   // is replaced by To, we don't want to replace of all its users with To
05869   // too. See PR3018 for more info.
05870   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05871   RAUWUpdateListener Listener(*this, UI, UE);
05872   while (UI != UE) {
05873     SDNode *User = *UI;
05874 
05875     // This node is about to morph, remove its old self from the CSE maps.
05876     RemoveNodeFromCSEMaps(User);
05877 
05878     // A user can appear in a use list multiple times, and when this
05879     // happens the uses are usually next to each other in the list.
05880     // To help reduce the number of CSE recomputations, process all
05881     // the uses of this user that we can find this way.
05882     do {
05883       SDUse &Use = UI.getUse();
05884       ++UI;
05885       Use.set(To);
05886     } while (UI != UE && *UI == User);
05887 
05888     // Now that we have modified User, add it back to the CSE maps.  If it
05889     // already exists there, recursively merge the results together.
05890     AddModifiedNodeToCSEMaps(User);
05891   }
05892 
05893   // If we just RAUW'd the root, take note.
05894   if (FromN == getRoot())
05895     setRoot(To);
05896 }
05897 
05898 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05899 /// This can cause recursive merging of nodes in the DAG.
05900 ///
05901 /// This version assumes that for each value of From, there is a
05902 /// corresponding value in To in the same position with the same type.
05903 ///
05904 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
05905 #ifndef NDEBUG
05906   for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
05907     assert((!From->hasAnyUseOfValue(i) ||
05908             From->getValueType(i) == To->getValueType(i)) &&
05909            "Cannot use this version of ReplaceAllUsesWith!");
05910 #endif
05911 
05912   // Handle the trivial case.
05913   if (From == To)
05914     return;
05915 
05916   // Iterate over just the existing users of From. See the comments in
05917   // the ReplaceAllUsesWith above.
05918   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05919   RAUWUpdateListener Listener(*this, UI, UE);
05920   while (UI != UE) {
05921     SDNode *User = *UI;
05922 
05923     // This node is about to morph, remove its old self from the CSE maps.
05924     RemoveNodeFromCSEMaps(User);
05925 
05926     // A user can appear in a use list multiple times, and when this
05927     // happens the uses are usually next to each other in the list.
05928     // To help reduce the number of CSE recomputations, process all
05929     // the uses of this user that we can find this way.
05930     do {
05931       SDUse &Use = UI.getUse();
05932       ++UI;
05933       Use.setNode(To);
05934     } while (UI != UE && *UI == User);
05935 
05936     // Now that we have modified User, add it back to the CSE maps.  If it
05937     // already exists there, recursively merge the results together.
05938     AddModifiedNodeToCSEMaps(User);
05939   }
05940 
05941   // If we just RAUW'd the root, take note.
05942   if (From == getRoot().getNode())
05943     setRoot(SDValue(To, getRoot().getResNo()));
05944 }
05945 
05946 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05947 /// This can cause recursive merging of nodes in the DAG.
05948 ///
05949 /// This version can replace From with any result values.  To must match the
05950 /// number and types of values returned by From.
05951 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
05952   if (From->getNumValues() == 1)  // Handle the simple case efficiently.
05953     return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
05954 
05955   // Iterate over just the existing users of From. See the comments in
05956   // the ReplaceAllUsesWith above.
05957   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05958   RAUWUpdateListener Listener(*this, UI, UE);
05959   while (UI != UE) {
05960     SDNode *User = *UI;
05961 
05962     // This node is about to morph, remove its old self from the CSE maps.
05963     RemoveNodeFromCSEMaps(User);
05964 
05965     // A user can appear in a use list multiple times, and when this
05966     // happens the uses are usually next to each other in the list.
05967     // To help reduce the number of CSE recomputations, process all
05968     // the uses of this user that we can find this way.
05969     do {
05970       SDUse &Use = UI.getUse();
05971       const SDValue &ToOp = To[Use.getResNo()];
05972       ++UI;
05973       Use.set(ToOp);
05974     } while (UI != UE && *UI == User);
05975 
05976     // Now that we have modified User, add it back to the CSE maps.  If it
05977     // already exists there, recursively merge the results together.
05978     AddModifiedNodeToCSEMaps(User);
05979   }
05980 
05981   // If we just RAUW'd the root, take note.
05982   if (From == getRoot().getNode())
05983     setRoot(SDValue(To[getRoot().getResNo()]));
05984 }
05985 
05986 /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
05987 /// uses of other values produced by From.getNode() alone.  The Deleted
05988 /// vector is handled the same way as for ReplaceAllUsesWith.
05989 void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
05990   // Handle the really simple, really trivial case efficiently.
05991   if (From == To) return;
05992 
05993   // Handle the simple, trivial, case efficiently.
05994   if (From.getNode()->getNumValues() == 1) {
05995     ReplaceAllUsesWith(From, To);
05996     return;
05997   }
05998 
05999   // Iterate over just the existing users of From. See the comments in
06000   // the ReplaceAllUsesWith above.
06001   SDNode::use_iterator UI = From.getNode()->use_begin(),
06002                        UE = From.getNode()->use_end();
06003   RAUWUpdateListener Listener(*this, UI, UE);
06004   while (UI != UE) {
06005     SDNode *User = *UI;
06006     bool UserRemovedFromCSEMaps = false;
06007 
06008     // A user can appear in a use list multiple times, and when this
06009     // happens the uses are usually next to each other in the list.
06010     // To help reduce the number of CSE recomputations, process all
06011     // the uses of this user that we can find this way.
06012     do {
06013       SDUse &Use = UI.getUse();
06014 
06015       // Skip uses of different values from the same node.
06016       if (Use.getResNo() != From.getResNo()) {
06017         ++UI;
06018         continue;
06019       }
06020 
06021       // If this node hasn't been modified yet, it's still in the CSE maps,
06022       // so remove its old self from the CSE maps.
06023       if (!UserRemovedFromCSEMaps) {
06024         RemoveNodeFromCSEMaps(User);
06025         UserRemovedFromCSEMaps = true;
06026       }
06027 
06028       ++UI;
06029       Use.set(To);
06030     } while (UI != UE && *UI == User);
06031 
06032     // We are iterating over all uses of the From node, so if a use
06033     // doesn't use the specific value, no changes are made.
06034     if (!UserRemovedFromCSEMaps)
06035       continue;
06036 
06037     // Now that we have modified User, add it back to the CSE maps.  If it
06038     // already exists there, recursively merge the results together.
06039     AddModifiedNodeToCSEMaps(User);
06040   }
06041 
06042   // If we just RAUW'd the root, take note.
06043   if (From == getRoot())
06044     setRoot(To);
06045 }
06046 
06047 namespace {
06048   /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
06049   /// to record information about a use.
06050   struct UseMemo {
06051     SDNode *User;
06052     unsigned Index;
06053     SDUse *Use;
06054   };
06055 
06056   /// operator< - Sort Memos by User.
06057   bool operator<(const UseMemo &L, const UseMemo &R) {
06058     return (intptr_t)L.User < (intptr_t)R.User;
06059   }
06060 }
06061 
06062 /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
06063 /// uses of other values produced by From.getNode() alone.  The same value
06064 /// may appear in both the From and To list.  The Deleted vector is
06065 /// handled the same way as for ReplaceAllUsesWith.
06066 void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
06067                                               const SDValue *To,
06068                                               unsigned Num){
06069   // Handle the simple, trivial case efficiently.
06070   if (Num == 1)
06071     return ReplaceAllUsesOfValueWith(*From, *To);
06072 
06073   // Read up all the uses and make records of them. This helps
06074   // processing new uses that are introduced during the
06075   // replacement process.
06076   SmallVector<UseMemo, 4> Uses;
06077   for (unsigned i = 0; i != Num; ++i) {
06078     unsigned FromResNo = From[i].getResNo();
06079     SDNode *FromNode = From[i].getNode();
06080     for (SDNode::use_iterator UI = FromNode->use_begin(),
06081          E = FromNode->use_end(); UI != E; ++UI) {
06082       SDUse &Use = UI.getUse();
06083       if (Use.getResNo() == FromResNo) {
06084         UseMemo Memo = { *UI, i, &Use };
06085         Uses.push_back(Memo);
06086       }
06087     }
06088   }
06089 
06090   // Sort the uses, so that all the uses from a given User are together.
06091   std::sort(Uses.begin(), Uses.end());
06092 
06093   for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
06094        UseIndex != UseIndexEnd; ) {
06095     // We know that this user uses some value of From.  If it is the right
06096     // value, update it.
06097     SDNode *User = Uses[UseIndex].User;
06098 
06099     // This node is about to morph, remove its old self from the CSE maps.
06100     RemoveNodeFromCSEMaps(User);
06101 
06102     // The Uses array is sorted, so all the uses for a given User
06103     // are next to each other in the list.
06104     // To help reduce the number of CSE recomputations, process all
06105     // the uses of this user that we can find this way.
06106     do {
06107       unsigned i = Uses[UseIndex].Index;
06108       SDUse &Use = *Uses[UseIndex].Use;
06109       ++UseIndex;
06110 
06111       Use.set(To[i]);
06112     } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
06113 
06114     // Now that we have modified User, add it back to the CSE maps.  If it
06115     // already exists there, recursively merge the results together.
06116     AddModifiedNodeToCSEMaps(User);
06117   }
06118 }
06119 
06120 /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
06121 /// based on their topological order. It returns the maximum id and a vector
06122 /// of the SDNodes* in assigned order by reference.
06123 unsigned SelectionDAG::AssignTopologicalOrder() {
06124 
06125   unsigned DAGSize = 0;
06126 
06127   // SortedPos tracks the progress of the algorithm. Nodes before it are
06128   // sorted, nodes after it are unsorted. When the algorithm completes
06129   // it is at the end of the list.
06130   allnodes_iterator SortedPos = allnodes_begin();
06131 
06132   // Visit all the nodes. Move nodes with no operands to the front of
06133   // the list immediately. Annotate nodes that do have operands with their
06134   // operand count. Before we do this, the Node Id fields of the nodes
06135   // may contain arbitrary values. After, the Node Id fields for nodes
06136   // before SortedPos will contain the topological sort index, and the
06137   // Node Id fields for nodes At SortedPos and after will contain the
06138   // count of outstanding operands.
06139   for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
06140     SDNode *N = I++;
06141     checkForCycles(N, this);
06142     unsigned Degree = N->getNumOperands();
06143     if (Degree == 0) {
06144       // A node with no uses, add it to the result array immediately.
06145       N->setNodeId(DAGSize++);
06146       allnodes_iterator Q = N;
06147       if (Q != SortedPos)
06148         SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
06149       assert(SortedPos != AllNodes.end() && "Overran node list");
06150       ++SortedPos;
06151     } else {
06152       // Temporarily use the Node Id as scratch space for the degree count.
06153       N->setNodeId(Degree);
06154     }
06155   }
06156 
06157   // Visit all the nodes. As we iterate, move nodes into sorted order,
06158   // such that by the time the end is reached all nodes will be sorted.
06159   for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) {
06160     SDNode *N = I;
06161     checkForCycles(N, this);
06162     // N is in sorted position, so all its uses have one less operand
06163     // that needs to be sorted.
06164     for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
06165          UI != UE; ++UI) {
06166       SDNode *P = *UI;
06167       unsigned Degree = P->getNodeId();
06168       assert(Degree != 0 && "Invalid node degree");
06169       --Degree;
06170       if (Degree == 0) {
06171         // All of P's operands are sorted, so P may sorted now.
06172         P->setNodeId(DAGSize++);
06173         if (P != SortedPos)
06174           SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
06175         assert(SortedPos != AllNodes.end() && "Overran node list");
06176         ++SortedPos;
06177       } else {
06178         // Update P's outstanding operand count.
06179         P->setNodeId(Degree);
06180       }
06181     }
06182     if (I == SortedPos) {
06183 #ifndef NDEBUG
06184       SDNode *S = ++I;
06185       dbgs() << "Overran sorted position:\n";
06186       S->dumprFull(this); dbgs() << "\n";
06187       dbgs() << "Checking if this is due to cycles\n";
06188       checkForCycles(this, true);
06189 #endif
06190       llvm_unreachable(nullptr);
06191     }
06192   }
06193 
06194   assert(SortedPos == AllNodes.end() &&
06195          "Topological sort incomplete!");
06196   assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
06197          "First node in topological sort is not the entry token!");
06198   assert(AllNodes.front().getNodeId() == 0 &&
06199          "First node in topological sort has non-zero id!");
06200   assert(AllNodes.front().getNumOperands() == 0 &&
06201          "First node in topological sort has operands!");
06202   assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
06203          "Last node in topologic sort has unexpected id!");
06204   assert(AllNodes.back().use_empty() &&
06205          "Last node in topologic sort has users!");
06206   assert(DAGSize == allnodes_size() && "Node count mismatch!");
06207   return DAGSize;
06208 }
06209 
06210 /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
06211 /// value is produced by SD.
06212 void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) {
06213   if (SD) {
06214     assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
06215     SD->setHasDebugValue(true);
06216   }
06217   DbgInfo->add(DB, SD, isParameter);
06218 }
06219 
06220 /// TransferDbgValues - Transfer SDDbgValues.
06221 void SelectionDAG::TransferDbgValues(SDValue From, SDValue To) {
06222   if (From == To || !From.getNode()->getHasDebugValue())
06223     return;
06224   SDNode *FromNode = From.getNode();
06225   SDNode *ToNode = To.getNode();
06226   ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode);
06227   SmallVector<SDDbgValue *, 2> ClonedDVs;
06228   for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end();
06229        I != E; ++I) {
06230     SDDbgValue *Dbg = *I;
06231     if (Dbg->getKind() == SDDbgValue::SDNODE) {
06232       SDDbgValue *Clone =
06233           getDbgValue(Dbg->getVariable(), Dbg->getExpression(), ToNode,
06234                       To.getResNo(), Dbg->isIndirect(), Dbg->getOffset(),
06235                       Dbg->getDebugLoc(), Dbg->getOrder());
06236       ClonedDVs.push_back(Clone);
06237     }
06238   }
06239   for (SmallVectorImpl<SDDbgValue *>::iterator I = ClonedDVs.begin(),
06240          E = ClonedDVs.end(); I != E; ++I)
06241     AddDbgValue(*I, ToNode, false);
06242 }
06243 
06244 //===----------------------------------------------------------------------===//
06245 //                              SDNode Class
06246 //===----------------------------------------------------------------------===//
06247 
06248 HandleSDNode::~HandleSDNode() {
06249   DropOperands();
06250 }
06251 
06252 GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
06253                                          DebugLoc DL, const GlobalValue *GA,
06254                                          EVT VT, int64_t o, unsigned char TF)
06255   : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
06256   TheGlobal = GA;
06257 }
06258 
06259 AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT,
06260                                          SDValue X, unsigned SrcAS,
06261                                          unsigned DestAS)
06262  : UnarySDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT), X),
06263    SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
06264 
06265 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
06266                      EVT memvt, MachineMemOperand *mmo)
06267  : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
06268   SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
06269                                       MMO->isNonTemporal(), MMO->isInvariant());
06270   assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
06271   assert(isNonTemporal() == MMO->isNonTemporal() &&
06272          "Non-temporal encoding error!");
06273   // We check here that the size of the memory operand fits within the size of
06274   // the MMO. This is because the MMO might indicate only a possible address
06275   // range instead of specifying the affected memory addresses precisely.
06276   assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
06277 }
06278 
06279 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
06280                      ArrayRef<SDValue> Ops, EVT memvt, MachineMemOperand *mmo)
06281    : SDNode(Opc, Order, dl, VTs, Ops),
06282      MemoryVT(memvt), MMO(mmo) {
06283   SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
06284