LLVM API Documentation

SelectionDAG.cpp
Go to the documentation of this file.
00001 //===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This implements the SelectionDAG class.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "llvm/CodeGen/SelectionDAG.h"
00015 #include "SDNodeDbgValue.h"
00016 #include "llvm/ADT/SetVector.h"
00017 #include "llvm/ADT/SmallPtrSet.h"
00018 #include "llvm/ADT/SmallSet.h"
00019 #include "llvm/ADT/SmallVector.h"
00020 #include "llvm/ADT/StringExtras.h"
00021 #include "llvm/Analysis/ValueTracking.h"
00022 #include "llvm/CodeGen/MachineBasicBlock.h"
00023 #include "llvm/CodeGen/MachineConstantPool.h"
00024 #include "llvm/CodeGen/MachineFrameInfo.h"
00025 #include "llvm/CodeGen/MachineModuleInfo.h"
00026 #include "llvm/IR/CallingConv.h"
00027 #include "llvm/IR/Constants.h"
00028 #include "llvm/IR/DataLayout.h"
00029 #include "llvm/IR/DebugInfo.h"
00030 #include "llvm/IR/DerivedTypes.h"
00031 #include "llvm/IR/Function.h"
00032 #include "llvm/IR/GlobalAlias.h"
00033 #include "llvm/IR/GlobalVariable.h"
00034 #include "llvm/IR/Intrinsics.h"
00035 #include "llvm/Support/CommandLine.h"
00036 #include "llvm/Support/Debug.h"
00037 #include "llvm/Support/ErrorHandling.h"
00038 #include "llvm/Support/ManagedStatic.h"
00039 #include "llvm/Support/MathExtras.h"
00040 #include "llvm/Support/Mutex.h"
00041 #include "llvm/Support/raw_ostream.h"
00042 #include "llvm/Target/TargetInstrInfo.h"
00043 #include "llvm/Target/TargetIntrinsicInfo.h"
00044 #include "llvm/Target/TargetLowering.h"
00045 #include "llvm/Target/TargetMachine.h"
00046 #include "llvm/Target/TargetOptions.h"
00047 #include "llvm/Target/TargetRegisterInfo.h"
00048 #include "llvm/Target/TargetSelectionDAGInfo.h"
00049 #include "llvm/Target/TargetSubtargetInfo.h"
00050 #include <algorithm>
00051 #include <cmath>
00052 
00053 using namespace llvm;
00054 
00055 /// makeVTList - Return an instance of the SDVTList struct initialized with the
00056 /// specified members.
00057 static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
00058   SDVTList Res = {VTs, NumVTs};
00059   return Res;
00060 }
00061 
00062 // Default null implementations of the callbacks.
00063 void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
00064 void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
00065 
00066 //===----------------------------------------------------------------------===//
00067 //                              ConstantFPSDNode Class
00068 //===----------------------------------------------------------------------===//
00069 
00070 /// isExactlyValue - We don't rely on operator== working on double values, as
00071 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
00072 /// As such, this method can be used to do an exact bit-for-bit comparison of
00073 /// two floating point values.
00074 bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
00075   return getValueAPF().bitwiseIsEqual(V);
00076 }
00077 
00078 bool ConstantFPSDNode::isValueValidForType(EVT VT,
00079                                            const APFloat& Val) {
00080   assert(VT.isFloatingPoint() && "Can only convert between FP types");
00081 
00082   // convert modifies in place, so make a copy.
00083   APFloat Val2 = APFloat(Val);
00084   bool losesInfo;
00085   (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
00086                       APFloat::rmNearestTiesToEven,
00087                       &losesInfo);
00088   return !losesInfo;
00089 }
00090 
00091 //===----------------------------------------------------------------------===//
00092 //                              ISD Namespace
00093 //===----------------------------------------------------------------------===//
00094 
00095 /// isBuildVectorAllOnes - Return true if the specified node is a
00096 /// BUILD_VECTOR where all of the elements are ~0 or undef.
00097 bool ISD::isBuildVectorAllOnes(const SDNode *N) {
00098   // Look through a bit convert.
00099   if (N->getOpcode() == ISD::BITCAST)
00100     N = N->getOperand(0).getNode();
00101 
00102   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
00103 
00104   unsigned i = 0, e = N->getNumOperands();
00105 
00106   // Skip over all of the undef values.
00107   while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF)
00108     ++i;
00109 
00110   // Do not accept an all-undef vector.
00111   if (i == e) return false;
00112 
00113   // Do not accept build_vectors that aren't all constants or which have non-~0
00114   // elements. We have to be a bit careful here, as the type of the constant
00115   // may not be the same as the type of the vector elements due to type
00116   // legalization (the elements are promoted to a legal type for the target and
00117   // a vector of a type may be legal when the base element type is not).
00118   // We only want to check enough bits to cover the vector elements, because
00119   // we care if the resultant vector is all ones, not whether the individual
00120   // constants are.
00121   SDValue NotZero = N->getOperand(i);
00122   unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
00123   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
00124     if (CN->getAPIntValue().countTrailingOnes() < EltSize)
00125       return false;
00126   } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
00127     if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
00128       return false;
00129   } else
00130     return false;
00131 
00132   // Okay, we have at least one ~0 value, check to see if the rest match or are
00133   // undefs. Even with the above element type twiddling, this should be OK, as
00134   // the same type legalization should have applied to all the elements.
00135   for (++i; i != e; ++i)
00136     if (N->getOperand(i) != NotZero &&
00137         N->getOperand(i).getOpcode() != ISD::UNDEF)
00138       return false;
00139   return true;
00140 }
00141 
00142 
00143 /// isBuildVectorAllZeros - Return true if the specified node is a
00144 /// BUILD_VECTOR where all of the elements are 0 or undef.
00145 bool ISD::isBuildVectorAllZeros(const SDNode *N) {
00146   // Look through a bit convert.
00147   if (N->getOpcode() == ISD::BITCAST)
00148     N = N->getOperand(0).getNode();
00149 
00150   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
00151 
00152   bool IsAllUndef = true;
00153   for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) {
00154     if (N->getOperand(i).getOpcode() == ISD::UNDEF)
00155       continue;
00156     IsAllUndef = false;
00157     // Do not accept build_vectors that aren't all constants or which have non-0
00158     // elements. We have to be a bit careful here, as the type of the constant
00159     // may not be the same as the type of the vector elements due to type
00160     // legalization (the elements are promoted to a legal type for the target
00161     // and a vector of a type may be legal when the base element type is not).
00162     // We only want to check enough bits to cover the vector elements, because
00163     // we care if the resultant vector is all zeros, not whether the individual
00164     // constants are.
00165     SDValue Zero = N->getOperand(i);
00166     unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
00167     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Zero)) {
00168       if (CN->getAPIntValue().countTrailingZeros() < EltSize)
00169         return false;
00170     } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Zero)) {
00171       if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
00172         return false;
00173     } else
00174       return false;
00175   }
00176 
00177   // Do not accept an all-undef vector.
00178   if (IsAllUndef)
00179     return false;
00180   return true;
00181 }
00182 
00183 /// \brief Return true if the specified node is a BUILD_VECTOR node of
00184 /// all ConstantSDNode or undef.
00185 bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
00186   if (N->getOpcode() != ISD::BUILD_VECTOR)
00187     return false;
00188 
00189   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
00190     SDValue Op = N->getOperand(i);
00191     if (Op.getOpcode() == ISD::UNDEF)
00192       continue;
00193     if (!isa<ConstantSDNode>(Op))
00194       return false;
00195   }
00196   return true;
00197 }
00198 
00199 /// isScalarToVector - Return true if the specified node is a
00200 /// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low
00201 /// element is not an undef.
00202 bool ISD::isScalarToVector(const SDNode *N) {
00203   if (N->getOpcode() == ISD::SCALAR_TO_VECTOR)
00204     return true;
00205 
00206   if (N->getOpcode() != ISD::BUILD_VECTOR)
00207     return false;
00208   if (N->getOperand(0).getOpcode() == ISD::UNDEF)
00209     return false;
00210   unsigned NumElems = N->getNumOperands();
00211   if (NumElems == 1)
00212     return false;
00213   for (unsigned i = 1; i < NumElems; ++i) {
00214     SDValue V = N->getOperand(i);
00215     if (V.getOpcode() != ISD::UNDEF)
00216       return false;
00217   }
00218   return true;
00219 }
00220 
00221 /// allOperandsUndef - Return true if the node has at least one operand
00222 /// and all operands of the specified node are ISD::UNDEF.
00223 bool ISD::allOperandsUndef(const SDNode *N) {
00224   // Return false if the node has no operands.
00225   // This is "logically inconsistent" with the definition of "all" but
00226   // is probably the desired behavior.
00227   if (N->getNumOperands() == 0)
00228     return false;
00229 
00230   for (unsigned i = 0, e = N->getNumOperands(); i != e ; ++i)
00231     if (N->getOperand(i).getOpcode() != ISD::UNDEF)
00232       return false;
00233 
00234   return true;
00235 }
00236 
00237 ISD::NodeType ISD::getExtForLoadExtType(ISD::LoadExtType ExtType) {
00238   switch (ExtType) {
00239   case ISD::EXTLOAD:
00240     return ISD::ANY_EXTEND;
00241   case ISD::SEXTLOAD:
00242     return ISD::SIGN_EXTEND;
00243   case ISD::ZEXTLOAD:
00244     return ISD::ZERO_EXTEND;
00245   default:
00246     break;
00247   }
00248 
00249   llvm_unreachable("Invalid LoadExtType");
00250 }
00251 
00252 /// getSetCCSwappedOperands - Return the operation corresponding to (Y op X)
00253 /// when given the operation for (X op Y).
00254 ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
00255   // To perform this operation, we just need to swap the L and G bits of the
00256   // operation.
00257   unsigned OldL = (Operation >> 2) & 1;
00258   unsigned OldG = (Operation >> 1) & 1;
00259   return ISD::CondCode((Operation & ~6) |  // Keep the N, U, E bits
00260                        (OldL << 1) |       // New G bit
00261                        (OldG << 2));       // New L bit.
00262 }
00263 
00264 /// getSetCCInverse - Return the operation corresponding to !(X op Y), where
00265 /// 'op' is a valid SetCC operation.
00266 ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) {
00267   unsigned Operation = Op;
00268   if (isInteger)
00269     Operation ^= 7;   // Flip L, G, E bits, but not U.
00270   else
00271     Operation ^= 15;  // Flip all of the condition bits.
00272 
00273   if (Operation > ISD::SETTRUE2)
00274     Operation &= ~8;  // Don't let N and U bits get set.
00275 
00276   return ISD::CondCode(Operation);
00277 }
00278 
00279 
00280 /// isSignedOp - For an integer comparison, return 1 if the comparison is a
00281 /// signed operation and 2 if the result is an unsigned comparison.  Return zero
00282 /// if the operation does not depend on the sign of the input (setne and seteq).
00283 static int isSignedOp(ISD::CondCode Opcode) {
00284   switch (Opcode) {
00285   default: llvm_unreachable("Illegal integer setcc operation!");
00286   case ISD::SETEQ:
00287   case ISD::SETNE: return 0;
00288   case ISD::SETLT:
00289   case ISD::SETLE:
00290   case ISD::SETGT:
00291   case ISD::SETGE: return 1;
00292   case ISD::SETULT:
00293   case ISD::SETULE:
00294   case ISD::SETUGT:
00295   case ISD::SETUGE: return 2;
00296   }
00297 }
00298 
00299 /// getSetCCOrOperation - Return the result of a logical OR between different
00300 /// comparisons of identical values: ((X op1 Y) | (X op2 Y)).  This function
00301 /// returns SETCC_INVALID if it is not possible to represent the resultant
00302 /// comparison.
00303 ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
00304                                        bool isInteger) {
00305   if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
00306     // Cannot fold a signed integer setcc with an unsigned integer setcc.
00307     return ISD::SETCC_INVALID;
00308 
00309   unsigned Op = Op1 | Op2;  // Combine all of the condition bits.
00310 
00311   // If the N and U bits get set then the resultant comparison DOES suddenly
00312   // care about orderedness, and is true when ordered.
00313   if (Op > ISD::SETTRUE2)
00314     Op &= ~16;     // Clear the U bit if the N bit is set.
00315 
00316   // Canonicalize illegal integer setcc's.
00317   if (isInteger && Op == ISD::SETUNE)  // e.g. SETUGT | SETULT
00318     Op = ISD::SETNE;
00319 
00320   return ISD::CondCode(Op);
00321 }
00322 
00323 /// getSetCCAndOperation - Return the result of a logical AND between different
00324 /// comparisons of identical values: ((X op1 Y) & (X op2 Y)).  This
00325 /// function returns zero if it is not possible to represent the resultant
00326 /// comparison.
00327 ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
00328                                         bool isInteger) {
00329   if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
00330     // Cannot fold a signed setcc with an unsigned setcc.
00331     return ISD::SETCC_INVALID;
00332 
00333   // Combine all of the condition bits.
00334   ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
00335 
00336   // Canonicalize illegal integer setcc's.
00337   if (isInteger) {
00338     switch (Result) {
00339     default: break;
00340     case ISD::SETUO : Result = ISD::SETFALSE; break;  // SETUGT & SETULT
00341     case ISD::SETOEQ:                                 // SETEQ  & SETU[LG]E
00342     case ISD::SETUEQ: Result = ISD::SETEQ   ; break;  // SETUGE & SETULE
00343     case ISD::SETOLT: Result = ISD::SETULT  ; break;  // SETULT & SETNE
00344     case ISD::SETOGT: Result = ISD::SETUGT  ; break;  // SETUGT & SETNE
00345     }
00346   }
00347 
00348   return Result;
00349 }
00350 
00351 //===----------------------------------------------------------------------===//
00352 //                           SDNode Profile Support
00353 //===----------------------------------------------------------------------===//
00354 
00355 /// AddNodeIDOpcode - Add the node opcode to the NodeID data.
00356 ///
00357 static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC)  {
00358   ID.AddInteger(OpC);
00359 }
00360 
00361 /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
00362 /// solely with their pointer.
00363 static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
00364   ID.AddPointer(VTList.VTs);
00365 }
00366 
00367 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
00368 ///
00369 static void AddNodeIDOperands(FoldingSetNodeID &ID,
00370                               ArrayRef<SDValue> Ops) {
00371   for (auto& Op : Ops) {
00372     ID.AddPointer(Op.getNode());
00373     ID.AddInteger(Op.getResNo());
00374   }
00375 }
00376 
00377 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
00378 ///
00379 static void AddNodeIDOperands(FoldingSetNodeID &ID,
00380                               ArrayRef<SDUse> Ops) {
00381   for (auto& Op : Ops) {
00382     ID.AddPointer(Op.getNode());
00383     ID.AddInteger(Op.getResNo());
00384   }
00385 }
00386 
00387 static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, bool nuw, bool nsw,
00388                                   bool exact) {
00389   ID.AddBoolean(nuw);
00390   ID.AddBoolean(nsw);
00391   ID.AddBoolean(exact);
00392 }
00393 
00394 /// AddBinaryNodeIDCustom - Add BinarySDNodes special infos
00395 static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, unsigned Opcode,
00396                                   bool nuw, bool nsw, bool exact) {
00397   if (isBinOpWithFlags(Opcode))
00398     AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
00399 }
00400 
00401 static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
00402                           SDVTList VTList, ArrayRef<SDValue> OpList) {
00403   AddNodeIDOpcode(ID, OpC);
00404   AddNodeIDValueTypes(ID, VTList);
00405   AddNodeIDOperands(ID, OpList);
00406 }
00407 
00408 /// AddNodeIDCustom - If this is an SDNode with special info, add this info to
00409 /// the NodeID data.
00410 static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
00411   switch (N->getOpcode()) {
00412   case ISD::TargetExternalSymbol:
00413   case ISD::ExternalSymbol:
00414     llvm_unreachable("Should only be used on nodes with operands");
00415   default: break;  // Normal nodes don't need extra info.
00416   case ISD::TargetConstant:
00417   case ISD::Constant: {
00418     const ConstantSDNode *C = cast<ConstantSDNode>(N);
00419     ID.AddPointer(C->getConstantIntValue());
00420     ID.AddBoolean(C->isOpaque());
00421     break;
00422   }
00423   case ISD::TargetConstantFP:
00424   case ISD::ConstantFP: {
00425     ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
00426     break;
00427   }
00428   case ISD::TargetGlobalAddress:
00429   case ISD::GlobalAddress:
00430   case ISD::TargetGlobalTLSAddress:
00431   case ISD::GlobalTLSAddress: {
00432     const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
00433     ID.AddPointer(GA->getGlobal());
00434     ID.AddInteger(GA->getOffset());
00435     ID.AddInteger(GA->getTargetFlags());
00436     ID.AddInteger(GA->getAddressSpace());
00437     break;
00438   }
00439   case ISD::BasicBlock:
00440     ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
00441     break;
00442   case ISD::Register:
00443     ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
00444     break;
00445   case ISD::RegisterMask:
00446     ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
00447     break;
00448   case ISD::SRCVALUE:
00449     ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
00450     break;
00451   case ISD::FrameIndex:
00452   case ISD::TargetFrameIndex:
00453     ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
00454     break;
00455   case ISD::JumpTable:
00456   case ISD::TargetJumpTable:
00457     ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
00458     ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
00459     break;
00460   case ISD::ConstantPool:
00461   case ISD::TargetConstantPool: {
00462     const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
00463     ID.AddInteger(CP->getAlignment());
00464     ID.AddInteger(CP->getOffset());
00465     if (CP->isMachineConstantPoolEntry())
00466       CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
00467     else
00468       ID.AddPointer(CP->getConstVal());
00469     ID.AddInteger(CP->getTargetFlags());
00470     break;
00471   }
00472   case ISD::TargetIndex: {
00473     const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
00474     ID.AddInteger(TI->getIndex());
00475     ID.AddInteger(TI->getOffset());
00476     ID.AddInteger(TI->getTargetFlags());
00477     break;
00478   }
00479   case ISD::LOAD: {
00480     const LoadSDNode *LD = cast<LoadSDNode>(N);
00481     ID.AddInteger(LD->getMemoryVT().getRawBits());
00482     ID.AddInteger(LD->getRawSubclassData());
00483     ID.AddInteger(LD->getPointerInfo().getAddrSpace());
00484     break;
00485   }
00486   case ISD::STORE: {
00487     const StoreSDNode *ST = cast<StoreSDNode>(N);
00488     ID.AddInteger(ST->getMemoryVT().getRawBits());
00489     ID.AddInteger(ST->getRawSubclassData());
00490     ID.AddInteger(ST->getPointerInfo().getAddrSpace());
00491     break;
00492   }
00493   case ISD::SDIV:
00494   case ISD::UDIV:
00495   case ISD::SRA:
00496   case ISD::SRL:
00497   case ISD::MUL:
00498   case ISD::ADD:
00499   case ISD::SUB:
00500   case ISD::SHL: {
00501     const BinaryWithFlagsSDNode *BinNode = cast<BinaryWithFlagsSDNode>(N);
00502     AddBinaryNodeIDCustom(ID, N->getOpcode(), BinNode->hasNoUnsignedWrap(),
00503                           BinNode->hasNoSignedWrap(), BinNode->isExact());
00504     break;
00505   }
00506   case ISD::ATOMIC_CMP_SWAP:
00507   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
00508   case ISD::ATOMIC_SWAP:
00509   case ISD::ATOMIC_LOAD_ADD:
00510   case ISD::ATOMIC_LOAD_SUB:
00511   case ISD::ATOMIC_LOAD_AND:
00512   case ISD::ATOMIC_LOAD_OR:
00513   case ISD::ATOMIC_LOAD_XOR:
00514   case ISD::ATOMIC_LOAD_NAND:
00515   case ISD::ATOMIC_LOAD_MIN:
00516   case ISD::ATOMIC_LOAD_MAX:
00517   case ISD::ATOMIC_LOAD_UMIN:
00518   case ISD::ATOMIC_LOAD_UMAX:
00519   case ISD::ATOMIC_LOAD:
00520   case ISD::ATOMIC_STORE: {
00521     const AtomicSDNode *AT = cast<AtomicSDNode>(N);
00522     ID.AddInteger(AT->getMemoryVT().getRawBits());
00523     ID.AddInteger(AT->getRawSubclassData());
00524     ID.AddInteger(AT->getPointerInfo().getAddrSpace());
00525     break;
00526   }
00527   case ISD::PREFETCH: {
00528     const MemSDNode *PF = cast<MemSDNode>(N);
00529     ID.AddInteger(PF->getPointerInfo().getAddrSpace());
00530     break;
00531   }
00532   case ISD::VECTOR_SHUFFLE: {
00533     const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
00534     for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
00535          i != e; ++i)
00536       ID.AddInteger(SVN->getMaskElt(i));
00537     break;
00538   }
00539   case ISD::TargetBlockAddress:
00540   case ISD::BlockAddress: {
00541     const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
00542     ID.AddPointer(BA->getBlockAddress());
00543     ID.AddInteger(BA->getOffset());
00544     ID.AddInteger(BA->getTargetFlags());
00545     break;
00546   }
00547   } // end switch (N->getOpcode())
00548 
00549   // Target specific memory nodes could also have address spaces to check.
00550   if (N->isTargetMemoryOpcode())
00551     ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
00552 }
00553 
00554 /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
00555 /// data.
00556 static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
00557   AddNodeIDOpcode(ID, N->getOpcode());
00558   // Add the return value info.
00559   AddNodeIDValueTypes(ID, N->getVTList());
00560   // Add the operand info.
00561   AddNodeIDOperands(ID, N->ops());
00562 
00563   // Handle SDNode leafs with special info.
00564   AddNodeIDCustom(ID, N);
00565 }
00566 
00567 /// encodeMemSDNodeFlags - Generic routine for computing a value for use in
00568 /// the CSE map that carries volatility, temporalness, indexing mode, and
00569 /// extension/truncation information.
00570 ///
00571 static inline unsigned
00572 encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile,
00573                      bool isNonTemporal, bool isInvariant) {
00574   assert((ConvType & 3) == ConvType &&
00575          "ConvType may not require more than 2 bits!");
00576   assert((AM & 7) == AM &&
00577          "AM may not require more than 3 bits!");
00578   return ConvType |
00579          (AM << 2) |
00580          (isVolatile << 5) |
00581          (isNonTemporal << 6) |
00582          (isInvariant << 7);
00583 }
00584 
00585 //===----------------------------------------------------------------------===//
00586 //                              SelectionDAG Class
00587 //===----------------------------------------------------------------------===//
00588 
00589 /// doNotCSE - Return true if CSE should not be performed for this node.
00590 static bool doNotCSE(SDNode *N) {
00591   if (N->getValueType(0) == MVT::Glue)
00592     return true; // Never CSE anything that produces a flag.
00593 
00594   switch (N->getOpcode()) {
00595   default: break;
00596   case ISD::HANDLENODE:
00597   case ISD::EH_LABEL:
00598     return true;   // Never CSE these nodes.
00599   }
00600 
00601   // Check that remaining values produced are not flags.
00602   for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
00603     if (N->getValueType(i) == MVT::Glue)
00604       return true; // Never CSE anything that produces a flag.
00605 
00606   return false;
00607 }
00608 
00609 /// RemoveDeadNodes - This method deletes all unreachable nodes in the
00610 /// SelectionDAG.
00611 void SelectionDAG::RemoveDeadNodes() {
00612   // Create a dummy node (which is not added to allnodes), that adds a reference
00613   // to the root node, preventing it from being deleted.
00614   HandleSDNode Dummy(getRoot());
00615 
00616   SmallVector<SDNode*, 128> DeadNodes;
00617 
00618   // Add all obviously-dead nodes to the DeadNodes worklist.
00619   for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I)
00620     if (I->use_empty())
00621       DeadNodes.push_back(I);
00622 
00623   RemoveDeadNodes(DeadNodes);
00624 
00625   // If the root changed (e.g. it was a dead load, update the root).
00626   setRoot(Dummy.getValue());
00627 }
00628 
00629 /// RemoveDeadNodes - This method deletes the unreachable nodes in the
00630 /// given list, and any nodes that become unreachable as a result.
00631 void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
00632 
00633   // Process the worklist, deleting the nodes and adding their uses to the
00634   // worklist.
00635   while (!DeadNodes.empty()) {
00636     SDNode *N = DeadNodes.pop_back_val();
00637 
00638     for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00639       DUL->NodeDeleted(N, nullptr);
00640 
00641     // Take the node out of the appropriate CSE map.
00642     RemoveNodeFromCSEMaps(N);
00643 
00644     // Next, brutally remove the operand list.  This is safe to do, as there are
00645     // no cycles in the graph.
00646     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
00647       SDUse &Use = *I++;
00648       SDNode *Operand = Use.getNode();
00649       Use.set(SDValue());
00650 
00651       // Now that we removed this operand, see if there are no uses of it left.
00652       if (Operand->use_empty())
00653         DeadNodes.push_back(Operand);
00654     }
00655 
00656     DeallocateNode(N);
00657   }
00658 }
00659 
00660 void SelectionDAG::RemoveDeadNode(SDNode *N){
00661   SmallVector<SDNode*, 16> DeadNodes(1, N);
00662 
00663   // Create a dummy node that adds a reference to the root node, preventing
00664   // it from being deleted.  (This matters if the root is an operand of the
00665   // dead node.)
00666   HandleSDNode Dummy(getRoot());
00667 
00668   RemoveDeadNodes(DeadNodes);
00669 }
00670 
00671 void SelectionDAG::DeleteNode(SDNode *N) {
00672   // First take this out of the appropriate CSE map.
00673   RemoveNodeFromCSEMaps(N);
00674 
00675   // Finally, remove uses due to operands of this node, remove from the
00676   // AllNodes list, and delete the node.
00677   DeleteNodeNotInCSEMaps(N);
00678 }
00679 
00680 void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
00681   assert(N != AllNodes.begin() && "Cannot delete the entry node!");
00682   assert(N->use_empty() && "Cannot delete a node that is not dead!");
00683 
00684   // Drop all of the operands and decrement used node's use counts.
00685   N->DropOperands();
00686 
00687   DeallocateNode(N);
00688 }
00689 
00690 void SelectionDAG::DeallocateNode(SDNode *N) {
00691   if (N->OperandsNeedDelete)
00692     delete[] N->OperandList;
00693 
00694   // Set the opcode to DELETED_NODE to help catch bugs when node
00695   // memory is reallocated.
00696   N->NodeType = ISD::DELETED_NODE;
00697 
00698   NodeAllocator.Deallocate(AllNodes.remove(N));
00699 
00700   // If any of the SDDbgValue nodes refer to this SDNode, invalidate them.
00701   ArrayRef<SDDbgValue*> DbgVals = DbgInfo->getSDDbgValues(N);
00702   for (unsigned i = 0, e = DbgVals.size(); i != e; ++i)
00703     DbgVals[i]->setIsInvalidated();
00704 }
00705 
00706 #ifndef NDEBUG
00707 /// VerifySDNode - Sanity check the given SDNode.  Aborts if it is invalid.
00708 static void VerifySDNode(SDNode *N) {
00709   switch (N->getOpcode()) {
00710   default:
00711     break;
00712   case ISD::BUILD_PAIR: {
00713     EVT VT = N->getValueType(0);
00714     assert(N->getNumValues() == 1 && "Too many results!");
00715     assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
00716            "Wrong return type!");
00717     assert(N->getNumOperands() == 2 && "Wrong number of operands!");
00718     assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
00719            "Mismatched operand types!");
00720     assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
00721            "Wrong operand type!");
00722     assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
00723            "Wrong return type size");
00724     break;
00725   }
00726   case ISD::BUILD_VECTOR: {
00727     assert(N->getNumValues() == 1 && "Too many results!");
00728     assert(N->getValueType(0).isVector() && "Wrong return type!");
00729     assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
00730            "Wrong number of operands!");
00731     EVT EltVT = N->getValueType(0).getVectorElementType();
00732     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
00733       assert((I->getValueType() == EltVT ||
00734              (EltVT.isInteger() && I->getValueType().isInteger() &&
00735               EltVT.bitsLE(I->getValueType()))) &&
00736             "Wrong operand type!");
00737       assert(I->getValueType() == N->getOperand(0).getValueType() &&
00738              "Operands must all have the same type");
00739     }
00740     break;
00741   }
00742   }
00743 }
00744 #endif // NDEBUG
00745 
00746 /// \brief Insert a newly allocated node into the DAG.
00747 ///
00748 /// Handles insertion into the all nodes list and CSE map, as well as
00749 /// verification and other common operations when a new node is allocated.
00750 void SelectionDAG::InsertNode(SDNode *N) {
00751   AllNodes.push_back(N);
00752 #ifndef NDEBUG
00753   VerifySDNode(N);
00754 #endif
00755 }
00756 
00757 /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
00758 /// correspond to it.  This is useful when we're about to delete or repurpose
00759 /// the node.  We don't want future request for structurally identical nodes
00760 /// to return N anymore.
00761 bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
00762   bool Erased = false;
00763   switch (N->getOpcode()) {
00764   case ISD::HANDLENODE: return false;  // noop.
00765   case ISD::CONDCODE:
00766     assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
00767            "Cond code doesn't exist!");
00768     Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
00769     CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
00770     break;
00771   case ISD::ExternalSymbol:
00772     Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
00773     break;
00774   case ISD::TargetExternalSymbol: {
00775     ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
00776     Erased = TargetExternalSymbols.erase(
00777                std::pair<std::string,unsigned char>(ESN->getSymbol(),
00778                                                     ESN->getTargetFlags()));
00779     break;
00780   }
00781   case ISD::VALUETYPE: {
00782     EVT VT = cast<VTSDNode>(N)->getVT();
00783     if (VT.isExtended()) {
00784       Erased = ExtendedValueTypeNodes.erase(VT);
00785     } else {
00786       Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
00787       ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
00788     }
00789     break;
00790   }
00791   default:
00792     // Remove it from the CSE Map.
00793     assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
00794     assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
00795     Erased = CSEMap.RemoveNode(N);
00796     break;
00797   }
00798 #ifndef NDEBUG
00799   // Verify that the node was actually in one of the CSE maps, unless it has a
00800   // flag result (which cannot be CSE'd) or is one of the special cases that are
00801   // not subject to CSE.
00802   if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
00803       !N->isMachineOpcode() && !doNotCSE(N)) {
00804     N->dump(this);
00805     dbgs() << "\n";
00806     llvm_unreachable("Node is not in map!");
00807   }
00808 #endif
00809   return Erased;
00810 }
00811 
00812 /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
00813 /// maps and modified in place. Add it back to the CSE maps, unless an identical
00814 /// node already exists, in which case transfer all its users to the existing
00815 /// node. This transfer can potentially trigger recursive merging.
00816 ///
00817 void
00818 SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
00819   // For node types that aren't CSE'd, just act as if no identical node
00820   // already exists.
00821   if (!doNotCSE(N)) {
00822     SDNode *Existing = CSEMap.GetOrInsertNode(N);
00823     if (Existing != N) {
00824       // If there was already an existing matching node, use ReplaceAllUsesWith
00825       // to replace the dead one with the existing one.  This can cause
00826       // recursive merging of other unrelated nodes down the line.
00827       ReplaceAllUsesWith(N, Existing);
00828 
00829       // N is now dead. Inform the listeners and delete it.
00830       for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00831         DUL->NodeDeleted(N, Existing);
00832       DeleteNodeNotInCSEMaps(N);
00833       return;
00834     }
00835   }
00836 
00837   // If the node doesn't already exist, we updated it.  Inform listeners.
00838   for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00839     DUL->NodeUpdated(N);
00840 }
00841 
00842 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00843 /// were replaced with those specified.  If this node is never memoized,
00844 /// return null, otherwise return a pointer to the slot it would take.  If a
00845 /// node already exists with these operands, the slot will be non-null.
00846 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
00847                                            void *&InsertPos) {
00848   if (doNotCSE(N))
00849     return nullptr;
00850 
00851   SDValue Ops[] = { Op };
00852   FoldingSetNodeID ID;
00853   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00854   AddNodeIDCustom(ID, N);
00855   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00856   return Node;
00857 }
00858 
00859 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00860 /// were replaced with those specified.  If this node is never memoized,
00861 /// return null, otherwise return a pointer to the slot it would take.  If a
00862 /// node already exists with these operands, the slot will be non-null.
00863 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
00864                                            SDValue Op1, SDValue Op2,
00865                                            void *&InsertPos) {
00866   if (doNotCSE(N))
00867     return nullptr;
00868 
00869   SDValue Ops[] = { Op1, Op2 };
00870   FoldingSetNodeID ID;
00871   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00872   AddNodeIDCustom(ID, N);
00873   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00874   return Node;
00875 }
00876 
00877 
00878 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00879 /// were replaced with those specified.  If this node is never memoized,
00880 /// return null, otherwise return a pointer to the slot it would take.  If a
00881 /// node already exists with these operands, the slot will be non-null.
00882 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
00883                                            void *&InsertPos) {
00884   if (doNotCSE(N))
00885     return nullptr;
00886 
00887   FoldingSetNodeID ID;
00888   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00889   AddNodeIDCustom(ID, N);
00890   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00891   return Node;
00892 }
00893 
00894 /// getEVTAlignment - Compute the default alignment value for the
00895 /// given type.
00896 ///
00897 unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
00898   Type *Ty = VT == MVT::iPTR ?
00899                    PointerType::get(Type::getInt8Ty(*getContext()), 0) :
00900                    VT.getTypeForEVT(*getContext());
00901 
00902   return TM.getSubtargetImpl()
00903       ->getTargetLowering()
00904       ->getDataLayout()
00905       ->getABITypeAlignment(Ty);
00906 }
00907 
00908 // EntryNode could meaningfully have debug info if we can find it...
00909 SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
00910     : TM(tm), TSI(tm.getSubtargetImpl()->getSelectionDAGInfo()), TLI(nullptr),
00911       OptLevel(OL),
00912       EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
00913       Root(getEntryNode()), NewNodesMustHaveLegalTypes(false),
00914       UpdateListeners(nullptr) {
00915   AllNodes.push_back(&EntryNode);
00916   DbgInfo = new SDDbgInfo();
00917 }
00918 
00919 void SelectionDAG::init(MachineFunction &mf, const TargetLowering *tli) {
00920   MF = &mf;
00921   TLI = tli;
00922   Context = &mf.getFunction()->getContext();
00923 }
00924 
00925 SelectionDAG::~SelectionDAG() {
00926   assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
00927   allnodes_clear();
00928   delete DbgInfo;
00929 }
00930 
00931 void SelectionDAG::allnodes_clear() {
00932   assert(&*AllNodes.begin() == &EntryNode);
00933   AllNodes.remove(AllNodes.begin());
00934   while (!AllNodes.empty())
00935     DeallocateNode(AllNodes.begin());
00936 }
00937 
00938 BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL,
00939                                             SDVTList VTs, SDValue N1,
00940                                             SDValue N2, bool nuw, bool nsw,
00941                                             bool exact) {
00942   if (isBinOpWithFlags(Opcode)) {
00943     BinaryWithFlagsSDNode *FN = new (NodeAllocator) BinaryWithFlagsSDNode(
00944         Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
00945     FN->setHasNoUnsignedWrap(nuw);
00946     FN->setHasNoSignedWrap(nsw);
00947     FN->setIsExact(exact);
00948 
00949     return FN;
00950   }
00951 
00952   BinarySDNode *N = new (NodeAllocator)
00953       BinarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
00954   return N;
00955 }
00956 
00957 void SelectionDAG::clear() {
00958   allnodes_clear();
00959   OperandAllocator.Reset();
00960   CSEMap.clear();
00961 
00962   ExtendedValueTypeNodes.clear();
00963   ExternalSymbols.clear();
00964   TargetExternalSymbols.clear();
00965   std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
00966             static_cast<CondCodeSDNode*>(nullptr));
00967   std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
00968             static_cast<SDNode*>(nullptr));
00969 
00970   EntryNode.UseList = nullptr;
00971   AllNodes.push_back(&EntryNode);
00972   Root = getEntryNode();
00973   DbgInfo->clear();
00974 }
00975 
00976 SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00977   return VT.bitsGT(Op.getValueType()) ?
00978     getNode(ISD::ANY_EXTEND, DL, VT, Op) :
00979     getNode(ISD::TRUNCATE, DL, VT, Op);
00980 }
00981 
00982 SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00983   return VT.bitsGT(Op.getValueType()) ?
00984     getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
00985     getNode(ISD::TRUNCATE, DL, VT, Op);
00986 }
00987 
00988 SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00989   return VT.bitsGT(Op.getValueType()) ?
00990     getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
00991     getNode(ISD::TRUNCATE, DL, VT, Op);
00992 }
00993 
00994 SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, SDLoc SL, EVT VT,
00995                                         EVT OpVT) {
00996   if (VT.bitsLE(Op.getValueType()))
00997     return getNode(ISD::TRUNCATE, SL, VT, Op);
00998 
00999   TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
01000   return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
01001 }
01002 
01003 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, SDLoc DL, EVT VT) {
01004   assert(!VT.isVector() &&
01005          "getZeroExtendInReg should use the vector element type instead of "
01006          "the vector type!");
01007   if (Op.getValueType() == VT) return Op;
01008   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01009   APInt Imm = APInt::getLowBitsSet(BitWidth,
01010                                    VT.getSizeInBits());
01011   return getNode(ISD::AND, DL, Op.getValueType(), Op,
01012                  getConstant(Imm, Op.getValueType()));
01013 }
01014 
01015 SDValue SelectionDAG::getAnyExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01016   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01017   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01018          "The sizes of the input and result must match in order to perform the "
01019          "extend in-register.");
01020   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01021          "The destination vector type must have fewer lanes than the input.");
01022   return getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Op);
01023 }
01024 
01025 SDValue SelectionDAG::getSignExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01026   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01027   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01028          "The sizes of the input and result must match in order to perform the "
01029          "extend in-register.");
01030   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01031          "The destination vector type must have fewer lanes than the input.");
01032   return getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Op);
01033 }
01034 
01035 SDValue SelectionDAG::getZeroExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01036   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01037   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01038          "The sizes of the input and result must match in order to perform the "
01039          "extend in-register.");
01040   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01041          "The destination vector type must have fewer lanes than the input.");
01042   return getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Op);
01043 }
01044 
01045 /// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
01046 ///
01047 SDValue SelectionDAG::getNOT(SDLoc DL, SDValue Val, EVT VT) {
01048   EVT EltVT = VT.getScalarType();
01049   SDValue NegOne =
01050     getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT);
01051   return getNode(ISD::XOR, DL, VT, Val, NegOne);
01052 }
01053 
01054 SDValue SelectionDAG::getLogicalNOT(SDLoc DL, SDValue Val, EVT VT) {
01055   EVT EltVT = VT.getScalarType();
01056   SDValue TrueValue;
01057   switch (TLI->getBooleanContents(VT)) {
01058     case TargetLowering::ZeroOrOneBooleanContent:
01059     case TargetLowering::UndefinedBooleanContent:
01060       TrueValue = getConstant(1, VT);
01061       break;
01062     case TargetLowering::ZeroOrNegativeOneBooleanContent:
01063       TrueValue = getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()),
01064                               VT);
01065       break;
01066   }
01067   return getNode(ISD::XOR, DL, VT, Val, TrueValue);
01068 }
01069 
01070 SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT, bool isO) {
01071   EVT EltVT = VT.getScalarType();
01072   assert((EltVT.getSizeInBits() >= 64 ||
01073          (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
01074          "getConstant with a uint64_t value that doesn't fit in the type!");
01075   return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT, isO);
01076 }
01077 
01078 SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT, bool isO)
01079 {
01080   return getConstant(*ConstantInt::get(*Context, Val), VT, isT, isO);
01081 }
01082 
01083 SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT,
01084                                   bool isO) {
01085   assert(VT.isInteger() && "Cannot create FP integer constant!");
01086 
01087   EVT EltVT = VT.getScalarType();
01088   const ConstantInt *Elt = &Val;
01089 
01090   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01091 
01092   // In some cases the vector type is legal but the element type is illegal and
01093   // needs to be promoted, for example v8i8 on ARM.  In this case, promote the
01094   // inserted value (the type does not need to match the vector element type).
01095   // Any extra bits introduced will be truncated away.
01096   if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
01097       TargetLowering::TypePromoteInteger) {
01098    EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
01099    APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits());
01100    Elt = ConstantInt::get(*getContext(), NewVal);
01101   }
01102   // In other cases the element type is illegal and needs to be expanded, for
01103   // example v2i64 on MIPS32. In this case, find the nearest legal type, split
01104   // the value into n parts and use a vector type with n-times the elements.
01105   // Then bitcast to the type requested.
01106   // Legalizing constants too early makes the DAGCombiner's job harder so we
01107   // only legalize if the DAG tells us we must produce legal types.
01108   else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
01109            TLI->getTypeAction(*getContext(), EltVT) ==
01110            TargetLowering::TypeExpandInteger) {
01111     APInt NewVal = Elt->getValue();
01112     EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
01113     unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
01114     unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
01115     EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
01116 
01117     // Check the temporary vector is the correct size. If this fails then
01118     // getTypeToTransformTo() probably returned a type whose size (in bits)
01119     // isn't a power-of-2 factor of the requested type size.
01120     assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
01121 
01122     SmallVector<SDValue, 2> EltParts;
01123     for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
01124       EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
01125                                            .trunc(ViaEltSizeInBits),
01126                                      ViaEltVT, isT, isO));
01127     }
01128 
01129     // EltParts is currently in little endian order. If we actually want
01130     // big-endian order then reverse it now.
01131     if (TLI->isBigEndian())
01132       std::reverse(EltParts.begin(), EltParts.end());
01133 
01134     // The elements must be reversed when the element order is different
01135     // to the endianness of the elements (because the BITCAST is itself a
01136     // vector shuffle in this situation). However, we do not need any code to
01137     // perform this reversal because getConstant() is producing a vector
01138     // splat.
01139     // This situation occurs in MIPS MSA.
01140 
01141     SmallVector<SDValue, 8> Ops;
01142     for (unsigned i = 0; i < VT.getVectorNumElements(); ++i)
01143       Ops.insert(Ops.end(), EltParts.begin(), EltParts.end());
01144 
01145     SDValue Result = getNode(ISD::BITCAST, SDLoc(), VT,
01146                              getNode(ISD::BUILD_VECTOR, SDLoc(), ViaVecVT,
01147                                      Ops));
01148     return Result;
01149   }
01150 
01151   assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
01152          "APInt size does not match type size!");
01153   unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
01154   FoldingSetNodeID ID;
01155   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
01156   ID.AddPointer(Elt);
01157   ID.AddBoolean(isO);
01158   void *IP = nullptr;
01159   SDNode *N = nullptr;
01160   if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
01161     if (!VT.isVector())
01162       return SDValue(N, 0);
01163 
01164   if (!N) {
01165     N = new (NodeAllocator) ConstantSDNode(isT, isO, Elt, EltVT);
01166     CSEMap.InsertNode(N, IP);
01167     InsertNode(N);
01168   }
01169 
01170   SDValue Result(N, 0);
01171   if (VT.isVector()) {
01172     SmallVector<SDValue, 8> Ops;
01173     Ops.assign(VT.getVectorNumElements(), Result);
01174     Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
01175   }
01176   return Result;
01177 }
01178 
01179 SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) {
01180   return getConstant(Val,
01181                      TM.getSubtargetImpl()->getTargetLowering()->getPointerTy(),
01182                      isTarget);
01183 }
01184 
01185 
01186 SDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) {
01187   return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget);
01188 }
01189 
01190 SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){
01191   assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
01192 
01193   EVT EltVT = VT.getScalarType();
01194 
01195   // Do the map lookup using the actual bit pattern for the floating point
01196   // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
01197   // we don't have issues with SNANs.
01198   unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
01199   FoldingSetNodeID ID;
01200   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
01201   ID.AddPointer(&V);
01202   void *IP = nullptr;
01203   SDNode *N = nullptr;
01204   if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
01205     if (!VT.isVector())
01206       return SDValue(N, 0);
01207 
01208   if (!N) {
01209     N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT);
01210     CSEMap.InsertNode(N, IP);
01211     InsertNode(N);
01212   }
01213 
01214   SDValue Result(N, 0);
01215   if (VT.isVector()) {
01216     SmallVector<SDValue, 8> Ops;
01217     Ops.assign(VT.getVectorNumElements(), Result);
01218     // FIXME SDLoc info might be appropriate here
01219     Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
01220   }
01221   return Result;
01222 }
01223 
01224 SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) {
01225   EVT EltVT = VT.getScalarType();
01226   if (EltVT==MVT::f32)
01227     return getConstantFP(APFloat((float)Val), VT, isTarget);
01228   else if (EltVT==MVT::f64)
01229     return getConstantFP(APFloat(Val), VT, isTarget);
01230   else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 ||
01231            EltVT==MVT::f16) {
01232     bool ignored;
01233     APFloat apf = APFloat(Val);
01234     apf.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
01235                 &ignored);
01236     return getConstantFP(apf, VT, isTarget);
01237   } else
01238     llvm_unreachable("Unsupported type in getConstantFP");
01239 }
01240 
01241 SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, SDLoc DL,
01242                                        EVT VT, int64_t Offset,
01243                                        bool isTargetGA,
01244                                        unsigned char TargetFlags) {
01245   assert((TargetFlags == 0 || isTargetGA) &&
01246          "Cannot set target flags on target-independent globals");
01247   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01248 
01249   // Truncate (with sign-extension) the offset value to the pointer size.
01250   unsigned BitWidth = TLI->getPointerTypeSizeInBits(GV->getType());
01251   if (BitWidth < 64)
01252     Offset = SignExtend64(Offset, BitWidth);
01253 
01254   unsigned Opc;
01255   if (GV->isThreadLocal())
01256     Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
01257   else
01258     Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
01259 
01260   FoldingSetNodeID ID;
01261   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01262   ID.AddPointer(GV);
01263   ID.AddInteger(Offset);
01264   ID.AddInteger(TargetFlags);
01265   ID.AddInteger(GV->getType()->getAddressSpace());
01266   void *IP = nullptr;
01267   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01268     return SDValue(E, 0);
01269 
01270   SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL.getIROrder(),
01271                                                       DL.getDebugLoc(), GV, VT,
01272                                                       Offset, TargetFlags);
01273   CSEMap.InsertNode(N, IP);
01274     InsertNode(N);
01275   return SDValue(N, 0);
01276 }
01277 
01278 SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
01279   unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
01280   FoldingSetNodeID ID;
01281   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01282   ID.AddInteger(FI);
01283   void *IP = nullptr;
01284   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01285     return SDValue(E, 0);
01286 
01287   SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget);
01288   CSEMap.InsertNode(N, IP);
01289   InsertNode(N);
01290   return SDValue(N, 0);
01291 }
01292 
01293 SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
01294                                    unsigned char TargetFlags) {
01295   assert((TargetFlags == 0 || isTarget) &&
01296          "Cannot set target flags on target-independent jump tables");
01297   unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
01298   FoldingSetNodeID ID;
01299   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01300   ID.AddInteger(JTI);
01301   ID.AddInteger(TargetFlags);
01302   void *IP = nullptr;
01303   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01304     return SDValue(E, 0);
01305 
01306   SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget,
01307                                                   TargetFlags);
01308   CSEMap.InsertNode(N, IP);
01309   InsertNode(N);
01310   return SDValue(N, 0);
01311 }
01312 
01313 SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
01314                                       unsigned Alignment, int Offset,
01315                                       bool isTarget,
01316                                       unsigned char TargetFlags) {
01317   assert((TargetFlags == 0 || isTarget) &&
01318          "Cannot set target flags on target-independent globals");
01319   if (Alignment == 0)
01320     Alignment = TM.getSubtargetImpl()
01321                     ->getTargetLowering()
01322                     ->getDataLayout()
01323                     ->getPrefTypeAlignment(C->getType());
01324   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
01325   FoldingSetNodeID ID;
01326   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01327   ID.AddInteger(Alignment);
01328   ID.AddInteger(Offset);
01329   ID.AddPointer(C);
01330   ID.AddInteger(TargetFlags);
01331   void *IP = nullptr;
01332   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01333     return SDValue(E, 0);
01334 
01335   SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
01336                                                      Alignment, TargetFlags);
01337   CSEMap.InsertNode(N, IP);
01338   InsertNode(N);
01339   return SDValue(N, 0);
01340 }
01341 
01342 
01343 SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
01344                                       unsigned Alignment, int Offset,
01345                                       bool isTarget,
01346                                       unsigned char TargetFlags) {
01347   assert((TargetFlags == 0 || isTarget) &&
01348          "Cannot set target flags on target-independent globals");
01349   if (Alignment == 0)
01350     Alignment = TM.getSubtargetImpl()
01351                     ->getTargetLowering()
01352                     ->getDataLayout()
01353                     ->getPrefTypeAlignment(C->getType());
01354   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
01355   FoldingSetNodeID ID;
01356   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01357   ID.AddInteger(Alignment);
01358   ID.AddInteger(Offset);
01359   C->addSelectionDAGCSEId(ID);
01360   ID.AddInteger(TargetFlags);
01361   void *IP = nullptr;
01362   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01363     return SDValue(E, 0);
01364 
01365   SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
01366                                                      Alignment, TargetFlags);
01367   CSEMap.InsertNode(N, IP);
01368   InsertNode(N);
01369   return SDValue(N, 0);
01370 }
01371 
01372 SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
01373                                      unsigned char TargetFlags) {
01374   FoldingSetNodeID ID;
01375   AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None);
01376   ID.AddInteger(Index);
01377   ID.AddInteger(Offset);
01378   ID.AddInteger(TargetFlags);
01379   void *IP = nullptr;
01380   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01381     return SDValue(E, 0);
01382 
01383   SDNode *N = new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset,
01384                                                     TargetFlags);
01385   CSEMap.InsertNode(N, IP);
01386   InsertNode(N);
01387   return SDValue(N, 0);
01388 }
01389 
01390 SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
01391   FoldingSetNodeID ID;
01392   AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
01393   ID.AddPointer(MBB);
01394   void *IP = nullptr;
01395   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01396     return SDValue(E, 0);
01397 
01398   SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB);
01399   CSEMap.InsertNode(N, IP);
01400   InsertNode(N);
01401   return SDValue(N, 0);
01402 }
01403 
01404 SDValue SelectionDAG::getValueType(EVT VT) {
01405   if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
01406       ValueTypeNodes.size())
01407     ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
01408 
01409   SDNode *&N = VT.isExtended() ?
01410     ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
01411 
01412   if (N) return SDValue(N, 0);
01413   N = new (NodeAllocator) VTSDNode(VT);
01414   InsertNode(N);
01415   return SDValue(N, 0);
01416 }
01417 
01418 SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
01419   SDNode *&N = ExternalSymbols[Sym];
01420   if (N) return SDValue(N, 0);
01421   N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT);
01422   InsertNode(N);
01423   return SDValue(N, 0);
01424 }
01425 
01426 SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
01427                                               unsigned char TargetFlags) {
01428   SDNode *&N =
01429     TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym,
01430                                                                TargetFlags)];
01431   if (N) return SDValue(N, 0);
01432   N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT);
01433   InsertNode(N);
01434   return SDValue(N, 0);
01435 }
01436 
01437 SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
01438   if ((unsigned)Cond >= CondCodeNodes.size())
01439     CondCodeNodes.resize(Cond+1);
01440 
01441   if (!CondCodeNodes[Cond]) {
01442     CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond);
01443     CondCodeNodes[Cond] = N;
01444     InsertNode(N);
01445   }
01446 
01447   return SDValue(CondCodeNodes[Cond], 0);
01448 }
01449 
01450 // commuteShuffle - swaps the values of N1 and N2, and swaps all indices in
01451 // the shuffle mask M that point at N1 to point at N2, and indices that point
01452 // N2 to point at N1.
01453 static void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) {
01454   std::swap(N1, N2);
01455   int NElts = M.size();
01456   for (int i = 0; i != NElts; ++i) {
01457     if (M[i] >= NElts)
01458       M[i] -= NElts;
01459     else if (M[i] >= 0)
01460       M[i] += NElts;
01461   }
01462 }
01463 
01464 SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1,
01465                                        SDValue N2, const int *Mask) {
01466   assert(VT == N1.getValueType() && VT == N2.getValueType() &&
01467          "Invalid VECTOR_SHUFFLE");
01468 
01469   // Canonicalize shuffle undef, undef -> undef
01470   if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF)
01471     return getUNDEF(VT);
01472 
01473   // Validate that all indices in Mask are within the range of the elements
01474   // input to the shuffle.
01475   unsigned NElts = VT.getVectorNumElements();
01476   SmallVector<int, 8> MaskVec;
01477   for (unsigned i = 0; i != NElts; ++i) {
01478     assert(Mask[i] < (int)(NElts * 2) && "Index out of range");
01479     MaskVec.push_back(Mask[i]);
01480   }
01481 
01482   // Canonicalize shuffle v, v -> v, undef
01483   if (N1 == N2) {
01484     N2 = getUNDEF(VT);
01485     for (unsigned i = 0; i != NElts; ++i)
01486       if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts;
01487   }
01488 
01489   // Canonicalize shuffle undef, v -> v, undef.  Commute the shuffle mask.
01490   if (N1.getOpcode() == ISD::UNDEF)
01491     commuteShuffle(N1, N2, MaskVec);
01492 
01493   // Canonicalize all index into lhs, -> shuffle lhs, undef
01494   // Canonicalize all index into rhs, -> shuffle rhs, undef
01495   bool AllLHS = true, AllRHS = true;
01496   bool N2Undef = N2.getOpcode() == ISD::UNDEF;
01497   for (unsigned i = 0; i != NElts; ++i) {
01498     if (MaskVec[i] >= (int)NElts) {
01499       if (N2Undef)
01500         MaskVec[i] = -1;
01501       else
01502         AllLHS = false;
01503     } else if (MaskVec[i] >= 0) {
01504       AllRHS = false;
01505     }
01506   }
01507   if (AllLHS && AllRHS)
01508     return getUNDEF(VT);
01509   if (AllLHS && !N2Undef)
01510     N2 = getUNDEF(VT);
01511   if (AllRHS) {
01512     N1 = getUNDEF(VT);
01513     commuteShuffle(N1, N2, MaskVec);
01514   }
01515   // Reset our undef status after accounting for the mask.
01516   N2Undef = N2.getOpcode() == ISD::UNDEF;
01517   // Re-check whether both sides ended up undef.
01518   if (N1.getOpcode() == ISD::UNDEF && N2Undef)
01519     return getUNDEF(VT);
01520 
01521   // If Identity shuffle return that node.
01522   bool Identity = true;
01523   for (unsigned i = 0; i != NElts; ++i) {
01524     if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false;
01525   }
01526   if (Identity && NElts)
01527     return N1;
01528 
01529   // Shuffling a constant splat doesn't change the result.
01530   if (N2Undef) {
01531     SDValue V = N1;
01532 
01533     // Look through any bitcasts. We check that these don't change the number
01534     // (and size) of elements and just changes their types.
01535     while (V.getOpcode() == ISD::BITCAST)
01536       V = V->getOperand(0);
01537 
01538     // A splat should always show up as a build vector node.
01539     if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
01540       BitVector UndefElements;
01541       SDValue Splat = BV->getSplatValue(&UndefElements);
01542       // If this is a splat of an undef, shuffling it is also undef.
01543       if (Splat && Splat.getOpcode() == ISD::UNDEF)
01544         return getUNDEF(VT);
01545 
01546       // We only have a splat which can skip shuffles if there is a splatted
01547       // value and no undef lanes rearranged by the shuffle.
01548       if (Splat && UndefElements.none()) {
01549         // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
01550         // number of elements match or the value splatted is a zero constant.
01551         if (V.getValueType().getVectorNumElements() ==
01552             VT.getVectorNumElements())
01553           return N1;
01554         if (auto *C = dyn_cast<ConstantSDNode>(Splat))
01555           if (C->isNullValue())
01556             return N1;
01557       }
01558     }
01559   }
01560 
01561   FoldingSetNodeID ID;
01562   SDValue Ops[2] = { N1, N2 };
01563   AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
01564   for (unsigned i = 0; i != NElts; ++i)
01565     ID.AddInteger(MaskVec[i]);
01566 
01567   void* IP = nullptr;
01568   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01569     return SDValue(E, 0);
01570 
01571   // Allocate the mask array for the node out of the BumpPtrAllocator, since
01572   // SDNode doesn't have access to it.  This memory will be "leaked" when
01573   // the node is deallocated, but recovered when the NodeAllocator is released.
01574   int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
01575   memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int));
01576 
01577   ShuffleVectorSDNode *N =
01578     new (NodeAllocator) ShuffleVectorSDNode(VT, dl.getIROrder(),
01579                                             dl.getDebugLoc(), N1, N2,
01580                                             MaskAlloc);
01581   CSEMap.InsertNode(N, IP);
01582   InsertNode(N);
01583   return SDValue(N, 0);
01584 }
01585 
01586 SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
01587   MVT VT = SV.getSimpleValueType(0);
01588   unsigned NumElems = VT.getVectorNumElements();
01589   SmallVector<int, 8> MaskVec;
01590 
01591   for (unsigned i = 0; i != NumElems; ++i) {
01592     int Idx = SV.getMaskElt(i);
01593     if (Idx >= 0) {
01594       if (Idx < (int)NumElems)
01595         Idx += NumElems;
01596       else
01597         Idx -= NumElems;
01598     }
01599     MaskVec.push_back(Idx);
01600   }
01601 
01602   SDValue Op0 = SV.getOperand(0);
01603   SDValue Op1 = SV.getOperand(1);
01604   return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, &MaskVec[0]);
01605 }
01606 
01607 SDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl,
01608                                        SDValue Val, SDValue DTy,
01609                                        SDValue STy, SDValue Rnd, SDValue Sat,
01610                                        ISD::CvtCode Code) {
01611   // If the src and dest types are the same and the conversion is between
01612   // integer types of the same sign or two floats, no conversion is necessary.
01613   if (DTy == STy &&
01614       (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF))
01615     return Val;
01616 
01617   FoldingSetNodeID ID;
01618   SDValue Ops[] = { Val, DTy, STy, Rnd, Sat };
01619   AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), Ops);
01620   void* IP = nullptr;
01621   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01622     return SDValue(E, 0);
01623 
01624   CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl.getIROrder(),
01625                                                            dl.getDebugLoc(),
01626                                                            Ops, Code);
01627   CSEMap.InsertNode(N, IP);
01628   InsertNode(N);
01629   return SDValue(N, 0);
01630 }
01631 
01632 SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
01633   FoldingSetNodeID ID;
01634   AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
01635   ID.AddInteger(RegNo);
01636   void *IP = nullptr;
01637   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01638     return SDValue(E, 0);
01639 
01640   SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT);
01641   CSEMap.InsertNode(N, IP);
01642   InsertNode(N);
01643   return SDValue(N, 0);
01644 }
01645 
01646 SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
01647   FoldingSetNodeID ID;
01648   AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
01649   ID.AddPointer(RegMask);
01650   void *IP = nullptr;
01651   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01652     return SDValue(E, 0);
01653 
01654   SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask);
01655   CSEMap.InsertNode(N, IP);
01656   InsertNode(N);
01657   return SDValue(N, 0);
01658 }
01659 
01660 SDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) {
01661   FoldingSetNodeID ID;
01662   SDValue Ops[] = { Root };
01663   AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), Ops);
01664   ID.AddPointer(Label);
01665   void *IP = nullptr;
01666   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01667     return SDValue(E, 0);
01668 
01669   SDNode *N = new (NodeAllocator) EHLabelSDNode(dl.getIROrder(),
01670                                                 dl.getDebugLoc(), Root, Label);
01671   CSEMap.InsertNode(N, IP);
01672   InsertNode(N);
01673   return SDValue(N, 0);
01674 }
01675 
01676 
01677 SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
01678                                       int64_t Offset,
01679                                       bool isTarget,
01680                                       unsigned char TargetFlags) {
01681   unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
01682 
01683   FoldingSetNodeID ID;
01684   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01685   ID.AddPointer(BA);
01686   ID.AddInteger(Offset);
01687   ID.AddInteger(TargetFlags);
01688   void *IP = nullptr;
01689   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01690     return SDValue(E, 0);
01691 
01692   SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset,
01693                                                      TargetFlags);
01694   CSEMap.InsertNode(N, IP);
01695   InsertNode(N);
01696   return SDValue(N, 0);
01697 }
01698 
01699 SDValue SelectionDAG::getSrcValue(const Value *V) {
01700   assert((!V || V->getType()->isPointerTy()) &&
01701          "SrcValue is not a pointer?");
01702 
01703   FoldingSetNodeID ID;
01704   AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None);
01705   ID.AddPointer(V);
01706 
01707   void *IP = nullptr;
01708   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01709     return SDValue(E, 0);
01710 
01711   SDNode *N = new (NodeAllocator) SrcValueSDNode(V);
01712   CSEMap.InsertNode(N, IP);
01713   InsertNode(N);
01714   return SDValue(N, 0);
01715 }
01716 
01717 /// getMDNode - Return an MDNodeSDNode which holds an MDNode.
01718 SDValue SelectionDAG::getMDNode(const MDNode *MD) {
01719   FoldingSetNodeID ID;
01720   AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None);
01721   ID.AddPointer(MD);
01722 
01723   void *IP = nullptr;
01724   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01725     return SDValue(E, 0);
01726 
01727   SDNode *N = new (NodeAllocator) MDNodeSDNode(MD);
01728   CSEMap.InsertNode(N, IP);
01729   InsertNode(N);
01730   return SDValue(N, 0);
01731 }
01732 
01733 /// getAddrSpaceCast - Return an AddrSpaceCastSDNode.
01734 SDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr,
01735                                        unsigned SrcAS, unsigned DestAS) {
01736   SDValue Ops[] = {Ptr};
01737   FoldingSetNodeID ID;
01738   AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
01739   ID.AddInteger(SrcAS);
01740   ID.AddInteger(DestAS);
01741 
01742   void *IP = nullptr;
01743   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01744     return SDValue(E, 0);
01745 
01746   SDNode *N = new (NodeAllocator) AddrSpaceCastSDNode(dl.getIROrder(),
01747                                                       dl.getDebugLoc(),
01748                                                       VT, Ptr, SrcAS, DestAS);
01749   CSEMap.InsertNode(N, IP);
01750   InsertNode(N);
01751   return SDValue(N, 0);
01752 }
01753 
01754 /// getShiftAmountOperand - Return the specified value casted to
01755 /// the target's desired shift amount type.
01756 SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
01757   EVT OpTy = Op.getValueType();
01758   EVT ShTy =
01759       TM.getSubtargetImpl()->getTargetLowering()->getShiftAmountTy(LHSTy);
01760   if (OpTy == ShTy || OpTy.isVector()) return Op;
01761 
01762   ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ?  ISD::TRUNCATE : ISD::ZERO_EXTEND;
01763   return getNode(Opcode, SDLoc(Op), ShTy, Op);
01764 }
01765 
01766 /// CreateStackTemporary - Create a stack temporary, suitable for holding the
01767 /// specified value type.
01768 SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
01769   MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
01770   unsigned ByteSize = VT.getStoreSize();
01771   Type *Ty = VT.getTypeForEVT(*getContext());
01772   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01773   unsigned StackAlign =
01774   std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty), minAlign);
01775 
01776   int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false);
01777   return getFrameIndex(FrameIdx, TLI->getPointerTy());
01778 }
01779 
01780 /// CreateStackTemporary - Create a stack temporary suitable for holding
01781 /// either of the specified value types.
01782 SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
01783   unsigned Bytes = std::max(VT1.getStoreSizeInBits(),
01784                             VT2.getStoreSizeInBits())/8;
01785   Type *Ty1 = VT1.getTypeForEVT(*getContext());
01786   Type *Ty2 = VT2.getTypeForEVT(*getContext());
01787   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01788   const DataLayout *TD = TLI->getDataLayout();
01789   unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1),
01790                             TD->getPrefTypeAlignment(Ty2));
01791 
01792   MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
01793   int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false);
01794   return getFrameIndex(FrameIdx, TLI->getPointerTy());
01795 }
01796 
01797 SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1,
01798                                 SDValue N2, ISD::CondCode Cond, SDLoc dl) {
01799   // These setcc operations always fold.
01800   switch (Cond) {
01801   default: break;
01802   case ISD::SETFALSE:
01803   case ISD::SETFALSE2: return getConstant(0, VT);
01804   case ISD::SETTRUE:
01805   case ISD::SETTRUE2: {
01806     const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01807     TargetLowering::BooleanContent Cnt =
01808         TLI->getBooleanContents(N1->getValueType(0));
01809     return getConstant(
01810         Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, VT);
01811   }
01812 
01813   case ISD::SETOEQ:
01814   case ISD::SETOGT:
01815   case ISD::SETOGE:
01816   case ISD::SETOLT:
01817   case ISD::SETOLE:
01818   case ISD::SETONE:
01819   case ISD::SETO:
01820   case ISD::SETUO:
01821   case ISD::SETUEQ:
01822   case ISD::SETUNE:
01823     assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!");
01824     break;
01825   }
01826 
01827   if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) {
01828     const APInt &C2 = N2C->getAPIntValue();
01829     if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
01830       const APInt &C1 = N1C->getAPIntValue();
01831 
01832       switch (Cond) {
01833       default: llvm_unreachable("Unknown integer setcc!");
01834       case ISD::SETEQ:  return getConstant(C1 == C2, VT);
01835       case ISD::SETNE:  return getConstant(C1 != C2, VT);
01836       case ISD::SETULT: return getConstant(C1.ult(C2), VT);
01837       case ISD::SETUGT: return getConstant(C1.ugt(C2), VT);
01838       case ISD::SETULE: return getConstant(C1.ule(C2), VT);
01839       case ISD::SETUGE: return getConstant(C1.uge(C2), VT);
01840       case ISD::SETLT:  return getConstant(C1.slt(C2), VT);
01841       case ISD::SETGT:  return getConstant(C1.sgt(C2), VT);
01842       case ISD::SETLE:  return getConstant(C1.sle(C2), VT);
01843       case ISD::SETGE:  return getConstant(C1.sge(C2), VT);
01844       }
01845     }
01846   }
01847   if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) {
01848     if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) {
01849       APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF());
01850       switch (Cond) {
01851       default: break;
01852       case ISD::SETEQ:  if (R==APFloat::cmpUnordered)
01853                           return getUNDEF(VT);
01854                         // fall through
01855       case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT);
01856       case ISD::SETNE:  if (R==APFloat::cmpUnordered)
01857                           return getUNDEF(VT);
01858                         // fall through
01859       case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan ||
01860                                            R==APFloat::cmpLessThan, VT);
01861       case ISD::SETLT:  if (R==APFloat::cmpUnordered)
01862                           return getUNDEF(VT);
01863                         // fall through
01864       case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT);
01865       case ISD::SETGT:  if (R==APFloat::cmpUnordered)
01866                           return getUNDEF(VT);
01867                         // fall through
01868       case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT);
01869       case ISD::SETLE:  if (R==APFloat::cmpUnordered)
01870                           return getUNDEF(VT);
01871                         // fall through
01872       case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan ||
01873                                            R==APFloat::cmpEqual, VT);
01874       case ISD::SETGE:  if (R==APFloat::cmpUnordered)
01875                           return getUNDEF(VT);
01876                         // fall through
01877       case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan ||
01878                                            R==APFloat::cmpEqual, VT);
01879       case ISD::SETO:   return getConstant(R!=APFloat::cmpUnordered, VT);
01880       case ISD::SETUO:  return getConstant(R==APFloat::cmpUnordered, VT);
01881       case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered ||
01882                                            R==APFloat::cmpEqual, VT);
01883       case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT);
01884       case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered ||
01885                                            R==APFloat::cmpLessThan, VT);
01886       case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan ||
01887                                            R==APFloat::cmpUnordered, VT);
01888       case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT);
01889       case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT);
01890       }
01891     } else {
01892       // Ensure that the constant occurs on the RHS.
01893       ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
01894       MVT CompVT = N1.getValueType().getSimpleVT();
01895       if (!TM.getSubtargetImpl()->getTargetLowering()->isCondCodeLegal(
01896               SwappedCond, CompVT))
01897         return SDValue();
01898 
01899       return getSetCC(dl, VT, N2, N1, SwappedCond);
01900     }
01901   }
01902 
01903   // Could not fold it.
01904   return SDValue();
01905 }
01906 
01907 /// SignBitIsZero - Return true if the sign bit of Op is known to be zero.  We
01908 /// use this predicate to simplify operations downstream.
01909 bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
01910   // This predicate is not safe for vector operations.
01911   if (Op.getValueType().isVector())
01912     return false;
01913 
01914   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01915   return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth);
01916 }
01917 
01918 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero.  We use
01919 /// this predicate to simplify operations downstream.  Mask is known to be zero
01920 /// for bits that V cannot have.
01921 bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask,
01922                                      unsigned Depth) const {
01923   APInt KnownZero, KnownOne;
01924   computeKnownBits(Op, KnownZero, KnownOne, Depth);
01925   return (KnownZero & Mask) == Mask;
01926 }
01927 
01928 /// Determine which bits of Op are known to be either zero or one and return
01929 /// them in the KnownZero/KnownOne bitsets.
01930 void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
01931                                     APInt &KnownOne, unsigned Depth) const {
01932   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
01933   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01934 
01935   KnownZero = KnownOne = APInt(BitWidth, 0);   // Don't know anything.
01936   if (Depth == 6)
01937     return;  // Limit search depth.
01938 
01939   APInt KnownZero2, KnownOne2;
01940 
01941   switch (Op.getOpcode()) {
01942   case ISD::Constant:
01943     // We know all of the bits for a constant!
01944     KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue();
01945     KnownZero = ~KnownOne;
01946     break;
01947   case ISD::AND:
01948     // If either the LHS or the RHS are Zero, the result is zero.
01949     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01950     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01951 
01952     // Output known-1 bits are only known if set in both the LHS & RHS.
01953     KnownOne &= KnownOne2;
01954     // Output known-0 are known to be clear if zero in either the LHS | RHS.
01955     KnownZero |= KnownZero2;
01956     break;
01957   case ISD::OR:
01958     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01959     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01960 
01961     // Output known-0 bits are only known if clear in both the LHS & RHS.
01962     KnownZero &= KnownZero2;
01963     // Output known-1 are known to be set if set in either the LHS | RHS.
01964     KnownOne |= KnownOne2;
01965     break;
01966   case ISD::XOR: {
01967     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01968     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01969 
01970     // Output known-0 bits are known if clear or set in both the LHS & RHS.
01971     APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2);
01972     // Output known-1 are known to be set if set in only one of the LHS, RHS.
01973     KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2);
01974     KnownZero = KnownZeroOut;
01975     break;
01976   }
01977   case ISD::MUL: {
01978     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01979     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01980 
01981     // If low bits are zero in either operand, output low known-0 bits.
01982     // Also compute a conserative estimate for high known-0 bits.
01983     // More trickiness is possible, but this is sufficient for the
01984     // interesting case of alignment computation.
01985     KnownOne.clearAllBits();
01986     unsigned TrailZ = KnownZero.countTrailingOnes() +
01987                       KnownZero2.countTrailingOnes();
01988     unsigned LeadZ =  std::max(KnownZero.countLeadingOnes() +
01989                                KnownZero2.countLeadingOnes(),
01990                                BitWidth) - BitWidth;
01991 
01992     TrailZ = std::min(TrailZ, BitWidth);
01993     LeadZ = std::min(LeadZ, BitWidth);
01994     KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) |
01995                 APInt::getHighBitsSet(BitWidth, LeadZ);
01996     break;
01997   }
01998   case ISD::UDIV: {
01999     // For the purposes of computing leading zeros we can conservatively
02000     // treat a udiv as a logical right shift by the power of 2 known to
02001     // be less than the denominator.
02002     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
02003     unsigned LeadZ = KnownZero2.countLeadingOnes();
02004 
02005     KnownOne2.clearAllBits();
02006     KnownZero2.clearAllBits();
02007     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02008     unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros();
02009     if (RHSUnknownLeadingOnes != BitWidth)
02010       LeadZ = std::min(BitWidth,
02011                        LeadZ + BitWidth - RHSUnknownLeadingOnes - 1);
02012 
02013     KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ);
02014     break;
02015   }
02016   case ISD::SELECT:
02017     computeKnownBits(Op.getOperand(2), KnownZero, KnownOne, Depth+1);
02018     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02019 
02020     // Only known if known in both the LHS and RHS.
02021     KnownOne &= KnownOne2;
02022     KnownZero &= KnownZero2;
02023     break;
02024   case ISD::SELECT_CC:
02025     computeKnownBits(Op.getOperand(3), KnownZero, KnownOne, Depth+1);
02026     computeKnownBits(Op.getOperand(2), KnownZero2, KnownOne2, Depth+1);
02027 
02028     // Only known if known in both the LHS and RHS.
02029     KnownOne &= KnownOne2;
02030     KnownZero &= KnownZero2;
02031     break;
02032   case ISD::SADDO:
02033   case ISD::UADDO:
02034   case ISD::SSUBO:
02035   case ISD::USUBO:
02036   case ISD::SMULO:
02037   case ISD::UMULO:
02038     if (Op.getResNo() != 1)
02039       break;
02040     // The boolean result conforms to getBooleanContents.
02041     // If we know the result of a setcc has the top bits zero, use this info.
02042     // We know that we have an integer-based boolean since these operations
02043     // are only available for integer.
02044     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
02045             TargetLowering::ZeroOrOneBooleanContent &&
02046         BitWidth > 1)
02047       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02048     break;
02049   case ISD::SETCC:
02050     // If we know the result of a setcc has the top bits zero, use this info.
02051     if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
02052             TargetLowering::ZeroOrOneBooleanContent &&
02053         BitWidth > 1)
02054       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02055     break;
02056   case ISD::SHL:
02057     // (shl X, C1) & C2 == 0   iff   (X & C2 >>u C1) == 0
02058     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02059       unsigned ShAmt = SA->getZExtValue();
02060 
02061       // If the shift count is an invalid immediate, don't do anything.
02062       if (ShAmt >= BitWidth)
02063         break;
02064 
02065       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02066       KnownZero <<= ShAmt;
02067       KnownOne  <<= ShAmt;
02068       // low bits known zero.
02069       KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt);
02070     }
02071     break;
02072   case ISD::SRL:
02073     // (ushr X, C1) & C2 == 0   iff  (-1 >> C1) & C2 == 0
02074     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02075       unsigned ShAmt = SA->getZExtValue();
02076 
02077       // If the shift count is an invalid immediate, don't do anything.
02078       if (ShAmt >= BitWidth)
02079         break;
02080 
02081       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02082       KnownZero = KnownZero.lshr(ShAmt);
02083       KnownOne  = KnownOne.lshr(ShAmt);
02084 
02085       APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
02086       KnownZero |= HighBits;  // High bits known zero.
02087     }
02088     break;
02089   case ISD::SRA:
02090     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02091       unsigned ShAmt = SA->getZExtValue();
02092 
02093       // If the shift count is an invalid immediate, don't do anything.
02094       if (ShAmt >= BitWidth)
02095         break;
02096 
02097       // If any of the demanded bits are produced by the sign extension, we also
02098       // demand the input sign bit.
02099       APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
02100 
02101       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02102       KnownZero = KnownZero.lshr(ShAmt);
02103       KnownOne  = KnownOne.lshr(ShAmt);
02104 
02105       // Handle the sign bits.
02106       APInt SignBit = APInt::getSignBit(BitWidth);
02107       SignBit = SignBit.lshr(ShAmt);  // Adjust to where it is now in the mask.
02108 
02109       if (KnownZero.intersects(SignBit)) {
02110         KnownZero |= HighBits;  // New bits are known zero.
02111       } else if (KnownOne.intersects(SignBit)) {
02112         KnownOne  |= HighBits;  // New bits are known one.
02113       }
02114     }
02115     break;
02116   case ISD::SIGN_EXTEND_INREG: {
02117     EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
02118     unsigned EBits = EVT.getScalarType().getSizeInBits();
02119 
02120     // Sign extension.  Compute the demanded bits in the result that are not
02121     // present in the input.
02122     APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits);
02123 
02124     APInt InSignBit = APInt::getSignBit(EBits);
02125     APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits);
02126 
02127     // If the sign extended bits are demanded, we know that the sign
02128     // bit is demanded.
02129     InSignBit = InSignBit.zext(BitWidth);
02130     if (NewBits.getBoolValue())
02131       InputDemandedBits |= InSignBit;
02132 
02133     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02134     KnownOne &= InputDemandedBits;
02135     KnownZero &= InputDemandedBits;
02136 
02137     // If the sign bit of the input is known set or clear, then we know the
02138     // top bits of the result.
02139     if (KnownZero.intersects(InSignBit)) {         // Input sign bit known clear
02140       KnownZero |= NewBits;
02141       KnownOne  &= ~NewBits;
02142     } else if (KnownOne.intersects(InSignBit)) {   // Input sign bit known set
02143       KnownOne  |= NewBits;
02144       KnownZero &= ~NewBits;
02145     } else {                              // Input sign bit unknown
02146       KnownZero &= ~NewBits;
02147       KnownOne  &= ~NewBits;
02148     }
02149     break;
02150   }
02151   case ISD::CTTZ:
02152   case ISD::CTTZ_ZERO_UNDEF:
02153   case ISD::CTLZ:
02154   case ISD::CTLZ_ZERO_UNDEF:
02155   case ISD::CTPOP: {
02156     unsigned LowBits = Log2_32(BitWidth)+1;
02157     KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits);
02158     KnownOne.clearAllBits();
02159     break;
02160   }
02161   case ISD::LOAD: {
02162     LoadSDNode *LD = cast<LoadSDNode>(Op);
02163     // If this is a ZEXTLoad and we are looking at the loaded value.
02164     if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
02165       EVT VT = LD->getMemoryVT();
02166       unsigned MemBits = VT.getScalarType().getSizeInBits();
02167       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
02168     } else if (const MDNode *Ranges = LD->getRanges()) {
02169       computeKnownBitsFromRangeMetadata(*Ranges, KnownZero);
02170     }
02171     break;
02172   }
02173   case ISD::ZERO_EXTEND: {
02174     EVT InVT = Op.getOperand(0).getValueType();
02175     unsigned InBits = InVT.getScalarType().getSizeInBits();
02176     APInt NewBits   = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
02177     KnownZero = KnownZero.trunc(InBits);
02178     KnownOne = KnownOne.trunc(InBits);
02179     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02180     KnownZero = KnownZero.zext(BitWidth);
02181     KnownOne = KnownOne.zext(BitWidth);
02182     KnownZero |= NewBits;
02183     break;
02184   }
02185   case ISD::SIGN_EXTEND: {
02186     EVT InVT = Op.getOperand(0).getValueType();
02187     unsigned InBits = InVT.getScalarType().getSizeInBits();
02188     APInt NewBits   = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
02189 
02190     KnownZero = KnownZero.trunc(InBits);
02191     KnownOne = KnownOne.trunc(InBits);
02192     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02193 
02194     // Note if the sign bit is known to be zero or one.
02195     bool SignBitKnownZero = KnownZero.isNegative();
02196     bool SignBitKnownOne  = KnownOne.isNegative();
02197 
02198     KnownZero = KnownZero.zext(BitWidth);
02199     KnownOne = KnownOne.zext(BitWidth);
02200 
02201     // If the sign bit is known zero or one, the top bits match.
02202     if (SignBitKnownZero)
02203       KnownZero |= NewBits;
02204     else if (SignBitKnownOne)
02205       KnownOne  |= NewBits;
02206     break;
02207   }
02208   case ISD::ANY_EXTEND: {
02209     EVT InVT = Op.getOperand(0).getValueType();
02210     unsigned InBits = InVT.getScalarType().getSizeInBits();
02211     KnownZero = KnownZero.trunc(InBits);
02212     KnownOne = KnownOne.trunc(InBits);
02213     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02214     KnownZero = KnownZero.zext(BitWidth);
02215     KnownOne = KnownOne.zext(BitWidth);
02216     break;
02217   }
02218   case ISD::TRUNCATE: {
02219     EVT InVT = Op.getOperand(0).getValueType();
02220     unsigned InBits = InVT.getScalarType().getSizeInBits();
02221     KnownZero = KnownZero.zext(InBits);
02222     KnownOne = KnownOne.zext(InBits);
02223     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02224     KnownZero = KnownZero.trunc(BitWidth);
02225     KnownOne = KnownOne.trunc(BitWidth);
02226     break;
02227   }
02228   case ISD::AssertZext: {
02229     EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
02230     APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
02231     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02232     KnownZero |= (~InMask);
02233     KnownOne  &= (~KnownZero);
02234     break;
02235   }
02236   case ISD::FGETSIGN:
02237     // All bits are zero except the low bit.
02238     KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02239     break;
02240 
02241   case ISD::SUB: {
02242     if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) {
02243       // We know that the top bits of C-X are clear if X contains less bits
02244       // than C (i.e. no wrap-around can happen).  For example, 20-X is
02245       // positive if we can prove that X is >= 0 and < 16.
02246       if (CLHS->getAPIntValue().isNonNegative()) {
02247         unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros();
02248         // NLZ can't be BitWidth with no sign bit
02249         APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1);
02250         computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02251 
02252         // If all of the MaskV bits are known to be zero, then we know the
02253         // output top bits are zero, because we now know that the output is
02254         // from [0-C].
02255         if ((KnownZero2 & MaskV) == MaskV) {
02256           unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros();
02257           // Top bits known zero.
02258           KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2);
02259         }
02260       }
02261     }
02262   }
02263   // fall through
02264   case ISD::ADD:
02265   case ISD::ADDE: {
02266     // Output known-0 bits are known if clear or set in both the low clear bits
02267     // common to both LHS & RHS.  For example, 8+(X<<3) is known to have the
02268     // low 3 bits clear.
02269     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
02270     unsigned KnownZeroOut = KnownZero2.countTrailingOnes();
02271 
02272     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02273     KnownZeroOut = std::min(KnownZeroOut,
02274                             KnownZero2.countTrailingOnes());
02275 
02276     if (Op.getOpcode() == ISD::ADD) {
02277       KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut);
02278       break;
02279     }
02280 
02281     // With ADDE, a carry bit may be added in, so we can only use this
02282     // information if we know (at least) that the low two bits are clear.  We
02283     // then return to the caller that the low bit is unknown but that other bits
02284     // are known zero.
02285     if (KnownZeroOut >= 2) // ADDE
02286       KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut);
02287     break;
02288   }
02289   case ISD::SREM:
02290     if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02291       const APInt &RA = Rem->getAPIntValue().abs();
02292       if (RA.isPowerOf2()) {
02293         APInt LowBits = RA - 1;
02294         computeKnownBits(Op.getOperand(0), KnownZero2,KnownOne2,Depth+1);
02295 
02296         // The low bits of the first operand are unchanged by the srem.
02297         KnownZero = KnownZero2 & LowBits;
02298         KnownOne = KnownOne2 & LowBits;
02299 
02300         // If the first operand is non-negative or has all low bits zero, then
02301         // the upper bits are all zero.
02302         if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits))
02303           KnownZero |= ~LowBits;
02304 
02305         // If the first operand is negative and not all low bits are zero, then
02306         // the upper bits are all one.
02307         if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0))
02308           KnownOne |= ~LowBits;
02309         assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?");
02310       }
02311     }
02312     break;
02313   case ISD::UREM: {
02314     if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02315       const APInt &RA = Rem->getAPIntValue();
02316       if (RA.isPowerOf2()) {
02317         APInt LowBits = (RA - 1);
02318         computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth + 1);
02319 
02320         // The upper bits are all zero, the lower ones are unchanged.
02321         KnownZero = KnownZero2 | ~LowBits;
02322         KnownOne = KnownOne2 & LowBits;
02323         break;
02324       }
02325     }
02326 
02327     // Since the result is less than or equal to either operand, any leading
02328     // zero bits in either operand must also exist in the result.
02329     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02330     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02331 
02332     uint32_t Leaders = std::max(KnownZero.countLeadingOnes(),
02333                                 KnownZero2.countLeadingOnes());
02334     KnownOne.clearAllBits();
02335     KnownZero = APInt::getHighBitsSet(BitWidth, Leaders);
02336     break;
02337   }
02338   case ISD::FrameIndex:
02339   case ISD::TargetFrameIndex:
02340     if (unsigned Align = InferPtrAlignment(Op)) {
02341       // The low bits are known zero if the pointer is aligned.
02342       KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align));
02343       break;
02344     }
02345     break;
02346 
02347   default:
02348     if (Op.getOpcode() < ISD::BUILTIN_OP_END)
02349       break;
02350     // Fallthrough
02351   case ISD::INTRINSIC_WO_CHAIN:
02352   case ISD::INTRINSIC_W_CHAIN:
02353   case ISD::INTRINSIC_VOID:
02354     // Allow the target to implement this method for its nodes.
02355     TLI->computeKnownBitsForTargetNode(Op, KnownZero, KnownOne, *this, Depth);
02356     break;
02357   }
02358 
02359   assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
02360 }
02361 
02362 /// ComputeNumSignBits - Return the number of times the sign bit of the
02363 /// register is replicated into the other bits.  We know that at least 1 bit
02364 /// is always equal to the sign bit (itself), but other cases can give us
02365 /// information.  For example, immediately after an "SRA X, 2", we know that
02366 /// the top 3 bits are all equal to each other, so we return 3.
02367 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{
02368   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
02369   EVT VT = Op.getValueType();
02370   assert(VT.isInteger() && "Invalid VT!");
02371   unsigned VTBits = VT.getScalarType().getSizeInBits();
02372   unsigned Tmp, Tmp2;
02373   unsigned FirstAnswer = 1;
02374 
02375   if (Depth == 6)
02376     return 1;  // Limit search depth.
02377 
02378   switch (Op.getOpcode()) {
02379   default: break;
02380   case ISD::AssertSext:
02381     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
02382     return VTBits-Tmp+1;
02383   case ISD::AssertZext:
02384     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
02385     return VTBits-Tmp;
02386 
02387   case ISD::Constant: {
02388     const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue();
02389     return Val.getNumSignBits();
02390   }
02391 
02392   case ISD::SIGN_EXTEND:
02393     Tmp =
02394         VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits();
02395     return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp;
02396 
02397   case ISD::SIGN_EXTEND_INREG:
02398     // Max of the input and what this extends.
02399     Tmp =
02400       cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits();
02401     Tmp = VTBits-Tmp+1;
02402 
02403     Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02404     return std::max(Tmp, Tmp2);
02405 
02406   case ISD::SRA:
02407     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02408     // SRA X, C   -> adds C sign bits.
02409     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02410       Tmp += C->getZExtValue();
02411       if (Tmp > VTBits) Tmp = VTBits;
02412     }
02413     return Tmp;
02414   case ISD::SHL:
02415     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02416       // shl destroys sign bits.
02417       Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02418       if (C->getZExtValue() >= VTBits ||      // Bad shift.
02419           C->getZExtValue() >= Tmp) break;    // Shifted all sign bits out.
02420       return Tmp - C->getZExtValue();
02421     }
02422     break;
02423   case ISD::AND:
02424   case ISD::OR:
02425   case ISD::XOR:    // NOT is handled here.
02426     // Logical binary ops preserve the number of sign bits at the worst.
02427     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02428     if (Tmp != 1) {
02429       Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02430       FirstAnswer = std::min(Tmp, Tmp2);
02431       // We computed what we know about the sign bits as our first
02432       // answer. Now proceed to the generic code that uses
02433       // computeKnownBits, and pick whichever answer is better.
02434     }
02435     break;
02436 
02437   case ISD::SELECT:
02438     Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02439     if (Tmp == 1) return 1;  // Early out.
02440     Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1);
02441     return std::min(Tmp, Tmp2);
02442 
02443   case ISD::SADDO:
02444   case ISD::UADDO:
02445   case ISD::SSUBO:
02446   case ISD::USUBO:
02447   case ISD::SMULO:
02448   case ISD::UMULO:
02449     if (Op.getResNo() != 1)
02450       break;
02451     // The boolean result conforms to getBooleanContents.  Fall through.
02452     // If setcc returns 0/-1, all bits are sign bits.
02453     // We know that we have an integer-based boolean since these operations
02454     // are only available for integer.
02455     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
02456         TargetLowering::ZeroOrNegativeOneBooleanContent)
02457       return VTBits;
02458     break;
02459   case ISD::SETCC:
02460     // If setcc returns 0/-1, all bits are sign bits.
02461     if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
02462         TargetLowering::ZeroOrNegativeOneBooleanContent)
02463       return VTBits;
02464     break;
02465   case ISD::ROTL:
02466   case ISD::ROTR:
02467     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02468       unsigned RotAmt = C->getZExtValue() & (VTBits-1);
02469 
02470       // Handle rotate right by N like a rotate left by 32-N.
02471       if (Op.getOpcode() == ISD::ROTR)
02472         RotAmt = (VTBits-RotAmt) & (VTBits-1);
02473 
02474       // If we aren't rotating out all of the known-in sign bits, return the
02475       // number that are left.  This handles rotl(sext(x), 1) for example.
02476       Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02477       if (Tmp > RotAmt+1) return Tmp-RotAmt;
02478     }
02479     break;
02480   case ISD::ADD:
02481     // Add can have at most one carry bit.  Thus we know that the output
02482     // is, at worst, one more bit than the inputs.
02483     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02484     if (Tmp == 1) return 1;  // Early out.
02485 
02486     // Special case decrementing a value (ADD X, -1):
02487     if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
02488       if (CRHS->isAllOnesValue()) {
02489         APInt KnownZero, KnownOne;
02490         computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02491 
02492         // If the input is known to be 0 or 1, the output is 0/-1, which is all
02493         // sign bits set.
02494         if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
02495           return VTBits;
02496 
02497         // If we are subtracting one from a positive number, there is no carry
02498         // out of the result.
02499         if (KnownZero.isNegative())
02500           return Tmp;
02501       }
02502 
02503     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02504     if (Tmp2 == 1) return 1;
02505     return std::min(Tmp, Tmp2)-1;
02506 
02507   case ISD::SUB:
02508     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02509     if (Tmp2 == 1) return 1;
02510 
02511     // Handle NEG.
02512     if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0)))
02513       if (CLHS->isNullValue()) {
02514         APInt KnownZero, KnownOne;
02515         computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
02516         // If the input is known to be 0 or 1, the output is 0/-1, which is all
02517         // sign bits set.
02518         if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
02519           return VTBits;
02520 
02521         // If the input is known to be positive (the sign bit is known clear),
02522         // the output of the NEG has the same number of sign bits as the input.
02523         if (KnownZero.isNegative())
02524           return Tmp2;
02525 
02526         // Otherwise, we treat this like a SUB.
02527       }
02528 
02529     // Sub can have at most one carry bit.  Thus we know that the output
02530     // is, at worst, one more bit than the inputs.
02531     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02532     if (Tmp == 1) return 1;  // Early out.
02533     return std::min(Tmp, Tmp2)-1;
02534   case ISD::TRUNCATE:
02535     // FIXME: it's tricky to do anything useful for this, but it is an important
02536     // case for targets like X86.
02537     break;
02538   }
02539 
02540   // If we are looking at the loaded value of the SDNode.
02541   if (Op.getResNo() == 0) {
02542     // Handle LOADX separately here. EXTLOAD case will fallthrough.
02543     if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
02544       unsigned ExtType = LD->getExtensionType();
02545       switch (ExtType) {
02546         default: break;
02547         case ISD::SEXTLOAD:    // '17' bits known
02548           Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
02549           return VTBits-Tmp+1;
02550         case ISD::ZEXTLOAD:    // '16' bits known
02551           Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
02552           return VTBits-Tmp;
02553       }
02554     }
02555   }
02556 
02557   // Allow the target to implement this method for its nodes.
02558   if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
02559       Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
02560       Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
02561       Op.getOpcode() == ISD::INTRINSIC_VOID) {
02562     unsigned NumBits = TLI->ComputeNumSignBitsForTargetNode(Op, *this, Depth);
02563     if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits);
02564   }
02565 
02566   // Finally, if we can prove that the top bits of the result are 0's or 1's,
02567   // use this information.
02568   APInt KnownZero, KnownOne;
02569   computeKnownBits(Op, KnownZero, KnownOne, Depth);
02570 
02571   APInt Mask;
02572   if (KnownZero.isNegative()) {        // sign bit is 0
02573     Mask = KnownZero;
02574   } else if (KnownOne.isNegative()) {  // sign bit is 1;
02575     Mask = KnownOne;
02576   } else {
02577     // Nothing known.
02578     return FirstAnswer;
02579   }
02580 
02581   // Okay, we know that the sign bit in Mask is set.  Use CLZ to determine
02582   // the number of identical bits in the top of the input value.
02583   Mask = ~Mask;
02584   Mask <<= Mask.getBitWidth()-VTBits;
02585   // Return # leading zeros.  We use 'min' here in case Val was zero before
02586   // shifting.  We don't want to return '64' as for an i32 "0".
02587   return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros()));
02588 }
02589 
02590 /// isBaseWithConstantOffset - Return true if the specified operand is an
02591 /// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an
02592 /// ISD::OR with a ConstantSDNode that is guaranteed to have the same
02593 /// semantics as an ADD.  This handles the equivalence:
02594 ///     X|Cst == X+Cst iff X&Cst = 0.
02595 bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
02596   if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
02597       !isa<ConstantSDNode>(Op.getOperand(1)))
02598     return false;
02599 
02600   if (Op.getOpcode() == ISD::OR &&
02601       !MaskedValueIsZero(Op.getOperand(0),
02602                      cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue()))
02603     return false;
02604 
02605   return true;
02606 }
02607 
02608 
02609 bool SelectionDAG::isKnownNeverNaN(SDValue Op) const {
02610   // If we're told that NaNs won't happen, assume they won't.
02611   if (getTarget().Options.NoNaNsFPMath)
02612     return true;
02613 
02614   // If the value is a constant, we can obviously see if it is a NaN or not.
02615   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
02616     return !C->getValueAPF().isNaN();
02617 
02618   // TODO: Recognize more cases here.
02619 
02620   return false;
02621 }
02622 
02623 bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
02624   // If the value is a constant, we can obviously see if it is a zero or not.
02625   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
02626     return !C->isZero();
02627 
02628   // TODO: Recognize more cases here.
02629   switch (Op.getOpcode()) {
02630   default: break;
02631   case ISD::OR:
02632     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
02633       return !C->isNullValue();
02634     break;
02635   }
02636 
02637   return false;
02638 }
02639 
02640 bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
02641   // Check the obvious case.
02642   if (A == B) return true;
02643 
02644   // For for negative and positive zero.
02645   if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
02646     if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
02647       if (CA->isZero() && CB->isZero()) return true;
02648 
02649   // Otherwise they may not be equal.
02650   return false;
02651 }
02652 
02653 /// getNode - Gets or creates the specified node.
02654 ///
02655 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT) {
02656   FoldingSetNodeID ID;
02657   AddNodeIDNode(ID, Opcode, getVTList(VT), None);
02658   void *IP = nullptr;
02659   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
02660     return SDValue(E, 0);
02661 
02662   SDNode *N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(),
02663                                          DL.getDebugLoc(), getVTList(VT));
02664   CSEMap.InsertNode(N, IP);
02665 
02666   InsertNode(N);
02667   return SDValue(N, 0);
02668 }
02669 
02670 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
02671                               EVT VT, SDValue Operand) {
02672   // Constant fold unary operations with an integer constant operand. Even
02673   // opaque constant will be folded, because the folding of unary operations
02674   // doesn't create new constants with different values. Nevertheless, the
02675   // opaque flag is preserved during folding to prevent future folding with
02676   // other constants.
02677   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) {
02678     const APInt &Val = C->getAPIntValue();
02679     switch (Opcode) {
02680     default: break;
02681     case ISD::SIGN_EXTEND:
02682       return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT,
02683                          C->isTargetOpcode(), C->isOpaque());
02684     case ISD::ANY_EXTEND:
02685     case ISD::ZERO_EXTEND:
02686     case ISD::TRUNCATE:
02687       return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT,
02688                          C->isTargetOpcode(), C->isOpaque());
02689     case ISD::UINT_TO_FP:
02690     case ISD::SINT_TO_FP: {
02691       APFloat apf(EVTToAPFloatSemantics(VT),
02692                   APInt::getNullValue(VT.getSizeInBits()));
02693       (void)apf.convertFromAPInt(Val,
02694                                  Opcode==ISD::SINT_TO_FP,
02695                                  APFloat::rmNearestTiesToEven);
02696       return getConstantFP(apf, VT);
02697     }
02698     case ISD::BITCAST:
02699       if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
02700         return getConstantFP(APFloat(APFloat::IEEEsingle, Val), VT);
02701       else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
02702         return getConstantFP(APFloat(APFloat::IEEEdouble, Val), VT);
02703       break;
02704     case ISD::BSWAP:
02705       return getConstant(Val.byteSwap(), VT, C->isTargetOpcode(),
02706                          C->isOpaque());
02707     case ISD::CTPOP:
02708       return getConstant(Val.countPopulation(), VT, C->isTargetOpcode(),
02709                          C->isOpaque());
02710     case ISD::CTLZ:
02711     case ISD::CTLZ_ZERO_UNDEF:
02712       return getConstant(Val.countLeadingZeros(), VT, C->isTargetOpcode(),
02713                          C->isOpaque());
02714     case ISD::CTTZ:
02715     case ISD::CTTZ_ZERO_UNDEF:
02716       return getConstant(Val.countTrailingZeros(), VT, C->isTargetOpcode(),
02717                          C->isOpaque());
02718     }
02719   }
02720 
02721   // Constant fold unary operations with a floating point constant operand.
02722   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) {
02723     APFloat V = C->getValueAPF();    // make copy
02724     switch (Opcode) {
02725     case ISD::FNEG:
02726       V.changeSign();
02727       return getConstantFP(V, VT);
02728     case ISD::FABS:
02729       V.clearSign();
02730       return getConstantFP(V, VT);
02731     case ISD::FCEIL: {
02732       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
02733       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02734         return getConstantFP(V, VT);
02735       break;
02736     }
02737     case ISD::FTRUNC: {
02738       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
02739       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02740         return getConstantFP(V, VT);
02741       break;
02742     }
02743     case ISD::FFLOOR: {
02744       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
02745       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02746         return getConstantFP(V, VT);
02747       break;
02748     }
02749     case ISD::FP_EXTEND: {
02750       bool ignored;
02751       // This can return overflow, underflow, or inexact; we don't care.
02752       // FIXME need to be more flexible about rounding mode.
02753       (void)V.convert(EVTToAPFloatSemantics(VT),
02754                       APFloat::rmNearestTiesToEven, &ignored);
02755       return getConstantFP(V, VT);
02756     }
02757     case ISD::FP_TO_SINT:
02758     case ISD::FP_TO_UINT: {
02759       integerPart x[2];
02760       bool ignored;
02761       assert(integerPartWidth >= 64);
02762       // FIXME need to be more flexible about rounding mode.
02763       APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(),
02764                             Opcode==ISD::FP_TO_SINT,
02765                             APFloat::rmTowardZero, &ignored);
02766       if (s==APFloat::opInvalidOp)     // inexact is OK, in fact usual
02767         break;
02768       APInt api(VT.getSizeInBits(), x);
02769       return getConstant(api, VT);
02770     }
02771     case ISD::BITCAST:
02772       if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
02773         return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT);
02774       else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
02775         return getConstant(V.bitcastToAPInt().getZExtValue(), VT);
02776       break;
02777     }
02778   }
02779 
02780   // Constant fold unary operations with a vector integer operand.
02781   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand.getNode())) {
02782     if (BV->isConstant()) {
02783       switch (Opcode) {
02784       default:
02785         // FIXME: Entirely reasonable to perform folding of other unary
02786         // operations here as the need arises.
02787         break;
02788       case ISD::UINT_TO_FP:
02789       case ISD::SINT_TO_FP: {
02790         SmallVector<SDValue, 8> Ops;
02791         for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
02792           SDValue OpN = BV->getOperand(i);
02793           // Let the above scalar folding handle the conversion of each
02794           // element.
02795           OpN = getNode(ISD::SINT_TO_FP, DL, VT.getVectorElementType(),
02796                         OpN);
02797           Ops.push_back(OpN);
02798         }
02799         return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
02800       }
02801       }
02802     }
02803   }
02804 
02805   unsigned OpOpcode = Operand.getNode()->getOpcode();
02806   switch (Opcode) {
02807   case ISD::TokenFactor:
02808   case ISD::MERGE_VALUES:
02809   case ISD::CONCAT_VECTORS:
02810     return Operand;         // Factor, merge or concat of one node?  No need.
02811   case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
02812   case ISD::FP_EXTEND:
02813     assert(VT.isFloatingPoint() &&
02814            Operand.getValueType().isFloatingPoint() && "Invalid FP cast!");
02815     if (Operand.getValueType() == VT) return Operand;  // noop conversion.
02816     assert((!VT.isVector() ||
02817             VT.getVectorNumElements() ==
02818             Operand.getValueType().getVectorNumElements()) &&
02819            "Vector element count mismatch!");
02820     if (Operand.getOpcode() == ISD::UNDEF)
02821       return getUNDEF(VT);
02822     break;
02823   case ISD::SIGN_EXTEND:
02824     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02825            "Invalid SIGN_EXTEND!");
02826     if (Operand.getValueType() == VT) return Operand;   // noop extension
02827     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02828            "Invalid sext node, dst < src!");
02829     assert((!VT.isVector() ||
02830             VT.getVectorNumElements() ==
02831             Operand.getValueType().getVectorNumElements()) &&
02832            "Vector element count mismatch!");
02833     if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
02834       return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02835     else if (OpOpcode == ISD::UNDEF)
02836       // sext(undef) = 0, because the top bits will all be the same.
02837       return getConstant(0, VT);
02838     break;
02839   case ISD::ZERO_EXTEND:
02840     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02841            "Invalid ZERO_EXTEND!");
02842     if (Operand.getValueType() == VT) return Operand;   // noop extension
02843     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02844            "Invalid zext node, dst < src!");
02845     assert((!VT.isVector() ||
02846             VT.getVectorNumElements() ==
02847             Operand.getValueType().getVectorNumElements()) &&
02848            "Vector element count mismatch!");
02849     if (OpOpcode == ISD::ZERO_EXTEND)   // (zext (zext x)) -> (zext x)
02850       return getNode(ISD::ZERO_EXTEND, DL, VT,
02851                      Operand.getNode()->getOperand(0));
02852     else if (OpOpcode == ISD::UNDEF)
02853       // zext(undef) = 0, because the top bits will be zero.
02854       return getConstant(0, VT);
02855     break;
02856   case ISD::ANY_EXTEND:
02857     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02858            "Invalid ANY_EXTEND!");
02859     if (Operand.getValueType() == VT) return Operand;   // noop extension
02860     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02861            "Invalid anyext node, dst < src!");
02862     assert((!VT.isVector() ||
02863             VT.getVectorNumElements() ==
02864             Operand.getValueType().getVectorNumElements()) &&
02865            "Vector element count mismatch!");
02866 
02867     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
02868         OpOpcode == ISD::ANY_EXTEND)
02869       // (ext (zext x)) -> (zext x)  and  (ext (sext x)) -> (sext x)
02870       return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02871     else if (OpOpcode == ISD::UNDEF)
02872       return getUNDEF(VT);
02873 
02874     // (ext (trunx x)) -> x
02875     if (OpOpcode == ISD::TRUNCATE) {
02876       SDValue OpOp = Operand.getNode()->getOperand(0);
02877       if (OpOp.getValueType() == VT)
02878         return OpOp;
02879     }
02880     break;
02881   case ISD::TRUNCATE:
02882     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02883            "Invalid TRUNCATE!");
02884     if (Operand.getValueType() == VT) return Operand;   // noop truncate
02885     assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) &&
02886            "Invalid truncate node, src < dst!");
02887     assert((!VT.isVector() ||
02888             VT.getVectorNumElements() ==
02889             Operand.getValueType().getVectorNumElements()) &&
02890            "Vector element count mismatch!");
02891     if (OpOpcode == ISD::TRUNCATE)
02892       return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
02893     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
02894         OpOpcode == ISD::ANY_EXTEND) {
02895       // If the source is smaller than the dest, we still need an extend.
02896       if (Operand.getNode()->getOperand(0).getValueType().getScalarType()
02897             .bitsLT(VT.getScalarType()))
02898         return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02899       if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT))
02900         return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
02901       return Operand.getNode()->getOperand(0);
02902     }
02903     if (OpOpcode == ISD::UNDEF)
02904       return getUNDEF(VT);
02905     break;
02906   case ISD::BITCAST:
02907     // Basic sanity checking.
02908     assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits()
02909            && "Cannot BITCAST between types of different sizes!");
02910     if (VT == Operand.getValueType()) return Operand;  // noop conversion.
02911     if (OpOpcode == ISD::BITCAST)  // bitconv(bitconv(x)) -> bitconv(x)
02912       return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
02913     if (OpOpcode == ISD::UNDEF)
02914       return getUNDEF(VT);
02915     break;
02916   case ISD::SCALAR_TO_VECTOR:
02917     assert(VT.isVector() && !Operand.getValueType().isVector() &&
02918            (VT.getVectorElementType() == Operand.getValueType() ||
02919             (VT.getVectorElementType().isInteger() &&
02920              Operand.getValueType().isInteger() &&
02921              VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&
02922            "Illegal SCALAR_TO_VECTOR node!");
02923     if (OpOpcode == ISD::UNDEF)
02924       return getUNDEF(VT);
02925     // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
02926     if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
02927         isa<ConstantSDNode>(Operand.getOperand(1)) &&
02928         Operand.getConstantOperandVal(1) == 0 &&
02929         Operand.getOperand(0).getValueType() == VT)
02930       return Operand.getOperand(0);
02931     break;
02932   case ISD::FNEG:
02933     // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0
02934     if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB)
02935       return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1),
02936                      Operand.getNode()->getOperand(0));
02937     if (OpOpcode == ISD::FNEG)  // --X -> X
02938       return Operand.getNode()->getOperand(0);
02939     break;
02940   case ISD::FABS:
02941     if (OpOpcode == ISD::FNEG)  // abs(-X) -> abs(X)
02942       return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0));
02943     break;
02944   }
02945 
02946   SDNode *N;
02947   SDVTList VTs = getVTList(VT);
02948   if (VT != MVT::Glue) { // Don't CSE flag producing nodes
02949     FoldingSetNodeID ID;
02950     SDValue Ops[1] = { Operand };
02951     AddNodeIDNode(ID, Opcode, VTs, Ops);
02952     void *IP = nullptr;
02953     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
02954       return SDValue(E, 0);
02955 
02956     N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
02957                                         DL.getDebugLoc(), VTs, Operand);
02958     CSEMap.InsertNode(N, IP);
02959   } else {
02960     N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
02961                                         DL.getDebugLoc(), VTs, Operand);
02962   }
02963 
02964   InsertNode(N);
02965   return SDValue(N, 0);
02966 }
02967 
02968 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT,
02969                                              SDNode *Cst1, SDNode *Cst2) {
02970   // If the opcode is a target-specific ISD node, there's nothing we can
02971   // do here and the operand rules may not line up with the below, so
02972   // bail early.
02973   if (Opcode >= ISD::BUILTIN_OP_END)
02974     return SDValue();
02975 
02976   SmallVector<std::pair<ConstantSDNode *, ConstantSDNode *>, 4> Inputs;
02977   SmallVector<SDValue, 4> Outputs;
02978   EVT SVT = VT.getScalarType();
02979 
02980   ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1);
02981   ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2);
02982   if (Scalar1 && Scalar2 && (Scalar1->isOpaque() || Scalar2->isOpaque()))
02983     return SDValue();
02984 
02985   if (Scalar1 && Scalar2)
02986     // Scalar instruction.
02987     Inputs.push_back(std::make_pair(Scalar1, Scalar2));
02988   else {
02989     // For vectors extract each constant element into Inputs so we can constant
02990     // fold them individually.
02991     BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1);
02992     BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2);
02993     if (!BV1 || !BV2)
02994       return SDValue();
02995 
02996     assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!");
02997 
02998     for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) {
02999       ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I));
03000       ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I));
03001       if (!V1 || !V2) // Not a constant, bail.
03002         return SDValue();
03003 
03004       if (V1->isOpaque() || V2->isOpaque())
03005         return SDValue();
03006 
03007       // Avoid BUILD_VECTOR nodes that perform implicit truncation.
03008       // FIXME: This is valid and could be handled by truncating the APInts.
03009       if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
03010         return SDValue();
03011 
03012       Inputs.push_back(std::make_pair(V1, V2));
03013     }
03014   }
03015 
03016   // We have a number of constant values, constant fold them element by element.
03017   for (unsigned I = 0, E = Inputs.size(); I != E; ++I) {
03018     const APInt &C1 = Inputs[I].first->getAPIntValue();
03019     const APInt &C2 = Inputs[I].second->getAPIntValue();
03020 
03021     switch (Opcode) {
03022     case ISD::ADD:
03023       Outputs.push_back(getConstant(C1 + C2, SVT));
03024       break;
03025     case ISD::SUB:
03026       Outputs.push_back(getConstant(C1 - C2, SVT));
03027       break;
03028     case ISD::MUL:
03029       Outputs.push_back(getConstant(C1 * C2, SVT));
03030       break;
03031     case ISD::UDIV:
03032       if (!C2.getBoolValue())
03033         return SDValue();
03034       Outputs.push_back(getConstant(C1.udiv(C2), SVT));
03035       break;
03036     case ISD::UREM:
03037       if (!C2.getBoolValue())
03038         return SDValue();
03039       Outputs.push_back(getConstant(C1.urem(C2), SVT));
03040       break;
03041     case ISD::SDIV:
03042       if (!C2.getBoolValue())
03043         return SDValue();
03044       Outputs.push_back(getConstant(C1.sdiv(C2), SVT));
03045       break;
03046     case ISD::SREM:
03047       if (!C2.getBoolValue())
03048         return SDValue();
03049       Outputs.push_back(getConstant(C1.srem(C2), SVT));
03050       break;
03051     case ISD::AND:
03052       Outputs.push_back(getConstant(C1 & C2, SVT));
03053       break;
03054     case ISD::OR:
03055       Outputs.push_back(getConstant(C1 | C2, SVT));
03056       break;
03057     case ISD::XOR:
03058       Outputs.push_back(getConstant(C1 ^ C2, SVT));
03059       break;
03060     case ISD::SHL:
03061       Outputs.push_back(getConstant(C1 << C2, SVT));
03062       break;
03063     case ISD::SRL:
03064       Outputs.push_back(getConstant(C1.lshr(C2), SVT));
03065       break;
03066     case ISD::SRA:
03067       Outputs.push_back(getConstant(C1.ashr(C2), SVT));
03068       break;
03069     case ISD::ROTL:
03070       Outputs.push_back(getConstant(C1.rotl(C2), SVT));
03071       break;
03072     case ISD::ROTR:
03073       Outputs.push_back(getConstant(C1.rotr(C2), SVT));
03074       break;
03075     default:
03076       return SDValue();
03077     }
03078   }
03079 
03080   assert((Scalar1 && Scalar2) || (VT.getVectorNumElements() == Outputs.size() &&
03081                                   "Expected a scalar or vector!"));
03082 
03083   // Handle the scalar case first.
03084   if (!VT.isVector())
03085     return Outputs.back();
03086 
03087   // We may have a vector type but a scalar result. Create a splat.
03088   Outputs.resize(VT.getVectorNumElements(), Outputs.back());
03089 
03090   // Build a big vector out of the scalar elements we generated.
03091   return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs);
03092 }
03093 
03094 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
03095                               SDValue N2, bool nuw, bool nsw, bool exact) {
03096   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
03097   ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode());
03098   switch (Opcode) {
03099   default: break;
03100   case ISD::TokenFactor:
03101     assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
03102            N2.getValueType() == MVT::Other && "Invalid token factor!");
03103     // Fold trivial token factors.
03104     if (N1.getOpcode() == ISD::EntryToken) return N2;
03105     if (N2.getOpcode() == ISD::EntryToken) return N1;
03106     if (N1 == N2) return N1;
03107     break;
03108   case ISD::CONCAT_VECTORS:
03109     // Concat of UNDEFs is UNDEF.
03110     if (N1.getOpcode() == ISD::UNDEF &&
03111         N2.getOpcode() == ISD::UNDEF)
03112       return getUNDEF(VT);
03113 
03114     // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
03115     // one big BUILD_VECTOR.
03116     if (N1.getOpcode() == ISD::BUILD_VECTOR &&
03117         N2.getOpcode() == ISD::BUILD_VECTOR) {
03118       SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
03119                                     N1.getNode()->op_end());
03120       Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
03121       return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
03122     }
03123     break;
03124   case ISD::AND:
03125     assert(VT.isInteger() && "This operator does not apply to FP types!");
03126     assert(N1.getValueType() == N2.getValueType() &&
03127            N1.getValueType() == VT && "Binary operator types must match!");
03128     // (X & 0) -> 0.  This commonly occurs when legalizing i64 values, so it's
03129     // worth handling here.
03130     if (N2C && N2C->isNullValue())
03131       return N2;
03132     if (N2C && N2C->isAllOnesValue())  // X & -1 -> X
03133       return N1;
03134     break;
03135   case ISD::OR:
03136   case ISD::XOR:
03137   case ISD::ADD:
03138   case ISD::SUB:
03139     assert(VT.isInteger() && "This operator does not apply to FP types!");
03140     assert(N1.getValueType() == N2.getValueType() &&
03141            N1.getValueType() == VT && "Binary operator types must match!");
03142     // (X ^|+- 0) -> X.  This commonly occurs when legalizing i64 values, so
03143     // it's worth handling here.
03144     if (N2C && N2C->isNullValue())
03145       return N1;
03146     break;
03147   case ISD::UDIV:
03148   case ISD::UREM:
03149   case ISD::MULHU:
03150   case ISD::MULHS:
03151   case ISD::MUL:
03152   case ISD::SDIV:
03153   case ISD::SREM:
03154     assert(VT.isInteger() && "This operator does not apply to FP types!");
03155     assert(N1.getValueType() == N2.getValueType() &&
03156            N1.getValueType() == VT && "Binary operator types must match!");
03157     break;
03158   case ISD::FADD:
03159   case ISD::FSUB:
03160   case ISD::FMUL:
03161   case ISD::FDIV:
03162   case ISD::FREM:
03163     if (getTarget().Options.UnsafeFPMath) {
03164       if (Opcode == ISD::FADD) {
03165         // 0+x --> x
03166         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1))
03167           if (CFP->getValueAPF().isZero())
03168             return N2;
03169         // x+0 --> x
03170         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
03171           if (CFP->getValueAPF().isZero())
03172             return N1;
03173       } else if (Opcode == ISD::FSUB) {
03174         // x-0 --> x
03175         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
03176           if (CFP->getValueAPF().isZero())
03177             return N1;
03178       } else if (Opcode == ISD::FMUL) {
03179         ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1);
03180         SDValue V = N2;
03181 
03182         // If the first operand isn't the constant, try the second
03183         if (!CFP) {
03184           CFP = dyn_cast<ConstantFPSDNode>(N2);
03185           V = N1;
03186         }
03187 
03188         if (CFP) {
03189           // 0*x --> 0
03190           if (CFP->isZero())
03191             return SDValue(CFP,0);
03192           // 1*x --> x
03193           if (CFP->isExactlyValue(1.0))
03194             return V;
03195         }
03196       }
03197     }
03198     assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
03199     assert(N1.getValueType() == N2.getValueType() &&
03200            N1.getValueType() == VT && "Binary operator types must match!");
03201     break;
03202   case ISD::FCOPYSIGN:   // N1 and result must match.  N1/N2 need not match.
03203     assert(N1.getValueType() == VT &&
03204            N1.getValueType().isFloatingPoint() &&
03205            N2.getValueType().isFloatingPoint() &&
03206            "Invalid FCOPYSIGN!");
03207     break;
03208   case ISD::SHL:
03209   case ISD::SRA:
03210   case ISD::SRL:
03211   case ISD::ROTL:
03212   case ISD::ROTR:
03213     assert(VT == N1.getValueType() &&
03214            "Shift operators return type must be the same as their first arg");
03215     assert(VT.isInteger() && N2.getValueType().isInteger() &&
03216            "Shifts only work on integers");
03217     assert((!VT.isVector() || VT == N2.getValueType()) &&
03218            "Vector shift amounts must be in the same as their first arg");
03219     // Verify that the shift amount VT is bit enough to hold valid shift
03220     // amounts.  This catches things like trying to shift an i1024 value by an
03221     // i8, which is easy to fall into in generic code that uses
03222     // TLI.getShiftAmount().
03223     assert(N2.getValueType().getSizeInBits() >=
03224                    Log2_32_Ceil(N1.getValueType().getSizeInBits()) &&
03225            "Invalid use of small shift amount with oversized value!");
03226 
03227     // Always fold shifts of i1 values so the code generator doesn't need to
03228     // handle them.  Since we know the size of the shift has to be less than the
03229     // size of the value, the shift/rotate count is guaranteed to be zero.
03230     if (VT == MVT::i1)
03231       return N1;
03232     if (N2C && N2C->isNullValue())
03233       return N1;
03234     break;
03235   case ISD::FP_ROUND_INREG: {
03236     EVT EVT = cast<VTSDNode>(N2)->getVT();
03237     assert(VT == N1.getValueType() && "Not an inreg round!");
03238     assert(VT.isFloatingPoint() && EVT.isFloatingPoint() &&
03239            "Cannot FP_ROUND_INREG integer types");
03240     assert(EVT.isVector() == VT.isVector() &&
03241            "FP_ROUND_INREG type should be vector iff the operand "
03242            "type is vector!");
03243     assert((!EVT.isVector() ||
03244             EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
03245            "Vector element counts must match in FP_ROUND_INREG");
03246     assert(EVT.bitsLE(VT) && "Not rounding down!");
03247     (void)EVT;
03248     if (cast<VTSDNode>(N2)->getVT() == VT) return N1;  // Not actually rounding.
03249     break;
03250   }
03251   case ISD::FP_ROUND:
03252     assert(VT.isFloatingPoint() &&
03253            N1.getValueType().isFloatingPoint() &&
03254            VT.bitsLE(N1.getValueType()) &&
03255            isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!");
03256     if (N1.getValueType() == VT) return N1;  // noop conversion.
03257     break;
03258   case ISD::AssertSext:
03259   case ISD::AssertZext: {
03260     EVT EVT = cast<VTSDNode>(N2)->getVT();
03261     assert(VT == N1.getValueType() && "Not an inreg extend!");
03262     assert(VT.isInteger() && EVT.isInteger() &&
03263            "Cannot *_EXTEND_INREG FP types");
03264     assert(!EVT.isVector() &&
03265            "AssertSExt/AssertZExt type should be the vector element type "
03266            "rather than the vector type!");
03267     assert(EVT.bitsLE(VT) && "Not extending!");
03268     if (VT == EVT) return N1; // noop assertion.
03269     break;
03270   }
03271   case ISD::SIGN_EXTEND_INREG: {
03272     EVT EVT = cast<VTSDNode>(N2)->getVT();
03273     assert(VT == N1.getValueType() && "Not an inreg extend!");
03274     assert(VT.isInteger() && EVT.isInteger() &&
03275            "Cannot *_EXTEND_INREG FP types");
03276     assert(EVT.isVector() == VT.isVector() &&
03277            "SIGN_EXTEND_INREG type should be vector iff the operand "
03278            "type is vector!");
03279     assert((!EVT.isVector() ||
03280             EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
03281            "Vector element counts must match in SIGN_EXTEND_INREG");
03282     assert(EVT.bitsLE(VT) && "Not extending!");
03283     if (EVT == VT) return N1;  // Not actually extending
03284 
03285     if (N1C) {
03286       APInt Val = N1C->getAPIntValue();
03287       unsigned FromBits = EVT.getScalarType().getSizeInBits();
03288       Val <<= Val.getBitWidth()-FromBits;
03289       Val = Val.ashr(Val.getBitWidth()-FromBits);
03290       return getConstant(Val, VT);
03291     }
03292     break;
03293   }
03294   case ISD::EXTRACT_VECTOR_ELT:
03295     // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF.
03296     if (N1.getOpcode() == ISD::UNDEF)
03297       return getUNDEF(VT);
03298 
03299     // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
03300     // expanding copies of large vectors from registers.
03301     if (N2C &&
03302         N1.getOpcode() == ISD::CONCAT_VECTORS &&
03303         N1.getNumOperands() > 0) {
03304       unsigned Factor =
03305         N1.getOperand(0).getValueType().getVectorNumElements();
03306       return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
03307                      N1.getOperand(N2C->getZExtValue() / Factor),
03308                      getConstant(N2C->getZExtValue() % Factor,
03309                                  N2.getValueType()));
03310     }
03311 
03312     // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
03313     // expanding large vector constants.
03314     if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) {
03315       SDValue Elt = N1.getOperand(N2C->getZExtValue());
03316 
03317       if (VT != Elt.getValueType())
03318         // If the vector element type is not legal, the BUILD_VECTOR operands
03319         // are promoted and implicitly truncated, and the result implicitly
03320         // extended. Make that explicit here.
03321         Elt = getAnyExtOrTrunc(Elt, DL, VT);
03322 
03323       return Elt;
03324     }
03325 
03326     // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
03327     // operations are lowered to scalars.
03328     if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
03329       // If the indices are the same, return the inserted element else
03330       // if the indices are known different, extract the element from
03331       // the original vector.
03332       SDValue N1Op2 = N1.getOperand(2);
03333       ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode());
03334 
03335       if (N1Op2C && N2C) {
03336         if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
03337           if (VT == N1.getOperand(1).getValueType())
03338             return N1.getOperand(1);
03339           else
03340             return getSExtOrTrunc(N1.getOperand(1), DL, VT);
03341         }
03342 
03343         return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
03344       }
03345     }
03346     break;
03347   case ISD::EXTRACT_ELEMENT:
03348     assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
03349     assert(!N1.getValueType().isVector() && !VT.isVector() &&
03350            (N1.getValueType().isInteger() == VT.isInteger()) &&
03351            N1.getValueType() != VT &&
03352            "Wrong types for EXTRACT_ELEMENT!");
03353 
03354     // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
03355     // 64-bit integers into 32-bit parts.  Instead of building the extract of
03356     // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
03357     if (N1.getOpcode() == ISD::BUILD_PAIR)
03358       return N1.getOperand(N2C->getZExtValue());
03359 
03360     // EXTRACT_ELEMENT of a constant int is also very common.
03361     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
03362       unsigned ElementSize = VT.getSizeInBits();
03363       unsigned Shift = ElementSize * N2C->getZExtValue();
03364       APInt ShiftedVal = C->getAPIntValue().lshr(Shift);
03365       return getConstant(ShiftedVal.trunc(ElementSize), VT);
03366     }
03367     break;
03368   case ISD::EXTRACT_SUBVECTOR: {
03369     SDValue Index = N2;
03370     if (VT.isSimple() && N1.getValueType().isSimple()) {
03371       assert(VT.isVector() && N1.getValueType().isVector() &&
03372              "Extract subvector VTs must be a vectors!");
03373       assert(VT.getVectorElementType() ==
03374              N1.getValueType().getVectorElementType() &&
03375              "Extract subvector VTs must have the same element type!");
03376       assert(VT.getSimpleVT() <= N1.getSimpleValueType() &&
03377              "Extract subvector must be from larger vector to smaller vector!");
03378 
03379       if (isa<ConstantSDNode>(Index.getNode())) {
03380         assert((VT.getVectorNumElements() +
03381                 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
03382                 <= N1.getValueType().getVectorNumElements())
03383                && "Extract subvector overflow!");
03384       }
03385 
03386       // Trivial extraction.
03387       if (VT.getSimpleVT() == N1.getSimpleValueType())
03388         return N1;
03389     }
03390     break;
03391   }
03392   }
03393 
03394   // Perform trivial constant folding.
03395   SDValue SV = FoldConstantArithmetic(Opcode, VT, N1.getNode(), N2.getNode());
03396   if (SV.getNode()) return SV;
03397 
03398   // Canonicalize constant to RHS if commutative.
03399   if (N1C && !N2C && isCommutativeBinOp(Opcode)) {
03400     std::swap(N1C, N2C);
03401     std::swap(N1, N2);
03402   }
03403 
03404   // Constant fold FP operations.
03405   bool HasFPExceptions = TLI->hasFloatingPointExceptions();
03406   ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode());
03407   ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode());
03408   if (N1CFP) {
03409     if (!N2CFP && isCommutativeBinOp(Opcode)) {
03410       // Canonicalize constant to RHS if commutative.
03411       std::swap(N1CFP, N2CFP);
03412       std::swap(N1, N2);
03413     } else if (N2CFP) {
03414       APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF();
03415       APFloat::opStatus s;
03416       switch (Opcode) {
03417       case ISD::FADD:
03418         s = V1.add(V2, APFloat::rmNearestTiesToEven);
03419         if (!HasFPExceptions || s != APFloat::opInvalidOp)
03420           return getConstantFP(V1, VT);
03421         break;
03422       case ISD::FSUB:
03423         s = V1.subtract(V2, APFloat::rmNearestTiesToEven);
03424         if (!HasFPExceptions || s!=APFloat::opInvalidOp)
03425           return getConstantFP(V1, VT);
03426         break;
03427       case ISD::FMUL:
03428         s = V1.multiply(V2, APFloat::rmNearestTiesToEven);
03429         if (!HasFPExceptions || s!=APFloat::opInvalidOp)
03430           return getConstantFP(V1, VT);
03431         break;
03432       case ISD::FDIV:
03433         s = V1.divide(V2, APFloat::rmNearestTiesToEven);
03434         if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
03435                                  s!=APFloat::opDivByZero)) {
03436           return getConstantFP(V1, VT);
03437         }
03438         break;
03439       case ISD::FREM :
03440         s = V1.mod(V2, APFloat::rmNearestTiesToEven);
03441         if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
03442                                  s!=APFloat::opDivByZero)) {
03443           return getConstantFP(V1, VT);
03444         }
03445         break;
03446       case ISD::FCOPYSIGN:
03447         V1.copySign(V2);
03448         return getConstantFP(V1, VT);
03449       default: break;
03450       }
03451     }
03452 
03453     if (Opcode == ISD::FP_ROUND) {
03454       APFloat V = N1CFP->getValueAPF();    // make copy
03455       bool ignored;
03456       // This can return overflow, underflow, or inexact; we don't care.
03457       // FIXME need to be more flexible about rounding mode.
03458       (void)V.convert(EVTToAPFloatSemantics(VT),
03459                       APFloat::rmNearestTiesToEven, &ignored);
03460       return getConstantFP(V, VT);
03461     }
03462   }
03463 
03464   // Canonicalize an UNDEF to the RHS, even over a constant.
03465   if (N1.getOpcode() == ISD::UNDEF) {
03466     if (isCommutativeBinOp(Opcode)) {
03467       std::swap(N1, N2);
03468     } else {
03469       switch (Opcode) {
03470       case ISD::FP_ROUND_INREG:
03471       case ISD::SIGN_EXTEND_INREG:
03472       case ISD::SUB:
03473       case ISD::FSUB:
03474       case ISD::FDIV:
03475       case ISD::FREM:
03476       case ISD::SRA:
03477         return N1;     // fold op(undef, arg2) -> undef
03478       case ISD::UDIV:
03479       case ISD::SDIV:
03480       case ISD::UREM:
03481       case ISD::SREM:
03482       case ISD::SRL:
03483       case ISD::SHL:
03484         if (!VT.isVector())
03485           return getConstant(0, VT);    // fold op(undef, arg2) -> 0
03486         // For vectors, we can't easily build an all zero vector, just return
03487         // the LHS.
03488         return N2;
03489       }
03490     }
03491   }
03492 
03493   // Fold a bunch of operators when the RHS is undef.
03494   if (N2.getOpcode() == ISD::UNDEF) {
03495     switch (Opcode) {
03496     case ISD::XOR:
03497       if (N1.getOpcode() == ISD::UNDEF)
03498         // Handle undef ^ undef -> 0 special case. This is a common
03499         // idiom (misuse).
03500         return getConstant(0, VT);
03501       // fallthrough
03502     case ISD::ADD:
03503     case ISD::ADDC:
03504     case ISD::ADDE:
03505     case ISD::SUB:
03506     case ISD::UDIV:
03507     case ISD::SDIV:
03508     case ISD::UREM:
03509     case ISD::SREM:
03510       return N2;       // fold op(arg1, undef) -> undef
03511     case ISD::FADD:
03512     case ISD::FSUB:
03513     case ISD::FMUL:
03514     case ISD::FDIV:
03515     case ISD::FREM:
03516       if (getTarget().Options.UnsafeFPMath)
03517         return N2;
03518       break;
03519     case ISD::MUL:
03520     case ISD::AND:
03521     case ISD::SRL:
03522     case ISD::SHL:
03523       if (!VT.isVector())
03524         return getConstant(0, VT);  // fold op(arg1, undef) -> 0
03525       // For vectors, we can't easily build an all zero vector, just return
03526       // the LHS.
03527       return N1;
03528     case ISD::OR:
03529       if (!VT.isVector())
03530         return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT);
03531       // For vectors, we can't easily build an all one vector, just return
03532       // the LHS.
03533       return N1;
03534     case ISD::SRA:
03535       return N1;
03536     }
03537   }
03538 
03539   // Memoize this node if possible.
03540   BinarySDNode *N;
03541   SDVTList VTs = getVTList(VT);
03542   const bool BinOpHasFlags = isBinOpWithFlags(Opcode);
03543   if (VT != MVT::Glue) {
03544     SDValue Ops[] = {N1, N2};
03545     FoldingSetNodeID ID;
03546     AddNodeIDNode(ID, Opcode, VTs, Ops);
03547     if (BinOpHasFlags)
03548       AddBinaryNodeIDCustom(ID, Opcode, nuw, nsw, exact);
03549     void *IP = nullptr;
03550     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
03551       return SDValue(E, 0);
03552 
03553     N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
03554 
03555     CSEMap.InsertNode(N, IP);
03556   } else {
03557 
03558     N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
03559   }
03560 
03561   InsertNode(N);
03562   return SDValue(N, 0);
03563 }
03564 
03565 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03566                               SDValue N1, SDValue N2, SDValue N3) {
03567   // Perform various simplifications.
03568   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
03569   switch (Opcode) {
03570   case ISD::FMA: {
03571     ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
03572     ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
03573     ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
03574     if (N1CFP && N2CFP && N3CFP) {
03575       APFloat  V1 = N1CFP->getValueAPF();
03576       const APFloat &V2 = N2CFP->getValueAPF();
03577       const APFloat &V3 = N3CFP->getValueAPF();
03578       APFloat::opStatus s =
03579         V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
03580       if (s != APFloat::opInvalidOp)
03581         return getConstantFP(V1, VT);
03582     }
03583     break;
03584   }
03585   case ISD::CONCAT_VECTORS:
03586     // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
03587     // one big BUILD_VECTOR.
03588     if (N1.getOpcode() == ISD::BUILD_VECTOR &&
03589         N2.getOpcode() == ISD::BUILD_VECTOR &&
03590         N3.getOpcode() == ISD::BUILD_VECTOR) {
03591       SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
03592                                     N1.getNode()->op_end());
03593       Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
03594       Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end());
03595       return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
03596     }
03597     break;
03598   case ISD::SETCC: {
03599     // Use FoldSetCC to simplify SETCC's.
03600     SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL);
03601     if (Simp.getNode()) return Simp;
03602     break;
03603   }
03604   case ISD::SELECT:
03605     if (N1C) {
03606      if (N1C->getZExtValue())
03607        return N2;             // select true, X, Y -> X
03608      return N3;             // select false, X, Y -> Y
03609     }
03610 
03611     if (N2 == N3) return N2;   // select C, X, X -> X
03612     break;
03613   case ISD::VECTOR_SHUFFLE:
03614     llvm_unreachable("should use getVectorShuffle constructor!");
03615   case ISD::INSERT_SUBVECTOR: {
03616     SDValue Index = N3;
03617     if (VT.isSimple() && N1.getValueType().isSimple()
03618         && N2.getValueType().isSimple()) {
03619       assert(VT.isVector() && N1.getValueType().isVector() &&
03620              N2.getValueType().isVector() &&
03621              "Insert subvector VTs must be a vectors");
03622       assert(VT == N1.getValueType() &&
03623              "Dest and insert subvector source types must match!");
03624       assert(N2.getSimpleValueType() <= N1.getSimpleValueType() &&
03625              "Insert subvector must be from smaller vector to larger vector!");
03626       if (isa<ConstantSDNode>(Index.getNode())) {
03627         assert((N2.getValueType().getVectorNumElements() +
03628                 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
03629                 <= VT.getVectorNumElements())
03630                && "Insert subvector overflow!");
03631       }
03632 
03633       // Trivial insertion.
03634       if (VT.getSimpleVT() == N2.getSimpleValueType())
03635         return N2;
03636     }
03637     break;
03638   }
03639   case ISD::BITCAST:
03640     // Fold bit_convert nodes from a type to themselves.
03641     if (N1.getValueType() == VT)
03642       return N1;
03643     break;
03644   }
03645 
03646   // Memoize node if it doesn't produce a flag.
03647   SDNode *N;
03648   SDVTList VTs = getVTList(VT);
03649   if (VT != MVT::Glue) {
03650     SDValue Ops[] = { N1, N2, N3 };
03651     FoldingSetNodeID ID;
03652     AddNodeIDNode(ID, Opcode, VTs, Ops);
03653     void *IP = nullptr;
03654     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
03655       return SDValue(E, 0);
03656 
03657     N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
03658                                           DL.getDebugLoc(), VTs, N1, N2, N3);
03659     CSEMap.InsertNode(N, IP);
03660   } else {
03661     N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
03662                                           DL.getDebugLoc(), VTs, N1, N2, N3);
03663   }
03664 
03665   InsertNode(N);
03666   return SDValue(N, 0);
03667 }
03668 
03669 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03670                               SDValue N1, SDValue N2, SDValue N3,
03671                               SDValue N4) {
03672   SDValue Ops[] = { N1, N2, N3, N4 };
03673   return getNode(Opcode, DL, VT, Ops);
03674 }
03675 
03676 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03677                               SDValue N1, SDValue N2, SDValue N3,
03678                               SDValue N4, SDValue N5) {
03679   SDValue Ops[] = { N1, N2, N3, N4, N5 };
03680   return getNode(Opcode, DL, VT, Ops);
03681 }
03682 
03683 /// getStackArgumentTokenFactor - Compute a TokenFactor to force all
03684 /// the incoming stack arguments to be loaded from the stack.
03685 SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
03686   SmallVector<SDValue, 8> ArgChains;
03687 
03688   // Include the original chain at the beginning of the list. When this is
03689   // used by target LowerCall hooks, this helps legalize find the
03690   // CALLSEQ_BEGIN node.
03691   ArgChains.push_back(Chain);
03692 
03693   // Add a chain value for each stack argument.
03694   for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(),
03695        UE = getEntryNode().getNode()->use_end(); U != UE; ++U)
03696     if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
03697       if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
03698         if (FI->getIndex() < 0)
03699           ArgChains.push_back(SDValue(L, 1));
03700 
03701   // Build a tokenfactor for all the chains.
03702   return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
03703 }
03704 
03705 /// getMemsetValue - Vectorized representation of the memset value
03706 /// operand.
03707 static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
03708                               SDLoc dl) {
03709   assert(Value.getOpcode() != ISD::UNDEF);
03710 
03711   unsigned NumBits = VT.getScalarType().getSizeInBits();
03712   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
03713     assert(C->getAPIntValue().getBitWidth() == 8);
03714     APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
03715     if (VT.isInteger())
03716       return DAG.getConstant(Val, VT);
03717     return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), VT);
03718   }
03719 
03720   Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value);
03721   if (NumBits > 8) {
03722     // Use a multiplication with 0x010101... to extend the input to the
03723     // required length.
03724     APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
03725     Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT));
03726   }
03727 
03728   return Value;
03729 }
03730 
03731 /// getMemsetStringVal - Similar to getMemsetValue. Except this is only
03732 /// used when a memcpy is turned into a memset when the source is a constant
03733 /// string ptr.
03734 static SDValue getMemsetStringVal(EVT VT, SDLoc dl, SelectionDAG &DAG,
03735                                   const TargetLowering &TLI, StringRef Str) {
03736   // Handle vector with all elements zero.
03737   if (Str.empty()) {
03738     if (VT.isInteger())
03739       return DAG.getConstant(0, VT);
03740     else if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
03741       return DAG.getConstantFP(0.0, VT);
03742     else if (VT.isVector()) {
03743       unsigned NumElts = VT.getVectorNumElements();
03744       MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
03745       return DAG.getNode(ISD::BITCAST, dl, VT,
03746                          DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(),
03747                                                              EltVT, NumElts)));
03748     } else
03749       llvm_unreachable("Expected type!");
03750   }
03751 
03752   assert(!VT.isVector() && "Can't handle vector type here!");
03753   unsigned NumVTBits = VT.getSizeInBits();
03754   unsigned NumVTBytes = NumVTBits / 8;
03755   unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size()));
03756 
03757   APInt Val(NumVTBits, 0);
03758   if (TLI.isLittleEndian()) {
03759     for (unsigned i = 0; i != NumBytes; ++i)
03760       Val |= (uint64_t)(unsigned char)Str[i] << i*8;
03761   } else {
03762     for (unsigned i = 0; i != NumBytes; ++i)
03763       Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8;
03764   }
03765 
03766   // If the "cost" of materializing the integer immediate is less than the cost
03767   // of a load, then it is cost effective to turn the load into the immediate.
03768   Type *Ty = VT.getTypeForEVT(*DAG.getContext());
03769   if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
03770     return DAG.getConstant(Val, VT);
03771   return SDValue(nullptr, 0);
03772 }
03773 
03774 /// getMemBasePlusOffset - Returns base and offset node for the
03775 ///
03776 static SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, SDLoc dl,
03777                                       SelectionDAG &DAG) {
03778   EVT VT = Base.getValueType();
03779   return DAG.getNode(ISD::ADD, dl,
03780                      VT, Base, DAG.getConstant(Offset, VT));
03781 }
03782 
03783 /// isMemSrcFromString - Returns true if memcpy source is a string constant.
03784 ///
03785 static bool isMemSrcFromString(SDValue Src, StringRef &Str) {
03786   unsigned SrcDelta = 0;
03787   GlobalAddressSDNode *G = nullptr;
03788   if (Src.getOpcode() == ISD::GlobalAddress)
03789     G = cast<GlobalAddressSDNode>(Src);
03790   else if (Src.getOpcode() == ISD::ADD &&
03791            Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
03792            Src.getOperand(1).getOpcode() == ISD::Constant) {
03793     G = cast<GlobalAddressSDNode>(Src.getOperand(0));
03794     SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue();
03795   }
03796   if (!G)
03797     return false;
03798 
03799   return getConstantStringInfo(G->getGlobal(), Str, SrcDelta, false);
03800 }
03801 
03802 /// FindOptimalMemOpLowering - Determines the optimial series memory ops
03803 /// to replace the memset / memcpy. Return true if the number of memory ops
03804 /// is below the threshold. It returns the types of the sequence of
03805 /// memory ops to perform memset / memcpy by reference.
03806 static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
03807                                      unsigned Limit, uint64_t Size,
03808                                      unsigned DstAlign, unsigned SrcAlign,
03809                                      bool IsMemset,
03810                                      bool ZeroMemset,
03811                                      bool MemcpyStrSrc,
03812                                      bool AllowOverlap,
03813                                      SelectionDAG &DAG,
03814                                      const TargetLowering &TLI) {
03815   assert((SrcAlign == 0 || SrcAlign >= DstAlign) &&
03816          "Expecting memcpy / memset source to meet alignment requirement!");
03817   // If 'SrcAlign' is zero, that means the memory operation does not need to
03818   // load the value, i.e. memset or memcpy from constant string. Otherwise,
03819   // it's the inferred alignment of the source. 'DstAlign', on the other hand,
03820   // is the specified alignment of the memory operation. If it is zero, that
03821   // means it's possible to change the alignment of the destination.
03822   // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does
03823   // not need to be loaded.
03824   EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign,
03825                                    IsMemset, ZeroMemset, MemcpyStrSrc,
03826                                    DAG.getMachineFunction());
03827 
03828   if (VT == MVT::Other) {
03829     unsigned AS = 0;
03830     if (DstAlign >= TLI.getDataLayout()->getPointerPrefAlignment(AS) ||
03831         TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign)) {
03832       VT = TLI.getPointerTy();
03833     } else {
03834       switch (DstAlign & 7) {
03835       case 0:  VT = MVT::i64; break;
03836       case 4:  VT = MVT::i32; break;
03837       case 2:  VT = MVT::i16; break;
03838       default: VT = MVT::i8;  break;
03839       }
03840     }
03841 
03842     MVT LVT = MVT::i64;
03843     while (!TLI.isTypeLegal(LVT))
03844       LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
03845     assert(LVT.isInteger());
03846 
03847     if (VT.bitsGT(LVT))
03848       VT = LVT;
03849   }
03850 
03851   unsigned NumMemOps = 0;
03852   while (Size != 0) {
03853     unsigned VTSize = VT.getSizeInBits() / 8;
03854     while (VTSize > Size) {
03855       // For now, only use non-vector load / store's for the left-over pieces.
03856       EVT NewVT = VT;
03857       unsigned NewVTSize;
03858 
03859       bool Found = false;
03860       if (VT.isVector() || VT.isFloatingPoint()) {
03861         NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
03862         if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) &&
03863             TLI.isSafeMemOpType(NewVT.getSimpleVT()))
03864           Found = true;
03865         else if (NewVT == MVT::i64 &&
03866                  TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
03867                  TLI.isSafeMemOpType(MVT::f64)) {
03868           // i64 is usually not legal on 32-bit targets, but f64 may be.
03869           NewVT = MVT::f64;
03870           Found = true;
03871         }
03872       }
03873 
03874       if (!Found) {
03875         do {
03876           NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
03877           if (NewVT == MVT::i8)
03878             break;
03879         } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT()));
03880       }
03881       NewVTSize = NewVT.getSizeInBits() / 8;
03882 
03883       // If the new VT cannot cover all of the remaining bits, then consider
03884       // issuing a (or a pair of) unaligned and overlapping load / store.
03885       // FIXME: Only does this for 64-bit or more since we don't have proper
03886       // cost model for unaligned load / store.
03887       bool Fast;
03888       unsigned AS = 0;
03889       if (NumMemOps && AllowOverlap &&
03890           VTSize >= 8 && NewVTSize < Size &&
03891           TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign, &Fast) && Fast)
03892         VTSize = Size;
03893       else {
03894         VT = NewVT;
03895         VTSize = NewVTSize;
03896       }
03897     }
03898 
03899     if (++NumMemOps > Limit)
03900       return false;
03901 
03902     MemOps.push_back(VT);
03903     Size -= VTSize;
03904   }
03905 
03906   return true;
03907 }
03908 
03909 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
03910                                        SDValue Chain, SDValue Dst,
03911                                        SDValue Src, uint64_t Size,
03912                                        unsigned Align, bool isVol,
03913                                        bool AlwaysInline,
03914                                        MachinePointerInfo DstPtrInfo,
03915                                        MachinePointerInfo SrcPtrInfo) {
03916   // Turn a memcpy of undef to nop.
03917   if (Src.getOpcode() == ISD::UNDEF)
03918     return Chain;
03919 
03920   // Expand memcpy to a series of load and store ops if the size operand falls
03921   // below a certain threshold.
03922   // TODO: In the AlwaysInline case, if the size is big then generate a loop
03923   // rather than maybe a humongous number of loads and stores.
03924   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03925   std::vector<EVT> MemOps;
03926   bool DstAlignCanChange = false;
03927   MachineFunction &MF = DAG.getMachineFunction();
03928   MachineFrameInfo *MFI = MF.getFrameInfo();
03929   bool OptSize =
03930     MF.getFunction()->getAttributes().
03931       hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
03932   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
03933   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
03934     DstAlignCanChange = true;
03935   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
03936   if (Align > SrcAlign)
03937     SrcAlign = Align;
03938   StringRef Str;
03939   bool CopyFromStr = isMemSrcFromString(Src, Str);
03940   bool isZeroStr = CopyFromStr && Str.empty();
03941   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
03942 
03943   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
03944                                 (DstAlignCanChange ? 0 : Align),
03945                                 (isZeroStr ? 0 : SrcAlign),
03946                                 false, false, CopyFromStr, true, DAG, TLI))
03947     return SDValue();
03948 
03949   if (DstAlignCanChange) {
03950     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
03951     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
03952 
03953     // Don't promote to an alignment that would require dynamic stack
03954     // realignment.
03955     const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
03956     if (!TRI->needsStackRealignment(MF))
03957        while (NewAlign > Align &&
03958              TLI.getDataLayout()->exceedsNaturalStackAlignment(NewAlign))
03959           NewAlign /= 2;
03960 
03961     if (NewAlign > Align) {
03962       // Give the stack frame object a larger alignment if needed.
03963       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
03964         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
03965       Align = NewAlign;
03966     }
03967   }
03968 
03969   SmallVector<SDValue, 8> OutChains;
03970   unsigned NumMemOps = MemOps.size();
03971   uint64_t SrcOff = 0, DstOff = 0;
03972   for (unsigned i = 0; i != NumMemOps; ++i) {
03973     EVT VT = MemOps[i];
03974     unsigned VTSize = VT.getSizeInBits() / 8;
03975     SDValue Value, Store;
03976 
03977     if (VTSize > Size) {
03978       // Issuing an unaligned load / store pair  that overlaps with the previous
03979       // pair. Adjust the offset accordingly.
03980       assert(i == NumMemOps-1 && i != 0);
03981       SrcOff -= VTSize - Size;
03982       DstOff -= VTSize - Size;
03983     }
03984 
03985     if (CopyFromStr &&
03986         (isZeroStr || (VT.isInteger() && !VT.isVector()))) {
03987       // It's unlikely a store of a vector immediate can be done in a single
03988       // instruction. It would require a load from a constantpool first.
03989       // We only handle zero vectors here.
03990       // FIXME: Handle other cases where store of vector immediate is done in
03991       // a single instruction.
03992       Value = getMemsetStringVal(VT, dl, DAG, TLI, Str.substr(SrcOff));
03993       if (Value.getNode())
03994         Store = DAG.getStore(Chain, dl, Value,
03995                              getMemBasePlusOffset(Dst, DstOff, dl, DAG),
03996                              DstPtrInfo.getWithOffset(DstOff), isVol,
03997                              false, Align);
03998     }
03999 
04000     if (!Store.getNode()) {
04001       // The type might not be legal for the target.  This should only happen
04002       // if the type is smaller than a legal type, as on PPC, so the right
04003       // thing to do is generate a LoadExt/StoreTrunc pair.  These simplify
04004       // to Load/Store if NVT==VT.
04005       // FIXME does the case above also need this?
04006       EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
04007       assert(NVT.bitsGE(VT));
04008       Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain,
04009                              getMemBasePlusOffset(Src, SrcOff, dl, DAG),
04010                              SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false,
04011                              false, MinAlign(SrcAlign, SrcOff));
04012       Store = DAG.getTruncStore(Chain, dl, Value,
04013                                 getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04014                                 DstPtrInfo.getWithOffset(DstOff), VT, isVol,
04015                                 false, Align);
04016     }
04017     OutChains.push_back(Store);
04018     SrcOff += VTSize;
04019     DstOff += VTSize;
04020     Size -= VTSize;
04021   }
04022 
04023   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04024 }
04025 
04026 static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
04027                                         SDValue Chain, SDValue Dst,
04028                                         SDValue Src, uint64_t Size,
04029                                         unsigned Align,  bool isVol,
04030                                         bool AlwaysInline,
04031                                         MachinePointerInfo DstPtrInfo,
04032                                         MachinePointerInfo SrcPtrInfo) {
04033   // Turn a memmove of undef to nop.
04034   if (Src.getOpcode() == ISD::UNDEF)
04035     return Chain;
04036 
04037   // Expand memmove to a series of load and store ops if the size operand falls
04038   // below a certain threshold.
04039   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04040   std::vector<EVT> MemOps;
04041   bool DstAlignCanChange = false;
04042   MachineFunction &MF = DAG.getMachineFunction();
04043   MachineFrameInfo *MFI = MF.getFrameInfo();
04044   bool OptSize = MF.getFunction()->getAttributes().
04045     hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
04046   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
04047   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
04048     DstAlignCanChange = true;
04049   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
04050   if (Align > SrcAlign)
04051     SrcAlign = Align;
04052   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
04053 
04054   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
04055                                 (DstAlignCanChange ? 0 : Align), SrcAlign,
04056                                 false, false, false, false, DAG, TLI))
04057     return SDValue();
04058 
04059   if (DstAlignCanChange) {
04060     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
04061     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
04062     if (NewAlign > Align) {
04063       // Give the stack frame object a larger alignment if needed.
04064       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
04065         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
04066       Align = NewAlign;
04067     }
04068   }
04069 
04070   uint64_t SrcOff = 0, DstOff = 0;
04071   SmallVector<SDValue, 8> LoadValues;
04072   SmallVector<SDValue, 8> LoadChains;
04073   SmallVector<SDValue, 8> OutChains;
04074   unsigned NumMemOps = MemOps.size();
04075   for (unsigned i = 0; i < NumMemOps; i++) {
04076     EVT VT = MemOps[i];
04077     unsigned VTSize = VT.getSizeInBits() / 8;
04078     SDValue Value;
04079 
04080     Value = DAG.getLoad(VT, dl, Chain,
04081                         getMemBasePlusOffset(Src, SrcOff, dl, DAG),
04082                         SrcPtrInfo.getWithOffset(SrcOff), isVol,
04083                         false, false, SrcAlign);
04084     LoadValues.push_back(Value);
04085     LoadChains.push_back(Value.getValue(1));
04086     SrcOff += VTSize;
04087   }
04088   Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
04089   OutChains.clear();
04090   for (unsigned i = 0; i < NumMemOps; i++) {
04091     EVT VT = MemOps[i];
04092     unsigned VTSize = VT.getSizeInBits() / 8;
04093     SDValue Store;
04094 
04095     Store = DAG.getStore(Chain, dl, LoadValues[i],
04096                          getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04097                          DstPtrInfo.getWithOffset(DstOff), isVol, false, Align);
04098     OutChains.push_back(Store);
04099     DstOff += VTSize;
04100   }
04101 
04102   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04103 }
04104 
04105 /// \brief Lower the call to 'memset' intrinsic function into a series of store
04106 /// operations.
04107 ///
04108 /// \param DAG Selection DAG where lowered code is placed.
04109 /// \param dl Link to corresponding IR location.
04110 /// \param Chain Control flow dependency.
04111 /// \param Dst Pointer to destination memory location.
04112 /// \param Src Value of byte to write into the memory.
04113 /// \param Size Number of bytes to write.
04114 /// \param Align Alignment of the destination in bytes.
04115 /// \param isVol True if destination is volatile.
04116 /// \param DstPtrInfo IR information on the memory pointer.
04117 /// \returns New head in the control flow, if lowering was successful, empty
04118 /// SDValue otherwise.
04119 ///
04120 /// The function tries to replace 'llvm.memset' intrinsic with several store
04121 /// operations and value calculation code. This is usually profitable for small
04122 /// memory size.
04123 static SDValue getMemsetStores(SelectionDAG &DAG, SDLoc dl,
04124                                SDValue Chain, SDValue Dst,
04125                                SDValue Src, uint64_t Size,
04126                                unsigned Align, bool isVol,
04127                                MachinePointerInfo DstPtrInfo) {
04128   // Turn a memset of undef to nop.
04129   if (Src.getOpcode() == ISD::UNDEF)
04130     return Chain;
04131 
04132   // Expand memset to a series of load/store ops if the size operand
04133   // falls below a certain threshold.
04134   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04135   std::vector<EVT> MemOps;
04136   bool DstAlignCanChange = false;
04137   MachineFunction &MF = DAG.getMachineFunction();
04138   MachineFrameInfo *MFI = MF.getFrameInfo();
04139   bool OptSize = MF.getFunction()->getAttributes().
04140     hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
04141   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
04142   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
04143     DstAlignCanChange = true;
04144   bool IsZeroVal =
04145     isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
04146   if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize),
04147                                 Size, (DstAlignCanChange ? 0 : Align), 0,
04148                                 true, IsZeroVal, false, true, DAG, TLI))
04149     return SDValue();
04150 
04151   if (DstAlignCanChange) {
04152     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
04153     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
04154     if (NewAlign > Align) {
04155       // Give the stack frame object a larger alignment if needed.
04156       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
04157         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
04158       Align = NewAlign;
04159     }
04160   }
04161 
04162   SmallVector<SDValue, 8> OutChains;
04163   uint64_t DstOff = 0;
04164   unsigned NumMemOps = MemOps.size();
04165 
04166   // Find the largest store and generate the bit pattern for it.
04167   EVT LargestVT = MemOps[0];
04168   for (unsigned i = 1; i < NumMemOps; i++)
04169     if (MemOps[i].bitsGT(LargestVT))
04170       LargestVT = MemOps[i];
04171   SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
04172 
04173   for (unsigned i = 0; i < NumMemOps; i++) {
04174     EVT VT = MemOps[i];
04175     unsigned VTSize = VT.getSizeInBits() / 8;
04176     if (VTSize > Size) {
04177       // Issuing an unaligned load / store pair  that overlaps with the previous
04178       // pair. Adjust the offset accordingly.
04179       assert(i == NumMemOps-1 && i != 0);
04180       DstOff -= VTSize - Size;
04181     }
04182 
04183     // If this store is smaller than the largest store see whether we can get
04184     // the smaller value for free with a truncate.
04185     SDValue Value = MemSetValue;
04186     if (VT.bitsLT(LargestVT)) {
04187       if (!LargestVT.isVector() && !VT.isVector() &&
04188           TLI.isTruncateFree(LargestVT, VT))
04189         Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
04190       else
04191         Value = getMemsetValue(Src, VT, DAG, dl);
04192     }
04193     assert(Value.getValueType() == VT && "Value with wrong type.");
04194     SDValue Store = DAG.getStore(Chain, dl, Value,
04195                                  getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04196                                  DstPtrInfo.getWithOffset(DstOff),
04197                                  isVol, false, Align);
04198     OutChains.push_back(Store);
04199     DstOff += VT.getSizeInBits() / 8;
04200     Size -= VTSize;
04201   }
04202 
04203   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04204 }
04205 
04206 SDValue SelectionDAG::getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst,
04207                                 SDValue Src, SDValue Size,
04208                                 unsigned Align, bool isVol, bool AlwaysInline,
04209                                 MachinePointerInfo DstPtrInfo,
04210                                 MachinePointerInfo SrcPtrInfo) {
04211   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04212 
04213   // Check to see if we should lower the memcpy to loads and stores first.
04214   // For cases within the target-specified limits, this is the best choice.
04215   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04216   if (ConstantSize) {
04217     // Memcpy with size zero? Just return the original chain.
04218     if (ConstantSize->isNullValue())
04219       return Chain;
04220 
04221     SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
04222                                              ConstantSize->getZExtValue(),Align,
04223                                 isVol, false, DstPtrInfo, SrcPtrInfo);
04224     if (Result.getNode())
04225       return Result;
04226   }
04227 
04228   // Then check to see if we should lower the memcpy with target-specific
04229   // code. If the target chooses to do this, this is the next best.
04230   SDValue Result =
04231       TSI->EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align,
04232                                    isVol, AlwaysInline, DstPtrInfo, SrcPtrInfo);
04233   if (Result.getNode())
04234     return Result;
04235 
04236   // If we really need inline code and the target declined to provide it,
04237   // use a (potentially long) sequence of loads and stores.
04238   if (AlwaysInline) {
04239     assert(ConstantSize && "AlwaysInline requires a constant size!");
04240     return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
04241                                    ConstantSize->getZExtValue(), Align, isVol,
04242                                    true, DstPtrInfo, SrcPtrInfo);
04243   }
04244 
04245   // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
04246   // memcpy is not guaranteed to be safe. libc memcpys aren't required to
04247   // respect volatile, so they may do things like read or write memory
04248   // beyond the given memory regions. But fixing this isn't easy, and most
04249   // people don't care.
04250 
04251   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
04252 
04253   // Emit a library call.
04254   TargetLowering::ArgListTy Args;
04255   TargetLowering::ArgListEntry Entry;
04256   Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
04257   Entry.Node = Dst; Args.push_back(Entry);
04258   Entry.Node = Src; Args.push_back(Entry);
04259   Entry.Node = Size; Args.push_back(Entry);
04260   // FIXME: pass in SDLoc
04261   TargetLowering::CallLoweringInfo CLI(*this);
04262   CLI.setDebugLoc(dl).setChain(Chain)
04263     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
04264                Type::getVoidTy(*getContext()),
04265                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
04266                                  TLI->getPointerTy()), std::move(Args), 0)
04267     .setDiscardResult();
04268   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04269 
04270   return CallResult.second;
04271 }
04272 
04273 SDValue SelectionDAG::getMemmove(SDValue Chain, SDLoc dl, SDValue Dst,
04274                                  SDValue Src, SDValue Size,
04275                                  unsigned Align, bool isVol,
04276                                  MachinePointerInfo DstPtrInfo,
04277                                  MachinePointerInfo SrcPtrInfo) {
04278   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04279 
04280   // Check to see if we should lower the memmove to loads and stores first.
04281   // For cases within the target-specified limits, this is the best choice.
04282   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04283   if (ConstantSize) {
04284     // Memmove with size zero? Just return the original chain.
04285     if (ConstantSize->isNullValue())
04286       return Chain;
04287 
04288     SDValue Result =
04289       getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src,
04290                                ConstantSize->getZExtValue(), Align, isVol,
04291                                false, DstPtrInfo, SrcPtrInfo);
04292     if (Result.getNode())
04293       return Result;
04294   }
04295 
04296   // Then check to see if we should lower the memmove with target-specific
04297   // code. If the target chooses to do this, this is the next best.
04298   SDValue Result = TSI->EmitTargetCodeForMemmove(
04299       *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo, SrcPtrInfo);
04300   if (Result.getNode())
04301     return Result;
04302 
04303   // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
04304   // not be safe.  See memcpy above for more details.
04305 
04306   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
04307 
04308   // Emit a library call.
04309   TargetLowering::ArgListTy Args;
04310   TargetLowering::ArgListEntry Entry;
04311   Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
04312   Entry.Node = Dst; Args.push_back(Entry);
04313   Entry.Node = Src; Args.push_back(Entry);
04314   Entry.Node = Size; Args.push_back(Entry);
04315   // FIXME:  pass in SDLoc
04316   TargetLowering::CallLoweringInfo CLI(*this);
04317   CLI.setDebugLoc(dl).setChain(Chain)
04318     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
04319                Type::getVoidTy(*getContext()),
04320                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
04321                                  TLI->getPointerTy()), std::move(Args), 0)
04322     .setDiscardResult();
04323   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04324 
04325   return CallResult.second;
04326 }
04327 
04328 SDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst,
04329                                 SDValue Src, SDValue Size,
04330                                 unsigned Align, bool isVol,
04331                                 MachinePointerInfo DstPtrInfo) {
04332   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04333 
04334   // Check to see if we should lower the memset to stores first.
04335   // For cases within the target-specified limits, this is the best choice.
04336   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04337   if (ConstantSize) {
04338     // Memset with size zero? Just return the original chain.
04339     if (ConstantSize->isNullValue())
04340       return Chain;
04341 
04342     SDValue Result =
04343       getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
04344                       Align, isVol, DstPtrInfo);
04345 
04346     if (Result.getNode())
04347       return Result;
04348   }
04349 
04350   // Then check to see if we should lower the memset with target-specific
04351   // code. If the target chooses to do this, this is the next best.
04352   SDValue Result = TSI->EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src,
04353                                                 Size, Align, isVol, DstPtrInfo);
04354   if (Result.getNode())
04355     return Result;
04356 
04357   // Emit a library call.
04358   const TargetLowering *TLI = TM.getSubtargetImpl()->getTargetLowering();
04359   Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(*getContext());
04360   TargetLowering::ArgListTy Args;
04361   TargetLowering::ArgListEntry Entry;
04362   Entry.Node = Dst; Entry.Ty = IntPtrTy;
04363   Args.push_back(Entry);
04364   // Extend or truncate the argument to be an i32 value for the call.
04365   if (Src.getValueType().bitsGT(MVT::i32))
04366     Src = getNode(ISD::TRUNCATE, dl, MVT::i32, Src);
04367   else
04368     Src = getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Src);
04369   Entry.Node = Src;
04370   Entry.Ty = Type::getInt32Ty(*getContext());
04371   Entry.isSExt = true;
04372   Args.push_back(Entry);
04373   Entry.Node = Size;
04374   Entry.Ty = IntPtrTy;
04375   Entry.isSExt = false;
04376   Args.push_back(Entry);
04377 
04378   // FIXME: pass in SDLoc
04379   TargetLowering::CallLoweringInfo CLI(*this);
04380   CLI.setDebugLoc(dl).setChain(Chain)
04381     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
04382                Type::getVoidTy(*getContext()),
04383                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
04384                                  TLI->getPointerTy()), std::move(Args), 0)
04385     .setDiscardResult();
04386 
04387   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04388   return CallResult.second;
04389 }
04390 
04391 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04392                                 SDVTList VTList, ArrayRef<SDValue> Ops,
04393                                 MachineMemOperand *MMO,
04394                                 AtomicOrdering SuccessOrdering,
04395                                 AtomicOrdering FailureOrdering,
04396                                 SynchronizationScope SynchScope) {
04397   FoldingSetNodeID ID;
04398   ID.AddInteger(MemVT.getRawBits());
04399   AddNodeIDNode(ID, Opcode, VTList, Ops);
04400   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04401   void* IP = nullptr;
04402   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04403     cast<AtomicSDNode>(E)->refineAlignment(MMO);
04404     return SDValue(E, 0);
04405   }
04406 
04407   // Allocate the operands array for the node out of the BumpPtrAllocator, since
04408   // SDNode doesn't have access to it.  This memory will be "leaked" when
04409   // the node is deallocated, but recovered when the allocator is released.
04410   // If the number of operands is less than 5 we use AtomicSDNode's internal
04411   // storage.
04412   unsigned NumOps = Ops.size();
04413   SDUse *DynOps = NumOps > 4 ? OperandAllocator.Allocate<SDUse>(NumOps)
04414                              : nullptr;
04415 
04416   SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(),
04417                                                dl.getDebugLoc(), VTList, MemVT,
04418                                                Ops.data(), DynOps, NumOps, MMO,
04419                                                SuccessOrdering, FailureOrdering,
04420                                                SynchScope);
04421   CSEMap.InsertNode(N, IP);
04422   InsertNode(N);
04423   return SDValue(N, 0);
04424 }
04425 
04426 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04427                                 SDVTList VTList, ArrayRef<SDValue> Ops,
04428                                 MachineMemOperand *MMO,
04429                                 AtomicOrdering Ordering,
04430                                 SynchronizationScope SynchScope) {
04431   return getAtomic(Opcode, dl, MemVT, VTList, Ops, MMO, Ordering,
04432                    Ordering, SynchScope);
04433 }
04434 
04435 SDValue SelectionDAG::getAtomicCmpSwap(
04436     unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs, SDValue Chain,
04437     SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo,
04438     unsigned Alignment, AtomicOrdering SuccessOrdering,
04439     AtomicOrdering FailureOrdering, SynchronizationScope SynchScope) {
04440   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
04441          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
04442   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
04443 
04444   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04445     Alignment = getEVTAlignment(MemVT);
04446 
04447   MachineFunction &MF = getMachineFunction();
04448 
04449   // FIXME: Volatile isn't really correct; we should keep track of atomic
04450   // orderings in the memoperand.
04451   unsigned Flags = MachineMemOperand::MOVolatile;
04452   Flags |= MachineMemOperand::MOLoad;
04453   Flags |= MachineMemOperand::MOStore;
04454 
04455   MachineMemOperand *MMO =
04456     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
04457 
04458   return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO,
04459                           SuccessOrdering, FailureOrdering, SynchScope);
04460 }
04461 
04462 SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, SDLoc dl, EVT MemVT,
04463                                        SDVTList VTs, SDValue Chain, SDValue Ptr,
04464                                        SDValue Cmp, SDValue Swp,
04465                                        MachineMemOperand *MMO,
04466                                        AtomicOrdering SuccessOrdering,
04467                                        AtomicOrdering FailureOrdering,
04468                                        SynchronizationScope SynchScope) {
04469   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
04470          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
04471   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
04472 
04473   SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
04474   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO,
04475                    SuccessOrdering, FailureOrdering, SynchScope);
04476 }
04477 
04478 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04479                                 SDValue Chain,
04480                                 SDValue Ptr, SDValue Val,
04481                                 const Value* PtrVal,
04482                                 unsigned Alignment,
04483                                 AtomicOrdering Ordering,
04484                                 SynchronizationScope SynchScope) {
04485   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04486     Alignment = getEVTAlignment(MemVT);
04487 
04488   MachineFunction &MF = getMachineFunction();
04489   // An atomic store does not load. An atomic load does not store.
04490   // (An atomicrmw obviously both loads and stores.)
04491   // For now, atomics are considered to be volatile always, and they are
04492   // chained as such.
04493   // FIXME: Volatile isn't really correct; we should keep track of atomic
04494   // orderings in the memoperand.
04495   unsigned Flags = MachineMemOperand::MOVolatile;
04496   if (Opcode != ISD::ATOMIC_STORE)
04497     Flags |= MachineMemOperand::MOLoad;
04498   if (Opcode != ISD::ATOMIC_LOAD)
04499     Flags |= MachineMemOperand::MOStore;
04500 
04501   MachineMemOperand *MMO =
04502     MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
04503                             MemVT.getStoreSize(), Alignment);
04504 
04505   return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO,
04506                    Ordering, SynchScope);
04507 }
04508 
04509 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04510                                 SDValue Chain,
04511                                 SDValue Ptr, SDValue Val,
04512                                 MachineMemOperand *MMO,
04513                                 AtomicOrdering Ordering,
04514                                 SynchronizationScope SynchScope) {
04515   assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
04516           Opcode == ISD::ATOMIC_LOAD_SUB ||
04517           Opcode == ISD::ATOMIC_LOAD_AND ||
04518           Opcode == ISD::ATOMIC_LOAD_OR ||
04519           Opcode == ISD::ATOMIC_LOAD_XOR ||
04520           Opcode == ISD::ATOMIC_LOAD_NAND ||
04521           Opcode == ISD::ATOMIC_LOAD_MIN ||
04522           Opcode == ISD::ATOMIC_LOAD_MAX ||
04523           Opcode == ISD::ATOMIC_LOAD_UMIN ||
04524           Opcode == ISD::ATOMIC_LOAD_UMAX ||
04525           Opcode == ISD::ATOMIC_SWAP ||
04526           Opcode == ISD::ATOMIC_STORE) &&
04527          "Invalid Atomic Op");
04528 
04529   EVT VT = Val.getValueType();
04530 
04531   SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
04532                                                getVTList(VT, MVT::Other);
04533   SDValue Ops[] = {Chain, Ptr, Val};
04534   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
04535 }
04536 
04537 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04538                                 EVT VT, SDValue Chain,
04539                                 SDValue Ptr,
04540                                 MachineMemOperand *MMO,
04541                                 AtomicOrdering Ordering,
04542                                 SynchronizationScope SynchScope) {
04543   assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
04544 
04545   SDVTList VTs = getVTList(VT, MVT::Other);
04546   SDValue Ops[] = {Chain, Ptr};
04547   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
04548 }
04549 
04550 /// getMergeValues - Create a MERGE_VALUES node from the given operands.
04551 SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, SDLoc dl) {
04552   if (Ops.size() == 1)
04553     return Ops[0];
04554 
04555   SmallVector<EVT, 4> VTs;
04556   VTs.reserve(Ops.size());
04557   for (unsigned i = 0; i < Ops.size(); ++i)
04558     VTs.push_back(Ops[i].getValueType());
04559   return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
04560 }
04561 
04562 SDValue
04563 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
04564                                   ArrayRef<SDValue> Ops,
04565                                   EVT MemVT, MachinePointerInfo PtrInfo,
04566                                   unsigned Align, bool Vol,
04567                                   bool ReadMem, bool WriteMem, unsigned Size) {
04568   if (Align == 0)  // Ensure that codegen never sees alignment 0
04569     Align = getEVTAlignment(MemVT);
04570 
04571   MachineFunction &MF = getMachineFunction();
04572   unsigned Flags = 0;
04573   if (WriteMem)
04574     Flags |= MachineMemOperand::MOStore;
04575   if (ReadMem)
04576     Flags |= MachineMemOperand::MOLoad;
04577   if (Vol)
04578     Flags |= MachineMemOperand::MOVolatile;
04579   if (!Size)
04580     Size = MemVT.getStoreSize();
04581   MachineMemOperand *MMO =
04582     MF.getMachineMemOperand(PtrInfo, Flags, Size, Align);
04583 
04584   return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
04585 }
04586 
04587 SDValue
04588 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
04589                                   ArrayRef<SDValue> Ops, EVT MemVT,
04590                                   MachineMemOperand *MMO) {
04591   assert((Opcode == ISD::INTRINSIC_VOID ||
04592           Opcode == ISD::INTRINSIC_W_CHAIN ||
04593           Opcode == ISD::PREFETCH ||
04594           Opcode == ISD::LIFETIME_START ||
04595           Opcode == ISD::LIFETIME_END ||
04596           (Opcode <= INT_MAX &&
04597            (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
04598          "Opcode is not a memory-accessing opcode!");
04599 
04600   // Memoize the node unless it returns a flag.
04601   MemIntrinsicSDNode *N;
04602   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
04603     FoldingSetNodeID ID;
04604     AddNodeIDNode(ID, Opcode, VTList, Ops);
04605     ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04606     void *IP = nullptr;
04607     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04608       cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
04609       return SDValue(E, 0);
04610     }
04611 
04612     N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
04613                                                dl.getDebugLoc(), VTList, Ops,
04614                                                MemVT, MMO);
04615     CSEMap.InsertNode(N, IP);
04616   } else {
04617     N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
04618                                                dl.getDebugLoc(), VTList, Ops,
04619                                                MemVT, MMO);
04620   }
04621   InsertNode(N);
04622   return SDValue(N, 0);
04623 }
04624 
04625 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
04626 /// MachinePointerInfo record from it.  This is particularly useful because the
04627 /// code generator has many cases where it doesn't bother passing in a
04628 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
04629 static MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) {
04630   // If this is FI+Offset, we can model it.
04631   if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
04632     return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset);
04633 
04634   // If this is (FI+Offset1)+Offset2, we can model it.
04635   if (Ptr.getOpcode() != ISD::ADD ||
04636       !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
04637       !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
04638     return MachinePointerInfo();
04639 
04640   int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
04641   return MachinePointerInfo::getFixedStack(FI, Offset+
04642                        cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
04643 }
04644 
04645 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
04646 /// MachinePointerInfo record from it.  This is particularly useful because the
04647 /// code generator has many cases where it doesn't bother passing in a
04648 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
04649 static MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) {
04650   // If the 'Offset' value isn't a constant, we can't handle this.
04651   if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
04652     return InferPointerInfo(Ptr, OffsetNode->getSExtValue());
04653   if (OffsetOp.getOpcode() == ISD::UNDEF)
04654     return InferPointerInfo(Ptr);
04655   return MachinePointerInfo();
04656 }
04657 
04658 
04659 SDValue
04660 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
04661                       EVT VT, SDLoc dl, SDValue Chain,
04662                       SDValue Ptr, SDValue Offset,
04663                       MachinePointerInfo PtrInfo, EVT MemVT,
04664                       bool isVolatile, bool isNonTemporal, bool isInvariant,
04665                       unsigned Alignment, const AAMDNodes &AAInfo,
04666                       const MDNode *Ranges) {
04667   assert(Chain.getValueType() == MVT::Other &&
04668         "Invalid chain type");
04669   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04670     Alignment = getEVTAlignment(VT);
04671 
04672   unsigned Flags = MachineMemOperand::MOLoad;
04673   if (isVolatile)
04674     Flags |= MachineMemOperand::MOVolatile;
04675   if (isNonTemporal)
04676     Flags |= MachineMemOperand::MONonTemporal;
04677   if (isInvariant)
04678     Flags |= MachineMemOperand::MOInvariant;
04679 
04680   // If we don't have a PtrInfo, infer the trivial frame index case to simplify
04681   // clients.
04682   if (PtrInfo.V.isNull())
04683     PtrInfo = InferPointerInfo(Ptr, Offset);
04684 
04685   MachineFunction &MF = getMachineFunction();
04686   MachineMemOperand *MMO =
04687     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment,
04688                             AAInfo, Ranges);
04689   return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
04690 }
04691 
04692 SDValue
04693 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
04694                       EVT VT, SDLoc dl, SDValue Chain,
04695                       SDValue Ptr, SDValue Offset, EVT MemVT,
04696                       MachineMemOperand *MMO) {
04697   if (VT == MemVT) {
04698     ExtType = ISD::NON_EXTLOAD;
04699   } else if (ExtType == ISD::NON_EXTLOAD) {
04700     assert(VT == MemVT && "Non-extending load from different memory type!");
04701   } else {
04702     // Extending load.
04703     assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
04704            "Should only be an extending load, not truncating!");
04705     assert(VT.isInteger() == MemVT.isInteger() &&
04706            "Cannot convert from FP to Int or Int -> FP!");
04707     assert(VT.isVector() == MemVT.isVector() &&
04708            "Cannot use trunc store to convert to or from a vector!");
04709     assert((!VT.isVector() ||
04710             VT.getVectorNumElements() == MemVT.getVectorNumElements()) &&
04711            "Cannot use trunc store to change the number of vector elements!");
04712   }
04713 
04714   bool Indexed = AM != ISD::UNINDEXED;
04715   assert((Indexed || Offset.getOpcode() == ISD::UNDEF) &&
04716          "Unindexed load with an offset!");
04717 
04718   SDVTList VTs = Indexed ?
04719     getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
04720   SDValue Ops[] = { Chain, Ptr, Offset };
04721   FoldingSetNodeID ID;
04722   AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
04723   ID.AddInteger(MemVT.getRawBits());
04724   ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(),
04725                                      MMO->isNonTemporal(),
04726                                      MMO->isInvariant()));
04727   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04728   void *IP = nullptr;
04729   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04730     cast<LoadSDNode>(E)->refineAlignment(MMO);
04731     return SDValue(E, 0);
04732   }
04733   SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl.getIROrder(),
04734                                              dl.getDebugLoc(), VTs, AM, ExtType,
04735                                              MemVT, MMO);
04736   CSEMap.InsertNode(N, IP);
04737   InsertNode(N);
04738   return SDValue(N, 0);
04739 }
04740 
04741 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
04742                               SDValue Chain, SDValue Ptr,
04743                               MachinePointerInfo PtrInfo,
04744                               bool isVolatile, bool isNonTemporal,
04745                               bool isInvariant, unsigned Alignment,
04746                               const AAMDNodes &AAInfo,
04747                               const MDNode *Ranges) {
04748   SDValue Undef = getUNDEF(Ptr.getValueType());
04749   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
04750                  PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment,
04751                  AAInfo, Ranges);
04752 }
04753 
04754 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
04755                               SDValue Chain, SDValue Ptr,
04756                               MachineMemOperand *MMO) {
04757   SDValue Undef = getUNDEF(Ptr.getValueType());
04758   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
04759                  VT, MMO);
04760 }
04761 
04762 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
04763                                  SDValue Chain, SDValue Ptr,
04764                                  MachinePointerInfo PtrInfo, EVT MemVT,
04765                                  bool isVolatile, bool isNonTemporal,
04766                                  bool isInvariant, unsigned Alignment,
04767                                  const AAMDNodes &AAInfo) {
04768   SDValue Undef = getUNDEF(Ptr.getValueType());
04769   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
04770                  PtrInfo, MemVT, isVolatile, isNonTemporal, isInvariant,
04771                  Alignment, AAInfo);
04772 }
04773 
04774 
04775 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
04776                                  SDValue Chain, SDValue Ptr, EVT MemVT,
04777                                  MachineMemOperand *MMO) {
04778   SDValue Undef = getUNDEF(Ptr.getValueType());
04779   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
04780                  MemVT, MMO);
04781 }
04782 
04783 SDValue
04784 SelectionDAG::getIndexedLoad(SDValue OrigLoad, SDLoc dl, SDValue Base,
04785                              SDValue Offset, ISD::MemIndexedMode AM) {
04786   LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
04787   assert(LD->getOffset().getOpcode() == ISD::UNDEF &&
04788          "Load is already a indexed load!");
04789   return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
04790                  LD->getChain(), Base, Offset, LD->getPointerInfo(),
04791                  LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(),
04792                  false, LD->getAlignment());
04793 }
04794 
04795 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
04796                                SDValue Ptr, MachinePointerInfo PtrInfo,
04797                                bool isVolatile, bool isNonTemporal,
04798                                unsigned Alignment, const AAMDNodes &AAInfo) {
04799   assert(Chain.getValueType() == MVT::Other &&
04800         "Invalid chain type");
04801   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04802     Alignment = getEVTAlignment(Val.getValueType());
04803 
04804   unsigned Flags = MachineMemOperand::MOStore;
04805   if (isVolatile)
04806     Flags |= MachineMemOperand::MOVolatile;
04807   if (isNonTemporal)
04808     Flags |= MachineMemOperand::MONonTemporal;
04809 
04810   if (PtrInfo.V.isNull())
04811     PtrInfo = InferPointerInfo(Ptr);
04812 
04813   MachineFunction &MF = getMachineFunction();
04814   MachineMemOperand *MMO =
04815     MF.getMachineMemOperand(PtrInfo, Flags,
04816                             Val.getValueType().getStoreSize(), Alignment,
04817                             AAInfo);
04818 
04819   return getStore(Chain, dl, Val, Ptr, MMO);
04820 }
04821 
04822 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
04823                                SDValue Ptr, MachineMemOperand *MMO) {
04824   assert(Chain.getValueType() == MVT::Other &&
04825         "Invalid chain type");
04826   EVT VT = Val.getValueType();
04827   SDVTList VTs = getVTList(MVT::Other);
04828   SDValue Undef = getUNDEF(Ptr.getValueType());
04829   SDValue Ops[] = { Chain, Val, Ptr, Undef };
04830   FoldingSetNodeID ID;
04831   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04832   ID.AddInteger(VT.getRawBits());
04833   ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
04834                                      MMO->isNonTemporal(), MMO->isInvariant()));
04835   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04836   void *IP = nullptr;
04837   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04838     cast<StoreSDNode>(E)->refineAlignment(MMO);
04839     return SDValue(E, 0);
04840   }
04841   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04842                                               dl.getDebugLoc(), VTs,
04843                                               ISD::UNINDEXED, false, VT, MMO);
04844   CSEMap.InsertNode(N, IP);
04845   InsertNode(N);
04846   return SDValue(N, 0);
04847 }
04848 
04849 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
04850                                     SDValue Ptr, MachinePointerInfo PtrInfo,
04851                                     EVT SVT,bool isVolatile, bool isNonTemporal,
04852                                     unsigned Alignment,
04853                                     const AAMDNodes &AAInfo) {
04854   assert(Chain.getValueType() == MVT::Other &&
04855         "Invalid chain type");
04856   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04857     Alignment = getEVTAlignment(SVT);
04858 
04859   unsigned Flags = MachineMemOperand::MOStore;
04860   if (isVolatile)
04861     Flags |= MachineMemOperand::MOVolatile;
04862   if (isNonTemporal)
04863     Flags |= MachineMemOperand::MONonTemporal;
04864 
04865   if (PtrInfo.V.isNull())
04866     PtrInfo = InferPointerInfo(Ptr);
04867 
04868   MachineFunction &MF = getMachineFunction();
04869   MachineMemOperand *MMO =
04870     MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment,
04871                             AAInfo);
04872 
04873   return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
04874 }
04875 
04876 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
04877                                     SDValue Ptr, EVT SVT,
04878                                     MachineMemOperand *MMO) {
04879   EVT VT = Val.getValueType();
04880 
04881   assert(Chain.getValueType() == MVT::Other &&
04882         "Invalid chain type");
04883   if (VT == SVT)
04884     return getStore(Chain, dl, Val, Ptr, MMO);
04885 
04886   assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
04887          "Should only be a truncating store, not extending!");
04888   assert(VT.isInteger() == SVT.isInteger() &&
04889          "Can't do FP-INT conversion!");
04890   assert(VT.isVector() == SVT.isVector() &&
04891          "Cannot use trunc store to convert to or from a vector!");
04892   assert((!VT.isVector() ||
04893           VT.getVectorNumElements() == SVT.getVectorNumElements()) &&
04894          "Cannot use trunc store to change the number of vector elements!");
04895 
04896   SDVTList VTs = getVTList(MVT::Other);
04897   SDValue Undef = getUNDEF(Ptr.getValueType());
04898   SDValue Ops[] = { Chain, Val, Ptr, Undef };
04899   FoldingSetNodeID ID;
04900   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04901   ID.AddInteger(SVT.getRawBits());
04902   ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(),
04903                                      MMO->isNonTemporal(), MMO->isInvariant()));
04904   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04905   void *IP = nullptr;
04906   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04907     cast<StoreSDNode>(E)->refineAlignment(MMO);
04908     return SDValue(E, 0);
04909   }
04910   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04911                                               dl.getDebugLoc(), VTs,
04912                                               ISD::UNINDEXED, true, SVT, MMO);
04913   CSEMap.InsertNode(N, IP);
04914   InsertNode(N);
04915   return SDValue(N, 0);
04916 }
04917 
04918 SDValue
04919 SelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base,
04920                               SDValue Offset, ISD::MemIndexedMode AM) {
04921   StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
04922   assert(ST->getOffset().getOpcode() == ISD::UNDEF &&
04923          "Store is already a indexed store!");
04924   SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
04925   SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
04926   FoldingSetNodeID ID;
04927   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04928   ID.AddInteger(ST->getMemoryVT().getRawBits());
04929   ID.AddInteger(ST->getRawSubclassData());
04930   ID.AddInteger(ST->getPointerInfo().getAddrSpace());
04931   void *IP = nullptr;
04932   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
04933     return SDValue(E, 0);
04934 
04935   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04936                                               dl.getDebugLoc(), VTs, AM,
04937                                               ST->isTruncatingStore(),
04938                                               ST->getMemoryVT(),
04939                                               ST->getMemOperand());
04940   CSEMap.InsertNode(N, IP);
04941   InsertNode(N);
04942   return SDValue(N, 0);
04943 }
04944 
04945 SDValue SelectionDAG::getVAArg(EVT VT, SDLoc dl,
04946                                SDValue Chain, SDValue Ptr,
04947                                SDValue SV,
04948                                unsigned Align) {
04949   SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) };
04950   return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
04951 }
04952 
04953 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
04954                               ArrayRef<SDUse> Ops) {
04955   switch (Ops.size()) {
04956   case 0: return getNode(Opcode, DL, VT);
04957   case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
04958   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
04959   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
04960   default: break;
04961   }
04962 
04963   // Copy from an SDUse array into an SDValue array for use with
04964   // the regular getNode logic.
04965   SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
04966   return getNode(Opcode, DL, VT, NewOps);
04967 }
04968 
04969 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
04970                               ArrayRef<SDValue> Ops) {
04971   unsigned NumOps = Ops.size();
04972   switch (NumOps) {
04973   case 0: return getNode(Opcode, DL, VT);
04974   case 1: return getNode(Opcode, DL, VT, Ops[0]);
04975   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
04976   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
04977   default: break;
04978   }
04979 
04980   switch (Opcode) {
04981   default: break;
04982   case ISD::SELECT_CC: {
04983     assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
04984     assert(Ops[0].getValueType() == Ops[1].getValueType() &&
04985            "LHS and RHS of condition must have same type!");
04986     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
04987            "True and False arms of SelectCC must have same type!");
04988     assert(Ops[2].getValueType() == VT &&
04989            "select_cc node must be of same type as true and false value!");
04990     break;
04991   }
04992   case ISD::BR_CC: {
04993     assert(NumOps == 5 && "BR_CC takes 5 operands!");
04994     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
04995            "LHS/RHS of comparison should match types!");
04996     break;
04997   }
04998   }
04999 
05000   // Memoize nodes.
05001   SDNode *N;
05002   SDVTList VTs = getVTList(VT);
05003 
05004   if (VT != MVT::Glue) {
05005     FoldingSetNodeID ID;
05006     AddNodeIDNode(ID, Opcode, VTs, Ops);
05007     void *IP = nullptr;
05008 
05009     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05010       return SDValue(E, 0);
05011 
05012     N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05013                                    VTs, Ops);
05014     CSEMap.InsertNode(N, IP);
05015   } else {
05016     N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05017                                    VTs, Ops);
05018   }
05019 
05020   InsertNode(N);
05021   return SDValue(N, 0);
05022 }
05023 
05024 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
05025                               ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
05026   return getNode(Opcode, DL, getVTList(ResultTys), Ops);
05027 }
05028 
05029 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05030                               ArrayRef<SDValue> Ops) {
05031   if (VTList.NumVTs == 1)
05032     return getNode(Opcode, DL, VTList.VTs[0], Ops);
05033 
05034 #if 0
05035   switch (Opcode) {
05036   // FIXME: figure out how to safely handle things like
05037   // int foo(int x) { return 1 << (x & 255); }
05038   // int bar() { return foo(256); }
05039   case ISD::SRA_PARTS:
05040   case ISD::SRL_PARTS:
05041   case ISD::SHL_PARTS:
05042     if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
05043         cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
05044       return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
05045     else if (N3.getOpcode() == ISD::AND)
05046       if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
05047         // If the and is only masking out bits that cannot effect the shift,
05048         // eliminate the and.
05049         unsigned NumBits = VT.getScalarType().getSizeInBits()*2;
05050         if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
05051           return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
05052       }
05053     break;
05054   }
05055 #endif
05056 
05057   // Memoize the node unless it returns a flag.
05058   SDNode *N;
05059   unsigned NumOps = Ops.size();
05060   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
05061     FoldingSetNodeID ID;
05062     AddNodeIDNode(ID, Opcode, VTList, Ops);
05063     void *IP = nullptr;
05064     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05065       return SDValue(E, 0);
05066 
05067     if (NumOps == 1) {
05068       N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
05069                                           DL.getDebugLoc(), VTList, Ops[0]);
05070     } else if (NumOps == 2) {
05071       N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
05072                                            DL.getDebugLoc(), VTList, Ops[0],
05073                                            Ops[1]);
05074     } else if (NumOps == 3) {
05075       N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
05076                                             DL.getDebugLoc(), VTList, Ops[0],
05077                                             Ops[1], Ops[2]);
05078     } else {
05079       N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05080                                      VTList, Ops);
05081     }
05082     CSEMap.InsertNode(N, IP);
05083   } else {
05084     if (NumOps == 1) {
05085       N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
05086                                           DL.getDebugLoc(), VTList, Ops[0]);
05087     } else if (NumOps == 2) {
05088       N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
05089                                            DL.getDebugLoc(), VTList, Ops[0],
05090                                            Ops[1]);
05091     } else if (NumOps == 3) {
05092       N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
05093                                             DL.getDebugLoc(), VTList, Ops[0],
05094                                             Ops[1], Ops[2]);
05095     } else {
05096       N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05097                                      VTList, Ops);
05098     }
05099   }
05100   InsertNode(N);
05101   return SDValue(N, 0);
05102 }
05103 
05104 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList) {
05105   return getNode(Opcode, DL, VTList, None);
05106 }
05107 
05108 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05109                               SDValue N1) {
05110   SDValue Ops[] = { N1 };
05111   return getNode(Opcode, DL, VTList, Ops);
05112 }
05113 
05114 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05115                               SDValue N1, SDValue N2) {
05116   SDValue Ops[] = { N1, N2 };
05117   return getNode(Opcode, DL, VTList, Ops);
05118 }
05119 
05120 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05121                               SDValue N1, SDValue N2, SDValue N3) {
05122   SDValue Ops[] = { N1, N2, N3 };
05123   return getNode(Opcode, DL, VTList, Ops);
05124 }
05125 
05126 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05127                               SDValue N1, SDValue N2, SDValue N3,
05128                               SDValue N4) {
05129   SDValue Ops[] = { N1, N2, N3, N4 };
05130   return getNode(Opcode, DL, VTList, Ops);
05131 }
05132 
05133 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05134                               SDValue N1, SDValue N2, SDValue N3,
05135                               SDValue N4, SDValue N5) {
05136   SDValue Ops[] = { N1, N2, N3, N4, N5 };
05137   return getNode(Opcode, DL, VTList, Ops);
05138 }
05139 
05140 SDVTList SelectionDAG::getVTList(EVT VT) {
05141   return makeVTList(SDNode::getValueTypeList(VT), 1);
05142 }
05143 
05144 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
05145   FoldingSetNodeID ID;
05146   ID.AddInteger(2U);
05147   ID.AddInteger(VT1.getRawBits());
05148   ID.AddInteger(VT2.getRawBits());
05149 
05150   void *IP = nullptr;
05151   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05152   if (!Result) {
05153     EVT *Array = Allocator.Allocate<EVT>(2);
05154     Array[0] = VT1;
05155     Array[1] = VT2;
05156     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
05157     VTListMap.InsertNode(Result, IP);
05158   }
05159   return Result->getSDVTList();
05160 }
05161 
05162 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
05163   FoldingSetNodeID ID;
05164   ID.AddInteger(3U);
05165   ID.AddInteger(VT1.getRawBits());
05166   ID.AddInteger(VT2.getRawBits());
05167   ID.AddInteger(VT3.getRawBits());
05168 
05169   void *IP = nullptr;
05170   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05171   if (!Result) {
05172     EVT *Array = Allocator.Allocate<EVT>(3);
05173     Array[0] = VT1;
05174     Array[1] = VT2;
05175     Array[2] = VT3;
05176     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
05177     VTListMap.InsertNode(Result, IP);
05178   }
05179   return Result->getSDVTList();
05180 }
05181 
05182 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
05183   FoldingSetNodeID ID;
05184   ID.AddInteger(4U);
05185   ID.AddInteger(VT1.getRawBits());
05186   ID.AddInteger(VT2.getRawBits());
05187   ID.AddInteger(VT3.getRawBits());
05188   ID.AddInteger(VT4.getRawBits());
05189 
05190   void *IP = nullptr;
05191   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05192   if (!Result) {
05193     EVT *Array = Allocator.Allocate<EVT>(4);
05194     Array[0] = VT1;
05195     Array[1] = VT2;
05196     Array[2] = VT3;
05197     Array[3] = VT4;
05198     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
05199     VTListMap.InsertNode(Result, IP);
05200   }
05201   return Result->getSDVTList();
05202 }
05203 
05204 SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
05205   unsigned NumVTs = VTs.size();
05206   FoldingSetNodeID ID;
05207   ID.AddInteger(NumVTs);
05208   for (unsigned index = 0; index < NumVTs; index++) {
05209     ID.AddInteger(VTs[index].getRawBits());
05210   }
05211 
05212   void *IP = nullptr;
05213   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05214   if (!Result) {
05215     EVT *Array = Allocator.Allocate<EVT>(NumVTs);
05216     std::copy(VTs.begin(), VTs.end(), Array);
05217     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
05218     VTListMap.InsertNode(Result, IP);
05219   }
05220   return Result->getSDVTList();
05221 }
05222 
05223 
05224 /// UpdateNodeOperands - *Mutate* the specified node in-place to have the
05225 /// specified operands.  If the resultant node already exists in the DAG,
05226 /// this does not modify the specified node, instead it returns the node that
05227 /// already exists.  If the resultant node does not exist in the DAG, the
05228 /// input node is returned.  As a degenerate case, if you specify the same
05229 /// input operands as the node already has, the input node is returned.
05230 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
05231   assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
05232 
05233   // Check to see if there is no change.
05234   if (Op == N->getOperand(0)) return N;
05235 
05236   // See if the modified node already exists.
05237   void *InsertPos = nullptr;
05238   if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
05239     return Existing;
05240 
05241   // Nope it doesn't.  Remove the node from its current place in the maps.
05242   if (InsertPos)
05243     if (!RemoveNodeFromCSEMaps(N))
05244       InsertPos = nullptr;
05245 
05246   // Now we update the operands.
05247   N->OperandList[0].set(Op);
05248 
05249   // If this gets put into a CSE map, add it.
05250   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05251   return N;
05252 }
05253 
05254 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
05255   assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
05256 
05257   // Check to see if there is no change.
05258   if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
05259     return N;   // No operands changed, just return the input node.
05260 
05261   // See if the modified node already exists.
05262   void *InsertPos = nullptr;
05263   if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
05264     return Existing;
05265 
05266   // Nope it doesn't.  Remove the node from its current place in the maps.
05267   if (InsertPos)
05268     if (!RemoveNodeFromCSEMaps(N))
05269       InsertPos = nullptr;
05270 
05271   // Now we update the operands.
05272   if (N->OperandList[0] != Op1)
05273     N->OperandList[0].set(Op1);
05274   if (N->OperandList[1] != Op2)
05275     N->OperandList[1].set(Op2);
05276 
05277   // If this gets put into a CSE map, add it.
05278   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05279   return N;
05280 }
05281 
05282 SDNode *SelectionDAG::
05283 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
05284   SDValue Ops[] = { Op1, Op2, Op3 };
05285   return UpdateNodeOperands(N, Ops);
05286 }
05287 
05288 SDNode *SelectionDAG::
05289 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
05290                    SDValue Op3, SDValue Op4) {
05291   SDValue Ops[] = { Op1, Op2, Op3, Op4 };
05292   return UpdateNodeOperands(N, Ops);
05293 }
05294 
05295 SDNode *SelectionDAG::
05296 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
05297                    SDValue Op3, SDValue Op4, SDValue Op5) {
05298   SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
05299   return UpdateNodeOperands(N, Ops);
05300 }
05301 
05302 SDNode *SelectionDAG::
05303 UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
05304   unsigned NumOps = Ops.size();
05305   assert(N->getNumOperands() == NumOps &&
05306          "Update with wrong number of operands");
05307 
05308   // Check to see if there is no change.
05309   bool AnyChange = false;
05310   for (unsigned i = 0; i != NumOps; ++i) {
05311     if (Ops[i] != N->getOperand(i)) {
05312       AnyChange = true;
05313       break;
05314     }
05315   }
05316 
05317   // No operands changed, just return the input node.
05318   if (!AnyChange) return N;
05319 
05320   // See if the modified node already exists.
05321   void *InsertPos = nullptr;
05322   if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
05323     return Existing;
05324 
05325   // Nope it doesn't.  Remove the node from its current place in the maps.
05326   if (InsertPos)
05327     if (!RemoveNodeFromCSEMaps(N))
05328       InsertPos = nullptr;
05329 
05330   // Now we update the operands.
05331   for (unsigned i = 0; i != NumOps; ++i)
05332     if (N->OperandList[i] != Ops[i])
05333       N->OperandList[i].set(Ops[i]);
05334 
05335   // If this gets put into a CSE map, add it.
05336   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05337   return N;
05338 }
05339 
05340 /// DropOperands - Release the operands and set this node to have
05341 /// zero operands.
05342 void SDNode::DropOperands() {
05343   // Unlike the code in MorphNodeTo that does this, we don't need to
05344   // watch for dead nodes here.
05345   for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
05346     SDUse &Use = *I++;
05347     Use.set(SDValue());
05348   }
05349 }
05350 
05351 /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
05352 /// machine opcode.
05353 ///
05354 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05355                                    EVT VT) {
05356   SDVTList VTs = getVTList(VT);
05357   return SelectNodeTo(N, MachineOpc, VTs, None);
05358 }
05359 
05360 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05361                                    EVT VT, SDValue Op1) {
05362   SDVTList VTs = getVTList(VT);
05363   SDValue Ops[] = { Op1 };
05364   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05365 }
05366 
05367 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05368                                    EVT VT, SDValue Op1,
05369                                    SDValue Op2) {
05370   SDVTList VTs = getVTList(VT);
05371   SDValue Ops[] = { Op1, Op2 };
05372   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05373 }
05374 
05375 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05376                                    EVT VT, SDValue Op1,
05377                                    SDValue Op2, SDValue Op3) {
05378   SDVTList VTs = getVTList(VT);
05379   SDValue Ops[] = { Op1, Op2, Op3 };
05380   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05381 }
05382 
05383 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05384                                    EVT VT, ArrayRef<SDValue> Ops) {
05385   SDVTList VTs = getVTList(VT);
05386   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05387 }
05388 
05389 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05390                                    EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
05391   SDVTList VTs = getVTList(VT1, VT2);
05392   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05393 }
05394 
05395 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05396                                    EVT VT1, EVT VT2) {
05397   SDVTList VTs = getVTList(VT1, VT2);
05398   return SelectNodeTo(N, MachineOpc, VTs, None);
05399 }
05400 
05401 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05402                                    EVT VT1, EVT VT2, EVT VT3,
05403                                    ArrayRef<SDValue> Ops) {
05404   SDVTList VTs = getVTList(VT1, VT2, VT3);
05405   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05406 }
05407 
05408 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05409                                    EVT VT1, EVT VT2, EVT VT3, EVT VT4,
05410                                    ArrayRef<SDValue> Ops) {
05411   SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
05412   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05413 }
05414 
05415 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05416                                    EVT VT1, EVT VT2,
05417                                    SDValue Op1) {
05418   SDVTList VTs = getVTList(VT1, VT2);
05419   SDValue Ops[] = { Op1 };
05420   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05421 }
05422 
05423 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05424                                    EVT VT1, EVT VT2,
05425                                    SDValue Op1, SDValue Op2) {
05426   SDVTList VTs = getVTList(VT1, VT2);
05427   SDValue Ops[] = { Op1, Op2 };
05428   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05429 }
05430 
05431 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05432                                    EVT VT1, EVT VT2,
05433                                    SDValue Op1, SDValue Op2,
05434                                    SDValue Op3) {
05435   SDVTList VTs = getVTList(VT1, VT2);
05436   SDValue Ops[] = { Op1, Op2, Op3 };
05437   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05438 }
05439 
05440 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05441                                    EVT VT1, EVT VT2, EVT VT3,
05442                                    SDValue Op1, SDValue Op2,
05443                                    SDValue Op3) {
05444   SDVTList VTs = getVTList(VT1, VT2, VT3);
05445   SDValue Ops[] = { Op1, Op2, Op3 };
05446   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05447 }
05448 
05449 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05450                                    SDVTList VTs,ArrayRef<SDValue> Ops) {
05451   N = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
05452   // Reset the NodeID to -1.
05453   N->setNodeId(-1);
05454   return N;
05455 }
05456 
05457 /// UpdadeSDLocOnMergedSDNode - If the opt level is -O0 then it throws away
05458 /// the line number information on the merged node since it is not possible to
05459 /// preserve the information that operation is associated with multiple lines.
05460 /// This will make the debugger working better at -O0, were there is a higher
05461 /// probability having other instructions associated with that line.
05462 ///
05463 /// For IROrder, we keep the smaller of the two
05464 SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) {
05465   DebugLoc NLoc = N->getDebugLoc();
05466   if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) &&
05467     (OLoc.getDebugLoc() != NLoc)) {
05468     N->setDebugLoc(DebugLoc());
05469   }
05470   unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
05471   N->setIROrder(Order);
05472   return N;
05473 }
05474 
05475 /// MorphNodeTo - This *mutates* the specified node to have the specified
05476 /// return type, opcode, and operands.
05477 ///
05478 /// Note that MorphNodeTo returns the resultant node.  If there is already a
05479 /// node of the specified opcode and operands, it returns that node instead of
05480 /// the current one.  Note that the SDLoc need not be the same.
05481 ///
05482 /// Using MorphNodeTo is faster than creating a new node and swapping it in
05483 /// with ReplaceAllUsesWith both because it often avoids allocating a new
05484 /// node, and because it doesn't require CSE recalculation for any of
05485 /// the node's users.
05486 ///
05487 /// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
05488 /// As a consequence it isn't appropriate to use from within the DAG combiner or
05489 /// the legalizer which maintain worklists that would need to be updated when
05490 /// deleting things.
05491 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
05492                                   SDVTList VTs, ArrayRef<SDValue> Ops) {
05493   unsigned NumOps = Ops.size();
05494   // If an identical node already exists, use it.
05495   void *IP = nullptr;
05496   if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
05497     FoldingSetNodeID ID;
05498     AddNodeIDNode(ID, Opc, VTs, Ops);
05499     if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
05500       return UpdadeSDLocOnMergedSDNode(ON, SDLoc(N));
05501   }
05502 
05503   if (!RemoveNodeFromCSEMaps(N))
05504     IP = nullptr;
05505 
05506   // Start the morphing.
05507   N->NodeType = Opc;
05508   N->ValueList = VTs.VTs;
05509   N->NumValues = VTs.NumVTs;
05510 
05511   // Clear the operands list, updating used nodes to remove this from their
05512   // use list.  Keep track of any operands that become dead as a result.
05513   SmallPtrSet<SDNode*, 16> DeadNodeSet;
05514   for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
05515     SDUse &Use = *I++;
05516     SDNode *Used = Use.getNode();
05517     Use.set(SDValue());
05518     if (Used->use_empty())
05519       DeadNodeSet.insert(Used);
05520   }
05521 
05522   if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) {
05523     // Initialize the memory references information.
05524     MN->setMemRefs(nullptr, nullptr);
05525     // If NumOps is larger than the # of operands we can have in a
05526     // MachineSDNode, reallocate the operand list.
05527     if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) {
05528       if (MN->OperandsNeedDelete)
05529         delete[] MN->OperandList;
05530       if (NumOps > array_lengthof(MN->LocalOperands))
05531         // We're creating a final node that will live unmorphed for the
05532         // remainder of the current SelectionDAG iteration, so we can allocate
05533         // the operands directly out of a pool with no recycling metadata.
05534         MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
05535                          Ops.data(), NumOps);
05536       else
05537         MN->InitOperands(MN->LocalOperands, Ops.data(), NumOps);
05538       MN->OperandsNeedDelete = false;
05539     } else
05540       MN->InitOperands(MN->OperandList, Ops.data(), NumOps);
05541   } else {
05542     // If NumOps is larger than the # of operands we currently have, reallocate
05543     // the operand list.
05544     if (NumOps > N->NumOperands) {
05545       if (N->OperandsNeedDelete)
05546         delete[] N->OperandList;
05547       N->InitOperands(new SDUse[NumOps], Ops.data(), NumOps);
05548       N->OperandsNeedDelete = true;
05549     } else
05550       N->InitOperands(N->OperandList, Ops.data(), NumOps);
05551   }
05552 
05553   // Delete any nodes that are still dead after adding the uses for the
05554   // new operands.
05555   if (!DeadNodeSet.empty()) {
05556     SmallVector<SDNode *, 16> DeadNodes;
05557     for (SDNode *N : DeadNodeSet)
05558       if (N->use_empty())
05559         DeadNodes.push_back(N);
05560     RemoveDeadNodes(DeadNodes);
05561   }
05562 
05563   if (IP)
05564     CSEMap.InsertNode(N, IP);   // Memoize the new node.
05565   return N;
05566 }
05567 
05568 
05569 /// getMachineNode - These are used for target selectors to create a new node
05570 /// with specified return type(s), MachineInstr opcode, and operands.
05571 ///
05572 /// Note that getMachineNode returns the resultant node.  If there is already a
05573 /// node of the specified opcode and operands, it returns that node instead of
05574 /// the current one.
05575 MachineSDNode *
05576 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT) {
05577   SDVTList VTs = getVTList(VT);
05578   return getMachineNode(Opcode, dl, VTs, None);
05579 }
05580 
05581 MachineSDNode *
05582 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, SDValue Op1) {
05583   SDVTList VTs = getVTList(VT);
05584   SDValue Ops[] = { Op1 };
05585   return getMachineNode(Opcode, dl, VTs, Ops);
05586 }
05587 
05588 MachineSDNode *
05589 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05590                              SDValue Op1, SDValue Op2) {
05591   SDVTList VTs = getVTList(VT);
05592   SDValue Ops[] = { Op1, Op2 };
05593   return getMachineNode(Opcode, dl, VTs, Ops);
05594 }
05595 
05596 MachineSDNode *
05597 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05598                              SDValue Op1, SDValue Op2, SDValue Op3) {
05599   SDVTList VTs = getVTList(VT);
05600   SDValue Ops[] = { Op1, Op2, Op3 };
05601   return getMachineNode(Opcode, dl, VTs, Ops);
05602 }
05603 
05604 MachineSDNode *
05605 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05606                              ArrayRef<SDValue> Ops) {
05607   SDVTList VTs = getVTList(VT);
05608   return getMachineNode(Opcode, dl, VTs, Ops);
05609 }
05610 
05611 MachineSDNode *
05612 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, EVT VT2) {
05613   SDVTList VTs = getVTList(VT1, VT2);
05614   return getMachineNode(Opcode, dl, VTs, None);
05615 }
05616 
05617 MachineSDNode *
05618 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05619                              EVT VT1, EVT VT2, SDValue Op1) {
05620   SDVTList VTs = getVTList(VT1, VT2);
05621   SDValue Ops[] = { Op1 };
05622   return getMachineNode(Opcode, dl, VTs, Ops);
05623 }
05624 
05625 MachineSDNode *
05626 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05627                              EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) {
05628   SDVTList VTs = getVTList(VT1, VT2);
05629   SDValue Ops[] = { Op1, Op2 };
05630   return getMachineNode(Opcode, dl, VTs, Ops);
05631 }
05632 
05633 MachineSDNode *
05634 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05635                              EVT VT1, EVT VT2, SDValue Op1,
05636                              SDValue Op2, SDValue Op3) {
05637   SDVTList VTs = getVTList(VT1, VT2);
05638   SDValue Ops[] = { Op1, Op2, Op3 };
05639   return getMachineNode(Opcode, dl, VTs, Ops);
05640 }
05641 
05642 MachineSDNode *
05643 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05644                              EVT VT1, EVT VT2,
05645                              ArrayRef<SDValue> Ops) {
05646   SDVTList VTs = getVTList(VT1, VT2);
05647   return getMachineNode(Opcode, dl, VTs, Ops);
05648 }
05649 
05650 MachineSDNode *
05651 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05652                              EVT VT1, EVT VT2, EVT VT3,
05653                              SDValue Op1, SDValue Op2) {
05654   SDVTList VTs = getVTList(VT1, VT2, VT3);
05655   SDValue Ops[] = { Op1, Op2 };
05656   return getMachineNode(Opcode, dl, VTs, Ops);
05657 }
05658 
05659 MachineSDNode *
05660 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05661                              EVT VT1, EVT VT2, EVT VT3,
05662                              SDValue Op1, SDValue Op2, SDValue Op3) {
05663   SDVTList VTs = getVTList(VT1, VT2, VT3);
05664   SDValue Ops[] = { Op1, Op2, Op3 };
05665   return getMachineNode(Opcode, dl, VTs, Ops);
05666 }
05667 
05668 MachineSDNode *
05669 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05670                              EVT VT1, EVT VT2, EVT VT3,
05671                              ArrayRef<SDValue> Ops) {
05672   SDVTList VTs = getVTList(VT1, VT2, VT3);
05673   return getMachineNode(Opcode, dl, VTs, Ops);
05674 }
05675 
05676 MachineSDNode *
05677 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1,
05678                              EVT VT2, EVT VT3, EVT VT4,
05679                              ArrayRef<SDValue> Ops) {
05680   SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
05681   return getMachineNode(Opcode, dl, VTs, Ops);
05682 }
05683 
05684 MachineSDNode *
05685 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05686                              ArrayRef<EVT> ResultTys,
05687                              ArrayRef<SDValue> Ops) {
05688   SDVTList VTs = getVTList(ResultTys);
05689   return getMachineNode(Opcode, dl, VTs, Ops);
05690 }
05691 
05692 MachineSDNode *
05693 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
05694                              ArrayRef<SDValue> OpsArray) {
05695   bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
05696   MachineSDNode *N;
05697   void *IP = nullptr;
05698   const SDValue *Ops = OpsArray.data();
05699   unsigned NumOps = OpsArray.size();
05700 
05701   if (DoCSE) {
05702     FoldingSetNodeID ID;
05703     AddNodeIDNode(ID, ~Opcode, VTs, OpsArray);
05704     IP = nullptr;
05705     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
05706       return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL));
05707     }
05708   }
05709 
05710   // Allocate a new MachineSDNode.
05711   N = new (NodeAllocator) MachineSDNode(~Opcode, DL.getIROrder(),
05712                                         DL.getDebugLoc(), VTs);
05713 
05714   // Initialize the operands list.
05715   if (NumOps > array_lengthof(N->LocalOperands))
05716     // We're creating a final node that will live unmorphed for the
05717     // remainder of the current SelectionDAG iteration, so we can allocate
05718     // the operands directly out of a pool with no recycling metadata.
05719     N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
05720                     Ops, NumOps);
05721   else
05722     N->InitOperands(N->LocalOperands, Ops, NumOps);
05723   N->OperandsNeedDelete = false;
05724 
05725   if (DoCSE)
05726     CSEMap.InsertNode(N, IP);
05727 
05728   InsertNode(N);
05729   return N;
05730 }
05731 
05732 /// getTargetExtractSubreg - A convenience function for creating
05733 /// TargetOpcode::EXTRACT_SUBREG nodes.
05734 SDValue
05735 SelectionDAG::getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT,
05736                                      SDValue Operand) {
05737   SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
05738   SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
05739                                   VT, Operand, SRIdxVal);
05740   return SDValue(Subreg, 0);
05741 }
05742 
05743 /// getTargetInsertSubreg - A convenience function for creating
05744 /// TargetOpcode::INSERT_SUBREG nodes.
05745 SDValue
05746 SelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT,
05747                                     SDValue Operand, SDValue Subreg) {
05748   SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
05749   SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
05750                                   VT, Operand, Subreg, SRIdxVal);
05751   return SDValue(Result, 0);
05752 }
05753 
05754 /// getNodeIfExists - Get the specified node if it's already available, or
05755 /// else return NULL.
05756 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
05757                                       ArrayRef<SDValue> Ops, bool nuw, bool nsw,
05758                                       bool exact) {
05759   if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
05760     FoldingSetNodeID ID;
05761     AddNodeIDNode(ID, Opcode, VTList, Ops);
05762     if (isBinOpWithFlags(Opcode))
05763       AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
05764     void *IP = nullptr;
05765     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05766       return E;
05767   }
05768   return nullptr;
05769 }
05770 
05771 /// getDbgValue - Creates a SDDbgValue node.
05772 ///
05773 /// SDNode
05774 SDDbgValue *
05775 SelectionDAG::getDbgValue(MDNode *MDPtr, SDNode *N, unsigned R,
05776         bool IsIndirect, uint64_t Off,
05777                           DebugLoc DL, unsigned O) {
05778   return new (Allocator) SDDbgValue(MDPtr, N, R, IsIndirect, Off, DL, O);
05779 }
05780 
05781 /// Constant
05782 SDDbgValue *
05783 SelectionDAG::getConstantDbgValue(MDNode *MDPtr, const Value *C,
05784           uint64_t Off,
05785           DebugLoc DL, unsigned O) {
05786   return new (Allocator) SDDbgValue(MDPtr, C, Off, DL, O);
05787 }
05788 
05789 /// FrameIndex
05790 SDDbgValue *
05791 SelectionDAG::getFrameIndexDbgValue(MDNode *MDPtr, unsigned FI, uint64_t Off,
05792             DebugLoc DL, unsigned O) {
05793   return new (Allocator) SDDbgValue(MDPtr, FI, Off, DL, O);
05794 }
05795 
05796 namespace {
05797 
05798 /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
05799 /// pointed to by a use iterator is deleted, increment the use iterator
05800 /// so that it doesn't dangle.
05801 ///
05802 class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
05803   SDNode::use_iterator &UI;
05804   SDNode::use_iterator &UE;
05805 
05806   void NodeDeleted(SDNode *N, SDNode *E) override {
05807     // Increment the iterator as needed.
05808     while (UI != UE && N == *UI)
05809       ++UI;
05810   }
05811 
05812 public:
05813   RAUWUpdateListener(SelectionDAG &d,
05814                      SDNode::use_iterator &ui,
05815                      SDNode::use_iterator &ue)
05816     : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
05817 };
05818 
05819 }
05820 
05821 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05822 /// This can cause recursive merging of nodes in the DAG.
05823 ///
05824 /// This version assumes From has a single result value.
05825 ///
05826 void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
05827   SDNode *From = FromN.getNode();
05828   assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
05829          "Cannot replace with this method!");
05830   assert(From != To.getNode() && "Cannot replace uses of with self");
05831 
05832   // Iterate over all the existing uses of From. New uses will be added
05833   // to the beginning of the use list, which we avoid visiting.
05834   // This specifically avoids visiting uses of From that arise while the
05835   // replacement is happening, because any such uses would be the result
05836   // of CSE: If an existing node looks like From after one of its operands
05837   // is replaced by To, we don't want to replace of all its users with To
05838   // too. See PR3018 for more info.
05839   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05840   RAUWUpdateListener Listener(*this, UI, UE);
05841   while (UI != UE) {
05842     SDNode *User = *UI;
05843 
05844     // This node is about to morph, remove its old self from the CSE maps.
05845     RemoveNodeFromCSEMaps(User);
05846 
05847     // A user can appear in a use list multiple times, and when this
05848     // happens the uses are usually next to each other in the list.
05849     // To help reduce the number of CSE recomputations, process all
05850     // the uses of this user that we can find this way.
05851     do {
05852       SDUse &Use = UI.getUse();
05853       ++UI;
05854       Use.set(To);
05855     } while (UI != UE && *UI == User);
05856 
05857     // Now that we have modified User, add it back to the CSE maps.  If it
05858     // already exists there, recursively merge the results together.
05859     AddModifiedNodeToCSEMaps(User);
05860   }
05861 
05862   // If we just RAUW'd the root, take note.
05863   if (FromN == getRoot())
05864     setRoot(To);
05865 }
05866 
05867 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05868 /// This can cause recursive merging of nodes in the DAG.
05869 ///
05870 /// This version assumes that for each value of From, there is a
05871 /// corresponding value in To in the same position with the same type.
05872 ///
05873 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
05874 #ifndef NDEBUG
05875   for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
05876     assert((!From->hasAnyUseOfValue(i) ||
05877             From->getValueType(i) == To->getValueType(i)) &&
05878            "Cannot use this version of ReplaceAllUsesWith!");
05879 #endif
05880 
05881   // Handle the trivial case.
05882   if (From == To)
05883     return;
05884 
05885   // Iterate over just the existing users of From. See the comments in
05886   // the ReplaceAllUsesWith above.
05887   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05888   RAUWUpdateListener Listener(*this, UI, UE);
05889   while (UI != UE) {
05890     SDNode *User = *UI;
05891 
05892     // This node is about to morph, remove its old self from the CSE maps.
05893     RemoveNodeFromCSEMaps(User);
05894 
05895     // A user can appear in a use list multiple times, and when this
05896     // happens the uses are usually next to each other in the list.
05897     // To help reduce the number of CSE recomputations, process all
05898     // the uses of this user that we can find this way.
05899     do {
05900       SDUse &Use = UI.getUse();
05901       ++UI;
05902       Use.setNode(To);
05903     } while (UI != UE && *UI == User);
05904 
05905     // Now that we have modified User, add it back to the CSE maps.  If it
05906     // already exists there, recursively merge the results together.
05907     AddModifiedNodeToCSEMaps(User);
05908   }
05909 
05910   // If we just RAUW'd the root, take note.
05911   if (From == getRoot().getNode())
05912     setRoot(SDValue(To, getRoot().getResNo()));
05913 }
05914 
05915 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05916 /// This can cause recursive merging of nodes in the DAG.
05917 ///
05918 /// This version can replace From with any result values.  To must match the
05919 /// number and types of values returned by From.
05920 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
05921   if (From->getNumValues() == 1)  // Handle the simple case efficiently.
05922     return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
05923 
05924   // Iterate over just the existing users of From. See the comments in
05925   // the ReplaceAllUsesWith above.
05926   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05927   RAUWUpdateListener Listener(*this, UI, UE);
05928   while (UI != UE) {
05929     SDNode *User = *UI;
05930 
05931     // This node is about to morph, remove its old self from the CSE maps.
05932     RemoveNodeFromCSEMaps(User);
05933 
05934     // A user can appear in a use list multiple times, and when this
05935     // happens the uses are usually next to each other in the list.
05936     // To help reduce the number of CSE recomputations, process all
05937     // the uses of this user that we can find this way.
05938     do {
05939       SDUse &Use = UI.getUse();
05940       const SDValue &ToOp = To[Use.getResNo()];
05941       ++UI;
05942       Use.set(ToOp);
05943     } while (UI != UE && *UI == User);
05944 
05945     // Now that we have modified User, add it back to the CSE maps.  If it
05946     // already exists there, recursively merge the results together.
05947     AddModifiedNodeToCSEMaps(User);
05948   }
05949 
05950   // If we just RAUW'd the root, take note.
05951   if (From == getRoot().getNode())
05952     setRoot(SDValue(To[getRoot().getResNo()]));
05953 }
05954 
05955 /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
05956 /// uses of other values produced by From.getNode() alone.  The Deleted
05957 /// vector is handled the same way as for ReplaceAllUsesWith.
05958 void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
05959   // Handle the really simple, really trivial case efficiently.
05960   if (From == To) return;
05961 
05962   // Handle the simple, trivial, case efficiently.
05963   if (From.getNode()->getNumValues() == 1) {
05964     ReplaceAllUsesWith(From, To);
05965     return;
05966   }
05967 
05968   // Iterate over just the existing users of From. See the comments in
05969   // the ReplaceAllUsesWith above.
05970   SDNode::use_iterator UI = From.getNode()->use_begin(),
05971                        UE = From.getNode()->use_end();
05972   RAUWUpdateListener Listener(*this, UI, UE);
05973   while (UI != UE) {
05974     SDNode *User = *UI;
05975     bool UserRemovedFromCSEMaps = false;
05976 
05977     // A user can appear in a use list multiple times, and when this
05978     // happens the uses are usually next to each other in the list.
05979     // To help reduce the number of CSE recomputations, process all
05980     // the uses of this user that we can find this way.
05981     do {
05982       SDUse &Use = UI.getUse();
05983 
05984       // Skip uses of different values from the same node.
05985       if (Use.getResNo() != From.getResNo()) {
05986         ++UI;
05987         continue;
05988       }
05989 
05990       // If this node hasn't been modified yet, it's still in the CSE maps,
05991       // so remove its old self from the CSE maps.
05992       if (!UserRemovedFromCSEMaps) {
05993         RemoveNodeFromCSEMaps(User);
05994         UserRemovedFromCSEMaps = true;
05995       }
05996 
05997       ++UI;
05998       Use.set(To);
05999     } while (UI != UE && *UI == User);
06000 
06001     // We are iterating over all uses of the From node, so if a use
06002     // doesn't use the specific value, no changes are made.
06003     if (!UserRemovedFromCSEMaps)
06004       continue;
06005 
06006     // Now that we have modified User, add it back to the CSE maps.  If it
06007     // already exists there, recursively merge the results together.
06008     AddModifiedNodeToCSEMaps(User);
06009   }
06010 
06011   // If we just RAUW'd the root, take note.
06012   if (From == getRoot())
06013     setRoot(To);
06014 }
06015 
06016 namespace {
06017   /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
06018   /// to record information about a use.
06019   struct UseMemo {
06020     SDNode *User;
06021     unsigned Index;
06022     SDUse *Use;
06023   };
06024 
06025   /// operator< - Sort Memos by User.
06026   bool operator<(const UseMemo &L, const UseMemo &R) {
06027     return (intptr_t)L.User < (intptr_t)R.User;
06028   }
06029 }
06030 
06031 /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
06032 /// uses of other values produced by From.getNode() alone.  The same value
06033 /// may appear in both the From and To list.  The Deleted vector is
06034 /// handled the same way as for ReplaceAllUsesWith.
06035 void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
06036                                               const SDValue *To,
06037                                               unsigned Num){
06038   // Handle the simple, trivial case efficiently.
06039   if (Num == 1)
06040     return ReplaceAllUsesOfValueWith(*From, *To);
06041 
06042   // Read up all the uses and make records of them. This helps
06043   // processing new uses that are introduced during the
06044   // replacement process.
06045   SmallVector<UseMemo, 4> Uses;
06046   for (unsigned i = 0; i != Num; ++i) {
06047     unsigned FromResNo = From[i].getResNo();
06048     SDNode *FromNode = From[i].getNode();
06049     for (SDNode::use_iterator UI = FromNode->use_begin(),
06050          E = FromNode->use_end(); UI != E; ++UI) {
06051       SDUse &Use = UI.getUse();
06052       if (Use.getResNo() == FromResNo) {
06053         UseMemo Memo = { *UI, i, &Use };
06054         Uses.push_back(Memo);
06055       }
06056     }
06057   }
06058 
06059   // Sort the uses, so that all the uses from a given User are together.
06060   std::sort(Uses.begin(), Uses.end());
06061 
06062   for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
06063        UseIndex != UseIndexEnd; ) {
06064     // We know that this user uses some value of From.  If it is the right
06065     // value, update it.
06066     SDNode *User = Uses[UseIndex].User;
06067 
06068     // This node is about to morph, remove its old self from the CSE maps.
06069     RemoveNodeFromCSEMaps(User);
06070 
06071     // The Uses array is sorted, so all the uses for a given User
06072     // are next to each other in the list.
06073     // To help reduce the number of CSE recomputations, process all
06074     // the uses of this user that we can find this way.
06075     do {
06076       unsigned i = Uses[UseIndex].Index;
06077       SDUse &Use = *Uses[UseIndex].Use;
06078       ++UseIndex;
06079 
06080       Use.set(To[i]);
06081     } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
06082 
06083     // Now that we have modified User, add it back to the CSE maps.  If it
06084     // already exists there, recursively merge the results together.
06085     AddModifiedNodeToCSEMaps(User);
06086   }
06087 }
06088 
06089 /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
06090 /// based on their topological order. It returns the maximum id and a vector
06091 /// of the SDNodes* in assigned order by reference.
06092 unsigned SelectionDAG::AssignTopologicalOrder() {
06093 
06094   unsigned DAGSize = 0;
06095 
06096   // SortedPos tracks the progress of the algorithm. Nodes before it are
06097   // sorted, nodes after it are unsorted. When the algorithm completes
06098   // it is at the end of the list.
06099   allnodes_iterator SortedPos = allnodes_begin();
06100 
06101   // Visit all the nodes. Move nodes with no operands to the front of
06102   // the list immediately. Annotate nodes that do have operands with their
06103   // operand count. Before we do this, the Node Id fields of the nodes
06104   // may contain arbitrary values. After, the Node Id fields for nodes
06105   // before SortedPos will contain the topological sort index, and the
06106   // Node Id fields for nodes At SortedPos and after will contain the
06107   // count of outstanding operands.
06108   for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
06109     SDNode *N = I++;
06110     checkForCycles(N, this);
06111     unsigned Degree = N->getNumOperands();
06112     if (Degree == 0) {
06113       // A node with no uses, add it to the result array immediately.
06114       N->setNodeId(DAGSize++);
06115       allnodes_iterator Q = N;
06116       if (Q != SortedPos)
06117         SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
06118       assert(SortedPos != AllNodes.end() && "Overran node list");
06119       ++SortedPos;
06120     } else {
06121       // Temporarily use the Node Id as scratch space for the degree count.
06122       N->setNodeId(Degree);
06123     }
06124   }
06125 
06126   // Visit all the nodes. As we iterate, move nodes into sorted order,
06127   // such that by the time the end is reached all nodes will be sorted.
06128   for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) {
06129     SDNode *N = I;
06130     checkForCycles(N, this);
06131     // N is in sorted position, so all its uses have one less operand
06132     // that needs to be sorted.
06133     for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
06134          UI != UE; ++UI) {
06135       SDNode *P = *UI;
06136       unsigned Degree = P->getNodeId();
06137       assert(Degree != 0 && "Invalid node degree");
06138       --Degree;
06139       if (Degree == 0) {
06140         // All of P's operands are sorted, so P may sorted now.
06141         P->setNodeId(DAGSize++);
06142         if (P != SortedPos)
06143           SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
06144         assert(SortedPos != AllNodes.end() && "Overran node list");
06145         ++SortedPos;
06146       } else {
06147         // Update P's outstanding operand count.
06148         P->setNodeId(Degree);
06149       }
06150     }
06151     if (I == SortedPos) {
06152 #ifndef NDEBUG
06153       SDNode *S = ++I;
06154       dbgs() << "Overran sorted position:\n";
06155       S->dumprFull(this); dbgs() << "\n";
06156       dbgs() << "Checking if this is due to cycles\n";
06157       checkForCycles(this, true);
06158 #endif
06159       llvm_unreachable(nullptr);
06160     }
06161   }
06162 
06163   assert(SortedPos == AllNodes.end() &&
06164          "Topological sort incomplete!");
06165   assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
06166          "First node in topological sort is not the entry token!");
06167   assert(AllNodes.front().getNodeId() == 0 &&
06168          "First node in topological sort has non-zero id!");
06169   assert(AllNodes.front().getNumOperands() == 0 &&
06170          "First node in topological sort has operands!");
06171   assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
06172          "Last node in topologic sort has unexpected id!");
06173   assert(AllNodes.back().use_empty() &&
06174          "Last node in topologic sort has users!");
06175   assert(DAGSize == allnodes_size() && "Node count mismatch!");
06176   return DAGSize;
06177 }
06178 
06179 /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
06180 /// value is produced by SD.
06181 void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) {
06182   DbgInfo->add(DB, SD, isParameter);
06183   if (SD)
06184     SD->setHasDebugValue(true);
06185 }
06186 
06187 /// TransferDbgValues - Transfer SDDbgValues.
06188 void SelectionDAG::TransferDbgValues(SDValue From, SDValue To) {
06189   if (From == To || !From.getNode()->getHasDebugValue())
06190     return;
06191   SDNode *FromNode = From.getNode();
06192   SDNode *ToNode = To.getNode();
06193   ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode);
06194   SmallVector<SDDbgValue *, 2> ClonedDVs;
06195   for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end();
06196        I != E; ++I) {
06197     SDDbgValue *Dbg = *I;
06198     if (Dbg->getKind() == SDDbgValue::SDNODE) {
06199       SDDbgValue *Clone = getDbgValue(Dbg->getMDPtr(), ToNode, To.getResNo(),
06200               Dbg->isIndirect(),
06201                                       Dbg->getOffset(), Dbg->getDebugLoc(),
06202                                       Dbg->getOrder());
06203       ClonedDVs.push_back(Clone);
06204     }
06205   }
06206   for (SmallVectorImpl<SDDbgValue *>::iterator I = ClonedDVs.begin(),
06207          E = ClonedDVs.end(); I != E; ++I)
06208     AddDbgValue(*I, ToNode, false);
06209 }
06210 
06211 //===----------------------------------------------------------------------===//
06212 //                              SDNode Class
06213 //===----------------------------------------------------------------------===//
06214 
06215 HandleSDNode::~HandleSDNode() {
06216   DropOperands();
06217 }
06218 
06219 GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
06220                                          DebugLoc DL, const GlobalValue *GA,
06221                                          EVT VT, int64_t o, unsigned char TF)
06222   : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
06223   TheGlobal = GA;
06224 }
06225 
06226 AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT,
06227                                          SDValue X, unsigned SrcAS,
06228                                          unsigned DestAS)
06229  : UnarySDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT), X),
06230    SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
06231 
06232 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
06233                      EVT memvt, MachineMemOperand *mmo)
06234  : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
06235   SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
06236                                       MMO->isNonTemporal(), MMO->isInvariant());
06237   assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
06238   assert(isNonTemporal() == MMO->isNonTemporal() &&
06239          "Non-temporal encoding error!");
06240   // We check here that the size of the memory operand fits within the size of
06241   // the MMO. This is because the MMO might indicate only a possible address
06242   // range instead of specifying the affected memory addresses precisely.
06243   assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
06244 }
06245 
06246 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
06247                      ArrayRef<SDValue> Ops, EVT memvt, MachineMemOperand *mmo)
06248    : SDNode(Opc, Order, dl, VTs, Ops),
06249      MemoryVT(memvt), MMO(mmo) {
06250   SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
06251                                       MMO->isNonTemporal(), MMO->isInvariant());
06252   assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
06253   assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
06254 }
06255 
06256 /// Profile - Gather unique data for the node.
06257 ///
06258 void SDNode::Profile(FoldingSetNodeID &ID) const {
06259   AddNodeIDNode(ID, this);
06260 }
06261 
06262 namespace {
06263   struct EVTArray {
06264     std::vector<EVT> VTs;
06265 
06266     EVTArray() {
06267       VTs.reserve(MVT::LAST_VALUETYPE);
06268       for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i)
06269         VTs.push_back(MVT((MVT::SimpleValueType)i));
06270     }
06271   };
06272 }
06273 
06274 static ManagedStatic<std::set<EVT, EVT::compareRawBits> > EVTs;
06275 static ManagedStatic<EVTArray> SimpleVTArray;
06276 static ManagedStatic<sys::SmartMutex<true> > VTMutex;
06277 
06278 /// getValueTypeList - Return a pointer to the specified value type.
06279 ///
06280 const EVT *SDNode::getValueTypeList(EVT VT) {
06281   if (VT.isExtended()) {
06282     sys::SmartScopedLock<true> Lock(*VTMutex);
06283     return &(*EVTs->insert(VT).first);
06284   } else {
06285     assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE &&
06286            "Value type out of range!");
06287     return &