LLVM API Documentation

SelectionDAG.cpp
Go to the documentation of this file.
00001 //===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This implements the SelectionDAG class.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "llvm/CodeGen/SelectionDAG.h"
00015 #include "SDNodeDbgValue.h"
00016 #include "llvm/ADT/SetVector.h"
00017 #include "llvm/ADT/SmallPtrSet.h"
00018 #include "llvm/ADT/SmallSet.h"
00019 #include "llvm/ADT/SmallVector.h"
00020 #include "llvm/ADT/StringExtras.h"
00021 #include "llvm/Analysis/ValueTracking.h"
00022 #include "llvm/CodeGen/MachineBasicBlock.h"
00023 #include "llvm/CodeGen/MachineConstantPool.h"
00024 #include "llvm/CodeGen/MachineFrameInfo.h"
00025 #include "llvm/CodeGen/MachineModuleInfo.h"
00026 #include "llvm/IR/CallingConv.h"
00027 #include "llvm/IR/Constants.h"
00028 #include "llvm/IR/DataLayout.h"
00029 #include "llvm/IR/DebugInfo.h"
00030 #include "llvm/IR/DerivedTypes.h"
00031 #include "llvm/IR/Function.h"
00032 #include "llvm/IR/GlobalAlias.h"
00033 #include "llvm/IR/GlobalVariable.h"
00034 #include "llvm/IR/Intrinsics.h"
00035 #include "llvm/Support/CommandLine.h"
00036 #include "llvm/Support/Debug.h"
00037 #include "llvm/Support/ErrorHandling.h"
00038 #include "llvm/Support/ManagedStatic.h"
00039 #include "llvm/Support/MathExtras.h"
00040 #include "llvm/Support/Mutex.h"
00041 #include "llvm/Support/raw_ostream.h"
00042 #include "llvm/Target/TargetInstrInfo.h"
00043 #include "llvm/Target/TargetIntrinsicInfo.h"
00044 #include "llvm/Target/TargetLowering.h"
00045 #include "llvm/Target/TargetMachine.h"
00046 #include "llvm/Target/TargetOptions.h"
00047 #include "llvm/Target/TargetRegisterInfo.h"
00048 #include "llvm/Target/TargetSelectionDAGInfo.h"
00049 #include "llvm/Target/TargetSubtargetInfo.h"
00050 #include <algorithm>
00051 #include <cmath>
00052 
00053 using namespace llvm;
00054 
00055 /// makeVTList - Return an instance of the SDVTList struct initialized with the
00056 /// specified members.
00057 static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
00058   SDVTList Res = {VTs, NumVTs};
00059   return Res;
00060 }
00061 
00062 // Default null implementations of the callbacks.
00063 void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
00064 void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
00065 
00066 //===----------------------------------------------------------------------===//
00067 //                              ConstantFPSDNode Class
00068 //===----------------------------------------------------------------------===//
00069 
00070 /// isExactlyValue - We don't rely on operator== working on double values, as
00071 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
00072 /// As such, this method can be used to do an exact bit-for-bit comparison of
00073 /// two floating point values.
00074 bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
00075   return getValueAPF().bitwiseIsEqual(V);
00076 }
00077 
00078 bool ConstantFPSDNode::isValueValidForType(EVT VT,
00079                                            const APFloat& Val) {
00080   assert(VT.isFloatingPoint() && "Can only convert between FP types");
00081 
00082   // convert modifies in place, so make a copy.
00083   APFloat Val2 = APFloat(Val);
00084   bool losesInfo;
00085   (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
00086                       APFloat::rmNearestTiesToEven,
00087                       &losesInfo);
00088   return !losesInfo;
00089 }
00090 
00091 //===----------------------------------------------------------------------===//
00092 //                              ISD Namespace
00093 //===----------------------------------------------------------------------===//
00094 
00095 /// isBuildVectorAllOnes - Return true if the specified node is a
00096 /// BUILD_VECTOR where all of the elements are ~0 or undef.
00097 bool ISD::isBuildVectorAllOnes(const SDNode *N) {
00098   // Look through a bit convert.
00099   while (N->getOpcode() == ISD::BITCAST)
00100     N = N->getOperand(0).getNode();
00101 
00102   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
00103 
00104   unsigned i = 0, e = N->getNumOperands();
00105 
00106   // Skip over all of the undef values.
00107   while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF)
00108     ++i;
00109 
00110   // Do not accept an all-undef vector.
00111   if (i == e) return false;
00112 
00113   // Do not accept build_vectors that aren't all constants or which have non-~0
00114   // elements. We have to be a bit careful here, as the type of the constant
00115   // may not be the same as the type of the vector elements due to type
00116   // legalization (the elements are promoted to a legal type for the target and
00117   // a vector of a type may be legal when the base element type is not).
00118   // We only want to check enough bits to cover the vector elements, because
00119   // we care if the resultant vector is all ones, not whether the individual
00120   // constants are.
00121   SDValue NotZero = N->getOperand(i);
00122   unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
00123   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
00124     if (CN->getAPIntValue().countTrailingOnes() < EltSize)
00125       return false;
00126   } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
00127     if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
00128       return false;
00129   } else
00130     return false;
00131 
00132   // Okay, we have at least one ~0 value, check to see if the rest match or are
00133   // undefs. Even with the above element type twiddling, this should be OK, as
00134   // the same type legalization should have applied to all the elements.
00135   for (++i; i != e; ++i)
00136     if (N->getOperand(i) != NotZero &&
00137         N->getOperand(i).getOpcode() != ISD::UNDEF)
00138       return false;
00139   return true;
00140 }
00141 
00142 
00143 /// isBuildVectorAllZeros - Return true if the specified node is a
00144 /// BUILD_VECTOR where all of the elements are 0 or undef.
00145 bool ISD::isBuildVectorAllZeros(const SDNode *N) {
00146   // Look through a bit convert.
00147   while (N->getOpcode() == ISD::BITCAST)
00148     N = N->getOperand(0).getNode();
00149 
00150   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
00151 
00152   bool IsAllUndef = true;
00153   for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) {
00154     if (N->getOperand(i).getOpcode() == ISD::UNDEF)
00155       continue;
00156     IsAllUndef = false;
00157     // Do not accept build_vectors that aren't all constants or which have non-0
00158     // elements. We have to be a bit careful here, as the type of the constant
00159     // may not be the same as the type of the vector elements due to type
00160     // legalization (the elements are promoted to a legal type for the target
00161     // and a vector of a type may be legal when the base element type is not).
00162     // We only want to check enough bits to cover the vector elements, because
00163     // we care if the resultant vector is all zeros, not whether the individual
00164     // constants are.
00165     SDValue Zero = N->getOperand(i);
00166     unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
00167     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Zero)) {
00168       if (CN->getAPIntValue().countTrailingZeros() < EltSize)
00169         return false;
00170     } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Zero)) {
00171       if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
00172         return false;
00173     } else
00174       return false;
00175   }
00176 
00177   // Do not accept an all-undef vector.
00178   if (IsAllUndef)
00179     return false;
00180   return true;
00181 }
00182 
00183 /// \brief Return true if the specified node is a BUILD_VECTOR node of
00184 /// all ConstantSDNode or undef.
00185 bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
00186   if (N->getOpcode() != ISD::BUILD_VECTOR)
00187     return false;
00188 
00189   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
00190     SDValue Op = N->getOperand(i);
00191     if (Op.getOpcode() == ISD::UNDEF)
00192       continue;
00193     if (!isa<ConstantSDNode>(Op))
00194       return false;
00195   }
00196   return true;
00197 }
00198 
00199 /// isScalarToVector - Return true if the specified node is a
00200 /// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low
00201 /// element is not an undef.
00202 bool ISD::isScalarToVector(const SDNode *N) {
00203   if (N->getOpcode() == ISD::SCALAR_TO_VECTOR)
00204     return true;
00205 
00206   if (N->getOpcode() != ISD::BUILD_VECTOR)
00207     return false;
00208   if (N->getOperand(0).getOpcode() == ISD::UNDEF)
00209     return false;
00210   unsigned NumElems = N->getNumOperands();
00211   if (NumElems == 1)
00212     return false;
00213   for (unsigned i = 1; i < NumElems; ++i) {
00214     SDValue V = N->getOperand(i);
00215     if (V.getOpcode() != ISD::UNDEF)
00216       return false;
00217   }
00218   return true;
00219 }
00220 
00221 /// allOperandsUndef - Return true if the node has at least one operand
00222 /// and all operands of the specified node are ISD::UNDEF.
00223 bool ISD::allOperandsUndef(const SDNode *N) {
00224   // Return false if the node has no operands.
00225   // This is "logically inconsistent" with the definition of "all" but
00226   // is probably the desired behavior.
00227   if (N->getNumOperands() == 0)
00228     return false;
00229 
00230   for (unsigned i = 0, e = N->getNumOperands(); i != e ; ++i)
00231     if (N->getOperand(i).getOpcode() != ISD::UNDEF)
00232       return false;
00233 
00234   return true;
00235 }
00236 
00237 ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
00238   switch (ExtType) {
00239   case ISD::EXTLOAD:
00240     return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
00241   case ISD::SEXTLOAD:
00242     return ISD::SIGN_EXTEND;
00243   case ISD::ZEXTLOAD:
00244     return ISD::ZERO_EXTEND;
00245   default:
00246     break;
00247   }
00248 
00249   llvm_unreachable("Invalid LoadExtType");
00250 }
00251 
00252 /// getSetCCSwappedOperands - Return the operation corresponding to (Y op X)
00253 /// when given the operation for (X op Y).
00254 ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
00255   // To perform this operation, we just need to swap the L and G bits of the
00256   // operation.
00257   unsigned OldL = (Operation >> 2) & 1;
00258   unsigned OldG = (Operation >> 1) & 1;
00259   return ISD::CondCode((Operation & ~6) |  // Keep the N, U, E bits
00260                        (OldL << 1) |       // New G bit
00261                        (OldG << 2));       // New L bit.
00262 }
00263 
00264 /// getSetCCInverse - Return the operation corresponding to !(X op Y), where
00265 /// 'op' is a valid SetCC operation.
00266 ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) {
00267   unsigned Operation = Op;
00268   if (isInteger)
00269     Operation ^= 7;   // Flip L, G, E bits, but not U.
00270   else
00271     Operation ^= 15;  // Flip all of the condition bits.
00272 
00273   if (Operation > ISD::SETTRUE2)
00274     Operation &= ~8;  // Don't let N and U bits get set.
00275 
00276   return ISD::CondCode(Operation);
00277 }
00278 
00279 
00280 /// isSignedOp - For an integer comparison, return 1 if the comparison is a
00281 /// signed operation and 2 if the result is an unsigned comparison.  Return zero
00282 /// if the operation does not depend on the sign of the input (setne and seteq).
00283 static int isSignedOp(ISD::CondCode Opcode) {
00284   switch (Opcode) {
00285   default: llvm_unreachable("Illegal integer setcc operation!");
00286   case ISD::SETEQ:
00287   case ISD::SETNE: return 0;
00288   case ISD::SETLT:
00289   case ISD::SETLE:
00290   case ISD::SETGT:
00291   case ISD::SETGE: return 1;
00292   case ISD::SETULT:
00293   case ISD::SETULE:
00294   case ISD::SETUGT:
00295   case ISD::SETUGE: return 2;
00296   }
00297 }
00298 
00299 /// getSetCCOrOperation - Return the result of a logical OR between different
00300 /// comparisons of identical values: ((X op1 Y) | (X op2 Y)).  This function
00301 /// returns SETCC_INVALID if it is not possible to represent the resultant
00302 /// comparison.
00303 ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
00304                                        bool isInteger) {
00305   if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
00306     // Cannot fold a signed integer setcc with an unsigned integer setcc.
00307     return ISD::SETCC_INVALID;
00308 
00309   unsigned Op = Op1 | Op2;  // Combine all of the condition bits.
00310 
00311   // If the N and U bits get set then the resultant comparison DOES suddenly
00312   // care about orderedness, and is true when ordered.
00313   if (Op > ISD::SETTRUE2)
00314     Op &= ~16;     // Clear the U bit if the N bit is set.
00315 
00316   // Canonicalize illegal integer setcc's.
00317   if (isInteger && Op == ISD::SETUNE)  // e.g. SETUGT | SETULT
00318     Op = ISD::SETNE;
00319 
00320   return ISD::CondCode(Op);
00321 }
00322 
00323 /// getSetCCAndOperation - Return the result of a logical AND between different
00324 /// comparisons of identical values: ((X op1 Y) & (X op2 Y)).  This
00325 /// function returns zero if it is not possible to represent the resultant
00326 /// comparison.
00327 ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
00328                                         bool isInteger) {
00329   if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
00330     // Cannot fold a signed setcc with an unsigned setcc.
00331     return ISD::SETCC_INVALID;
00332 
00333   // Combine all of the condition bits.
00334   ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
00335 
00336   // Canonicalize illegal integer setcc's.
00337   if (isInteger) {
00338     switch (Result) {
00339     default: break;
00340     case ISD::SETUO : Result = ISD::SETFALSE; break;  // SETUGT & SETULT
00341     case ISD::SETOEQ:                                 // SETEQ  & SETU[LG]E
00342     case ISD::SETUEQ: Result = ISD::SETEQ   ; break;  // SETUGE & SETULE
00343     case ISD::SETOLT: Result = ISD::SETULT  ; break;  // SETULT & SETNE
00344     case ISD::SETOGT: Result = ISD::SETUGT  ; break;  // SETUGT & SETNE
00345     }
00346   }
00347 
00348   return Result;
00349 }
00350 
00351 //===----------------------------------------------------------------------===//
00352 //                           SDNode Profile Support
00353 //===----------------------------------------------------------------------===//
00354 
00355 /// AddNodeIDOpcode - Add the node opcode to the NodeID data.
00356 ///
00357 static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC)  {
00358   ID.AddInteger(OpC);
00359 }
00360 
00361 /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
00362 /// solely with their pointer.
00363 static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
00364   ID.AddPointer(VTList.VTs);
00365 }
00366 
00367 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
00368 ///
00369 static void AddNodeIDOperands(FoldingSetNodeID &ID,
00370                               ArrayRef<SDValue> Ops) {
00371   for (auto& Op : Ops) {
00372     ID.AddPointer(Op.getNode());
00373     ID.AddInteger(Op.getResNo());
00374   }
00375 }
00376 
00377 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
00378 ///
00379 static void AddNodeIDOperands(FoldingSetNodeID &ID,
00380                               ArrayRef<SDUse> Ops) {
00381   for (auto& Op : Ops) {
00382     ID.AddPointer(Op.getNode());
00383     ID.AddInteger(Op.getResNo());
00384   }
00385 }
00386 
00387 static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, bool nuw, bool nsw,
00388                                   bool exact) {
00389   ID.AddBoolean(nuw);
00390   ID.AddBoolean(nsw);
00391   ID.AddBoolean(exact);
00392 }
00393 
00394 /// AddBinaryNodeIDCustom - Add BinarySDNodes special infos
00395 static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, unsigned Opcode,
00396                                   bool nuw, bool nsw, bool exact) {
00397   if (isBinOpWithFlags(Opcode))
00398     AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
00399 }
00400 
00401 static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
00402                           SDVTList VTList, ArrayRef<SDValue> OpList) {
00403   AddNodeIDOpcode(ID, OpC);
00404   AddNodeIDValueTypes(ID, VTList);
00405   AddNodeIDOperands(ID, OpList);
00406 }
00407 
00408 /// AddNodeIDCustom - If this is an SDNode with special info, add this info to
00409 /// the NodeID data.
00410 static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
00411   switch (N->getOpcode()) {
00412   case ISD::TargetExternalSymbol:
00413   case ISD::ExternalSymbol:
00414     llvm_unreachable("Should only be used on nodes with operands");
00415   default: break;  // Normal nodes don't need extra info.
00416   case ISD::TargetConstant:
00417   case ISD::Constant: {
00418     const ConstantSDNode *C = cast<ConstantSDNode>(N);
00419     ID.AddPointer(C->getConstantIntValue());
00420     ID.AddBoolean(C->isOpaque());
00421     break;
00422   }
00423   case ISD::TargetConstantFP:
00424   case ISD::ConstantFP: {
00425     ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
00426     break;
00427   }
00428   case ISD::TargetGlobalAddress:
00429   case ISD::GlobalAddress:
00430   case ISD::TargetGlobalTLSAddress:
00431   case ISD::GlobalTLSAddress: {
00432     const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
00433     ID.AddPointer(GA->getGlobal());
00434     ID.AddInteger(GA->getOffset());
00435     ID.AddInteger(GA->getTargetFlags());
00436     ID.AddInteger(GA->getAddressSpace());
00437     break;
00438   }
00439   case ISD::BasicBlock:
00440     ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
00441     break;
00442   case ISD::Register:
00443     ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
00444     break;
00445   case ISD::RegisterMask:
00446     ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
00447     break;
00448   case ISD::SRCVALUE:
00449     ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
00450     break;
00451   case ISD::FrameIndex:
00452   case ISD::TargetFrameIndex:
00453     ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
00454     break;
00455   case ISD::JumpTable:
00456   case ISD::TargetJumpTable:
00457     ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
00458     ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
00459     break;
00460   case ISD::ConstantPool:
00461   case ISD::TargetConstantPool: {
00462     const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
00463     ID.AddInteger(CP->getAlignment());
00464     ID.AddInteger(CP->getOffset());
00465     if (CP->isMachineConstantPoolEntry())
00466       CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
00467     else
00468       ID.AddPointer(CP->getConstVal());
00469     ID.AddInteger(CP->getTargetFlags());
00470     break;
00471   }
00472   case ISD::TargetIndex: {
00473     const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
00474     ID.AddInteger(TI->getIndex());
00475     ID.AddInteger(TI->getOffset());
00476     ID.AddInteger(TI->getTargetFlags());
00477     break;
00478   }
00479   case ISD::LOAD: {
00480     const LoadSDNode *LD = cast<LoadSDNode>(N);
00481     ID.AddInteger(LD->getMemoryVT().getRawBits());
00482     ID.AddInteger(LD->getRawSubclassData());
00483     ID.AddInteger(LD->getPointerInfo().getAddrSpace());
00484     break;
00485   }
00486   case ISD::STORE: {
00487     const StoreSDNode *ST = cast<StoreSDNode>(N);
00488     ID.AddInteger(ST->getMemoryVT().getRawBits());
00489     ID.AddInteger(ST->getRawSubclassData());
00490     ID.AddInteger(ST->getPointerInfo().getAddrSpace());
00491     break;
00492   }
00493   case ISD::SDIV:
00494   case ISD::UDIV:
00495   case ISD::SRA:
00496   case ISD::SRL:
00497   case ISD::MUL:
00498   case ISD::ADD:
00499   case ISD::SUB:
00500   case ISD::SHL: {
00501     const BinaryWithFlagsSDNode *BinNode = cast<BinaryWithFlagsSDNode>(N);
00502     AddBinaryNodeIDCustom(ID, N->getOpcode(), BinNode->hasNoUnsignedWrap(),
00503                           BinNode->hasNoSignedWrap(), BinNode->isExact());
00504     break;
00505   }
00506   case ISD::ATOMIC_CMP_SWAP:
00507   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
00508   case ISD::ATOMIC_SWAP:
00509   case ISD::ATOMIC_LOAD_ADD:
00510   case ISD::ATOMIC_LOAD_SUB:
00511   case ISD::ATOMIC_LOAD_AND:
00512   case ISD::ATOMIC_LOAD_OR:
00513   case ISD::ATOMIC_LOAD_XOR:
00514   case ISD::ATOMIC_LOAD_NAND:
00515   case ISD::ATOMIC_LOAD_MIN:
00516   case ISD::ATOMIC_LOAD_MAX:
00517   case ISD::ATOMIC_LOAD_UMIN:
00518   case ISD::ATOMIC_LOAD_UMAX:
00519   case ISD::ATOMIC_LOAD:
00520   case ISD::ATOMIC_STORE: {
00521     const AtomicSDNode *AT = cast<AtomicSDNode>(N);
00522     ID.AddInteger(AT->getMemoryVT().getRawBits());
00523     ID.AddInteger(AT->getRawSubclassData());
00524     ID.AddInteger(AT->getPointerInfo().getAddrSpace());
00525     break;
00526   }
00527   case ISD::PREFETCH: {
00528     const MemSDNode *PF = cast<MemSDNode>(N);
00529     ID.AddInteger(PF->getPointerInfo().getAddrSpace());
00530     break;
00531   }
00532   case ISD::VECTOR_SHUFFLE: {
00533     const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
00534     for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
00535          i != e; ++i)
00536       ID.AddInteger(SVN->getMaskElt(i));
00537     break;
00538   }
00539   case ISD::TargetBlockAddress:
00540   case ISD::BlockAddress: {
00541     const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
00542     ID.AddPointer(BA->getBlockAddress());
00543     ID.AddInteger(BA->getOffset());
00544     ID.AddInteger(BA->getTargetFlags());
00545     break;
00546   }
00547   } // end switch (N->getOpcode())
00548 
00549   // Target specific memory nodes could also have address spaces to check.
00550   if (N->isTargetMemoryOpcode())
00551     ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
00552 }
00553 
00554 /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
00555 /// data.
00556 static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
00557   AddNodeIDOpcode(ID, N->getOpcode());
00558   // Add the return value info.
00559   AddNodeIDValueTypes(ID, N->getVTList());
00560   // Add the operand info.
00561   AddNodeIDOperands(ID, N->ops());
00562 
00563   // Handle SDNode leafs with special info.
00564   AddNodeIDCustom(ID, N);
00565 }
00566 
00567 /// encodeMemSDNodeFlags - Generic routine for computing a value for use in
00568 /// the CSE map that carries volatility, temporalness, indexing mode, and
00569 /// extension/truncation information.
00570 ///
00571 static inline unsigned
00572 encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile,
00573                      bool isNonTemporal, bool isInvariant) {
00574   assert((ConvType & 3) == ConvType &&
00575          "ConvType may not require more than 2 bits!");
00576   assert((AM & 7) == AM &&
00577          "AM may not require more than 3 bits!");
00578   return ConvType |
00579          (AM << 2) |
00580          (isVolatile << 5) |
00581          (isNonTemporal << 6) |
00582          (isInvariant << 7);
00583 }
00584 
00585 //===----------------------------------------------------------------------===//
00586 //                              SelectionDAG Class
00587 //===----------------------------------------------------------------------===//
00588 
00589 /// doNotCSE - Return true if CSE should not be performed for this node.
00590 static bool doNotCSE(SDNode *N) {
00591   if (N->getValueType(0) == MVT::Glue)
00592     return true; // Never CSE anything that produces a flag.
00593 
00594   switch (N->getOpcode()) {
00595   default: break;
00596   case ISD::HANDLENODE:
00597   case ISD::EH_LABEL:
00598     return true;   // Never CSE these nodes.
00599   }
00600 
00601   // Check that remaining values produced are not flags.
00602   for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
00603     if (N->getValueType(i) == MVT::Glue)
00604       return true; // Never CSE anything that produces a flag.
00605 
00606   return false;
00607 }
00608 
00609 /// RemoveDeadNodes - This method deletes all unreachable nodes in the
00610 /// SelectionDAG.
00611 void SelectionDAG::RemoveDeadNodes() {
00612   // Create a dummy node (which is not added to allnodes), that adds a reference
00613   // to the root node, preventing it from being deleted.
00614   HandleSDNode Dummy(getRoot());
00615 
00616   SmallVector<SDNode*, 128> DeadNodes;
00617 
00618   // Add all obviously-dead nodes to the DeadNodes worklist.
00619   for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I)
00620     if (I->use_empty())
00621       DeadNodes.push_back(I);
00622 
00623   RemoveDeadNodes(DeadNodes);
00624 
00625   // If the root changed (e.g. it was a dead load, update the root).
00626   setRoot(Dummy.getValue());
00627 }
00628 
00629 /// RemoveDeadNodes - This method deletes the unreachable nodes in the
00630 /// given list, and any nodes that become unreachable as a result.
00631 void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
00632 
00633   // Process the worklist, deleting the nodes and adding their uses to the
00634   // worklist.
00635   while (!DeadNodes.empty()) {
00636     SDNode *N = DeadNodes.pop_back_val();
00637 
00638     for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00639       DUL->NodeDeleted(N, nullptr);
00640 
00641     // Take the node out of the appropriate CSE map.
00642     RemoveNodeFromCSEMaps(N);
00643 
00644     // Next, brutally remove the operand list.  This is safe to do, as there are
00645     // no cycles in the graph.
00646     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
00647       SDUse &Use = *I++;
00648       SDNode *Operand = Use.getNode();
00649       Use.set(SDValue());
00650 
00651       // Now that we removed this operand, see if there are no uses of it left.
00652       if (Operand->use_empty())
00653         DeadNodes.push_back(Operand);
00654     }
00655 
00656     DeallocateNode(N);
00657   }
00658 }
00659 
00660 void SelectionDAG::RemoveDeadNode(SDNode *N){
00661   SmallVector<SDNode*, 16> DeadNodes(1, N);
00662 
00663   // Create a dummy node that adds a reference to the root node, preventing
00664   // it from being deleted.  (This matters if the root is an operand of the
00665   // dead node.)
00666   HandleSDNode Dummy(getRoot());
00667 
00668   RemoveDeadNodes(DeadNodes);
00669 }
00670 
00671 void SelectionDAG::DeleteNode(SDNode *N) {
00672   // First take this out of the appropriate CSE map.
00673   RemoveNodeFromCSEMaps(N);
00674 
00675   // Finally, remove uses due to operands of this node, remove from the
00676   // AllNodes list, and delete the node.
00677   DeleteNodeNotInCSEMaps(N);
00678 }
00679 
00680 void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
00681   assert(N != AllNodes.begin() && "Cannot delete the entry node!");
00682   assert(N->use_empty() && "Cannot delete a node that is not dead!");
00683 
00684   // Drop all of the operands and decrement used node's use counts.
00685   N->DropOperands();
00686 
00687   DeallocateNode(N);
00688 }
00689 
00690 void SDDbgInfo::erase(const SDNode *Node) {
00691   DbgValMapType::iterator I = DbgValMap.find(Node);
00692   if (I == DbgValMap.end())
00693     return;
00694   for (auto &Val: I->second)
00695     Val->setIsInvalidated();
00696   DbgValMap.erase(I);
00697 }
00698 
00699 void SelectionDAG::DeallocateNode(SDNode *N) {
00700   if (N->OperandsNeedDelete)
00701     delete[] N->OperandList;
00702 
00703   // Set the opcode to DELETED_NODE to help catch bugs when node
00704   // memory is reallocated.
00705   N->NodeType = ISD::DELETED_NODE;
00706 
00707   NodeAllocator.Deallocate(AllNodes.remove(N));
00708 
00709   // If any of the SDDbgValue nodes refer to this SDNode, invalidate
00710   // them and forget about that node.
00711   DbgInfo->erase(N);
00712 }
00713 
00714 #ifndef NDEBUG
00715 /// VerifySDNode - Sanity check the given SDNode.  Aborts if it is invalid.
00716 static void VerifySDNode(SDNode *N) {
00717   switch (N->getOpcode()) {
00718   default:
00719     break;
00720   case ISD::BUILD_PAIR: {
00721     EVT VT = N->getValueType(0);
00722     assert(N->getNumValues() == 1 && "Too many results!");
00723     assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
00724            "Wrong return type!");
00725     assert(N->getNumOperands() == 2 && "Wrong number of operands!");
00726     assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
00727            "Mismatched operand types!");
00728     assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
00729            "Wrong operand type!");
00730     assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
00731            "Wrong return type size");
00732     break;
00733   }
00734   case ISD::BUILD_VECTOR: {
00735     assert(N->getNumValues() == 1 && "Too many results!");
00736     assert(N->getValueType(0).isVector() && "Wrong return type!");
00737     assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
00738            "Wrong number of operands!");
00739     EVT EltVT = N->getValueType(0).getVectorElementType();
00740     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
00741       assert((I->getValueType() == EltVT ||
00742              (EltVT.isInteger() && I->getValueType().isInteger() &&
00743               EltVT.bitsLE(I->getValueType()))) &&
00744             "Wrong operand type!");
00745       assert(I->getValueType() == N->getOperand(0).getValueType() &&
00746              "Operands must all have the same type");
00747     }
00748     break;
00749   }
00750   }
00751 }
00752 #endif // NDEBUG
00753 
00754 /// \brief Insert a newly allocated node into the DAG.
00755 ///
00756 /// Handles insertion into the all nodes list and CSE map, as well as
00757 /// verification and other common operations when a new node is allocated.
00758 void SelectionDAG::InsertNode(SDNode *N) {
00759   AllNodes.push_back(N);
00760 #ifndef NDEBUG
00761   VerifySDNode(N);
00762 #endif
00763 }
00764 
00765 /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
00766 /// correspond to it.  This is useful when we're about to delete or repurpose
00767 /// the node.  We don't want future request for structurally identical nodes
00768 /// to return N anymore.
00769 bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
00770   bool Erased = false;
00771   switch (N->getOpcode()) {
00772   case ISD::HANDLENODE: return false;  // noop.
00773   case ISD::CONDCODE:
00774     assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
00775            "Cond code doesn't exist!");
00776     Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
00777     CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
00778     break;
00779   case ISD::ExternalSymbol:
00780     Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
00781     break;
00782   case ISD::TargetExternalSymbol: {
00783     ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
00784     Erased = TargetExternalSymbols.erase(
00785                std::pair<std::string,unsigned char>(ESN->getSymbol(),
00786                                                     ESN->getTargetFlags()));
00787     break;
00788   }
00789   case ISD::VALUETYPE: {
00790     EVT VT = cast<VTSDNode>(N)->getVT();
00791     if (VT.isExtended()) {
00792       Erased = ExtendedValueTypeNodes.erase(VT);
00793     } else {
00794       Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
00795       ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
00796     }
00797     break;
00798   }
00799   default:
00800     // Remove it from the CSE Map.
00801     assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
00802     assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
00803     Erased = CSEMap.RemoveNode(N);
00804     break;
00805   }
00806 #ifndef NDEBUG
00807   // Verify that the node was actually in one of the CSE maps, unless it has a
00808   // flag result (which cannot be CSE'd) or is one of the special cases that are
00809   // not subject to CSE.
00810   if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
00811       !N->isMachineOpcode() && !doNotCSE(N)) {
00812     N->dump(this);
00813     dbgs() << "\n";
00814     llvm_unreachable("Node is not in map!");
00815   }
00816 #endif
00817   return Erased;
00818 }
00819 
00820 /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
00821 /// maps and modified in place. Add it back to the CSE maps, unless an identical
00822 /// node already exists, in which case transfer all its users to the existing
00823 /// node. This transfer can potentially trigger recursive merging.
00824 ///
00825 void
00826 SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
00827   // For node types that aren't CSE'd, just act as if no identical node
00828   // already exists.
00829   if (!doNotCSE(N)) {
00830     SDNode *Existing = CSEMap.GetOrInsertNode(N);
00831     if (Existing != N) {
00832       // If there was already an existing matching node, use ReplaceAllUsesWith
00833       // to replace the dead one with the existing one.  This can cause
00834       // recursive merging of other unrelated nodes down the line.
00835       ReplaceAllUsesWith(N, Existing);
00836 
00837       // N is now dead. Inform the listeners and delete it.
00838       for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00839         DUL->NodeDeleted(N, Existing);
00840       DeleteNodeNotInCSEMaps(N);
00841       return;
00842     }
00843   }
00844 
00845   // If the node doesn't already exist, we updated it.  Inform listeners.
00846   for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00847     DUL->NodeUpdated(N);
00848 }
00849 
00850 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00851 /// were replaced with those specified.  If this node is never memoized,
00852 /// return null, otherwise return a pointer to the slot it would take.  If a
00853 /// node already exists with these operands, the slot will be non-null.
00854 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
00855                                            void *&InsertPos) {
00856   if (doNotCSE(N))
00857     return nullptr;
00858 
00859   SDValue Ops[] = { Op };
00860   FoldingSetNodeID ID;
00861   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00862   AddNodeIDCustom(ID, N);
00863   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00864   return Node;
00865 }
00866 
00867 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00868 /// were replaced with those specified.  If this node is never memoized,
00869 /// return null, otherwise return a pointer to the slot it would take.  If a
00870 /// node already exists with these operands, the slot will be non-null.
00871 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
00872                                            SDValue Op1, SDValue Op2,
00873                                            void *&InsertPos) {
00874   if (doNotCSE(N))
00875     return nullptr;
00876 
00877   SDValue Ops[] = { Op1, Op2 };
00878   FoldingSetNodeID ID;
00879   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00880   AddNodeIDCustom(ID, N);
00881   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00882   return Node;
00883 }
00884 
00885 
00886 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00887 /// were replaced with those specified.  If this node is never memoized,
00888 /// return null, otherwise return a pointer to the slot it would take.  If a
00889 /// node already exists with these operands, the slot will be non-null.
00890 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
00891                                            void *&InsertPos) {
00892   if (doNotCSE(N))
00893     return nullptr;
00894 
00895   FoldingSetNodeID ID;
00896   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00897   AddNodeIDCustom(ID, N);
00898   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00899   return Node;
00900 }
00901 
00902 /// getEVTAlignment - Compute the default alignment value for the
00903 /// given type.
00904 ///
00905 unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
00906   Type *Ty = VT == MVT::iPTR ?
00907                    PointerType::get(Type::getInt8Ty(*getContext()), 0) :
00908                    VT.getTypeForEVT(*getContext());
00909 
00910   return TLI->getDataLayout()->getABITypeAlignment(Ty);
00911 }
00912 
00913 // EntryNode could meaningfully have debug info if we can find it...
00914 SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
00915     : TM(tm), TSI(nullptr), TLI(nullptr), OptLevel(OL),
00916       EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
00917       Root(getEntryNode()), NewNodesMustHaveLegalTypes(false),
00918       UpdateListeners(nullptr) {
00919   AllNodes.push_back(&EntryNode);
00920   DbgInfo = new SDDbgInfo();
00921 }
00922 
00923 void SelectionDAG::init(MachineFunction &mf) {
00924   MF = &mf;
00925   TLI = getSubtarget().getTargetLowering();
00926   TSI = getSubtarget().getSelectionDAGInfo();
00927   Context = &mf.getFunction()->getContext();
00928 }
00929 
00930 SelectionDAG::~SelectionDAG() {
00931   assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
00932   allnodes_clear();
00933   delete DbgInfo;
00934 }
00935 
00936 void SelectionDAG::allnodes_clear() {
00937   assert(&*AllNodes.begin() == &EntryNode);
00938   AllNodes.remove(AllNodes.begin());
00939   while (!AllNodes.empty())
00940     DeallocateNode(AllNodes.begin());
00941 }
00942 
00943 BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL,
00944                                             SDVTList VTs, SDValue N1,
00945                                             SDValue N2, bool nuw, bool nsw,
00946                                             bool exact) {
00947   if (isBinOpWithFlags(Opcode)) {
00948     BinaryWithFlagsSDNode *FN = new (NodeAllocator) BinaryWithFlagsSDNode(
00949         Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
00950     FN->setHasNoUnsignedWrap(nuw);
00951     FN->setHasNoSignedWrap(nsw);
00952     FN->setIsExact(exact);
00953 
00954     return FN;
00955   }
00956 
00957   BinarySDNode *N = new (NodeAllocator)
00958       BinarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
00959   return N;
00960 }
00961 
00962 void SelectionDAG::clear() {
00963   allnodes_clear();
00964   OperandAllocator.Reset();
00965   CSEMap.clear();
00966 
00967   ExtendedValueTypeNodes.clear();
00968   ExternalSymbols.clear();
00969   TargetExternalSymbols.clear();
00970   std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
00971             static_cast<CondCodeSDNode*>(nullptr));
00972   std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
00973             static_cast<SDNode*>(nullptr));
00974 
00975   EntryNode.UseList = nullptr;
00976   AllNodes.push_back(&EntryNode);
00977   Root = getEntryNode();
00978   DbgInfo->clear();
00979 }
00980 
00981 SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00982   return VT.bitsGT(Op.getValueType()) ?
00983     getNode(ISD::ANY_EXTEND, DL, VT, Op) :
00984     getNode(ISD::TRUNCATE, DL, VT, Op);
00985 }
00986 
00987 SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00988   return VT.bitsGT(Op.getValueType()) ?
00989     getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
00990     getNode(ISD::TRUNCATE, DL, VT, Op);
00991 }
00992 
00993 SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00994   return VT.bitsGT(Op.getValueType()) ?
00995     getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
00996     getNode(ISD::TRUNCATE, DL, VT, Op);
00997 }
00998 
00999 SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, SDLoc SL, EVT VT,
01000                                         EVT OpVT) {
01001   if (VT.bitsLE(Op.getValueType()))
01002     return getNode(ISD::TRUNCATE, SL, VT, Op);
01003 
01004   TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
01005   return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
01006 }
01007 
01008 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, SDLoc DL, EVT VT) {
01009   assert(!VT.isVector() &&
01010          "getZeroExtendInReg should use the vector element type instead of "
01011          "the vector type!");
01012   if (Op.getValueType() == VT) return Op;
01013   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01014   APInt Imm = APInt::getLowBitsSet(BitWidth,
01015                                    VT.getSizeInBits());
01016   return getNode(ISD::AND, DL, Op.getValueType(), Op,
01017                  getConstant(Imm, Op.getValueType()));
01018 }
01019 
01020 SDValue SelectionDAG::getAnyExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01021   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01022   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01023          "The sizes of the input and result must match in order to perform the "
01024          "extend in-register.");
01025   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01026          "The destination vector type must have fewer lanes than the input.");
01027   return getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Op);
01028 }
01029 
01030 SDValue SelectionDAG::getSignExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01031   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01032   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01033          "The sizes of the input and result must match in order to perform the "
01034          "extend in-register.");
01035   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01036          "The destination vector type must have fewer lanes than the input.");
01037   return getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Op);
01038 }
01039 
01040 SDValue SelectionDAG::getZeroExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01041   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01042   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01043          "The sizes of the input and result must match in order to perform the "
01044          "extend in-register.");
01045   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01046          "The destination vector type must have fewer lanes than the input.");
01047   return getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Op);
01048 }
01049 
01050 /// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
01051 ///
01052 SDValue SelectionDAG::getNOT(SDLoc DL, SDValue Val, EVT VT) {
01053   EVT EltVT = VT.getScalarType();
01054   SDValue NegOne =
01055     getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT);
01056   return getNode(ISD::XOR, DL, VT, Val, NegOne);
01057 }
01058 
01059 SDValue SelectionDAG::getLogicalNOT(SDLoc DL, SDValue Val, EVT VT) {
01060   EVT EltVT = VT.getScalarType();
01061   SDValue TrueValue;
01062   switch (TLI->getBooleanContents(VT)) {
01063     case TargetLowering::ZeroOrOneBooleanContent:
01064     case TargetLowering::UndefinedBooleanContent:
01065       TrueValue = getConstant(1, VT);
01066       break;
01067     case TargetLowering::ZeroOrNegativeOneBooleanContent:
01068       TrueValue = getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()),
01069                               VT);
01070       break;
01071   }
01072   return getNode(ISD::XOR, DL, VT, Val, TrueValue);
01073 }
01074 
01075 SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT, bool isO) {
01076   EVT EltVT = VT.getScalarType();
01077   assert((EltVT.getSizeInBits() >= 64 ||
01078          (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
01079          "getConstant with a uint64_t value that doesn't fit in the type!");
01080   return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT, isO);
01081 }
01082 
01083 SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT, bool isO)
01084 {
01085   return getConstant(*ConstantInt::get(*Context, Val), VT, isT, isO);
01086 }
01087 
01088 SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT,
01089                                   bool isO) {
01090   assert(VT.isInteger() && "Cannot create FP integer constant!");
01091 
01092   EVT EltVT = VT.getScalarType();
01093   const ConstantInt *Elt = &Val;
01094 
01095   // In some cases the vector type is legal but the element type is illegal and
01096   // needs to be promoted, for example v8i8 on ARM.  In this case, promote the
01097   // inserted value (the type does not need to match the vector element type).
01098   // Any extra bits introduced will be truncated away.
01099   if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
01100       TargetLowering::TypePromoteInteger) {
01101    EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
01102    APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits());
01103    Elt = ConstantInt::get(*getContext(), NewVal);
01104   }
01105   // In other cases the element type is illegal and needs to be expanded, for
01106   // example v2i64 on MIPS32. In this case, find the nearest legal type, split
01107   // the value into n parts and use a vector type with n-times the elements.
01108   // Then bitcast to the type requested.
01109   // Legalizing constants too early makes the DAGCombiner's job harder so we
01110   // only legalize if the DAG tells us we must produce legal types.
01111   else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
01112            TLI->getTypeAction(*getContext(), EltVT) ==
01113            TargetLowering::TypeExpandInteger) {
01114     APInt NewVal = Elt->getValue();
01115     EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
01116     unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
01117     unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
01118     EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
01119 
01120     // Check the temporary vector is the correct size. If this fails then
01121     // getTypeToTransformTo() probably returned a type whose size (in bits)
01122     // isn't a power-of-2 factor of the requested type size.
01123     assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
01124 
01125     SmallVector<SDValue, 2> EltParts;
01126     for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
01127       EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
01128                                            .trunc(ViaEltSizeInBits),
01129                                      ViaEltVT, isT, isO));
01130     }
01131 
01132     // EltParts is currently in little endian order. If we actually want
01133     // big-endian order then reverse it now.
01134     if (TLI->isBigEndian())
01135       std::reverse(EltParts.begin(), EltParts.end());
01136 
01137     // The elements must be reversed when the element order is different
01138     // to the endianness of the elements (because the BITCAST is itself a
01139     // vector shuffle in this situation). However, we do not need any code to
01140     // perform this reversal because getConstant() is producing a vector
01141     // splat.
01142     // This situation occurs in MIPS MSA.
01143 
01144     SmallVector<SDValue, 8> Ops;
01145     for (unsigned i = 0; i < VT.getVectorNumElements(); ++i)
01146       Ops.insert(Ops.end(), EltParts.begin(), EltParts.end());
01147 
01148     SDValue Result = getNode(ISD::BITCAST, SDLoc(), VT,
01149                              getNode(ISD::BUILD_VECTOR, SDLoc(), ViaVecVT,
01150                                      Ops));
01151     return Result;
01152   }
01153 
01154   assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
01155          "APInt size does not match type size!");
01156   unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
01157   FoldingSetNodeID ID;
01158   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
01159   ID.AddPointer(Elt);
01160   ID.AddBoolean(isO);
01161   void *IP = nullptr;
01162   SDNode *N = nullptr;
01163   if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
01164     if (!VT.isVector())
01165       return SDValue(N, 0);
01166 
01167   if (!N) {
01168     N = new (NodeAllocator) ConstantSDNode(isT, isO, Elt, EltVT);
01169     CSEMap.InsertNode(N, IP);
01170     InsertNode(N);
01171   }
01172 
01173   SDValue Result(N, 0);
01174   if (VT.isVector()) {
01175     SmallVector<SDValue, 8> Ops;
01176     Ops.assign(VT.getVectorNumElements(), Result);
01177     Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
01178   }
01179   return Result;
01180 }
01181 
01182 SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) {
01183   return getConstant(Val, TLI->getPointerTy(), isTarget);
01184 }
01185 
01186 
01187 SDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) {
01188   return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget);
01189 }
01190 
01191 SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){
01192   assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
01193 
01194   EVT EltVT = VT.getScalarType();
01195 
01196   // Do the map lookup using the actual bit pattern for the floating point
01197   // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
01198   // we don't have issues with SNANs.
01199   unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
01200   FoldingSetNodeID ID;
01201   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
01202   ID.AddPointer(&V);
01203   void *IP = nullptr;
01204   SDNode *N = nullptr;
01205   if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
01206     if (!VT.isVector())
01207       return SDValue(N, 0);
01208 
01209   if (!N) {
01210     N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT);
01211     CSEMap.InsertNode(N, IP);
01212     InsertNode(N);
01213   }
01214 
01215   SDValue Result(N, 0);
01216   if (VT.isVector()) {
01217     SmallVector<SDValue, 8> Ops;
01218     Ops.assign(VT.getVectorNumElements(), Result);
01219     // FIXME SDLoc info might be appropriate here
01220     Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
01221   }
01222   return Result;
01223 }
01224 
01225 SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) {
01226   EVT EltVT = VT.getScalarType();
01227   if (EltVT==MVT::f32)
01228     return getConstantFP(APFloat((float)Val), VT, isTarget);
01229   else if (EltVT==MVT::f64)
01230     return getConstantFP(APFloat(Val), VT, isTarget);
01231   else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 ||
01232            EltVT==MVT::f16) {
01233     bool ignored;
01234     APFloat apf = APFloat(Val);
01235     apf.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
01236                 &ignored);
01237     return getConstantFP(apf, VT, isTarget);
01238   } else
01239     llvm_unreachable("Unsupported type in getConstantFP");
01240 }
01241 
01242 SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, SDLoc DL,
01243                                        EVT VT, int64_t Offset,
01244                                        bool isTargetGA,
01245                                        unsigned char TargetFlags) {
01246   assert((TargetFlags == 0 || isTargetGA) &&
01247          "Cannot set target flags on target-independent globals");
01248 
01249   // Truncate (with sign-extension) the offset value to the pointer size.
01250   unsigned BitWidth = TLI->getPointerTypeSizeInBits(GV->getType());
01251   if (BitWidth < 64)
01252     Offset = SignExtend64(Offset, BitWidth);
01253 
01254   unsigned Opc;
01255   if (GV->isThreadLocal())
01256     Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
01257   else
01258     Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
01259 
01260   FoldingSetNodeID ID;
01261   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01262   ID.AddPointer(GV);
01263   ID.AddInteger(Offset);
01264   ID.AddInteger(TargetFlags);
01265   ID.AddInteger(GV->getType()->getAddressSpace());
01266   void *IP = nullptr;
01267   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01268     return SDValue(E, 0);
01269 
01270   SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL.getIROrder(),
01271                                                       DL.getDebugLoc(), GV, VT,
01272                                                       Offset, TargetFlags);
01273   CSEMap.InsertNode(N, IP);
01274     InsertNode(N);
01275   return SDValue(N, 0);
01276 }
01277 
01278 SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
01279   unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
01280   FoldingSetNodeID ID;
01281   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01282   ID.AddInteger(FI);
01283   void *IP = nullptr;
01284   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01285     return SDValue(E, 0);
01286 
01287   SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget);
01288   CSEMap.InsertNode(N, IP);
01289   InsertNode(N);
01290   return SDValue(N, 0);
01291 }
01292 
01293 SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
01294                                    unsigned char TargetFlags) {
01295   assert((TargetFlags == 0 || isTarget) &&
01296          "Cannot set target flags on target-independent jump tables");
01297   unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
01298   FoldingSetNodeID ID;
01299   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01300   ID.AddInteger(JTI);
01301   ID.AddInteger(TargetFlags);
01302   void *IP = nullptr;
01303   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01304     return SDValue(E, 0);
01305 
01306   SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget,
01307                                                   TargetFlags);
01308   CSEMap.InsertNode(N, IP);
01309   InsertNode(N);
01310   return SDValue(N, 0);
01311 }
01312 
01313 SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
01314                                       unsigned Alignment, int Offset,
01315                                       bool isTarget,
01316                                       unsigned char TargetFlags) {
01317   assert((TargetFlags == 0 || isTarget) &&
01318          "Cannot set target flags on target-independent globals");
01319   if (Alignment == 0)
01320     Alignment = TLI->getDataLayout()->getPrefTypeAlignment(C->getType());
01321   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
01322   FoldingSetNodeID ID;
01323   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01324   ID.AddInteger(Alignment);
01325   ID.AddInteger(Offset);
01326   ID.AddPointer(C);
01327   ID.AddInteger(TargetFlags);
01328   void *IP = nullptr;
01329   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01330     return SDValue(E, 0);
01331 
01332   SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
01333                                                      Alignment, TargetFlags);
01334   CSEMap.InsertNode(N, IP);
01335   InsertNode(N);
01336   return SDValue(N, 0);
01337 }
01338 
01339 
01340 SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
01341                                       unsigned Alignment, int Offset,
01342                                       bool isTarget,
01343                                       unsigned char TargetFlags) {
01344   assert((TargetFlags == 0 || isTarget) &&
01345          "Cannot set target flags on target-independent globals");
01346   if (Alignment == 0)
01347     Alignment = TLI->getDataLayout()->getPrefTypeAlignment(C->getType());
01348   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
01349   FoldingSetNodeID ID;
01350   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01351   ID.AddInteger(Alignment);
01352   ID.AddInteger(Offset);
01353   C->addSelectionDAGCSEId(ID);
01354   ID.AddInteger(TargetFlags);
01355   void *IP = nullptr;
01356   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01357     return SDValue(E, 0);
01358 
01359   SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
01360                                                      Alignment, TargetFlags);
01361   CSEMap.InsertNode(N, IP);
01362   InsertNode(N);
01363   return SDValue(N, 0);
01364 }
01365 
01366 SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
01367                                      unsigned char TargetFlags) {
01368   FoldingSetNodeID ID;
01369   AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None);
01370   ID.AddInteger(Index);
01371   ID.AddInteger(Offset);
01372   ID.AddInteger(TargetFlags);
01373   void *IP = nullptr;
01374   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01375     return SDValue(E, 0);
01376 
01377   SDNode *N = new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset,
01378                                                     TargetFlags);
01379   CSEMap.InsertNode(N, IP);
01380   InsertNode(N);
01381   return SDValue(N, 0);
01382 }
01383 
01384 SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
01385   FoldingSetNodeID ID;
01386   AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
01387   ID.AddPointer(MBB);
01388   void *IP = nullptr;
01389   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01390     return SDValue(E, 0);
01391 
01392   SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB);
01393   CSEMap.InsertNode(N, IP);
01394   InsertNode(N);
01395   return SDValue(N, 0);
01396 }
01397 
01398 SDValue SelectionDAG::getValueType(EVT VT) {
01399   if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
01400       ValueTypeNodes.size())
01401     ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
01402 
01403   SDNode *&N = VT.isExtended() ?
01404     ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
01405 
01406   if (N) return SDValue(N, 0);
01407   N = new (NodeAllocator) VTSDNode(VT);
01408   InsertNode(N);
01409   return SDValue(N, 0);
01410 }
01411 
01412 SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
01413   SDNode *&N = ExternalSymbols[Sym];
01414   if (N) return SDValue(N, 0);
01415   N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT);
01416   InsertNode(N);
01417   return SDValue(N, 0);
01418 }
01419 
01420 SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
01421                                               unsigned char TargetFlags) {
01422   SDNode *&N =
01423     TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym,
01424                                                                TargetFlags)];
01425   if (N) return SDValue(N, 0);
01426   N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT);
01427   InsertNode(N);
01428   return SDValue(N, 0);
01429 }
01430 
01431 SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
01432   if ((unsigned)Cond >= CondCodeNodes.size())
01433     CondCodeNodes.resize(Cond+1);
01434 
01435   if (!CondCodeNodes[Cond]) {
01436     CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond);
01437     CondCodeNodes[Cond] = N;
01438     InsertNode(N);
01439   }
01440 
01441   return SDValue(CondCodeNodes[Cond], 0);
01442 }
01443 
01444 // commuteShuffle - swaps the values of N1 and N2, and swaps all indices in
01445 // the shuffle mask M that point at N1 to point at N2, and indices that point
01446 // N2 to point at N1.
01447 static void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) {
01448   std::swap(N1, N2);
01449   int NElts = M.size();
01450   for (int i = 0; i != NElts; ++i) {
01451     if (M[i] >= NElts)
01452       M[i] -= NElts;
01453     else if (M[i] >= 0)
01454       M[i] += NElts;
01455   }
01456 }
01457 
01458 SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1,
01459                                        SDValue N2, const int *Mask) {
01460   assert(VT == N1.getValueType() && VT == N2.getValueType() &&
01461          "Invalid VECTOR_SHUFFLE");
01462 
01463   // Canonicalize shuffle undef, undef -> undef
01464   if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF)
01465     return getUNDEF(VT);
01466 
01467   // Validate that all indices in Mask are within the range of the elements
01468   // input to the shuffle.
01469   unsigned NElts = VT.getVectorNumElements();
01470   SmallVector<int, 8> MaskVec;
01471   for (unsigned i = 0; i != NElts; ++i) {
01472     assert(Mask[i] < (int)(NElts * 2) && "Index out of range");
01473     MaskVec.push_back(Mask[i]);
01474   }
01475 
01476   // Canonicalize shuffle v, v -> v, undef
01477   if (N1 == N2) {
01478     N2 = getUNDEF(VT);
01479     for (unsigned i = 0; i != NElts; ++i)
01480       if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts;
01481   }
01482 
01483   // Canonicalize shuffle undef, v -> v, undef.  Commute the shuffle mask.
01484   if (N1.getOpcode() == ISD::UNDEF)
01485     commuteShuffle(N1, N2, MaskVec);
01486 
01487   // Canonicalize all index into lhs, -> shuffle lhs, undef
01488   // Canonicalize all index into rhs, -> shuffle rhs, undef
01489   bool AllLHS = true, AllRHS = true;
01490   bool N2Undef = N2.getOpcode() == ISD::UNDEF;
01491   for (unsigned i = 0; i != NElts; ++i) {
01492     if (MaskVec[i] >= (int)NElts) {
01493       if (N2Undef)
01494         MaskVec[i] = -1;
01495       else
01496         AllLHS = false;
01497     } else if (MaskVec[i] >= 0) {
01498       AllRHS = false;
01499     }
01500   }
01501   if (AllLHS && AllRHS)
01502     return getUNDEF(VT);
01503   if (AllLHS && !N2Undef)
01504     N2 = getUNDEF(VT);
01505   if (AllRHS) {
01506     N1 = getUNDEF(VT);
01507     commuteShuffle(N1, N2, MaskVec);
01508   }
01509   // Reset our undef status after accounting for the mask.
01510   N2Undef = N2.getOpcode() == ISD::UNDEF;
01511   // Re-check whether both sides ended up undef.
01512   if (N1.getOpcode() == ISD::UNDEF && N2Undef)
01513     return getUNDEF(VT);
01514 
01515   // If Identity shuffle return that node.
01516   bool Identity = true, AllSame = true;
01517   for (unsigned i = 0; i != NElts; ++i) {
01518     if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false;
01519     if (MaskVec[i] != MaskVec[0]) AllSame = false;
01520   }
01521   if (Identity && NElts)
01522     return N1;
01523 
01524   // Shuffling a constant splat doesn't change the result.
01525   if (N2Undef) {
01526     SDValue V = N1;
01527 
01528     // Look through any bitcasts. We check that these don't change the number
01529     // (and size) of elements and just changes their types.
01530     while (V.getOpcode() == ISD::BITCAST)
01531       V = V->getOperand(0);
01532 
01533     // A splat should always show up as a build vector node.
01534     if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
01535       BitVector UndefElements;
01536       SDValue Splat = BV->getSplatValue(&UndefElements);
01537       // If this is a splat of an undef, shuffling it is also undef.
01538       if (Splat && Splat.getOpcode() == ISD::UNDEF)
01539         return getUNDEF(VT);
01540 
01541       bool SameNumElts =
01542           V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
01543 
01544       // We only have a splat which can skip shuffles if there is a splatted
01545       // value and no undef lanes rearranged by the shuffle.
01546       if (Splat && UndefElements.none()) {
01547         // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
01548         // number of elements match or the value splatted is a zero constant.
01549         if (SameNumElts)
01550           return N1;
01551         if (auto *C = dyn_cast<ConstantSDNode>(Splat))
01552           if (C->isNullValue())
01553             return N1;
01554       }
01555 
01556       // If the shuffle itself creates a constant splat, build the vector
01557       // directly.
01558       if (AllSame && SameNumElts) {
01559          const SDValue &Splatted = BV->getOperand(MaskVec[0]);
01560          if (isa<ConstantSDNode>(Splatted) || isa<ConstantFPSDNode>(Splatted)) {
01561            SmallVector<SDValue, 8> Ops;
01562            for (unsigned i = 0; i != NElts; ++i)
01563              Ops.push_back(Splatted);
01564 
01565            SDValue NewBV =
01566                getNode(ISD::BUILD_VECTOR, dl, BV->getValueType(0), Ops);
01567 
01568            // We may have jumped through bitcasts, so the type of the
01569            // BUILD_VECTOR may not match the type of the shuffle.
01570            if (BV->getValueType(0) != VT)
01571              NewBV = getNode(ISD::BITCAST, dl, VT, NewBV);
01572            return NewBV;
01573          }
01574       }
01575     }
01576   }
01577 
01578   FoldingSetNodeID ID;
01579   SDValue Ops[2] = { N1, N2 };
01580   AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
01581   for (unsigned i = 0; i != NElts; ++i)
01582     ID.AddInteger(MaskVec[i]);
01583 
01584   void* IP = nullptr;
01585   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01586     return SDValue(E, 0);
01587 
01588   // Allocate the mask array for the node out of the BumpPtrAllocator, since
01589   // SDNode doesn't have access to it.  This memory will be "leaked" when
01590   // the node is deallocated, but recovered when the NodeAllocator is released.
01591   int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
01592   memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int));
01593 
01594   ShuffleVectorSDNode *N =
01595     new (NodeAllocator) ShuffleVectorSDNode(VT, dl.getIROrder(),
01596                                             dl.getDebugLoc(), N1, N2,
01597                                             MaskAlloc);
01598   CSEMap.InsertNode(N, IP);
01599   InsertNode(N);
01600   return SDValue(N, 0);
01601 }
01602 
01603 SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
01604   MVT VT = SV.getSimpleValueType(0);
01605   unsigned NumElems = VT.getVectorNumElements();
01606   SmallVector<int, 8> MaskVec;
01607 
01608   for (unsigned i = 0; i != NumElems; ++i) {
01609     int Idx = SV.getMaskElt(i);
01610     if (Idx >= 0) {
01611       if (Idx < (int)NumElems)
01612         Idx += NumElems;
01613       else
01614         Idx -= NumElems;
01615     }
01616     MaskVec.push_back(Idx);
01617   }
01618 
01619   SDValue Op0 = SV.getOperand(0);
01620   SDValue Op1 = SV.getOperand(1);
01621   return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, &MaskVec[0]);
01622 }
01623 
01624 SDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl,
01625                                        SDValue Val, SDValue DTy,
01626                                        SDValue STy, SDValue Rnd, SDValue Sat,
01627                                        ISD::CvtCode Code) {
01628   // If the src and dest types are the same and the conversion is between
01629   // integer types of the same sign or two floats, no conversion is necessary.
01630   if (DTy == STy &&
01631       (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF))
01632     return Val;
01633 
01634   FoldingSetNodeID ID;
01635   SDValue Ops[] = { Val, DTy, STy, Rnd, Sat };
01636   AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), Ops);
01637   void* IP = nullptr;
01638   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01639     return SDValue(E, 0);
01640 
01641   CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl.getIROrder(),
01642                                                            dl.getDebugLoc(),
01643                                                            Ops, Code);
01644   CSEMap.InsertNode(N, IP);
01645   InsertNode(N);
01646   return SDValue(N, 0);
01647 }
01648 
01649 SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
01650   FoldingSetNodeID ID;
01651   AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
01652   ID.AddInteger(RegNo);
01653   void *IP = nullptr;
01654   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01655     return SDValue(E, 0);
01656 
01657   SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT);
01658   CSEMap.InsertNode(N, IP);
01659   InsertNode(N);
01660   return SDValue(N, 0);
01661 }
01662 
01663 SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
01664   FoldingSetNodeID ID;
01665   AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
01666   ID.AddPointer(RegMask);
01667   void *IP = nullptr;
01668   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01669     return SDValue(E, 0);
01670 
01671   SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask);
01672   CSEMap.InsertNode(N, IP);
01673   InsertNode(N);
01674   return SDValue(N, 0);
01675 }
01676 
01677 SDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) {
01678   FoldingSetNodeID ID;
01679   SDValue Ops[] = { Root };
01680   AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), Ops);
01681   ID.AddPointer(Label);
01682   void *IP = nullptr;
01683   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01684     return SDValue(E, 0);
01685 
01686   SDNode *N = new (NodeAllocator) EHLabelSDNode(dl.getIROrder(),
01687                                                 dl.getDebugLoc(), Root, Label);
01688   CSEMap.InsertNode(N, IP);
01689   InsertNode(N);
01690   return SDValue(N, 0);
01691 }
01692 
01693 
01694 SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
01695                                       int64_t Offset,
01696                                       bool isTarget,
01697                                       unsigned char TargetFlags) {
01698   unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
01699 
01700   FoldingSetNodeID ID;
01701   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01702   ID.AddPointer(BA);
01703   ID.AddInteger(Offset);
01704   ID.AddInteger(TargetFlags);
01705   void *IP = nullptr;
01706   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01707     return SDValue(E, 0);
01708 
01709   SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset,
01710                                                      TargetFlags);
01711   CSEMap.InsertNode(N, IP);
01712   InsertNode(N);
01713   return SDValue(N, 0);
01714 }
01715 
01716 SDValue SelectionDAG::getSrcValue(const Value *V) {
01717   assert((!V || V->getType()->isPointerTy()) &&
01718          "SrcValue is not a pointer?");
01719 
01720   FoldingSetNodeID ID;
01721   AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None);
01722   ID.AddPointer(V);
01723 
01724   void *IP = nullptr;
01725   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01726     return SDValue(E, 0);
01727 
01728   SDNode *N = new (NodeAllocator) SrcValueSDNode(V);
01729   CSEMap.InsertNode(N, IP);
01730   InsertNode(N);
01731   return SDValue(N, 0);
01732 }
01733 
01734 /// getMDNode - Return an MDNodeSDNode which holds an MDNode.
01735 SDValue SelectionDAG::getMDNode(const MDNode *MD) {
01736   FoldingSetNodeID ID;
01737   AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None);
01738   ID.AddPointer(MD);
01739 
01740   void *IP = nullptr;
01741   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01742     return SDValue(E, 0);
01743 
01744   SDNode *N = new (NodeAllocator) MDNodeSDNode(MD);
01745   CSEMap.InsertNode(N, IP);
01746   InsertNode(N);
01747   return SDValue(N, 0);
01748 }
01749 
01750 /// getAddrSpaceCast - Return an AddrSpaceCastSDNode.
01751 SDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr,
01752                                        unsigned SrcAS, unsigned DestAS) {
01753   SDValue Ops[] = {Ptr};
01754   FoldingSetNodeID ID;
01755   AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
01756   ID.AddInteger(SrcAS);
01757   ID.AddInteger(DestAS);
01758 
01759   void *IP = nullptr;
01760   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01761     return SDValue(E, 0);
01762 
01763   SDNode *N = new (NodeAllocator) AddrSpaceCastSDNode(dl.getIROrder(),
01764                                                       dl.getDebugLoc(),
01765                                                       VT, Ptr, SrcAS, DestAS);
01766   CSEMap.InsertNode(N, IP);
01767   InsertNode(N);
01768   return SDValue(N, 0);
01769 }
01770 
01771 /// getShiftAmountOperand - Return the specified value casted to
01772 /// the target's desired shift amount type.
01773 SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
01774   EVT OpTy = Op.getValueType();
01775   EVT ShTy = TLI->getShiftAmountTy(LHSTy);
01776   if (OpTy == ShTy || OpTy.isVector()) return Op;
01777 
01778   ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ?  ISD::TRUNCATE : ISD::ZERO_EXTEND;
01779   return getNode(Opcode, SDLoc(Op), ShTy, Op);
01780 }
01781 
01782 /// CreateStackTemporary - Create a stack temporary, suitable for holding the
01783 /// specified value type.
01784 SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
01785   MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
01786   unsigned ByteSize = VT.getStoreSize();
01787   Type *Ty = VT.getTypeForEVT(*getContext());
01788   unsigned StackAlign =
01789   std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty), minAlign);
01790 
01791   int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false);
01792   return getFrameIndex(FrameIdx, TLI->getPointerTy());
01793 }
01794 
01795 /// CreateStackTemporary - Create a stack temporary suitable for holding
01796 /// either of the specified value types.
01797 SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
01798   unsigned Bytes = std::max(VT1.getStoreSizeInBits(),
01799                             VT2.getStoreSizeInBits())/8;
01800   Type *Ty1 = VT1.getTypeForEVT(*getContext());
01801   Type *Ty2 = VT2.getTypeForEVT(*getContext());
01802   const DataLayout *TD = TLI->getDataLayout();
01803   unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1),
01804                             TD->getPrefTypeAlignment(Ty2));
01805 
01806   MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
01807   int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false);
01808   return getFrameIndex(FrameIdx, TLI->getPointerTy());
01809 }
01810 
01811 SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1,
01812                                 SDValue N2, ISD::CondCode Cond, SDLoc dl) {
01813   // These setcc operations always fold.
01814   switch (Cond) {
01815   default: break;
01816   case ISD::SETFALSE:
01817   case ISD::SETFALSE2: return getConstant(0, VT);
01818   case ISD::SETTRUE:
01819   case ISD::SETTRUE2: {
01820     TargetLowering::BooleanContent Cnt =
01821         TLI->getBooleanContents(N1->getValueType(0));
01822     return getConstant(
01823         Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, VT);
01824   }
01825 
01826   case ISD::SETOEQ:
01827   case ISD::SETOGT:
01828   case ISD::SETOGE:
01829   case ISD::SETOLT:
01830   case ISD::SETOLE:
01831   case ISD::SETONE:
01832   case ISD::SETO:
01833   case ISD::SETUO:
01834   case ISD::SETUEQ:
01835   case ISD::SETUNE:
01836     assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!");
01837     break;
01838   }
01839 
01840   if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) {
01841     const APInt &C2 = N2C->getAPIntValue();
01842     if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
01843       const APInt &C1 = N1C->getAPIntValue();
01844 
01845       switch (Cond) {
01846       default: llvm_unreachable("Unknown integer setcc!");
01847       case ISD::SETEQ:  return getConstant(C1 == C2, VT);
01848       case ISD::SETNE:  return getConstant(C1 != C2, VT);
01849       case ISD::SETULT: return getConstant(C1.ult(C2), VT);
01850       case ISD::SETUGT: return getConstant(C1.ugt(C2), VT);
01851       case ISD::SETULE: return getConstant(C1.ule(C2), VT);
01852       case ISD::SETUGE: return getConstant(C1.uge(C2), VT);
01853       case ISD::SETLT:  return getConstant(C1.slt(C2), VT);
01854       case ISD::SETGT:  return getConstant(C1.sgt(C2), VT);
01855       case ISD::SETLE:  return getConstant(C1.sle(C2), VT);
01856       case ISD::SETGE:  return getConstant(C1.sge(C2), VT);
01857       }
01858     }
01859   }
01860   if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) {
01861     if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) {
01862       APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF());
01863       switch (Cond) {
01864       default: break;
01865       case ISD::SETEQ:  if (R==APFloat::cmpUnordered)
01866                           return getUNDEF(VT);
01867                         // fall through
01868       case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT);
01869       case ISD::SETNE:  if (R==APFloat::cmpUnordered)
01870                           return getUNDEF(VT);
01871                         // fall through
01872       case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan ||
01873                                            R==APFloat::cmpLessThan, VT);
01874       case ISD::SETLT:  if (R==APFloat::cmpUnordered)
01875                           return getUNDEF(VT);
01876                         // fall through
01877       case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT);
01878       case ISD::SETGT:  if (R==APFloat::cmpUnordered)
01879                           return getUNDEF(VT);
01880                         // fall through
01881       case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT);
01882       case ISD::SETLE:  if (R==APFloat::cmpUnordered)
01883                           return getUNDEF(VT);
01884                         // fall through
01885       case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan ||
01886                                            R==APFloat::cmpEqual, VT);
01887       case ISD::SETGE:  if (R==APFloat::cmpUnordered)
01888                           return getUNDEF(VT);
01889                         // fall through
01890       case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan ||
01891                                            R==APFloat::cmpEqual, VT);
01892       case ISD::SETO:   return getConstant(R!=APFloat::cmpUnordered, VT);
01893       case ISD::SETUO:  return getConstant(R==APFloat::cmpUnordered, VT);
01894       case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered ||
01895                                            R==APFloat::cmpEqual, VT);
01896       case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT);
01897       case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered ||
01898                                            R==APFloat::cmpLessThan, VT);
01899       case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan ||
01900                                            R==APFloat::cmpUnordered, VT);
01901       case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT);
01902       case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT);
01903       }
01904     } else {
01905       // Ensure that the constant occurs on the RHS.
01906       ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
01907       MVT CompVT = N1.getValueType().getSimpleVT();
01908       if (!TLI->isCondCodeLegal(SwappedCond, CompVT))
01909         return SDValue();
01910 
01911       return getSetCC(dl, VT, N2, N1, SwappedCond);
01912     }
01913   }
01914 
01915   // Could not fold it.
01916   return SDValue();
01917 }
01918 
01919 /// SignBitIsZero - Return true if the sign bit of Op is known to be zero.  We
01920 /// use this predicate to simplify operations downstream.
01921 bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
01922   // This predicate is not safe for vector operations.
01923   if (Op.getValueType().isVector())
01924     return false;
01925 
01926   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01927   return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth);
01928 }
01929 
01930 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero.  We use
01931 /// this predicate to simplify operations downstream.  Mask is known to be zero
01932 /// for bits that V cannot have.
01933 bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask,
01934                                      unsigned Depth) const {
01935   APInt KnownZero, KnownOne;
01936   computeKnownBits(Op, KnownZero, KnownOne, Depth);
01937   return (KnownZero & Mask) == Mask;
01938 }
01939 
01940 /// Determine which bits of Op are known to be either zero or one and return
01941 /// them in the KnownZero/KnownOne bitsets.
01942 void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
01943                                     APInt &KnownOne, unsigned Depth) const {
01944   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01945 
01946   KnownZero = KnownOne = APInt(BitWidth, 0);   // Don't know anything.
01947   if (Depth == 6)
01948     return;  // Limit search depth.
01949 
01950   APInt KnownZero2, KnownOne2;
01951 
01952   switch (Op.getOpcode()) {
01953   case ISD::Constant:
01954     // We know all of the bits for a constant!
01955     KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue();
01956     KnownZero = ~KnownOne;
01957     break;
01958   case ISD::AND:
01959     // If either the LHS or the RHS are Zero, the result is zero.
01960     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01961     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01962 
01963     // Output known-1 bits are only known if set in both the LHS & RHS.
01964     KnownOne &= KnownOne2;
01965     // Output known-0 are known to be clear if zero in either the LHS | RHS.
01966     KnownZero |= KnownZero2;
01967     break;
01968   case ISD::OR:
01969     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01970     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01971 
01972     // Output known-0 bits are only known if clear in both the LHS & RHS.
01973     KnownZero &= KnownZero2;
01974     // Output known-1 are known to be set if set in either the LHS | RHS.
01975     KnownOne |= KnownOne2;
01976     break;
01977   case ISD::XOR: {
01978     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01979     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01980 
01981     // Output known-0 bits are known if clear or set in both the LHS & RHS.
01982     APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2);
01983     // Output known-1 are known to be set if set in only one of the LHS, RHS.
01984     KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2);
01985     KnownZero = KnownZeroOut;
01986     break;
01987   }
01988   case ISD::MUL: {
01989     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01990     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01991 
01992     // If low bits are zero in either operand, output low known-0 bits.
01993     // Also compute a conserative estimate for high known-0 bits.
01994     // More trickiness is possible, but this is sufficient for the
01995     // interesting case of alignment computation.
01996     KnownOne.clearAllBits();
01997     unsigned TrailZ = KnownZero.countTrailingOnes() +
01998                       KnownZero2.countTrailingOnes();
01999     unsigned LeadZ =  std::max(KnownZero.countLeadingOnes() +
02000                                KnownZero2.countLeadingOnes(),
02001                                BitWidth) - BitWidth;
02002 
02003     TrailZ = std::min(TrailZ, BitWidth);
02004     LeadZ = std::min(LeadZ, BitWidth);
02005     KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) |
02006                 APInt::getHighBitsSet(BitWidth, LeadZ);
02007     break;
02008   }
02009   case ISD::UDIV: {
02010     // For the purposes of computing leading zeros we can conservatively
02011     // treat a udiv as a logical right shift by the power of 2 known to
02012     // be less than the denominator.
02013     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
02014     unsigned LeadZ = KnownZero2.countLeadingOnes();
02015 
02016     KnownOne2.clearAllBits();
02017     KnownZero2.clearAllBits();
02018     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02019     unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros();
02020     if (RHSUnknownLeadingOnes != BitWidth)
02021       LeadZ = std::min(BitWidth,
02022                        LeadZ + BitWidth - RHSUnknownLeadingOnes - 1);
02023 
02024     KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ);
02025     break;
02026   }
02027   case ISD::SELECT:
02028     computeKnownBits(Op.getOperand(2), KnownZero, KnownOne, Depth+1);
02029     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02030 
02031     // Only known if known in both the LHS and RHS.
02032     KnownOne &= KnownOne2;
02033     KnownZero &= KnownZero2;
02034     break;
02035   case ISD::SELECT_CC:
02036     computeKnownBits(Op.getOperand(3), KnownZero, KnownOne, Depth+1);
02037     computeKnownBits(Op.getOperand(2), KnownZero2, KnownOne2, Depth+1);
02038 
02039     // Only known if known in both the LHS and RHS.
02040     KnownOne &= KnownOne2;
02041     KnownZero &= KnownZero2;
02042     break;
02043   case ISD::SADDO:
02044   case ISD::UADDO:
02045   case ISD::SSUBO:
02046   case ISD::USUBO:
02047   case ISD::SMULO:
02048   case ISD::UMULO:
02049     if (Op.getResNo() != 1)
02050       break;
02051     // The boolean result conforms to getBooleanContents.
02052     // If we know the result of a setcc has the top bits zero, use this info.
02053     // We know that we have an integer-based boolean since these operations
02054     // are only available for integer.
02055     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
02056             TargetLowering::ZeroOrOneBooleanContent &&
02057         BitWidth > 1)
02058       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02059     break;
02060   case ISD::SETCC:
02061     // If we know the result of a setcc has the top bits zero, use this info.
02062     if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
02063             TargetLowering::ZeroOrOneBooleanContent &&
02064         BitWidth > 1)
02065       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02066     break;
02067   case ISD::SHL:
02068     // (shl X, C1) & C2 == 0   iff   (X & C2 >>u C1) == 0
02069     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02070       unsigned ShAmt = SA->getZExtValue();
02071 
02072       // If the shift count is an invalid immediate, don't do anything.
02073       if (ShAmt >= BitWidth)
02074         break;
02075 
02076       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02077       KnownZero <<= ShAmt;
02078       KnownOne  <<= ShAmt;
02079       // low bits known zero.
02080       KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt);
02081     }
02082     break;
02083   case ISD::SRL:
02084     // (ushr X, C1) & C2 == 0   iff  (-1 >> C1) & C2 == 0
02085     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02086       unsigned ShAmt = SA->getZExtValue();
02087 
02088       // If the shift count is an invalid immediate, don't do anything.
02089       if (ShAmt >= BitWidth)
02090         break;
02091 
02092       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02093       KnownZero = KnownZero.lshr(ShAmt);
02094       KnownOne  = KnownOne.lshr(ShAmt);
02095 
02096       APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
02097       KnownZero |= HighBits;  // High bits known zero.
02098     }
02099     break;
02100   case ISD::SRA:
02101     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02102       unsigned ShAmt = SA->getZExtValue();
02103 
02104       // If the shift count is an invalid immediate, don't do anything.
02105       if (ShAmt >= BitWidth)
02106         break;
02107 
02108       // If any of the demanded bits are produced by the sign extension, we also
02109       // demand the input sign bit.
02110       APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
02111 
02112       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02113       KnownZero = KnownZero.lshr(ShAmt);
02114       KnownOne  = KnownOne.lshr(ShAmt);
02115 
02116       // Handle the sign bits.
02117       APInt SignBit = APInt::getSignBit(BitWidth);
02118       SignBit = SignBit.lshr(ShAmt);  // Adjust to where it is now in the mask.
02119 
02120       if (KnownZero.intersects(SignBit)) {
02121         KnownZero |= HighBits;  // New bits are known zero.
02122       } else if (KnownOne.intersects(SignBit)) {
02123         KnownOne  |= HighBits;  // New bits are known one.
02124       }
02125     }
02126     break;
02127   case ISD::SIGN_EXTEND_INREG: {
02128     EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
02129     unsigned EBits = EVT.getScalarType().getSizeInBits();
02130 
02131     // Sign extension.  Compute the demanded bits in the result that are not
02132     // present in the input.
02133     APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits);
02134 
02135     APInt InSignBit = APInt::getSignBit(EBits);
02136     APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits);
02137 
02138     // If the sign extended bits are demanded, we know that the sign
02139     // bit is demanded.
02140     InSignBit = InSignBit.zext(BitWidth);
02141     if (NewBits.getBoolValue())
02142       InputDemandedBits |= InSignBit;
02143 
02144     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02145     KnownOne &= InputDemandedBits;
02146     KnownZero &= InputDemandedBits;
02147 
02148     // If the sign bit of the input is known set or clear, then we know the
02149     // top bits of the result.
02150     if (KnownZero.intersects(InSignBit)) {         // Input sign bit known clear
02151       KnownZero |= NewBits;
02152       KnownOne  &= ~NewBits;
02153     } else if (KnownOne.intersects(InSignBit)) {   // Input sign bit known set
02154       KnownOne  |= NewBits;
02155       KnownZero &= ~NewBits;
02156     } else {                              // Input sign bit unknown
02157       KnownZero &= ~NewBits;
02158       KnownOne  &= ~NewBits;
02159     }
02160     break;
02161   }
02162   case ISD::CTTZ:
02163   case ISD::CTTZ_ZERO_UNDEF:
02164   case ISD::CTLZ:
02165   case ISD::CTLZ_ZERO_UNDEF:
02166   case ISD::CTPOP: {
02167     unsigned LowBits = Log2_32(BitWidth)+1;
02168     KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits);
02169     KnownOne.clearAllBits();
02170     break;
02171   }
02172   case ISD::LOAD: {
02173     LoadSDNode *LD = cast<LoadSDNode>(Op);
02174     // If this is a ZEXTLoad and we are looking at the loaded value.
02175     if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
02176       EVT VT = LD->getMemoryVT();
02177       unsigned MemBits = VT.getScalarType().getSizeInBits();
02178       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
02179     } else if (const MDNode *Ranges = LD->getRanges()) {
02180       computeKnownBitsFromRangeMetadata(*Ranges, KnownZero);
02181     }
02182     break;
02183   }
02184   case ISD::ZERO_EXTEND: {
02185     EVT InVT = Op.getOperand(0).getValueType();
02186     unsigned InBits = InVT.getScalarType().getSizeInBits();
02187     APInt NewBits   = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
02188     KnownZero = KnownZero.trunc(InBits);
02189     KnownOne = KnownOne.trunc(InBits);
02190     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02191     KnownZero = KnownZero.zext(BitWidth);
02192     KnownOne = KnownOne.zext(BitWidth);
02193     KnownZero |= NewBits;
02194     break;
02195   }
02196   case ISD::SIGN_EXTEND: {
02197     EVT InVT = Op.getOperand(0).getValueType();
02198     unsigned InBits = InVT.getScalarType().getSizeInBits();
02199     APInt NewBits   = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
02200 
02201     KnownZero = KnownZero.trunc(InBits);
02202     KnownOne = KnownOne.trunc(InBits);
02203     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02204 
02205     // Note if the sign bit is known to be zero or one.
02206     bool SignBitKnownZero = KnownZero.isNegative();
02207     bool SignBitKnownOne  = KnownOne.isNegative();
02208 
02209     KnownZero = KnownZero.zext(BitWidth);
02210     KnownOne = KnownOne.zext(BitWidth);
02211 
02212     // If the sign bit is known zero or one, the top bits match.
02213     if (SignBitKnownZero)
02214       KnownZero |= NewBits;
02215     else if (SignBitKnownOne)
02216       KnownOne  |= NewBits;
02217     break;
02218   }
02219   case ISD::ANY_EXTEND: {
02220     EVT InVT = Op.getOperand(0).getValueType();
02221     unsigned InBits = InVT.getScalarType().getSizeInBits();
02222     KnownZero = KnownZero.trunc(InBits);
02223     KnownOne = KnownOne.trunc(InBits);
02224     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02225     KnownZero = KnownZero.zext(BitWidth);
02226     KnownOne = KnownOne.zext(BitWidth);
02227     break;
02228   }
02229   case ISD::TRUNCATE: {
02230     EVT InVT = Op.getOperand(0).getValueType();
02231     unsigned InBits = InVT.getScalarType().getSizeInBits();
02232     KnownZero = KnownZero.zext(InBits);
02233     KnownOne = KnownOne.zext(InBits);
02234     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02235     KnownZero = KnownZero.trunc(BitWidth);
02236     KnownOne = KnownOne.trunc(BitWidth);
02237     break;
02238   }
02239   case ISD::AssertZext: {
02240     EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
02241     APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
02242     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02243     KnownZero |= (~InMask);
02244     KnownOne  &= (~KnownZero);
02245     break;
02246   }
02247   case ISD::FGETSIGN:
02248     // All bits are zero except the low bit.
02249     KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02250     break;
02251 
02252   case ISD::SUB: {
02253     if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) {
02254       // We know that the top bits of C-X are clear if X contains less bits
02255       // than C (i.e. no wrap-around can happen).  For example, 20-X is
02256       // positive if we can prove that X is >= 0 and < 16.
02257       if (CLHS->getAPIntValue().isNonNegative()) {
02258         unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros();
02259         // NLZ can't be BitWidth with no sign bit
02260         APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1);
02261         computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02262 
02263         // If all of the MaskV bits are known to be zero, then we know the
02264         // output top bits are zero, because we now know that the output is
02265         // from [0-C].
02266         if ((KnownZero2 & MaskV) == MaskV) {
02267           unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros();
02268           // Top bits known zero.
02269           KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2);
02270         }
02271       }
02272     }
02273   }
02274   // fall through
02275   case ISD::ADD:
02276   case ISD::ADDE: {
02277     // Output known-0 bits are known if clear or set in both the low clear bits
02278     // common to both LHS & RHS.  For example, 8+(X<<3) is known to have the
02279     // low 3 bits clear.
02280     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
02281     unsigned KnownZeroOut = KnownZero2.countTrailingOnes();
02282 
02283     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02284     KnownZeroOut = std::min(KnownZeroOut,
02285                             KnownZero2.countTrailingOnes());
02286 
02287     if (Op.getOpcode() == ISD::ADD) {
02288       KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut);
02289       break;
02290     }
02291 
02292     // With ADDE, a carry bit may be added in, so we can only use this
02293     // information if we know (at least) that the low two bits are clear.  We
02294     // then return to the caller that the low bit is unknown but that other bits
02295     // are known zero.
02296     if (KnownZeroOut >= 2) // ADDE
02297       KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut);
02298     break;
02299   }
02300   case ISD::SREM:
02301     if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02302       const APInt &RA = Rem->getAPIntValue().abs();
02303       if (RA.isPowerOf2()) {
02304         APInt LowBits = RA - 1;
02305         computeKnownBits(Op.getOperand(0), KnownZero2,KnownOne2,Depth+1);
02306 
02307         // The low bits of the first operand are unchanged by the srem.
02308         KnownZero = KnownZero2 & LowBits;
02309         KnownOne = KnownOne2 & LowBits;
02310 
02311         // If the first operand is non-negative or has all low bits zero, then
02312         // the upper bits are all zero.
02313         if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits))
02314           KnownZero |= ~LowBits;
02315 
02316         // If the first operand is negative and not all low bits are zero, then
02317         // the upper bits are all one.
02318         if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0))
02319           KnownOne |= ~LowBits;
02320         assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?");
02321       }
02322     }
02323     break;
02324   case ISD::UREM: {
02325     if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02326       const APInt &RA = Rem->getAPIntValue();
02327       if (RA.isPowerOf2()) {
02328         APInt LowBits = (RA - 1);
02329         computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth + 1);
02330 
02331         // The upper bits are all zero, the lower ones are unchanged.
02332         KnownZero = KnownZero2 | ~LowBits;
02333         KnownOne = KnownOne2 & LowBits;
02334         break;
02335       }
02336     }
02337 
02338     // Since the result is less than or equal to either operand, any leading
02339     // zero bits in either operand must also exist in the result.
02340     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02341     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02342 
02343     uint32_t Leaders = std::max(KnownZero.countLeadingOnes(),
02344                                 KnownZero2.countLeadingOnes());
02345     KnownOne.clearAllBits();
02346     KnownZero = APInt::getHighBitsSet(BitWidth, Leaders);
02347     break;
02348   }
02349   case ISD::EXTRACT_ELEMENT: {
02350     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02351     const unsigned Index =
02352       cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
02353     const unsigned BitWidth = Op.getValueType().getSizeInBits();
02354 
02355     // Remove low part of known bits mask
02356     KnownZero = KnownZero.getHiBits(KnownZero.getBitWidth() - Index * BitWidth);
02357     KnownOne = KnownOne.getHiBits(KnownOne.getBitWidth() - Index * BitWidth);
02358 
02359     // Remove high part of known bit mask
02360     KnownZero = KnownZero.trunc(BitWidth);
02361     KnownOne = KnownOne.trunc(BitWidth);
02362     break;
02363   }
02364   case ISD::FrameIndex:
02365   case ISD::TargetFrameIndex:
02366     if (unsigned Align = InferPtrAlignment(Op)) {
02367       // The low bits are known zero if the pointer is aligned.
02368       KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align));
02369       break;
02370     }
02371     break;
02372 
02373   default:
02374     if (Op.getOpcode() < ISD::BUILTIN_OP_END)
02375       break;
02376     // Fallthrough
02377   case ISD::INTRINSIC_WO_CHAIN:
02378   case ISD::INTRINSIC_W_CHAIN:
02379   case ISD::INTRINSIC_VOID:
02380     // Allow the target to implement this method for its nodes.
02381     TLI->computeKnownBitsForTargetNode(Op, KnownZero, KnownOne, *this, Depth);
02382     break;
02383   }
02384 
02385   assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
02386 }
02387 
02388 /// ComputeNumSignBits - Return the number of times the sign bit of the
02389 /// register is replicated into the other bits.  We know that at least 1 bit
02390 /// is always equal to the sign bit (itself), but other cases can give us
02391 /// information.  For example, immediately after an "SRA X, 2", we know that
02392 /// the top 3 bits are all equal to each other, so we return 3.
02393 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{
02394   EVT VT = Op.getValueType();
02395   assert(VT.isInteger() && "Invalid VT!");
02396   unsigned VTBits = VT.getScalarType().getSizeInBits();
02397   unsigned Tmp, Tmp2;
02398   unsigned FirstAnswer = 1;
02399 
02400   if (Depth == 6)
02401     return 1;  // Limit search depth.
02402 
02403   switch (Op.getOpcode()) {
02404   default: break;
02405   case ISD::AssertSext:
02406     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
02407     return VTBits-Tmp+1;
02408   case ISD::AssertZext:
02409     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
02410     return VTBits-Tmp;
02411 
02412   case ISD::Constant: {
02413     const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue();
02414     return Val.getNumSignBits();
02415   }
02416 
02417   case ISD::SIGN_EXTEND:
02418     Tmp =
02419         VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits();
02420     return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp;
02421 
02422   case ISD::SIGN_EXTEND_INREG:
02423     // Max of the input and what this extends.
02424     Tmp =
02425       cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits();
02426     Tmp = VTBits-Tmp+1;
02427 
02428     Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02429     return std::max(Tmp, Tmp2);
02430 
02431   case ISD::SRA:
02432     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02433     // SRA X, C   -> adds C sign bits.
02434     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02435       Tmp += C->getZExtValue();
02436       if (Tmp > VTBits) Tmp = VTBits;
02437     }
02438     return Tmp;
02439   case ISD::SHL:
02440     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02441       // shl destroys sign bits.
02442       Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02443       if (C->getZExtValue() >= VTBits ||      // Bad shift.
02444           C->getZExtValue() >= Tmp) break;    // Shifted all sign bits out.
02445       return Tmp - C->getZExtValue();
02446     }
02447     break;
02448   case ISD::AND:
02449   case ISD::OR:
02450   case ISD::XOR:    // NOT is handled here.
02451     // Logical binary ops preserve the number of sign bits at the worst.
02452     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02453     if (Tmp != 1) {
02454       Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02455       FirstAnswer = std::min(Tmp, Tmp2);
02456       // We computed what we know about the sign bits as our first
02457       // answer. Now proceed to the generic code that uses
02458       // computeKnownBits, and pick whichever answer is better.
02459     }
02460     break;
02461 
02462   case ISD::SELECT:
02463     Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02464     if (Tmp == 1) return 1;  // Early out.
02465     Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1);
02466     return std::min(Tmp, Tmp2);
02467 
02468   case ISD::SADDO:
02469   case ISD::UADDO:
02470   case ISD::SSUBO:
02471   case ISD::USUBO:
02472   case ISD::SMULO:
02473   case ISD::UMULO:
02474     if (Op.getResNo() != 1)
02475       break;
02476     // The boolean result conforms to getBooleanContents.  Fall through.
02477     // If setcc returns 0/-1, all bits are sign bits.
02478     // We know that we have an integer-based boolean since these operations
02479     // are only available for integer.
02480     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
02481         TargetLowering::ZeroOrNegativeOneBooleanContent)
02482       return VTBits;
02483     break;
02484   case ISD::SETCC:
02485     // If setcc returns 0/-1, all bits are sign bits.
02486     if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
02487         TargetLowering::ZeroOrNegativeOneBooleanContent)
02488       return VTBits;
02489     break;
02490   case ISD::ROTL:
02491   case ISD::ROTR:
02492     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02493       unsigned RotAmt = C->getZExtValue() & (VTBits-1);
02494 
02495       // Handle rotate right by N like a rotate left by 32-N.
02496       if (Op.getOpcode() == ISD::ROTR)
02497         RotAmt = (VTBits-RotAmt) & (VTBits-1);
02498 
02499       // If we aren't rotating out all of the known-in sign bits, return the
02500       // number that are left.  This handles rotl(sext(x), 1) for example.
02501       Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02502       if (Tmp > RotAmt+1) return Tmp-RotAmt;
02503     }
02504     break;
02505   case ISD::ADD:
02506     // Add can have at most one carry bit.  Thus we know that the output
02507     // is, at worst, one more bit than the inputs.
02508     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02509     if (Tmp == 1) return 1;  // Early out.
02510 
02511     // Special case decrementing a value (ADD X, -1):
02512     if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
02513       if (CRHS->isAllOnesValue()) {
02514         APInt KnownZero, KnownOne;
02515         computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02516 
02517         // If the input is known to be 0 or 1, the output is 0/-1, which is all
02518         // sign bits set.
02519         if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
02520           return VTBits;
02521 
02522         // If we are subtracting one from a positive number, there is no carry
02523         // out of the result.
02524         if (KnownZero.isNegative())
02525           return Tmp;
02526       }
02527 
02528     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02529     if (Tmp2 == 1) return 1;
02530     return std::min(Tmp, Tmp2)-1;
02531 
02532   case ISD::SUB:
02533     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02534     if (Tmp2 == 1) return 1;
02535 
02536     // Handle NEG.
02537     if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0)))
02538       if (CLHS->isNullValue()) {
02539         APInt KnownZero, KnownOne;
02540         computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
02541         // If the input is known to be 0 or 1, the output is 0/-1, which is all
02542         // sign bits set.
02543         if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
02544           return VTBits;
02545 
02546         // If the input is known to be positive (the sign bit is known clear),
02547         // the output of the NEG has the same number of sign bits as the input.
02548         if (KnownZero.isNegative())
02549           return Tmp2;
02550 
02551         // Otherwise, we treat this like a SUB.
02552       }
02553 
02554     // Sub can have at most one carry bit.  Thus we know that the output
02555     // is, at worst, one more bit than the inputs.
02556     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02557     if (Tmp == 1) return 1;  // Early out.
02558     return std::min(Tmp, Tmp2)-1;
02559   case ISD::TRUNCATE:
02560     // FIXME: it's tricky to do anything useful for this, but it is an important
02561     // case for targets like X86.
02562     break;
02563   case ISD::EXTRACT_ELEMENT: {
02564     const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02565     const int BitWidth = Op.getValueType().getSizeInBits();
02566     const int Items =
02567       Op.getOperand(0).getValueType().getSizeInBits() / BitWidth;
02568 
02569     // Get reverse index (starting from 1), Op1 value indexes elements from
02570     // little end. Sign starts at big end.
02571     const int rIndex = Items - 1 -
02572       cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
02573 
02574     // If the sign portion ends in our element the substraction gives correct
02575     // result. Otherwise it gives either negative or > bitwidth result
02576     return std::max(std::min(KnownSign - rIndex * BitWidth, BitWidth), 0);
02577   }
02578   }
02579 
02580   // If we are looking at the loaded value of the SDNode.
02581   if (Op.getResNo() == 0) {
02582     // Handle LOADX separately here. EXTLOAD case will fallthrough.
02583     if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
02584       unsigned ExtType = LD->getExtensionType();
02585       switch (ExtType) {
02586         default: break;
02587         case ISD::SEXTLOAD:    // '17' bits known
02588           Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
02589           return VTBits-Tmp+1;
02590         case ISD::ZEXTLOAD:    // '16' bits known
02591           Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
02592           return VTBits-Tmp;
02593       }
02594     }
02595   }
02596 
02597   // Allow the target to implement this method for its nodes.
02598   if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
02599       Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
02600       Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
02601       Op.getOpcode() == ISD::INTRINSIC_VOID) {
02602     unsigned NumBits = TLI->ComputeNumSignBitsForTargetNode(Op, *this, Depth);
02603     if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits);
02604   }
02605 
02606   // Finally, if we can prove that the top bits of the result are 0's or 1's,
02607   // use this information.
02608   APInt KnownZero, KnownOne;
02609   computeKnownBits(Op, KnownZero, KnownOne, Depth);
02610 
02611   APInt Mask;
02612   if (KnownZero.isNegative()) {        // sign bit is 0
02613     Mask = KnownZero;
02614   } else if (KnownOne.isNegative()) {  // sign bit is 1;
02615     Mask = KnownOne;
02616   } else {
02617     // Nothing known.
02618     return FirstAnswer;
02619   }
02620 
02621   // Okay, we know that the sign bit in Mask is set.  Use CLZ to determine
02622   // the number of identical bits in the top of the input value.
02623   Mask = ~Mask;
02624   Mask <<= Mask.getBitWidth()-VTBits;
02625   // Return # leading zeros.  We use 'min' here in case Val was zero before
02626   // shifting.  We don't want to return '64' as for an i32 "0".
02627   return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros()));
02628 }
02629 
02630 /// isBaseWithConstantOffset - Return true if the specified operand is an
02631 /// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an
02632 /// ISD::OR with a ConstantSDNode that is guaranteed to have the same
02633 /// semantics as an ADD.  This handles the equivalence:
02634 ///     X|Cst == X+Cst iff X&Cst = 0.
02635 bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
02636   if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
02637       !isa<ConstantSDNode>(Op.getOperand(1)))
02638     return false;
02639 
02640   if (Op.getOpcode() == ISD::OR &&
02641       !MaskedValueIsZero(Op.getOperand(0),
02642                      cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue()))
02643     return false;
02644 
02645   return true;
02646 }
02647 
02648 
02649 bool SelectionDAG::isKnownNeverNaN(SDValue Op) const {
02650   // If we're told that NaNs won't happen, assume they won't.
02651   if (getTarget().Options.NoNaNsFPMath)
02652     return true;
02653 
02654   // If the value is a constant, we can obviously see if it is a NaN or not.
02655   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
02656     return !C->getValueAPF().isNaN();
02657 
02658   // TODO: Recognize more cases here.
02659 
02660   return false;
02661 }
02662 
02663 bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
02664   // If the value is a constant, we can obviously see if it is a zero or not.
02665   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
02666     return !C->isZero();
02667 
02668   // TODO: Recognize more cases here.
02669   switch (Op.getOpcode()) {
02670   default: break;
02671   case ISD::OR:
02672     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
02673       return !C->isNullValue();
02674     break;
02675   }
02676 
02677   return false;
02678 }
02679 
02680 bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
02681   // Check the obvious case.
02682   if (A == B) return true;
02683 
02684   // For for negative and positive zero.
02685   if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
02686     if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
02687       if (CA->isZero() && CB->isZero()) return true;
02688 
02689   // Otherwise they may not be equal.
02690   return false;
02691 }
02692 
02693 /// getNode - Gets or creates the specified node.
02694 ///
02695 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT) {
02696   FoldingSetNodeID ID;
02697   AddNodeIDNode(ID, Opcode, getVTList(VT), None);
02698   void *IP = nullptr;
02699   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
02700     return SDValue(E, 0);
02701 
02702   SDNode *N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(),
02703                                          DL.getDebugLoc(), getVTList(VT));
02704   CSEMap.InsertNode(N, IP);
02705 
02706   InsertNode(N);
02707   return SDValue(N, 0);
02708 }
02709 
02710 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
02711                               EVT VT, SDValue Operand) {
02712   // Constant fold unary operations with an integer constant operand. Even
02713   // opaque constant will be folded, because the folding of unary operations
02714   // doesn't create new constants with different values. Nevertheless, the
02715   // opaque flag is preserved during folding to prevent future folding with
02716   // other constants.
02717   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) {
02718     const APInt &Val = C->getAPIntValue();
02719     switch (Opcode) {
02720     default: break;
02721     case ISD::SIGN_EXTEND:
02722       return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT,
02723                          C->isTargetOpcode(), C->isOpaque());
02724     case ISD::ANY_EXTEND:
02725     case ISD::ZERO_EXTEND:
02726     case ISD::TRUNCATE:
02727       return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT,
02728                          C->isTargetOpcode(), C->isOpaque());
02729     case ISD::UINT_TO_FP:
02730     case ISD::SINT_TO_FP: {
02731       APFloat apf(EVTToAPFloatSemantics(VT),
02732                   APInt::getNullValue(VT.getSizeInBits()));
02733       (void)apf.convertFromAPInt(Val,
02734                                  Opcode==ISD::SINT_TO_FP,
02735                                  APFloat::rmNearestTiesToEven);
02736       return getConstantFP(apf, VT);
02737     }
02738     case ISD::BITCAST:
02739       if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
02740         return getConstantFP(APFloat(APFloat::IEEEhalf, Val), VT);
02741       if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
02742         return getConstantFP(APFloat(APFloat::IEEEsingle, Val), VT);
02743       else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
02744         return getConstantFP(APFloat(APFloat::IEEEdouble, Val), VT);
02745       break;
02746     case ISD::BSWAP:
02747       return getConstant(Val.byteSwap(), VT, C->isTargetOpcode(),
02748                          C->isOpaque());
02749     case ISD::CTPOP:
02750       return getConstant(Val.countPopulation(), VT, C->isTargetOpcode(),
02751                          C->isOpaque());
02752     case ISD::CTLZ:
02753     case ISD::CTLZ_ZERO_UNDEF:
02754       return getConstant(Val.countLeadingZeros(), VT, C->isTargetOpcode(),
02755                          C->isOpaque());
02756     case ISD::CTTZ:
02757     case ISD::CTTZ_ZERO_UNDEF:
02758       return getConstant(Val.countTrailingZeros(), VT, C->isTargetOpcode(),
02759                          C->isOpaque());
02760     }
02761   }
02762 
02763   // Constant fold unary operations with a floating point constant operand.
02764   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) {
02765     APFloat V = C->getValueAPF();    // make copy
02766     switch (Opcode) {
02767     case ISD::FNEG:
02768       V.changeSign();
02769       return getConstantFP(V, VT);
02770     case ISD::FABS:
02771       V.clearSign();
02772       return getConstantFP(V, VT);
02773     case ISD::FCEIL: {
02774       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
02775       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02776         return getConstantFP(V, VT);
02777       break;
02778     }
02779     case ISD::FTRUNC: {
02780       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
02781       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02782         return getConstantFP(V, VT);
02783       break;
02784     }
02785     case ISD::FFLOOR: {
02786       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
02787       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02788         return getConstantFP(V, VT);
02789       break;
02790     }
02791     case ISD::FP_EXTEND: {
02792       bool ignored;
02793       // This can return overflow, underflow, or inexact; we don't care.
02794       // FIXME need to be more flexible about rounding mode.
02795       (void)V.convert(EVTToAPFloatSemantics(VT),
02796                       APFloat::rmNearestTiesToEven, &ignored);
02797       return getConstantFP(V, VT);
02798     }
02799     case ISD::FP_TO_SINT:
02800     case ISD::FP_TO_UINT: {
02801       integerPart x[2];
02802       bool ignored;
02803       static_assert(integerPartWidth >= 64, "APFloat parts too small!");
02804       // FIXME need to be more flexible about rounding mode.
02805       APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(),
02806                             Opcode==ISD::FP_TO_SINT,
02807                             APFloat::rmTowardZero, &ignored);
02808       if (s==APFloat::opInvalidOp)     // inexact is OK, in fact usual
02809         break;
02810       APInt api(VT.getSizeInBits(), x);
02811       return getConstant(api, VT);
02812     }
02813     case ISD::BITCAST:
02814       if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
02815         return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), VT);
02816       else if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
02817         return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT);
02818       else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
02819         return getConstant(V.bitcastToAPInt().getZExtValue(), VT);
02820       break;
02821     }
02822   }
02823 
02824   // Constant fold unary operations with a vector integer operand.
02825   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand.getNode())) {
02826     if (BV->isConstant()) {
02827       switch (Opcode) {
02828       default:
02829         // FIXME: Entirely reasonable to perform folding of other unary
02830         // operations here as the need arises.
02831         break;
02832       case ISD::UINT_TO_FP:
02833       case ISD::SINT_TO_FP: {
02834         SmallVector<SDValue, 8> Ops;
02835         for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
02836           SDValue OpN = BV->getOperand(i);
02837           // Let the above scalar folding handle the conversion of each
02838           // element.
02839           OpN = getNode(ISD::SINT_TO_FP, DL, VT.getVectorElementType(),
02840                         OpN);
02841           Ops.push_back(OpN);
02842         }
02843         return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
02844       }
02845       }
02846     }
02847   }
02848 
02849   unsigned OpOpcode = Operand.getNode()->getOpcode();
02850   switch (Opcode) {
02851   case ISD::TokenFactor:
02852   case ISD::MERGE_VALUES:
02853   case ISD::CONCAT_VECTORS:
02854     return Operand;         // Factor, merge or concat of one node?  No need.
02855   case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
02856   case ISD::FP_EXTEND:
02857     assert(VT.isFloatingPoint() &&
02858            Operand.getValueType().isFloatingPoint() && "Invalid FP cast!");
02859     if (Operand.getValueType() == VT) return Operand;  // noop conversion.
02860     assert((!VT.isVector() ||
02861             VT.getVectorNumElements() ==
02862             Operand.getValueType().getVectorNumElements()) &&
02863            "Vector element count mismatch!");
02864     if (Operand.getOpcode() == ISD::UNDEF)
02865       return getUNDEF(VT);
02866     break;
02867   case ISD::SIGN_EXTEND:
02868     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02869            "Invalid SIGN_EXTEND!");
02870     if (Operand.getValueType() == VT) return Operand;   // noop extension
02871     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02872            "Invalid sext node, dst < src!");
02873     assert((!VT.isVector() ||
02874             VT.getVectorNumElements() ==
02875             Operand.getValueType().getVectorNumElements()) &&
02876            "Vector element count mismatch!");
02877     if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
02878       return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02879     else if (OpOpcode == ISD::UNDEF)
02880       // sext(undef) = 0, because the top bits will all be the same.
02881       return getConstant(0, VT);
02882     break;
02883   case ISD::ZERO_EXTEND:
02884     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02885            "Invalid ZERO_EXTEND!");
02886     if (Operand.getValueType() == VT) return Operand;   // noop extension
02887     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02888            "Invalid zext node, dst < src!");
02889     assert((!VT.isVector() ||
02890             VT.getVectorNumElements() ==
02891             Operand.getValueType().getVectorNumElements()) &&
02892            "Vector element count mismatch!");
02893     if (OpOpcode == ISD::ZERO_EXTEND)   // (zext (zext x)) -> (zext x)
02894       return getNode(ISD::ZERO_EXTEND, DL, VT,
02895                      Operand.getNode()->getOperand(0));
02896     else if (OpOpcode == ISD::UNDEF)
02897       // zext(undef) = 0, because the top bits will be zero.
02898       return getConstant(0, VT);
02899     break;
02900   case ISD::ANY_EXTEND:
02901     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02902            "Invalid ANY_EXTEND!");
02903     if (Operand.getValueType() == VT) return Operand;   // noop extension
02904     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02905            "Invalid anyext node, dst < src!");
02906     assert((!VT.isVector() ||
02907             VT.getVectorNumElements() ==
02908             Operand.getValueType().getVectorNumElements()) &&
02909            "Vector element count mismatch!");
02910 
02911     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
02912         OpOpcode == ISD::ANY_EXTEND)
02913       // (ext (zext x)) -> (zext x)  and  (ext (sext x)) -> (sext x)
02914       return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02915     else if (OpOpcode == ISD::UNDEF)
02916       return getUNDEF(VT);
02917 
02918     // (ext (trunx x)) -> x
02919     if (OpOpcode == ISD::TRUNCATE) {
02920       SDValue OpOp = Operand.getNode()->getOperand(0);
02921       if (OpOp.getValueType() == VT)
02922         return OpOp;
02923     }
02924     break;
02925   case ISD::TRUNCATE:
02926     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02927            "Invalid TRUNCATE!");
02928     if (Operand.getValueType() == VT) return Operand;   // noop truncate
02929     assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) &&
02930            "Invalid truncate node, src < dst!");
02931     assert((!VT.isVector() ||
02932             VT.getVectorNumElements() ==
02933             Operand.getValueType().getVectorNumElements()) &&
02934            "Vector element count mismatch!");
02935     if (OpOpcode == ISD::TRUNCATE)
02936       return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
02937     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
02938         OpOpcode == ISD::ANY_EXTEND) {
02939       // If the source is smaller than the dest, we still need an extend.
02940       if (Operand.getNode()->getOperand(0).getValueType().getScalarType()
02941             .bitsLT(VT.getScalarType()))
02942         return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02943       if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT))
02944         return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
02945       return Operand.getNode()->getOperand(0);
02946     }
02947     if (OpOpcode == ISD::UNDEF)
02948       return getUNDEF(VT);
02949     break;
02950   case ISD::BITCAST:
02951     // Basic sanity checking.
02952     assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits()
02953            && "Cannot BITCAST between types of different sizes!");
02954     if (VT == Operand.getValueType()) return Operand;  // noop conversion.
02955     if (OpOpcode == ISD::BITCAST)  // bitconv(bitconv(x)) -> bitconv(x)
02956       return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
02957     if (OpOpcode == ISD::UNDEF)
02958       return getUNDEF(VT);
02959     break;
02960   case ISD::SCALAR_TO_VECTOR:
02961     assert(VT.isVector() && !Operand.getValueType().isVector() &&
02962            (VT.getVectorElementType() == Operand.getValueType() ||
02963             (VT.getVectorElementType().isInteger() &&
02964              Operand.getValueType().isInteger() &&
02965              VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&
02966            "Illegal SCALAR_TO_VECTOR node!");
02967     if (OpOpcode == ISD::UNDEF)
02968       return getUNDEF(VT);
02969     // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
02970     if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
02971         isa<ConstantSDNode>(Operand.getOperand(1)) &&
02972         Operand.getConstantOperandVal(1) == 0 &&
02973         Operand.getOperand(0).getValueType() == VT)
02974       return Operand.getOperand(0);
02975     break;
02976   case ISD::FNEG:
02977     // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0
02978     if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB)
02979       return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1),
02980                      Operand.getNode()->getOperand(0));
02981     if (OpOpcode == ISD::FNEG)  // --X -> X
02982       return Operand.getNode()->getOperand(0);
02983     break;
02984   case ISD::FABS:
02985     if (OpOpcode == ISD::FNEG)  // abs(-X) -> abs(X)
02986       return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0));
02987     break;
02988   }
02989 
02990   SDNode *N;
02991   SDVTList VTs = getVTList(VT);
02992   if (VT != MVT::Glue) { // Don't CSE flag producing nodes
02993     FoldingSetNodeID ID;
02994     SDValue Ops[1] = { Operand };
02995     AddNodeIDNode(ID, Opcode, VTs, Ops);
02996     void *IP = nullptr;
02997     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
02998       return SDValue(E, 0);
02999 
03000     N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
03001                                         DL.getDebugLoc(), VTs, Operand);
03002     CSEMap.InsertNode(N, IP);
03003   } else {
03004     N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
03005                                         DL.getDebugLoc(), VTs, Operand);
03006   }
03007 
03008   InsertNode(N);
03009   return SDValue(N, 0);
03010 }
03011 
03012 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT,
03013                                              SDNode *Cst1, SDNode *Cst2) {
03014   // If the opcode is a target-specific ISD node, there's nothing we can
03015   // do here and the operand rules may not line up with the below, so
03016   // bail early.
03017   if (Opcode >= ISD::BUILTIN_OP_END)
03018     return SDValue();
03019 
03020   SmallVector<std::pair<ConstantSDNode *, ConstantSDNode *>, 4> Inputs;
03021   SmallVector<SDValue, 4> Outputs;
03022   EVT SVT = VT.getScalarType();
03023 
03024   ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1);
03025   ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2);
03026   if (Scalar1 && Scalar2 && (Scalar1->isOpaque() || Scalar2->isOpaque()))
03027     return SDValue();
03028 
03029   if (Scalar1 && Scalar2)
03030     // Scalar instruction.
03031     Inputs.push_back(std::make_pair(Scalar1, Scalar2));
03032   else {
03033     // For vectors extract each constant element into Inputs so we can constant
03034     // fold them individually.
03035     BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1);
03036     BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2);
03037     if (!BV1 || !BV2)
03038       return SDValue();
03039 
03040     assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!");
03041 
03042     for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) {
03043       ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I));
03044       ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I));
03045       if (!V1 || !V2) // Not a constant, bail.
03046         return SDValue();
03047 
03048       if (V1->isOpaque() || V2->isOpaque())
03049         return SDValue();
03050 
03051       // Avoid BUILD_VECTOR nodes that perform implicit truncation.
03052       // FIXME: This is valid and could be handled by truncating the APInts.
03053       if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
03054         return SDValue();
03055 
03056       Inputs.push_back(std::make_pair(V1, V2));
03057     }
03058   }
03059 
03060   // We have a number of constant values, constant fold them element by element.
03061   for (unsigned I = 0, E = Inputs.size(); I != E; ++I) {
03062     const APInt &C1 = Inputs[I].first->getAPIntValue();
03063     const APInt &C2 = Inputs[I].second->getAPIntValue();
03064 
03065     switch (Opcode) {
03066     case ISD::ADD:
03067       Outputs.push_back(getConstant(C1 + C2, SVT));
03068       break;
03069     case ISD::SUB:
03070       Outputs.push_back(getConstant(C1 - C2, SVT));
03071       break;
03072     case ISD::MUL:
03073       Outputs.push_back(getConstant(C1 * C2, SVT));
03074       break;
03075     case ISD::UDIV:
03076       if (!C2.getBoolValue())
03077         return SDValue();
03078       Outputs.push_back(getConstant(C1.udiv(C2), SVT));
03079       break;
03080     case ISD::UREM:
03081       if (!C2.getBoolValue())
03082         return SDValue();
03083       Outputs.push_back(getConstant(C1.urem(C2), SVT));
03084       break;
03085     case ISD::SDIV:
03086       if (!C2.getBoolValue())
03087         return SDValue();
03088       Outputs.push_back(getConstant(C1.sdiv(C2), SVT));
03089       break;
03090     case ISD::SREM:
03091       if (!C2.getBoolValue())
03092         return SDValue();
03093       Outputs.push_back(getConstant(C1.srem(C2), SVT));
03094       break;
03095     case ISD::AND:
03096       Outputs.push_back(getConstant(C1 & C2, SVT));
03097       break;
03098     case ISD::OR:
03099       Outputs.push_back(getConstant(C1 | C2, SVT));
03100       break;
03101     case ISD::XOR:
03102       Outputs.push_back(getConstant(C1 ^ C2, SVT));
03103       break;
03104     case ISD::SHL:
03105       Outputs.push_back(getConstant(C1 << C2, SVT));
03106       break;
03107     case ISD::SRL:
03108       Outputs.push_back(getConstant(C1.lshr(C2), SVT));
03109       break;
03110     case ISD::SRA:
03111       Outputs.push_back(getConstant(C1.ashr(C2), SVT));
03112       break;
03113     case ISD::ROTL:
03114       Outputs.push_back(getConstant(C1.rotl(C2), SVT));
03115       break;
03116     case ISD::ROTR:
03117       Outputs.push_back(getConstant(C1.rotr(C2), SVT));
03118       break;
03119     default:
03120       return SDValue();
03121     }
03122   }
03123 
03124   assert((Scalar1 && Scalar2) || (VT.getVectorNumElements() == Outputs.size() &&
03125                                   "Expected a scalar or vector!"));
03126 
03127   // Handle the scalar case first.
03128   if (!VT.isVector())
03129     return Outputs.back();
03130 
03131   // We may have a vector type but a scalar result. Create a splat.
03132   Outputs.resize(VT.getVectorNumElements(), Outputs.back());
03133 
03134   // Build a big vector out of the scalar elements we generated.
03135   return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs);
03136 }
03137 
03138 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
03139                               SDValue N2, bool nuw, bool nsw, bool exact) {
03140   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
03141   ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode());
03142   switch (Opcode) {
03143   default: break;
03144   case ISD::TokenFactor:
03145     assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
03146            N2.getValueType() == MVT::Other && "Invalid token factor!");
03147     // Fold trivial token factors.
03148     if (N1.getOpcode() == ISD::EntryToken) return N2;
03149     if (N2.getOpcode() == ISD::EntryToken) return N1;
03150     if (N1 == N2) return N1;
03151     break;
03152   case ISD::CONCAT_VECTORS:
03153     // Concat of UNDEFs is UNDEF.
03154     if (N1.getOpcode() == ISD::UNDEF &&
03155         N2.getOpcode() == ISD::UNDEF)
03156       return getUNDEF(VT);
03157 
03158     // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
03159     // one big BUILD_VECTOR.
03160     if (N1.getOpcode() == ISD::BUILD_VECTOR &&
03161         N2.getOpcode() == ISD::BUILD_VECTOR) {
03162       SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
03163                                     N1.getNode()->op_end());
03164       Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
03165       return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
03166     }
03167     break;
03168   case ISD::AND:
03169     assert(VT.isInteger() && "This operator does not apply to FP types!");
03170     assert(N1.getValueType() == N2.getValueType() &&
03171            N1.getValueType() == VT && "Binary operator types must match!");
03172     // (X & 0) -> 0.  This commonly occurs when legalizing i64 values, so it's
03173     // worth handling here.
03174     if (N2C && N2C->isNullValue())
03175       return N2;
03176     if (N2C && N2C->isAllOnesValue())  // X & -1 -> X
03177       return N1;
03178     break;
03179   case ISD::OR:
03180   case ISD::XOR:
03181   case ISD::ADD:
03182   case ISD::SUB:
03183     assert(VT.isInteger() && "This operator does not apply to FP types!");
03184     assert(N1.getValueType() == N2.getValueType() &&
03185            N1.getValueType() == VT && "Binary operator types must match!");
03186     // (X ^|+- 0) -> X.  This commonly occurs when legalizing i64 values, so
03187     // it's worth handling here.
03188     if (N2C && N2C->isNullValue())
03189       return N1;
03190     break;
03191   case ISD::UDIV:
03192   case ISD::UREM:
03193   case ISD::MULHU:
03194   case ISD::MULHS:
03195   case ISD::MUL:
03196   case ISD::SDIV:
03197   case ISD::SREM:
03198     assert(VT.isInteger() && "This operator does not apply to FP types!");
03199     assert(N1.getValueType() == N2.getValueType() &&
03200            N1.getValueType() == VT && "Binary operator types must match!");
03201     break;
03202   case ISD::FADD:
03203   case ISD::FSUB:
03204   case ISD::FMUL:
03205   case ISD::FDIV:
03206   case ISD::FREM:
03207     if (getTarget().Options.UnsafeFPMath) {
03208       if (Opcode == ISD::FADD) {
03209         // 0+x --> x
03210         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1))
03211           if (CFP->getValueAPF().isZero())
03212             return N2;
03213         // x+0 --> x
03214         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
03215           if (CFP->getValueAPF().isZero())
03216             return N1;
03217       } else if (Opcode == ISD::FSUB) {
03218         // x-0 --> x
03219         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
03220           if (CFP->getValueAPF().isZero())
03221             return N1;
03222       } else if (Opcode == ISD::FMUL) {
03223         ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1);
03224         SDValue V = N2;
03225 
03226         // If the first operand isn't the constant, try the second
03227         if (!CFP) {
03228           CFP = dyn_cast<ConstantFPSDNode>(N2);
03229           V = N1;
03230         }
03231 
03232         if (CFP) {
03233           // 0*x --> 0
03234           if (CFP->isZero())
03235             return SDValue(CFP,0);
03236           // 1*x --> x
03237           if (CFP->isExactlyValue(1.0))
03238             return V;
03239         }
03240       }
03241     }
03242     assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
03243     assert(N1.getValueType() == N2.getValueType() &&
03244            N1.getValueType() == VT && "Binary operator types must match!");
03245     break;
03246   case ISD::FCOPYSIGN:   // N1 and result must match.  N1/N2 need not match.
03247     assert(N1.getValueType() == VT &&
03248            N1.getValueType().isFloatingPoint() &&
03249            N2.getValueType().isFloatingPoint() &&
03250            "Invalid FCOPYSIGN!");
03251     break;
03252   case ISD::SHL:
03253   case ISD::SRA:
03254   case ISD::SRL:
03255   case ISD::ROTL:
03256   case ISD::ROTR:
03257     assert(VT == N1.getValueType() &&
03258            "Shift operators return type must be the same as their first arg");
03259     assert(VT.isInteger() && N2.getValueType().isInteger() &&
03260            "Shifts only work on integers");
03261     assert((!VT.isVector() || VT == N2.getValueType()) &&
03262            "Vector shift amounts must be in the same as their first arg");
03263     // Verify that the shift amount VT is bit enough to hold valid shift
03264     // amounts.  This catches things like trying to shift an i1024 value by an
03265     // i8, which is easy to fall into in generic code that uses
03266     // TLI.getShiftAmount().
03267     assert(N2.getValueType().getSizeInBits() >=
03268                    Log2_32_Ceil(N1.getValueType().getSizeInBits()) &&
03269            "Invalid use of small shift amount with oversized value!");
03270 
03271     // Always fold shifts of i1 values so the code generator doesn't need to
03272     // handle them.  Since we know the size of the shift has to be less than the
03273     // size of the value, the shift/rotate count is guaranteed to be zero.
03274     if (VT == MVT::i1)
03275       return N1;
03276     if (N2C && N2C->isNullValue())
03277       return N1;
03278     break;
03279   case ISD::FP_ROUND_INREG: {
03280     EVT EVT = cast<VTSDNode>(N2)->getVT();
03281     assert(VT == N1.getValueType() && "Not an inreg round!");
03282     assert(VT.isFloatingPoint() && EVT.isFloatingPoint() &&
03283            "Cannot FP_ROUND_INREG integer types");
03284     assert(EVT.isVector() == VT.isVector() &&
03285            "FP_ROUND_INREG type should be vector iff the operand "
03286            "type is vector!");
03287     assert((!EVT.isVector() ||
03288             EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
03289            "Vector element counts must match in FP_ROUND_INREG");
03290     assert(EVT.bitsLE(VT) && "Not rounding down!");
03291     (void)EVT;
03292     if (cast<VTSDNode>(N2)->getVT() == VT) return N1;  // Not actually rounding.
03293     break;
03294   }
03295   case ISD::FP_ROUND:
03296     assert(VT.isFloatingPoint() &&
03297            N1.getValueType().isFloatingPoint() &&
03298            VT.bitsLE(N1.getValueType()) &&
03299            isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!");
03300     if (N1.getValueType() == VT) return N1;  // noop conversion.
03301     break;
03302   case ISD::AssertSext:
03303   case ISD::AssertZext: {
03304     EVT EVT = cast<VTSDNode>(N2)->getVT();
03305     assert(VT == N1.getValueType() && "Not an inreg extend!");
03306     assert(VT.isInteger() && EVT.isInteger() &&
03307            "Cannot *_EXTEND_INREG FP types");
03308     assert(!EVT.isVector() &&
03309            "AssertSExt/AssertZExt type should be the vector element type "
03310            "rather than the vector type!");
03311     assert(EVT.bitsLE(VT) && "Not extending!");
03312     if (VT == EVT) return N1; // noop assertion.
03313     break;
03314   }
03315   case ISD::SIGN_EXTEND_INREG: {
03316     EVT EVT = cast<VTSDNode>(N2)->getVT();
03317     assert(VT == N1.getValueType() && "Not an inreg extend!");
03318     assert(VT.isInteger() && EVT.isInteger() &&
03319            "Cannot *_EXTEND_INREG FP types");
03320     assert(EVT.isVector() == VT.isVector() &&
03321            "SIGN_EXTEND_INREG type should be vector iff the operand "
03322            "type is vector!");
03323     assert((!EVT.isVector() ||
03324             EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
03325            "Vector element counts must match in SIGN_EXTEND_INREG");
03326     assert(EVT.bitsLE(VT) && "Not extending!");
03327     if (EVT == VT) return N1;  // Not actually extending
03328 
03329     if (N1C) {
03330       APInt Val = N1C->getAPIntValue();
03331       unsigned FromBits = EVT.getScalarType().getSizeInBits();
03332       Val <<= Val.getBitWidth()-FromBits;
03333       Val = Val.ashr(Val.getBitWidth()-FromBits);
03334       return getConstant(Val, VT);
03335     }
03336     break;
03337   }
03338   case ISD::EXTRACT_VECTOR_ELT:
03339     // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF.
03340     if (N1.getOpcode() == ISD::UNDEF)
03341       return getUNDEF(VT);
03342 
03343     // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
03344     // expanding copies of large vectors from registers.
03345     if (N2C &&
03346         N1.getOpcode() == ISD::CONCAT_VECTORS &&
03347         N1.getNumOperands() > 0) {
03348       unsigned Factor =
03349         N1.getOperand(0).getValueType().getVectorNumElements();
03350       return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
03351                      N1.getOperand(N2C->getZExtValue() / Factor),
03352                      getConstant(N2C->getZExtValue() % Factor,
03353                                  N2.getValueType()));
03354     }
03355 
03356     // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
03357     // expanding large vector constants.
03358     if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) {
03359       SDValue Elt = N1.getOperand(N2C->getZExtValue());
03360 
03361       if (VT != Elt.getValueType())
03362         // If the vector element type is not legal, the BUILD_VECTOR operands
03363         // are promoted and implicitly truncated, and the result implicitly
03364         // extended. Make that explicit here.
03365         Elt = getAnyExtOrTrunc(Elt, DL, VT);
03366 
03367       return Elt;
03368     }
03369 
03370     // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
03371     // operations are lowered to scalars.
03372     if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
03373       // If the indices are the same, return the inserted element else
03374       // if the indices are known different, extract the element from
03375       // the original vector.
03376       SDValue N1Op2 = N1.getOperand(2);
03377       ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode());
03378 
03379       if (N1Op2C && N2C) {
03380         if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
03381           if (VT == N1.getOperand(1).getValueType())
03382             return N1.getOperand(1);
03383           else
03384             return getSExtOrTrunc(N1.getOperand(1), DL, VT);
03385         }
03386 
03387         return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
03388       }
03389     }
03390     break;
03391   case ISD::EXTRACT_ELEMENT:
03392     assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
03393     assert(!N1.getValueType().isVector() && !VT.isVector() &&
03394            (N1.getValueType().isInteger() == VT.isInteger()) &&
03395            N1.getValueType() != VT &&
03396            "Wrong types for EXTRACT_ELEMENT!");
03397 
03398     // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
03399     // 64-bit integers into 32-bit parts.  Instead of building the extract of
03400     // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
03401     if (N1.getOpcode() == ISD::BUILD_PAIR)
03402       return N1.getOperand(N2C->getZExtValue());
03403 
03404     // EXTRACT_ELEMENT of a constant int is also very common.
03405     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
03406       unsigned ElementSize = VT.getSizeInBits();
03407       unsigned Shift = ElementSize * N2C->getZExtValue();
03408       APInt ShiftedVal = C->getAPIntValue().lshr(Shift);
03409       return getConstant(ShiftedVal.trunc(ElementSize), VT);
03410     }
03411     break;
03412   case ISD::EXTRACT_SUBVECTOR: {
03413     SDValue Index = N2;
03414     if (VT.isSimple() && N1.getValueType().isSimple()) {
03415       assert(VT.isVector() && N1.getValueType().isVector() &&
03416              "Extract subvector VTs must be a vectors!");
03417       assert(VT.getVectorElementType() ==
03418              N1.getValueType().getVectorElementType() &&
03419              "Extract subvector VTs must have the same element type!");
03420       assert(VT.getSimpleVT() <= N1.getSimpleValueType() &&
03421              "Extract subvector must be from larger vector to smaller vector!");
03422 
03423       if (isa<ConstantSDNode>(Index.getNode())) {
03424         assert((VT.getVectorNumElements() +
03425                 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
03426                 <= N1.getValueType().getVectorNumElements())
03427                && "Extract subvector overflow!");
03428       }
03429 
03430       // Trivial extraction.
03431       if (VT.getSimpleVT() == N1.getSimpleValueType())
03432         return N1;
03433     }
03434     break;
03435   }
03436   }
03437 
03438   // Perform trivial constant folding.
03439   if (SDValue SV =
03440           FoldConstantArithmetic(Opcode, VT, N1.getNode(), N2.getNode()))
03441     return SV;
03442 
03443   // Canonicalize constant to RHS if commutative.
03444   if (N1C && !N2C && isCommutativeBinOp(Opcode)) {
03445     std::swap(N1C, N2C);
03446     std::swap(N1, N2);
03447   }
03448 
03449   // Constant fold FP operations.
03450   bool HasFPExceptions = TLI->hasFloatingPointExceptions();
03451   ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode());
03452   ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode());
03453   if (N1CFP) {
03454     if (!N2CFP && isCommutativeBinOp(Opcode)) {
03455       // Canonicalize constant to RHS if commutative.
03456       std::swap(N1CFP, N2CFP);
03457       std::swap(N1, N2);
03458     } else if (N2CFP) {
03459       APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF();
03460       APFloat::opStatus s;
03461       switch (Opcode) {
03462       case ISD::FADD:
03463         s = V1.add(V2, APFloat::rmNearestTiesToEven);
03464         if (!HasFPExceptions || s != APFloat::opInvalidOp)
03465           return getConstantFP(V1, VT);
03466         break;
03467       case ISD::FSUB:
03468         s = V1.subtract(V2, APFloat::rmNearestTiesToEven);
03469         if (!HasFPExceptions || s!=APFloat::opInvalidOp)
03470           return getConstantFP(V1, VT);
03471         break;
03472       case ISD::FMUL:
03473         s = V1.multiply(V2, APFloat::rmNearestTiesToEven);
03474         if (!HasFPExceptions || s!=APFloat::opInvalidOp)
03475           return getConstantFP(V1, VT);
03476         break;
03477       case ISD::FDIV:
03478         s = V1.divide(V2, APFloat::rmNearestTiesToEven);
03479         if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
03480                                  s!=APFloat::opDivByZero)) {
03481           return getConstantFP(V1, VT);
03482         }
03483         break;
03484       case ISD::FREM :
03485         s = V1.mod(V2, APFloat::rmNearestTiesToEven);
03486         if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
03487                                  s!=APFloat::opDivByZero)) {
03488           return getConstantFP(V1, VT);
03489         }
03490         break;
03491       case ISD::FCOPYSIGN:
03492         V1.copySign(V2);
03493         return getConstantFP(V1, VT);
03494       default: break;
03495       }
03496     }
03497 
03498     if (Opcode == ISD::FP_ROUND) {
03499       APFloat V = N1CFP->getValueAPF();    // make copy
03500       bool ignored;
03501       // This can return overflow, underflow, or inexact; we don't care.
03502       // FIXME need to be more flexible about rounding mode.
03503       (void)V.convert(EVTToAPFloatSemantics(VT),
03504                       APFloat::rmNearestTiesToEven, &ignored);
03505       return getConstantFP(V, VT);
03506     }
03507   }
03508 
03509   // Canonicalize an UNDEF to the RHS, even over a constant.
03510   if (N1.getOpcode() == ISD::UNDEF) {
03511     if (isCommutativeBinOp(Opcode)) {
03512       std::swap(N1, N2);
03513     } else {
03514       switch (Opcode) {
03515       case ISD::FP_ROUND_INREG:
03516       case ISD::SIGN_EXTEND_INREG:
03517       case ISD::SUB:
03518       case ISD::FSUB:
03519       case ISD::FDIV:
03520       case ISD::FREM:
03521       case ISD::SRA:
03522         return N1;     // fold op(undef, arg2) -> undef
03523       case ISD::UDIV:
03524       case ISD::SDIV:
03525       case ISD::UREM:
03526       case ISD::SREM:
03527       case ISD::SRL:
03528       case ISD::SHL:
03529         if (!VT.isVector())
03530           return getConstant(0, VT);    // fold op(undef, arg2) -> 0
03531         // For vectors, we can't easily build an all zero vector, just return
03532         // the LHS.
03533         return N2;
03534       }
03535     }
03536   }
03537 
03538   // Fold a bunch of operators when the RHS is undef.
03539   if (N2.getOpcode() == ISD::UNDEF) {
03540     switch (Opcode) {
03541     case ISD::XOR:
03542       if (N1.getOpcode() == ISD::UNDEF)
03543         // Handle undef ^ undef -> 0 special case. This is a common
03544         // idiom (misuse).
03545         return getConstant(0, VT);
03546       // fallthrough
03547     case ISD::ADD:
03548     case ISD::ADDC:
03549     case ISD::ADDE:
03550     case ISD::SUB:
03551     case ISD::UDIV:
03552     case ISD::SDIV:
03553     case ISD::UREM:
03554     case ISD::SREM:
03555       return N2;       // fold op(arg1, undef) -> undef
03556     case ISD::FADD:
03557     case ISD::FSUB:
03558     case ISD::FMUL:
03559     case ISD::FDIV:
03560     case ISD::FREM:
03561       if (getTarget().Options.UnsafeFPMath)
03562         return N2;
03563       break;
03564     case ISD::MUL:
03565     case ISD::AND:
03566     case ISD::SRL:
03567     case ISD::SHL:
03568       if (!VT.isVector())
03569         return getConstant(0, VT);  // fold op(arg1, undef) -> 0
03570       // For vectors, we can't easily build an all zero vector, just return
03571       // the LHS.
03572       return N1;
03573     case ISD::OR:
03574       if (!VT.isVector())
03575         return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT);
03576       // For vectors, we can't easily build an all one vector, just return
03577       // the LHS.
03578       return N1;
03579     case ISD::SRA:
03580       return N1;
03581     }
03582   }
03583 
03584   // Memoize this node if possible.
03585   BinarySDNode *N;
03586   SDVTList VTs = getVTList(VT);
03587   const bool BinOpHasFlags = isBinOpWithFlags(Opcode);
03588   if (VT != MVT::Glue) {
03589     SDValue Ops[] = {N1, N2};
03590     FoldingSetNodeID ID;
03591     AddNodeIDNode(ID, Opcode, VTs, Ops);
03592     if (BinOpHasFlags)
03593       AddBinaryNodeIDCustom(ID, Opcode, nuw, nsw, exact);
03594     void *IP = nullptr;
03595     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
03596       return SDValue(E, 0);
03597 
03598     N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
03599 
03600     CSEMap.InsertNode(N, IP);
03601   } else {
03602 
03603     N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
03604   }
03605 
03606   InsertNode(N);
03607   return SDValue(N, 0);
03608 }
03609 
03610 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03611                               SDValue N1, SDValue N2, SDValue N3) {
03612   // Perform various simplifications.
03613   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
03614   switch (Opcode) {
03615   case ISD::FMA: {
03616     ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
03617     ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
03618     ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
03619     if (N1CFP && N2CFP && N3CFP) {
03620       APFloat  V1 = N1CFP->getValueAPF();
03621       const APFloat &V2 = N2CFP->getValueAPF();
03622       const APFloat &V3 = N3CFP->getValueAPF();
03623       APFloat::opStatus s =
03624         V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
03625       if (!TLI->hasFloatingPointExceptions() || s != APFloat::opInvalidOp)
03626         return getConstantFP(V1, VT);
03627     }
03628     break;
03629   }
03630   case ISD::CONCAT_VECTORS:
03631     // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
03632     // one big BUILD_VECTOR.
03633     if (N1.getOpcode() == ISD::BUILD_VECTOR &&
03634         N2.getOpcode() == ISD::BUILD_VECTOR &&
03635         N3.getOpcode() == ISD::BUILD_VECTOR) {
03636       SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
03637                                     N1.getNode()->op_end());
03638       Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
03639       Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end());
03640       return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
03641     }
03642     break;
03643   case ISD::SETCC: {
03644     // Use FoldSetCC to simplify SETCC's.
03645     SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL);
03646     if (Simp.getNode()) return Simp;
03647     break;
03648   }
03649   case ISD::SELECT:
03650     if (N1C) {
03651      if (N1C->getZExtValue())
03652        return N2;             // select true, X, Y -> X
03653      return N3;             // select false, X, Y -> Y
03654     }
03655 
03656     if (N2 == N3) return N2;   // select C, X, X -> X
03657     break;
03658   case ISD::VECTOR_SHUFFLE:
03659     llvm_unreachable("should use getVectorShuffle constructor!");
03660   case ISD::INSERT_SUBVECTOR: {
03661     SDValue Index = N3;
03662     if (VT.isSimple() && N1.getValueType().isSimple()
03663         && N2.getValueType().isSimple()) {
03664       assert(VT.isVector() && N1.getValueType().isVector() &&
03665              N2.getValueType().isVector() &&
03666              "Insert subvector VTs must be a vectors");
03667       assert(VT == N1.getValueType() &&
03668              "Dest and insert subvector source types must match!");
03669       assert(N2.getSimpleValueType() <= N1.getSimpleValueType() &&
03670              "Insert subvector must be from smaller vector to larger vector!");
03671       if (isa<ConstantSDNode>(Index.getNode())) {
03672         assert((N2.getValueType().getVectorNumElements() +
03673                 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
03674                 <= VT.getVectorNumElements())
03675                && "Insert subvector overflow!");
03676       }
03677 
03678       // Trivial insertion.
03679       if (VT.getSimpleVT() == N2.getSimpleValueType())
03680         return N2;
03681     }
03682     break;
03683   }
03684   case ISD::BITCAST:
03685     // Fold bit_convert nodes from a type to themselves.
03686     if (N1.getValueType() == VT)
03687       return N1;
03688     break;
03689   }
03690 
03691   // Memoize node if it doesn't produce a flag.
03692   SDNode *N;
03693   SDVTList VTs = getVTList(VT);
03694   if (VT != MVT::Glue) {
03695     SDValue Ops[] = { N1, N2, N3 };
03696     FoldingSetNodeID ID;
03697     AddNodeIDNode(ID, Opcode, VTs, Ops);
03698     void *IP = nullptr;
03699     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
03700       return SDValue(E, 0);
03701 
03702     N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
03703                                           DL.getDebugLoc(), VTs, N1, N2, N3);
03704     CSEMap.InsertNode(N, IP);
03705   } else {
03706     N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
03707                                           DL.getDebugLoc(), VTs, N1, N2, N3);
03708   }
03709 
03710   InsertNode(N);
03711   return SDValue(N, 0);
03712 }
03713 
03714 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03715                               SDValue N1, SDValue N2, SDValue N3,
03716                               SDValue N4) {
03717   SDValue Ops[] = { N1, N2, N3, N4 };
03718   return getNode(Opcode, DL, VT, Ops);
03719 }
03720 
03721 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03722                               SDValue N1, SDValue N2, SDValue N3,
03723                               SDValue N4, SDValue N5) {
03724   SDValue Ops[] = { N1, N2, N3, N4, N5 };
03725   return getNode(Opcode, DL, VT, Ops);
03726 }
03727 
03728 /// getStackArgumentTokenFactor - Compute a TokenFactor to force all
03729 /// the incoming stack arguments to be loaded from the stack.
03730 SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
03731   SmallVector<SDValue, 8> ArgChains;
03732 
03733   // Include the original chain at the beginning of the list. When this is
03734   // used by target LowerCall hooks, this helps legalize find the
03735   // CALLSEQ_BEGIN node.
03736   ArgChains.push_back(Chain);
03737 
03738   // Add a chain value for each stack argument.
03739   for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(),
03740        UE = getEntryNode().getNode()->use_end(); U != UE; ++U)
03741     if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
03742       if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
03743         if (FI->getIndex() < 0)
03744           ArgChains.push_back(SDValue(L, 1));
03745 
03746   // Build a tokenfactor for all the chains.
03747   return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
03748 }
03749 
03750 /// getMemsetValue - Vectorized representation of the memset value
03751 /// operand.
03752 static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
03753                               SDLoc dl) {
03754   assert(Value.getOpcode() != ISD::UNDEF);
03755 
03756   unsigned NumBits = VT.getScalarType().getSizeInBits();
03757   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
03758     assert(C->getAPIntValue().getBitWidth() == 8);
03759     APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
03760     if (VT.isInteger())
03761       return DAG.getConstant(Val, VT);
03762     return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), VT);
03763   }
03764 
03765   Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value);
03766   if (NumBits > 8) {
03767     // Use a multiplication with 0x010101... to extend the input to the
03768     // required length.
03769     APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
03770     Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT));
03771   }
03772 
03773   return Value;
03774 }
03775 
03776 /// getMemsetStringVal - Similar to getMemsetValue. Except this is only
03777 /// used when a memcpy is turned into a memset when the source is a constant
03778 /// string ptr.
03779 static SDValue getMemsetStringVal(EVT VT, SDLoc dl, SelectionDAG &DAG,
03780                                   const TargetLowering &TLI, StringRef Str) {
03781   // Handle vector with all elements zero.
03782   if (Str.empty()) {
03783     if (VT.isInteger())
03784       return DAG.getConstant(0, VT);
03785     else if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
03786       return DAG.getConstantFP(0.0, VT);
03787     else if (VT.isVector()) {
03788       unsigned NumElts = VT.getVectorNumElements();
03789       MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
03790       return DAG.getNode(ISD::BITCAST, dl, VT,
03791                          DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(),
03792                                                              EltVT, NumElts)));
03793     } else
03794       llvm_unreachable("Expected type!");
03795   }
03796 
03797   assert(!VT.isVector() && "Can't handle vector type here!");
03798   unsigned NumVTBits = VT.getSizeInBits();
03799   unsigned NumVTBytes = NumVTBits / 8;
03800   unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size()));
03801 
03802   APInt Val(NumVTBits, 0);
03803   if (TLI.isLittleEndian()) {
03804     for (unsigned i = 0; i != NumBytes; ++i)
03805       Val |= (uint64_t)(unsigned char)Str[i] << i*8;
03806   } else {
03807     for (unsigned i = 0; i != NumBytes; ++i)
03808       Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8;
03809   }
03810 
03811   // If the "cost" of materializing the integer immediate is less than the cost
03812   // of a load, then it is cost effective to turn the load into the immediate.
03813   Type *Ty = VT.getTypeForEVT(*DAG.getContext());
03814   if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
03815     return DAG.getConstant(Val, VT);
03816   return SDValue(nullptr, 0);
03817 }
03818 
03819 /// getMemBasePlusOffset - Returns base and offset node for the
03820 ///
03821 static SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, SDLoc dl,
03822                                       SelectionDAG &DAG) {
03823   EVT VT = Base.getValueType();
03824   return DAG.getNode(ISD::ADD, dl,
03825                      VT, Base, DAG.getConstant(Offset, VT));
03826 }
03827 
03828 /// isMemSrcFromString - Returns true if memcpy source is a string constant.
03829 ///
03830 static bool isMemSrcFromString(SDValue Src, StringRef &Str) {
03831   unsigned SrcDelta = 0;
03832   GlobalAddressSDNode *G = nullptr;
03833   if (Src.getOpcode() == ISD::GlobalAddress)
03834     G = cast<GlobalAddressSDNode>(Src);
03835   else if (Src.getOpcode() == ISD::ADD &&
03836            Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
03837            Src.getOperand(1).getOpcode() == ISD::Constant) {
03838     G = cast<GlobalAddressSDNode>(Src.getOperand(0));
03839     SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue();
03840   }
03841   if (!G)
03842     return false;
03843 
03844   return getConstantStringInfo(G->getGlobal(), Str, SrcDelta, false);
03845 }
03846 
03847 /// FindOptimalMemOpLowering - Determines the optimial series memory ops
03848 /// to replace the memset / memcpy. Return true if the number of memory ops
03849 /// is below the threshold. It returns the types of the sequence of
03850 /// memory ops to perform memset / memcpy by reference.
03851 static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
03852                                      unsigned Limit, uint64_t Size,
03853                                      unsigned DstAlign, unsigned SrcAlign,
03854                                      bool IsMemset,
03855                                      bool ZeroMemset,
03856                                      bool MemcpyStrSrc,
03857                                      bool AllowOverlap,
03858                                      SelectionDAG &DAG,
03859                                      const TargetLowering &TLI) {
03860   assert((SrcAlign == 0 || SrcAlign >= DstAlign) &&
03861          "Expecting memcpy / memset source to meet alignment requirement!");
03862   // If 'SrcAlign' is zero, that means the memory operation does not need to
03863   // load the value, i.e. memset or memcpy from constant string. Otherwise,
03864   // it's the inferred alignment of the source. 'DstAlign', on the other hand,
03865   // is the specified alignment of the memory operation. If it is zero, that
03866   // means it's possible to change the alignment of the destination.
03867   // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does
03868   // not need to be loaded.
03869   EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign,
03870                                    IsMemset, ZeroMemset, MemcpyStrSrc,
03871                                    DAG.getMachineFunction());
03872 
03873   if (VT == MVT::Other) {
03874     unsigned AS = 0;
03875     if (DstAlign >= TLI.getDataLayout()->getPointerPrefAlignment(AS) ||
03876         TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign)) {
03877       VT = TLI.getPointerTy();
03878     } else {
03879       switch (DstAlign & 7) {
03880       case 0:  VT = MVT::i64; break;
03881       case 4:  VT = MVT::i32; break;
03882       case 2:  VT = MVT::i16; break;
03883       default: VT = MVT::i8;  break;
03884       }
03885     }
03886 
03887     MVT LVT = MVT::i64;
03888     while (!TLI.isTypeLegal(LVT))
03889       LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
03890     assert(LVT.isInteger());
03891 
03892     if (VT.bitsGT(LVT))
03893       VT = LVT;
03894   }
03895 
03896   unsigned NumMemOps = 0;
03897   while (Size != 0) {
03898     unsigned VTSize = VT.getSizeInBits() / 8;
03899     while (VTSize > Size) {
03900       // For now, only use non-vector load / store's for the left-over pieces.
03901       EVT NewVT = VT;
03902       unsigned NewVTSize;
03903 
03904       bool Found = false;
03905       if (VT.isVector() || VT.isFloatingPoint()) {
03906         NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
03907         if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) &&
03908             TLI.isSafeMemOpType(NewVT.getSimpleVT()))
03909           Found = true;
03910         else if (NewVT == MVT::i64 &&
03911                  TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
03912                  TLI.isSafeMemOpType(MVT::f64)) {
03913           // i64 is usually not legal on 32-bit targets, but f64 may be.
03914           NewVT = MVT::f64;
03915           Found = true;
03916         }
03917       }
03918 
03919       if (!Found) {
03920         do {
03921           NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
03922           if (NewVT == MVT::i8)
03923             break;
03924         } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT()));
03925       }
03926       NewVTSize = NewVT.getSizeInBits() / 8;
03927 
03928       // If the new VT cannot cover all of the remaining bits, then consider
03929       // issuing a (or a pair of) unaligned and overlapping load / store.
03930       // FIXME: Only does this for 64-bit or more since we don't have proper
03931       // cost model for unaligned load / store.
03932       bool Fast;
03933       unsigned AS = 0;
03934       if (NumMemOps && AllowOverlap &&
03935           VTSize >= 8 && NewVTSize < Size &&
03936           TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign, &Fast) && Fast)
03937         VTSize = Size;
03938       else {
03939         VT = NewVT;
03940         VTSize = NewVTSize;
03941       }
03942     }
03943 
03944     if (++NumMemOps > Limit)
03945       return false;
03946 
03947     MemOps.push_back(VT);
03948     Size -= VTSize;
03949   }
03950 
03951   return true;
03952 }
03953 
03954 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
03955                                        SDValue Chain, SDValue Dst,
03956                                        SDValue Src, uint64_t Size,
03957                                        unsigned Align, bool isVol,
03958                                        bool AlwaysInline,
03959                                        MachinePointerInfo DstPtrInfo,
03960                                        MachinePointerInfo SrcPtrInfo) {
03961   // Turn a memcpy of undef to nop.
03962   if (Src.getOpcode() == ISD::UNDEF)
03963     return Chain;
03964 
03965   // Expand memcpy to a series of load and store ops if the size operand falls
03966   // below a certain threshold.
03967   // TODO: In the AlwaysInline case, if the size is big then generate a loop
03968   // rather than maybe a humongous number of loads and stores.
03969   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03970   std::vector<EVT> MemOps;
03971   bool DstAlignCanChange = false;
03972   MachineFunction &MF = DAG.getMachineFunction();
03973   MachineFrameInfo *MFI = MF.getFrameInfo();
03974   bool OptSize =
03975     MF.getFunction()->getAttributes().
03976       hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
03977   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
03978   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
03979     DstAlignCanChange = true;
03980   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
03981   if (Align > SrcAlign)
03982     SrcAlign = Align;
03983   StringRef Str;
03984   bool CopyFromStr = isMemSrcFromString(Src, Str);
03985   bool isZeroStr = CopyFromStr && Str.empty();
03986   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
03987 
03988   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
03989                                 (DstAlignCanChange ? 0 : Align),
03990                                 (isZeroStr ? 0 : SrcAlign),
03991                                 false, false, CopyFromStr, true, DAG, TLI))
03992     return SDValue();
03993 
03994   if (DstAlignCanChange) {
03995     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
03996     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
03997 
03998     // Don't promote to an alignment that would require dynamic stack
03999     // realignment.
04000     const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
04001     if (!TRI->needsStackRealignment(MF))
04002        while (NewAlign > Align &&
04003              TLI.getDataLayout()->exceedsNaturalStackAlignment(NewAlign))
04004           NewAlign /= 2;
04005 
04006     if (NewAlign > Align) {
04007       // Give the stack frame object a larger alignment if needed.
04008       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
04009         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
04010       Align = NewAlign;
04011     }
04012   }
04013 
04014   SmallVector<SDValue, 8> OutChains;
04015   unsigned NumMemOps = MemOps.size();
04016   uint64_t SrcOff = 0, DstOff = 0;
04017   for (unsigned i = 0; i != NumMemOps; ++i) {
04018     EVT VT = MemOps[i];
04019     unsigned VTSize = VT.getSizeInBits() / 8;
04020     SDValue Value, Store;
04021 
04022     if (VTSize > Size) {
04023       // Issuing an unaligned load / store pair  that overlaps with the previous
04024       // pair. Adjust the offset accordingly.
04025       assert(i == NumMemOps-1 && i != 0);
04026       SrcOff -= VTSize - Size;
04027       DstOff -= VTSize - Size;
04028     }
04029 
04030     if (CopyFromStr &&
04031         (isZeroStr || (VT.isInteger() && !VT.isVector()))) {
04032       // It's unlikely a store of a vector immediate can be done in a single
04033       // instruction. It would require a load from a constantpool first.
04034       // We only handle zero vectors here.
04035       // FIXME: Handle other cases where store of vector immediate is done in
04036       // a single instruction.
04037       Value = getMemsetStringVal(VT, dl, DAG, TLI, Str.substr(SrcOff));
04038       if (Value.getNode())
04039         Store = DAG.getStore(Chain, dl, Value,
04040                              getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04041                              DstPtrInfo.getWithOffset(DstOff), isVol,
04042                              false, Align);
04043     }
04044 
04045     if (!Store.getNode()) {
04046       // The type might not be legal for the target.  This should only happen
04047       // if the type is smaller than a legal type, as on PPC, so the right
04048       // thing to do is generate a LoadExt/StoreTrunc pair.  These simplify
04049       // to Load/Store if NVT==VT.
04050       // FIXME does the case above also need this?
04051       EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
04052       assert(NVT.bitsGE(VT));
04053       Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain,
04054                              getMemBasePlusOffset(Src, SrcOff, dl, DAG),
04055                              SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false,
04056                              false, MinAlign(SrcAlign, SrcOff));
04057       Store = DAG.getTruncStore(Chain, dl, Value,
04058                                 getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04059                                 DstPtrInfo.getWithOffset(DstOff), VT, isVol,
04060                                 false, Align);
04061     }
04062     OutChains.push_back(Store);
04063     SrcOff += VTSize;
04064     DstOff += VTSize;
04065     Size -= VTSize;
04066   }
04067 
04068   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04069 }
04070 
04071 static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
04072                                         SDValue Chain, SDValue Dst,
04073                                         SDValue Src, uint64_t Size,
04074                                         unsigned Align,  bool isVol,
04075                                         bool AlwaysInline,
04076                                         MachinePointerInfo DstPtrInfo,
04077                                         MachinePointerInfo SrcPtrInfo) {
04078   // Turn a memmove of undef to nop.
04079   if (Src.getOpcode() == ISD::UNDEF)
04080     return Chain;
04081 
04082   // Expand memmove to a series of load and store ops if the size operand falls
04083   // below a certain threshold.
04084   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04085   std::vector<EVT> MemOps;
04086   bool DstAlignCanChange = false;
04087   MachineFunction &MF = DAG.getMachineFunction();
04088   MachineFrameInfo *MFI = MF.getFrameInfo();
04089   bool OptSize = MF.getFunction()->getAttributes().
04090     hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
04091   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
04092   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
04093     DstAlignCanChange = true;
04094   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
04095   if (Align > SrcAlign)
04096     SrcAlign = Align;
04097   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
04098 
04099   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
04100                                 (DstAlignCanChange ? 0 : Align), SrcAlign,
04101                                 false, false, false, false, DAG, TLI))
04102     return SDValue();
04103 
04104   if (DstAlignCanChange) {
04105     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
04106     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
04107     if (NewAlign > Align) {
04108       // Give the stack frame object a larger alignment if needed.
04109       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
04110         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
04111       Align = NewAlign;
04112     }
04113   }
04114 
04115   uint64_t SrcOff = 0, DstOff = 0;
04116   SmallVector<SDValue, 8> LoadValues;
04117   SmallVector<SDValue, 8> LoadChains;
04118   SmallVector<SDValue, 8> OutChains;
04119   unsigned NumMemOps = MemOps.size();
04120   for (unsigned i = 0; i < NumMemOps; i++) {
04121     EVT VT = MemOps[i];
04122     unsigned VTSize = VT.getSizeInBits() / 8;
04123     SDValue Value;
04124 
04125     Value = DAG.getLoad(VT, dl, Chain,
04126                         getMemBasePlusOffset(Src, SrcOff, dl, DAG),
04127                         SrcPtrInfo.getWithOffset(SrcOff), isVol,
04128                         false, false, SrcAlign);
04129     LoadValues.push_back(Value);
04130     LoadChains.push_back(Value.getValue(1));
04131     SrcOff += VTSize;
04132   }
04133   Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
04134   OutChains.clear();
04135   for (unsigned i = 0; i < NumMemOps; i++) {
04136     EVT VT = MemOps[i];
04137     unsigned VTSize = VT.getSizeInBits() / 8;
04138     SDValue Store;
04139 
04140     Store = DAG.getStore(Chain, dl, LoadValues[i],
04141                          getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04142                          DstPtrInfo.getWithOffset(DstOff), isVol, false, Align);
04143     OutChains.push_back(Store);
04144     DstOff += VTSize;
04145   }
04146 
04147   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04148 }
04149 
04150 /// \brief Lower the call to 'memset' intrinsic function into a series of store
04151 /// operations.
04152 ///
04153 /// \param DAG Selection DAG where lowered code is placed.
04154 /// \param dl Link to corresponding IR location.
04155 /// \param Chain Control flow dependency.
04156 /// \param Dst Pointer to destination memory location.
04157 /// \param Src Value of byte to write into the memory.
04158 /// \param Size Number of bytes to write.
04159 /// \param Align Alignment of the destination in bytes.
04160 /// \param isVol True if destination is volatile.
04161 /// \param DstPtrInfo IR information on the memory pointer.
04162 /// \returns New head in the control flow, if lowering was successful, empty
04163 /// SDValue otherwise.
04164 ///
04165 /// The function tries to replace 'llvm.memset' intrinsic with several store
04166 /// operations and value calculation code. This is usually profitable for small
04167 /// memory size.
04168 static SDValue getMemsetStores(SelectionDAG &DAG, SDLoc dl,
04169                                SDValue Chain, SDValue Dst,
04170                                SDValue Src, uint64_t Size,
04171                                unsigned Align, bool isVol,
04172                                MachinePointerInfo DstPtrInfo) {
04173   // Turn a memset of undef to nop.
04174   if (Src.getOpcode() == ISD::UNDEF)
04175     return Chain;
04176 
04177   // Expand memset to a series of load/store ops if the size operand
04178   // falls below a certain threshold.
04179   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04180   std::vector<EVT> MemOps;
04181   bool DstAlignCanChange = false;
04182   MachineFunction &MF = DAG.getMachineFunction();
04183   MachineFrameInfo *MFI = MF.getFrameInfo();
04184   bool OptSize = MF.getFunction()->getAttributes().
04185     hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
04186   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
04187   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
04188     DstAlignCanChange = true;
04189   bool IsZeroVal =
04190     isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
04191   if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize),
04192                                 Size, (DstAlignCanChange ? 0 : Align), 0,
04193                                 true, IsZeroVal, false, true, DAG, TLI))
04194     return SDValue();
04195 
04196   if (DstAlignCanChange) {
04197     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
04198     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
04199     if (NewAlign > Align) {
04200       // Give the stack frame object a larger alignment if needed.
04201       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
04202         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
04203       Align = NewAlign;
04204     }
04205   }
04206 
04207   SmallVector<SDValue, 8> OutChains;
04208   uint64_t DstOff = 0;
04209   unsigned NumMemOps = MemOps.size();
04210 
04211   // Find the largest store and generate the bit pattern for it.
04212   EVT LargestVT = MemOps[0];
04213   for (unsigned i = 1; i < NumMemOps; i++)
04214     if (MemOps[i].bitsGT(LargestVT))
04215       LargestVT = MemOps[i];
04216   SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
04217 
04218   for (unsigned i = 0; i < NumMemOps; i++) {
04219     EVT VT = MemOps[i];
04220     unsigned VTSize = VT.getSizeInBits() / 8;
04221     if (VTSize > Size) {
04222       // Issuing an unaligned load / store pair  that overlaps with the previous
04223       // pair. Adjust the offset accordingly.
04224       assert(i == NumMemOps-1 && i != 0);
04225       DstOff -= VTSize - Size;
04226     }
04227 
04228     // If this store is smaller than the largest store see whether we can get
04229     // the smaller value for free with a truncate.
04230     SDValue Value = MemSetValue;
04231     if (VT.bitsLT(LargestVT)) {
04232       if (!LargestVT.isVector() && !VT.isVector() &&
04233           TLI.isTruncateFree(LargestVT, VT))
04234         Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
04235       else
04236         Value = getMemsetValue(Src, VT, DAG, dl);
04237     }
04238     assert(Value.getValueType() == VT && "Value with wrong type.");
04239     SDValue Store = DAG.getStore(Chain, dl, Value,
04240                                  getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04241                                  DstPtrInfo.getWithOffset(DstOff),
04242                                  isVol, false, Align);
04243     OutChains.push_back(Store);
04244     DstOff += VT.getSizeInBits() / 8;
04245     Size -= VTSize;
04246   }
04247 
04248   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04249 }
04250 
04251 SDValue SelectionDAG::getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst,
04252                                 SDValue Src, SDValue Size,
04253                                 unsigned Align, bool isVol, bool AlwaysInline,
04254                                 MachinePointerInfo DstPtrInfo,
04255                                 MachinePointerInfo SrcPtrInfo) {
04256   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04257 
04258   // Check to see if we should lower the memcpy to loads and stores first.
04259   // For cases within the target-specified limits, this is the best choice.
04260   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04261   if (ConstantSize) {
04262     // Memcpy with size zero? Just return the original chain.
04263     if (ConstantSize->isNullValue())
04264       return Chain;
04265 
04266     SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
04267                                              ConstantSize->getZExtValue(),Align,
04268                                 isVol, false, DstPtrInfo, SrcPtrInfo);
04269     if (Result.getNode())
04270       return Result;
04271   }
04272 
04273   // Then check to see if we should lower the memcpy with target-specific
04274   // code. If the target chooses to do this, this is the next best.
04275   if (TSI) {
04276     SDValue Result = TSI->EmitTargetCodeForMemcpy(
04277         *this, dl, Chain, Dst, Src, Size, Align, isVol, AlwaysInline,
04278         DstPtrInfo, SrcPtrInfo);
04279     if (Result.getNode())
04280       return Result;
04281   }
04282 
04283   // If we really need inline code and the target declined to provide it,
04284   // use a (potentially long) sequence of loads and stores.
04285   if (AlwaysInline) {
04286     assert(ConstantSize && "AlwaysInline requires a constant size!");
04287     return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
04288                                    ConstantSize->getZExtValue(), Align, isVol,
04289                                    true, DstPtrInfo, SrcPtrInfo);
04290   }
04291 
04292   // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
04293   // memcpy is not guaranteed to be safe. libc memcpys aren't required to
04294   // respect volatile, so they may do things like read or write memory
04295   // beyond the given memory regions. But fixing this isn't easy, and most
04296   // people don't care.
04297 
04298   // Emit a library call.
04299   TargetLowering::ArgListTy Args;
04300   TargetLowering::ArgListEntry Entry;
04301   Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
04302   Entry.Node = Dst; Args.push_back(Entry);
04303   Entry.Node = Src; Args.push_back(Entry);
04304   Entry.Node = Size; Args.push_back(Entry);
04305   // FIXME: pass in SDLoc
04306   TargetLowering::CallLoweringInfo CLI(*this);
04307   CLI.setDebugLoc(dl).setChain(Chain)
04308     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
04309                Type::getVoidTy(*getContext()),
04310                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
04311                                  TLI->getPointerTy()), std::move(Args), 0)
04312     .setDiscardResult();
04313   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04314 
04315   return CallResult.second;
04316 }
04317 
04318 SDValue SelectionDAG::getMemmove(SDValue Chain, SDLoc dl, SDValue Dst,
04319                                  SDValue Src, SDValue Size,
04320                                  unsigned Align, bool isVol,
04321                                  MachinePointerInfo DstPtrInfo,
04322                                  MachinePointerInfo SrcPtrInfo) {
04323   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04324 
04325   // Check to see if we should lower the memmove to loads and stores first.
04326   // For cases within the target-specified limits, this is the best choice.
04327   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04328   if (ConstantSize) {
04329     // Memmove with size zero? Just return the original chain.
04330     if (ConstantSize->isNullValue())
04331       return Chain;
04332 
04333     SDValue Result =
04334       getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src,
04335                                ConstantSize->getZExtValue(), Align, isVol,
04336                                false, DstPtrInfo, SrcPtrInfo);
04337     if (Result.getNode())
04338       return Result;
04339   }
04340 
04341   // Then check to see if we should lower the memmove with target-specific
04342   // code. If the target chooses to do this, this is the next best.
04343   if (TSI) {
04344     SDValue Result = TSI->EmitTargetCodeForMemmove(
04345         *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo, SrcPtrInfo);
04346     if (Result.getNode())
04347       return Result;
04348   }
04349 
04350   // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
04351   // not be safe.  See memcpy above for more details.
04352 
04353   // Emit a library call.
04354   TargetLowering::ArgListTy Args;
04355   TargetLowering::ArgListEntry Entry;
04356   Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
04357   Entry.Node = Dst; Args.push_back(Entry);
04358   Entry.Node = Src; Args.push_back(Entry);
04359   Entry.Node = Size; Args.push_back(Entry);
04360   // FIXME:  pass in SDLoc
04361   TargetLowering::CallLoweringInfo CLI(*this);
04362   CLI.setDebugLoc(dl).setChain(Chain)
04363     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
04364                Type::getVoidTy(*getContext()),
04365                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
04366                                  TLI->getPointerTy()), std::move(Args), 0)
04367     .setDiscardResult();
04368   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04369 
04370   return CallResult.second;
04371 }
04372 
04373 SDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst,
04374                                 SDValue Src, SDValue Size,
04375                                 unsigned Align, bool isVol,
04376                                 MachinePointerInfo DstPtrInfo) {
04377   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04378 
04379   // Check to see if we should lower the memset to stores first.
04380   // For cases within the target-specified limits, this is the best choice.
04381   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04382   if (ConstantSize) {
04383     // Memset with size zero? Just return the original chain.
04384     if (ConstantSize->isNullValue())
04385       return Chain;
04386 
04387     SDValue Result =
04388       getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
04389                       Align, isVol, DstPtrInfo);
04390 
04391     if (Result.getNode())
04392       return Result;
04393   }
04394 
04395   // Then check to see if we should lower the memset with target-specific
04396   // code. If the target chooses to do this, this is the next best.
04397   if (TSI) {
04398     SDValue Result = TSI->EmitTargetCodeForMemset(
04399         *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo);
04400     if (Result.getNode())
04401       return Result;
04402   }
04403 
04404   // Emit a library call.
04405   Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(*getContext());
04406   TargetLowering::ArgListTy Args;
04407   TargetLowering::ArgListEntry Entry;
04408   Entry.Node = Dst; Entry.Ty = IntPtrTy;
04409   Args.push_back(Entry);
04410   Entry.Node = Src;
04411   Entry.Ty = Src.getValueType().getTypeForEVT(*getContext());
04412   Args.push_back(Entry);
04413   Entry.Node = Size;
04414   Entry.Ty = IntPtrTy;
04415   Args.push_back(Entry);
04416 
04417   // FIXME: pass in SDLoc
04418   TargetLowering::CallLoweringInfo CLI(*this);
04419   CLI.setDebugLoc(dl).setChain(Chain)
04420     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
04421                Type::getVoidTy(*getContext()),
04422                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
04423                                  TLI->getPointerTy()), std::move(Args), 0)
04424     .setDiscardResult();
04425 
04426   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04427   return CallResult.second;
04428 }
04429 
04430 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04431                                 SDVTList VTList, ArrayRef<SDValue> Ops,
04432                                 MachineMemOperand *MMO,
04433                                 AtomicOrdering SuccessOrdering,
04434                                 AtomicOrdering FailureOrdering,
04435                                 SynchronizationScope SynchScope) {
04436   FoldingSetNodeID ID;
04437   ID.AddInteger(MemVT.getRawBits());
04438   AddNodeIDNode(ID, Opcode, VTList, Ops);
04439   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04440   void* IP = nullptr;
04441   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04442     cast<AtomicSDNode>(E)->refineAlignment(MMO);
04443     return SDValue(E, 0);
04444   }
04445 
04446   // Allocate the operands array for the node out of the BumpPtrAllocator, since
04447   // SDNode doesn't have access to it.  This memory will be "leaked" when
04448   // the node is deallocated, but recovered when the allocator is released.
04449   // If the number of operands is less than 5 we use AtomicSDNode's internal
04450   // storage.
04451   unsigned NumOps = Ops.size();
04452   SDUse *DynOps = NumOps > 4 ? OperandAllocator.Allocate<SDUse>(NumOps)
04453                              : nullptr;
04454 
04455   SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(),
04456                                                dl.getDebugLoc(), VTList, MemVT,
04457                                                Ops.data(), DynOps, NumOps, MMO,
04458                                                SuccessOrdering, FailureOrdering,
04459                                                SynchScope);
04460   CSEMap.InsertNode(N, IP);
04461   InsertNode(N);
04462   return SDValue(N, 0);
04463 }
04464 
04465 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04466                                 SDVTList VTList, ArrayRef<SDValue> Ops,
04467                                 MachineMemOperand *MMO,
04468                                 AtomicOrdering Ordering,
04469                                 SynchronizationScope SynchScope) {
04470   return getAtomic(Opcode, dl, MemVT, VTList, Ops, MMO, Ordering,
04471                    Ordering, SynchScope);
04472 }
04473 
04474 SDValue SelectionDAG::getAtomicCmpSwap(
04475     unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs, SDValue Chain,
04476     SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo,
04477     unsigned Alignment, AtomicOrdering SuccessOrdering,
04478     AtomicOrdering FailureOrdering, SynchronizationScope SynchScope) {
04479   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
04480          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
04481   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
04482 
04483   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04484     Alignment = getEVTAlignment(MemVT);
04485 
04486   MachineFunction &MF = getMachineFunction();
04487 
04488   // FIXME: Volatile isn't really correct; we should keep track of atomic
04489   // orderings in the memoperand.
04490   unsigned Flags = MachineMemOperand::MOVolatile;
04491   Flags |= MachineMemOperand::MOLoad;
04492   Flags |= MachineMemOperand::MOStore;
04493 
04494   MachineMemOperand *MMO =
04495     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
04496 
04497   return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO,
04498                           SuccessOrdering, FailureOrdering, SynchScope);
04499 }
04500 
04501 SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, SDLoc dl, EVT MemVT,
04502                                        SDVTList VTs, SDValue Chain, SDValue Ptr,
04503                                        SDValue Cmp, SDValue Swp,
04504                                        MachineMemOperand *MMO,
04505                                        AtomicOrdering SuccessOrdering,
04506                                        AtomicOrdering FailureOrdering,
04507                                        SynchronizationScope SynchScope) {
04508   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
04509          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
04510   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
04511 
04512   SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
04513   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO,
04514                    SuccessOrdering, FailureOrdering, SynchScope);
04515 }
04516 
04517 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04518                                 SDValue Chain,
04519                                 SDValue Ptr, SDValue Val,
04520                                 const Value* PtrVal,
04521                                 unsigned Alignment,
04522                                 AtomicOrdering Ordering,
04523                                 SynchronizationScope SynchScope) {
04524   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04525     Alignment = getEVTAlignment(MemVT);
04526 
04527   MachineFunction &MF = getMachineFunction();
04528   // An atomic store does not load. An atomic load does not store.
04529   // (An atomicrmw obviously both loads and stores.)
04530   // For now, atomics are considered to be volatile always, and they are
04531   // chained as such.
04532   // FIXME: Volatile isn't really correct; we should keep track of atomic
04533   // orderings in the memoperand.
04534   unsigned Flags = MachineMemOperand::MOVolatile;
04535   if (Opcode != ISD::ATOMIC_STORE)
04536     Flags |= MachineMemOperand::MOLoad;
04537   if (Opcode != ISD::ATOMIC_LOAD)
04538     Flags |= MachineMemOperand::MOStore;
04539 
04540   MachineMemOperand *MMO =
04541     MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
04542                             MemVT.getStoreSize(), Alignment);
04543 
04544   return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO,
04545                    Ordering, SynchScope);
04546 }
04547 
04548 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04549                                 SDValue Chain,
04550                                 SDValue Ptr, SDValue Val,
04551                                 MachineMemOperand *MMO,
04552                                 AtomicOrdering Ordering,
04553                                 SynchronizationScope SynchScope) {
04554   assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
04555           Opcode == ISD::ATOMIC_LOAD_SUB ||
04556           Opcode == ISD::ATOMIC_LOAD_AND ||
04557           Opcode == ISD::ATOMIC_LOAD_OR ||
04558           Opcode == ISD::ATOMIC_LOAD_XOR ||
04559           Opcode == ISD::ATOMIC_LOAD_NAND ||
04560           Opcode == ISD::ATOMIC_LOAD_MIN ||
04561           Opcode == ISD::ATOMIC_LOAD_MAX ||
04562           Opcode == ISD::ATOMIC_LOAD_UMIN ||
04563           Opcode == ISD::ATOMIC_LOAD_UMAX ||
04564           Opcode == ISD::ATOMIC_SWAP ||
04565           Opcode == ISD::ATOMIC_STORE) &&
04566          "Invalid Atomic Op");
04567 
04568   EVT VT = Val.getValueType();
04569 
04570   SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
04571                                                getVTList(VT, MVT::Other);
04572   SDValue Ops[] = {Chain, Ptr, Val};
04573   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
04574 }
04575 
04576 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04577                                 EVT VT, SDValue Chain,
04578                                 SDValue Ptr,
04579                                 MachineMemOperand *MMO,
04580                                 AtomicOrdering Ordering,
04581                                 SynchronizationScope SynchScope) {
04582   assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
04583 
04584   SDVTList VTs = getVTList(VT, MVT::Other);
04585   SDValue Ops[] = {Chain, Ptr};
04586   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
04587 }
04588 
04589 /// getMergeValues - Create a MERGE_VALUES node from the given operands.
04590 SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, SDLoc dl) {
04591   if (Ops.size() == 1)
04592     return Ops[0];
04593 
04594   SmallVector<EVT, 4> VTs;
04595   VTs.reserve(Ops.size());
04596   for (unsigned i = 0; i < Ops.size(); ++i)
04597     VTs.push_back(Ops[i].getValueType());
04598   return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
04599 }
04600 
04601 SDValue
04602 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
04603                                   ArrayRef<SDValue> Ops,
04604                                   EVT MemVT, MachinePointerInfo PtrInfo,
04605                                   unsigned Align, bool Vol,
04606                                   bool ReadMem, bool WriteMem, unsigned Size) {
04607   if (Align == 0)  // Ensure that codegen never sees alignment 0
04608     Align = getEVTAlignment(MemVT);
04609 
04610   MachineFunction &MF = getMachineFunction();
04611   unsigned Flags = 0;
04612   if (WriteMem)
04613     Flags |= MachineMemOperand::MOStore;
04614   if (ReadMem)
04615     Flags |= MachineMemOperand::MOLoad;
04616   if (Vol)
04617     Flags |= MachineMemOperand::MOVolatile;
04618   if (!Size)
04619     Size = MemVT.getStoreSize();
04620   MachineMemOperand *MMO =
04621     MF.getMachineMemOperand(PtrInfo, Flags, Size, Align);
04622 
04623   return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
04624 }
04625 
04626 SDValue
04627 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
04628                                   ArrayRef<SDValue> Ops, EVT MemVT,
04629                                   MachineMemOperand *MMO) {
04630   assert((Opcode == ISD::INTRINSIC_VOID ||
04631           Opcode == ISD::INTRINSIC_W_CHAIN ||
04632           Opcode == ISD::PREFETCH ||
04633           Opcode == ISD::LIFETIME_START ||
04634           Opcode == ISD::LIFETIME_END ||
04635           (Opcode <= INT_MAX &&
04636            (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
04637          "Opcode is not a memory-accessing opcode!");
04638 
04639   // Memoize the node unless it returns a flag.
04640   MemIntrinsicSDNode *N;
04641   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
04642     FoldingSetNodeID ID;
04643     AddNodeIDNode(ID, Opcode, VTList, Ops);
04644     ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04645     void *IP = nullptr;
04646     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04647       cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
04648       return SDValue(E, 0);
04649     }
04650 
04651     N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
04652                                                dl.getDebugLoc(), VTList, Ops,
04653                                                MemVT, MMO);
04654     CSEMap.InsertNode(N, IP);
04655   } else {
04656     N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
04657                                                dl.getDebugLoc(), VTList, Ops,
04658                                                MemVT, MMO);
04659   }
04660   InsertNode(N);
04661   return SDValue(N, 0);
04662 }
04663 
04664 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
04665 /// MachinePointerInfo record from it.  This is particularly useful because the
04666 /// code generator has many cases where it doesn't bother passing in a
04667 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
04668 static MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) {
04669   // If this is FI+Offset, we can model it.
04670   if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
04671     return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset);
04672 
04673   // If this is (FI+Offset1)+Offset2, we can model it.
04674   if (Ptr.getOpcode() != ISD::ADD ||
04675       !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
04676       !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
04677     return MachinePointerInfo();
04678 
04679   int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
04680   return MachinePointerInfo::getFixedStack(FI, Offset+
04681                        cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
04682 }
04683 
04684 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
04685 /// MachinePointerInfo record from it.  This is particularly useful because the
04686 /// code generator has many cases where it doesn't bother passing in a
04687 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
04688 static MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) {
04689   // If the 'Offset' value isn't a constant, we can't handle this.
04690   if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
04691     return InferPointerInfo(Ptr, OffsetNode->getSExtValue());
04692   if (OffsetOp.getOpcode() == ISD::UNDEF)
04693     return InferPointerInfo(Ptr);
04694   return MachinePointerInfo();
04695 }
04696 
04697 
04698 SDValue
04699 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
04700                       EVT VT, SDLoc dl, SDValue Chain,
04701                       SDValue Ptr, SDValue Offset,
04702                       MachinePointerInfo PtrInfo, EVT MemVT,
04703                       bool isVolatile, bool isNonTemporal, bool isInvariant,
04704                       unsigned Alignment, const AAMDNodes &AAInfo,
04705                       const MDNode *Ranges) {
04706   assert(Chain.getValueType() == MVT::Other &&
04707         "Invalid chain type");
04708   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04709     Alignment = getEVTAlignment(VT);
04710 
04711   unsigned Flags = MachineMemOperand::MOLoad;
04712   if (isVolatile)
04713     Flags |= MachineMemOperand::MOVolatile;
04714   if (isNonTemporal)
04715     Flags |= MachineMemOperand::MONonTemporal;
04716   if (isInvariant)
04717     Flags |= MachineMemOperand::MOInvariant;
04718 
04719   // If we don't have a PtrInfo, infer the trivial frame index case to simplify
04720   // clients.
04721   if (PtrInfo.V.isNull())
04722     PtrInfo = InferPointerInfo(Ptr, Offset);
04723 
04724   MachineFunction &MF = getMachineFunction();
04725   MachineMemOperand *MMO =
04726     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment,
04727                             AAInfo, Ranges);
04728   return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
04729 }
04730 
04731 SDValue
04732 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
04733                       EVT VT, SDLoc dl, SDValue Chain,
04734                       SDValue Ptr, SDValue Offset, EVT MemVT,
04735                       MachineMemOperand *MMO) {
04736   if (VT == MemVT) {
04737     ExtType = ISD::NON_EXTLOAD;
04738   } else if (ExtType == ISD::NON_EXTLOAD) {
04739     assert(VT == MemVT && "Non-extending load from different memory type!");
04740   } else {
04741     // Extending load.
04742     assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
04743            "Should only be an extending load, not truncating!");
04744     assert(VT.isInteger() == MemVT.isInteger() &&
04745            "Cannot convert from FP to Int or Int -> FP!");
04746     assert(VT.isVector() == MemVT.isVector() &&
04747            "Cannot use an ext load to convert to or from a vector!");
04748     assert((!VT.isVector() ||
04749             VT.getVectorNumElements() == MemVT.getVectorNumElements()) &&
04750            "Cannot use an ext load to change the number of vector elements!");
04751   }
04752 
04753   bool Indexed = AM != ISD::UNINDEXED;
04754   assert((Indexed || Offset.getOpcode() == ISD::UNDEF) &&
04755          "Unindexed load with an offset!");
04756 
04757   SDVTList VTs = Indexed ?
04758     getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
04759   SDValue Ops[] = { Chain, Ptr, Offset };
04760   FoldingSetNodeID ID;
04761   AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
04762   ID.AddInteger(MemVT.getRawBits());
04763   ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(),
04764                                      MMO->isNonTemporal(),
04765                                      MMO->isInvariant()));
04766   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04767   void *IP = nullptr;
04768   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04769     cast<LoadSDNode>(E)->refineAlignment(MMO);
04770     return SDValue(E, 0);
04771   }
04772   SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl.getIROrder(),
04773                                              dl.getDebugLoc(), VTs, AM, ExtType,
04774                                              MemVT, MMO);
04775   CSEMap.InsertNode(N, IP);
04776   InsertNode(N);
04777   return SDValue(N, 0);
04778 }
04779 
04780 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
04781                               SDValue Chain, SDValue Ptr,
04782                               MachinePointerInfo PtrInfo,
04783                               bool isVolatile, bool isNonTemporal,
04784                               bool isInvariant, unsigned Alignment,
04785                               const AAMDNodes &AAInfo,
04786                               const MDNode *Ranges) {
04787   SDValue Undef = getUNDEF(Ptr.getValueType());
04788   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
04789                  PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment,
04790                  AAInfo, Ranges);
04791 }
04792 
04793 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
04794                               SDValue Chain, SDValue Ptr,
04795                               MachineMemOperand *MMO) {
04796   SDValue Undef = getUNDEF(Ptr.getValueType());
04797   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
04798                  VT, MMO);
04799 }
04800 
04801 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
04802                                  SDValue Chain, SDValue Ptr,
04803                                  MachinePointerInfo PtrInfo, EVT MemVT,
04804                                  bool isVolatile, bool isNonTemporal,
04805                                  bool isInvariant, unsigned Alignment,
04806                                  const AAMDNodes &AAInfo) {
04807   SDValue Undef = getUNDEF(Ptr.getValueType());
04808   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
04809                  PtrInfo, MemVT, isVolatile, isNonTemporal, isInvariant,
04810                  Alignment, AAInfo);
04811 }
04812 
04813 
04814 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
04815                                  SDValue Chain, SDValue Ptr, EVT MemVT,
04816                                  MachineMemOperand *MMO) {
04817   SDValue Undef = getUNDEF(Ptr.getValueType());
04818   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
04819                  MemVT, MMO);
04820 }
04821 
04822 SDValue
04823 SelectionDAG::getIndexedLoad(SDValue OrigLoad, SDLoc dl, SDValue Base,
04824                              SDValue Offset, ISD::MemIndexedMode AM) {
04825   LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
04826   assert(LD->getOffset().getOpcode() == ISD::UNDEF &&
04827          "Load is already a indexed load!");
04828   return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
04829                  LD->getChain(), Base, Offset, LD->getPointerInfo(),
04830                  LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(),
04831                  false, LD->getAlignment());
04832 }
04833 
04834 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
04835                                SDValue Ptr, MachinePointerInfo PtrInfo,
04836                                bool isVolatile, bool isNonTemporal,
04837                                unsigned Alignment, const AAMDNodes &AAInfo) {
04838   assert(Chain.getValueType() == MVT::Other &&
04839         "Invalid chain type");
04840   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04841     Alignment = getEVTAlignment(Val.getValueType());
04842 
04843   unsigned Flags = MachineMemOperand::MOStore;
04844   if (isVolatile)
04845     Flags |= MachineMemOperand::MOVolatile;
04846   if (isNonTemporal)
04847     Flags |= MachineMemOperand::MONonTemporal;
04848 
04849   if (PtrInfo.V.isNull())
04850     PtrInfo = InferPointerInfo(Ptr);
04851 
04852   MachineFunction &MF = getMachineFunction();
04853   MachineMemOperand *MMO =
04854     MF.getMachineMemOperand(PtrInfo, Flags,
04855                             Val.getValueType().getStoreSize(), Alignment,
04856                             AAInfo);
04857 
04858   return getStore(Chain, dl, Val, Ptr, MMO);
04859 }
04860 
04861 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
04862                                SDValue Ptr, MachineMemOperand *MMO) {
04863   assert(Chain.getValueType() == MVT::Other &&
04864         "Invalid chain type");
04865   EVT VT = Val.getValueType();
04866   SDVTList VTs = getVTList(MVT::Other);
04867   SDValue Undef = getUNDEF(Ptr.getValueType());
04868   SDValue Ops[] = { Chain, Val, Ptr, Undef };
04869   FoldingSetNodeID ID;
04870   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04871   ID.AddInteger(VT.getRawBits());
04872   ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
04873                                      MMO->isNonTemporal(), MMO->isInvariant()));
04874   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04875   void *IP = nullptr;
04876   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04877     cast<StoreSDNode>(E)->refineAlignment(MMO);
04878     return SDValue(E, 0);
04879   }
04880   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04881                                               dl.getDebugLoc(), VTs,
04882                                               ISD::UNINDEXED, false, VT, MMO);
04883   CSEMap.InsertNode(N, IP);
04884   InsertNode(N);
04885   return SDValue(N, 0);
04886 }
04887 
04888 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
04889                                     SDValue Ptr, MachinePointerInfo PtrInfo,
04890                                     EVT SVT,bool isVolatile, bool isNonTemporal,
04891                                     unsigned Alignment,
04892                                     const AAMDNodes &AAInfo) {
04893   assert(Chain.getValueType() == MVT::Other &&
04894         "Invalid chain type");
04895   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04896     Alignment = getEVTAlignment(SVT);
04897 
04898   unsigned Flags = MachineMemOperand::MOStore;
04899   if (isVolatile)
04900     Flags |= MachineMemOperand::MOVolatile;
04901   if (isNonTemporal)
04902     Flags |= MachineMemOperand::MONonTemporal;
04903 
04904   if (PtrInfo.V.isNull())
04905     PtrInfo = InferPointerInfo(Ptr);
04906 
04907   MachineFunction &MF = getMachineFunction();
04908   MachineMemOperand *MMO =
04909     MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment,
04910                             AAInfo);
04911 
04912   return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
04913 }
04914 
04915 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
04916                                     SDValue Ptr, EVT SVT,
04917                                     MachineMemOperand *MMO) {
04918   EVT VT = Val.getValueType();
04919 
04920   assert(Chain.getValueType() == MVT::Other &&
04921         "Invalid chain type");
04922   if (VT == SVT)
04923     return getStore(Chain, dl, Val, Ptr, MMO);
04924 
04925   assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
04926          "Should only be a truncating store, not extending!");
04927   assert(VT.isInteger() == SVT.isInteger() &&
04928          "Can't do FP-INT conversion!");
04929   assert(VT.isVector() == SVT.isVector() &&
04930          "Cannot use trunc store to convert to or from a vector!");
04931   assert((!VT.isVector() ||
04932           VT.getVectorNumElements() == SVT.getVectorNumElements()) &&
04933          "Cannot use trunc store to change the number of vector elements!");
04934 
04935   SDVTList VTs = getVTList(MVT::Other);
04936   SDValue Undef = getUNDEF(Ptr.getValueType());
04937   SDValue Ops[] = { Chain, Val, Ptr, Undef };
04938   FoldingSetNodeID ID;
04939   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04940   ID.AddInteger(SVT.getRawBits());
04941   ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(),
04942                                      MMO->isNonTemporal(), MMO->isInvariant()));
04943   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04944   void *IP = nullptr;
04945   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04946     cast<StoreSDNode>(E)->refineAlignment(MMO);
04947     return SDValue(E, 0);
04948   }
04949   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04950                                               dl.getDebugLoc(), VTs,
04951                                               ISD::UNINDEXED, true, SVT, MMO);
04952   CSEMap.InsertNode(N, IP);
04953   InsertNode(N);
04954   return SDValue(N, 0);
04955 }
04956 
04957 SDValue
04958 SelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base,
04959                               SDValue Offset, ISD::MemIndexedMode AM) {
04960   StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
04961   assert(ST->getOffset().getOpcode() == ISD::UNDEF &&
04962          "Store is already a indexed store!");
04963   SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
04964   SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
04965   FoldingSetNodeID ID;
04966   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04967   ID.AddInteger(ST->getMemoryVT().getRawBits());
04968   ID.AddInteger(ST->getRawSubclassData());
04969   ID.AddInteger(ST->getPointerInfo().getAddrSpace());
04970   void *IP = nullptr;
04971   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
04972     return SDValue(E, 0);
04973 
04974   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04975                                               dl.getDebugLoc(), VTs, AM,
04976                                               ST->isTruncatingStore(),
04977                                               ST->getMemoryVT(),
04978                                               ST->getMemOperand());
04979   CSEMap.InsertNode(N, IP);
04980   InsertNode(N);
04981   return SDValue(N, 0);
04982 }
04983 
04984 SDValue
04985 SelectionDAG::getMaskedLoad(EVT VT, SDLoc dl, SDValue Chain,
04986                             SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT,
04987                             MachineMemOperand *MMO, ISD::LoadExtType ExtTy) {
04988 
04989   SDVTList VTs = getVTList(VT, MVT::Other);
04990   SDValue Ops[] = { Chain, Ptr, Mask, Src0 };
04991   FoldingSetNodeID ID;
04992   AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops);
04993   ID.AddInteger(VT.getRawBits());
04994   ID.AddInteger(encodeMemSDNodeFlags(ExtTy, ISD::UNINDEXED,
04995                                      MMO->isVolatile(),
04996                                      MMO->isNonTemporal(),
04997                                      MMO->isInvariant()));
04998   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04999   void *IP = nullptr;
05000   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
05001     cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
05002     return SDValue(E, 0);
05003   }
05004   SDNode *N = new (NodeAllocator) MaskedLoadSDNode(dl.getIROrder(),
05005                                              dl.getDebugLoc(), Ops, 4, VTs,
05006                                              ExtTy, MemVT, MMO);
05007   CSEMap.InsertNode(N, IP);
05008   InsertNode(N);
05009   return SDValue(N, 0);
05010 }
05011 
05012 SDValue SelectionDAG::getMaskedStore(SDValue Chain, SDLoc dl, SDValue Val,
05013                                      SDValue Ptr, SDValue Mask, EVT MemVT,
05014                                      MachineMemOperand *MMO, bool isTrunc) {
05015   assert(Chain.getValueType() == MVT::Other &&
05016         "Invalid chain type");
05017   EVT VT = Val.getValueType();
05018   SDVTList VTs = getVTList(MVT::Other);
05019   SDValue Ops[] = { Chain, Ptr, Mask, Val };
05020   FoldingSetNodeID ID;
05021   AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops);
05022   ID.AddInteger(VT.getRawBits());
05023   ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
05024                                      MMO->isNonTemporal(), MMO->isInvariant()));
05025   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
05026   void *IP = nullptr;
05027   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
05028     cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
05029     return SDValue(E, 0);
05030   }
05031   SDNode *N = new (NodeAllocator) MaskedStoreSDNode(dl.getIROrder(),
05032                                                     dl.getDebugLoc(), Ops, 4,
05033                                                     VTs, isTrunc, MemVT, MMO);
05034   CSEMap.InsertNode(N, IP);
05035   InsertNode(N);
05036   return SDValue(N, 0);
05037 }
05038 
05039 SDValue SelectionDAG::getVAArg(EVT VT, SDLoc dl,
05040                                SDValue Chain, SDValue Ptr,
05041                                SDValue SV,
05042                                unsigned Align) {
05043   SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) };
05044   return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
05045 }
05046 
05047 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
05048                               ArrayRef<SDUse> Ops) {
05049   switch (Ops.size()) {
05050   case 0: return getNode(Opcode, DL, VT);
05051   case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
05052   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
05053   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
05054   default: break;
05055   }
05056 
05057   // Copy from an SDUse array into an SDValue array for use with
05058   // the regular getNode logic.
05059   SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
05060   return getNode(Opcode, DL, VT, NewOps);
05061 }
05062 
05063 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
05064                               ArrayRef<SDValue> Ops) {
05065   unsigned NumOps = Ops.size();
05066   switch (NumOps) {
05067   case 0: return getNode(Opcode, DL, VT);
05068   case 1: return getNode(Opcode, DL, VT, Ops[0]);
05069   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
05070   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
05071   default: break;
05072   }
05073 
05074   switch (Opcode) {
05075   default: break;
05076   case ISD::SELECT_CC: {
05077     assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
05078     assert(Ops[0].getValueType() == Ops[1].getValueType() &&
05079            "LHS and RHS of condition must have same type!");
05080     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
05081            "True and False arms of SelectCC must have same type!");
05082     assert(Ops[2].getValueType() == VT &&
05083            "select_cc node must be of same type as true and false value!");
05084     break;
05085   }
05086   case ISD::BR_CC: {
05087     assert(NumOps == 5 && "BR_CC takes 5 operands!");
05088     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
05089            "LHS/RHS of comparison should match types!");
05090     break;
05091   }
05092   }
05093 
05094   // Memoize nodes.
05095   SDNode *N;
05096   SDVTList VTs = getVTList(VT);
05097 
05098   if (VT != MVT::Glue) {
05099     FoldingSetNodeID ID;
05100     AddNodeIDNode(ID, Opcode, VTs, Ops);
05101     void *IP = nullptr;
05102 
05103     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05104       return SDValue(E, 0);
05105 
05106     N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05107                                    VTs, Ops);
05108     CSEMap.InsertNode(N, IP);
05109   } else {
05110     N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05111                                    VTs, Ops);
05112   }
05113 
05114   InsertNode(N);
05115   return SDValue(N, 0);
05116 }
05117 
05118 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
05119                               ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
05120   return getNode(Opcode, DL, getVTList(ResultTys), Ops);
05121 }
05122 
05123 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05124                               ArrayRef<SDValue> Ops) {
05125   if (VTList.NumVTs == 1)
05126     return getNode(Opcode, DL, VTList.VTs[0], Ops);
05127 
05128 #if 0
05129   switch (Opcode) {
05130   // FIXME: figure out how to safely handle things like
05131   // int foo(int x) { return 1 << (x & 255); }
05132   // int bar() { return foo(256); }
05133   case ISD::SRA_PARTS:
05134   case ISD::SRL_PARTS:
05135   case ISD::SHL_PARTS:
05136     if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
05137         cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
05138       return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
05139     else if (N3.getOpcode() == ISD::AND)
05140       if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
05141         // If the and is only masking out bits that cannot effect the shift,
05142         // eliminate the and.
05143         unsigned NumBits = VT.getScalarType().getSizeInBits()*2;
05144         if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
05145           return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
05146       }
05147     break;
05148   }
05149 #endif
05150 
05151   // Memoize the node unless it returns a flag.
05152   SDNode *N;
05153   unsigned NumOps = Ops.size();
05154   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
05155     FoldingSetNodeID ID;
05156     AddNodeIDNode(ID, Opcode, VTList, Ops);
05157     void *IP = nullptr;
05158     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05159       return SDValue(E, 0);
05160 
05161     if (NumOps == 1) {
05162       N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
05163                                           DL.getDebugLoc(), VTList, Ops[0]);
05164     } else if (NumOps == 2) {
05165       N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
05166                                            DL.getDebugLoc(), VTList, Ops[0],
05167                                            Ops[1]);
05168     } else if (NumOps == 3) {
05169       N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
05170                                             DL.getDebugLoc(), VTList, Ops[0],
05171                                             Ops[1], Ops[2]);
05172     } else {
05173       N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05174                                      VTList, Ops);
05175     }
05176     CSEMap.InsertNode(N, IP);
05177   } else {
05178     if (NumOps == 1) {
05179       N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
05180                                           DL.getDebugLoc(), VTList, Ops[0]);
05181     } else if (NumOps == 2) {
05182       N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
05183                                            DL.getDebugLoc(), VTList, Ops[0],
05184                                            Ops[1]);
05185     } else if (NumOps == 3) {
05186       N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
05187                                             DL.getDebugLoc(), VTList, Ops[0],
05188                                             Ops[1], Ops[2]);
05189     } else {
05190       N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05191                                      VTList, Ops);
05192     }
05193   }
05194   InsertNode(N);
05195   return SDValue(N, 0);
05196 }
05197 
05198 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList) {
05199   return getNode(Opcode, DL, VTList, None);
05200 }
05201 
05202 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05203                               SDValue N1) {
05204   SDValue Ops[] = { N1 };
05205   return getNode(Opcode, DL, VTList, Ops);
05206 }
05207 
05208 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05209                               SDValue N1, SDValue N2) {
05210   SDValue Ops[] = { N1, N2 };
05211   return getNode(Opcode, DL, VTList, Ops);
05212 }
05213 
05214 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05215                               SDValue N1, SDValue N2, SDValue N3) {
05216   SDValue Ops[] = { N1, N2, N3 };
05217   return getNode(Opcode, DL, VTList, Ops);
05218 }
05219 
05220 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05221                               SDValue N1, SDValue N2, SDValue N3,
05222                               SDValue N4) {
05223   SDValue Ops[] = { N1, N2, N3, N4 };
05224   return getNode(Opcode, DL, VTList, Ops);
05225 }
05226 
05227 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05228                               SDValue N1, SDValue N2, SDValue N3,
05229                               SDValue N4, SDValue N5) {
05230   SDValue Ops[] = { N1, N2, N3, N4, N5 };
05231   return getNode(Opcode, DL, VTList, Ops);
05232 }
05233 
05234 SDVTList SelectionDAG::getVTList(EVT VT) {
05235   return makeVTList(SDNode::getValueTypeList(VT), 1);
05236 }
05237 
05238 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
05239   FoldingSetNodeID ID;
05240   ID.AddInteger(2U);
05241   ID.AddInteger(VT1.getRawBits());
05242   ID.AddInteger(VT2.getRawBits());
05243 
05244   void *IP = nullptr;
05245   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05246   if (!Result) {
05247     EVT *Array = Allocator.Allocate<EVT>(2);
05248     Array[0] = VT1;
05249     Array[1] = VT2;
05250     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
05251     VTListMap.InsertNode(Result, IP);
05252   }
05253   return Result->getSDVTList();
05254 }
05255 
05256 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
05257   FoldingSetNodeID ID;
05258   ID.AddInteger(3U);
05259   ID.AddInteger(VT1.getRawBits());
05260   ID.AddInteger(VT2.getRawBits());
05261   ID.AddInteger(VT3.getRawBits());
05262 
05263   void *IP = nullptr;
05264   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05265   if (!Result) {
05266     EVT *Array = Allocator.Allocate<EVT>(3);
05267     Array[0] = VT1;
05268     Array[1] = VT2;
05269     Array[2] = VT3;
05270     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
05271     VTListMap.InsertNode(Result, IP);
05272   }
05273   return Result->getSDVTList();
05274 }
05275 
05276 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
05277   FoldingSetNodeID ID;
05278   ID.AddInteger(4U);
05279   ID.AddInteger(VT1.getRawBits());
05280   ID.AddInteger(VT2.getRawBits());
05281   ID.AddInteger(VT3.getRawBits());
05282   ID.AddInteger(VT4.getRawBits());
05283 
05284   void *IP = nullptr;
05285   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05286   if (!Result) {
05287     EVT *Array = Allocator.Allocate<EVT>(4);
05288     Array[0] = VT1;
05289     Array[1] = VT2;
05290     Array[2] = VT3;
05291     Array[3] = VT4;
05292     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
05293     VTListMap.InsertNode(Result, IP);
05294   }
05295   return Result->getSDVTList();
05296 }
05297 
05298 SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
05299   unsigned NumVTs = VTs.size();
05300   FoldingSetNodeID ID;
05301   ID.AddInteger(NumVTs);
05302   for (unsigned index = 0; index < NumVTs; index++) {
05303     ID.AddInteger(VTs[index].getRawBits());
05304   }
05305 
05306   void *IP = nullptr;
05307   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05308   if (!Result) {
05309     EVT *Array = Allocator.Allocate<EVT>(NumVTs);
05310     std::copy(VTs.begin(), VTs.end(), Array);
05311     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
05312     VTListMap.InsertNode(Result, IP);
05313   }
05314   return Result->getSDVTList();
05315 }
05316 
05317 
05318 /// UpdateNodeOperands - *Mutate* the specified node in-place to have the
05319 /// specified operands.  If the resultant node already exists in the DAG,
05320 /// this does not modify the specified node, instead it returns the node that
05321 /// already exists.  If the resultant node does not exist in the DAG, the
05322 /// input node is returned.  As a degenerate case, if you specify the same
05323 /// input operands as the node already has, the input node is returned.
05324 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
05325   assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
05326 
05327   // Check to see if there is no change.
05328   if (Op == N->getOperand(0)) return N;
05329 
05330   // See if the modified node already exists.
05331   void *InsertPos = nullptr;
05332   if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
05333     return Existing;
05334 
05335   // Nope it doesn't.  Remove the node from its current place in the maps.
05336   if (InsertPos)
05337     if (!RemoveNodeFromCSEMaps(N))
05338       InsertPos = nullptr;
05339 
05340   // Now we update the operands.
05341   N->OperandList[0].set(Op);
05342 
05343   // If this gets put into a CSE map, add it.
05344   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05345   return N;
05346 }
05347 
05348 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
05349   assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
05350 
05351   // Check to see if there is no change.
05352   if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
05353     return N;   // No operands changed, just return the input node.
05354 
05355   // See if the modified node already exists.
05356   void *InsertPos = nullptr;
05357   if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
05358     return Existing;
05359 
05360   // Nope it doesn't.  Remove the node from its current place in the maps.
05361   if (InsertPos)
05362     if (!RemoveNodeFromCSEMaps(N))
05363       InsertPos = nullptr;
05364 
05365   // Now we update the operands.
05366   if (N->OperandList[0] != Op1)
05367     N->OperandList[0].set(Op1);
05368   if (N->OperandList[1] != Op2)
05369     N->OperandList[1].set(Op2);
05370 
05371   // If this gets put into a CSE map, add it.
05372   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05373   return N;
05374 }
05375 
05376 SDNode *SelectionDAG::
05377 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
05378   SDValue Ops[] = { Op1, Op2, Op3 };
05379   return UpdateNodeOperands(N, Ops);
05380 }
05381 
05382 SDNode *SelectionDAG::
05383 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
05384                    SDValue Op3, SDValue Op4) {
05385   SDValue Ops[] = { Op1, Op2, Op3, Op4 };
05386   return UpdateNodeOperands(N, Ops);
05387 }
05388 
05389 SDNode *SelectionDAG::
05390 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
05391                    SDValue Op3, SDValue Op4, SDValue Op5) {
05392   SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
05393   return UpdateNodeOperands(N, Ops);
05394 }
05395 
05396 SDNode *SelectionDAG::
05397 UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
05398   unsigned NumOps = Ops.size();
05399   assert(N->getNumOperands() == NumOps &&
05400          "Update with wrong number of operands");
05401 
05402   // Check to see if there is no change.
05403   bool AnyChange = false;
05404   for (unsigned i = 0; i != NumOps; ++i) {
05405     if (Ops[i] != N->getOperand(i)) {
05406       AnyChange = true;
05407       break;
05408     }
05409   }
05410 
05411   // No operands changed, just return the input node.
05412   if (!AnyChange) return N;
05413 
05414   // See if the modified node already exists.
05415   void *InsertPos = nullptr;
05416   if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
05417     return Existing;
05418 
05419   // Nope it doesn't.  Remove the node from its current place in the maps.
05420   if (InsertPos)
05421     if (!RemoveNodeFromCSEMaps(N))
05422       InsertPos = nullptr;
05423 
05424   // Now we update the operands.
05425   for (unsigned i = 0; i != NumOps; ++i)
05426     if (N->OperandList[i] != Ops[i])
05427       N->OperandList[i].set(Ops[i]);
05428 
05429   // If this gets put into a CSE map, add it.
05430   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05431   return N;
05432 }
05433 
05434 /// DropOperands - Release the operands and set this node to have
05435 /// zero operands.
05436 void SDNode::DropOperands() {
05437   // Unlike the code in MorphNodeTo that does this, we don't need to
05438   // watch for dead nodes here.
05439   for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
05440     SDUse &Use = *I++;
05441     Use.set(SDValue());
05442   }
05443 }
05444 
05445 /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
05446 /// machine opcode.
05447 ///
05448 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05449                                    EVT VT) {
05450   SDVTList VTs = getVTList(VT);
05451   return SelectNodeTo(N, MachineOpc, VTs, None);
05452 }
05453 
05454 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05455                                    EVT VT, SDValue Op1) {
05456   SDVTList VTs = getVTList(VT);
05457   SDValue Ops[] = { Op1 };
05458   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05459 }
05460 
05461 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05462                                    EVT VT, SDValue Op1,
05463                                    SDValue Op2) {
05464   SDVTList VTs = getVTList(VT);
05465   SDValue Ops[] = { Op1, Op2 };
05466   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05467 }
05468 
05469 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05470                                    EVT VT, SDValue Op1,
05471                                    SDValue Op2, SDValue Op3) {
05472   SDVTList VTs = getVTList(VT);
05473   SDValue Ops[] = { Op1, Op2, Op3 };
05474   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05475 }
05476 
05477 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05478                                    EVT VT, ArrayRef<SDValue> Ops) {
05479   SDVTList VTs = getVTList(VT);
05480   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05481 }
05482 
05483 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05484                                    EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
05485   SDVTList VTs = getVTList(VT1, VT2);
05486   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05487 }
05488 
05489 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05490                                    EVT VT1, EVT VT2) {
05491   SDVTList VTs = getVTList(VT1, VT2);
05492   return SelectNodeTo(N, MachineOpc, VTs, None);
05493 }
05494 
05495 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05496                                    EVT VT1, EVT VT2, EVT VT3,
05497                                    ArrayRef<SDValue> Ops) {
05498   SDVTList VTs = getVTList(VT1, VT2, VT3);
05499   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05500 }
05501 
05502 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05503                                    EVT VT1, EVT VT2, EVT VT3, EVT VT4,
05504                                    ArrayRef<SDValue> Ops) {
05505   SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
05506   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05507 }
05508 
05509 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05510                                    EVT VT1, EVT VT2,
05511                                    SDValue Op1) {
05512   SDVTList VTs = getVTList(VT1, VT2);
05513   SDValue Ops[] = { Op1 };
05514   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05515 }
05516 
05517 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05518                                    EVT VT1, EVT VT2,
05519                                    SDValue Op1, SDValue Op2) {
05520   SDVTList VTs = getVTList(VT1, VT2);
05521   SDValue Ops[] = { Op1, Op2 };
05522   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05523 }
05524 
05525 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05526                                    EVT VT1, EVT VT2,
05527                                    SDValue Op1, SDValue Op2,
05528                                    SDValue Op3) {
05529   SDVTList VTs = getVTList(VT1, VT2);
05530   SDValue Ops[] = { Op1, Op2, Op3 };
05531   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05532 }
05533 
05534 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05535                                    EVT VT1, EVT VT2, EVT VT3,
05536                                    SDValue Op1, SDValue Op2,
05537                                    SDValue Op3) {
05538   SDVTList VTs = getVTList(VT1, VT2, VT3);
05539   SDValue Ops[] = { Op1, Op2, Op3 };
05540   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05541 }
05542 
05543 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05544                                    SDVTList VTs,ArrayRef<SDValue> Ops) {
05545   N = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
05546   // Reset the NodeID to -1.
05547   N->setNodeId(-1);
05548   return N;
05549 }
05550 
05551 /// UpdadeSDLocOnMergedSDNode - If the opt level is -O0 then it throws away
05552 /// the line number information on the merged node since it is not possible to
05553 /// preserve the information that operation is associated with multiple lines.
05554 /// This will make the debugger working better at -O0, were there is a higher
05555 /// probability having other instructions associated with that line.
05556 ///
05557 /// For IROrder, we keep the smaller of the two
05558 SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) {
05559   DebugLoc NLoc = N->getDebugLoc();
05560   if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) &&
05561     (OLoc.getDebugLoc() != NLoc)) {
05562     N->setDebugLoc(DebugLoc());
05563   }
05564   unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
05565   N->setIROrder(Order);
05566   return N;
05567 }
05568 
05569 /// MorphNodeTo - This *mutates* the specified node to have the specified
05570 /// return type, opcode, and operands.
05571 ///
05572 /// Note that MorphNodeTo returns the resultant node.  If there is already a
05573 /// node of the specified opcode and operands, it returns that node instead of
05574 /// the current one.  Note that the SDLoc need not be the same.
05575 ///
05576 /// Using MorphNodeTo is faster than creating a new node and swapping it in
05577 /// with ReplaceAllUsesWith both because it often avoids allocating a new
05578 /// node, and because it doesn't require CSE recalculation for any of
05579 /// the node's users.
05580 ///
05581 /// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
05582 /// As a consequence it isn't appropriate to use from within the DAG combiner or
05583 /// the legalizer which maintain worklists that would need to be updated when
05584 /// deleting things.
05585 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
05586                                   SDVTList VTs, ArrayRef<SDValue> Ops) {
05587   unsigned NumOps = Ops.size();
05588   // If an identical node already exists, use it.
05589   void *IP = nullptr;
05590   if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
05591     FoldingSetNodeID ID;
05592     AddNodeIDNode(ID, Opc, VTs, Ops);
05593     if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
05594       return UpdadeSDLocOnMergedSDNode(ON, SDLoc(N));
05595   }
05596 
05597   if (!RemoveNodeFromCSEMaps(N))
05598     IP = nullptr;
05599 
05600   // Start the morphing.
05601   N->NodeType = Opc;
05602   N->ValueList = VTs.VTs;
05603   N->NumValues = VTs.NumVTs;
05604 
05605   // Clear the operands list, updating used nodes to remove this from their
05606   // use list.  Keep track of any operands that become dead as a result.
05607   SmallPtrSet<SDNode*, 16> DeadNodeSet;
05608   for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
05609     SDUse &Use = *I++;
05610     SDNode *Used = Use.getNode();
05611     Use.set(SDValue());
05612     if (Used->use_empty())
05613       DeadNodeSet.insert(Used);
05614   }
05615 
05616   if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) {
05617     // Initialize the memory references information.
05618     MN->setMemRefs(nullptr, nullptr);
05619     // If NumOps is larger than the # of operands we can have in a
05620     // MachineSDNode, reallocate the operand list.
05621     if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) {
05622       if (MN->OperandsNeedDelete)
05623         delete[] MN->OperandList;
05624       if (NumOps > array_lengthof(MN->LocalOperands))
05625         // We're creating a final node that will live unmorphed for the
05626         // remainder of the current SelectionDAG iteration, so we can allocate
05627         // the operands directly out of a pool with no recycling metadata.
05628         MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
05629                          Ops.data(), NumOps);
05630       else
05631         MN->InitOperands(MN->LocalOperands, Ops.data(), NumOps);
05632       MN->OperandsNeedDelete = false;
05633     } else
05634       MN->InitOperands(MN->OperandList, Ops.data(), NumOps);
05635   } else {
05636     // If NumOps is larger than the # of operands we currently have, reallocate
05637     // the operand list.
05638     if (NumOps > N->NumOperands) {
05639       if (N->OperandsNeedDelete)
05640         delete[] N->OperandList;
05641       N->InitOperands(new SDUse[NumOps], Ops.data(), NumOps);
05642       N->OperandsNeedDelete = true;
05643     } else
05644       N->InitOperands(N->OperandList, Ops.data(), NumOps);
05645   }
05646 
05647   // Delete any nodes that are still dead after adding the uses for the
05648   // new operands.
05649   if (!DeadNodeSet.empty()) {
05650     SmallVector<SDNode *, 16> DeadNodes;
05651     for (SDNode *N : DeadNodeSet)
05652       if (N->use_empty())
05653         DeadNodes.push_back(N);
05654     RemoveDeadNodes(DeadNodes);
05655   }
05656 
05657   if (IP)
05658     CSEMap.InsertNode(N, IP);   // Memoize the new node.
05659   return N;
05660 }
05661 
05662 
05663 /// getMachineNode - These are used for target selectors to create a new node
05664 /// with specified return type(s), MachineInstr opcode, and operands.
05665 ///
05666 /// Note that getMachineNode returns the resultant node.  If there is already a
05667 /// node of the specified opcode and operands, it returns that node instead of
05668 /// the current one.
05669 MachineSDNode *
05670 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT) {
05671   SDVTList VTs = getVTList(VT);
05672   return getMachineNode(Opcode, dl, VTs, None);
05673 }
05674 
05675 MachineSDNode *
05676 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, SDValue Op1) {
05677   SDVTList VTs = getVTList(VT);
05678   SDValue Ops[] = { Op1 };
05679   return getMachineNode(Opcode, dl, VTs, Ops);
05680 }
05681 
05682 MachineSDNode *
05683 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05684                              SDValue Op1, SDValue Op2) {
05685   SDVTList VTs = getVTList(VT);
05686   SDValue Ops[] = { Op1, Op2 };
05687   return getMachineNode(Opcode, dl, VTs, Ops);
05688 }
05689 
05690 MachineSDNode *
05691 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05692                              SDValue Op1, SDValue Op2, SDValue Op3) {
05693   SDVTList VTs = getVTList(VT);
05694   SDValue Ops[] = { Op1, Op2, Op3 };
05695   return getMachineNode(Opcode, dl, VTs, Ops);
05696 }
05697 
05698 MachineSDNode *
05699 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05700                              ArrayRef<SDValue> Ops) {
05701   SDVTList VTs = getVTList(VT);
05702   return getMachineNode(Opcode, dl, VTs, Ops);
05703 }
05704 
05705 MachineSDNode *
05706 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, EVT VT2) {
05707   SDVTList VTs = getVTList(VT1, VT2);
05708   return getMachineNode(Opcode, dl, VTs, None);
05709 }
05710 
05711 MachineSDNode *
05712 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05713                              EVT VT1, EVT VT2, SDValue Op1) {
05714   SDVTList VTs = getVTList(VT1, VT2);
05715   SDValue Ops[] = { Op1 };
05716   return getMachineNode(Opcode, dl, VTs, Ops);
05717 }
05718 
05719 MachineSDNode *
05720 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05721                              EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) {
05722   SDVTList VTs = getVTList(VT1, VT2);
05723   SDValue Ops[] = { Op1, Op2 };
05724   return getMachineNode(Opcode, dl, VTs, Ops);
05725 }
05726 
05727 MachineSDNode *
05728 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05729                              EVT VT1, EVT VT2, SDValue Op1,
05730                              SDValue Op2, SDValue Op3) {
05731   SDVTList VTs = getVTList(VT1, VT2);
05732   SDValue Ops[] = { Op1, Op2, Op3 };
05733   return getMachineNode(Opcode, dl, VTs, Ops);
05734 }
05735 
05736 MachineSDNode *
05737 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05738                              EVT VT1, EVT VT2,
05739                              ArrayRef<SDValue> Ops) {
05740   SDVTList VTs = getVTList(VT1, VT2);
05741   return getMachineNode(Opcode, dl, VTs, Ops);
05742 }
05743 
05744 MachineSDNode *
05745 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05746                              EVT VT1, EVT VT2, EVT VT3,
05747                              SDValue Op1, SDValue Op2) {
05748   SDVTList VTs = getVTList(VT1, VT2, VT3);
05749   SDValue Ops[] = { Op1, Op2 };
05750   return getMachineNode(Opcode, dl, VTs, Ops);
05751 }
05752 
05753 MachineSDNode *
05754 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05755                              EVT VT1, EVT VT2, EVT VT3,
05756                              SDValue Op1, SDValue Op2, SDValue Op3) {
05757   SDVTList VTs = getVTList(VT1, VT2, VT3);
05758   SDValue Ops[] = { Op1, Op2, Op3 };
05759   return getMachineNode(Opcode, dl, VTs, Ops);
05760 }
05761 
05762 MachineSDNode *
05763 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05764                              EVT VT1, EVT VT2, EVT VT3,
05765                              ArrayRef<SDValue> Ops) {
05766   SDVTList VTs = getVTList(VT1, VT2, VT3);
05767   return getMachineNode(Opcode, dl, VTs, Ops);
05768 }
05769 
05770 MachineSDNode *
05771 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1,
05772                              EVT VT2, EVT VT3, EVT VT4,
05773                              ArrayRef<SDValue> Ops) {
05774   SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
05775   return getMachineNode(Opcode, dl, VTs, Ops);
05776 }
05777 
05778 MachineSDNode *
05779 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05780                              ArrayRef<EVT> ResultTys,
05781                              ArrayRef<SDValue> Ops) {
05782   SDVTList VTs = getVTList(ResultTys);
05783   return getMachineNode(Opcode, dl, VTs, Ops);
05784 }
05785 
05786 MachineSDNode *
05787 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
05788                              ArrayRef<SDValue> OpsArray) {
05789   bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
05790   MachineSDNode *N;
05791   void *IP = nullptr;
05792   const SDValue *Ops = OpsArray.data();
05793   unsigned NumOps = OpsArray.size();
05794 
05795   if (DoCSE) {
05796     FoldingSetNodeID ID;
05797     AddNodeIDNode(ID, ~Opcode, VTs, OpsArray);
05798     IP = nullptr;
05799     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
05800       return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL));
05801     }
05802   }
05803 
05804   // Allocate a new MachineSDNode.
05805   N = new (NodeAllocator) MachineSDNode(~Opcode, DL.getIROrder(),
05806                                         DL.getDebugLoc(), VTs);
05807 
05808   // Initialize the operands list.
05809   if (NumOps > array_lengthof(N->LocalOperands))
05810     // We're creating a final node that will live unmorphed for the
05811     // remainder of the current SelectionDAG iteration, so we can allocate
05812     // the operands directly out of a pool with no recycling metadata.
05813     N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
05814                     Ops, NumOps);
05815   else
05816     N->InitOperands(N->LocalOperands, Ops, NumOps);
05817   N->OperandsNeedDelete = false;
05818 
05819   if (DoCSE)
05820     CSEMap.InsertNode(N, IP);
05821 
05822   InsertNode(N);
05823   return N;
05824 }
05825 
05826 /// getTargetExtractSubreg - A convenience function for creating
05827 /// TargetOpcode::EXTRACT_SUBREG nodes.
05828 SDValue
05829 SelectionDAG::getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT,
05830                                      SDValue Operand) {
05831   SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
05832   SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
05833                                   VT, Operand, SRIdxVal);
05834   return SDValue(Subreg, 0);
05835 }
05836 
05837 /// getTargetInsertSubreg - A convenience function for creating
05838 /// TargetOpcode::INSERT_SUBREG nodes.
05839 SDValue
05840 SelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT,
05841                                     SDValue Operand, SDValue Subreg) {
05842   SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
05843   SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
05844                                   VT, Operand, Subreg, SRIdxVal);
05845   return SDValue(Result, 0);
05846 }
05847 
05848 /// getNodeIfExists - Get the specified node if it's already available, or
05849 /// else return NULL.
05850 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
05851                                       ArrayRef<SDValue> Ops, bool nuw, bool nsw,
05852                                       bool exact) {
05853   if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
05854     FoldingSetNodeID ID;
05855     AddNodeIDNode(ID, Opcode, VTList, Ops);
05856     if (isBinOpWithFlags(Opcode))
05857       AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
05858     void *IP = nullptr;
05859     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05860       return E;
05861   }
05862   return nullptr;
05863 }
05864 
05865 /// getDbgValue - Creates a SDDbgValue node.
05866 ///
05867 /// SDNode
05868 SDDbgValue *SelectionDAG::getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N,
05869                                       unsigned R, bool IsIndirect, uint64_t Off,
05870                                       DebugLoc DL, unsigned O) {
05871   return new (Allocator) SDDbgValue(Var, Expr, N, R, IsIndirect, Off, DL, O);
05872 }
05873 
05874 /// Constant
05875 SDDbgValue *SelectionDAG::getConstantDbgValue(MDNode *Var, MDNode *Expr,
05876                                               const Value *C, uint64_t Off,
05877                                               DebugLoc DL, unsigned O) {
05878   return new (Allocator) SDDbgValue(Var, Expr, C, Off, DL, O);
05879 }
05880 
05881 /// FrameIndex
05882 SDDbgValue *SelectionDAG::getFrameIndexDbgValue(MDNode *Var, MDNode *Expr,
05883                                                 unsigned FI, uint64_t Off,
05884                                                 DebugLoc DL, unsigned O) {
05885   return new (Allocator) SDDbgValue(Var, Expr, FI, Off, DL, O);
05886 }
05887 
05888 namespace {
05889 
05890 /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
05891 /// pointed to by a use iterator is deleted, increment the use iterator
05892 /// so that it doesn't dangle.
05893 ///
05894 class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
05895   SDNode::use_iterator &UI;
05896   SDNode::use_iterator &UE;
05897 
05898   void NodeDeleted(SDNode *N, SDNode *E) override {
05899     // Increment the iterator as needed.
05900     while (UI != UE && N == *UI)
05901       ++UI;
05902   }
05903 
05904 public:
05905   RAUWUpdateListener(SelectionDAG &d,
05906                      SDNode::use_iterator &ui,
05907                      SDNode::use_iterator &ue)
05908     : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
05909 };
05910 
05911 }
05912 
05913 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05914 /// This can cause recursive merging of nodes in the DAG.
05915 ///
05916 /// This version assumes From has a single result value.
05917 ///
05918 void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
05919   SDNode *From = FromN.getNode();
05920   assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
05921          "Cannot replace with this method!");
05922   assert(From != To.getNode() && "Cannot replace uses of with self");
05923 
05924   // Iterate over all the existing uses of From. New uses will be added
05925   // to the beginning of the use list, which we avoid visiting.
05926   // This specifically avoids visiting uses of From that arise while the
05927   // replacement is happening, because any such uses would be the result
05928   // of CSE: If an existing node looks like From after one of its operands
05929   // is replaced by To, we don't want to replace of all its users with To
05930   // too. See PR3018 for more info.
05931   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05932   RAUWUpdateListener Listener(*this, UI, UE);
05933   while (UI != UE) {
05934     SDNode *User = *UI;
05935 
05936     // This node is about to morph, remove its old self from the CSE maps.
05937     RemoveNodeFromCSEMaps(User);
05938 
05939     // A user can appear in a use list multiple times, and when this
05940     // happens the uses are usually next to each other in the list.
05941     // To help reduce the number of CSE recomputations, process all
05942     // the uses of this user that we can find this way.
05943     do {
05944       SDUse &Use = UI.getUse();
05945       ++UI;
05946       Use.set(To);
05947     } while (UI != UE && *UI == User);
05948 
05949     // Now that we have modified User, add it back to the CSE maps.  If it
05950     // already exists there, recursively merge the results together.
05951     AddModifiedNodeToCSEMaps(User);
05952   }
05953 
05954   // If we just RAUW'd the root, take note.
05955   if (FromN == getRoot())
05956     setRoot(To);
05957 }
05958 
05959 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05960 /// This can cause recursive merging of nodes in the DAG.
05961 ///
05962 /// This version assumes that for each value of From, there is a
05963 /// corresponding value in To in the same position with the same type.
05964 ///
05965 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
05966 #ifndef NDEBUG
05967   for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
05968     assert((!From->hasAnyUseOfValue(i) ||
05969             From->getValueType(i) == To->getValueType(i)) &&
05970            "Cannot use this version of ReplaceAllUsesWith!");
05971 #endif
05972 
05973   // Handle the trivial case.
05974   if (From == To)
05975     return;
05976 
05977   // Iterate over just the existing users of From. See the comments in
05978   // the ReplaceAllUsesWith above.
05979   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05980   RAUWUpdateListener Listener(*this, UI, UE);
05981   while (UI != UE) {
05982     SDNode *User = *UI;
05983 
05984     // This node is about to morph, remove its old self from the CSE maps.
05985     RemoveNodeFromCSEMaps(User);
05986 
05987     // A user can appear in a use list multiple times, and when this
05988     // happens the uses are usually next to each other in the list.
05989     // To help reduce the number of CSE recomputations, process all
05990     // the uses of this user that we can find this way.
05991     do {
05992       SDUse &Use = UI.getUse();
05993       ++UI;
05994       Use.setNode(To);
05995     } while (UI != UE && *UI == User);
05996 
05997     // Now that we have modified User, add it back to the CSE maps.  If it
05998     // already exists there, recursively merge the results together.
05999     AddModifiedNodeToCSEMaps(User);
06000   }
06001 
06002   // If we just RAUW'd the root, take note.
06003   if (From == getRoot().getNode())
06004     setRoot(SDValue(To, getRoot().getResNo()));
06005 }
06006 
06007 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
06008 /// This can cause recursive merging of nodes in the DAG.
06009 ///
06010 /// This version can replace From with any result values.  To must match the
06011 /// number and types of values returned by From.
06012 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
06013   if (From->getNumValues() == 1)  // Handle the simple case efficiently.
06014     return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
06015 
06016   // Iterate over just the existing users of From. See the comments in
06017   // the ReplaceAllUsesWith above.
06018   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
06019   RAUWUpdateListener Listener(*this, UI, UE);
06020   while (UI != UE) {
06021     SDNode *User = *UI;
06022 
06023     // This node is about to morph, remove its old self from the CSE maps.
06024     RemoveNodeFromCSEMaps(User);
06025 
06026     // A user can appear in a use list multiple times, and when this
06027     // happens the uses are usually next to each other in the list.
06028     // To help reduce the number of CSE recomputations, process all
06029     // the uses of this user that we can find this way.
06030     do {
06031       SDUse &Use = UI.getUse();
06032       const SDValue &ToOp = To[Use.getResNo()];
06033       ++UI;
06034       Use.set(ToOp);
06035     } while (UI != UE && *UI == User);
06036 
06037     // Now that we have modified User, add it back to the CSE maps.  If it
06038     // already exists there, recursively merge the results together.
06039     AddModifiedNodeToCSEMaps(User);
06040   }
06041 
06042   // If we just RAUW'd the root, take note.
06043   if (From == getRoot().getNode())
06044     setRoot(SDValue(To[getRoot().getResNo()]));
06045 }
06046 
06047 /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
06048 /// uses of other values produced by From.getNode() alone.  The Deleted
06049 /// vector is handled the same way as for ReplaceAllUsesWith.
06050 void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
06051   // Handle the really simple, really trivial case efficiently.
06052   if (From == To) return;
06053 
06054   // Handle the simple, trivial, case efficiently.
06055   if (From.getNode()->getNumValues() == 1) {
06056     ReplaceAllUsesWith(From, To);
06057     return;
06058   }
06059 
06060   // Iterate over just the existing users of From. See the comments in
06061   // the ReplaceAllUsesWith above.
06062   SDNode::use_iterator UI = From.getNode()->use_begin(),
06063                        UE = From.getNode()->use_end();
06064   RAUWUpdateListener Listener(*this, UI, UE);
06065   while (UI != UE) {
06066     SDNode *User = *UI;
06067     bool UserRemovedFromCSEMaps = false;
06068 
06069     // A user can appear in a use list multiple times, and when this
06070     // happens the uses are usually next to each other in the list.
06071     // To help reduce the number of CSE recomputations, process all
06072     // the uses of this user that we can find this way.
06073     do {
06074       SDUse &Use = UI.getUse();
06075 
06076       // Skip uses of different values from the same node.
06077       if (Use.getResNo() != From.getResNo()) {
06078         ++UI;
06079         continue;
06080       }
06081 
06082       // If this node hasn't been modified yet, it's still in the CSE maps,
06083       // so remove its old self from the CSE maps.
06084       if (!UserRemovedFromCSEMaps) {
06085         RemoveNodeFromCSEMaps(User);
06086         UserRemovedFromCSEMaps = true;
06087       }
06088 
06089       ++UI;
06090       Use.set(To);
06091     } while (UI != UE && *UI == User);
06092 
06093     // We are iterating over all uses of the From node, so if a use
06094     // doesn't use the specific value, no changes are made.
06095     if (!UserRemovedFromCSEMaps)
06096       continue;
06097 
06098     // Now that we have modified User, add it back to the CSE maps.  If it
06099     // already exists there, recursively merge the results together.
06100     AddModifiedNodeToCSEMaps(User);
06101   }
06102 
06103   // If we just RAUW'd the root, take note.
06104   if (From == getRoot())
06105     setRoot(To);
06106 }
06107 
06108 namespace {
06109   /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
06110   /// to record information about a use.
06111   struct UseMemo {
06112     SDNode *User;
06113     unsigned Index;
06114     SDUse *Use;
06115   };
06116 
06117   /// operator< - Sort Memos by User.
06118   bool operator<(const UseMemo &L, const UseMemo &R) {
06119     return (intptr_t)L.User < (intptr_t)R.User;
06120   }
06121 }
06122 
06123 /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
06124 /// uses of other values produced by From.getNode() alone.  The same value
06125 /// may appear in both the From and To list.  The Deleted vector is
06126 /// handled the same way as for ReplaceAllUsesWith.
06127 void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
06128                                               const SDValue *To,
06129                                               unsigned Num){
06130   // Handle the simple, trivial case efficiently.
06131   if (Num == 1)
06132     return ReplaceAllUsesOfValueWith(*From, *To);
06133 
06134   // Read up all the uses and make records of them. This helps
06135   // processing new uses that are introduced during the
06136   // replacement process.
06137   SmallVector<UseMemo, 4> Uses;
06138   for (unsigned i = 0; i != Num; ++i) {
06139     unsigned FromResNo = From[i].getResNo();
06140     SDNode *FromNode = From[i].getNode();
06141     for (SDNode::use_iterator UI = FromNode->use_begin(),
06142          E = FromNode->use_end(); UI != E; ++UI) {
06143       SDUse &Use = UI.getUse();
06144       if (Use.getResNo() == FromResNo) {
06145         UseMemo Memo = { *UI, i, &Use };
06146         Uses.push_back(Memo);
06147       }
06148     }
06149   }
06150 
06151   // Sort the uses, so that all the uses from a given User are together.
06152   std::sort(Uses.begin(), Uses.end());
06153 
06154   for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
06155        UseIndex != UseIndexEnd; ) {
06156     // We know that this user uses some value of From.  If it is the right
06157     // value, update it.
06158     SDNode *User = Uses[UseIndex].User;
06159 
06160     // This node is about to morph, remove its old self from the CSE maps.
06161     RemoveNodeFromCSEMaps(User);
06162 
06163     // The Uses array is sorted, so all the uses for a given User
06164     // are next to each other in the list.
06165     // To help reduce the number of CSE recomputations, process all
06166     // the uses of this user that we can find this way.
06167     do {
06168       unsigned i = Uses[UseIndex].Index;
06169       SDUse &Use = *Uses[UseIndex].Use;
06170       ++UseIndex;
06171 
06172       Use.set(To[i]);
06173     } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
06174 
06175     // Now that we have modified User, add it back to the CSE maps.  If it
06176     // already exists there, recursively merge the results together.
06177     AddModifiedNodeToCSEMaps(User);
06178   }
06179 }
06180 
06181 /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
06182 /// based on their topological order. It returns the maximum id and a vector
06183 /// of the SDNodes* in assigned order by reference.
06184 unsigned SelectionDAG::AssignTopologicalOrder() {
06185 
06186   unsigned DAGSize = 0;
06187 
06188   // SortedPos tracks the progress of the algorithm. Nodes before it are
06189   // sorted, nodes after it are unsorted. When the algorithm completes
06190   // it is at the end of the list.
06191   allnodes_iterator SortedPos = allnodes_begin();
06192 
06193   // Visit all the nodes. Move nodes with no operands to the front of
06194   // the list immediately. Annotate nodes that do have operands with their
06195   // operand count. Before we do this, the Node Id fields of the nodes
06196   // may contain arbitrary values. After, the Node Id fields for nodes
06197   // before SortedPos will contain the topological sort index, and the
06198   // Node Id fields for nodes At SortedPos and after will contain the
06199   // count of outstanding operands.
06200   for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
06201     SDNode *N = I++;
06202     checkForCycles(N, this);
06203     unsigned Degree = N->getNumOperands();
06204     if (Degree == 0) {
06205       // A node with no uses, add it to the result array immediately.
06206       N->setNodeId(DAGSize++);
06207       allnodes_iterator Q = N;
06208       if (Q != SortedPos)
06209         SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
06210       assert(SortedPos != AllNodes.end() && "Overran node list");
06211       ++SortedPos;
06212     } else {
06213       // Temporarily use the Node Id as scratch space for the degree count.
06214       N->setNodeId(Degree);
06215     }
06216   }
06217 
06218   // Visit all the nodes. As we iterate, move nodes into sorted order,
06219   // such that by the time the end is reached all nodes will be sorted.
06220   for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) {
06221     SDNode *N = I;
06222     checkForCycles(N, this);
06223     // N is in sorted position, so all its uses have one less operand
06224     // that needs to be sorted.
06225     for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
06226          UI != UE; ++UI) {
06227       SDNode *P = *UI;
06228       unsigned Degree = P->getNodeId();
06229       assert(Degree != 0 && "Invalid node degree");
06230       --Degree;
06231       if (Degree == 0) {
06232         // All of P's operands are sorted, so P may sorted now.
06233         P->setNodeId(DAGSize++);
06234         if (P != SortedPos)
06235           SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
06236         assert(SortedPos != AllNodes.end() && "Overran node list");
06237         ++SortedPos;
06238       } else {
06239         // Update P's outstanding operand count.
06240         P->setNodeId(Degree);
06241       }
06242     }
06243     if (I == SortedPos) {
06244 #ifndef NDEBUG
06245       SDNode *S = ++I;
06246       dbgs() << "Overran sorted position:\n";
06247       S->dumprFull(this); dbgs() << "\n";
06248       dbgs() << "Checking if this is due to cycles\n";
06249       checkForCycles(this, true);
06250 #endif
06251       llvm_unreachable(nullptr);
06252     }
06253   }
06254 
06255   assert(SortedPos == AllNodes.end() &&
06256          "Topological sort incomplete!");
06257   assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
06258          "First node in topological sort is not the entry token!");
06259   assert(AllNodes.front().getNodeId() == 0 &&
06260          "First node in topological sort has non-zero id!");
06261   assert(AllNodes.front().getNumOperands() == 0 &&
06262          "First node in topological sort has operands!");
06263   assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
06264          "Last node in topologic sort has unexpected id!");
06265   assert(AllNodes.back().use_empty() &&
06266          "Last node in topologic sort has users!");
06267   assert(DAGSize == allnodes_size() && "Node count mismatch!");
06268   return DAGSize;
06269 }
06270 
06271 /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
06272 /// value is produced by SD.
06273 void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) {
06274   if (SD) {
06275     assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
06276     SD->setHasDebugValue(true);
06277   }
06278   DbgInfo->add(DB, SD, isParameter);
06279 }
06280 
06281 /// TransferDbgValues - Transfer SDDbgValues.
06282 void SelectionDAG::TransferDbgValues(SDValue From, SDValue To) {
06283   if (From == To || !From.getNode()->getHasDebugValue())
06284     return;
06285   SDNode *FromNode = From.getNode();
06286   SDNode *ToNode = To.getNode();
06287   ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode);
06288   SmallVector<SDDbgValue *, 2> ClonedDVs;
06289   for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end();
06290        I != E; ++I) {
06291     SDDbgValue *Dbg = *I;
06292     if (Dbg->getKind() == SDDbgValue::SDNODE) {
06293       SDDbgValue *Clone =
06294           getDbgValue(Dbg->getVariable(), Dbg->getExpression(), ToNode,
06295                       To.getResNo(), Dbg->isIndirect(), Dbg->getOffset(),
06296                       Dbg->getDebugLoc(), Dbg->getOrder());
06297       ClonedDVs.push_back(Clone);
06298     }
06299   }
06300   for (SmallVectorImpl<SDDbgValue *>::iterator I = ClonedDVs.begin(),
06301          E = ClonedDVs.end(); I != E; ++I)
06302     AddDbgValue(*I, ToNode, false);
06303 }
06304 
06305 //===----------------------------------------------------------------------===//
06306 //                              SDNode Class
06307 //===----------------------------------------------------------------------===//
06308