LLVM API Documentation

SelectionDAG.cpp
Go to the documentation of this file.
00001 //===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This implements the SelectionDAG class.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "llvm/CodeGen/SelectionDAG.h"
00015 #include "SDNodeDbgValue.h"
00016 #include "llvm/ADT/SetVector.h"
00017 #include "llvm/ADT/SmallPtrSet.h"
00018 #include "llvm/ADT/SmallSet.h"
00019 #include "llvm/ADT/SmallVector.h"
00020 #include "llvm/ADT/StringExtras.h"
00021 #include "llvm/Analysis/ValueTracking.h"
00022 #include "llvm/CodeGen/MachineBasicBlock.h"
00023 #include "llvm/CodeGen/MachineConstantPool.h"
00024 #include "llvm/CodeGen/MachineFrameInfo.h"
00025 #include "llvm/CodeGen/MachineModuleInfo.h"
00026 #include "llvm/IR/CallingConv.h"
00027 #include "llvm/IR/Constants.h"
00028 #include "llvm/IR/DataLayout.h"
00029 #include "llvm/IR/DebugInfo.h"
00030 #include "llvm/IR/DerivedTypes.h"
00031 #include "llvm/IR/Function.h"
00032 #include "llvm/IR/GlobalAlias.h"
00033 #include "llvm/IR/GlobalVariable.h"
00034 #include "llvm/IR/Intrinsics.h"
00035 #include "llvm/Support/CommandLine.h"
00036 #include "llvm/Support/Debug.h"
00037 #include "llvm/Support/ErrorHandling.h"
00038 #include "llvm/Support/ManagedStatic.h"
00039 #include "llvm/Support/MathExtras.h"
00040 #include "llvm/Support/Mutex.h"
00041 #include "llvm/Support/raw_ostream.h"
00042 #include "llvm/Target/TargetInstrInfo.h"
00043 #include "llvm/Target/TargetIntrinsicInfo.h"
00044 #include "llvm/Target/TargetLowering.h"
00045 #include "llvm/Target/TargetMachine.h"
00046 #include "llvm/Target/TargetOptions.h"
00047 #include "llvm/Target/TargetRegisterInfo.h"
00048 #include "llvm/Target/TargetSelectionDAGInfo.h"
00049 #include "llvm/Target/TargetSubtargetInfo.h"
00050 #include <algorithm>
00051 #include <cmath>
00052 
00053 using namespace llvm;
00054 
00055 /// makeVTList - Return an instance of the SDVTList struct initialized with the
00056 /// specified members.
00057 static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
00058   SDVTList Res = {VTs, NumVTs};
00059   return Res;
00060 }
00061 
00062 // Default null implementations of the callbacks.
00063 void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
00064 void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
00065 
00066 //===----------------------------------------------------------------------===//
00067 //                              ConstantFPSDNode Class
00068 //===----------------------------------------------------------------------===//
00069 
00070 /// isExactlyValue - We don't rely on operator== working on double values, as
00071 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
00072 /// As such, this method can be used to do an exact bit-for-bit comparison of
00073 /// two floating point values.
00074 bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
00075   return getValueAPF().bitwiseIsEqual(V);
00076 }
00077 
00078 bool ConstantFPSDNode::isValueValidForType(EVT VT,
00079                                            const APFloat& Val) {
00080   assert(VT.isFloatingPoint() && "Can only convert between FP types");
00081 
00082   // convert modifies in place, so make a copy.
00083   APFloat Val2 = APFloat(Val);
00084   bool losesInfo;
00085   (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
00086                       APFloat::rmNearestTiesToEven,
00087                       &losesInfo);
00088   return !losesInfo;
00089 }
00090 
00091 //===----------------------------------------------------------------------===//
00092 //                              ISD Namespace
00093 //===----------------------------------------------------------------------===//
00094 
00095 /// isBuildVectorAllOnes - Return true if the specified node is a
00096 /// BUILD_VECTOR where all of the elements are ~0 or undef.
00097 bool ISD::isBuildVectorAllOnes(const SDNode *N) {
00098   // Look through a bit convert.
00099   while (N->getOpcode() == ISD::BITCAST)
00100     N = N->getOperand(0).getNode();
00101 
00102   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
00103 
00104   unsigned i = 0, e = N->getNumOperands();
00105 
00106   // Skip over all of the undef values.
00107   while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF)
00108     ++i;
00109 
00110   // Do not accept an all-undef vector.
00111   if (i == e) return false;
00112 
00113   // Do not accept build_vectors that aren't all constants or which have non-~0
00114   // elements. We have to be a bit careful here, as the type of the constant
00115   // may not be the same as the type of the vector elements due to type
00116   // legalization (the elements are promoted to a legal type for the target and
00117   // a vector of a type may be legal when the base element type is not).
00118   // We only want to check enough bits to cover the vector elements, because
00119   // we care if the resultant vector is all ones, not whether the individual
00120   // constants are.
00121   SDValue NotZero = N->getOperand(i);
00122   unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
00123   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
00124     if (CN->getAPIntValue().countTrailingOnes() < EltSize)
00125       return false;
00126   } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
00127     if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
00128       return false;
00129   } else
00130     return false;
00131 
00132   // Okay, we have at least one ~0 value, check to see if the rest match or are
00133   // undefs. Even with the above element type twiddling, this should be OK, as
00134   // the same type legalization should have applied to all the elements.
00135   for (++i; i != e; ++i)
00136     if (N->getOperand(i) != NotZero &&
00137         N->getOperand(i).getOpcode() != ISD::UNDEF)
00138       return false;
00139   return true;
00140 }
00141 
00142 
00143 /// isBuildVectorAllZeros - Return true if the specified node is a
00144 /// BUILD_VECTOR where all of the elements are 0 or undef.
00145 bool ISD::isBuildVectorAllZeros(const SDNode *N) {
00146   // Look through a bit convert.
00147   while (N->getOpcode() == ISD::BITCAST)
00148     N = N->getOperand(0).getNode();
00149 
00150   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
00151 
00152   bool IsAllUndef = true;
00153   for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) {
00154     if (N->getOperand(i).getOpcode() == ISD::UNDEF)
00155       continue;
00156     IsAllUndef = false;
00157     // Do not accept build_vectors that aren't all constants or which have non-0
00158     // elements. We have to be a bit careful here, as the type of the constant
00159     // may not be the same as the type of the vector elements due to type
00160     // legalization (the elements are promoted to a legal type for the target
00161     // and a vector of a type may be legal when the base element type is not).
00162     // We only want to check enough bits to cover the vector elements, because
00163     // we care if the resultant vector is all zeros, not whether the individual
00164     // constants are.
00165     SDValue Zero = N->getOperand(i);
00166     unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
00167     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Zero)) {
00168       if (CN->getAPIntValue().countTrailingZeros() < EltSize)
00169         return false;
00170     } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Zero)) {
00171       if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
00172         return false;
00173     } else
00174       return false;
00175   }
00176 
00177   // Do not accept an all-undef vector.
00178   if (IsAllUndef)
00179     return false;
00180   return true;
00181 }
00182 
00183 /// \brief Return true if the specified node is a BUILD_VECTOR node of
00184 /// all ConstantSDNode or undef.
00185 bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
00186   if (N->getOpcode() != ISD::BUILD_VECTOR)
00187     return false;
00188 
00189   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
00190     SDValue Op = N->getOperand(i);
00191     if (Op.getOpcode() == ISD::UNDEF)
00192       continue;
00193     if (!isa<ConstantSDNode>(Op))
00194       return false;
00195   }
00196   return true;
00197 }
00198 
00199 /// isScalarToVector - Return true if the specified node is a
00200 /// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low
00201 /// element is not an undef.
00202 bool ISD::isScalarToVector(const SDNode *N) {
00203   if (N->getOpcode() == ISD::SCALAR_TO_VECTOR)
00204     return true;
00205 
00206   if (N->getOpcode() != ISD::BUILD_VECTOR)
00207     return false;
00208   if (N->getOperand(0).getOpcode() == ISD::UNDEF)
00209     return false;
00210   unsigned NumElems = N->getNumOperands();
00211   if (NumElems == 1)
00212     return false;
00213   for (unsigned i = 1; i < NumElems; ++i) {
00214     SDValue V = N->getOperand(i);
00215     if (V.getOpcode() != ISD::UNDEF)
00216       return false;
00217   }
00218   return true;
00219 }
00220 
00221 /// allOperandsUndef - Return true if the node has at least one operand
00222 /// and all operands of the specified node are ISD::UNDEF.
00223 bool ISD::allOperandsUndef(const SDNode *N) {
00224   // Return false if the node has no operands.
00225   // This is "logically inconsistent" with the definition of "all" but
00226   // is probably the desired behavior.
00227   if (N->getNumOperands() == 0)
00228     return false;
00229 
00230   for (unsigned i = 0, e = N->getNumOperands(); i != e ; ++i)
00231     if (N->getOperand(i).getOpcode() != ISD::UNDEF)
00232       return false;
00233 
00234   return true;
00235 }
00236 
00237 ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
00238   switch (ExtType) {
00239   case ISD::EXTLOAD:
00240     return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
00241   case ISD::SEXTLOAD:
00242     return ISD::SIGN_EXTEND;
00243   case ISD::ZEXTLOAD:
00244     return ISD::ZERO_EXTEND;
00245   default:
00246     break;
00247   }
00248 
00249   llvm_unreachable("Invalid LoadExtType");
00250 }
00251 
00252 /// getSetCCSwappedOperands - Return the operation corresponding to (Y op X)
00253 /// when given the operation for (X op Y).
00254 ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
00255   // To perform this operation, we just need to swap the L and G bits of the
00256   // operation.
00257   unsigned OldL = (Operation >> 2) & 1;
00258   unsigned OldG = (Operation >> 1) & 1;
00259   return ISD::CondCode((Operation & ~6) |  // Keep the N, U, E bits
00260                        (OldL << 1) |       // New G bit
00261                        (OldG << 2));       // New L bit.
00262 }
00263 
00264 /// getSetCCInverse - Return the operation corresponding to !(X op Y), where
00265 /// 'op' is a valid SetCC operation.
00266 ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) {
00267   unsigned Operation = Op;
00268   if (isInteger)
00269     Operation ^= 7;   // Flip L, G, E bits, but not U.
00270   else
00271     Operation ^= 15;  // Flip all of the condition bits.
00272 
00273   if (Operation > ISD::SETTRUE2)
00274     Operation &= ~8;  // Don't let N and U bits get set.
00275 
00276   return ISD::CondCode(Operation);
00277 }
00278 
00279 
00280 /// isSignedOp - For an integer comparison, return 1 if the comparison is a
00281 /// signed operation and 2 if the result is an unsigned comparison.  Return zero
00282 /// if the operation does not depend on the sign of the input (setne and seteq).
00283 static int isSignedOp(ISD::CondCode Opcode) {
00284   switch (Opcode) {
00285   default: llvm_unreachable("Illegal integer setcc operation!");
00286   case ISD::SETEQ:
00287   case ISD::SETNE: return 0;
00288   case ISD::SETLT:
00289   case ISD::SETLE:
00290   case ISD::SETGT:
00291   case ISD::SETGE: return 1;
00292   case ISD::SETULT:
00293   case ISD::SETULE:
00294   case ISD::SETUGT:
00295   case ISD::SETUGE: return 2;
00296   }
00297 }
00298 
00299 /// getSetCCOrOperation - Return the result of a logical OR between different
00300 /// comparisons of identical values: ((X op1 Y) | (X op2 Y)).  This function
00301 /// returns SETCC_INVALID if it is not possible to represent the resultant
00302 /// comparison.
00303 ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
00304                                        bool isInteger) {
00305   if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
00306     // Cannot fold a signed integer setcc with an unsigned integer setcc.
00307     return ISD::SETCC_INVALID;
00308 
00309   unsigned Op = Op1 | Op2;  // Combine all of the condition bits.
00310 
00311   // If the N and U bits get set then the resultant comparison DOES suddenly
00312   // care about orderedness, and is true when ordered.
00313   if (Op > ISD::SETTRUE2)
00314     Op &= ~16;     // Clear the U bit if the N bit is set.
00315 
00316   // Canonicalize illegal integer setcc's.
00317   if (isInteger && Op == ISD::SETUNE)  // e.g. SETUGT | SETULT
00318     Op = ISD::SETNE;
00319 
00320   return ISD::CondCode(Op);
00321 }
00322 
00323 /// getSetCCAndOperation - Return the result of a logical AND between different
00324 /// comparisons of identical values: ((X op1 Y) & (X op2 Y)).  This
00325 /// function returns zero if it is not possible to represent the resultant
00326 /// comparison.
00327 ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
00328                                         bool isInteger) {
00329   if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
00330     // Cannot fold a signed setcc with an unsigned setcc.
00331     return ISD::SETCC_INVALID;
00332 
00333   // Combine all of the condition bits.
00334   ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
00335 
00336   // Canonicalize illegal integer setcc's.
00337   if (isInteger) {
00338     switch (Result) {
00339     default: break;
00340     case ISD::SETUO : Result = ISD::SETFALSE; break;  // SETUGT & SETULT
00341     case ISD::SETOEQ:                                 // SETEQ  & SETU[LG]E
00342     case ISD::SETUEQ: Result = ISD::SETEQ   ; break;  // SETUGE & SETULE
00343     case ISD::SETOLT: Result = ISD::SETULT  ; break;  // SETULT & SETNE
00344     case ISD::SETOGT: Result = ISD::SETUGT  ; break;  // SETUGT & SETNE
00345     }
00346   }
00347 
00348   return Result;
00349 }
00350 
00351 //===----------------------------------------------------------------------===//
00352 //                           SDNode Profile Support
00353 //===----------------------------------------------------------------------===//
00354 
00355 /// AddNodeIDOpcode - Add the node opcode to the NodeID data.
00356 ///
00357 static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC)  {
00358   ID.AddInteger(OpC);
00359 }
00360 
00361 /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
00362 /// solely with their pointer.
00363 static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
00364   ID.AddPointer(VTList.VTs);
00365 }
00366 
00367 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
00368 ///
00369 static void AddNodeIDOperands(FoldingSetNodeID &ID,
00370                               ArrayRef<SDValue> Ops) {
00371   for (auto& Op : Ops) {
00372     ID.AddPointer(Op.getNode());
00373     ID.AddInteger(Op.getResNo());
00374   }
00375 }
00376 
00377 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
00378 ///
00379 static void AddNodeIDOperands(FoldingSetNodeID &ID,
00380                               ArrayRef<SDUse> Ops) {
00381   for (auto& Op : Ops) {
00382     ID.AddPointer(Op.getNode());
00383     ID.AddInteger(Op.getResNo());
00384   }
00385 }
00386 
00387 static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, bool nuw, bool nsw,
00388                                   bool exact) {
00389   ID.AddBoolean(nuw);
00390   ID.AddBoolean(nsw);
00391   ID.AddBoolean(exact);
00392 }
00393 
00394 /// AddBinaryNodeIDCustom - Add BinarySDNodes special infos
00395 static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, unsigned Opcode,
00396                                   bool nuw, bool nsw, bool exact) {
00397   if (isBinOpWithFlags(Opcode))
00398     AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
00399 }
00400 
00401 static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
00402                           SDVTList VTList, ArrayRef<SDValue> OpList) {
00403   AddNodeIDOpcode(ID, OpC);
00404   AddNodeIDValueTypes(ID, VTList);
00405   AddNodeIDOperands(ID, OpList);
00406 }
00407 
00408 /// AddNodeIDCustom - If this is an SDNode with special info, add this info to
00409 /// the NodeID data.
00410 static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
00411   switch (N->getOpcode()) {
00412   case ISD::TargetExternalSymbol:
00413   case ISD::ExternalSymbol:
00414     llvm_unreachable("Should only be used on nodes with operands");
00415   default: break;  // Normal nodes don't need extra info.
00416   case ISD::TargetConstant:
00417   case ISD::Constant: {
00418     const ConstantSDNode *C = cast<ConstantSDNode>(N);
00419     ID.AddPointer(C->getConstantIntValue());
00420     ID.AddBoolean(C->isOpaque());
00421     break;
00422   }
00423   case ISD::TargetConstantFP:
00424   case ISD::ConstantFP: {
00425     ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
00426     break;
00427   }
00428   case ISD::TargetGlobalAddress:
00429   case ISD::GlobalAddress:
00430   case ISD::TargetGlobalTLSAddress:
00431   case ISD::GlobalTLSAddress: {
00432     const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
00433     ID.AddPointer(GA->getGlobal());
00434     ID.AddInteger(GA->getOffset());
00435     ID.AddInteger(GA->getTargetFlags());
00436     ID.AddInteger(GA->getAddressSpace());
00437     break;
00438   }
00439   case ISD::BasicBlock:
00440     ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
00441     break;
00442   case ISD::Register:
00443     ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
00444     break;
00445   case ISD::RegisterMask:
00446     ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
00447     break;
00448   case ISD::SRCVALUE:
00449     ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
00450     break;
00451   case ISD::FrameIndex:
00452   case ISD::TargetFrameIndex:
00453     ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
00454     break;
00455   case ISD::JumpTable:
00456   case ISD::TargetJumpTable:
00457     ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
00458     ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
00459     break;
00460   case ISD::ConstantPool:
00461   case ISD::TargetConstantPool: {
00462     const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
00463     ID.AddInteger(CP->getAlignment());
00464     ID.AddInteger(CP->getOffset());
00465     if (CP->isMachineConstantPoolEntry())
00466       CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
00467     else
00468       ID.AddPointer(CP->getConstVal());
00469     ID.AddInteger(CP->getTargetFlags());
00470     break;
00471   }
00472   case ISD::TargetIndex: {
00473     const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
00474     ID.AddInteger(TI->getIndex());
00475     ID.AddInteger(TI->getOffset());
00476     ID.AddInteger(TI->getTargetFlags());
00477     break;
00478   }
00479   case ISD::LOAD: {
00480     const LoadSDNode *LD = cast<LoadSDNode>(N);
00481     ID.AddInteger(LD->getMemoryVT().getRawBits());
00482     ID.AddInteger(LD->getRawSubclassData());
00483     ID.AddInteger(LD->getPointerInfo().getAddrSpace());
00484     break;
00485   }
00486   case ISD::STORE: {
00487     const StoreSDNode *ST = cast<StoreSDNode>(N);
00488     ID.AddInteger(ST->getMemoryVT().getRawBits());
00489     ID.AddInteger(ST->getRawSubclassData());
00490     ID.AddInteger(ST->getPointerInfo().getAddrSpace());
00491     break;
00492   }
00493   case ISD::SDIV:
00494   case ISD::UDIV:
00495   case ISD::SRA:
00496   case ISD::SRL:
00497   case ISD::MUL:
00498   case ISD::ADD:
00499   case ISD::SUB:
00500   case ISD::SHL: {
00501     const BinaryWithFlagsSDNode *BinNode = cast<BinaryWithFlagsSDNode>(N);
00502     AddBinaryNodeIDCustom(ID, N->getOpcode(), BinNode->hasNoUnsignedWrap(),
00503                           BinNode->hasNoSignedWrap(), BinNode->isExact());
00504     break;
00505   }
00506   case ISD::ATOMIC_CMP_SWAP:
00507   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
00508   case ISD::ATOMIC_SWAP:
00509   case ISD::ATOMIC_LOAD_ADD:
00510   case ISD::ATOMIC_LOAD_SUB:
00511   case ISD::ATOMIC_LOAD_AND:
00512   case ISD::ATOMIC_LOAD_OR:
00513   case ISD::ATOMIC_LOAD_XOR:
00514   case ISD::ATOMIC_LOAD_NAND:
00515   case ISD::ATOMIC_LOAD_MIN:
00516   case ISD::ATOMIC_LOAD_MAX:
00517   case ISD::ATOMIC_LOAD_UMIN:
00518   case ISD::ATOMIC_LOAD_UMAX:
00519   case ISD::ATOMIC_LOAD:
00520   case ISD::ATOMIC_STORE: {
00521     const AtomicSDNode *AT = cast<AtomicSDNode>(N);
00522     ID.AddInteger(AT->getMemoryVT().getRawBits());
00523     ID.AddInteger(AT->getRawSubclassData());
00524     ID.AddInteger(AT->getPointerInfo().getAddrSpace());
00525     break;
00526   }
00527   case ISD::PREFETCH: {
00528     const MemSDNode *PF = cast<MemSDNode>(N);
00529     ID.AddInteger(PF->getPointerInfo().getAddrSpace());
00530     break;
00531   }
00532   case ISD::VECTOR_SHUFFLE: {
00533     const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
00534     for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
00535          i != e; ++i)
00536       ID.AddInteger(SVN->getMaskElt(i));
00537     break;
00538   }
00539   case ISD::TargetBlockAddress:
00540   case ISD::BlockAddress: {
00541     const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
00542     ID.AddPointer(BA->getBlockAddress());
00543     ID.AddInteger(BA->getOffset());
00544     ID.AddInteger(BA->getTargetFlags());
00545     break;
00546   }
00547   } // end switch (N->getOpcode())
00548 
00549   // Target specific memory nodes could also have address spaces to check.
00550   if (N->isTargetMemoryOpcode())
00551     ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
00552 }
00553 
00554 /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
00555 /// data.
00556 static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
00557   AddNodeIDOpcode(ID, N->getOpcode());
00558   // Add the return value info.
00559   AddNodeIDValueTypes(ID, N->getVTList());
00560   // Add the operand info.
00561   AddNodeIDOperands(ID, N->ops());
00562 
00563   // Handle SDNode leafs with special info.
00564   AddNodeIDCustom(ID, N);
00565 }
00566 
00567 /// encodeMemSDNodeFlags - Generic routine for computing a value for use in
00568 /// the CSE map that carries volatility, temporalness, indexing mode, and
00569 /// extension/truncation information.
00570 ///
00571 static inline unsigned
00572 encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile,
00573                      bool isNonTemporal, bool isInvariant) {
00574   assert((ConvType & 3) == ConvType &&
00575          "ConvType may not require more than 2 bits!");
00576   assert((AM & 7) == AM &&
00577          "AM may not require more than 3 bits!");
00578   return ConvType |
00579          (AM << 2) |
00580          (isVolatile << 5) |
00581          (isNonTemporal << 6) |
00582          (isInvariant << 7);
00583 }
00584 
00585 //===----------------------------------------------------------------------===//
00586 //                              SelectionDAG Class
00587 //===----------------------------------------------------------------------===//
00588 
00589 /// doNotCSE - Return true if CSE should not be performed for this node.
00590 static bool doNotCSE(SDNode *N) {
00591   if (N->getValueType(0) == MVT::Glue)
00592     return true; // Never CSE anything that produces a flag.
00593 
00594   switch (N->getOpcode()) {
00595   default: break;
00596   case ISD::HANDLENODE:
00597   case ISD::EH_LABEL:
00598     return true;   // Never CSE these nodes.
00599   }
00600 
00601   // Check that remaining values produced are not flags.
00602   for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
00603     if (N->getValueType(i) == MVT::Glue)
00604       return true; // Never CSE anything that produces a flag.
00605 
00606   return false;
00607 }
00608 
00609 /// RemoveDeadNodes - This method deletes all unreachable nodes in the
00610 /// SelectionDAG.
00611 void SelectionDAG::RemoveDeadNodes() {
00612   // Create a dummy node (which is not added to allnodes), that adds a reference
00613   // to the root node, preventing it from being deleted.
00614   HandleSDNode Dummy(getRoot());
00615 
00616   SmallVector<SDNode*, 128> DeadNodes;
00617 
00618   // Add all obviously-dead nodes to the DeadNodes worklist.
00619   for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I)
00620     if (I->use_empty())
00621       DeadNodes.push_back(I);
00622 
00623   RemoveDeadNodes(DeadNodes);
00624 
00625   // If the root changed (e.g. it was a dead load, update the root).
00626   setRoot(Dummy.getValue());
00627 }
00628 
00629 /// RemoveDeadNodes - This method deletes the unreachable nodes in the
00630 /// given list, and any nodes that become unreachable as a result.
00631 void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
00632 
00633   // Process the worklist, deleting the nodes and adding their uses to the
00634   // worklist.
00635   while (!DeadNodes.empty()) {
00636     SDNode *N = DeadNodes.pop_back_val();
00637 
00638     for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00639       DUL->NodeDeleted(N, nullptr);
00640 
00641     // Take the node out of the appropriate CSE map.
00642     RemoveNodeFromCSEMaps(N);
00643 
00644     // Next, brutally remove the operand list.  This is safe to do, as there are
00645     // no cycles in the graph.
00646     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
00647       SDUse &Use = *I++;
00648       SDNode *Operand = Use.getNode();
00649       Use.set(SDValue());
00650 
00651       // Now that we removed this operand, see if there are no uses of it left.
00652       if (Operand->use_empty())
00653         DeadNodes.push_back(Operand);
00654     }
00655 
00656     DeallocateNode(N);
00657   }
00658 }
00659 
00660 void SelectionDAG::RemoveDeadNode(SDNode *N){
00661   SmallVector<SDNode*, 16> DeadNodes(1, N);
00662 
00663   // Create a dummy node that adds a reference to the root node, preventing
00664   // it from being deleted.  (This matters if the root is an operand of the
00665   // dead node.)
00666   HandleSDNode Dummy(getRoot());
00667 
00668   RemoveDeadNodes(DeadNodes);
00669 }
00670 
00671 void SelectionDAG::DeleteNode(SDNode *N) {
00672   // First take this out of the appropriate CSE map.
00673   RemoveNodeFromCSEMaps(N);
00674 
00675   // Finally, remove uses due to operands of this node, remove from the
00676   // AllNodes list, and delete the node.
00677   DeleteNodeNotInCSEMaps(N);
00678 }
00679 
00680 void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
00681   assert(N != AllNodes.begin() && "Cannot delete the entry node!");
00682   assert(N->use_empty() && "Cannot delete a node that is not dead!");
00683 
00684   // Drop all of the operands and decrement used node's use counts.
00685   N->DropOperands();
00686 
00687   DeallocateNode(N);
00688 }
00689 
00690 void SDDbgInfo::erase(const SDNode *Node) {
00691   DbgValMapType::iterator I = DbgValMap.find(Node);
00692   if (I == DbgValMap.end())
00693     return;
00694   for (auto &Val: I->second)
00695     Val->setIsInvalidated();
00696   DbgValMap.erase(I);
00697 }
00698 
00699 void SelectionDAG::DeallocateNode(SDNode *N) {
00700   if (N->OperandsNeedDelete)
00701     delete[] N->OperandList;
00702 
00703   // Set the opcode to DELETED_NODE to help catch bugs when node
00704   // memory is reallocated.
00705   N->NodeType = ISD::DELETED_NODE;
00706 
00707   NodeAllocator.Deallocate(AllNodes.remove(N));
00708 
00709   // If any of the SDDbgValue nodes refer to this SDNode, invalidate
00710   // them and forget about that node.
00711   DbgInfo->erase(N);
00712 }
00713 
00714 #ifndef NDEBUG
00715 /// VerifySDNode - Sanity check the given SDNode.  Aborts if it is invalid.
00716 static void VerifySDNode(SDNode *N) {
00717   switch (N->getOpcode()) {
00718   default:
00719     break;
00720   case ISD::BUILD_PAIR: {
00721     EVT VT = N->getValueType(0);
00722     assert(N->getNumValues() == 1 && "Too many results!");
00723     assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
00724            "Wrong return type!");
00725     assert(N->getNumOperands() == 2 && "Wrong number of operands!");
00726     assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
00727            "Mismatched operand types!");
00728     assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
00729            "Wrong operand type!");
00730     assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
00731            "Wrong return type size");
00732     break;
00733   }
00734   case ISD::BUILD_VECTOR: {
00735     assert(N->getNumValues() == 1 && "Too many results!");
00736     assert(N->getValueType(0).isVector() && "Wrong return type!");
00737     assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
00738            "Wrong number of operands!");
00739     EVT EltVT = N->getValueType(0).getVectorElementType();
00740     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
00741       assert((I->getValueType() == EltVT ||
00742              (EltVT.isInteger() && I->getValueType().isInteger() &&
00743               EltVT.bitsLE(I->getValueType()))) &&
00744             "Wrong operand type!");
00745       assert(I->getValueType() == N->getOperand(0).getValueType() &&
00746              "Operands must all have the same type");
00747     }
00748     break;
00749   }
00750   }
00751 }
00752 #endif // NDEBUG
00753 
00754 /// \brief Insert a newly allocated node into the DAG.
00755 ///
00756 /// Handles insertion into the all nodes list and CSE map, as well as
00757 /// verification and other common operations when a new node is allocated.
00758 void SelectionDAG::InsertNode(SDNode *N) {
00759   AllNodes.push_back(N);
00760 #ifndef NDEBUG
00761   VerifySDNode(N);
00762 #endif
00763 }
00764 
00765 /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
00766 /// correspond to it.  This is useful when we're about to delete or repurpose
00767 /// the node.  We don't want future request for structurally identical nodes
00768 /// to return N anymore.
00769 bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
00770   bool Erased = false;
00771   switch (N->getOpcode()) {
00772   case ISD::HANDLENODE: return false;  // noop.
00773   case ISD::CONDCODE:
00774     assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
00775            "Cond code doesn't exist!");
00776     Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
00777     CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
00778     break;
00779   case ISD::ExternalSymbol:
00780     Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
00781     break;
00782   case ISD::TargetExternalSymbol: {
00783     ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
00784     Erased = TargetExternalSymbols.erase(
00785                std::pair<std::string,unsigned char>(ESN->getSymbol(),
00786                                                     ESN->getTargetFlags()));
00787     break;
00788   }
00789   case ISD::VALUETYPE: {
00790     EVT VT = cast<VTSDNode>(N)->getVT();
00791     if (VT.isExtended()) {
00792       Erased = ExtendedValueTypeNodes.erase(VT);
00793     } else {
00794       Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
00795       ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
00796     }
00797     break;
00798   }
00799   default:
00800     // Remove it from the CSE Map.
00801     assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
00802     assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
00803     Erased = CSEMap.RemoveNode(N);
00804     break;
00805   }
00806 #ifndef NDEBUG
00807   // Verify that the node was actually in one of the CSE maps, unless it has a
00808   // flag result (which cannot be CSE'd) or is one of the special cases that are
00809   // not subject to CSE.
00810   if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
00811       !N->isMachineOpcode() && !doNotCSE(N)) {
00812     N->dump(this);
00813     dbgs() << "\n";
00814     llvm_unreachable("Node is not in map!");
00815   }
00816 #endif
00817   return Erased;
00818 }
00819 
00820 /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
00821 /// maps and modified in place. Add it back to the CSE maps, unless an identical
00822 /// node already exists, in which case transfer all its users to the existing
00823 /// node. This transfer can potentially trigger recursive merging.
00824 ///
00825 void
00826 SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
00827   // For node types that aren't CSE'd, just act as if no identical node
00828   // already exists.
00829   if (!doNotCSE(N)) {
00830     SDNode *Existing = CSEMap.GetOrInsertNode(N);
00831     if (Existing != N) {
00832       // If there was already an existing matching node, use ReplaceAllUsesWith
00833       // to replace the dead one with the existing one.  This can cause
00834       // recursive merging of other unrelated nodes down the line.
00835       ReplaceAllUsesWith(N, Existing);
00836 
00837       // N is now dead. Inform the listeners and delete it.
00838       for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00839         DUL->NodeDeleted(N, Existing);
00840       DeleteNodeNotInCSEMaps(N);
00841       return;
00842     }
00843   }
00844 
00845   // If the node doesn't already exist, we updated it.  Inform listeners.
00846   for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00847     DUL->NodeUpdated(N);
00848 }
00849 
00850 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00851 /// were replaced with those specified.  If this node is never memoized,
00852 /// return null, otherwise return a pointer to the slot it would take.  If a
00853 /// node already exists with these operands, the slot will be non-null.
00854 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
00855                                            void *&InsertPos) {
00856   if (doNotCSE(N))
00857     return nullptr;
00858 
00859   SDValue Ops[] = { Op };
00860   FoldingSetNodeID ID;
00861   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00862   AddNodeIDCustom(ID, N);
00863   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00864   return Node;
00865 }
00866 
00867 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00868 /// were replaced with those specified.  If this node is never memoized,
00869 /// return null, otherwise return a pointer to the slot it would take.  If a
00870 /// node already exists with these operands, the slot will be non-null.
00871 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
00872                                            SDValue Op1, SDValue Op2,
00873                                            void *&InsertPos) {
00874   if (doNotCSE(N))
00875     return nullptr;
00876 
00877   SDValue Ops[] = { Op1, Op2 };
00878   FoldingSetNodeID ID;
00879   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00880   AddNodeIDCustom(ID, N);
00881   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00882   return Node;
00883 }
00884 
00885 
00886 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00887 /// were replaced with those specified.  If this node is never memoized,
00888 /// return null, otherwise return a pointer to the slot it would take.  If a
00889 /// node already exists with these operands, the slot will be non-null.
00890 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
00891                                            void *&InsertPos) {
00892   if (doNotCSE(N))
00893     return nullptr;
00894 
00895   FoldingSetNodeID ID;
00896   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00897   AddNodeIDCustom(ID, N);
00898   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00899   return Node;
00900 }
00901 
00902 /// getEVTAlignment - Compute the default alignment value for the
00903 /// given type.
00904 ///
00905 unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
00906   Type *Ty = VT == MVT::iPTR ?
00907                    PointerType::get(Type::getInt8Ty(*getContext()), 0) :
00908                    VT.getTypeForEVT(*getContext());
00909 
00910   return TLI->getDataLayout()->getABITypeAlignment(Ty);
00911 }
00912 
00913 // EntryNode could meaningfully have debug info if we can find it...
00914 SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
00915     : TM(tm), TSI(nullptr), TLI(nullptr), OptLevel(OL),
00916       EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
00917       Root(getEntryNode()), NewNodesMustHaveLegalTypes(false),
00918       UpdateListeners(nullptr) {
00919   AllNodes.push_back(&EntryNode);
00920   DbgInfo = new SDDbgInfo();
00921 }
00922 
00923 void SelectionDAG::init(MachineFunction &mf) {
00924   MF = &mf;
00925   TLI = getSubtarget().getTargetLowering();
00926   TSI = getSubtarget().getSelectionDAGInfo();
00927   Context = &mf.getFunction()->getContext();
00928 }
00929 
00930 SelectionDAG::~SelectionDAG() {
00931   assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
00932   allnodes_clear();
00933   delete DbgInfo;
00934 }
00935 
00936 void SelectionDAG::allnodes_clear() {
00937   assert(&*AllNodes.begin() == &EntryNode);
00938   AllNodes.remove(AllNodes.begin());
00939   while (!AllNodes.empty())
00940     DeallocateNode(AllNodes.begin());
00941 }
00942 
00943 BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL,
00944                                             SDVTList VTs, SDValue N1,
00945                                             SDValue N2, bool nuw, bool nsw,
00946                                             bool exact) {
00947   if (isBinOpWithFlags(Opcode)) {
00948     BinaryWithFlagsSDNode *FN = new (NodeAllocator) BinaryWithFlagsSDNode(
00949         Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
00950     FN->setHasNoUnsignedWrap(nuw);
00951     FN->setHasNoSignedWrap(nsw);
00952     FN->setIsExact(exact);
00953 
00954     return FN;
00955   }
00956 
00957   BinarySDNode *N = new (NodeAllocator)
00958       BinarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
00959   return N;
00960 }
00961 
00962 void SelectionDAG::clear() {
00963   allnodes_clear();
00964   OperandAllocator.Reset();
00965   CSEMap.clear();
00966 
00967   ExtendedValueTypeNodes.clear();
00968   ExternalSymbols.clear();
00969   TargetExternalSymbols.clear();
00970   std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
00971             static_cast<CondCodeSDNode*>(nullptr));
00972   std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
00973             static_cast<SDNode*>(nullptr));
00974 
00975   EntryNode.UseList = nullptr;
00976   AllNodes.push_back(&EntryNode);
00977   Root = getEntryNode();
00978   DbgInfo->clear();
00979 }
00980 
00981 SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00982   return VT.bitsGT(Op.getValueType()) ?
00983     getNode(ISD::ANY_EXTEND, DL, VT, Op) :
00984     getNode(ISD::TRUNCATE, DL, VT, Op);
00985 }
00986 
00987 SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00988   return VT.bitsGT(Op.getValueType()) ?
00989     getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
00990     getNode(ISD::TRUNCATE, DL, VT, Op);
00991 }
00992 
00993 SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00994   return VT.bitsGT(Op.getValueType()) ?
00995     getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
00996     getNode(ISD::TRUNCATE, DL, VT, Op);
00997 }
00998 
00999 SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, SDLoc SL, EVT VT,
01000                                         EVT OpVT) {
01001   if (VT.bitsLE(Op.getValueType()))
01002     return getNode(ISD::TRUNCATE, SL, VT, Op);
01003 
01004   TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
01005   return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
01006 }
01007 
01008 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, SDLoc DL, EVT VT) {
01009   assert(!VT.isVector() &&
01010          "getZeroExtendInReg should use the vector element type instead of "
01011          "the vector type!");
01012   if (Op.getValueType() == VT) return Op;
01013   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01014   APInt Imm = APInt::getLowBitsSet(BitWidth,
01015                                    VT.getSizeInBits());
01016   return getNode(ISD::AND, DL, Op.getValueType(), Op,
01017                  getConstant(Imm, Op.getValueType()));
01018 }
01019 
01020 SDValue SelectionDAG::getAnyExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01021   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01022   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01023          "The sizes of the input and result must match in order to perform the "
01024          "extend in-register.");
01025   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01026          "The destination vector type must have fewer lanes than the input.");
01027   return getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Op);
01028 }
01029 
01030 SDValue SelectionDAG::getSignExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01031   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01032   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01033          "The sizes of the input and result must match in order to perform the "
01034          "extend in-register.");
01035   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01036          "The destination vector type must have fewer lanes than the input.");
01037   return getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Op);
01038 }
01039 
01040 SDValue SelectionDAG::getZeroExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01041   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01042   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01043          "The sizes of the input and result must match in order to perform the "
01044          "extend in-register.");
01045   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01046          "The destination vector type must have fewer lanes than the input.");
01047   return getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Op);
01048 }
01049 
01050 /// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
01051 ///
01052 SDValue SelectionDAG::getNOT(SDLoc DL, SDValue Val, EVT VT) {
01053   EVT EltVT = VT.getScalarType();
01054   SDValue NegOne =
01055     getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT);
01056   return getNode(ISD::XOR, DL, VT, Val, NegOne);
01057 }
01058 
01059 SDValue SelectionDAG::getLogicalNOT(SDLoc DL, SDValue Val, EVT VT) {
01060   EVT EltVT = VT.getScalarType();
01061   SDValue TrueValue;
01062   switch (TLI->getBooleanContents(VT)) {
01063     case TargetLowering::ZeroOrOneBooleanContent:
01064     case TargetLowering::UndefinedBooleanContent:
01065       TrueValue = getConstant(1, VT);
01066       break;
01067     case TargetLowering::ZeroOrNegativeOneBooleanContent:
01068       TrueValue = getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()),
01069                               VT);
01070       break;
01071   }
01072   return getNode(ISD::XOR, DL, VT, Val, TrueValue);
01073 }
01074 
01075 SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT, bool isO) {
01076   EVT EltVT = VT.getScalarType();
01077   assert((EltVT.getSizeInBits() >= 64 ||
01078          (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
01079          "getConstant with a uint64_t value that doesn't fit in the type!");
01080   return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT, isO);
01081 }
01082 
01083 SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT, bool isO)
01084 {
01085   return getConstant(*ConstantInt::get(*Context, Val), VT, isT, isO);
01086 }
01087 
01088 SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT,
01089                                   bool isO) {
01090   assert(VT.isInteger() && "Cannot create FP integer constant!");
01091 
01092   EVT EltVT = VT.getScalarType();
01093   const ConstantInt *Elt = &Val;
01094 
01095   // In some cases the vector type is legal but the element type is illegal and
01096   // needs to be promoted, for example v8i8 on ARM.  In this case, promote the
01097   // inserted value (the type does not need to match the vector element type).
01098   // Any extra bits introduced will be truncated away.
01099   if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
01100       TargetLowering::TypePromoteInteger) {
01101    EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
01102    APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits());
01103    Elt = ConstantInt::get(*getContext(), NewVal);
01104   }
01105   // In other cases the element type is illegal and needs to be expanded, for
01106   // example v2i64 on MIPS32. In this case, find the nearest legal type, split
01107   // the value into n parts and use a vector type with n-times the elements.
01108   // Then bitcast to the type requested.
01109   // Legalizing constants too early makes the DAGCombiner's job harder so we
01110   // only legalize if the DAG tells us we must produce legal types.
01111   else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
01112            TLI->getTypeAction(*getContext(), EltVT) ==
01113            TargetLowering::TypeExpandInteger) {
01114     APInt NewVal = Elt->getValue();
01115     EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
01116     unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
01117     unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
01118     EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
01119 
01120     // Check the temporary vector is the correct size. If this fails then
01121     // getTypeToTransformTo() probably returned a type whose size (in bits)
01122     // isn't a power-of-2 factor of the requested type size.
01123     assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
01124 
01125     SmallVector<SDValue, 2> EltParts;
01126     for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
01127       EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
01128                                            .trunc(ViaEltSizeInBits),
01129                                      ViaEltVT, isT, isO));
01130     }
01131 
01132     // EltParts is currently in little endian order. If we actually want
01133     // big-endian order then reverse it now.
01134     if (TLI->isBigEndian())
01135       std::reverse(EltParts.begin(), EltParts.end());
01136 
01137     // The elements must be reversed when the element order is different
01138     // to the endianness of the elements (because the BITCAST is itself a
01139     // vector shuffle in this situation). However, we do not need any code to
01140     // perform this reversal because getConstant() is producing a vector
01141     // splat.
01142     // This situation occurs in MIPS MSA.
01143 
01144     SmallVector<SDValue, 8> Ops;
01145     for (unsigned i = 0; i < VT.getVectorNumElements(); ++i)
01146       Ops.insert(Ops.end(), EltParts.begin(), EltParts.end());
01147 
01148     SDValue Result = getNode(ISD::BITCAST, SDLoc(), VT,
01149                              getNode(ISD::BUILD_VECTOR, SDLoc(), ViaVecVT,
01150                                      Ops));
01151     return Result;
01152   }
01153 
01154   assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
01155          "APInt size does not match type size!");
01156   unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
01157   FoldingSetNodeID ID;
01158   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
01159   ID.AddPointer(Elt);
01160   ID.AddBoolean(isO);
01161   void *IP = nullptr;
01162   SDNode *N = nullptr;
01163   if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
01164     if (!VT.isVector())
01165       return SDValue(N, 0);
01166 
01167   if (!N) {
01168     N = new (NodeAllocator) ConstantSDNode(isT, isO, Elt, EltVT);
01169     CSEMap.InsertNode(N, IP);
01170     InsertNode(N);
01171   }
01172 
01173   SDValue Result(N, 0);
01174   if (VT.isVector()) {
01175     SmallVector<SDValue, 8> Ops;
01176     Ops.assign(VT.getVectorNumElements(), Result);
01177     Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
01178   }
01179   return Result;
01180 }
01181 
01182 SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) {
01183   return getConstant(Val, TLI->getPointerTy(), isTarget);
01184 }
01185 
01186 
01187 SDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) {
01188   return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget);
01189 }
01190 
01191 SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){
01192   assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
01193 
01194   EVT EltVT = VT.getScalarType();
01195 
01196   // Do the map lookup using the actual bit pattern for the floating point
01197   // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
01198   // we don't have issues with SNANs.
01199   unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
01200   FoldingSetNodeID ID;
01201   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
01202   ID.AddPointer(&V);
01203   void *IP = nullptr;
01204   SDNode *N = nullptr;
01205   if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
01206     if (!VT.isVector())
01207       return SDValue(N, 0);
01208 
01209   if (!N) {
01210     N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT);
01211     CSEMap.InsertNode(N, IP);
01212     InsertNode(N);
01213   }
01214 
01215   SDValue Result(N, 0);
01216   if (VT.isVector()) {
01217     SmallVector<SDValue, 8> Ops;
01218     Ops.assign(VT.getVectorNumElements(), Result);
01219     // FIXME SDLoc info might be appropriate here
01220     Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
01221   }
01222   return Result;
01223 }
01224 
01225 SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) {
01226   EVT EltVT = VT.getScalarType();
01227   if (EltVT==MVT::f32)
01228     return getConstantFP(APFloat((float)Val), VT, isTarget);
01229   else if (EltVT==MVT::f64)
01230     return getConstantFP(APFloat(Val), VT, isTarget);
01231   else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 ||
01232            EltVT==MVT::f16) {
01233     bool ignored;
01234     APFloat apf = APFloat(Val);
01235     apf.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
01236                 &ignored);
01237     return getConstantFP(apf, VT, isTarget);
01238   } else
01239     llvm_unreachable("Unsupported type in getConstantFP");
01240 }
01241 
01242 SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, SDLoc DL,
01243                                        EVT VT, int64_t Offset,
01244                                        bool isTargetGA,
01245                                        unsigned char TargetFlags) {
01246   assert((TargetFlags == 0 || isTargetGA) &&
01247          "Cannot set target flags on target-independent globals");
01248 
01249   // Truncate (with sign-extension) the offset value to the pointer size.
01250   unsigned BitWidth = TLI->getPointerTypeSizeInBits(GV->getType());
01251   if (BitWidth < 64)
01252     Offset = SignExtend64(Offset, BitWidth);
01253 
01254   unsigned Opc;
01255   if (GV->isThreadLocal())
01256     Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
01257   else
01258     Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
01259 
01260   FoldingSetNodeID ID;
01261   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01262   ID.AddPointer(GV);
01263   ID.AddInteger(Offset);
01264   ID.AddInteger(TargetFlags);
01265   ID.AddInteger(GV->getType()->getAddressSpace());
01266   void *IP = nullptr;
01267   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01268     return SDValue(E, 0);
01269 
01270   SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL.getIROrder(),
01271                                                       DL.getDebugLoc(), GV, VT,
01272                                                       Offset, TargetFlags);
01273   CSEMap.InsertNode(N, IP);
01274     InsertNode(N);
01275   return SDValue(N, 0);
01276 }
01277 
01278 SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
01279   unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
01280   FoldingSetNodeID ID;
01281   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01282   ID.AddInteger(FI);
01283   void *IP = nullptr;
01284   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01285     return SDValue(E, 0);
01286 
01287   SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget);
01288   CSEMap.InsertNode(N, IP);
01289   InsertNode(N);
01290   return SDValue(N, 0);
01291 }
01292 
01293 SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
01294                                    unsigned char TargetFlags) {
01295   assert((TargetFlags == 0 || isTarget) &&
01296          "Cannot set target flags on target-independent jump tables");
01297   unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
01298   FoldingSetNodeID ID;
01299   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01300   ID.AddInteger(JTI);
01301   ID.AddInteger(TargetFlags);
01302   void *IP = nullptr;
01303   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01304     return SDValue(E, 0);
01305 
01306   SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget,
01307                                                   TargetFlags);
01308   CSEMap.InsertNode(N, IP);
01309   InsertNode(N);
01310   return SDValue(N, 0);
01311 }
01312 
01313 SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
01314                                       unsigned Alignment, int Offset,
01315                                       bool isTarget,
01316                                       unsigned char TargetFlags) {
01317   assert((TargetFlags == 0 || isTarget) &&
01318          "Cannot set target flags on target-independent globals");
01319   if (Alignment == 0)
01320     Alignment = TLI->getDataLayout()->getPrefTypeAlignment(C->getType());
01321   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
01322   FoldingSetNodeID ID;
01323   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01324   ID.AddInteger(Alignment);
01325   ID.AddInteger(Offset);
01326   ID.AddPointer(C);
01327   ID.AddInteger(TargetFlags);
01328   void *IP = nullptr;
01329   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01330     return SDValue(E, 0);
01331 
01332   SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
01333                                                      Alignment, TargetFlags);
01334   CSEMap.InsertNode(N, IP);
01335   InsertNode(N);
01336   return SDValue(N, 0);
01337 }
01338 
01339 
01340 SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
01341                                       unsigned Alignment, int Offset,
01342                                       bool isTarget,
01343                                       unsigned char TargetFlags) {
01344   assert((TargetFlags == 0 || isTarget) &&
01345          "Cannot set target flags on target-independent globals");
01346   if (Alignment == 0)
01347     Alignment = TLI->getDataLayout()->getPrefTypeAlignment(C->getType());
01348   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
01349   FoldingSetNodeID ID;
01350   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01351   ID.AddInteger(Alignment);
01352   ID.AddInteger(Offset);
01353   C->addSelectionDAGCSEId(ID);
01354   ID.AddInteger(TargetFlags);
01355   void *IP = nullptr;
01356   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01357     return SDValue(E, 0);
01358 
01359   SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
01360                                                      Alignment, TargetFlags);
01361   CSEMap.InsertNode(N, IP);
01362   InsertNode(N);
01363   return SDValue(N, 0);
01364 }
01365 
01366 SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
01367                                      unsigned char TargetFlags) {
01368   FoldingSetNodeID ID;
01369   AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None);
01370   ID.AddInteger(Index);
01371   ID.AddInteger(Offset);
01372   ID.AddInteger(TargetFlags);
01373   void *IP = nullptr;
01374   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01375     return SDValue(E, 0);
01376 
01377   SDNode *N = new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset,
01378                                                     TargetFlags);
01379   CSEMap.InsertNode(N, IP);
01380   InsertNode(N);
01381   return SDValue(N, 0);
01382 }
01383 
01384 SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
01385   FoldingSetNodeID ID;
01386   AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
01387   ID.AddPointer(MBB);
01388   void *IP = nullptr;
01389   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01390     return SDValue(E, 0);
01391 
01392   SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB);
01393   CSEMap.InsertNode(N, IP);
01394   InsertNode(N);
01395   return SDValue(N, 0);
01396 }
01397 
01398 SDValue SelectionDAG::getValueType(EVT VT) {
01399   if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
01400       ValueTypeNodes.size())
01401     ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
01402 
01403   SDNode *&N = VT.isExtended() ?
01404     ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
01405 
01406   if (N) return SDValue(N, 0);
01407   N = new (NodeAllocator) VTSDNode(VT);
01408   InsertNode(N);
01409   return SDValue(N, 0);
01410 }
01411 
01412 SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
01413   SDNode *&N = ExternalSymbols[Sym];
01414   if (N) return SDValue(N, 0);
01415   N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT);
01416   InsertNode(N);
01417   return SDValue(N, 0);
01418 }
01419 
01420 SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
01421                                               unsigned char TargetFlags) {
01422   SDNode *&N =
01423     TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym,
01424                                                                TargetFlags)];
01425   if (N) return SDValue(N, 0);
01426   N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT);
01427   InsertNode(N);
01428   return SDValue(N, 0);
01429 }
01430 
01431 SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
01432   if ((unsigned)Cond >= CondCodeNodes.size())
01433     CondCodeNodes.resize(Cond+1);
01434 
01435   if (!CondCodeNodes[Cond]) {
01436     CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond);
01437     CondCodeNodes[Cond] = N;
01438     InsertNode(N);
01439   }
01440 
01441   return SDValue(CondCodeNodes[Cond], 0);
01442 }
01443 
01444 // commuteShuffle - swaps the values of N1 and N2, and swaps all indices in
01445 // the shuffle mask M that point at N1 to point at N2, and indices that point
01446 // N2 to point at N1.
01447 static void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) {
01448   std::swap(N1, N2);
01449   int NElts = M.size();
01450   for (int i = 0; i != NElts; ++i) {
01451     if (M[i] >= NElts)
01452       M[i] -= NElts;
01453     else if (M[i] >= 0)
01454       M[i] += NElts;
01455   }
01456 }
01457 
01458 SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1,
01459                                        SDValue N2, const int *Mask) {
01460   assert(VT == N1.getValueType() && VT == N2.getValueType() &&
01461          "Invalid VECTOR_SHUFFLE");
01462 
01463   // Canonicalize shuffle undef, undef -> undef
01464   if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF)
01465     return getUNDEF(VT);
01466 
01467   // Validate that all indices in Mask are within the range of the elements
01468   // input to the shuffle.
01469   unsigned NElts = VT.getVectorNumElements();
01470   SmallVector<int, 8> MaskVec;
01471   for (unsigned i = 0; i != NElts; ++i) {
01472     assert(Mask[i] < (int)(NElts * 2) && "Index out of range");
01473     MaskVec.push_back(Mask[i]);
01474   }
01475 
01476   // Canonicalize shuffle v, v -> v, undef
01477   if (N1 == N2) {
01478     N2 = getUNDEF(VT);
01479     for (unsigned i = 0; i != NElts; ++i)
01480       if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts;
01481   }
01482 
01483   // Canonicalize shuffle undef, v -> v, undef.  Commute the shuffle mask.
01484   if (N1.getOpcode() == ISD::UNDEF)
01485     commuteShuffle(N1, N2, MaskVec);
01486 
01487   // If shuffling a splat, try to blend the splat instead. We do this here so
01488   // that even when this arises during lowering we don't have to re-handle it.
01489   auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
01490     BitVector UndefElements;
01491     SDValue Splat = BV->getSplatValue(&UndefElements);
01492     if (!Splat)
01493       return;
01494 
01495     for (int i = 0; i < (int)NElts; ++i) {
01496       if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + (int)NElts))
01497         continue;
01498 
01499       // If this input comes from undef, mark it as such.
01500       if (UndefElements[MaskVec[i] - Offset]) {
01501         MaskVec[i] = -1;
01502         continue;
01503       }
01504 
01505       // If we can blend a non-undef lane, use that instead.
01506       if (!UndefElements[i])
01507         MaskVec[i] = i + Offset;
01508     }
01509   };
01510   if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
01511     BlendSplat(N1BV, 0);
01512   if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
01513     BlendSplat(N2BV, NElts);
01514 
01515   // Canonicalize all index into lhs, -> shuffle lhs, undef
01516   // Canonicalize all index into rhs, -> shuffle rhs, undef
01517   bool AllLHS = true, AllRHS = true;
01518   bool N2Undef = N2.getOpcode() == ISD::UNDEF;
01519   for (unsigned i = 0; i != NElts; ++i) {
01520     if (MaskVec[i] >= (int)NElts) {
01521       if (N2Undef)
01522         MaskVec[i] = -1;
01523       else
01524         AllLHS = false;
01525     } else if (MaskVec[i] >= 0) {
01526       AllRHS = false;
01527     }
01528   }
01529   if (AllLHS && AllRHS)
01530     return getUNDEF(VT);
01531   if (AllLHS && !N2Undef)
01532     N2 = getUNDEF(VT);
01533   if (AllRHS) {
01534     N1 = getUNDEF(VT);
01535     commuteShuffle(N1, N2, MaskVec);
01536   }
01537   // Reset our undef status after accounting for the mask.
01538   N2Undef = N2.getOpcode() == ISD::UNDEF;
01539   // Re-check whether both sides ended up undef.
01540   if (N1.getOpcode() == ISD::UNDEF && N2Undef)
01541     return getUNDEF(VT);
01542 
01543   // If Identity shuffle return that node.
01544   bool Identity = true, AllSame = true;
01545   for (unsigned i = 0; i != NElts; ++i) {
01546     if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false;
01547     if (MaskVec[i] != MaskVec[0]) AllSame = false;
01548   }
01549   if (Identity && NElts)
01550     return N1;
01551 
01552   // Shuffling a constant splat doesn't change the result.
01553   if (N2Undef) {
01554     SDValue V = N1;
01555 
01556     // Look through any bitcasts. We check that these don't change the number
01557     // (and size) of elements and just changes their types.
01558     while (V.getOpcode() == ISD::BITCAST)
01559       V = V->getOperand(0);
01560 
01561     // A splat should always show up as a build vector node.
01562     if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
01563       BitVector UndefElements;
01564       SDValue Splat = BV->getSplatValue(&UndefElements);
01565       // If this is a splat of an undef, shuffling it is also undef.
01566       if (Splat && Splat.getOpcode() == ISD::UNDEF)
01567         return getUNDEF(VT);
01568 
01569       bool SameNumElts =
01570           V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
01571 
01572       // We only have a splat which can skip shuffles if there is a splatted
01573       // value and no undef lanes rearranged by the shuffle.
01574       if (Splat && UndefElements.none()) {
01575         // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
01576         // number of elements match or the value splatted is a zero constant.
01577         if (SameNumElts)
01578           return N1;
01579         if (auto *C = dyn_cast<ConstantSDNode>(Splat))
01580           if (C->isNullValue())
01581             return N1;
01582       }
01583 
01584       // If the shuffle itself creates a splat, build the vector directly.
01585       if (AllSame && SameNumElts) {
01586         const SDValue &Splatted = BV->getOperand(MaskVec[0]);
01587         SmallVector<SDValue, 8> Ops(NElts, Splatted);
01588 
01589         EVT BuildVT = BV->getValueType(0);
01590         SDValue NewBV = getNode(ISD::BUILD_VECTOR, dl, BuildVT, Ops);
01591 
01592         // We may have jumped through bitcasts, so the type of the
01593         // BUILD_VECTOR may not match the type of the shuffle.
01594         if (BuildVT != VT)
01595           NewBV = getNode(ISD::BITCAST, dl, VT, NewBV);
01596         return NewBV;
01597       }
01598     }
01599   }
01600 
01601   FoldingSetNodeID ID;
01602   SDValue Ops[2] = { N1, N2 };
01603   AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
01604   for (unsigned i = 0; i != NElts; ++i)
01605     ID.AddInteger(MaskVec[i]);
01606 
01607   void* IP = nullptr;
01608   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01609     return SDValue(E, 0);
01610 
01611   // Allocate the mask array for the node out of the BumpPtrAllocator, since
01612   // SDNode doesn't have access to it.  This memory will be "leaked" when
01613   // the node is deallocated, but recovered when the NodeAllocator is released.
01614   int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
01615   memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int));
01616 
01617   ShuffleVectorSDNode *N =
01618     new (NodeAllocator) ShuffleVectorSDNode(VT, dl.getIROrder(),
01619                                             dl.getDebugLoc(), N1, N2,
01620                                             MaskAlloc);
01621   CSEMap.InsertNode(N, IP);
01622   InsertNode(N);
01623   return SDValue(N, 0);
01624 }
01625 
01626 SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
01627   MVT VT = SV.getSimpleValueType(0);
01628   unsigned NumElems = VT.getVectorNumElements();
01629   SmallVector<int, 8> MaskVec;
01630 
01631   for (unsigned i = 0; i != NumElems; ++i) {
01632     int Idx = SV.getMaskElt(i);
01633     if (Idx >= 0) {
01634       if (Idx < (int)NumElems)
01635         Idx += NumElems;
01636       else
01637         Idx -= NumElems;
01638     }
01639     MaskVec.push_back(Idx);
01640   }
01641 
01642   SDValue Op0 = SV.getOperand(0);
01643   SDValue Op1 = SV.getOperand(1);
01644   return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, &MaskVec[0]);
01645 }
01646 
01647 SDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl,
01648                                        SDValue Val, SDValue DTy,
01649                                        SDValue STy, SDValue Rnd, SDValue Sat,
01650                                        ISD::CvtCode Code) {
01651   // If the src and dest types are the same and the conversion is between
01652   // integer types of the same sign or two floats, no conversion is necessary.
01653   if (DTy == STy &&
01654       (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF))
01655     return Val;
01656 
01657   FoldingSetNodeID ID;
01658   SDValue Ops[] = { Val, DTy, STy, Rnd, Sat };
01659   AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), Ops);
01660   void* IP = nullptr;
01661   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01662     return SDValue(E, 0);
01663 
01664   CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl.getIROrder(),
01665                                                            dl.getDebugLoc(),
01666                                                            Ops, Code);
01667   CSEMap.InsertNode(N, IP);
01668   InsertNode(N);
01669   return SDValue(N, 0);
01670 }
01671 
01672 SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
01673   FoldingSetNodeID ID;
01674   AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
01675   ID.AddInteger(RegNo);
01676   void *IP = nullptr;
01677   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01678     return SDValue(E, 0);
01679 
01680   SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT);
01681   CSEMap.InsertNode(N, IP);
01682   InsertNode(N);
01683   return SDValue(N, 0);
01684 }
01685 
01686 SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
01687   FoldingSetNodeID ID;
01688   AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
01689   ID.AddPointer(RegMask);
01690   void *IP = nullptr;
01691   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01692     return SDValue(E, 0);
01693 
01694   SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask);
01695   CSEMap.InsertNode(N, IP);
01696   InsertNode(N);
01697   return SDValue(N, 0);
01698 }
01699 
01700 SDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) {
01701   FoldingSetNodeID ID;
01702   SDValue Ops[] = { Root };
01703   AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), Ops);
01704   ID.AddPointer(Label);
01705   void *IP = nullptr;
01706   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01707     return SDValue(E, 0);
01708 
01709   SDNode *N = new (NodeAllocator) EHLabelSDNode(dl.getIROrder(),
01710                                                 dl.getDebugLoc(), Root, Label);
01711   CSEMap.InsertNode(N, IP);
01712   InsertNode(N);
01713   return SDValue(N, 0);
01714 }
01715 
01716 
01717 SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
01718                                       int64_t Offset,
01719                                       bool isTarget,
01720                                       unsigned char TargetFlags) {
01721   unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
01722 
01723   FoldingSetNodeID ID;
01724   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01725   ID.AddPointer(BA);
01726   ID.AddInteger(Offset);
01727   ID.AddInteger(TargetFlags);
01728   void *IP = nullptr;
01729   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01730     return SDValue(E, 0);
01731 
01732   SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset,
01733                                                      TargetFlags);
01734   CSEMap.InsertNode(N, IP);
01735   InsertNode(N);
01736   return SDValue(N, 0);
01737 }
01738 
01739 SDValue SelectionDAG::getSrcValue(const Value *V) {
01740   assert((!V || V->getType()->isPointerTy()) &&
01741          "SrcValue is not a pointer?");
01742 
01743   FoldingSetNodeID ID;
01744   AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None);
01745   ID.AddPointer(V);
01746 
01747   void *IP = nullptr;
01748   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01749     return SDValue(E, 0);
01750 
01751   SDNode *N = new (NodeAllocator) SrcValueSDNode(V);
01752   CSEMap.InsertNode(N, IP);
01753   InsertNode(N);
01754   return SDValue(N, 0);
01755 }
01756 
01757 /// getMDNode - Return an MDNodeSDNode which holds an MDNode.
01758 SDValue SelectionDAG::getMDNode(const MDNode *MD) {
01759   FoldingSetNodeID ID;
01760   AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None);
01761   ID.AddPointer(MD);
01762 
01763   void *IP = nullptr;
01764   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01765     return SDValue(E, 0);
01766 
01767   SDNode *N = new (NodeAllocator) MDNodeSDNode(MD);
01768   CSEMap.InsertNode(N, IP);
01769   InsertNode(N);
01770   return SDValue(N, 0);
01771 }
01772 
01773 /// getAddrSpaceCast - Return an AddrSpaceCastSDNode.
01774 SDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr,
01775                                        unsigned SrcAS, unsigned DestAS) {
01776   SDValue Ops[] = {Ptr};
01777   FoldingSetNodeID ID;
01778   AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
01779   ID.AddInteger(SrcAS);
01780   ID.AddInteger(DestAS);
01781 
01782   void *IP = nullptr;
01783   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01784     return SDValue(E, 0);
01785 
01786   SDNode *N = new (NodeAllocator) AddrSpaceCastSDNode(dl.getIROrder(),
01787                                                       dl.getDebugLoc(),
01788                                                       VT, Ptr, SrcAS, DestAS);
01789   CSEMap.InsertNode(N, IP);
01790   InsertNode(N);
01791   return SDValue(N, 0);
01792 }
01793 
01794 /// getShiftAmountOperand - Return the specified value casted to
01795 /// the target's desired shift amount type.
01796 SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
01797   EVT OpTy = Op.getValueType();
01798   EVT ShTy = TLI->getShiftAmountTy(LHSTy);
01799   if (OpTy == ShTy || OpTy.isVector()) return Op;
01800 
01801   ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ?  ISD::TRUNCATE : ISD::ZERO_EXTEND;
01802   return getNode(Opcode, SDLoc(Op), ShTy, Op);
01803 }
01804 
01805 /// CreateStackTemporary - Create a stack temporary, suitable for holding the
01806 /// specified value type.
01807 SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
01808   MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
01809   unsigned ByteSize = VT.getStoreSize();
01810   Type *Ty = VT.getTypeForEVT(*getContext());
01811   unsigned StackAlign =
01812   std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty), minAlign);
01813 
01814   int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false);
01815   return getFrameIndex(FrameIdx, TLI->getPointerTy());
01816 }
01817 
01818 /// CreateStackTemporary - Create a stack temporary suitable for holding
01819 /// either of the specified value types.
01820 SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
01821   unsigned Bytes = std::max(VT1.getStoreSizeInBits(),
01822                             VT2.getStoreSizeInBits())/8;
01823   Type *Ty1 = VT1.getTypeForEVT(*getContext());
01824   Type *Ty2 = VT2.getTypeForEVT(*getContext());
01825   const DataLayout *TD = TLI->getDataLayout();
01826   unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1),
01827                             TD->getPrefTypeAlignment(Ty2));
01828 
01829   MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
01830   int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false);
01831   return getFrameIndex(FrameIdx, TLI->getPointerTy());
01832 }
01833 
01834 SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1,
01835                                 SDValue N2, ISD::CondCode Cond, SDLoc dl) {
01836   // These setcc operations always fold.
01837   switch (Cond) {
01838   default: break;
01839   case ISD::SETFALSE:
01840   case ISD::SETFALSE2: return getConstant(0, VT);
01841   case ISD::SETTRUE:
01842   case ISD::SETTRUE2: {
01843     TargetLowering::BooleanContent Cnt =
01844         TLI->getBooleanContents(N1->getValueType(0));
01845     return getConstant(
01846         Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, VT);
01847   }
01848 
01849   case ISD::SETOEQ:
01850   case ISD::SETOGT:
01851   case ISD::SETOGE:
01852   case ISD::SETOLT:
01853   case ISD::SETOLE:
01854   case ISD::SETONE:
01855   case ISD::SETO:
01856   case ISD::SETUO:
01857   case ISD::SETUEQ:
01858   case ISD::SETUNE:
01859     assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!");
01860     break;
01861   }
01862 
01863   if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) {
01864     const APInt &C2 = N2C->getAPIntValue();
01865     if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
01866       const APInt &C1 = N1C->getAPIntValue();
01867 
01868       switch (Cond) {
01869       default: llvm_unreachable("Unknown integer setcc!");
01870       case ISD::SETEQ:  return getConstant(C1 == C2, VT);
01871       case ISD::SETNE:  return getConstant(C1 != C2, VT);
01872       case ISD::SETULT: return getConstant(C1.ult(C2), VT);
01873       case ISD::SETUGT: return getConstant(C1.ugt(C2), VT);
01874       case ISD::SETULE: return getConstant(C1.ule(C2), VT);
01875       case ISD::SETUGE: return getConstant(C1.uge(C2), VT);
01876       case ISD::SETLT:  return getConstant(C1.slt(C2), VT);
01877       case ISD::SETGT:  return getConstant(C1.sgt(C2), VT);
01878       case ISD::SETLE:  return getConstant(C1.sle(C2), VT);
01879       case ISD::SETGE:  return getConstant(C1.sge(C2), VT);
01880       }
01881     }
01882   }
01883   if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) {
01884     if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) {
01885       APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF());
01886       switch (Cond) {
01887       default: break;
01888       case ISD::SETEQ:  if (R==APFloat::cmpUnordered)
01889                           return getUNDEF(VT);
01890                         // fall through
01891       case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT);
01892       case ISD::SETNE:  if (R==APFloat::cmpUnordered)
01893                           return getUNDEF(VT);
01894                         // fall through
01895       case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan ||
01896                                            R==APFloat::cmpLessThan, VT);
01897       case ISD::SETLT:  if (R==APFloat::cmpUnordered)
01898                           return getUNDEF(VT);
01899                         // fall through
01900       case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT);
01901       case ISD::SETGT:  if (R==APFloat::cmpUnordered)
01902                           return getUNDEF(VT);
01903                         // fall through
01904       case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT);
01905       case ISD::SETLE:  if (R==APFloat::cmpUnordered)
01906                           return getUNDEF(VT);
01907                         // fall through
01908       case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan ||
01909                                            R==APFloat::cmpEqual, VT);
01910       case ISD::SETGE:  if (R==APFloat::cmpUnordered)
01911                           return getUNDEF(VT);
01912                         // fall through
01913       case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan ||
01914                                            R==APFloat::cmpEqual, VT);
01915       case ISD::SETO:   return getConstant(R!=APFloat::cmpUnordered, VT);
01916       case ISD::SETUO:  return getConstant(R==APFloat::cmpUnordered, VT);
01917       case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered ||
01918                                            R==APFloat::cmpEqual, VT);
01919       case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT);
01920       case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered ||
01921                                            R==APFloat::cmpLessThan, VT);
01922       case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan ||
01923                                            R==APFloat::cmpUnordered, VT);
01924       case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT);
01925       case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT);
01926       }
01927     } else {
01928       // Ensure that the constant occurs on the RHS.
01929       ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
01930       MVT CompVT = N1.getValueType().getSimpleVT();
01931       if (!TLI->isCondCodeLegal(SwappedCond, CompVT))
01932         return SDValue();
01933 
01934       return getSetCC(dl, VT, N2, N1, SwappedCond);
01935     }
01936   }
01937 
01938   // Could not fold it.
01939   return SDValue();
01940 }
01941 
01942 /// SignBitIsZero - Return true if the sign bit of Op is known to be zero.  We
01943 /// use this predicate to simplify operations downstream.
01944 bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
01945   // This predicate is not safe for vector operations.
01946   if (Op.getValueType().isVector())
01947     return false;
01948 
01949   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01950   return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth);
01951 }
01952 
01953 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero.  We use
01954 /// this predicate to simplify operations downstream.  Mask is known to be zero
01955 /// for bits that V cannot have.
01956 bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask,
01957                                      unsigned Depth) const {
01958   APInt KnownZero, KnownOne;
01959   computeKnownBits(Op, KnownZero, KnownOne, Depth);
01960   return (KnownZero & Mask) == Mask;
01961 }
01962 
01963 /// Determine which bits of Op are known to be either zero or one and return
01964 /// them in the KnownZero/KnownOne bitsets.
01965 void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
01966                                     APInt &KnownOne, unsigned Depth) const {
01967   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01968 
01969   KnownZero = KnownOne = APInt(BitWidth, 0);   // Don't know anything.
01970   if (Depth == 6)
01971     return;  // Limit search depth.
01972 
01973   APInt KnownZero2, KnownOne2;
01974 
01975   switch (Op.getOpcode()) {
01976   case ISD::Constant:
01977     // We know all of the bits for a constant!
01978     KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue();
01979     KnownZero = ~KnownOne;
01980     break;
01981   case ISD::AND:
01982     // If either the LHS or the RHS are Zero, the result is zero.
01983     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01984     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01985 
01986     // Output known-1 bits are only known if set in both the LHS & RHS.
01987     KnownOne &= KnownOne2;
01988     // Output known-0 are known to be clear if zero in either the LHS | RHS.
01989     KnownZero |= KnownZero2;
01990     break;
01991   case ISD::OR:
01992     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01993     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01994 
01995     // Output known-0 bits are only known if clear in both the LHS & RHS.
01996     KnownZero &= KnownZero2;
01997     // Output known-1 are known to be set if set in either the LHS | RHS.
01998     KnownOne |= KnownOne2;
01999     break;
02000   case ISD::XOR: {
02001     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
02002     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
02003 
02004     // Output known-0 bits are known if clear or set in both the LHS & RHS.
02005     APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2);
02006     // Output known-1 are known to be set if set in only one of the LHS, RHS.
02007     KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2);
02008     KnownZero = KnownZeroOut;
02009     break;
02010   }
02011   case ISD::MUL: {
02012     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
02013     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
02014 
02015     // If low bits are zero in either operand, output low known-0 bits.
02016     // Also compute a conserative estimate for high known-0 bits.
02017     // More trickiness is possible, but this is sufficient for the
02018     // interesting case of alignment computation.
02019     KnownOne.clearAllBits();
02020     unsigned TrailZ = KnownZero.countTrailingOnes() +
02021                       KnownZero2.countTrailingOnes();
02022     unsigned LeadZ =  std::max(KnownZero.countLeadingOnes() +
02023                                KnownZero2.countLeadingOnes(),
02024                                BitWidth) - BitWidth;
02025 
02026     TrailZ = std::min(TrailZ, BitWidth);
02027     LeadZ = std::min(LeadZ, BitWidth);
02028     KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) |
02029                 APInt::getHighBitsSet(BitWidth, LeadZ);
02030     break;
02031   }
02032   case ISD::UDIV: {
02033     // For the purposes of computing leading zeros we can conservatively
02034     // treat a udiv as a logical right shift by the power of 2 known to
02035     // be less than the denominator.
02036     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
02037     unsigned LeadZ = KnownZero2.countLeadingOnes();
02038 
02039     KnownOne2.clearAllBits();
02040     KnownZero2.clearAllBits();
02041     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02042     unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros();
02043     if (RHSUnknownLeadingOnes != BitWidth)
02044       LeadZ = std::min(BitWidth,
02045                        LeadZ + BitWidth - RHSUnknownLeadingOnes - 1);
02046 
02047     KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ);
02048     break;
02049   }
02050   case ISD::SELECT:
02051     computeKnownBits(Op.getOperand(2), KnownZero, KnownOne, Depth+1);
02052     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02053 
02054     // Only known if known in both the LHS and RHS.
02055     KnownOne &= KnownOne2;
02056     KnownZero &= KnownZero2;
02057     break;
02058   case ISD::SELECT_CC:
02059     computeKnownBits(Op.getOperand(3), KnownZero, KnownOne, Depth+1);
02060     computeKnownBits(Op.getOperand(2), KnownZero2, KnownOne2, Depth+1);
02061 
02062     // Only known if known in both the LHS and RHS.
02063     KnownOne &= KnownOne2;
02064     KnownZero &= KnownZero2;
02065     break;
02066   case ISD::SADDO:
02067   case ISD::UADDO:
02068   case ISD::SSUBO:
02069   case ISD::USUBO:
02070   case ISD::SMULO:
02071   case ISD::UMULO:
02072     if (Op.getResNo() != 1)
02073       break;
02074     // The boolean result conforms to getBooleanContents.
02075     // If we know the result of a setcc has the top bits zero, use this info.
02076     // We know that we have an integer-based boolean since these operations
02077     // are only available for integer.
02078     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
02079             TargetLowering::ZeroOrOneBooleanContent &&
02080         BitWidth > 1)
02081       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02082     break;
02083   case ISD::SETCC:
02084     // If we know the result of a setcc has the top bits zero, use this info.
02085     if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
02086             TargetLowering::ZeroOrOneBooleanContent &&
02087         BitWidth > 1)
02088       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02089     break;
02090   case ISD::SHL:
02091     // (shl X, C1) & C2 == 0   iff   (X & C2 >>u C1) == 0
02092     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02093       unsigned ShAmt = SA->getZExtValue();
02094 
02095       // If the shift count is an invalid immediate, don't do anything.
02096       if (ShAmt >= BitWidth)
02097         break;
02098 
02099       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02100       KnownZero <<= ShAmt;
02101       KnownOne  <<= ShAmt;
02102       // low bits known zero.
02103       KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt);
02104     }
02105     break;
02106   case ISD::SRL:
02107     // (ushr X, C1) & C2 == 0   iff  (-1 >> C1) & C2 == 0
02108     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02109       unsigned ShAmt = SA->getZExtValue();
02110 
02111       // If the shift count is an invalid immediate, don't do anything.
02112       if (ShAmt >= BitWidth)
02113         break;
02114 
02115       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02116       KnownZero = KnownZero.lshr(ShAmt);
02117       KnownOne  = KnownOne.lshr(ShAmt);
02118 
02119       APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
02120       KnownZero |= HighBits;  // High bits known zero.
02121     }
02122     break;
02123   case ISD::SRA:
02124     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02125       unsigned ShAmt = SA->getZExtValue();
02126 
02127       // If the shift count is an invalid immediate, don't do anything.
02128       if (ShAmt >= BitWidth)
02129         break;
02130 
02131       // If any of the demanded bits are produced by the sign extension, we also
02132       // demand the input sign bit.
02133       APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
02134 
02135       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02136       KnownZero = KnownZero.lshr(ShAmt);
02137       KnownOne  = KnownOne.lshr(ShAmt);
02138 
02139       // Handle the sign bits.
02140       APInt SignBit = APInt::getSignBit(BitWidth);
02141       SignBit = SignBit.lshr(ShAmt);  // Adjust to where it is now in the mask.
02142 
02143       if (KnownZero.intersects(SignBit)) {
02144         KnownZero |= HighBits;  // New bits are known zero.
02145       } else if (KnownOne.intersects(SignBit)) {
02146         KnownOne  |= HighBits;  // New bits are known one.
02147       }
02148     }
02149     break;
02150   case ISD::SIGN_EXTEND_INREG: {
02151     EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
02152     unsigned EBits = EVT.getScalarType().getSizeInBits();
02153 
02154     // Sign extension.  Compute the demanded bits in the result that are not
02155     // present in the input.
02156     APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits);
02157 
02158     APInt InSignBit = APInt::getSignBit(EBits);
02159     APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits);
02160 
02161     // If the sign extended bits are demanded, we know that the sign
02162     // bit is demanded.
02163     InSignBit = InSignBit.zext(BitWidth);
02164     if (NewBits.getBoolValue())
02165       InputDemandedBits |= InSignBit;
02166 
02167     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02168     KnownOne &= InputDemandedBits;
02169     KnownZero &= InputDemandedBits;
02170 
02171     // If the sign bit of the input is known set or clear, then we know the
02172     // top bits of the result.
02173     if (KnownZero.intersects(InSignBit)) {         // Input sign bit known clear
02174       KnownZero |= NewBits;
02175       KnownOne  &= ~NewBits;
02176     } else if (KnownOne.intersects(InSignBit)) {   // Input sign bit known set
02177       KnownOne  |= NewBits;
02178       KnownZero &= ~NewBits;
02179     } else {                              // Input sign bit unknown
02180       KnownZero &= ~NewBits;
02181       KnownOne  &= ~NewBits;
02182     }
02183     break;
02184   }
02185   case ISD::CTTZ:
02186   case ISD::CTTZ_ZERO_UNDEF:
02187   case ISD::CTLZ:
02188   case ISD::CTLZ_ZERO_UNDEF:
02189   case ISD::CTPOP: {
02190     unsigned LowBits = Log2_32(BitWidth)+1;
02191     KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits);
02192     KnownOne.clearAllBits();
02193     break;
02194   }
02195   case ISD::LOAD: {
02196     LoadSDNode *LD = cast<LoadSDNode>(Op);
02197     // If this is a ZEXTLoad and we are looking at the loaded value.
02198     if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
02199       EVT VT = LD->getMemoryVT();
02200       unsigned MemBits = VT.getScalarType().getSizeInBits();
02201       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
02202     } else if (const MDNode *Ranges = LD->getRanges()) {
02203       computeKnownBitsFromRangeMetadata(*Ranges, KnownZero);
02204     }
02205     break;
02206   }
02207   case ISD::ZERO_EXTEND: {
02208     EVT InVT = Op.getOperand(0).getValueType();
02209     unsigned InBits = InVT.getScalarType().getSizeInBits();
02210     APInt NewBits   = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
02211     KnownZero = KnownZero.trunc(InBits);
02212     KnownOne = KnownOne.trunc(InBits);
02213     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02214     KnownZero = KnownZero.zext(BitWidth);
02215     KnownOne = KnownOne.zext(BitWidth);
02216     KnownZero |= NewBits;
02217     break;
02218   }
02219   case ISD::SIGN_EXTEND: {
02220     EVT InVT = Op.getOperand(0).getValueType();
02221     unsigned InBits = InVT.getScalarType().getSizeInBits();
02222     APInt NewBits   = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
02223 
02224     KnownZero = KnownZero.trunc(InBits);
02225     KnownOne = KnownOne.trunc(InBits);
02226     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02227 
02228     // Note if the sign bit is known to be zero or one.
02229     bool SignBitKnownZero = KnownZero.isNegative();
02230     bool SignBitKnownOne  = KnownOne.isNegative();
02231 
02232     KnownZero = KnownZero.zext(BitWidth);
02233     KnownOne = KnownOne.zext(BitWidth);
02234 
02235     // If the sign bit is known zero or one, the top bits match.
02236     if (SignBitKnownZero)
02237       KnownZero |= NewBits;
02238     else if (SignBitKnownOne)
02239       KnownOne  |= NewBits;
02240     break;
02241   }
02242   case ISD::ANY_EXTEND: {
02243     EVT InVT = Op.getOperand(0).getValueType();
02244     unsigned InBits = InVT.getScalarType().getSizeInBits();
02245     KnownZero = KnownZero.trunc(InBits);
02246     KnownOne = KnownOne.trunc(InBits);
02247     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02248     KnownZero = KnownZero.zext(BitWidth);
02249     KnownOne = KnownOne.zext(BitWidth);
02250     break;
02251   }
02252   case ISD::TRUNCATE: {
02253     EVT InVT = Op.getOperand(0).getValueType();
02254     unsigned InBits = InVT.getScalarType().getSizeInBits();
02255     KnownZero = KnownZero.zext(InBits);
02256     KnownOne = KnownOne.zext(InBits);
02257     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02258     KnownZero = KnownZero.trunc(BitWidth);
02259     KnownOne = KnownOne.trunc(BitWidth);
02260     break;
02261   }
02262   case ISD::AssertZext: {
02263     EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
02264     APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
02265     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02266     KnownZero |= (~InMask);
02267     KnownOne  &= (~KnownZero);
02268     break;
02269   }
02270   case ISD::FGETSIGN:
02271     // All bits are zero except the low bit.
02272     KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02273     break;
02274 
02275   case ISD::SUB: {
02276     if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) {
02277       // We know that the top bits of C-X are clear if X contains less bits
02278       // than C (i.e. no wrap-around can happen).  For example, 20-X is
02279       // positive if we can prove that X is >= 0 and < 16.
02280       if (CLHS->getAPIntValue().isNonNegative()) {
02281         unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros();
02282         // NLZ can't be BitWidth with no sign bit
02283         APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1);
02284         computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02285 
02286         // If all of the MaskV bits are known to be zero, then we know the
02287         // output top bits are zero, because we now know that the output is
02288         // from [0-C].
02289         if ((KnownZero2 & MaskV) == MaskV) {
02290           unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros();
02291           // Top bits known zero.
02292           KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2);
02293         }
02294       }
02295     }
02296   }
02297   // fall through
02298   case ISD::ADD:
02299   case ISD::ADDE: {
02300     // Output known-0 bits are known if clear or set in both the low clear bits
02301     // common to both LHS & RHS.  For example, 8+(X<<3) is known to have the
02302     // low 3 bits clear.
02303     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
02304     unsigned KnownZeroOut = KnownZero2.countTrailingOnes();
02305 
02306     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02307     KnownZeroOut = std::min(KnownZeroOut,
02308                             KnownZero2.countTrailingOnes());
02309 
02310     if (Op.getOpcode() == ISD::ADD) {
02311       KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut);
02312       break;
02313     }
02314 
02315     // With ADDE, a carry bit may be added in, so we can only use this
02316     // information if we know (at least) that the low two bits are clear.  We
02317     // then return to the caller that the low bit is unknown but that other bits
02318     // are known zero.
02319     if (KnownZeroOut >= 2) // ADDE
02320       KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut);
02321     break;
02322   }
02323   case ISD::SREM:
02324     if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02325       const APInt &RA = Rem->getAPIntValue().abs();
02326       if (RA.isPowerOf2()) {
02327         APInt LowBits = RA - 1;
02328         computeKnownBits(Op.getOperand(0), KnownZero2,KnownOne2,Depth+1);
02329 
02330         // The low bits of the first operand are unchanged by the srem.
02331         KnownZero = KnownZero2 & LowBits;
02332         KnownOne = KnownOne2 & LowBits;
02333 
02334         // If the first operand is non-negative or has all low bits zero, then
02335         // the upper bits are all zero.
02336         if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits))
02337           KnownZero |= ~LowBits;
02338 
02339         // If the first operand is negative and not all low bits are zero, then
02340         // the upper bits are all one.
02341         if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0))
02342           KnownOne |= ~LowBits;
02343         assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?");
02344       }
02345     }
02346     break;
02347   case ISD::UREM: {
02348     if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02349       const APInt &RA = Rem->getAPIntValue();
02350       if (RA.isPowerOf2()) {
02351         APInt LowBits = (RA - 1);
02352         computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth + 1);
02353 
02354         // The upper bits are all zero, the lower ones are unchanged.
02355         KnownZero = KnownZero2 | ~LowBits;
02356         KnownOne = KnownOne2 & LowBits;
02357         break;
02358       }
02359     }
02360 
02361     // Since the result is less than or equal to either operand, any leading
02362     // zero bits in either operand must also exist in the result.
02363     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02364     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02365 
02366     uint32_t Leaders = std::max(KnownZero.countLeadingOnes(),
02367                                 KnownZero2.countLeadingOnes());
02368     KnownOne.clearAllBits();
02369     KnownZero = APInt::getHighBitsSet(BitWidth, Leaders);
02370     break;
02371   }
02372   case ISD::EXTRACT_ELEMENT: {
02373     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02374     const unsigned Index =
02375       cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
02376     const unsigned BitWidth = Op.getValueType().getSizeInBits();
02377 
02378     // Remove low part of known bits mask
02379     KnownZero = KnownZero.getHiBits(KnownZero.getBitWidth() - Index * BitWidth);
02380     KnownOne = KnownOne.getHiBits(KnownOne.getBitWidth() - Index * BitWidth);
02381 
02382     // Remove high part of known bit mask
02383     KnownZero = KnownZero.trunc(BitWidth);
02384     KnownOne = KnownOne.trunc(BitWidth);
02385     break;
02386   }
02387   case ISD::FrameIndex:
02388   case ISD::TargetFrameIndex:
02389     if (unsigned Align = InferPtrAlignment(Op)) {
02390       // The low bits are known zero if the pointer is aligned.
02391       KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align));
02392       break;
02393     }
02394     break;
02395 
02396   default:
02397     if (Op.getOpcode() < ISD::BUILTIN_OP_END)
02398       break;
02399     // Fallthrough
02400   case ISD::INTRINSIC_WO_CHAIN:
02401   case ISD::INTRINSIC_W_CHAIN:
02402   case ISD::INTRINSIC_VOID:
02403     // Allow the target to implement this method for its nodes.
02404     TLI->computeKnownBitsForTargetNode(Op, KnownZero, KnownOne, *this, Depth);
02405     break;
02406   }
02407 
02408   assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
02409 }
02410 
02411 /// ComputeNumSignBits - Return the number of times the sign bit of the
02412 /// register is replicated into the other bits.  We know that at least 1 bit
02413 /// is always equal to the sign bit (itself), but other cases can give us
02414 /// information.  For example, immediately after an "SRA X, 2", we know that
02415 /// the top 3 bits are all equal to each other, so we return 3.
02416 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{
02417   EVT VT = Op.getValueType();
02418   assert(VT.isInteger() && "Invalid VT!");
02419   unsigned VTBits = VT.getScalarType().getSizeInBits();
02420   unsigned Tmp, Tmp2;
02421   unsigned FirstAnswer = 1;
02422 
02423   if (Depth == 6)
02424     return 1;  // Limit search depth.
02425 
02426   switch (Op.getOpcode()) {
02427   default: break;
02428   case ISD::AssertSext:
02429     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
02430     return VTBits-Tmp+1;
02431   case ISD::AssertZext:
02432     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
02433     return VTBits-Tmp;
02434 
02435   case ISD::Constant: {
02436     const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue();
02437     return Val.getNumSignBits();
02438   }
02439 
02440   case ISD::SIGN_EXTEND:
02441     Tmp =
02442         VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits();
02443     return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp;
02444 
02445   case ISD::SIGN_EXTEND_INREG:
02446     // Max of the input and what this extends.
02447     Tmp =
02448       cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits();
02449     Tmp = VTBits-Tmp+1;
02450 
02451     Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02452     return std::max(Tmp, Tmp2);
02453 
02454   case ISD::SRA:
02455     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02456     // SRA X, C   -> adds C sign bits.
02457     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02458       Tmp += C->getZExtValue();
02459       if (Tmp > VTBits) Tmp = VTBits;
02460     }
02461     return Tmp;
02462   case ISD::SHL:
02463     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02464       // shl destroys sign bits.
02465       Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02466       if (C->getZExtValue() >= VTBits ||      // Bad shift.
02467           C->getZExtValue() >= Tmp) break;    // Shifted all sign bits out.
02468       return Tmp - C->getZExtValue();
02469     }
02470     break;
02471   case ISD::AND:
02472   case ISD::OR:
02473   case ISD::XOR:    // NOT is handled here.
02474     // Logical binary ops preserve the number of sign bits at the worst.
02475     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02476     if (Tmp != 1) {
02477       Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02478       FirstAnswer = std::min(Tmp, Tmp2);
02479       // We computed what we know about the sign bits as our first
02480       // answer. Now proceed to the generic code that uses
02481       // computeKnownBits, and pick whichever answer is better.
02482     }
02483     break;
02484 
02485   case ISD::SELECT:
02486     Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02487     if (Tmp == 1) return 1;  // Early out.
02488     Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1);
02489     return std::min(Tmp, Tmp2);
02490 
02491   case ISD::SADDO:
02492   case ISD::UADDO:
02493   case ISD::SSUBO:
02494   case ISD::USUBO:
02495   case ISD::SMULO:
02496   case ISD::UMULO:
02497     if (Op.getResNo() != 1)
02498       break;
02499     // The boolean result conforms to getBooleanContents.  Fall through.
02500     // If setcc returns 0/-1, all bits are sign bits.
02501     // We know that we have an integer-based boolean since these operations
02502     // are only available for integer.
02503     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
02504         TargetLowering::ZeroOrNegativeOneBooleanContent)
02505       return VTBits;
02506     break;
02507   case ISD::SETCC:
02508     // If setcc returns 0/-1, all bits are sign bits.
02509     if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
02510         TargetLowering::ZeroOrNegativeOneBooleanContent)
02511       return VTBits;
02512     break;
02513   case ISD::ROTL:
02514   case ISD::ROTR:
02515     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02516       unsigned RotAmt = C->getZExtValue() & (VTBits-1);
02517 
02518       // Handle rotate right by N like a rotate left by 32-N.
02519       if (Op.getOpcode() == ISD::ROTR)
02520         RotAmt = (VTBits-RotAmt) & (VTBits-1);
02521 
02522       // If we aren't rotating out all of the known-in sign bits, return the
02523       // number that are left.  This handles rotl(sext(x), 1) for example.
02524       Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02525       if (Tmp > RotAmt+1) return Tmp-RotAmt;
02526     }
02527     break;
02528   case ISD::ADD:
02529     // Add can have at most one carry bit.  Thus we know that the output
02530     // is, at worst, one more bit than the inputs.
02531     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02532     if (Tmp == 1) return 1;  // Early out.
02533 
02534     // Special case decrementing a value (ADD X, -1):
02535     if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
02536       if (CRHS->isAllOnesValue()) {
02537         APInt KnownZero, KnownOne;
02538         computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02539 
02540         // If the input is known to be 0 or 1, the output is 0/-1, which is all
02541         // sign bits set.
02542         if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
02543           return VTBits;
02544 
02545         // If we are subtracting one from a positive number, there is no carry
02546         // out of the result.
02547         if (KnownZero.isNegative())
02548           return Tmp;
02549       }
02550 
02551     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02552     if (Tmp2 == 1) return 1;
02553     return std::min(Tmp, Tmp2)-1;
02554 
02555   case ISD::SUB:
02556     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02557     if (Tmp2 == 1) return 1;
02558 
02559     // Handle NEG.
02560     if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0)))
02561       if (CLHS->isNullValue()) {
02562         APInt KnownZero, KnownOne;
02563         computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
02564         // If the input is known to be 0 or 1, the output is 0/-1, which is all
02565         // sign bits set.
02566         if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
02567           return VTBits;
02568 
02569         // If the input is known to be positive (the sign bit is known clear),
02570         // the output of the NEG has the same number of sign bits as the input.
02571         if (KnownZero.isNegative())
02572           return Tmp2;
02573 
02574         // Otherwise, we treat this like a SUB.
02575       }
02576 
02577     // Sub can have at most one carry bit.  Thus we know that the output
02578     // is, at worst, one more bit than the inputs.
02579     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02580     if (Tmp == 1) return 1;  // Early out.
02581     return std::min(Tmp, Tmp2)-1;
02582   case ISD::TRUNCATE:
02583     // FIXME: it's tricky to do anything useful for this, but it is an important
02584     // case for targets like X86.
02585     break;
02586   case ISD::EXTRACT_ELEMENT: {
02587     const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02588     const int BitWidth = Op.getValueType().getSizeInBits();
02589     const int Items =
02590       Op.getOperand(0).getValueType().getSizeInBits() / BitWidth;
02591 
02592     // Get reverse index (starting from 1), Op1 value indexes elements from
02593     // little end. Sign starts at big end.
02594     const int rIndex = Items - 1 -
02595       cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
02596 
02597     // If the sign portion ends in our element the substraction gives correct
02598     // result. Otherwise it gives either negative or > bitwidth result
02599     return std::max(std::min(KnownSign - rIndex * BitWidth, BitWidth), 0);
02600   }
02601   }
02602 
02603   // If we are looking at the loaded value of the SDNode.
02604   if (Op.getResNo() == 0) {
02605     // Handle LOADX separately here. EXTLOAD case will fallthrough.
02606     if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
02607       unsigned ExtType = LD->getExtensionType();
02608       switch (ExtType) {
02609         default: break;
02610         case ISD::SEXTLOAD:    // '17' bits known
02611           Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
02612           return VTBits-Tmp+1;
02613         case ISD::ZEXTLOAD:    // '16' bits known
02614           Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
02615           return VTBits-Tmp;
02616       }
02617     }
02618   }
02619 
02620   // Allow the target to implement this method for its nodes.
02621   if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
02622       Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
02623       Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
02624       Op.getOpcode() == ISD::INTRINSIC_VOID) {
02625     unsigned NumBits = TLI->ComputeNumSignBitsForTargetNode(Op, *this, Depth);
02626     if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits);
02627   }
02628 
02629   // Finally, if we can prove that the top bits of the result are 0's or 1's,
02630   // use this information.
02631   APInt KnownZero, KnownOne;
02632   computeKnownBits(Op, KnownZero, KnownOne, Depth);
02633 
02634   APInt Mask;
02635   if (KnownZero.isNegative()) {        // sign bit is 0
02636     Mask = KnownZero;
02637   } else if (KnownOne.isNegative()) {  // sign bit is 1;
02638     Mask = KnownOne;
02639   } else {
02640     // Nothing known.
02641     return FirstAnswer;
02642   }
02643 
02644   // Okay, we know that the sign bit in Mask is set.  Use CLZ to determine
02645   // the number of identical bits in the top of the input value.
02646   Mask = ~Mask;
02647   Mask <<= Mask.getBitWidth()-VTBits;
02648   // Return # leading zeros.  We use 'min' here in case Val was zero before
02649   // shifting.  We don't want to return '64' as for an i32 "0".
02650   return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros()));
02651 }
02652 
02653 /// isBaseWithConstantOffset - Return true if the specified operand is an
02654 /// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an
02655 /// ISD::OR with a ConstantSDNode that is guaranteed to have the same
02656 /// semantics as an ADD.  This handles the equivalence:
02657 ///     X|Cst == X+Cst iff X&Cst = 0.
02658 bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
02659   if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
02660       !isa<ConstantSDNode>(Op.getOperand(1)))
02661     return false;
02662 
02663   if (Op.getOpcode() == ISD::OR &&
02664       !MaskedValueIsZero(Op.getOperand(0),
02665                      cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue()))
02666     return false;
02667 
02668   return true;
02669 }
02670 
02671 
02672 bool SelectionDAG::isKnownNeverNaN(SDValue Op) const {
02673   // If we're told that NaNs won't happen, assume they won't.
02674   if (getTarget().Options.NoNaNsFPMath)
02675     return true;
02676 
02677   // If the value is a constant, we can obviously see if it is a NaN or not.
02678   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
02679     return !C->getValueAPF().isNaN();
02680 
02681   // TODO: Recognize more cases here.
02682 
02683   return false;
02684 }
02685 
02686 bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
02687   // If the value is a constant, we can obviously see if it is a zero or not.
02688   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
02689     return !C->isZero();
02690 
02691   // TODO: Recognize more cases here.
02692   switch (Op.getOpcode()) {
02693   default: break;
02694   case ISD::OR:
02695     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
02696       return !C->isNullValue();
02697     break;
02698   }
02699 
02700   return false;
02701 }
02702 
02703 bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
02704   // Check the obvious case.
02705   if (A == B) return true;
02706 
02707   // For for negative and positive zero.
02708   if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
02709     if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
02710       if (CA->isZero() && CB->isZero()) return true;
02711 
02712   // Otherwise they may not be equal.
02713   return false;
02714 }
02715 
02716 /// getNode - Gets or creates the specified node.
02717 ///
02718 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT) {
02719   FoldingSetNodeID ID;
02720   AddNodeIDNode(ID, Opcode, getVTList(VT), None);
02721   void *IP = nullptr;
02722   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
02723     return SDValue(E, 0);
02724 
02725   SDNode *N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(),
02726                                          DL.getDebugLoc(), getVTList(VT));
02727   CSEMap.InsertNode(N, IP);
02728 
02729   InsertNode(N);
02730   return SDValue(N, 0);
02731 }
02732 
02733 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
02734                               EVT VT, SDValue Operand) {
02735   // Constant fold unary operations with an integer constant operand. Even
02736   // opaque constant will be folded, because the folding of unary operations
02737   // doesn't create new constants with different values. Nevertheless, the
02738   // opaque flag is preserved during folding to prevent future folding with
02739   // other constants.
02740   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) {
02741     const APInt &Val = C->getAPIntValue();
02742     switch (Opcode) {
02743     default: break;
02744     case ISD::SIGN_EXTEND:
02745       return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT,
02746                          C->isTargetOpcode(), C->isOpaque());
02747     case ISD::ANY_EXTEND:
02748     case ISD::ZERO_EXTEND:
02749     case ISD::TRUNCATE:
02750       return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT,
02751                          C->isTargetOpcode(), C->isOpaque());
02752     case ISD::UINT_TO_FP:
02753     case ISD::SINT_TO_FP: {
02754       APFloat apf(EVTToAPFloatSemantics(VT),
02755                   APInt::getNullValue(VT.getSizeInBits()));
02756       (void)apf.convertFromAPInt(Val,
02757                                  Opcode==ISD::SINT_TO_FP,
02758                                  APFloat::rmNearestTiesToEven);
02759       return getConstantFP(apf, VT);
02760     }
02761     case ISD::BITCAST:
02762       if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
02763         return getConstantFP(APFloat(APFloat::IEEEhalf, Val), VT);
02764       if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
02765         return getConstantFP(APFloat(APFloat::IEEEsingle, Val), VT);
02766       else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
02767         return getConstantFP(APFloat(APFloat::IEEEdouble, Val), VT);
02768       break;
02769     case ISD::BSWAP:
02770       return getConstant(Val.byteSwap(), VT, C->isTargetOpcode(),
02771                          C->isOpaque());
02772     case ISD::CTPOP:
02773       return getConstant(Val.countPopulation(), VT, C->isTargetOpcode(),
02774                          C->isOpaque());
02775     case ISD::CTLZ:
02776     case ISD::CTLZ_ZERO_UNDEF:
02777       return getConstant(Val.countLeadingZeros(), VT, C->isTargetOpcode(),
02778                          C->isOpaque());
02779     case ISD::CTTZ:
02780     case ISD::CTTZ_ZERO_UNDEF:
02781       return getConstant(Val.countTrailingZeros(), VT, C->isTargetOpcode(),
02782                          C->isOpaque());
02783     }
02784   }
02785 
02786   // Constant fold unary operations with a floating point constant operand.
02787   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) {
02788     APFloat V = C->getValueAPF();    // make copy
02789     switch (Opcode) {
02790     case ISD::FNEG:
02791       V.changeSign();
02792       return getConstantFP(V, VT);
02793     case ISD::FABS:
02794       V.clearSign();
02795       return getConstantFP(V, VT);
02796     case ISD::FCEIL: {
02797       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
02798       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02799         return getConstantFP(V, VT);
02800       break;
02801     }
02802     case ISD::FTRUNC: {
02803       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
02804       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02805         return getConstantFP(V, VT);
02806       break;
02807     }
02808     case ISD::FFLOOR: {
02809       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
02810       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02811         return getConstantFP(V, VT);
02812       break;
02813     }
02814     case ISD::FP_EXTEND: {
02815       bool ignored;
02816       // This can return overflow, underflow, or inexact; we don't care.
02817       // FIXME need to be more flexible about rounding mode.
02818       (void)V.convert(EVTToAPFloatSemantics(VT),
02819                       APFloat::rmNearestTiesToEven, &ignored);
02820       return getConstantFP(V, VT);
02821     }
02822     case ISD::FP_TO_SINT:
02823     case ISD::FP_TO_UINT: {
02824       integerPart x[2];
02825       bool ignored;
02826       static_assert(integerPartWidth >= 64, "APFloat parts too small!");
02827       // FIXME need to be more flexible about rounding mode.
02828       APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(),
02829                             Opcode==ISD::FP_TO_SINT,
02830                             APFloat::rmTowardZero, &ignored);
02831       if (s==APFloat::opInvalidOp)     // inexact is OK, in fact usual
02832         break;
02833       APInt api(VT.getSizeInBits(), x);
02834       return getConstant(api, VT);
02835     }
02836     case ISD::BITCAST:
02837       if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
02838         return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), VT);
02839       else if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
02840         return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT);
02841       else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
02842         return getConstant(V.bitcastToAPInt().getZExtValue(), VT);
02843       break;
02844     }
02845   }
02846 
02847   // Constant fold unary operations with a vector integer operand.
02848   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand.getNode())) {
02849     if (BV->isConstant()) {
02850       switch (Opcode) {
02851       default:
02852         // FIXME: Entirely reasonable to perform folding of other unary
02853         // operations here as the need arises.
02854         break;
02855       case ISD::UINT_TO_FP:
02856       case ISD::SINT_TO_FP: {
02857         SmallVector<SDValue, 8> Ops;
02858         for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
02859           SDValue OpN = BV->getOperand(i);
02860           // Let the above scalar folding handle the conversion of each
02861           // element.
02862           OpN = getNode(ISD::SINT_TO_FP, DL, VT.getVectorElementType(),
02863                         OpN);
02864           Ops.push_back(OpN);
02865         }
02866         return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
02867       }
02868       }
02869     }
02870   }
02871 
02872   unsigned OpOpcode = Operand.getNode()->getOpcode();
02873   switch (Opcode) {
02874   case ISD::TokenFactor:
02875   case ISD::MERGE_VALUES:
02876   case ISD::CONCAT_VECTORS:
02877     return Operand;         // Factor, merge or concat of one node?  No need.
02878   case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
02879   case ISD::FP_EXTEND:
02880     assert(VT.isFloatingPoint() &&
02881            Operand.getValueType().isFloatingPoint() && "Invalid FP cast!");
02882     if (Operand.getValueType() == VT) return Operand;  // noop conversion.
02883     assert((!VT.isVector() ||
02884             VT.getVectorNumElements() ==
02885             Operand.getValueType().getVectorNumElements()) &&
02886            "Vector element count mismatch!");
02887     if (Operand.getOpcode() == ISD::UNDEF)
02888       return getUNDEF(VT);
02889     break;
02890   case ISD::SIGN_EXTEND:
02891     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02892            "Invalid SIGN_EXTEND!");
02893     if (Operand.getValueType() == VT) return Operand;   // noop extension
02894     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02895            "Invalid sext node, dst < src!");
02896     assert((!VT.isVector() ||
02897             VT.getVectorNumElements() ==
02898             Operand.getValueType().getVectorNumElements()) &&
02899            "Vector element count mismatch!");
02900     if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
02901       return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02902     else if (OpOpcode == ISD::UNDEF)
02903       // sext(undef) = 0, because the top bits will all be the same.
02904       return getConstant(0, VT);
02905     break;
02906   case ISD::ZERO_EXTEND:
02907     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02908            "Invalid ZERO_EXTEND!");
02909     if (Operand.getValueType() == VT) return Operand;   // noop extension
02910     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02911            "Invalid zext node, dst < src!");
02912     assert((!VT.isVector() ||
02913             VT.getVectorNumElements() ==
02914             Operand.getValueType().getVectorNumElements()) &&
02915            "Vector element count mismatch!");
02916     if (OpOpcode == ISD::ZERO_EXTEND)   // (zext (zext x)) -> (zext x)
02917       return getNode(ISD::ZERO_EXTEND, DL, VT,
02918                      Operand.getNode()->getOperand(0));
02919     else if (OpOpcode == ISD::UNDEF)
02920       // zext(undef) = 0, because the top bits will be zero.
02921       return getConstant(0, VT);
02922     break;
02923   case ISD::ANY_EXTEND:
02924     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02925            "Invalid ANY_EXTEND!");
02926     if (Operand.getValueType() == VT) return Operand;   // noop extension
02927     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02928            "Invalid anyext node, dst < src!");
02929     assert((!VT.isVector() ||
02930             VT.getVectorNumElements() ==
02931             Operand.getValueType().getVectorNumElements()) &&
02932            "Vector element count mismatch!");
02933 
02934     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
02935         OpOpcode == ISD::ANY_EXTEND)
02936       // (ext (zext x)) -> (zext x)  and  (ext (sext x)) -> (sext x)
02937       return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02938     else if (OpOpcode == ISD::UNDEF)
02939       return getUNDEF(VT);
02940 
02941     // (ext (trunx x)) -> x
02942     if (OpOpcode == ISD::TRUNCATE) {
02943       SDValue OpOp = Operand.getNode()->getOperand(0);
02944       if (OpOp.getValueType() == VT)
02945         return OpOp;
02946     }
02947     break;
02948   case ISD::TRUNCATE:
02949     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02950            "Invalid TRUNCATE!");
02951     if (Operand.getValueType() == VT) return Operand;   // noop truncate
02952     assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) &&
02953            "Invalid truncate node, src < dst!");
02954     assert((!VT.isVector() ||
02955             VT.getVectorNumElements() ==
02956             Operand.getValueType().getVectorNumElements()) &&
02957            "Vector element count mismatch!");
02958     if (OpOpcode == ISD::TRUNCATE)
02959       return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
02960     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
02961         OpOpcode == ISD::ANY_EXTEND) {
02962       // If the source is smaller than the dest, we still need an extend.
02963       if (Operand.getNode()->getOperand(0).getValueType().getScalarType()
02964             .bitsLT(VT.getScalarType()))
02965         return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02966       if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT))
02967         return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
02968       return Operand.getNode()->getOperand(0);
02969     }
02970     if (OpOpcode == ISD::UNDEF)
02971       return getUNDEF(VT);
02972     break;
02973   case ISD::BITCAST:
02974     // Basic sanity checking.
02975     assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits()
02976            && "Cannot BITCAST between types of different sizes!");
02977     if (VT == Operand.getValueType()) return Operand;  // noop conversion.
02978     if (OpOpcode == ISD::BITCAST)  // bitconv(bitconv(x)) -> bitconv(x)
02979       return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
02980     if (OpOpcode == ISD::UNDEF)
02981       return getUNDEF(VT);
02982     break;
02983   case ISD::SCALAR_TO_VECTOR:
02984     assert(VT.isVector() && !Operand.getValueType().isVector() &&
02985            (VT.getVectorElementType() == Operand.getValueType() ||
02986             (VT.getVectorElementType().isInteger() &&
02987              Operand.getValueType().isInteger() &&
02988              VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&
02989            "Illegal SCALAR_TO_VECTOR node!");
02990     if (OpOpcode == ISD::UNDEF)
02991       return getUNDEF(VT);
02992     // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
02993     if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
02994         isa<ConstantSDNode>(Operand.getOperand(1)) &&
02995         Operand.getConstantOperandVal(1) == 0 &&
02996         Operand.getOperand(0).getValueType() == VT)
02997       return Operand.getOperand(0);
02998     break;
02999   case ISD::FNEG:
03000     // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0
03001     if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB)
03002       return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1),
03003                      Operand.getNode()->getOperand(0));
03004     if (OpOpcode == ISD::FNEG)  // --X -> X
03005       return Operand.getNode()->getOperand(0);
03006     break;
03007   case ISD::FABS:
03008     if (OpOpcode == ISD::FNEG)  // abs(-X) -> abs(X)
03009       return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0));
03010     break;
03011   }
03012 
03013   SDNode *N;
03014   SDVTList VTs = getVTList(VT);
03015   if (VT != MVT::Glue) { // Don't CSE flag producing nodes
03016     FoldingSetNodeID ID;
03017     SDValue Ops[1] = { Operand };
03018     AddNodeIDNode(ID, Opcode, VTs, Ops);
03019     void *IP = nullptr;
03020     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
03021       return SDValue(E, 0);
03022 
03023     N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
03024                                         DL.getDebugLoc(), VTs, Operand);
03025     CSEMap.InsertNode(N, IP);
03026   } else {
03027     N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
03028                                         DL.getDebugLoc(), VTs, Operand);
03029   }
03030 
03031   InsertNode(N);
03032   return SDValue(N, 0);
03033 }
03034 
03035 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT,
03036                                              SDNode *Cst1, SDNode *Cst2) {
03037   // If the opcode is a target-specific ISD node, there's nothing we can
03038   // do here and the operand rules may not line up with the below, so
03039   // bail early.
03040   if (Opcode >= ISD::BUILTIN_OP_END)
03041     return SDValue();
03042 
03043   SmallVector<std::pair<ConstantSDNode *, ConstantSDNode *>, 4> Inputs;
03044   SmallVector<SDValue, 4> Outputs;
03045   EVT SVT = VT.getScalarType();
03046 
03047   ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1);
03048   ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2);
03049   if (Scalar1 && Scalar2 && (Scalar1->isOpaque() || Scalar2->isOpaque()))
03050     return SDValue();
03051 
03052   if (Scalar1 && Scalar2)
03053     // Scalar instruction.
03054     Inputs.push_back(std::make_pair(Scalar1, Scalar2));
03055   else {
03056     // For vectors extract each constant element into Inputs so we can constant
03057     // fold them individually.
03058     BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1);
03059     BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2);
03060     if (!BV1 || !BV2)
03061       return SDValue();
03062 
03063     assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!");
03064 
03065     for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) {
03066       ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I));
03067       ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I));
03068       if (!V1 || !V2) // Not a constant, bail.
03069         return SDValue();
03070 
03071       if (V1->isOpaque() || V2->isOpaque())
03072         return SDValue();
03073 
03074       // Avoid BUILD_VECTOR nodes that perform implicit truncation.
03075       // FIXME: This is valid and could be handled by truncating the APInts.
03076       if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
03077         return SDValue();
03078 
03079       Inputs.push_back(std::make_pair(V1, V2));
03080     }
03081   }
03082 
03083   // We have a number of constant values, constant fold them element by element.
03084   for (unsigned I = 0, E = Inputs.size(); I != E; ++I) {
03085     const APInt &C1 = Inputs[I].first->getAPIntValue();
03086     const APInt &C2 = Inputs[I].second->getAPIntValue();
03087 
03088     switch (Opcode) {
03089     case ISD::ADD:
03090       Outputs.push_back(getConstant(C1 + C2, SVT));
03091       break;
03092     case ISD::SUB:
03093       Outputs.push_back(getConstant(C1 - C2, SVT));
03094       break;
03095     case ISD::MUL:
03096       Outputs.push_back(getConstant(C1 * C2, SVT));
03097       break;
03098     case ISD::UDIV:
03099       if (!C2.getBoolValue())
03100         return SDValue();
03101       Outputs.push_back(getConstant(C1.udiv(C2), SVT));
03102       break;
03103     case ISD::UREM:
03104       if (!C2.getBoolValue())
03105         return SDValue();
03106       Outputs.push_back(getConstant(C1.urem(C2), SVT));
03107       break;
03108     case ISD::SDIV:
03109       if (!C2.getBoolValue())
03110         return SDValue();
03111       Outputs.push_back(getConstant(C1.sdiv(C2), SVT));
03112       break;
03113     case ISD::SREM:
03114       if (!C2.getBoolValue())
03115         return SDValue();
03116       Outputs.push_back(getConstant(C1.srem(C2), SVT));
03117       break;
03118     case ISD::AND:
03119       Outputs.push_back(getConstant(C1 & C2, SVT));
03120       break;
03121     case ISD::OR:
03122       Outputs.push_back(getConstant(C1 | C2, SVT));
03123       break;
03124     case ISD::XOR:
03125       Outputs.push_back(getConstant(C1 ^ C2, SVT));
03126       break;
03127     case ISD::SHL:
03128       Outputs.push_back(getConstant(C1 << C2, SVT));
03129       break;
03130     case ISD::SRL:
03131       Outputs.push_back(getConstant(C1.lshr(C2), SVT));
03132       break;
03133     case ISD::SRA:
03134       Outputs.push_back(getConstant(C1.ashr(C2), SVT));
03135       break;
03136     case ISD::ROTL:
03137       Outputs.push_back(getConstant(C1.rotl(C2), SVT));
03138       break;
03139     case ISD::ROTR:
03140       Outputs.push_back(getConstant(C1.rotr(C2), SVT));
03141       break;
03142     default:
03143       return SDValue();
03144     }
03145   }
03146 
03147   assert((Scalar1 && Scalar2) || (VT.getVectorNumElements() == Outputs.size() &&
03148                                   "Expected a scalar or vector!"));
03149 
03150   // Handle the scalar case first.
03151   if (!VT.isVector())
03152     return Outputs.back();
03153 
03154   // We may have a vector type but a scalar result. Create a splat.
03155   Outputs.resize(VT.getVectorNumElements(), Outputs.back());
03156 
03157   // Build a big vector out of the scalar elements we generated.
03158   return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs);
03159 }
03160 
03161 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
03162                               SDValue N2, bool nuw, bool nsw, bool exact) {
03163   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
03164   ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode());
03165   switch (Opcode) {
03166   default: break;
03167   case ISD::TokenFactor:
03168     assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
03169            N2.getValueType() == MVT::Other && "Invalid token factor!");
03170     // Fold trivial token factors.
03171     if (N1.getOpcode() == ISD::EntryToken) return N2;
03172     if (N2.getOpcode() == ISD::EntryToken) return N1;
03173     if (N1 == N2) return N1;
03174     break;
03175   case ISD::CONCAT_VECTORS:
03176     // Concat of UNDEFs is UNDEF.
03177     if (N1.getOpcode() == ISD::UNDEF &&
03178         N2.getOpcode() == ISD::UNDEF)
03179       return getUNDEF(VT);
03180 
03181     // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
03182     // one big BUILD_VECTOR.
03183     if (N1.getOpcode() == ISD::BUILD_VECTOR &&
03184         N2.getOpcode() == ISD::BUILD_VECTOR) {
03185       SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
03186                                     N1.getNode()->op_end());
03187       Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
03188       return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
03189     }
03190     break;
03191   case ISD::AND:
03192     assert(VT.isInteger() && "This operator does not apply to FP types!");
03193     assert(N1.getValueType() == N2.getValueType() &&
03194            N1.getValueType() == VT && "Binary operator types must match!");
03195     // (X & 0) -> 0.  This commonly occurs when legalizing i64 values, so it's
03196     // worth handling here.
03197     if (N2C && N2C->isNullValue())
03198       return N2;
03199     if (N2C && N2C->isAllOnesValue())  // X & -1 -> X
03200       return N1;
03201     break;
03202   case ISD::OR:
03203   case ISD::XOR:
03204   case ISD::ADD:
03205   case ISD::SUB:
03206     assert(VT.isInteger() && "This operator does not apply to FP types!");
03207     assert(N1.getValueType() == N2.getValueType() &&
03208            N1.getValueType() == VT && "Binary operator types must match!");
03209     // (X ^|+- 0) -> X.  This commonly occurs when legalizing i64 values, so
03210     // it's worth handling here.
03211     if (N2C && N2C->isNullValue())
03212       return N1;
03213     break;
03214   case ISD::UDIV:
03215   case ISD::UREM:
03216   case ISD::MULHU:
03217   case ISD::MULHS:
03218   case ISD::MUL:
03219   case ISD::SDIV:
03220   case ISD::SREM:
03221     assert(VT.isInteger() && "This operator does not apply to FP types!");
03222     assert(N1.getValueType() == N2.getValueType() &&
03223            N1.getValueType() == VT && "Binary operator types must match!");
03224     break;
03225   case ISD::FADD:
03226   case ISD::FSUB:
03227   case ISD::FMUL:
03228   case ISD::FDIV:
03229   case ISD::FREM:
03230     if (getTarget().Options.UnsafeFPMath) {
03231       if (Opcode == ISD::FADD) {
03232         // 0+x --> x
03233         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1))
03234           if (CFP->getValueAPF().isZero())
03235             return N2;
03236         // x+0 --> x
03237         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
03238           if (CFP->getValueAPF().isZero())
03239             return N1;
03240       } else if (Opcode == ISD::FSUB) {
03241         // x-0 --> x
03242         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
03243           if (CFP->getValueAPF().isZero())
03244             return N1;
03245       } else if (Opcode == ISD::FMUL) {
03246         ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1);
03247         SDValue V = N2;
03248 
03249         // If the first operand isn't the constant, try the second
03250         if (!CFP) {
03251           CFP = dyn_cast<ConstantFPSDNode>(N2);
03252           V = N1;
03253         }
03254 
03255         if (CFP) {
03256           // 0*x --> 0
03257           if (CFP->isZero())
03258             return SDValue(CFP,0);
03259           // 1*x --> x
03260           if (CFP->isExactlyValue(1.0))
03261             return V;
03262         }
03263       }
03264     }
03265     assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
03266     assert(N1.getValueType() == N2.getValueType() &&
03267            N1.getValueType() == VT && "Binary operator types must match!");
03268     break;
03269   case ISD::FCOPYSIGN:   // N1 and result must match.  N1/N2 need not match.
03270     assert(N1.getValueType() == VT &&
03271            N1.getValueType().isFloatingPoint() &&
03272            N2.getValueType().isFloatingPoint() &&
03273            "Invalid FCOPYSIGN!");
03274     break;
03275   case ISD::SHL:
03276   case ISD::SRA:
03277   case ISD::SRL:
03278   case ISD::ROTL:
03279   case ISD::ROTR:
03280     assert(VT == N1.getValueType() &&
03281            "Shift operators return type must be the same as their first arg");
03282     assert(VT.isInteger() && N2.getValueType().isInteger() &&
03283            "Shifts only work on integers");
03284     assert((!VT.isVector() || VT == N2.getValueType()) &&
03285            "Vector shift amounts must be in the same as their first arg");
03286     // Verify that the shift amount VT is bit enough to hold valid shift
03287     // amounts.  This catches things like trying to shift an i1024 value by an
03288     // i8, which is easy to fall into in generic code that uses
03289     // TLI.getShiftAmount().
03290     assert(N2.getValueType().getSizeInBits() >=
03291                    Log2_32_Ceil(N1.getValueType().getSizeInBits()) &&
03292            "Invalid use of small shift amount with oversized value!");
03293 
03294     // Always fold shifts of i1 values so the code generator doesn't need to
03295     // handle them.  Since we know the size of the shift has to be less than the
03296     // size of the value, the shift/rotate count is guaranteed to be zero.
03297     if (VT == MVT::i1)
03298       return N1;
03299     if (N2C && N2C->isNullValue())
03300       return N1;
03301     break;
03302   case ISD::FP_ROUND_INREG: {
03303     EVT EVT = cast<VTSDNode>(N2)->getVT();
03304     assert(VT == N1.getValueType() && "Not an inreg round!");
03305     assert(VT.isFloatingPoint() && EVT.isFloatingPoint() &&
03306            "Cannot FP_ROUND_INREG integer types");
03307     assert(EVT.isVector() == VT.isVector() &&
03308            "FP_ROUND_INREG type should be vector iff the operand "
03309            "type is vector!");
03310     assert((!EVT.isVector() ||
03311             EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
03312            "Vector element counts must match in FP_ROUND_INREG");
03313     assert(EVT.bitsLE(VT) && "Not rounding down!");
03314     (void)EVT;
03315     if (cast<VTSDNode>(N2)->getVT() == VT) return N1;  // Not actually rounding.
03316     break;
03317   }
03318   case ISD::FP_ROUND:
03319     assert(VT.isFloatingPoint() &&
03320            N1.getValueType().isFloatingPoint() &&
03321            VT.bitsLE(N1.getValueType()) &&
03322            isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!");
03323     if (N1.getValueType() == VT) return N1;  // noop conversion.
03324     break;
03325   case ISD::AssertSext:
03326   case ISD::AssertZext: {
03327     EVT EVT = cast<VTSDNode>(N2)->getVT();
03328     assert(VT == N1.getValueType() && "Not an inreg extend!");
03329     assert(VT.isInteger() && EVT.isInteger() &&
03330            "Cannot *_EXTEND_INREG FP types");
03331     assert(!EVT.isVector() &&
03332            "AssertSExt/AssertZExt type should be the vector element type "
03333            "rather than the vector type!");
03334     assert(EVT.bitsLE(VT) && "Not extending!");
03335     if (VT == EVT) return N1; // noop assertion.
03336     break;
03337   }
03338   case ISD::SIGN_EXTEND_INREG: {
03339     EVT EVT = cast<VTSDNode>(N2)->getVT();
03340     assert(VT == N1.getValueType() && "Not an inreg extend!");
03341     assert(VT.isInteger() && EVT.isInteger() &&
03342            "Cannot *_EXTEND_INREG FP types");
03343     assert(EVT.isVector() == VT.isVector() &&
03344            "SIGN_EXTEND_INREG type should be vector iff the operand "
03345            "type is vector!");
03346     assert((!EVT.isVector() ||
03347             EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
03348            "Vector element counts must match in SIGN_EXTEND_INREG");
03349     assert(EVT.bitsLE(VT) && "Not extending!");
03350     if (EVT == VT) return N1;  // Not actually extending
03351 
03352     if (N1C) {
03353       APInt Val = N1C->getAPIntValue();
03354       unsigned FromBits = EVT.getScalarType().getSizeInBits();
03355       Val <<= Val.getBitWidth()-FromBits;
03356       Val = Val.ashr(Val.getBitWidth()-FromBits);
03357       return getConstant(Val, VT);
03358     }
03359     break;
03360   }
03361   case ISD::EXTRACT_VECTOR_ELT:
03362     // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF.
03363     if (N1.getOpcode() == ISD::UNDEF)
03364       return getUNDEF(VT);
03365 
03366     // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
03367     // expanding copies of large vectors from registers.
03368     if (N2C &&
03369         N1.getOpcode() == ISD::CONCAT_VECTORS &&
03370         N1.getNumOperands() > 0) {
03371       unsigned Factor =
03372         N1.getOperand(0).getValueType().getVectorNumElements();
03373       return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
03374                      N1.getOperand(N2C->getZExtValue() / Factor),
03375                      getConstant(N2C->getZExtValue() % Factor,
03376                                  N2.getValueType()));
03377     }
03378 
03379     // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
03380     // expanding large vector constants.
03381     if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) {
03382       SDValue Elt = N1.getOperand(N2C->getZExtValue());
03383 
03384       if (VT != Elt.getValueType())
03385         // If the vector element type is not legal, the BUILD_VECTOR operands
03386         // are promoted and implicitly truncated, and the result implicitly
03387         // extended. Make that explicit here.
03388         Elt = getAnyExtOrTrunc(Elt, DL, VT);
03389 
03390       return Elt;
03391     }
03392 
03393     // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
03394     // operations are lowered to scalars.
03395     if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
03396       // If the indices are the same, return the inserted element else
03397       // if the indices are known different, extract the element from
03398       // the original vector.
03399       SDValue N1Op2 = N1.getOperand(2);
03400       ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode());
03401 
03402       if (N1Op2C && N2C) {
03403         if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
03404           if (VT == N1.getOperand(1).getValueType())
03405             return N1.getOperand(1);
03406           else
03407             return getSExtOrTrunc(N1.getOperand(1), DL, VT);
03408         }
03409 
03410         return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
03411       }
03412     }
03413     break;
03414   case ISD::EXTRACT_ELEMENT:
03415     assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
03416     assert(!N1.getValueType().isVector() && !VT.isVector() &&
03417            (N1.getValueType().isInteger() == VT.isInteger()) &&
03418            N1.getValueType() != VT &&
03419            "Wrong types for EXTRACT_ELEMENT!");
03420 
03421     // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
03422     // 64-bit integers into 32-bit parts.  Instead of building the extract of
03423     // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
03424     if (N1.getOpcode() == ISD::BUILD_PAIR)
03425       return N1.getOperand(N2C->getZExtValue());
03426 
03427     // EXTRACT_ELEMENT of a constant int is also very common.
03428     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
03429       unsigned ElementSize = VT.getSizeInBits();
03430       unsigned Shift = ElementSize * N2C->getZExtValue();
03431       APInt ShiftedVal = C->getAPIntValue().lshr(Shift);
03432       return getConstant(ShiftedVal.trunc(ElementSize), VT);
03433     }
03434     break;
03435   case ISD::EXTRACT_SUBVECTOR: {
03436     SDValue Index = N2;
03437     if (VT.isSimple() && N1.getValueType().isSimple()) {
03438       assert(VT.isVector() && N1.getValueType().isVector() &&
03439              "Extract subvector VTs must be a vectors!");
03440       assert(VT.getVectorElementType() ==
03441              N1.getValueType().getVectorElementType() &&
03442              "Extract subvector VTs must have the same element type!");
03443       assert(VT.getSimpleVT() <= N1.getSimpleValueType() &&
03444              "Extract subvector must be from larger vector to smaller vector!");
03445 
03446       if (isa<ConstantSDNode>(Index.getNode())) {
03447         assert((VT.getVectorNumElements() +
03448                 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
03449                 <= N1.getValueType().getVectorNumElements())
03450                && "Extract subvector overflow!");
03451       }
03452 
03453       // Trivial extraction.
03454       if (VT.getSimpleVT() == N1.getSimpleValueType())
03455         return N1;
03456     }
03457     break;
03458   }
03459   }
03460 
03461   // Perform trivial constant folding.
03462   if (SDValue SV =
03463           FoldConstantArithmetic(Opcode, VT, N1.getNode(), N2.getNode()))
03464     return SV;
03465 
03466   // Canonicalize constant to RHS if commutative.
03467   if (N1C && !N2C && isCommutativeBinOp(Opcode)) {
03468     std::swap(N1C, N2C);
03469     std::swap(N1, N2);
03470   }
03471 
03472   // Constant fold FP operations.
03473   bool HasFPExceptions = TLI->hasFloatingPointExceptions();
03474   ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode());
03475   ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode());
03476   if (N1CFP) {
03477     if (!N2CFP && isCommutativeBinOp(Opcode)) {
03478       // Canonicalize constant to RHS if commutative.
03479       std::swap(N1CFP, N2CFP);
03480       std::swap(N1, N2);
03481     } else if (N2CFP) {
03482       APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF();
03483       APFloat::opStatus s;
03484       switch (Opcode) {
03485       case ISD::FADD:
03486         s = V1.add(V2, APFloat::rmNearestTiesToEven);
03487         if (!HasFPExceptions || s != APFloat::opInvalidOp)
03488           return getConstantFP(V1, VT);
03489         break;
03490       case ISD::FSUB:
03491         s = V1.subtract(V2, APFloat::rmNearestTiesToEven);
03492         if (!HasFPExceptions || s!=APFloat::opInvalidOp)
03493           return getConstantFP(V1, VT);
03494         break;
03495       case ISD::FMUL:
03496         s = V1.multiply(V2, APFloat::rmNearestTiesToEven);
03497         if (!HasFPExceptions || s!=APFloat::opInvalidOp)
03498           return getConstantFP(V1, VT);
03499         break;
03500       case ISD::FDIV:
03501         s = V1.divide(V2, APFloat::rmNearestTiesToEven);
03502         if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
03503                                  s!=APFloat::opDivByZero)) {
03504           return getConstantFP(V1, VT);
03505         }
03506         break;
03507       case ISD::FREM :
03508         s = V1.mod(V2, APFloat::rmNearestTiesToEven);
03509         if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
03510                                  s!=APFloat::opDivByZero)) {
03511           return getConstantFP(V1, VT);
03512         }
03513         break;
03514       case ISD::FCOPYSIGN:
03515         V1.copySign(V2);
03516         return getConstantFP(V1, VT);
03517       default: break;
03518       }
03519     }
03520 
03521     if (Opcode == ISD::FP_ROUND) {
03522       APFloat V = N1CFP->getValueAPF();    // make copy
03523       bool ignored;
03524       // This can return overflow, underflow, or inexact; we don't care.
03525       // FIXME need to be more flexible about rounding mode.
03526       (void)V.convert(EVTToAPFloatSemantics(VT),
03527                       APFloat::rmNearestTiesToEven, &ignored);
03528       return getConstantFP(V, VT);
03529     }
03530   }
03531 
03532   // Canonicalize an UNDEF to the RHS, even over a constant.
03533   if (N1.getOpcode() == ISD::UNDEF) {
03534     if (isCommutativeBinOp(Opcode)) {
03535       std::swap(N1, N2);
03536     } else {
03537       switch (Opcode) {
03538       case ISD::FP_ROUND_INREG:
03539       case ISD::SIGN_EXTEND_INREG:
03540       case ISD::SUB:
03541       case ISD::FSUB:
03542       case ISD::FDIV:
03543       case ISD::FREM:
03544       case ISD::SRA:
03545         return N1;     // fold op(undef, arg2) -> undef
03546       case ISD::UDIV:
03547       case ISD::SDIV:
03548       case ISD::UREM:
03549       case ISD::SREM:
03550       case ISD::SRL:
03551       case ISD::SHL:
03552         if (!VT.isVector())
03553           return getConstant(0, VT);    // fold op(undef, arg2) -> 0
03554         // For vectors, we can't easily build an all zero vector, just return
03555         // the LHS.
03556         return N2;
03557       }
03558     }
03559   }
03560 
03561   // Fold a bunch of operators when the RHS is undef.
03562   if (N2.getOpcode() == ISD::UNDEF) {
03563     switch (Opcode) {
03564     case ISD::XOR:
03565       if (N1.getOpcode() == ISD::UNDEF)
03566         // Handle undef ^ undef -> 0 special case. This is a common
03567         // idiom (misuse).
03568         return getConstant(0, VT);
03569       // fallthrough
03570     case ISD::ADD:
03571     case ISD::ADDC:
03572     case ISD::ADDE:
03573     case ISD::SUB:
03574     case ISD::UDIV:
03575     case ISD::SDIV:
03576     case ISD::UREM:
03577     case ISD::SREM:
03578       return N2;       // fold op(arg1, undef) -> undef
03579     case ISD::FADD:
03580     case ISD::FSUB:
03581     case ISD::FMUL:
03582     case ISD::FDIV:
03583     case ISD::FREM:
03584       if (getTarget().Options.UnsafeFPMath)
03585         return N2;
03586       break;
03587     case ISD::MUL:
03588     case ISD::AND:
03589     case ISD::SRL:
03590     case ISD::SHL:
03591       if (!VT.isVector())
03592         return getConstant(0, VT);  // fold op(arg1, undef) -> 0
03593       // For vectors, we can't easily build an all zero vector, just return
03594       // the LHS.
03595       return N1;
03596     case ISD::OR:
03597       if (!VT.isVector())
03598         return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT);
03599       // For vectors, we can't easily build an all one vector, just return
03600       // the LHS.
03601       return N1;
03602     case ISD::SRA:
03603       return N1;
03604     }
03605   }
03606 
03607   // Memoize this node if possible.
03608   BinarySDNode *N;
03609   SDVTList VTs = getVTList(VT);
03610   const bool BinOpHasFlags = isBinOpWithFlags(Opcode);
03611   if (VT != MVT::Glue) {
03612     SDValue Ops[] = {N1, N2};
03613     FoldingSetNodeID ID;
03614     AddNodeIDNode(ID, Opcode, VTs, Ops);
03615     if (BinOpHasFlags)
03616       AddBinaryNodeIDCustom(ID, Opcode, nuw, nsw, exact);
03617     void *IP = nullptr;
03618     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
03619       return SDValue(E, 0);
03620 
03621     N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
03622 
03623     CSEMap.InsertNode(N, IP);
03624   } else {
03625 
03626     N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
03627   }
03628 
03629   InsertNode(N);
03630   return SDValue(N, 0);
03631 }
03632 
03633 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03634                               SDValue N1, SDValue N2, SDValue N3) {
03635   // Perform various simplifications.
03636   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
03637   switch (Opcode) {
03638   case ISD::FMA: {
03639     ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
03640     ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
03641     ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
03642     if (N1CFP && N2CFP && N3CFP) {
03643       APFloat  V1 = N1CFP->getValueAPF();
03644       const APFloat &V2 = N2CFP->getValueAPF();
03645       const APFloat &V3 = N3CFP->getValueAPF();
03646       APFloat::opStatus s =
03647         V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
03648       if (!TLI->hasFloatingPointExceptions() || s != APFloat::opInvalidOp)
03649         return getConstantFP(V1, VT);
03650     }
03651     break;
03652   }
03653   case ISD::CONCAT_VECTORS:
03654     // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
03655     // one big BUILD_VECTOR.
03656     if (N1.getOpcode() == ISD::BUILD_VECTOR &&
03657         N2.getOpcode() == ISD::BUILD_VECTOR &&
03658         N3.getOpcode() == ISD::BUILD_VECTOR) {
03659       SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
03660                                     N1.getNode()->op_end());
03661       Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
03662       Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end());
03663       return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
03664     }
03665     break;
03666   case ISD::SETCC: {
03667     // Use FoldSetCC to simplify SETCC's.
03668     SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL);
03669     if (Simp.getNode()) return Simp;
03670     break;
03671   }
03672   case ISD::SELECT:
03673     if (N1C) {
03674      if (N1C->getZExtValue())
03675        return N2;             // select true, X, Y -> X
03676      return N3;             // select false, X, Y -> Y
03677     }
03678 
03679     if (N2 == N3) return N2;   // select C, X, X -> X
03680     break;
03681   case ISD::VECTOR_SHUFFLE:
03682     llvm_unreachable("should use getVectorShuffle constructor!");
03683   case ISD::INSERT_SUBVECTOR: {
03684     SDValue Index = N3;
03685     if (VT.isSimple() && N1.getValueType().isSimple()
03686         && N2.getValueType().isSimple()) {
03687       assert(VT.isVector() && N1.getValueType().isVector() &&
03688              N2.getValueType().isVector() &&
03689              "Insert subvector VTs must be a vectors");
03690       assert(VT == N1.getValueType() &&
03691              "Dest and insert subvector source types must match!");
03692       assert(N2.getSimpleValueType() <= N1.getSimpleValueType() &&
03693              "Insert subvector must be from smaller vector to larger vector!");
03694       if (isa<ConstantSDNode>(Index.getNode())) {
03695         assert((N2.getValueType().getVectorNumElements() +
03696                 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
03697                 <= VT.getVectorNumElements())
03698                && "Insert subvector overflow!");
03699       }
03700 
03701       // Trivial insertion.
03702       if (VT.getSimpleVT() == N2.getSimpleValueType())
03703         return N2;
03704     }
03705     break;
03706   }
03707   case ISD::BITCAST:
03708     // Fold bit_convert nodes from a type to themselves.
03709     if (N1.getValueType() == VT)
03710       return N1;
03711     break;
03712   }
03713 
03714   // Memoize node if it doesn't produce a flag.
03715   SDNode *N;
03716   SDVTList VTs = getVTList(VT);
03717   if (VT != MVT::Glue) {
03718     SDValue Ops[] = { N1, N2, N3 };
03719     FoldingSetNodeID ID;
03720     AddNodeIDNode(ID, Opcode, VTs, Ops);
03721     void *IP = nullptr;
03722     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
03723       return SDValue(E, 0);
03724 
03725     N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
03726                                           DL.getDebugLoc(), VTs, N1, N2, N3);
03727     CSEMap.InsertNode(N, IP);
03728   } else {
03729     N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
03730                                           DL.getDebugLoc(), VTs, N1, N2, N3);
03731   }
03732 
03733   InsertNode(N);
03734   return SDValue(N, 0);
03735 }
03736 
03737 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03738                               SDValue N1, SDValue N2, SDValue N3,
03739                               SDValue N4) {
03740   SDValue Ops[] = { N1, N2, N3, N4 };
03741   return getNode(Opcode, DL, VT, Ops);
03742 }
03743 
03744 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03745                               SDValue N1, SDValue N2, SDValue N3,
03746                               SDValue N4, SDValue N5) {
03747   SDValue Ops[] = { N1, N2, N3, N4, N5 };
03748   return getNode(Opcode, DL, VT, Ops);
03749 }
03750 
03751 /// getStackArgumentTokenFactor - Compute a TokenFactor to force all
03752 /// the incoming stack arguments to be loaded from the stack.
03753 SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
03754   SmallVector<SDValue, 8> ArgChains;
03755 
03756   // Include the original chain at the beginning of the list. When this is
03757   // used by target LowerCall hooks, this helps legalize find the
03758   // CALLSEQ_BEGIN node.
03759   ArgChains.push_back(Chain);
03760 
03761   // Add a chain value for each stack argument.
03762   for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(),
03763        UE = getEntryNode().getNode()->use_end(); U != UE; ++U)
03764     if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
03765       if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
03766         if (FI->getIndex() < 0)
03767           ArgChains.push_back(SDValue(L, 1));
03768 
03769   // Build a tokenfactor for all the chains.
03770   return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
03771 }
03772 
03773 /// getMemsetValue - Vectorized representation of the memset value
03774 /// operand.
03775 static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
03776                               SDLoc dl) {
03777   assert(Value.getOpcode() != ISD::UNDEF);
03778 
03779   unsigned NumBits = VT.getScalarType().getSizeInBits();
03780   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
03781     assert(C->getAPIntValue().getBitWidth() == 8);
03782     APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
03783     if (VT.isInteger())
03784       return DAG.getConstant(Val, VT);
03785     return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), VT);
03786   }
03787 
03788   Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value);
03789   if (NumBits > 8) {
03790     // Use a multiplication with 0x010101... to extend the input to the
03791     // required length.
03792     APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
03793     Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT));
03794   }
03795 
03796   return Value;
03797 }
03798 
03799 /// getMemsetStringVal - Similar to getMemsetValue. Except this is only
03800 /// used when a memcpy is turned into a memset when the source is a constant
03801 /// string ptr.
03802 static SDValue getMemsetStringVal(EVT VT, SDLoc dl, SelectionDAG &DAG,
03803                                   const TargetLowering &TLI, StringRef Str) {
03804   // Handle vector with all elements zero.
03805   if (Str.empty()) {
03806     if (VT.isInteger())
03807       return DAG.getConstant(0, VT);
03808     else if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
03809       return DAG.getConstantFP(0.0, VT);
03810     else if (VT.isVector()) {
03811       unsigned NumElts = VT.getVectorNumElements();
03812       MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
03813       return DAG.getNode(ISD::BITCAST, dl, VT,
03814                          DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(),
03815                                                              EltVT, NumElts)));
03816     } else
03817       llvm_unreachable("Expected type!");
03818   }
03819 
03820   assert(!VT.isVector() && "Can't handle vector type here!");
03821   unsigned NumVTBits = VT.getSizeInBits();
03822   unsigned NumVTBytes = NumVTBits / 8;
03823   unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size()));
03824 
03825   APInt Val(NumVTBits, 0);
03826   if (TLI.isLittleEndian()) {
03827     for (unsigned i = 0; i != NumBytes; ++i)
03828       Val |= (uint64_t)(unsigned char)Str[i] << i*8;
03829   } else {
03830     for (unsigned i = 0; i != NumBytes; ++i)
03831       Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8;
03832   }
03833 
03834   // If the "cost" of materializing the integer immediate is less than the cost
03835   // of a load, then it is cost effective to turn the load into the immediate.
03836   Type *Ty = VT.getTypeForEVT(*DAG.getContext());
03837   if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
03838     return DAG.getConstant(Val, VT);
03839   return SDValue(nullptr, 0);
03840 }
03841 
03842 /// getMemBasePlusOffset - Returns base and offset node for the
03843 ///
03844 static SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, SDLoc dl,
03845                                       SelectionDAG &DAG) {
03846   EVT VT = Base.getValueType();
03847   return DAG.getNode(ISD::ADD, dl,
03848                      VT, Base, DAG.getConstant(Offset, VT));
03849 }
03850 
03851 /// isMemSrcFromString - Returns true if memcpy source is a string constant.
03852 ///
03853 static bool isMemSrcFromString(SDValue Src, StringRef &Str) {
03854   unsigned SrcDelta = 0;
03855   GlobalAddressSDNode *G = nullptr;
03856   if (Src.getOpcode() == ISD::GlobalAddress)
03857     G = cast<GlobalAddressSDNode>(Src);
03858   else if (Src.getOpcode() == ISD::ADD &&
03859            Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
03860            Src.getOperand(1).getOpcode() == ISD::Constant) {
03861     G = cast<GlobalAddressSDNode>(Src.getOperand(0));
03862     SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue();
03863   }
03864   if (!G)
03865     return false;
03866 
03867   return getConstantStringInfo(G->getGlobal(), Str, SrcDelta, false);
03868 }
03869 
03870 /// FindOptimalMemOpLowering - Determines the optimial series memory ops
03871 /// to replace the memset / memcpy. Return true if the number of memory ops
03872 /// is below the threshold. It returns the types of the sequence of
03873 /// memory ops to perform memset / memcpy by reference.
03874 static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
03875                                      unsigned Limit, uint64_t Size,
03876                                      unsigned DstAlign, unsigned SrcAlign,
03877                                      bool IsMemset,
03878                                      bool ZeroMemset,
03879                                      bool MemcpyStrSrc,
03880                                      bool AllowOverlap,
03881                                      SelectionDAG &DAG,
03882                                      const TargetLowering &TLI) {
03883   assert((SrcAlign == 0 || SrcAlign >= DstAlign) &&
03884          "Expecting memcpy / memset source to meet alignment requirement!");
03885   // If 'SrcAlign' is zero, that means the memory operation does not need to
03886   // load the value, i.e. memset or memcpy from constant string. Otherwise,
03887   // it's the inferred alignment of the source. 'DstAlign', on the other hand,
03888   // is the specified alignment of the memory operation. If it is zero, that
03889   // means it's possible to change the alignment of the destination.
03890   // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does
03891   // not need to be loaded.
03892   EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign,
03893                                    IsMemset, ZeroMemset, MemcpyStrSrc,
03894                                    DAG.getMachineFunction());
03895 
03896   if (VT == MVT::Other) {
03897     unsigned AS = 0;
03898     if (DstAlign >= TLI.getDataLayout()->getPointerPrefAlignment(AS) ||
03899         TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign)) {
03900       VT = TLI.getPointerTy();
03901     } else {
03902       switch (DstAlign & 7) {
03903       case 0:  VT = MVT::i64; break;
03904       case 4:  VT = MVT::i32; break;
03905       case 2:  VT = MVT::i16; break;
03906       default: VT = MVT::i8;  break;
03907       }
03908     }
03909 
03910     MVT LVT = MVT::i64;
03911     while (!TLI.isTypeLegal(LVT))
03912       LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
03913     assert(LVT.isInteger());
03914 
03915     if (VT.bitsGT(LVT))
03916       VT = LVT;
03917   }
03918 
03919   unsigned NumMemOps = 0;
03920   while (Size != 0) {
03921     unsigned VTSize = VT.getSizeInBits() / 8;
03922     while (VTSize > Size) {
03923       // For now, only use non-vector load / store's for the left-over pieces.
03924       EVT NewVT = VT;
03925       unsigned NewVTSize;
03926 
03927       bool Found = false;
03928       if (VT.isVector() || VT.isFloatingPoint()) {
03929         NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
03930         if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) &&
03931             TLI.isSafeMemOpType(NewVT.getSimpleVT()))
03932           Found = true;
03933         else if (NewVT == MVT::i64 &&
03934                  TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
03935                  TLI.isSafeMemOpType(MVT::f64)) {
03936           // i64 is usually not legal on 32-bit targets, but f64 may be.
03937           NewVT = MVT::f64;
03938           Found = true;
03939         }
03940       }
03941 
03942       if (!Found) {
03943         do {
03944           NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
03945           if (NewVT == MVT::i8)
03946             break;
03947         } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT()));
03948       }
03949       NewVTSize = NewVT.getSizeInBits() / 8;
03950 
03951       // If the new VT cannot cover all of the remaining bits, then consider
03952       // issuing a (or a pair of) unaligned and overlapping load / store.
03953       // FIXME: Only does this for 64-bit or more since we don't have proper
03954       // cost model for unaligned load / store.
03955       bool Fast;
03956       unsigned AS = 0;
03957       if (NumMemOps && AllowOverlap &&
03958           VTSize >= 8 && NewVTSize < Size &&
03959           TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign, &Fast) && Fast)
03960         VTSize = Size;
03961       else {
03962         VT = NewVT;
03963         VTSize = NewVTSize;
03964       }
03965     }
03966 
03967     if (++NumMemOps > Limit)
03968       return false;
03969 
03970     MemOps.push_back(VT);
03971     Size -= VTSize;
03972   }
03973 
03974   return true;
03975 }
03976 
03977 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
03978                                        SDValue Chain, SDValue Dst,
03979                                        SDValue Src, uint64_t Size,
03980                                        unsigned Align, bool isVol,
03981                                        bool AlwaysInline,
03982                                        MachinePointerInfo DstPtrInfo,
03983                                        MachinePointerInfo SrcPtrInfo) {
03984   // Turn a memcpy of undef to nop.
03985   if (Src.getOpcode() == ISD::UNDEF)
03986     return Chain;
03987 
03988   // Expand memcpy to a series of load and store ops if the size operand falls
03989   // below a certain threshold.
03990   // TODO: In the AlwaysInline case, if the size is big then generate a loop
03991   // rather than maybe a humongous number of loads and stores.
03992   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03993   std::vector<EVT> MemOps;
03994   bool DstAlignCanChange = false;
03995   MachineFunction &MF = DAG.getMachineFunction();
03996   MachineFrameInfo *MFI = MF.getFrameInfo();
03997   bool OptSize = MF.getFunction()->hasFnAttribute(Attribute::OptimizeForSize);
03998   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
03999   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
04000     DstAlignCanChange = true;
04001   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
04002   if (Align > SrcAlign)
04003     SrcAlign = Align;
04004   StringRef Str;
04005   bool CopyFromStr = isMemSrcFromString(Src, Str);
04006   bool isZeroStr = CopyFromStr && Str.empty();
04007   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
04008 
04009   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
04010                                 (DstAlignCanChange ? 0 : Align),
04011                                 (isZeroStr ? 0 : SrcAlign),
04012                                 false, false, CopyFromStr, true, DAG, TLI))
04013     return SDValue();
04014 
04015   if (DstAlignCanChange) {
04016     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
04017     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
04018 
04019     // Don't promote to an alignment that would require dynamic stack
04020     // realignment.
04021     const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
04022     if (!TRI->needsStackRealignment(MF))
04023        while (NewAlign > Align &&
04024              TLI.getDataLayout()->exceedsNaturalStackAlignment(NewAlign))
04025           NewAlign /= 2;
04026 
04027     if (NewAlign > Align) {
04028       // Give the stack frame object a larger alignment if needed.
04029       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
04030         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
04031       Align = NewAlign;
04032     }
04033   }
04034 
04035   SmallVector<SDValue, 8> OutChains;
04036   unsigned NumMemOps = MemOps.size();
04037   uint64_t SrcOff = 0, DstOff = 0;
04038   for (unsigned i = 0; i != NumMemOps; ++i) {
04039     EVT VT = MemOps[i];
04040     unsigned VTSize = VT.getSizeInBits() / 8;
04041     SDValue Value, Store;
04042 
04043     if (VTSize > Size) {
04044       // Issuing an unaligned load / store pair  that overlaps with the previous
04045       // pair. Adjust the offset accordingly.
04046       assert(i == NumMemOps-1 && i != 0);
04047       SrcOff -= VTSize - Size;
04048       DstOff -= VTSize - Size;
04049     }
04050 
04051     if (CopyFromStr &&
04052         (isZeroStr || (VT.isInteger() && !VT.isVector()))) {
04053       // It's unlikely a store of a vector immediate can be done in a single
04054       // instruction. It would require a load from a constantpool first.
04055       // We only handle zero vectors here.
04056       // FIXME: Handle other cases where store of vector immediate is done in
04057       // a single instruction.
04058       Value = getMemsetStringVal(VT, dl, DAG, TLI, Str.substr(SrcOff));
04059       if (Value.getNode())
04060         Store = DAG.getStore(Chain, dl, Value,
04061                              getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04062                              DstPtrInfo.getWithOffset(DstOff), isVol,
04063                              false, Align);
04064     }
04065 
04066     if (!Store.getNode()) {
04067       // The type might not be legal for the target.  This should only happen
04068       // if the type is smaller than a legal type, as on PPC, so the right
04069       // thing to do is generate a LoadExt/StoreTrunc pair.  These simplify
04070       // to Load/Store if NVT==VT.
04071       // FIXME does the case above also need this?
04072       EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
04073       assert(NVT.bitsGE(VT));
04074       Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain,
04075                              getMemBasePlusOffset(Src, SrcOff, dl, DAG),
04076                              SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false,
04077                              false, MinAlign(SrcAlign, SrcOff));
04078       Store = DAG.getTruncStore(Chain, dl, Value,
04079                                 getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04080                                 DstPtrInfo.getWithOffset(DstOff), VT, isVol,
04081                                 false, Align);
04082     }
04083     OutChains.push_back(Store);
04084     SrcOff += VTSize;
04085     DstOff += VTSize;
04086     Size -= VTSize;
04087   }
04088 
04089   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04090 }
04091 
04092 static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
04093                                         SDValue Chain, SDValue Dst,
04094                                         SDValue Src, uint64_t Size,
04095                                         unsigned Align,  bool isVol,
04096                                         bool AlwaysInline,
04097                                         MachinePointerInfo DstPtrInfo,
04098                                         MachinePointerInfo SrcPtrInfo) {
04099   // Turn a memmove of undef to nop.
04100   if (Src.getOpcode() == ISD::UNDEF)
04101     return Chain;
04102 
04103   // Expand memmove to a series of load and store ops if the size operand falls
04104   // below a certain threshold.
04105   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04106   std::vector<EVT> MemOps;
04107   bool DstAlignCanChange = false;
04108   MachineFunction &MF = DAG.getMachineFunction();
04109   MachineFrameInfo *MFI = MF.getFrameInfo();
04110   bool OptSize = MF.getFunction()->hasFnAttribute(Attribute::OptimizeForSize);
04111   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
04112   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
04113     DstAlignCanChange = true;
04114   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
04115   if (Align > SrcAlign)
04116     SrcAlign = Align;
04117   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
04118 
04119   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
04120                                 (DstAlignCanChange ? 0 : Align), SrcAlign,
04121                                 false, false, false, false, DAG, TLI))
04122     return SDValue();
04123 
04124   if (DstAlignCanChange) {
04125     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
04126     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
04127     if (NewAlign > Align) {
04128       // Give the stack frame object a larger alignment if needed.
04129       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
04130         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
04131       Align = NewAlign;
04132     }
04133   }
04134 
04135   uint64_t SrcOff = 0, DstOff = 0;
04136   SmallVector<SDValue, 8> LoadValues;
04137   SmallVector<SDValue, 8> LoadChains;
04138   SmallVector<SDValue, 8> OutChains;
04139   unsigned NumMemOps = MemOps.size();
04140   for (unsigned i = 0; i < NumMemOps; i++) {
04141     EVT VT = MemOps[i];
04142     unsigned VTSize = VT.getSizeInBits() / 8;
04143     SDValue Value;
04144 
04145     Value = DAG.getLoad(VT, dl, Chain,
04146                         getMemBasePlusOffset(Src, SrcOff, dl, DAG),
04147                         SrcPtrInfo.getWithOffset(SrcOff), isVol,
04148                         false, false, SrcAlign);
04149     LoadValues.push_back(Value);
04150     LoadChains.push_back(Value.getValue(1));
04151     SrcOff += VTSize;
04152   }
04153   Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
04154   OutChains.clear();
04155   for (unsigned i = 0; i < NumMemOps; i++) {
04156     EVT VT = MemOps[i];
04157     unsigned VTSize = VT.getSizeInBits() / 8;
04158     SDValue Store;
04159 
04160     Store = DAG.getStore(Chain, dl, LoadValues[i],
04161                          getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04162                          DstPtrInfo.getWithOffset(DstOff), isVol, false, Align);
04163     OutChains.push_back(Store);
04164     DstOff += VTSize;
04165   }
04166 
04167   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04168 }
04169 
04170 /// \brief Lower the call to 'memset' intrinsic function into a series of store
04171 /// operations.
04172 ///
04173 /// \param DAG Selection DAG where lowered code is placed.
04174 /// \param dl Link to corresponding IR location.
04175 /// \param Chain Control flow dependency.
04176 /// \param Dst Pointer to destination memory location.
04177 /// \param Src Value of byte to write into the memory.
04178 /// \param Size Number of bytes to write.
04179 /// \param Align Alignment of the destination in bytes.
04180 /// \param isVol True if destination is volatile.
04181 /// \param DstPtrInfo IR information on the memory pointer.
04182 /// \returns New head in the control flow, if lowering was successful, empty
04183 /// SDValue otherwise.
04184 ///
04185 /// The function tries to replace 'llvm.memset' intrinsic with several store
04186 /// operations and value calculation code. This is usually profitable for small
04187 /// memory size.
04188 static SDValue getMemsetStores(SelectionDAG &DAG, SDLoc dl,
04189                                SDValue Chain, SDValue Dst,
04190                                SDValue Src, uint64_t Size,
04191                                unsigned Align, bool isVol,
04192                                MachinePointerInfo DstPtrInfo) {
04193   // Turn a memset of undef to nop.
04194   if (Src.getOpcode() == ISD::UNDEF)
04195     return Chain;
04196 
04197   // Expand memset to a series of load/store ops if the size operand
04198   // falls below a certain threshold.
04199   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04200   std::vector<EVT> MemOps;
04201   bool DstAlignCanChange = false;
04202   MachineFunction &MF = DAG.getMachineFunction();
04203   MachineFrameInfo *MFI = MF.getFrameInfo();
04204   bool OptSize = MF.getFunction()->hasFnAttribute(Attribute::OptimizeForSize);
04205   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
04206   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
04207     DstAlignCanChange = true;
04208   bool IsZeroVal =
04209     isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
04210   if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize),
04211                                 Size, (DstAlignCanChange ? 0 : Align), 0,
04212                                 true, IsZeroVal, false, true, DAG, TLI))
04213     return SDValue();
04214 
04215   if (DstAlignCanChange) {
04216     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
04217     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
04218     if (NewAlign > Align) {
04219       // Give the stack frame object a larger alignment if needed.
04220       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
04221         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
04222       Align = NewAlign;
04223     }
04224   }
04225 
04226   SmallVector<SDValue, 8> OutChains;
04227   uint64_t DstOff = 0;
04228   unsigned NumMemOps = MemOps.size();
04229 
04230   // Find the largest store and generate the bit pattern for it.
04231   EVT LargestVT = MemOps[0];
04232   for (unsigned i = 1; i < NumMemOps; i++)
04233     if (MemOps[i].bitsGT(LargestVT))
04234       LargestVT = MemOps[i];
04235   SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
04236 
04237   for (unsigned i = 0; i < NumMemOps; i++) {
04238     EVT VT = MemOps[i];
04239     unsigned VTSize = VT.getSizeInBits() / 8;
04240     if (VTSize > Size) {
04241       // Issuing an unaligned load / store pair  that overlaps with the previous
04242       // pair. Adjust the offset accordingly.
04243       assert(i == NumMemOps-1 && i != 0);
04244       DstOff -= VTSize - Size;
04245     }
04246 
04247     // If this store is smaller than the largest store see whether we can get
04248     // the smaller value for free with a truncate.
04249     SDValue Value = MemSetValue;
04250     if (VT.bitsLT(LargestVT)) {
04251       if (!LargestVT.isVector() && !VT.isVector() &&
04252           TLI.isTruncateFree(LargestVT, VT))
04253         Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
04254       else
04255         Value = getMemsetValue(Src, VT, DAG, dl);
04256     }
04257     assert(Value.getValueType() == VT && "Value with wrong type.");
04258     SDValue Store = DAG.getStore(Chain, dl, Value,
04259                                  getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04260                                  DstPtrInfo.getWithOffset(DstOff),
04261                                  isVol, false, Align);
04262     OutChains.push_back(Store);
04263     DstOff += VT.getSizeInBits() / 8;
04264     Size -= VTSize;
04265   }
04266 
04267   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04268 }
04269 
04270 SDValue SelectionDAG::getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst,
04271                                 SDValue Src, SDValue Size,
04272                                 unsigned Align, bool isVol, bool AlwaysInline,
04273                                 MachinePointerInfo DstPtrInfo,
04274                                 MachinePointerInfo SrcPtrInfo) {
04275   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04276 
04277   // Check to see if we should lower the memcpy to loads and stores first.
04278   // For cases within the target-specified limits, this is the best choice.
04279   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04280   if (ConstantSize) {
04281     // Memcpy with size zero? Just return the original chain.
04282     if (ConstantSize->isNullValue())
04283       return Chain;
04284 
04285     SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
04286                                              ConstantSize->getZExtValue(),Align,
04287                                 isVol, false, DstPtrInfo, SrcPtrInfo);
04288     if (Result.getNode())
04289       return Result;
04290   }
04291 
04292   // Then check to see if we should lower the memcpy with target-specific
04293   // code. If the target chooses to do this, this is the next best.
04294   if (TSI) {
04295     SDValue Result = TSI->EmitTargetCodeForMemcpy(
04296         *this, dl, Chain, Dst, Src, Size, Align, isVol, AlwaysInline,
04297         DstPtrInfo, SrcPtrInfo);
04298     if (Result.getNode())
04299       return Result;
04300   }
04301 
04302   // If we really need inline code and the target declined to provide it,
04303   // use a (potentially long) sequence of loads and stores.
04304   if (AlwaysInline) {
04305     assert(ConstantSize && "AlwaysInline requires a constant size!");
04306     return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
04307                                    ConstantSize->getZExtValue(), Align, isVol,
04308                                    true, DstPtrInfo, SrcPtrInfo);
04309   }
04310 
04311   // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
04312   // memcpy is not guaranteed to be safe. libc memcpys aren't required to
04313   // respect volatile, so they may do things like read or write memory
04314   // beyond the given memory regions. But fixing this isn't easy, and most
04315   // people don't care.
04316 
04317   // Emit a library call.
04318   TargetLowering::ArgListTy Args;
04319   TargetLowering::ArgListEntry Entry;
04320   Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
04321   Entry.Node = Dst; Args.push_back(Entry);
04322   Entry.Node = Src; Args.push_back(Entry);
04323   Entry.Node = Size; Args.push_back(Entry);
04324   // FIXME: pass in SDLoc
04325   TargetLowering::CallLoweringInfo CLI(*this);
04326   CLI.setDebugLoc(dl).setChain(Chain)
04327     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
04328                Type::getVoidTy(*getContext()),
04329                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
04330                                  TLI->getPointerTy()), std::move(Args), 0)
04331     .setDiscardResult();
04332   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04333 
04334   return CallResult.second;
04335 }
04336 
04337 SDValue SelectionDAG::getMemmove(SDValue Chain, SDLoc dl, SDValue Dst,
04338                                  SDValue Src, SDValue Size,
04339                                  unsigned Align, bool isVol,
04340                                  MachinePointerInfo DstPtrInfo,
04341                                  MachinePointerInfo SrcPtrInfo) {
04342   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04343 
04344   // Check to see if we should lower the memmove to loads and stores first.
04345   // For cases within the target-specified limits, this is the best choice.
04346   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04347   if (ConstantSize) {
04348     // Memmove with size zero? Just return the original chain.
04349     if (ConstantSize->isNullValue())
04350       return Chain;
04351 
04352     SDValue Result =
04353       getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src,
04354                                ConstantSize->getZExtValue(), Align, isVol,
04355                                false, DstPtrInfo, SrcPtrInfo);
04356     if (Result.getNode())
04357       return Result;
04358   }
04359 
04360   // Then check to see if we should lower the memmove with target-specific
04361   // code. If the target chooses to do this, this is the next best.
04362   if (TSI) {
04363     SDValue Result = TSI->EmitTargetCodeForMemmove(
04364         *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo, SrcPtrInfo);
04365     if (Result.getNode())
04366       return Result;
04367   }
04368 
04369   // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
04370   // not be safe.  See memcpy above for more details.
04371 
04372   // Emit a library call.
04373   TargetLowering::ArgListTy Args;
04374   TargetLowering::ArgListEntry Entry;
04375   Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
04376   Entry.Node = Dst; Args.push_back(Entry);
04377   Entry.Node = Src; Args.push_back(Entry);
04378   Entry.Node = Size; Args.push_back(Entry);
04379   // FIXME:  pass in SDLoc
04380   TargetLowering::CallLoweringInfo CLI(*this);
04381   CLI.setDebugLoc(dl).setChain(Chain)
04382     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
04383                Type::getVoidTy(*getContext()),
04384                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
04385                                  TLI->getPointerTy()), std::move(Args), 0)
04386     .setDiscardResult();
04387   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04388 
04389   return CallResult.second;
04390 }
04391 
04392 SDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst,
04393                                 SDValue Src, SDValue Size,
04394                                 unsigned Align, bool isVol,
04395                                 MachinePointerInfo DstPtrInfo) {
04396   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04397 
04398   // Check to see if we should lower the memset to stores first.
04399   // For cases within the target-specified limits, this is the best choice.
04400   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04401   if (ConstantSize) {
04402     // Memset with size zero? Just return the original chain.
04403     if (ConstantSize->isNullValue())
04404       return Chain;
04405 
04406     SDValue Result =
04407       getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
04408                       Align, isVol, DstPtrInfo);
04409 
04410     if (Result.getNode())
04411       return Result;
04412   }
04413 
04414   // Then check to see if we should lower the memset with target-specific
04415   // code. If the target chooses to do this, this is the next best.
04416   if (TSI) {
04417     SDValue Result = TSI->EmitTargetCodeForMemset(
04418         *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo);
04419     if (Result.getNode())
04420       return Result;
04421   }
04422 
04423   // Emit a library call.
04424   Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(*getContext());
04425   TargetLowering::ArgListTy Args;
04426   TargetLowering::ArgListEntry Entry;
04427   Entry.Node = Dst; Entry.Ty = IntPtrTy;
04428   Args.push_back(Entry);
04429   Entry.Node = Src;
04430   Entry.Ty = Src.getValueType().getTypeForEVT(*getContext());
04431   Args.push_back(Entry);
04432   Entry.Node = Size;
04433   Entry.Ty = IntPtrTy;
04434   Args.push_back(Entry);
04435 
04436   // FIXME: pass in SDLoc
04437   TargetLowering::CallLoweringInfo CLI(*this);
04438   CLI.setDebugLoc(dl).setChain(Chain)
04439     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
04440                Type::getVoidTy(*getContext()),
04441                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
04442                                  TLI->getPointerTy()), std::move(Args), 0)
04443     .setDiscardResult();
04444 
04445   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04446   return CallResult.second;
04447 }
04448 
04449 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04450                                 SDVTList VTList, ArrayRef<SDValue> Ops,
04451                                 MachineMemOperand *MMO,
04452                                 AtomicOrdering SuccessOrdering,
04453                                 AtomicOrdering FailureOrdering,
04454                                 SynchronizationScope SynchScope) {
04455   FoldingSetNodeID ID;
04456   ID.AddInteger(MemVT.getRawBits());
04457   AddNodeIDNode(ID, Opcode, VTList, Ops);
04458   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04459   void* IP = nullptr;
04460   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04461     cast<AtomicSDNode>(E)->refineAlignment(MMO);
04462     return SDValue(E, 0);
04463   }
04464 
04465   // Allocate the operands array for the node out of the BumpPtrAllocator, since
04466   // SDNode doesn't have access to it.  This memory will be "leaked" when
04467   // the node is deallocated, but recovered when the allocator is released.
04468   // If the number of operands is less than 5 we use AtomicSDNode's internal
04469   // storage.
04470   unsigned NumOps = Ops.size();
04471   SDUse *DynOps = NumOps > 4 ? OperandAllocator.Allocate<SDUse>(NumOps)
04472                              : nullptr;
04473 
04474   SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(),
04475                                                dl.getDebugLoc(), VTList, MemVT,
04476                                                Ops.data(), DynOps, NumOps, MMO,
04477                                                SuccessOrdering, FailureOrdering,
04478                                                SynchScope);
04479   CSEMap.InsertNode(N, IP);
04480   InsertNode(N);
04481   return SDValue(N, 0);
04482 }
04483 
04484 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04485                                 SDVTList VTList, ArrayRef<SDValue> Ops,
04486                                 MachineMemOperand *MMO,
04487                                 AtomicOrdering Ordering,
04488                                 SynchronizationScope SynchScope) {
04489   return getAtomic(Opcode, dl, MemVT, VTList, Ops, MMO, Ordering,
04490                    Ordering, SynchScope);
04491 }
04492 
04493 SDValue SelectionDAG::getAtomicCmpSwap(
04494     unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs, SDValue Chain,
04495     SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo,
04496     unsigned Alignment, AtomicOrdering SuccessOrdering,
04497     AtomicOrdering FailureOrdering, SynchronizationScope SynchScope) {
04498   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
04499          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
04500   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
04501 
04502   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04503     Alignment = getEVTAlignment(MemVT);
04504 
04505   MachineFunction &MF = getMachineFunction();
04506 
04507   // FIXME: Volatile isn't really correct; we should keep track of atomic
04508   // orderings in the memoperand.
04509   unsigned Flags = MachineMemOperand::MOVolatile;
04510   Flags |= MachineMemOperand::MOLoad;
04511   Flags |= MachineMemOperand::MOStore;
04512 
04513   MachineMemOperand *MMO =
04514     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
04515 
04516   return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO,
04517                           SuccessOrdering, FailureOrdering, SynchScope);
04518 }
04519 
04520 SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, SDLoc dl, EVT MemVT,
04521                                        SDVTList VTs, SDValue Chain, SDValue Ptr,
04522                                        SDValue Cmp, SDValue Swp,
04523                                        MachineMemOperand *MMO,
04524                                        AtomicOrdering SuccessOrdering,
04525                                        AtomicOrdering FailureOrdering,
04526                                        SynchronizationScope SynchScope) {
04527   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
04528          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
04529   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
04530 
04531   SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
04532   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO,
04533                    SuccessOrdering, FailureOrdering, SynchScope);
04534 }
04535 
04536 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04537                                 SDValue Chain,
04538                                 SDValue Ptr, SDValue Val,
04539                                 const Value* PtrVal,
04540                                 unsigned Alignment,
04541                                 AtomicOrdering Ordering,
04542                                 SynchronizationScope SynchScope) {
04543   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04544     Alignment = getEVTAlignment(MemVT);
04545 
04546   MachineFunction &MF = getMachineFunction();
04547   // An atomic store does not load. An atomic load does not store.
04548   // (An atomicrmw obviously both loads and stores.)
04549   // For now, atomics are considered to be volatile always, and they are
04550   // chained as such.
04551   // FIXME: Volatile isn't really correct; we should keep track of atomic
04552   // orderings in the memoperand.
04553   unsigned Flags = MachineMemOperand::MOVolatile;
04554   if (Opcode != ISD::ATOMIC_STORE)
04555     Flags |= MachineMemOperand::MOLoad;
04556   if (Opcode != ISD::ATOMIC_LOAD)
04557     Flags |= MachineMemOperand::MOStore;
04558 
04559   MachineMemOperand *MMO =
04560     MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
04561                             MemVT.getStoreSize(), Alignment);
04562 
04563   return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO,
04564                    Ordering, SynchScope);
04565 }
04566 
04567 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04568                                 SDValue Chain,
04569                                 SDValue Ptr, SDValue Val,
04570                                 MachineMemOperand *MMO,
04571                                 AtomicOrdering Ordering,
04572                                 SynchronizationScope SynchScope) {
04573   assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
04574           Opcode == ISD::ATOMIC_LOAD_SUB ||
04575           Opcode == ISD::ATOMIC_LOAD_AND ||
04576           Opcode == ISD::ATOMIC_LOAD_OR ||
04577           Opcode == ISD::ATOMIC_LOAD_XOR ||
04578           Opcode == ISD::ATOMIC_LOAD_NAND ||
04579           Opcode == ISD::ATOMIC_LOAD_MIN ||
04580           Opcode == ISD::ATOMIC_LOAD_MAX ||
04581           Opcode == ISD::ATOMIC_LOAD_UMIN ||
04582           Opcode == ISD::ATOMIC_LOAD_UMAX ||
04583           Opcode == ISD::ATOMIC_SWAP ||
04584           Opcode == ISD::ATOMIC_STORE) &&
04585          "Invalid Atomic Op");
04586 
04587   EVT VT = Val.getValueType();
04588 
04589   SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
04590                                                getVTList(VT, MVT::Other);
04591   SDValue Ops[] = {Chain, Ptr, Val};
04592   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
04593 }
04594 
04595 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04596                                 EVT VT, SDValue Chain,
04597                                 SDValue Ptr,
04598                                 MachineMemOperand *MMO,
04599                                 AtomicOrdering Ordering,
04600                                 SynchronizationScope SynchScope) {
04601   assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
04602 
04603   SDVTList VTs = getVTList(VT, MVT::Other);
04604   SDValue Ops[] = {Chain, Ptr};
04605   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
04606 }
04607 
04608 /// getMergeValues - Create a MERGE_VALUES node from the given operands.
04609 SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, SDLoc dl) {
04610   if (Ops.size() == 1)
04611     return Ops[0];
04612 
04613   SmallVector<EVT, 4> VTs;
04614   VTs.reserve(Ops.size());
04615   for (unsigned i = 0; i < Ops.size(); ++i)
04616     VTs.push_back(Ops[i].getValueType());
04617   return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
04618 }
04619 
04620 SDValue
04621 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
04622                                   ArrayRef<SDValue> Ops,
04623                                   EVT MemVT, MachinePointerInfo PtrInfo,
04624                                   unsigned Align, bool Vol,
04625                                   bool ReadMem, bool WriteMem, unsigned Size) {
04626   if (Align == 0)  // Ensure that codegen never sees alignment 0
04627     Align = getEVTAlignment(MemVT);
04628 
04629   MachineFunction &MF = getMachineFunction();
04630   unsigned Flags = 0;
04631   if (WriteMem)
04632     Flags |= MachineMemOperand::MOStore;
04633   if (ReadMem)
04634     Flags |= MachineMemOperand::MOLoad;
04635   if (Vol)
04636     Flags |= MachineMemOperand::MOVolatile;
04637   if (!Size)
04638     Size = MemVT.getStoreSize();
04639   MachineMemOperand *MMO =
04640     MF.getMachineMemOperand(PtrInfo, Flags, Size, Align);
04641 
04642   return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
04643 }
04644 
04645 SDValue
04646 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
04647                                   ArrayRef<SDValue> Ops, EVT MemVT,
04648                                   MachineMemOperand *MMO) {
04649   assert((Opcode == ISD::INTRINSIC_VOID ||
04650           Opcode == ISD::INTRINSIC_W_CHAIN ||
04651           Opcode == ISD::PREFETCH ||
04652           Opcode == ISD::LIFETIME_START ||
04653           Opcode == ISD::LIFETIME_END ||
04654           (Opcode <= INT_MAX &&
04655            (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
04656          "Opcode is not a memory-accessing opcode!");
04657 
04658   // Memoize the node unless it returns a flag.
04659   MemIntrinsicSDNode *N;
04660   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
04661     FoldingSetNodeID ID;
04662     AddNodeIDNode(ID, Opcode, VTList, Ops);
04663     ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04664     void *IP = nullptr;
04665     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04666       cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
04667       return SDValue(E, 0);
04668     }
04669 
04670     N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
04671                                                dl.getDebugLoc(), VTList, Ops,
04672                                                MemVT, MMO);
04673     CSEMap.InsertNode(N, IP);
04674   } else {
04675     N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
04676                                                dl.getDebugLoc(), VTList, Ops,
04677                                                MemVT, MMO);
04678   }
04679   InsertNode(N);
04680   return SDValue(N, 0);
04681 }
04682 
04683 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
04684 /// MachinePointerInfo record from it.  This is particularly useful because the
04685 /// code generator has many cases where it doesn't bother passing in a
04686 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
04687 static MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) {
04688   // If this is FI+Offset, we can model it.
04689   if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
04690     return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset);
04691 
04692   // If this is (FI+Offset1)+Offset2, we can model it.
04693   if (Ptr.getOpcode() != ISD::ADD ||
04694       !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
04695       !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
04696     return MachinePointerInfo();
04697 
04698   int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
04699   return MachinePointerInfo::getFixedStack(FI, Offset+
04700                        cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
04701 }
04702 
04703 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
04704 /// MachinePointerInfo record from it.  This is particularly useful because the
04705 /// code generator has many cases where it doesn't bother passing in a
04706 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
04707 static MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) {
04708   // If the 'Offset' value isn't a constant, we can't handle this.
04709   if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
04710     return InferPointerInfo(Ptr, OffsetNode->getSExtValue());
04711   if (OffsetOp.getOpcode() == ISD::UNDEF)
04712     return InferPointerInfo(Ptr);
04713   return MachinePointerInfo();
04714 }
04715 
04716 
04717 SDValue
04718 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
04719                       EVT VT, SDLoc dl, SDValue Chain,
04720                       SDValue Ptr, SDValue Offset,
04721                       MachinePointerInfo PtrInfo, EVT MemVT,
04722                       bool isVolatile, bool isNonTemporal, bool isInvariant,
04723                       unsigned Alignment, const AAMDNodes &AAInfo,
04724                       const MDNode *Ranges) {
04725   assert(Chain.getValueType() == MVT::Other &&
04726         "Invalid chain type");
04727   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04728     Alignment = getEVTAlignment(VT);
04729 
04730   unsigned Flags = MachineMemOperand::MOLoad;
04731   if (isVolatile)
04732     Flags |= MachineMemOperand::MOVolatile;
04733   if (isNonTemporal)
04734     Flags |= MachineMemOperand::MONonTemporal;
04735   if (isInvariant)
04736     Flags |= MachineMemOperand::MOInvariant;
04737 
04738   // If we don't have a PtrInfo, infer the trivial frame index case to simplify
04739   // clients.
04740   if (PtrInfo.V.isNull())
04741     PtrInfo = InferPointerInfo(Ptr, Offset);
04742 
04743   MachineFunction &MF = getMachineFunction();
04744   MachineMemOperand *MMO =
04745     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment,
04746                             AAInfo, Ranges);
04747   return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
04748 }
04749 
04750 SDValue
04751 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
04752                       EVT VT, SDLoc dl, SDValue Chain,
04753                       SDValue Ptr, SDValue Offset, EVT MemVT,
04754                       MachineMemOperand *MMO) {
04755   if (VT == MemVT) {
04756     ExtType = ISD::NON_EXTLOAD;
04757   } else if (ExtType == ISD::NON_EXTLOAD) {
04758     assert(VT == MemVT && "Non-extending load from different memory type!");
04759   } else {
04760     // Extending load.
04761     assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
04762            "Should only be an extending load, not truncating!");
04763     assert(VT.isInteger() == MemVT.isInteger() &&
04764            "Cannot convert from FP to Int or Int -> FP!");
04765     assert(VT.isVector() == MemVT.isVector() &&
04766            "Cannot use an ext load to convert to or from a vector!");
04767     assert((!VT.isVector() ||
04768             VT.getVectorNumElements() == MemVT.getVectorNumElements()) &&
04769            "Cannot use an ext load to change the number of vector elements!");
04770   }
04771 
04772   bool Indexed = AM != ISD::UNINDEXED;
04773   assert((Indexed || Offset.getOpcode() == ISD::UNDEF) &&
04774          "Unindexed load with an offset!");
04775 
04776   SDVTList VTs = Indexed ?
04777     getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
04778   SDValue Ops[] = { Chain, Ptr, Offset };
04779   FoldingSetNodeID ID;
04780   AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
04781   ID.AddInteger(MemVT.getRawBits());
04782   ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(),
04783                                      MMO->isNonTemporal(),
04784                                      MMO->isInvariant()));
04785   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04786   void *IP = nullptr;
04787   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04788     cast<LoadSDNode>(E)->refineAlignment(MMO);
04789     return SDValue(E, 0);
04790   }
04791   SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl.getIROrder(),
04792                                              dl.getDebugLoc(), VTs, AM, ExtType,
04793                                              MemVT, MMO);
04794   CSEMap.InsertNode(N, IP);
04795   InsertNode(N);
04796   return SDValue(N, 0);
04797 }
04798 
04799 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
04800                               SDValue Chain, SDValue Ptr,
04801                               MachinePointerInfo PtrInfo,
04802                               bool isVolatile, bool isNonTemporal,
04803                               bool isInvariant, unsigned Alignment,
04804                               const AAMDNodes &AAInfo,
04805                               const MDNode *Ranges) {
04806   SDValue Undef = getUNDEF(Ptr.getValueType());
04807   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
04808                  PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment,
04809                  AAInfo, Ranges);
04810 }
04811 
04812 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
04813                               SDValue Chain, SDValue Ptr,
04814                               MachineMemOperand *MMO) {
04815   SDValue Undef = getUNDEF(Ptr.getValueType());
04816   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
04817                  VT, MMO);
04818 }
04819 
04820 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
04821                                  SDValue Chain, SDValue Ptr,
04822                                  MachinePointerInfo PtrInfo, EVT MemVT,
04823                                  bool isVolatile, bool isNonTemporal,
04824                                  bool isInvariant, unsigned Alignment,
04825                                  const AAMDNodes &AAInfo) {
04826   SDValue Undef = getUNDEF(Ptr.getValueType());
04827   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
04828                  PtrInfo, MemVT, isVolatile, isNonTemporal, isInvariant,
04829                  Alignment, AAInfo);
04830 }
04831 
04832 
04833 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
04834                                  SDValue Chain, SDValue Ptr, EVT MemVT,
04835                                  MachineMemOperand *MMO) {
04836   SDValue Undef = getUNDEF(Ptr.getValueType());
04837   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
04838                  MemVT, MMO);
04839 }
04840 
04841 SDValue
04842 SelectionDAG::getIndexedLoad(SDValue OrigLoad, SDLoc dl, SDValue Base,
04843                              SDValue Offset, ISD::MemIndexedMode AM) {
04844   LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
04845   assert(LD->getOffset().getOpcode() == ISD::UNDEF &&
04846          "Load is already a indexed load!");
04847   return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
04848                  LD->getChain(), Base, Offset, LD->getPointerInfo(),
04849                  LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(),
04850                  false, LD->getAlignment());
04851 }
04852 
04853 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
04854                                SDValue Ptr, MachinePointerInfo PtrInfo,
04855                                bool isVolatile, bool isNonTemporal,
04856                                unsigned Alignment, const AAMDNodes &AAInfo) {
04857   assert(Chain.getValueType() == MVT::Other &&
04858         "Invalid chain type");
04859   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04860     Alignment = getEVTAlignment(Val.getValueType());
04861 
04862   unsigned Flags = MachineMemOperand::MOStore;
04863   if (isVolatile)
04864     Flags |= MachineMemOperand::MOVolatile;
04865   if (isNonTemporal)
04866     Flags |= MachineMemOperand::MONonTemporal;
04867 
04868   if (PtrInfo.V.isNull())
04869     PtrInfo = InferPointerInfo(Ptr);
04870 
04871   MachineFunction &MF = getMachineFunction();
04872   MachineMemOperand *MMO =
04873     MF.getMachineMemOperand(PtrInfo, Flags,
04874                             Val.getValueType().getStoreSize(), Alignment,
04875                             AAInfo);
04876 
04877   return getStore(Chain, dl, Val, Ptr, MMO);
04878 }
04879 
04880 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
04881                                SDValue Ptr, MachineMemOperand *MMO) {
04882   assert(Chain.getValueType() == MVT::Other &&
04883         "Invalid chain type");
04884   EVT VT = Val.getValueType();
04885   SDVTList VTs = getVTList(MVT::Other);
04886   SDValue Undef = getUNDEF(Ptr.getValueType());
04887   SDValue Ops[] = { Chain, Val, Ptr, Undef };
04888   FoldingSetNodeID ID;
04889   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04890   ID.AddInteger(VT.getRawBits());
04891   ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
04892                                      MMO->isNonTemporal(), MMO->isInvariant()));
04893   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04894   void *IP = nullptr;
04895   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04896     cast<StoreSDNode>(E)->refineAlignment(MMO);
04897     return SDValue(E, 0);
04898   }
04899   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04900                                               dl.getDebugLoc(), VTs,
04901                                               ISD::UNINDEXED, false, VT, MMO);
04902   CSEMap.InsertNode(N, IP);
04903   InsertNode(N);
04904   return SDValue(N, 0);
04905 }
04906 
04907 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
04908                                     SDValue Ptr, MachinePointerInfo PtrInfo,
04909                                     EVT SVT,bool isVolatile, bool isNonTemporal,
04910                                     unsigned Alignment,
04911                                     const AAMDNodes &AAInfo) {
04912   assert(Chain.getValueType() == MVT::Other &&
04913         "Invalid chain type");
04914   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04915     Alignment = getEVTAlignment(SVT);
04916 
04917   unsigned Flags = MachineMemOperand::MOStore;
04918   if (isVolatile)
04919     Flags |= MachineMemOperand::MOVolatile;
04920   if (isNonTemporal)
04921     Flags |= MachineMemOperand::MONonTemporal;
04922 
04923   if (PtrInfo.V.isNull())
04924     PtrInfo = InferPointerInfo(Ptr);
04925 
04926   MachineFunction &MF = getMachineFunction();
04927   MachineMemOperand *MMO =
04928     MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment,
04929                             AAInfo);
04930 
04931   return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
04932 }
04933 
04934 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
04935                                     SDValue Ptr, EVT SVT,
04936                                     MachineMemOperand *MMO) {
04937   EVT VT = Val.getValueType();
04938 
04939   assert(Chain.getValueType() == MVT::Other &&
04940         "Invalid chain type");
04941   if (VT == SVT)
04942     return getStore(Chain, dl, Val, Ptr, MMO);
04943 
04944   assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
04945          "Should only be a truncating store, not extending!");
04946   assert(VT.isInteger() == SVT.isInteger() &&
04947          "Can't do FP-INT conversion!");
04948   assert(VT.isVector() == SVT.isVector() &&
04949          "Cannot use trunc store to convert to or from a vector!");
04950   assert((!VT.isVector() ||
04951           VT.getVectorNumElements() == SVT.getVectorNumElements()) &&
04952          "Cannot use trunc store to change the number of vector elements!");
04953 
04954   SDVTList VTs = getVTList(MVT::Other);
04955   SDValue Undef = getUNDEF(Ptr.getValueType());
04956   SDValue Ops[] = { Chain, Val, Ptr, Undef };
04957   FoldingSetNodeID ID;
04958   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04959   ID.AddInteger(SVT.getRawBits());
04960   ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(),
04961                                      MMO->isNonTemporal(), MMO->isInvariant()));
04962   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04963   void *IP = nullptr;
04964   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04965     cast<StoreSDNode>(E)->refineAlignment(MMO);
04966     return SDValue(E, 0);
04967   }
04968   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04969                                               dl.getDebugLoc(), VTs,
04970                                               ISD::UNINDEXED, true, SVT, MMO);
04971   CSEMap.InsertNode(N, IP);
04972   InsertNode(N);
04973   return SDValue(N, 0);
04974 }
04975 
04976 SDValue
04977 SelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base,
04978                               SDValue Offset, ISD::MemIndexedMode AM) {
04979   StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
04980   assert(ST->getOffset().getOpcode() == ISD::UNDEF &&
04981          "Store is already a indexed store!");
04982   SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
04983   SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
04984   FoldingSetNodeID ID;
04985   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04986   ID.AddInteger(ST->getMemoryVT().getRawBits());
04987   ID.AddInteger(ST->getRawSubclassData());
04988   ID.AddInteger(ST->getPointerInfo().getAddrSpace());
04989   void *IP = nullptr;
04990   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
04991     return SDValue(E, 0);
04992 
04993   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04994                                               dl.getDebugLoc(), VTs, AM,
04995                                               ST->isTruncatingStore(),
04996                                               ST->getMemoryVT(),
04997                                               ST->getMemOperand());
04998   CSEMap.InsertNode(N, IP);
04999   InsertNode(N);
05000   return SDValue(N, 0);
05001 }
05002 
05003 SDValue
05004 SelectionDAG::getMaskedLoad(EVT VT, SDLoc dl, SDValue Chain,
05005                             SDValue Ptr, SDValue Mask, SDValue Src0, EVT MemVT,
05006                             MachineMemOperand *MMO, ISD::LoadExtType ExtTy) {
05007 
05008   SDVTList VTs = getVTList(VT, MVT::Other);
05009   SDValue Ops[] = { Chain, Ptr, Mask, Src0 };
05010   FoldingSetNodeID ID;
05011   AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops);
05012   ID.AddInteger(VT.getRawBits());
05013   ID.AddInteger(encodeMemSDNodeFlags(ExtTy, ISD::UNINDEXED,
05014                                      MMO->isVolatile(),
05015                                      MMO->isNonTemporal(),
05016                                      MMO->isInvariant()));
05017   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
05018   void *IP = nullptr;
05019   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
05020     cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
05021     return SDValue(E, 0);
05022   }
05023   SDNode *N = new (NodeAllocator) MaskedLoadSDNode(dl.getIROrder(),
05024                                              dl.getDebugLoc(), Ops, 4, VTs,
05025                                              ExtTy, MemVT, MMO);
05026   CSEMap.InsertNode(N, IP);
05027   InsertNode(N);
05028   return SDValue(N, 0);
05029 }
05030 
05031 SDValue SelectionDAG::getMaskedStore(SDValue Chain, SDLoc dl, SDValue Val,
05032                                      SDValue Ptr, SDValue Mask, EVT MemVT,
05033                                      MachineMemOperand *MMO, bool isTrunc) {
05034   assert(Chain.getValueType() == MVT::Other &&
05035         "Invalid chain type");
05036   EVT VT = Val.getValueType();
05037   SDVTList VTs = getVTList(MVT::Other);
05038   SDValue Ops[] = { Chain, Ptr, Mask, Val };
05039   FoldingSetNodeID ID;
05040   AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops);
05041   ID.AddInteger(VT.getRawBits());
05042   ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
05043                                      MMO->isNonTemporal(), MMO->isInvariant()));
05044   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
05045   void *IP = nullptr;
05046   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
05047     cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
05048     return SDValue(E, 0);
05049   }
05050   SDNode *N = new (NodeAllocator) MaskedStoreSDNode(dl.getIROrder(),
05051                                                     dl.getDebugLoc(), Ops, 4,
05052                                                     VTs, isTrunc, MemVT, MMO);
05053   CSEMap.InsertNode(N, IP);
05054   InsertNode(N);
05055   return SDValue(N, 0);
05056 }
05057 
05058 SDValue SelectionDAG::getVAArg(EVT VT, SDLoc dl,
05059                                SDValue Chain, SDValue Ptr,
05060                                SDValue SV,
05061                                unsigned Align) {
05062   SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) };
05063   return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
05064 }
05065 
05066 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
05067                               ArrayRef<SDUse> Ops) {
05068   switch (Ops.size()) {
05069   case 0: return getNode(Opcode, DL, VT);
05070   case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
05071   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
05072   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
05073   default: break;
05074   }
05075 
05076   // Copy from an SDUse array into an SDValue array for use with
05077   // the regular getNode logic.
05078   SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
05079   return getNode(Opcode, DL, VT, NewOps);
05080 }
05081 
05082 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
05083                               ArrayRef<SDValue> Ops) {
05084   unsigned NumOps = Ops.size();
05085   switch (NumOps) {
05086   case 0: return getNode(Opcode, DL, VT);
05087   case 1: return getNode(Opcode, DL, VT, Ops[0]);
05088   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
05089   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
05090   default: break;
05091   }
05092 
05093   switch (Opcode) {
05094   default: break;
05095   case ISD::SELECT_CC: {
05096     assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
05097     assert(Ops[0].getValueType() == Ops[1].getValueType() &&
05098            "LHS and RHS of condition must have same type!");
05099     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
05100            "True and False arms of SelectCC must have same type!");
05101     assert(Ops[2].getValueType() == VT &&
05102            "select_cc node must be of same type as true and false value!");
05103     break;
05104   }
05105   case ISD::BR_CC: {
05106     assert(NumOps == 5 && "BR_CC takes 5 operands!");
05107     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
05108            "LHS/RHS of comparison should match types!");
05109     break;
05110   }
05111   }
05112 
05113   // Memoize nodes.
05114   SDNode *N;
05115   SDVTList VTs = getVTList(VT);
05116 
05117   if (VT != MVT::Glue) {
05118     FoldingSetNodeID ID;
05119     AddNodeIDNode(ID, Opcode, VTs, Ops);
05120     void *IP = nullptr;
05121 
05122     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05123       return SDValue(E, 0);
05124 
05125     N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05126                                    VTs, Ops);
05127     CSEMap.InsertNode(N, IP);
05128   } else {
05129     N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05130                                    VTs, Ops);
05131   }
05132 
05133   InsertNode(N);
05134   return SDValue(N, 0);
05135 }
05136 
05137 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
05138                               ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
05139   return getNode(Opcode, DL, getVTList(ResultTys), Ops);
05140 }
05141 
05142 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05143                               ArrayRef<SDValue> Ops) {
05144   if (VTList.NumVTs == 1)
05145     return getNode(Opcode, DL, VTList.VTs[0], Ops);
05146 
05147 #if 0
05148   switch (Opcode) {
05149   // FIXME: figure out how to safely handle things like
05150   // int foo(int x) { return 1 << (x & 255); }
05151   // int bar() { return foo(256); }
05152   case ISD::SRA_PARTS:
05153   case ISD::SRL_PARTS:
05154   case ISD::SHL_PARTS:
05155     if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
05156         cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
05157       return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
05158     else if (N3.getOpcode() == ISD::AND)
05159       if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
05160         // If the and is only masking out bits that cannot effect the shift,
05161         // eliminate the and.
05162         unsigned NumBits = VT.getScalarType().getSizeInBits()*2;
05163         if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
05164           return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
05165       }
05166     break;
05167   }
05168 #endif
05169 
05170   // Memoize the node unless it returns a flag.
05171   SDNode *N;
05172   unsigned NumOps = Ops.size();
05173   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
05174     FoldingSetNodeID ID;
05175     AddNodeIDNode(ID, Opcode, VTList, Ops);
05176     void *IP = nullptr;
05177     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05178       return SDValue(E, 0);
05179 
05180     if (NumOps == 1) {
05181       N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
05182                                           DL.getDebugLoc(), VTList, Ops[0]);
05183     } else if (NumOps == 2) {
05184       N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
05185                                            DL.getDebugLoc(), VTList, Ops[0],
05186                                            Ops[1]);
05187     } else if (NumOps == 3) {
05188       N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
05189                                             DL.getDebugLoc(), VTList, Ops[0],
05190                                             Ops[1], Ops[2]);
05191     } else {
05192       N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05193                                      VTList, Ops);
05194     }
05195     CSEMap.InsertNode(N, IP);
05196   } else {
05197     if (NumOps == 1) {
05198       N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
05199                                           DL.getDebugLoc(), VTList, Ops[0]);
05200     } else if (NumOps == 2) {
05201       N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
05202                                            DL.getDebugLoc(), VTList, Ops[0],
05203                                            Ops[1]);
05204     } else if (NumOps == 3) {
05205       N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
05206                                             DL.getDebugLoc(), VTList, Ops[0],
05207                                             Ops[1], Ops[2]);
05208     } else {
05209       N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05210                                      VTList, Ops);
05211     }
05212   }
05213   InsertNode(N);
05214   return SDValue(N, 0);
05215 }
05216 
05217 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList) {
05218   return getNode(Opcode, DL, VTList, None);
05219 }
05220 
05221 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05222                               SDValue N1) {
05223   SDValue Ops[] = { N1 };
05224   return getNode(Opcode, DL, VTList, Ops);
05225 }
05226 
05227 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05228                               SDValue N1, SDValue N2) {
05229   SDValue Ops[] = { N1, N2 };
05230   return getNode(Opcode, DL, VTList, Ops);
05231 }
05232 
05233 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05234                               SDValue N1, SDValue N2, SDValue N3) {
05235   SDValue Ops[] = { N1, N2, N3 };
05236   return getNode(Opcode, DL, VTList, Ops);
05237 }
05238 
05239 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05240                               SDValue N1, SDValue N2, SDValue N3,
05241                               SDValue N4) {
05242   SDValue Ops[] = { N1, N2, N3, N4 };
05243   return getNode(Opcode, DL, VTList, Ops);
05244 }
05245 
05246 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05247                               SDValue N1, SDValue N2, SDValue N3,
05248                               SDValue N4, SDValue N5) {
05249   SDValue Ops[] = { N1, N2, N3, N4, N5 };
05250   return getNode(Opcode, DL, VTList, Ops);
05251 }
05252 
05253 SDVTList SelectionDAG::getVTList(EVT VT) {
05254   return makeVTList(SDNode::getValueTypeList(VT), 1);
05255 }
05256 
05257 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
05258   FoldingSetNodeID ID;
05259   ID.AddInteger(2U);
05260   ID.AddInteger(VT1.getRawBits());
05261   ID.AddInteger(VT2.getRawBits());
05262 
05263   void *IP = nullptr;
05264   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05265   if (!Result) {
05266     EVT *Array = Allocator.Allocate<EVT>(2);
05267     Array[0] = VT1;
05268     Array[1] = VT2;
05269     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
05270     VTListMap.InsertNode(Result, IP);
05271   }
05272   return Result->getSDVTList();
05273 }
05274 
05275 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
05276   FoldingSetNodeID ID;
05277   ID.AddInteger(3U);
05278   ID.AddInteger(VT1.getRawBits());
05279   ID.AddInteger(VT2.getRawBits());
05280   ID.AddInteger(VT3.getRawBits());
05281 
05282   void *IP = nullptr;
05283   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05284   if (!Result) {
05285     EVT *Array = Allocator.Allocate<EVT>(3);
05286     Array[0] = VT1;
05287     Array[1] = VT2;
05288     Array[2] = VT3;
05289     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
05290     VTListMap.InsertNode(Result, IP);
05291   }
05292   return Result->getSDVTList();
05293 }
05294 
05295 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
05296   FoldingSetNodeID ID;
05297   ID.AddInteger(4U);
05298   ID.AddInteger(VT1.getRawBits());
05299   ID.AddInteger(VT2.getRawBits());
05300   ID.AddInteger(VT3.getRawBits());
05301   ID.AddInteger(VT4.getRawBits());
05302 
05303   void *IP = nullptr;
05304   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05305   if (!Result) {
05306     EVT *Array = Allocator.Allocate<EVT>(4);
05307     Array[0] = VT1;
05308     Array[1] = VT2;
05309     Array[2] = VT3;
05310     Array[3] = VT4;
05311     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
05312     VTListMap.InsertNode(Result, IP);
05313   }
05314   return Result->getSDVTList();
05315 }
05316 
05317 SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
05318   unsigned NumVTs = VTs.size();
05319   FoldingSetNodeID ID;
05320   ID.AddInteger(NumVTs);
05321   for (unsigned index = 0; index < NumVTs; index++) {
05322     ID.AddInteger(VTs[index].getRawBits());
05323   }
05324 
05325   void *IP = nullptr;
05326   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05327   if (!Result) {
05328     EVT *Array = Allocator.Allocate<EVT>(NumVTs);
05329     std::copy(VTs.begin(), VTs.end(), Array);
05330     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
05331     VTListMap.InsertNode(Result, IP);
05332   }
05333   return Result->getSDVTList();
05334 }
05335 
05336 
05337 /// UpdateNodeOperands - *Mutate* the specified node in-place to have the
05338 /// specified operands.  If the resultant node already exists in the DAG,
05339 /// this does not modify the specified node, instead it returns the node that
05340 /// already exists.  If the resultant node does not exist in the DAG, the
05341 /// input node is returned.  As a degenerate case, if you specify the same
05342 /// input operands as the node already has, the input node is returned.
05343 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
05344   assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
05345 
05346   // Check to see if there is no change.
05347   if (Op == N->getOperand(0)) return N;
05348 
05349   // See if the modified node already exists.
05350   void *InsertPos = nullptr;
05351   if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
05352     return Existing;
05353 
05354   // Nope it doesn't.  Remove the node from its current place in the maps.
05355   if (InsertPos)
05356     if (!RemoveNodeFromCSEMaps(N))
05357       InsertPos = nullptr;
05358 
05359   // Now we update the operands.
05360   N->OperandList[0].set(Op);
05361 
05362   // If this gets put into a CSE map, add it.
05363   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05364   return N;
05365 }
05366 
05367 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
05368   assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
05369 
05370   // Check to see if there is no change.
05371   if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
05372     return N;   // No operands changed, just return the input node.
05373 
05374   // See if the modified node already exists.
05375   void *InsertPos = nullptr;
05376   if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
05377     return Existing;
05378 
05379   // Nope it doesn't.  Remove the node from its current place in the maps.
05380   if (InsertPos)
05381     if (!RemoveNodeFromCSEMaps(N))
05382       InsertPos = nullptr;
05383 
05384   // Now we update the operands.
05385   if (N->OperandList[0] != Op1)
05386     N->OperandList[0].set(Op1);
05387   if (N->OperandList[1] != Op2)
05388     N->OperandList[1].set(Op2);
05389 
05390   // If this gets put into a CSE map, add it.
05391   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05392   return N;
05393 }
05394 
05395 SDNode *SelectionDAG::
05396 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
05397   SDValue Ops[] = { Op1, Op2, Op3 };
05398   return UpdateNodeOperands(N, Ops);
05399 }
05400 
05401 SDNode *SelectionDAG::
05402 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
05403                    SDValue Op3, SDValue Op4) {
05404   SDValue Ops[] = { Op1, Op2, Op3, Op4 };
05405   return UpdateNodeOperands(N, Ops);
05406 }
05407 
05408 SDNode *SelectionDAG::
05409 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
05410                    SDValue Op3, SDValue Op4, SDValue Op5) {
05411   SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
05412   return UpdateNodeOperands(N, Ops);
05413 }
05414 
05415 SDNode *SelectionDAG::
05416 UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
05417   unsigned NumOps = Ops.size();
05418   assert(N->getNumOperands() == NumOps &&
05419          "Update with wrong number of operands");
05420 
05421   // If no operands changed just return the input node.
05422   if (Ops.empty() || std::equal(Ops.begin(), Ops.end(), N->op_begin()))
05423     return N;
05424 
05425   // See if the modified node already exists.
05426   void *InsertPos = nullptr;
05427   if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
05428     return Existing;
05429 
05430   // Nope it doesn't.  Remove the node from its current place in the maps.
05431   if (InsertPos)
05432     if (!RemoveNodeFromCSEMaps(N))
05433       InsertPos = nullptr;
05434 
05435   // Now we update the operands.
05436   for (unsigned i = 0; i != NumOps; ++i)
05437     if (N->OperandList[i] != Ops[i])
05438       N->OperandList[i].set(Ops[i]);
05439 
05440   // If this gets put into a CSE map, add it.
05441   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05442   return N;
05443 }
05444 
05445 /// DropOperands - Release the operands and set this node to have
05446 /// zero operands.
05447 void SDNode::DropOperands() {
05448   // Unlike the code in MorphNodeTo that does this, we don't need to
05449   // watch for dead nodes here.
05450   for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
05451     SDUse &Use = *I++;
05452     Use.set(SDValue());
05453   }
05454 }
05455 
05456 /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
05457 /// machine opcode.
05458 ///
05459 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05460                                    EVT VT) {
05461   SDVTList VTs = getVTList(VT);
05462   return SelectNodeTo(N, MachineOpc, VTs, None);
05463 }
05464 
05465 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05466                                    EVT VT, SDValue Op1) {
05467   SDVTList VTs = getVTList(VT);
05468   SDValue Ops[] = { Op1 };
05469   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05470 }
05471 
05472 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05473                                    EVT VT, SDValue Op1,
05474                                    SDValue Op2) {
05475   SDVTList VTs = getVTList(VT);
05476   SDValue Ops[] = { Op1, Op2 };
05477   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05478 }
05479 
05480 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05481                                    EVT VT, SDValue Op1,
05482                                    SDValue Op2, SDValue Op3) {
05483   SDVTList VTs = getVTList(VT);
05484   SDValue Ops[] = { Op1, Op2, Op3 };
05485   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05486 }
05487 
05488 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05489                                    EVT VT, ArrayRef<SDValue> Ops) {
05490   SDVTList VTs = getVTList(VT);
05491   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05492 }
05493 
05494 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05495                                    EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
05496   SDVTList VTs = getVTList(VT1, VT2);
05497   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05498 }
05499 
05500 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05501                                    EVT VT1, EVT VT2) {
05502   SDVTList VTs = getVTList(VT1, VT2);
05503   return SelectNodeTo(N, MachineOpc, VTs, None);
05504 }
05505 
05506 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05507                                    EVT VT1, EVT VT2, EVT VT3,
05508                                    ArrayRef<SDValue> Ops) {
05509   SDVTList VTs = getVTList(VT1, VT2, VT3);
05510   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05511 }
05512 
05513 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05514                                    EVT VT1, EVT VT2, EVT VT3, EVT VT4,
05515                                    ArrayRef<SDValue> Ops) {
05516   SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
05517   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05518 }
05519 
05520 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05521                                    EVT VT1, EVT VT2,
05522                                    SDValue Op1) {
05523   SDVTList VTs = getVTList(VT1, VT2);
05524   SDValue Ops[] = { Op1 };
05525   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05526 }
05527 
05528 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05529                                    EVT VT1, EVT VT2,
05530                                    SDValue Op1, SDValue Op2) {
05531   SDVTList VTs = getVTList(VT1, VT2);
05532   SDValue Ops[] = { Op1, Op2 };
05533   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05534 }
05535 
05536 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05537                                    EVT VT1, EVT VT2,
05538                                    SDValue Op1, SDValue Op2,
05539                                    SDValue Op3) {
05540   SDVTList VTs = getVTList(VT1, VT2);
05541   SDValue Ops[] = { Op1, Op2, Op3 };
05542   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05543 }
05544 
05545 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05546                                    EVT VT1, EVT VT2, EVT VT3,
05547                                    SDValue Op1, SDValue Op2,
05548                                    SDValue Op3) {
05549   SDVTList VTs = getVTList(VT1, VT2, VT3);
05550   SDValue Ops[] = { Op1, Op2, Op3 };
05551   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05552 }
05553 
05554 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05555                                    SDVTList VTs,ArrayRef<SDValue> Ops) {
05556   N = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
05557   // Reset the NodeID to -1.
05558   N->setNodeId(-1);
05559   return N;
05560 }
05561 
05562 /// UpdadeSDLocOnMergedSDNode - If the opt level is -O0 then it throws away
05563 /// the line number information on the merged node since it is not possible to
05564 /// preserve the information that operation is associated with multiple lines.
05565 /// This will make the debugger working better at -O0, were there is a higher
05566 /// probability having other instructions associated with that line.
05567 ///
05568 /// For IROrder, we keep the smaller of the two
05569 SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) {
05570   DebugLoc NLoc = N->getDebugLoc();
05571   if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) &&
05572     (OLoc.getDebugLoc() != NLoc)) {
05573     N->setDebugLoc(DebugLoc());
05574   }
05575   unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
05576   N->setIROrder(Order);
05577   return N;
05578 }
05579 
05580 /// MorphNodeTo - This *mutates* the specified node to have the specified
05581 /// return type, opcode, and operands.
05582 ///
05583 /// Note that MorphNodeTo returns the resultant node.  If there is already a
05584 /// node of the specified opcode and operands, it returns that node instead of
05585 /// the current one.  Note that the SDLoc need not be the same.
05586 ///
05587 /// Using MorphNodeTo is faster than creating a new node and swapping it in
05588 /// with ReplaceAllUsesWith both because it often avoids allocating a new
05589 /// node, and because it doesn't require CSE recalculation for any of
05590 /// the node's users.
05591 ///
05592 /// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
05593 /// As a consequence it isn't appropriate to use from within the DAG combiner or
05594 /// the legalizer which maintain worklists that would need to be updated when
05595 /// deleting things.
05596 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
05597                                   SDVTList VTs, ArrayRef<SDValue> Ops) {
05598   unsigned NumOps = Ops.size();
05599   // If an identical node already exists, use it.
05600   void *IP = nullptr;
05601   if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
05602     FoldingSetNodeID ID;
05603     AddNodeIDNode(ID, Opc, VTs, Ops);
05604     if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
05605       return UpdadeSDLocOnMergedSDNode(ON, SDLoc(N));
05606   }
05607 
05608   if (!RemoveNodeFromCSEMaps(N))
05609     IP = nullptr;
05610 
05611   // Start the morphing.
05612   N->NodeType = Opc;
05613   N->ValueList = VTs.VTs;
05614   N->NumValues = VTs.NumVTs;
05615 
05616   // Clear the operands list, updating used nodes to remove this from their
05617   // use list.  Keep track of any operands that become dead as a result.
05618   SmallPtrSet<SDNode*, 16> DeadNodeSet;
05619   for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
05620     SDUse &Use = *I++;
05621     SDNode *Used = Use.getNode();
05622     Use.set(SDValue());
05623     if (Used->use_empty())
05624       DeadNodeSet.insert(Used);
05625   }
05626 
05627   if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) {
05628     // Initialize the memory references information.
05629     MN->setMemRefs(nullptr, nullptr);
05630     // If NumOps is larger than the # of operands we can have in a
05631     // MachineSDNode, reallocate the operand list.
05632     if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) {
05633       if (MN->OperandsNeedDelete)
05634         delete[] MN->OperandList;
05635       if (NumOps > array_lengthof(MN->LocalOperands))
05636         // We're creating a final node that will live unmorphed for the
05637         // remainder of the current SelectionDAG iteration, so we can allocate
05638         // the operands directly out of a pool with no recycling metadata.
05639         MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
05640                          Ops.data(), NumOps);
05641       else
05642         MN->InitOperands(MN->LocalOperands, Ops.data(), NumOps);
05643       MN->OperandsNeedDelete = false;
05644     } else
05645       MN->InitOperands(MN->OperandList, Ops.data(), NumOps);
05646   } else {
05647     // If NumOps is larger than the # of operands we currently have, reallocate
05648     // the operand list.
05649     if (NumOps > N->NumOperands) {
05650       if (N->OperandsNeedDelete)
05651         delete[] N->OperandList;
05652       N->InitOperands(new SDUse[NumOps], Ops.data(), NumOps);
05653       N->OperandsNeedDelete = true;
05654     } else
05655       N->InitOperands(N->OperandList, Ops.data(), NumOps);
05656   }
05657 
05658   // Delete any nodes that are still dead after adding the uses for the
05659   // new operands.
05660   if (!DeadNodeSet.empty()) {
05661     SmallVector<SDNode *, 16> DeadNodes;
05662     for (SDNode *N : DeadNodeSet)
05663       if (N->use_empty())
05664         DeadNodes.push_back(N);
05665     RemoveDeadNodes(DeadNodes);
05666   }
05667 
05668   if (IP)
05669     CSEMap.InsertNode(N, IP);   // Memoize the new node.
05670   return N;
05671 }
05672 
05673 
05674 /// getMachineNode - These are used for target selectors to create a new node
05675 /// with specified return type(s), MachineInstr opcode, and operands.
05676 ///
05677 /// Note that getMachineNode returns the resultant node.  If there is already a
05678 /// node of the specified opcode and operands, it returns that node instead of
05679 /// the current one.
05680 MachineSDNode *
05681 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT) {
05682   SDVTList VTs = getVTList(VT);
05683   return getMachineNode(Opcode, dl, VTs, None);
05684 }
05685 
05686 MachineSDNode *
05687 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, SDValue Op1) {
05688   SDVTList VTs = getVTList(VT);
05689   SDValue Ops[] = { Op1 };
05690   return getMachineNode(Opcode, dl, VTs, Ops);
05691 }
05692 
05693 MachineSDNode *
05694 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05695                              SDValue Op1, SDValue Op2) {
05696   SDVTList VTs = getVTList(VT);
05697   SDValue Ops[] = { Op1, Op2 };
05698   return getMachineNode(Opcode, dl, VTs, Ops);
05699 }
05700 
05701 MachineSDNode *
05702 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05703                              SDValue Op1, SDValue Op2, SDValue Op3) {
05704   SDVTList VTs = getVTList(VT);
05705   SDValue Ops[] = { Op1, Op2, Op3 };
05706   return getMachineNode(Opcode, dl, VTs, Ops);
05707 }
05708 
05709 MachineSDNode *
05710 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05711                              ArrayRef<SDValue> Ops) {
05712   SDVTList VTs = getVTList(VT);
05713   return getMachineNode(Opcode, dl, VTs, Ops);
05714 }
05715 
05716 MachineSDNode *
05717 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, EVT VT2) {
05718   SDVTList VTs = getVTList(VT1, VT2);
05719   return getMachineNode(Opcode, dl, VTs, None);
05720 }
05721 
05722 MachineSDNode *
05723 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05724                              EVT VT1, EVT VT2, SDValue Op1) {
05725   SDVTList VTs = getVTList(VT1, VT2);
05726   SDValue Ops[] = { Op1 };
05727   return getMachineNode(Opcode, dl, VTs, Ops);
05728 }
05729 
05730 MachineSDNode *
05731 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05732                              EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) {
05733   SDVTList VTs = getVTList(VT1, VT2);
05734   SDValue Ops[] = { Op1, Op2 };
05735   return getMachineNode(Opcode, dl, VTs, Ops);
05736 }
05737 
05738 MachineSDNode *
05739 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05740                              EVT VT1, EVT VT2, SDValue Op1,
05741                              SDValue Op2, SDValue Op3) {
05742   SDVTList VTs = getVTList(VT1, VT2);
05743   SDValue Ops[] = { Op1, Op2, Op3 };
05744   return getMachineNode(Opcode, dl, VTs, Ops);
05745 }
05746 
05747 MachineSDNode *
05748 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05749                              EVT VT1, EVT VT2,
05750                              ArrayRef<SDValue> Ops) {
05751   SDVTList VTs = getVTList(VT1, VT2);
05752   return getMachineNode(Opcode, dl, VTs, Ops);
05753 }
05754 
05755 MachineSDNode *
05756 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05757                              EVT VT1, EVT VT2, EVT VT3,
05758                              SDValue Op1, SDValue Op2) {
05759   SDVTList VTs = getVTList(VT1, VT2, VT3);
05760   SDValue Ops[] = { Op1, Op2 };
05761   return getMachineNode(Opcode, dl, VTs, Ops);
05762 }
05763 
05764 MachineSDNode *
05765 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05766                              EVT VT1, EVT VT2, EVT VT3,
05767                              SDValue Op1, SDValue Op2, SDValue Op3) {
05768   SDVTList VTs = getVTList(VT1, VT2, VT3);
05769   SDValue Ops[] = { Op1, Op2, Op3 };
05770   return getMachineNode(Opcode, dl, VTs, Ops);
05771 }
05772 
05773 MachineSDNode *
05774 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05775                              EVT VT1, EVT VT2, EVT VT3,
05776                              ArrayRef<SDValue> Ops) {
05777   SDVTList VTs = getVTList(VT1, VT2, VT3);
05778   return getMachineNode(Opcode, dl, VTs, Ops);
05779 }
05780 
05781 MachineSDNode *
05782 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1,
05783                              EVT VT2, EVT VT3, EVT VT4,
05784                              ArrayRef<SDValue> Ops) {
05785   SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
05786   return getMachineNode(Opcode, dl, VTs, Ops);
05787 }
05788 
05789 MachineSDNode *
05790 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05791                              ArrayRef<EVT> ResultTys,
05792                              ArrayRef<SDValue> Ops) {
05793   SDVTList VTs = getVTList(ResultTys);
05794   return getMachineNode(Opcode, dl, VTs, Ops);
05795 }
05796 
05797 MachineSDNode *
05798 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
05799                              ArrayRef<SDValue> OpsArray) {
05800   bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
05801   MachineSDNode *N;
05802   void *IP = nullptr;
05803   const SDValue *Ops = OpsArray.data();
05804   unsigned NumOps = OpsArray.size();
05805 
05806   if (DoCSE) {
05807     FoldingSetNodeID ID;
05808     AddNodeIDNode(ID, ~Opcode, VTs, OpsArray);
05809     IP = nullptr;
05810     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
05811       return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL));
05812     }
05813   }
05814 
05815   // Allocate a new MachineSDNode.
05816   N = new (NodeAllocator) MachineSDNode(~Opcode, DL.getIROrder(),
05817                                         DL.getDebugLoc(), VTs);
05818 
05819   // Initialize the operands list.
05820   if (NumOps > array_lengthof(N->LocalOperands))
05821     // We're creating a final node that will live unmorphed for the
05822     // remainder of the current SelectionDAG iteration, so we can allocate
05823     // the operands directly out of a pool with no recycling metadata.
05824     N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
05825                     Ops, NumOps);
05826   else
05827     N->InitOperands(N->LocalOperands, Ops, NumOps);
05828   N->OperandsNeedDelete = false;
05829 
05830   if (DoCSE)
05831     CSEMap.InsertNode(N, IP);
05832 
05833   InsertNode(N);
05834   return N;
05835 }
05836 
05837 /// getTargetExtractSubreg - A convenience function for creating
05838 /// TargetOpcode::EXTRACT_SUBREG nodes.
05839 SDValue
05840 SelectionDAG::getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT,
05841                                      SDValue Operand) {
05842   SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
05843   SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
05844                                   VT, Operand, SRIdxVal);
05845   return SDValue(Subreg, 0);
05846 }
05847 
05848 /// getTargetInsertSubreg - A convenience function for creating
05849 /// TargetOpcode::INSERT_SUBREG nodes.
05850 SDValue
05851 SelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT,
05852                                     SDValue Operand, SDValue Subreg) {
05853   SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
05854   SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
05855                                   VT, Operand, Subreg, SRIdxVal);
05856   return SDValue(Result, 0);
05857 }
05858 
05859 /// getNodeIfExists - Get the specified node if it's already available, or
05860 /// else return NULL.
05861 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
05862                                       ArrayRef<SDValue> Ops, bool nuw, bool nsw,
05863                                       bool exact) {
05864   if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
05865     FoldingSetNodeID ID;
05866     AddNodeIDNode(ID, Opcode, VTList, Ops);
05867     if (isBinOpWithFlags(Opcode))
05868       AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
05869     void *IP = nullptr;
05870     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05871       return E;
05872   }
05873   return nullptr;
05874 }
05875 
05876 /// getDbgValue - Creates a SDDbgValue node.
05877 ///
05878 /// SDNode
05879 SDDbgValue *SelectionDAG::getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N,
05880                                       unsigned R, bool IsIndirect, uint64_t Off,
05881                                       DebugLoc DL, unsigned O) {
05882   return new (Allocator) SDDbgValue(Var, Expr, N, R, IsIndirect, Off, DL, O);
05883 }
05884 
05885 /// Constant
05886 SDDbgValue *SelectionDAG::getConstantDbgValue(MDNode *Var, MDNode *Expr,
05887                                               const Value *C, uint64_t Off,
05888                                               DebugLoc DL, unsigned O) {
05889   return new (Allocator) SDDbgValue(Var, Expr, C, Off, DL, O);
05890 }
05891 
05892 /// FrameIndex
05893 SDDbgValue *SelectionDAG::getFrameIndexDbgValue(MDNode *Var, MDNode *Expr,
05894                                                 unsigned FI, uint64_t Off,
05895                                                 DebugLoc DL, unsigned O) {
05896   return new (Allocator) SDDbgValue(Var, Expr, FI, Off, DL, O);
05897 }
05898 
05899 namespace {
05900 
05901 /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
05902 /// pointed to by a use iterator is deleted, increment the use iterator
05903 /// so that it doesn't dangle.
05904 ///
05905 class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
05906   SDNode::use_iterator &UI;
05907   SDNode::use_iterator &UE;
05908 
05909   void NodeDeleted(SDNode *N, SDNode *E) override {
05910     // Increment the iterator as needed.
05911     while (UI != UE && N == *UI)
05912       ++UI;
05913   }
05914 
05915 public:
05916   RAUWUpdateListener(SelectionDAG &d,
05917                      SDNode::use_iterator &ui,
05918                      SDNode::use_iterator &ue)
05919     : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
05920 };
05921 
05922 }
05923 
05924 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05925 /// This can cause recursive merging of nodes in the DAG.
05926 ///
05927 /// This version assumes From has a single result value.
05928 ///
05929 void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
05930   SDNode *From = FromN.getNode();
05931   assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
05932          "Cannot replace with this method!");
05933   assert(From != To.getNode() && "Cannot replace uses of with self");
05934 
05935   // Iterate over all the existing uses of From. New uses will be added
05936   // to the beginning of the use list, which we avoid visiting.
05937   // This specifically avoids visiting uses of From that arise while the
05938   // replacement is happening, because any such uses would be the result
05939   // of CSE: If an existing node looks like From after one of its operands
05940   // is replaced by To, we don't want to replace of all its users with To
05941   // too. See PR3018 for more info.
05942   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05943   RAUWUpdateListener Listener(*this, UI, UE);
05944   while (UI != UE) {
05945     SDNode *User = *UI;
05946 
05947     // This node is about to morph, remove its old self from the CSE maps.
05948     RemoveNodeFromCSEMaps(User);
05949 
05950     // A user can appear in a use list multiple times, and when this
05951     // happens the uses are usually next to each other in the list.
05952     // To help reduce the number of CSE recomputations, process all
05953     // the uses of this user that we can find this way.
05954     do {
05955       SDUse &Use = UI.getUse();
05956       ++UI;
05957       Use.set(To);
05958     } while (UI != UE && *UI == User);
05959 
05960     // Now that we have modified User, add it back to the CSE maps.  If it
05961     // already exists there, recursively merge the results together.
05962     AddModifiedNodeToCSEMaps(User);
05963   }
05964 
05965   // If we just RAUW'd the root, take note.
05966   if (FromN == getRoot())
05967     setRoot(To);
05968 }
05969 
05970 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05971 /// This can cause recursive merging of nodes in the DAG.
05972 ///
05973 /// This version assumes that for each value of From, there is a
05974 /// corresponding value in To in the same position with the same type.
05975 ///
05976 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
05977 #ifndef NDEBUG
05978   for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
05979     assert((!From->hasAnyUseOfValue(i) ||
05980             From->getValueType(i) == To->getValueType(i)) &&
05981            "Cannot use this version of ReplaceAllUsesWith!");
05982 #endif
05983 
05984   // Handle the trivial case.
05985   if (From == To)
05986     return;
05987 
05988   // Iterate over just the existing users of From. See the comments in
05989   // the ReplaceAllUsesWith above.
05990   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05991   RAUWUpdateListener Listener(*this, UI, UE);
05992   while (UI != UE) {
05993     SDNode *User = *UI;
05994 
05995     // This node is about to morph, remove its old self from the CSE maps.
05996     RemoveNodeFromCSEMaps(User);
05997 
05998     // A user can appear in a use list multiple times, and when this
05999     // happens the uses are usually next to each other in the list.
06000     // To help reduce the number of CSE recomputations, process all
06001     // the uses of this user that we can find this way.
06002     do {
06003       SDUse &Use = UI.getUse();
06004       ++UI;
06005       Use.setNode(To);
06006     } while (UI != UE && *UI == User);
06007 
06008     // Now that we have modified User, add it back to the CSE maps.  If it
06009     // already exists there, recursively merge the results together.
06010     AddModifiedNodeToCSEMaps(User);
06011   }
06012 
06013   // If we just RAUW'd the root, take note.
06014   if (From == getRoot().getNode())
06015     setRoot(SDValue(To, getRoot().getResNo()));
06016 }
06017 
06018 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
06019 /// This can cause recursive merging of nodes in the DAG.
06020 ///
06021 /// This version can replace From with any result values.  To must match the
06022 /// number and types of values returned by From.
06023 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
06024   if (From->getNumValues() == 1)  // Handle the simple case efficiently.
06025     return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
06026 
06027   // Iterate over just the existing users of From. See the comments in
06028   // the ReplaceAllUsesWith above.
06029   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
06030   RAUWUpdateListener Listener(*this, UI, UE);
06031   while (UI != UE) {
06032     SDNode *User = *UI;
06033 
06034     // This node is about to morph, remove its old self from the CSE maps.
06035     RemoveNodeFromCSEMaps(User);
06036 
06037     // A user can appear in a use list multiple times, and when this
06038     // happens the uses are usually next to each other in the list.
06039     // To help reduce the number of CSE recomputations, process all
06040     // the uses of this user that we can find this way.
06041     do {
06042       SDUse &Use = UI.getUse();
06043       const SDValue &ToOp = To[Use.getResNo()];
06044       ++UI;
06045       Use.set(ToOp);
06046     } while (UI != UE && *UI == User);
06047 
06048     // Now that we have modified User, add it back to the CSE maps.  If it
06049     // already exists there, recursively merge the results together.
06050     AddModifiedNodeToCSEMaps(User);
06051   }
06052 
06053   // If we just RAUW'd the root, take note.
06054   if (From == getRoot().getNode())
06055     setRoot(SDValue(To[getRoot().getResNo()]));
06056 }
06057 
06058 /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
06059 /// uses of other values produced by From.getNode() alone.  The Deleted
06060 /// vector is handled the same way as for ReplaceAllUsesWith.
06061 void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
06062   // Handle the really simple, really trivial case efficiently.
06063   if (From == To) return;
06064 
06065   // Handle the simple, trivial, case efficiently.
06066   if (From.getNode()->getNumValues() == 1) {
06067     ReplaceAllUsesWith(From, To);
06068     return;
06069   }
06070 
06071   // Iterate over just the existing users of From. See the comments in
06072   // the ReplaceAllUsesWith above.
06073   SDNode::use_iterator UI = From.getNode()->use_begin(),
06074                        UE = From.getNode()->use_end();
06075   RAUWUpdateListener Listener(*this, UI, UE);
06076   while (UI != UE) {
06077     SDNode *User = *UI;
06078     bool UserRemovedFromCSEMaps = false;
06079 
06080     // A user can appear in a use list multiple times, and when this
06081     // happens the uses are usually next to each other in the list.
06082     // To help reduce the number of CSE recomputations, process all
06083     // the uses of this user that we can find this way.
06084     do {
06085       SDUse &Use = UI.getUse();
06086 
06087       // Skip uses of different values from the same node.
06088       if (Use.getResNo() != From.getResNo()) {
06089         ++UI;
06090         continue;
06091       }
06092 
06093       // If this node hasn't been modified yet, it's still in the CSE maps,
06094       // so remove its old self from the CSE maps.
06095       if (!UserRemovedFromCSEMaps) {
06096         RemoveNodeFromCSEMaps(User);
06097         UserRemovedFromCSEMaps = true;
06098       }
06099 
06100       ++UI;
06101       Use.set(To);
06102     } while (UI != UE && *UI == User);
06103 
06104     // We are iterating over all uses of the From node, so if a use
06105     // doesn't use the specific value, no changes are made.
06106     if (!UserRemovedFromCSEMaps)
06107       continue;
06108 
06109     // Now that we have modified User, add it back to the CSE maps.  If it
06110     // already exists there, recursively merge the results together.
06111     AddModifiedNodeToCSEMaps(User);
06112   }
06113 
06114   // If we just RAUW'd the root, take note.
06115   if (From == getRoot())
06116     setRoot(To);
06117 }
06118 
06119 namespace {
06120   /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
06121   /// to record information about a use.
06122   struct UseMemo {
06123     SDNode *User;
06124     unsigned Index;
06125     SDUse *Use;
06126   };
06127 
06128   /// operator< - Sort Memos by User.
06129   bool operator<(const UseMemo &L, const UseMemo &R) {
06130     return (intptr_t)L.User < (intptr_t)R.User;
06131   }
06132 }
06133 
06134 /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
06135 /// uses of other values produced by From.getNode() alone.  The same value
06136 /// may appear in both the From and To list.  The Deleted vector is
06137 /// handled the same way as for ReplaceAllUsesWith.
06138 void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
06139                                               const SDValue *To,
06140                                               unsigned Num){
06141   // Handle the simple, trivial case efficiently.
06142   if (Num == 1)
06143     return ReplaceAllUsesOfValueWith(*From, *To);
06144 
06145   // Read up all the uses and make records of them. This helps
06146   // processing new uses that are introduced during the
06147   // replacement process.
06148   SmallVector<UseMemo, 4> Uses;
06149   for (unsigned i = 0; i != Num; ++i) {
06150     unsigned FromResNo = From[i].getResNo();
06151     SDNode *FromNode = From[i].getNode();
06152     for (SDNode::use_iterator UI = FromNode->use_begin(),
06153          E = FromNode->use_end(); UI != E; ++UI) {
06154       SDUse &Use = UI.getUse();
06155       if (Use.getResNo() == FromResNo) {
06156         UseMemo Memo = { *UI, i, &Use };
06157         Uses.push_back(Memo);
06158       }
06159     }
06160   }
06161 
06162   // Sort the uses, so that all the uses from a given User are together.
06163   std::sort(Uses.begin(), Uses.end());
06164 
06165   for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
06166        UseIndex != UseIndexEnd; ) {
06167     // We know that this user uses some value of From.  If it is the right
06168     // value, update it.
06169     SDNode *User = Uses[UseIndex].User;
06170 
06171     // This node is about to morph, remove its old self from the CSE maps.
06172     RemoveNodeFromCSEMaps(User);
06173 
06174     // The Uses array is sorted, so all the uses for a given User
06175     // are next to each other in the list.
06176     // To help reduce the number of CSE recomputations, process all
06177     // the uses of this user that we can find this way.
06178     do {
06179       unsigned i = Uses[UseIndex].Index;
06180       SDUse &Use = *Uses[UseIndex].Use;
06181       ++UseIndex;
06182 
06183       Use.set(To[i]);
06184     } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
06185 
06186     // Now that we have modified User, add it back to the CSE maps.  If it
06187     // already exists there, recursively merge the results together.
06188     AddModifiedNodeToCSEMaps(User);
06189   }
06190 }
06191 
06192 /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
06193 /// based on their topological order. It returns the maximum id and a vector
06194 /// of the SDNodes* in assigned order by reference.
06195 unsigned SelectionDAG::AssignTopologicalOrder() {
06196 
06197   unsigned DAGSize = 0;
06198 
06199   // SortedPos tracks the progress of the algorithm. Nodes before it are
06200   // sorted, nodes after it are unsorted. When the algorithm completes
06201   // it is at the end of the list.
06202   allnodes_iterator SortedPos = allnodes_begin();
06203 
06204   // Visit all the nodes. Move nodes with no operands to the front of
06205   // the list immediately. Annotate nodes that do have operands with their
06206   // operand count. Before we do this, the Node Id fields of the nodes
06207   // may contain arbitrary values. After, the Node Id fields for nodes
06208   // before SortedPos will contain the topological sort index, and the
06209   // Node Id fields for nodes At SortedPos and after will contain the
06210   // count of outstanding operands.
06211   for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
06212     SDNode *N = I++;
06213     checkForCycles(N, this);
06214     unsigned Degree = N->getNumOperands();
06215     if (Degree == 0) {
06216       // A node with no uses, add it to the result array immediately.
06217       N->setNodeId(DAGSize++);
06218       allnodes_iterator Q = N;
06219       if (Q != SortedPos)
06220         SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
06221       assert(SortedPos != AllNodes.end() && "Overran node list");
06222       ++SortedPos;
06223     } else {
06224       // Temporarily use the Node Id as scratch space for the degree count.
06225       N->setNodeId(Degree);
06226     }
06227   }
06228 
06229   // Visit all the nodes. As we iterate, move nodes into sorted order,
06230   // such that by the time the end is reached all nodes will be sorted.
06231   for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) {
06232     SDNode *N = I;
06233     checkForCycles(N, this);
06234     // N is in sorted position, so all its uses have one less operand
06235     // that needs to be sorted.
06236     for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
06237          UI != UE; ++UI) {
06238       SDNode *P = *UI;
06239       unsigned Degree = P->getNodeId();
06240       assert(Degree != 0 && "Invalid node degree");
06241       --Degree;
06242       if (Degree == 0) {
06243         // All of P's operands are sorted, so P may sorted now.
06244         P->setNodeId(DAGSize++);
06245         if (P != SortedPos)
06246           SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
06247         assert(SortedPos != AllNodes.end() && "Overran node list");
06248         ++SortedPos;
06249       } else {
06250         // Update P's outstanding operand count.
06251         P->setNodeId(Degree);
06252       }
06253     }
06254     if (I == SortedPos) {
06255 #ifndef NDEBUG
06256       SDNode *S = ++I;
06257       dbgs() << "Overran sorted position:\n";
06258       S->dumprFull(this); dbgs() << "\n";
06259       dbgs() << "Checking if this is due to cycles\n";
06260       checkForCycles(this, true);
06261 #endif
06262       llvm_unreachable(nullptr);
06263     }
06264   }
06265 
06266   assert(SortedPos == AllNodes.end() &&
06267          "Topological sort incomplete!");
06268   assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
06269          "First node in topological sort is not the entry token!");
06270   assert(AllNodes.front().getNodeId() == 0 &&
06271          "First node in topological sort has non-zero id!");
06272   assert(AllNodes.front().getNumOperands() == 0 &&
06273          "First node in topological sort has operands!");
06274   assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
06275          "Last node in topologic sort has unexpected id!");
06276   assert(AllNodes.back().use_empty() &&
06277          "Last node in topologic sort has users!");
06278   assert(DAGSize == allnodes_size() && "Node count mismatch!");
06279   return DAGSize;
06280 }
06281 
06282 /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
06283 /// value is produced by SD.
06284 void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) {
06285   if (SD) {
06286     assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
06287     SD->setHasDebugValue(true);
06288   }
06289   DbgInfo->add(DB, SD, isParameter);
06290 }
06291 
06292 /// TransferDbgValues - Transfer SDDbgValues.
06293 void SelectionDAG::TransferDbgValues(SDValue From, SDValue To) {
06294   if (From == To || !From.getNode()->getHasDebugValue())
06295     return;