LLVM API Documentation

SelectionDAG.cpp
Go to the documentation of this file.
00001 //===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===//
00002 //
00003 //                     The LLVM Compiler Infrastructure
00004 //
00005 // This file is distributed under the University of Illinois Open Source
00006 // License. See LICENSE.TXT for details.
00007 //
00008 //===----------------------------------------------------------------------===//
00009 //
00010 // This implements the SelectionDAG class.
00011 //
00012 //===----------------------------------------------------------------------===//
00013 
00014 #include "llvm/CodeGen/SelectionDAG.h"
00015 #include "SDNodeDbgValue.h"
00016 #include "llvm/ADT/SetVector.h"
00017 #include "llvm/ADT/SmallPtrSet.h"
00018 #include "llvm/ADT/SmallSet.h"
00019 #include "llvm/ADT/SmallVector.h"
00020 #include "llvm/ADT/StringExtras.h"
00021 #include "llvm/Analysis/ValueTracking.h"
00022 #include "llvm/CodeGen/MachineBasicBlock.h"
00023 #include "llvm/CodeGen/MachineConstantPool.h"
00024 #include "llvm/CodeGen/MachineFrameInfo.h"
00025 #include "llvm/CodeGen/MachineModuleInfo.h"
00026 #include "llvm/IR/CallingConv.h"
00027 #include "llvm/IR/Constants.h"
00028 #include "llvm/IR/DataLayout.h"
00029 #include "llvm/IR/DebugInfo.h"
00030 #include "llvm/IR/DerivedTypes.h"
00031 #include "llvm/IR/Function.h"
00032 #include "llvm/IR/GlobalAlias.h"
00033 #include "llvm/IR/GlobalVariable.h"
00034 #include "llvm/IR/Intrinsics.h"
00035 #include "llvm/Support/CommandLine.h"
00036 #include "llvm/Support/Debug.h"
00037 #include "llvm/Support/ErrorHandling.h"
00038 #include "llvm/Support/ManagedStatic.h"
00039 #include "llvm/Support/MathExtras.h"
00040 #include "llvm/Support/Mutex.h"
00041 #include "llvm/Support/raw_ostream.h"
00042 #include "llvm/Target/TargetInstrInfo.h"
00043 #include "llvm/Target/TargetIntrinsicInfo.h"
00044 #include "llvm/Target/TargetLowering.h"
00045 #include "llvm/Target/TargetMachine.h"
00046 #include "llvm/Target/TargetOptions.h"
00047 #include "llvm/Target/TargetRegisterInfo.h"
00048 #include "llvm/Target/TargetSelectionDAGInfo.h"
00049 #include "llvm/Target/TargetSubtargetInfo.h"
00050 #include <algorithm>
00051 #include <cmath>
00052 
00053 using namespace llvm;
00054 
00055 /// makeVTList - Return an instance of the SDVTList struct initialized with the
00056 /// specified members.
00057 static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
00058   SDVTList Res = {VTs, NumVTs};
00059   return Res;
00060 }
00061 
00062 // Default null implementations of the callbacks.
00063 void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
00064 void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
00065 
00066 //===----------------------------------------------------------------------===//
00067 //                              ConstantFPSDNode Class
00068 //===----------------------------------------------------------------------===//
00069 
00070 /// isExactlyValue - We don't rely on operator== working on double values, as
00071 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
00072 /// As such, this method can be used to do an exact bit-for-bit comparison of
00073 /// two floating point values.
00074 bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
00075   return getValueAPF().bitwiseIsEqual(V);
00076 }
00077 
00078 bool ConstantFPSDNode::isValueValidForType(EVT VT,
00079                                            const APFloat& Val) {
00080   assert(VT.isFloatingPoint() && "Can only convert between FP types");
00081 
00082   // convert modifies in place, so make a copy.
00083   APFloat Val2 = APFloat(Val);
00084   bool losesInfo;
00085   (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
00086                       APFloat::rmNearestTiesToEven,
00087                       &losesInfo);
00088   return !losesInfo;
00089 }
00090 
00091 //===----------------------------------------------------------------------===//
00092 //                              ISD Namespace
00093 //===----------------------------------------------------------------------===//
00094 
00095 /// isBuildVectorAllOnes - Return true if the specified node is a
00096 /// BUILD_VECTOR where all of the elements are ~0 or undef.
00097 bool ISD::isBuildVectorAllOnes(const SDNode *N) {
00098   // Look through a bit convert.
00099   while (N->getOpcode() == ISD::BITCAST)
00100     N = N->getOperand(0).getNode();
00101 
00102   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
00103 
00104   unsigned i = 0, e = N->getNumOperands();
00105 
00106   // Skip over all of the undef values.
00107   while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF)
00108     ++i;
00109 
00110   // Do not accept an all-undef vector.
00111   if (i == e) return false;
00112 
00113   // Do not accept build_vectors that aren't all constants or which have non-~0
00114   // elements. We have to be a bit careful here, as the type of the constant
00115   // may not be the same as the type of the vector elements due to type
00116   // legalization (the elements are promoted to a legal type for the target and
00117   // a vector of a type may be legal when the base element type is not).
00118   // We only want to check enough bits to cover the vector elements, because
00119   // we care if the resultant vector is all ones, not whether the individual
00120   // constants are.
00121   SDValue NotZero = N->getOperand(i);
00122   unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
00123   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
00124     if (CN->getAPIntValue().countTrailingOnes() < EltSize)
00125       return false;
00126   } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
00127     if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
00128       return false;
00129   } else
00130     return false;
00131 
00132   // Okay, we have at least one ~0 value, check to see if the rest match or are
00133   // undefs. Even with the above element type twiddling, this should be OK, as
00134   // the same type legalization should have applied to all the elements.
00135   for (++i; i != e; ++i)
00136     if (N->getOperand(i) != NotZero &&
00137         N->getOperand(i).getOpcode() != ISD::UNDEF)
00138       return false;
00139   return true;
00140 }
00141 
00142 
00143 /// isBuildVectorAllZeros - Return true if the specified node is a
00144 /// BUILD_VECTOR where all of the elements are 0 or undef.
00145 bool ISD::isBuildVectorAllZeros(const SDNode *N) {
00146   // Look through a bit convert.
00147   while (N->getOpcode() == ISD::BITCAST)
00148     N = N->getOperand(0).getNode();
00149 
00150   if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
00151 
00152   bool IsAllUndef = true;
00153   for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) {
00154     if (N->getOperand(i).getOpcode() == ISD::UNDEF)
00155       continue;
00156     IsAllUndef = false;
00157     // Do not accept build_vectors that aren't all constants or which have non-0
00158     // elements. We have to be a bit careful here, as the type of the constant
00159     // may not be the same as the type of the vector elements due to type
00160     // legalization (the elements are promoted to a legal type for the target
00161     // and a vector of a type may be legal when the base element type is not).
00162     // We only want to check enough bits to cover the vector elements, because
00163     // we care if the resultant vector is all zeros, not whether the individual
00164     // constants are.
00165     SDValue Zero = N->getOperand(i);
00166     unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
00167     if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Zero)) {
00168       if (CN->getAPIntValue().countTrailingZeros() < EltSize)
00169         return false;
00170     } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Zero)) {
00171       if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
00172         return false;
00173     } else
00174       return false;
00175   }
00176 
00177   // Do not accept an all-undef vector.
00178   if (IsAllUndef)
00179     return false;
00180   return true;
00181 }
00182 
00183 /// \brief Return true if the specified node is a BUILD_VECTOR node of
00184 /// all ConstantSDNode or undef.
00185 bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
00186   if (N->getOpcode() != ISD::BUILD_VECTOR)
00187     return false;
00188 
00189   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
00190     SDValue Op = N->getOperand(i);
00191     if (Op.getOpcode() == ISD::UNDEF)
00192       continue;
00193     if (!isa<ConstantSDNode>(Op))
00194       return false;
00195   }
00196   return true;
00197 }
00198 
00199 /// isScalarToVector - Return true if the specified node is a
00200 /// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low
00201 /// element is not an undef.
00202 bool ISD::isScalarToVector(const SDNode *N) {
00203   if (N->getOpcode() == ISD::SCALAR_TO_VECTOR)
00204     return true;
00205 
00206   if (N->getOpcode() != ISD::BUILD_VECTOR)
00207     return false;
00208   if (N->getOperand(0).getOpcode() == ISD::UNDEF)
00209     return false;
00210   unsigned NumElems = N->getNumOperands();
00211   if (NumElems == 1)
00212     return false;
00213   for (unsigned i = 1; i < NumElems; ++i) {
00214     SDValue V = N->getOperand(i);
00215     if (V.getOpcode() != ISD::UNDEF)
00216       return false;
00217   }
00218   return true;
00219 }
00220 
00221 /// allOperandsUndef - Return true if the node has at least one operand
00222 /// and all operands of the specified node are ISD::UNDEF.
00223 bool ISD::allOperandsUndef(const SDNode *N) {
00224   // Return false if the node has no operands.
00225   // This is "logically inconsistent" with the definition of "all" but
00226   // is probably the desired behavior.
00227   if (N->getNumOperands() == 0)
00228     return false;
00229 
00230   for (unsigned i = 0, e = N->getNumOperands(); i != e ; ++i)
00231     if (N->getOperand(i).getOpcode() != ISD::UNDEF)
00232       return false;
00233 
00234   return true;
00235 }
00236 
00237 ISD::NodeType ISD::getExtForLoadExtType(ISD::LoadExtType ExtType) {
00238   switch (ExtType) {
00239   case ISD::EXTLOAD:
00240     return ISD::ANY_EXTEND;
00241   case ISD::SEXTLOAD:
00242     return ISD::SIGN_EXTEND;
00243   case ISD::ZEXTLOAD:
00244     return ISD::ZERO_EXTEND;
00245   default:
00246     break;
00247   }
00248 
00249   llvm_unreachable("Invalid LoadExtType");
00250 }
00251 
00252 /// getSetCCSwappedOperands - Return the operation corresponding to (Y op X)
00253 /// when given the operation for (X op Y).
00254 ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
00255   // To perform this operation, we just need to swap the L and G bits of the
00256   // operation.
00257   unsigned OldL = (Operation >> 2) & 1;
00258   unsigned OldG = (Operation >> 1) & 1;
00259   return ISD::CondCode((Operation & ~6) |  // Keep the N, U, E bits
00260                        (OldL << 1) |       // New G bit
00261                        (OldG << 2));       // New L bit.
00262 }
00263 
00264 /// getSetCCInverse - Return the operation corresponding to !(X op Y), where
00265 /// 'op' is a valid SetCC operation.
00266 ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) {
00267   unsigned Operation = Op;
00268   if (isInteger)
00269     Operation ^= 7;   // Flip L, G, E bits, but not U.
00270   else
00271     Operation ^= 15;  // Flip all of the condition bits.
00272 
00273   if (Operation > ISD::SETTRUE2)
00274     Operation &= ~8;  // Don't let N and U bits get set.
00275 
00276   return ISD::CondCode(Operation);
00277 }
00278 
00279 
00280 /// isSignedOp - For an integer comparison, return 1 if the comparison is a
00281 /// signed operation and 2 if the result is an unsigned comparison.  Return zero
00282 /// if the operation does not depend on the sign of the input (setne and seteq).
00283 static int isSignedOp(ISD::CondCode Opcode) {
00284   switch (Opcode) {
00285   default: llvm_unreachable("Illegal integer setcc operation!");
00286   case ISD::SETEQ:
00287   case ISD::SETNE: return 0;
00288   case ISD::SETLT:
00289   case ISD::SETLE:
00290   case ISD::SETGT:
00291   case ISD::SETGE: return 1;
00292   case ISD::SETULT:
00293   case ISD::SETULE:
00294   case ISD::SETUGT:
00295   case ISD::SETUGE: return 2;
00296   }
00297 }
00298 
00299 /// getSetCCOrOperation - Return the result of a logical OR between different
00300 /// comparisons of identical values: ((X op1 Y) | (X op2 Y)).  This function
00301 /// returns SETCC_INVALID if it is not possible to represent the resultant
00302 /// comparison.
00303 ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
00304                                        bool isInteger) {
00305   if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
00306     // Cannot fold a signed integer setcc with an unsigned integer setcc.
00307     return ISD::SETCC_INVALID;
00308 
00309   unsigned Op = Op1 | Op2;  // Combine all of the condition bits.
00310 
00311   // If the N and U bits get set then the resultant comparison DOES suddenly
00312   // care about orderedness, and is true when ordered.
00313   if (Op > ISD::SETTRUE2)
00314     Op &= ~16;     // Clear the U bit if the N bit is set.
00315 
00316   // Canonicalize illegal integer setcc's.
00317   if (isInteger && Op == ISD::SETUNE)  // e.g. SETUGT | SETULT
00318     Op = ISD::SETNE;
00319 
00320   return ISD::CondCode(Op);
00321 }
00322 
00323 /// getSetCCAndOperation - Return the result of a logical AND between different
00324 /// comparisons of identical values: ((X op1 Y) & (X op2 Y)).  This
00325 /// function returns zero if it is not possible to represent the resultant
00326 /// comparison.
00327 ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
00328                                         bool isInteger) {
00329   if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
00330     // Cannot fold a signed setcc with an unsigned setcc.
00331     return ISD::SETCC_INVALID;
00332 
00333   // Combine all of the condition bits.
00334   ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
00335 
00336   // Canonicalize illegal integer setcc's.
00337   if (isInteger) {
00338     switch (Result) {
00339     default: break;
00340     case ISD::SETUO : Result = ISD::SETFALSE; break;  // SETUGT & SETULT
00341     case ISD::SETOEQ:                                 // SETEQ  & SETU[LG]E
00342     case ISD::SETUEQ: Result = ISD::SETEQ   ; break;  // SETUGE & SETULE
00343     case ISD::SETOLT: Result = ISD::SETULT  ; break;  // SETULT & SETNE
00344     case ISD::SETOGT: Result = ISD::SETUGT  ; break;  // SETUGT & SETNE
00345     }
00346   }
00347 
00348   return Result;
00349 }
00350 
00351 //===----------------------------------------------------------------------===//
00352 //                           SDNode Profile Support
00353 //===----------------------------------------------------------------------===//
00354 
00355 /// AddNodeIDOpcode - Add the node opcode to the NodeID data.
00356 ///
00357 static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC)  {
00358   ID.AddInteger(OpC);
00359 }
00360 
00361 /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
00362 /// solely with their pointer.
00363 static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
00364   ID.AddPointer(VTList.VTs);
00365 }
00366 
00367 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
00368 ///
00369 static void AddNodeIDOperands(FoldingSetNodeID &ID,
00370                               ArrayRef<SDValue> Ops) {
00371   for (auto& Op : Ops) {
00372     ID.AddPointer(Op.getNode());
00373     ID.AddInteger(Op.getResNo());
00374   }
00375 }
00376 
00377 /// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
00378 ///
00379 static void AddNodeIDOperands(FoldingSetNodeID &ID,
00380                               ArrayRef<SDUse> Ops) {
00381   for (auto& Op : Ops) {
00382     ID.AddPointer(Op.getNode());
00383     ID.AddInteger(Op.getResNo());
00384   }
00385 }
00386 
00387 static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, bool nuw, bool nsw,
00388                                   bool exact) {
00389   ID.AddBoolean(nuw);
00390   ID.AddBoolean(nsw);
00391   ID.AddBoolean(exact);
00392 }
00393 
00394 /// AddBinaryNodeIDCustom - Add BinarySDNodes special infos
00395 static void AddBinaryNodeIDCustom(FoldingSetNodeID &ID, unsigned Opcode,
00396                                   bool nuw, bool nsw, bool exact) {
00397   if (isBinOpWithFlags(Opcode))
00398     AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
00399 }
00400 
00401 static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
00402                           SDVTList VTList, ArrayRef<SDValue> OpList) {
00403   AddNodeIDOpcode(ID, OpC);
00404   AddNodeIDValueTypes(ID, VTList);
00405   AddNodeIDOperands(ID, OpList);
00406 }
00407 
00408 /// AddNodeIDCustom - If this is an SDNode with special info, add this info to
00409 /// the NodeID data.
00410 static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
00411   switch (N->getOpcode()) {
00412   case ISD::TargetExternalSymbol:
00413   case ISD::ExternalSymbol:
00414     llvm_unreachable("Should only be used on nodes with operands");
00415   default: break;  // Normal nodes don't need extra info.
00416   case ISD::TargetConstant:
00417   case ISD::Constant: {
00418     const ConstantSDNode *C = cast<ConstantSDNode>(N);
00419     ID.AddPointer(C->getConstantIntValue());
00420     ID.AddBoolean(C->isOpaque());
00421     break;
00422   }
00423   case ISD::TargetConstantFP:
00424   case ISD::ConstantFP: {
00425     ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
00426     break;
00427   }
00428   case ISD::TargetGlobalAddress:
00429   case ISD::GlobalAddress:
00430   case ISD::TargetGlobalTLSAddress:
00431   case ISD::GlobalTLSAddress: {
00432     const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
00433     ID.AddPointer(GA->getGlobal());
00434     ID.AddInteger(GA->getOffset());
00435     ID.AddInteger(GA->getTargetFlags());
00436     ID.AddInteger(GA->getAddressSpace());
00437     break;
00438   }
00439   case ISD::BasicBlock:
00440     ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
00441     break;
00442   case ISD::Register:
00443     ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
00444     break;
00445   case ISD::RegisterMask:
00446     ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
00447     break;
00448   case ISD::SRCVALUE:
00449     ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
00450     break;
00451   case ISD::FrameIndex:
00452   case ISD::TargetFrameIndex:
00453     ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
00454     break;
00455   case ISD::JumpTable:
00456   case ISD::TargetJumpTable:
00457     ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
00458     ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
00459     break;
00460   case ISD::ConstantPool:
00461   case ISD::TargetConstantPool: {
00462     const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
00463     ID.AddInteger(CP->getAlignment());
00464     ID.AddInteger(CP->getOffset());
00465     if (CP->isMachineConstantPoolEntry())
00466       CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
00467     else
00468       ID.AddPointer(CP->getConstVal());
00469     ID.AddInteger(CP->getTargetFlags());
00470     break;
00471   }
00472   case ISD::TargetIndex: {
00473     const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
00474     ID.AddInteger(TI->getIndex());
00475     ID.AddInteger(TI->getOffset());
00476     ID.AddInteger(TI->getTargetFlags());
00477     break;
00478   }
00479   case ISD::LOAD: {
00480     const LoadSDNode *LD = cast<LoadSDNode>(N);
00481     ID.AddInteger(LD->getMemoryVT().getRawBits());
00482     ID.AddInteger(LD->getRawSubclassData());
00483     ID.AddInteger(LD->getPointerInfo().getAddrSpace());
00484     break;
00485   }
00486   case ISD::STORE: {
00487     const StoreSDNode *ST = cast<StoreSDNode>(N);
00488     ID.AddInteger(ST->getMemoryVT().getRawBits());
00489     ID.AddInteger(ST->getRawSubclassData());
00490     ID.AddInteger(ST->getPointerInfo().getAddrSpace());
00491     break;
00492   }
00493   case ISD::SDIV:
00494   case ISD::UDIV:
00495   case ISD::SRA:
00496   case ISD::SRL:
00497   case ISD::MUL:
00498   case ISD::ADD:
00499   case ISD::SUB:
00500   case ISD::SHL: {
00501     const BinaryWithFlagsSDNode *BinNode = cast<BinaryWithFlagsSDNode>(N);
00502     AddBinaryNodeIDCustom(ID, N->getOpcode(), BinNode->hasNoUnsignedWrap(),
00503                           BinNode->hasNoSignedWrap(), BinNode->isExact());
00504     break;
00505   }
00506   case ISD::ATOMIC_CMP_SWAP:
00507   case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
00508   case ISD::ATOMIC_SWAP:
00509   case ISD::ATOMIC_LOAD_ADD:
00510   case ISD::ATOMIC_LOAD_SUB:
00511   case ISD::ATOMIC_LOAD_AND:
00512   case ISD::ATOMIC_LOAD_OR:
00513   case ISD::ATOMIC_LOAD_XOR:
00514   case ISD::ATOMIC_LOAD_NAND:
00515   case ISD::ATOMIC_LOAD_MIN:
00516   case ISD::ATOMIC_LOAD_MAX:
00517   case ISD::ATOMIC_LOAD_UMIN:
00518   case ISD::ATOMIC_LOAD_UMAX:
00519   case ISD::ATOMIC_LOAD:
00520   case ISD::ATOMIC_STORE: {
00521     const AtomicSDNode *AT = cast<AtomicSDNode>(N);
00522     ID.AddInteger(AT->getMemoryVT().getRawBits());
00523     ID.AddInteger(AT->getRawSubclassData());
00524     ID.AddInteger(AT->getPointerInfo().getAddrSpace());
00525     break;
00526   }
00527   case ISD::PREFETCH: {
00528     const MemSDNode *PF = cast<MemSDNode>(N);
00529     ID.AddInteger(PF->getPointerInfo().getAddrSpace());
00530     break;
00531   }
00532   case ISD::VECTOR_SHUFFLE: {
00533     const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
00534     for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
00535          i != e; ++i)
00536       ID.AddInteger(SVN->getMaskElt(i));
00537     break;
00538   }
00539   case ISD::TargetBlockAddress:
00540   case ISD::BlockAddress: {
00541     const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
00542     ID.AddPointer(BA->getBlockAddress());
00543     ID.AddInteger(BA->getOffset());
00544     ID.AddInteger(BA->getTargetFlags());
00545     break;
00546   }
00547   } // end switch (N->getOpcode())
00548 
00549   // Target specific memory nodes could also have address spaces to check.
00550   if (N->isTargetMemoryOpcode())
00551     ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
00552 }
00553 
00554 /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
00555 /// data.
00556 static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
00557   AddNodeIDOpcode(ID, N->getOpcode());
00558   // Add the return value info.
00559   AddNodeIDValueTypes(ID, N->getVTList());
00560   // Add the operand info.
00561   AddNodeIDOperands(ID, N->ops());
00562 
00563   // Handle SDNode leafs with special info.
00564   AddNodeIDCustom(ID, N);
00565 }
00566 
00567 /// encodeMemSDNodeFlags - Generic routine for computing a value for use in
00568 /// the CSE map that carries volatility, temporalness, indexing mode, and
00569 /// extension/truncation information.
00570 ///
00571 static inline unsigned
00572 encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile,
00573                      bool isNonTemporal, bool isInvariant) {
00574   assert((ConvType & 3) == ConvType &&
00575          "ConvType may not require more than 2 bits!");
00576   assert((AM & 7) == AM &&
00577          "AM may not require more than 3 bits!");
00578   return ConvType |
00579          (AM << 2) |
00580          (isVolatile << 5) |
00581          (isNonTemporal << 6) |
00582          (isInvariant << 7);
00583 }
00584 
00585 //===----------------------------------------------------------------------===//
00586 //                              SelectionDAG Class
00587 //===----------------------------------------------------------------------===//
00588 
00589 /// doNotCSE - Return true if CSE should not be performed for this node.
00590 static bool doNotCSE(SDNode *N) {
00591   if (N->getValueType(0) == MVT::Glue)
00592     return true; // Never CSE anything that produces a flag.
00593 
00594   switch (N->getOpcode()) {
00595   default: break;
00596   case ISD::HANDLENODE:
00597   case ISD::EH_LABEL:
00598     return true;   // Never CSE these nodes.
00599   }
00600 
00601   // Check that remaining values produced are not flags.
00602   for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
00603     if (N->getValueType(i) == MVT::Glue)
00604       return true; // Never CSE anything that produces a flag.
00605 
00606   return false;
00607 }
00608 
00609 /// RemoveDeadNodes - This method deletes all unreachable nodes in the
00610 /// SelectionDAG.
00611 void SelectionDAG::RemoveDeadNodes() {
00612   // Create a dummy node (which is not added to allnodes), that adds a reference
00613   // to the root node, preventing it from being deleted.
00614   HandleSDNode Dummy(getRoot());
00615 
00616   SmallVector<SDNode*, 128> DeadNodes;
00617 
00618   // Add all obviously-dead nodes to the DeadNodes worklist.
00619   for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I)
00620     if (I->use_empty())
00621       DeadNodes.push_back(I);
00622 
00623   RemoveDeadNodes(DeadNodes);
00624 
00625   // If the root changed (e.g. it was a dead load, update the root).
00626   setRoot(Dummy.getValue());
00627 }
00628 
00629 /// RemoveDeadNodes - This method deletes the unreachable nodes in the
00630 /// given list, and any nodes that become unreachable as a result.
00631 void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
00632 
00633   // Process the worklist, deleting the nodes and adding their uses to the
00634   // worklist.
00635   while (!DeadNodes.empty()) {
00636     SDNode *N = DeadNodes.pop_back_val();
00637 
00638     for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00639       DUL->NodeDeleted(N, nullptr);
00640 
00641     // Take the node out of the appropriate CSE map.
00642     RemoveNodeFromCSEMaps(N);
00643 
00644     // Next, brutally remove the operand list.  This is safe to do, as there are
00645     // no cycles in the graph.
00646     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
00647       SDUse &Use = *I++;
00648       SDNode *Operand = Use.getNode();
00649       Use.set(SDValue());
00650 
00651       // Now that we removed this operand, see if there are no uses of it left.
00652       if (Operand->use_empty())
00653         DeadNodes.push_back(Operand);
00654     }
00655 
00656     DeallocateNode(N);
00657   }
00658 }
00659 
00660 void SelectionDAG::RemoveDeadNode(SDNode *N){
00661   SmallVector<SDNode*, 16> DeadNodes(1, N);
00662 
00663   // Create a dummy node that adds a reference to the root node, preventing
00664   // it from being deleted.  (This matters if the root is an operand of the
00665   // dead node.)
00666   HandleSDNode Dummy(getRoot());
00667 
00668   RemoveDeadNodes(DeadNodes);
00669 }
00670 
00671 void SelectionDAG::DeleteNode(SDNode *N) {
00672   // First take this out of the appropriate CSE map.
00673   RemoveNodeFromCSEMaps(N);
00674 
00675   // Finally, remove uses due to operands of this node, remove from the
00676   // AllNodes list, and delete the node.
00677   DeleteNodeNotInCSEMaps(N);
00678 }
00679 
00680 void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
00681   assert(N != AllNodes.begin() && "Cannot delete the entry node!");
00682   assert(N->use_empty() && "Cannot delete a node that is not dead!");
00683 
00684   // Drop all of the operands and decrement used node's use counts.
00685   N->DropOperands();
00686 
00687   DeallocateNode(N);
00688 }
00689 
00690 void SDDbgInfo::erase(const SDNode *Node) {
00691   DbgValMapType::iterator I = DbgValMap.find(Node);
00692   if (I == DbgValMap.end())
00693     return;
00694   for (auto &Val: I->second)
00695     Val->setIsInvalidated();
00696   DbgValMap.erase(I);
00697 }
00698 
00699 void SelectionDAG::DeallocateNode(SDNode *N) {
00700   if (N->OperandsNeedDelete)
00701     delete[] N->OperandList;
00702 
00703   // Set the opcode to DELETED_NODE to help catch bugs when node
00704   // memory is reallocated.
00705   N->NodeType = ISD::DELETED_NODE;
00706 
00707   NodeAllocator.Deallocate(AllNodes.remove(N));
00708 
00709   // If any of the SDDbgValue nodes refer to this SDNode, invalidate
00710   // them and forget about that node.
00711   DbgInfo->erase(N);
00712 }
00713 
00714 #ifndef NDEBUG
00715 /// VerifySDNode - Sanity check the given SDNode.  Aborts if it is invalid.
00716 static void VerifySDNode(SDNode *N) {
00717   switch (N->getOpcode()) {
00718   default:
00719     break;
00720   case ISD::BUILD_PAIR: {
00721     EVT VT = N->getValueType(0);
00722     assert(N->getNumValues() == 1 && "Too many results!");
00723     assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
00724            "Wrong return type!");
00725     assert(N->getNumOperands() == 2 && "Wrong number of operands!");
00726     assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
00727            "Mismatched operand types!");
00728     assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
00729            "Wrong operand type!");
00730     assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
00731            "Wrong return type size");
00732     break;
00733   }
00734   case ISD::BUILD_VECTOR: {
00735     assert(N->getNumValues() == 1 && "Too many results!");
00736     assert(N->getValueType(0).isVector() && "Wrong return type!");
00737     assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
00738            "Wrong number of operands!");
00739     EVT EltVT = N->getValueType(0).getVectorElementType();
00740     for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
00741       assert((I->getValueType() == EltVT ||
00742              (EltVT.isInteger() && I->getValueType().isInteger() &&
00743               EltVT.bitsLE(I->getValueType()))) &&
00744             "Wrong operand type!");
00745       assert(I->getValueType() == N->getOperand(0).getValueType() &&
00746              "Operands must all have the same type");
00747     }
00748     break;
00749   }
00750   }
00751 }
00752 #endif // NDEBUG
00753 
00754 /// \brief Insert a newly allocated node into the DAG.
00755 ///
00756 /// Handles insertion into the all nodes list and CSE map, as well as
00757 /// verification and other common operations when a new node is allocated.
00758 void SelectionDAG::InsertNode(SDNode *N) {
00759   AllNodes.push_back(N);
00760 #ifndef NDEBUG
00761   VerifySDNode(N);
00762 #endif
00763 }
00764 
00765 /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
00766 /// correspond to it.  This is useful when we're about to delete or repurpose
00767 /// the node.  We don't want future request for structurally identical nodes
00768 /// to return N anymore.
00769 bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
00770   bool Erased = false;
00771   switch (N->getOpcode()) {
00772   case ISD::HANDLENODE: return false;  // noop.
00773   case ISD::CONDCODE:
00774     assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
00775            "Cond code doesn't exist!");
00776     Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
00777     CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
00778     break;
00779   case ISD::ExternalSymbol:
00780     Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
00781     break;
00782   case ISD::TargetExternalSymbol: {
00783     ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
00784     Erased = TargetExternalSymbols.erase(
00785                std::pair<std::string,unsigned char>(ESN->getSymbol(),
00786                                                     ESN->getTargetFlags()));
00787     break;
00788   }
00789   case ISD::VALUETYPE: {
00790     EVT VT = cast<VTSDNode>(N)->getVT();
00791     if (VT.isExtended()) {
00792       Erased = ExtendedValueTypeNodes.erase(VT);
00793     } else {
00794       Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
00795       ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
00796     }
00797     break;
00798   }
00799   default:
00800     // Remove it from the CSE Map.
00801     assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
00802     assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
00803     Erased = CSEMap.RemoveNode(N);
00804     break;
00805   }
00806 #ifndef NDEBUG
00807   // Verify that the node was actually in one of the CSE maps, unless it has a
00808   // flag result (which cannot be CSE'd) or is one of the special cases that are
00809   // not subject to CSE.
00810   if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
00811       !N->isMachineOpcode() && !doNotCSE(N)) {
00812     N->dump(this);
00813     dbgs() << "\n";
00814     llvm_unreachable("Node is not in map!");
00815   }
00816 #endif
00817   return Erased;
00818 }
00819 
00820 /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
00821 /// maps and modified in place. Add it back to the CSE maps, unless an identical
00822 /// node already exists, in which case transfer all its users to the existing
00823 /// node. This transfer can potentially trigger recursive merging.
00824 ///
00825 void
00826 SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
00827   // For node types that aren't CSE'd, just act as if no identical node
00828   // already exists.
00829   if (!doNotCSE(N)) {
00830     SDNode *Existing = CSEMap.GetOrInsertNode(N);
00831     if (Existing != N) {
00832       // If there was already an existing matching node, use ReplaceAllUsesWith
00833       // to replace the dead one with the existing one.  This can cause
00834       // recursive merging of other unrelated nodes down the line.
00835       ReplaceAllUsesWith(N, Existing);
00836 
00837       // N is now dead. Inform the listeners and delete it.
00838       for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00839         DUL->NodeDeleted(N, Existing);
00840       DeleteNodeNotInCSEMaps(N);
00841       return;
00842     }
00843   }
00844 
00845   // If the node doesn't already exist, we updated it.  Inform listeners.
00846   for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
00847     DUL->NodeUpdated(N);
00848 }
00849 
00850 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00851 /// were replaced with those specified.  If this node is never memoized,
00852 /// return null, otherwise return a pointer to the slot it would take.  If a
00853 /// node already exists with these operands, the slot will be non-null.
00854 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
00855                                            void *&InsertPos) {
00856   if (doNotCSE(N))
00857     return nullptr;
00858 
00859   SDValue Ops[] = { Op };
00860   FoldingSetNodeID ID;
00861   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00862   AddNodeIDCustom(ID, N);
00863   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00864   return Node;
00865 }
00866 
00867 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00868 /// were replaced with those specified.  If this node is never memoized,
00869 /// return null, otherwise return a pointer to the slot it would take.  If a
00870 /// node already exists with these operands, the slot will be non-null.
00871 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
00872                                            SDValue Op1, SDValue Op2,
00873                                            void *&InsertPos) {
00874   if (doNotCSE(N))
00875     return nullptr;
00876 
00877   SDValue Ops[] = { Op1, Op2 };
00878   FoldingSetNodeID ID;
00879   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00880   AddNodeIDCustom(ID, N);
00881   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00882   return Node;
00883 }
00884 
00885 
00886 /// FindModifiedNodeSlot - Find a slot for the specified node if its operands
00887 /// were replaced with those specified.  If this node is never memoized,
00888 /// return null, otherwise return a pointer to the slot it would take.  If a
00889 /// node already exists with these operands, the slot will be non-null.
00890 SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
00891                                            void *&InsertPos) {
00892   if (doNotCSE(N))
00893     return nullptr;
00894 
00895   FoldingSetNodeID ID;
00896   AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
00897   AddNodeIDCustom(ID, N);
00898   SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
00899   return Node;
00900 }
00901 
00902 /// getEVTAlignment - Compute the default alignment value for the
00903 /// given type.
00904 ///
00905 unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
00906   Type *Ty = VT == MVT::iPTR ?
00907                    PointerType::get(Type::getInt8Ty(*getContext()), 0) :
00908                    VT.getTypeForEVT(*getContext());
00909 
00910   return TLI->getDataLayout()->getABITypeAlignment(Ty);
00911 }
00912 
00913 // EntryNode could meaningfully have debug info if we can find it...
00914 SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
00915     : TM(tm), TSI(nullptr), TLI(nullptr), OptLevel(OL),
00916       EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
00917       Root(getEntryNode()), NewNodesMustHaveLegalTypes(false),
00918       UpdateListeners(nullptr) {
00919   AllNodes.push_back(&EntryNode);
00920   DbgInfo = new SDDbgInfo();
00921 }
00922 
00923 void SelectionDAG::init(MachineFunction &mf) {
00924   MF = &mf;
00925   TLI = getSubtarget().getTargetLowering();
00926   TSI = getSubtarget().getSelectionDAGInfo();
00927   Context = &mf.getFunction()->getContext();
00928 }
00929 
00930 SelectionDAG::~SelectionDAG() {
00931   assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
00932   allnodes_clear();
00933   delete DbgInfo;
00934 }
00935 
00936 void SelectionDAG::allnodes_clear() {
00937   assert(&*AllNodes.begin() == &EntryNode);
00938   AllNodes.remove(AllNodes.begin());
00939   while (!AllNodes.empty())
00940     DeallocateNode(AllNodes.begin());
00941 }
00942 
00943 BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL,
00944                                             SDVTList VTs, SDValue N1,
00945                                             SDValue N2, bool nuw, bool nsw,
00946                                             bool exact) {
00947   if (isBinOpWithFlags(Opcode)) {
00948     BinaryWithFlagsSDNode *FN = new (NodeAllocator) BinaryWithFlagsSDNode(
00949         Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
00950     FN->setHasNoUnsignedWrap(nuw);
00951     FN->setHasNoSignedWrap(nsw);
00952     FN->setIsExact(exact);
00953 
00954     return FN;
00955   }
00956 
00957   BinarySDNode *N = new (NodeAllocator)
00958       BinarySDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2);
00959   return N;
00960 }
00961 
00962 void SelectionDAG::clear() {
00963   allnodes_clear();
00964   OperandAllocator.Reset();
00965   CSEMap.clear();
00966 
00967   ExtendedValueTypeNodes.clear();
00968   ExternalSymbols.clear();
00969   TargetExternalSymbols.clear();
00970   std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
00971             static_cast<CondCodeSDNode*>(nullptr));
00972   std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
00973             static_cast<SDNode*>(nullptr));
00974 
00975   EntryNode.UseList = nullptr;
00976   AllNodes.push_back(&EntryNode);
00977   Root = getEntryNode();
00978   DbgInfo->clear();
00979 }
00980 
00981 SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00982   return VT.bitsGT(Op.getValueType()) ?
00983     getNode(ISD::ANY_EXTEND, DL, VT, Op) :
00984     getNode(ISD::TRUNCATE, DL, VT, Op);
00985 }
00986 
00987 SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00988   return VT.bitsGT(Op.getValueType()) ?
00989     getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
00990     getNode(ISD::TRUNCATE, DL, VT, Op);
00991 }
00992 
00993 SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) {
00994   return VT.bitsGT(Op.getValueType()) ?
00995     getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
00996     getNode(ISD::TRUNCATE, DL, VT, Op);
00997 }
00998 
00999 SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, SDLoc SL, EVT VT,
01000                                         EVT OpVT) {
01001   if (VT.bitsLE(Op.getValueType()))
01002     return getNode(ISD::TRUNCATE, SL, VT, Op);
01003 
01004   TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
01005   return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
01006 }
01007 
01008 SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, SDLoc DL, EVT VT) {
01009   assert(!VT.isVector() &&
01010          "getZeroExtendInReg should use the vector element type instead of "
01011          "the vector type!");
01012   if (Op.getValueType() == VT) return Op;
01013   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01014   APInt Imm = APInt::getLowBitsSet(BitWidth,
01015                                    VT.getSizeInBits());
01016   return getNode(ISD::AND, DL, Op.getValueType(), Op,
01017                  getConstant(Imm, Op.getValueType()));
01018 }
01019 
01020 SDValue SelectionDAG::getAnyExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01021   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01022   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01023          "The sizes of the input and result must match in order to perform the "
01024          "extend in-register.");
01025   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01026          "The destination vector type must have fewer lanes than the input.");
01027   return getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Op);
01028 }
01029 
01030 SDValue SelectionDAG::getSignExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01031   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01032   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01033          "The sizes of the input and result must match in order to perform the "
01034          "extend in-register.");
01035   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01036          "The destination vector type must have fewer lanes than the input.");
01037   return getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Op);
01038 }
01039 
01040 SDValue SelectionDAG::getZeroExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT) {
01041   assert(VT.isVector() && "This DAG node is restricted to vector types.");
01042   assert(VT.getSizeInBits() == Op.getValueType().getSizeInBits() &&
01043          "The sizes of the input and result must match in order to perform the "
01044          "extend in-register.");
01045   assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
01046          "The destination vector type must have fewer lanes than the input.");
01047   return getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Op);
01048 }
01049 
01050 /// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
01051 ///
01052 SDValue SelectionDAG::getNOT(SDLoc DL, SDValue Val, EVT VT) {
01053   EVT EltVT = VT.getScalarType();
01054   SDValue NegOne =
01055     getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT);
01056   return getNode(ISD::XOR, DL, VT, Val, NegOne);
01057 }
01058 
01059 SDValue SelectionDAG::getLogicalNOT(SDLoc DL, SDValue Val, EVT VT) {
01060   EVT EltVT = VT.getScalarType();
01061   SDValue TrueValue;
01062   switch (TLI->getBooleanContents(VT)) {
01063     case TargetLowering::ZeroOrOneBooleanContent:
01064     case TargetLowering::UndefinedBooleanContent:
01065       TrueValue = getConstant(1, VT);
01066       break;
01067     case TargetLowering::ZeroOrNegativeOneBooleanContent:
01068       TrueValue = getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()),
01069                               VT);
01070       break;
01071   }
01072   return getNode(ISD::XOR, DL, VT, Val, TrueValue);
01073 }
01074 
01075 SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT, bool isO) {
01076   EVT EltVT = VT.getScalarType();
01077   assert((EltVT.getSizeInBits() >= 64 ||
01078          (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
01079          "getConstant with a uint64_t value that doesn't fit in the type!");
01080   return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT, isO);
01081 }
01082 
01083 SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT, bool isO)
01084 {
01085   return getConstant(*ConstantInt::get(*Context, Val), VT, isT, isO);
01086 }
01087 
01088 SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT,
01089                                   bool isO) {
01090   assert(VT.isInteger() && "Cannot create FP integer constant!");
01091 
01092   EVT EltVT = VT.getScalarType();
01093   const ConstantInt *Elt = &Val;
01094 
01095   // In some cases the vector type is legal but the element type is illegal and
01096   // needs to be promoted, for example v8i8 on ARM.  In this case, promote the
01097   // inserted value (the type does not need to match the vector element type).
01098   // Any extra bits introduced will be truncated away.
01099   if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
01100       TargetLowering::TypePromoteInteger) {
01101    EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
01102    APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits());
01103    Elt = ConstantInt::get(*getContext(), NewVal);
01104   }
01105   // In other cases the element type is illegal and needs to be expanded, for
01106   // example v2i64 on MIPS32. In this case, find the nearest legal type, split
01107   // the value into n parts and use a vector type with n-times the elements.
01108   // Then bitcast to the type requested.
01109   // Legalizing constants too early makes the DAGCombiner's job harder so we
01110   // only legalize if the DAG tells us we must produce legal types.
01111   else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
01112            TLI->getTypeAction(*getContext(), EltVT) ==
01113            TargetLowering::TypeExpandInteger) {
01114     APInt NewVal = Elt->getValue();
01115     EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
01116     unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
01117     unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
01118     EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
01119 
01120     // Check the temporary vector is the correct size. If this fails then
01121     // getTypeToTransformTo() probably returned a type whose size (in bits)
01122     // isn't a power-of-2 factor of the requested type size.
01123     assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
01124 
01125     SmallVector<SDValue, 2> EltParts;
01126     for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i) {
01127       EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
01128                                            .trunc(ViaEltSizeInBits),
01129                                      ViaEltVT, isT, isO));
01130     }
01131 
01132     // EltParts is currently in little endian order. If we actually want
01133     // big-endian order then reverse it now.
01134     if (TLI->isBigEndian())
01135       std::reverse(EltParts.begin(), EltParts.end());
01136 
01137     // The elements must be reversed when the element order is different
01138     // to the endianness of the elements (because the BITCAST is itself a
01139     // vector shuffle in this situation). However, we do not need any code to
01140     // perform this reversal because getConstant() is producing a vector
01141     // splat.
01142     // This situation occurs in MIPS MSA.
01143 
01144     SmallVector<SDValue, 8> Ops;
01145     for (unsigned i = 0; i < VT.getVectorNumElements(); ++i)
01146       Ops.insert(Ops.end(), EltParts.begin(), EltParts.end());
01147 
01148     SDValue Result = getNode(ISD::BITCAST, SDLoc(), VT,
01149                              getNode(ISD::BUILD_VECTOR, SDLoc(), ViaVecVT,
01150                                      Ops));
01151     return Result;
01152   }
01153 
01154   assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
01155          "APInt size does not match type size!");
01156   unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
01157   FoldingSetNodeID ID;
01158   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
01159   ID.AddPointer(Elt);
01160   ID.AddBoolean(isO);
01161   void *IP = nullptr;
01162   SDNode *N = nullptr;
01163   if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
01164     if (!VT.isVector())
01165       return SDValue(N, 0);
01166 
01167   if (!N) {
01168     N = new (NodeAllocator) ConstantSDNode(isT, isO, Elt, EltVT);
01169     CSEMap.InsertNode(N, IP);
01170     InsertNode(N);
01171   }
01172 
01173   SDValue Result(N, 0);
01174   if (VT.isVector()) {
01175     SmallVector<SDValue, 8> Ops;
01176     Ops.assign(VT.getVectorNumElements(), Result);
01177     Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
01178   }
01179   return Result;
01180 }
01181 
01182 SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) {
01183   return getConstant(Val, TLI->getPointerTy(), isTarget);
01184 }
01185 
01186 
01187 SDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) {
01188   return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget);
01189 }
01190 
01191 SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){
01192   assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
01193 
01194   EVT EltVT = VT.getScalarType();
01195 
01196   // Do the map lookup using the actual bit pattern for the floating point
01197   // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
01198   // we don't have issues with SNANs.
01199   unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
01200   FoldingSetNodeID ID;
01201   AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
01202   ID.AddPointer(&V);
01203   void *IP = nullptr;
01204   SDNode *N = nullptr;
01205   if ((N = CSEMap.FindNodeOrInsertPos(ID, IP)))
01206     if (!VT.isVector())
01207       return SDValue(N, 0);
01208 
01209   if (!N) {
01210     N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT);
01211     CSEMap.InsertNode(N, IP);
01212     InsertNode(N);
01213   }
01214 
01215   SDValue Result(N, 0);
01216   if (VT.isVector()) {
01217     SmallVector<SDValue, 8> Ops;
01218     Ops.assign(VT.getVectorNumElements(), Result);
01219     // FIXME SDLoc info might be appropriate here
01220     Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Ops);
01221   }
01222   return Result;
01223 }
01224 
01225 SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) {
01226   EVT EltVT = VT.getScalarType();
01227   if (EltVT==MVT::f32)
01228     return getConstantFP(APFloat((float)Val), VT, isTarget);
01229   else if (EltVT==MVT::f64)
01230     return getConstantFP(APFloat(Val), VT, isTarget);
01231   else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 ||
01232            EltVT==MVT::f16) {
01233     bool ignored;
01234     APFloat apf = APFloat(Val);
01235     apf.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
01236                 &ignored);
01237     return getConstantFP(apf, VT, isTarget);
01238   } else
01239     llvm_unreachable("Unsupported type in getConstantFP");
01240 }
01241 
01242 SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, SDLoc DL,
01243                                        EVT VT, int64_t Offset,
01244                                        bool isTargetGA,
01245                                        unsigned char TargetFlags) {
01246   assert((TargetFlags == 0 || isTargetGA) &&
01247          "Cannot set target flags on target-independent globals");
01248 
01249   // Truncate (with sign-extension) the offset value to the pointer size.
01250   unsigned BitWidth = TLI->getPointerTypeSizeInBits(GV->getType());
01251   if (BitWidth < 64)
01252     Offset = SignExtend64(Offset, BitWidth);
01253 
01254   unsigned Opc;
01255   if (GV->isThreadLocal())
01256     Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
01257   else
01258     Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
01259 
01260   FoldingSetNodeID ID;
01261   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01262   ID.AddPointer(GV);
01263   ID.AddInteger(Offset);
01264   ID.AddInteger(TargetFlags);
01265   ID.AddInteger(GV->getType()->getAddressSpace());
01266   void *IP = nullptr;
01267   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01268     return SDValue(E, 0);
01269 
01270   SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL.getIROrder(),
01271                                                       DL.getDebugLoc(), GV, VT,
01272                                                       Offset, TargetFlags);
01273   CSEMap.InsertNode(N, IP);
01274     InsertNode(N);
01275   return SDValue(N, 0);
01276 }
01277 
01278 SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
01279   unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
01280   FoldingSetNodeID ID;
01281   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01282   ID.AddInteger(FI);
01283   void *IP = nullptr;
01284   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01285     return SDValue(E, 0);
01286 
01287   SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget);
01288   CSEMap.InsertNode(N, IP);
01289   InsertNode(N);
01290   return SDValue(N, 0);
01291 }
01292 
01293 SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
01294                                    unsigned char TargetFlags) {
01295   assert((TargetFlags == 0 || isTarget) &&
01296          "Cannot set target flags on target-independent jump tables");
01297   unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
01298   FoldingSetNodeID ID;
01299   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01300   ID.AddInteger(JTI);
01301   ID.AddInteger(TargetFlags);
01302   void *IP = nullptr;
01303   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01304     return SDValue(E, 0);
01305 
01306   SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget,
01307                                                   TargetFlags);
01308   CSEMap.InsertNode(N, IP);
01309   InsertNode(N);
01310   return SDValue(N, 0);
01311 }
01312 
01313 SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
01314                                       unsigned Alignment, int Offset,
01315                                       bool isTarget,
01316                                       unsigned char TargetFlags) {
01317   assert((TargetFlags == 0 || isTarget) &&
01318          "Cannot set target flags on target-independent globals");
01319   if (Alignment == 0)
01320     Alignment = TLI->getDataLayout()->getPrefTypeAlignment(C->getType());
01321   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
01322   FoldingSetNodeID ID;
01323   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01324   ID.AddInteger(Alignment);
01325   ID.AddInteger(Offset);
01326   ID.AddPointer(C);
01327   ID.AddInteger(TargetFlags);
01328   void *IP = nullptr;
01329   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01330     return SDValue(E, 0);
01331 
01332   SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
01333                                                      Alignment, TargetFlags);
01334   CSEMap.InsertNode(N, IP);
01335   InsertNode(N);
01336   return SDValue(N, 0);
01337 }
01338 
01339 
01340 SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
01341                                       unsigned Alignment, int Offset,
01342                                       bool isTarget,
01343                                       unsigned char TargetFlags) {
01344   assert((TargetFlags == 0 || isTarget) &&
01345          "Cannot set target flags on target-independent globals");
01346   if (Alignment == 0)
01347     Alignment = TLI->getDataLayout()->getPrefTypeAlignment(C->getType());
01348   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
01349   FoldingSetNodeID ID;
01350   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01351   ID.AddInteger(Alignment);
01352   ID.AddInteger(Offset);
01353   C->addSelectionDAGCSEId(ID);
01354   ID.AddInteger(TargetFlags);
01355   void *IP = nullptr;
01356   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01357     return SDValue(E, 0);
01358 
01359   SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset,
01360                                                      Alignment, TargetFlags);
01361   CSEMap.InsertNode(N, IP);
01362   InsertNode(N);
01363   return SDValue(N, 0);
01364 }
01365 
01366 SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
01367                                      unsigned char TargetFlags) {
01368   FoldingSetNodeID ID;
01369   AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None);
01370   ID.AddInteger(Index);
01371   ID.AddInteger(Offset);
01372   ID.AddInteger(TargetFlags);
01373   void *IP = nullptr;
01374   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01375     return SDValue(E, 0);
01376 
01377   SDNode *N = new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset,
01378                                                     TargetFlags);
01379   CSEMap.InsertNode(N, IP);
01380   InsertNode(N);
01381   return SDValue(N, 0);
01382 }
01383 
01384 SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
01385   FoldingSetNodeID ID;
01386   AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
01387   ID.AddPointer(MBB);
01388   void *IP = nullptr;
01389   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01390     return SDValue(E, 0);
01391 
01392   SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB);
01393   CSEMap.InsertNode(N, IP);
01394   InsertNode(N);
01395   return SDValue(N, 0);
01396 }
01397 
01398 SDValue SelectionDAG::getValueType(EVT VT) {
01399   if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
01400       ValueTypeNodes.size())
01401     ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
01402 
01403   SDNode *&N = VT.isExtended() ?
01404     ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
01405 
01406   if (N) return SDValue(N, 0);
01407   N = new (NodeAllocator) VTSDNode(VT);
01408   InsertNode(N);
01409   return SDValue(N, 0);
01410 }
01411 
01412 SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
01413   SDNode *&N = ExternalSymbols[Sym];
01414   if (N) return SDValue(N, 0);
01415   N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT);
01416   InsertNode(N);
01417   return SDValue(N, 0);
01418 }
01419 
01420 SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
01421                                               unsigned char TargetFlags) {
01422   SDNode *&N =
01423     TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym,
01424                                                                TargetFlags)];
01425   if (N) return SDValue(N, 0);
01426   N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT);
01427   InsertNode(N);
01428   return SDValue(N, 0);
01429 }
01430 
01431 SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
01432   if ((unsigned)Cond >= CondCodeNodes.size())
01433     CondCodeNodes.resize(Cond+1);
01434 
01435   if (!CondCodeNodes[Cond]) {
01436     CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond);
01437     CondCodeNodes[Cond] = N;
01438     InsertNode(N);
01439   }
01440 
01441   return SDValue(CondCodeNodes[Cond], 0);
01442 }
01443 
01444 // commuteShuffle - swaps the values of N1 and N2, and swaps all indices in
01445 // the shuffle mask M that point at N1 to point at N2, and indices that point
01446 // N2 to point at N1.
01447 static void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) {
01448   std::swap(N1, N2);
01449   int NElts = M.size();
01450   for (int i = 0; i != NElts; ++i) {
01451     if (M[i] >= NElts)
01452       M[i] -= NElts;
01453     else if (M[i] >= 0)
01454       M[i] += NElts;
01455   }
01456 }
01457 
01458 SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1,
01459                                        SDValue N2, const int *Mask) {
01460   assert(VT == N1.getValueType() && VT == N2.getValueType() &&
01461          "Invalid VECTOR_SHUFFLE");
01462 
01463   // Canonicalize shuffle undef, undef -> undef
01464   if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF)
01465     return getUNDEF(VT);
01466 
01467   // Validate that all indices in Mask are within the range of the elements
01468   // input to the shuffle.
01469   unsigned NElts = VT.getVectorNumElements();
01470   SmallVector<int, 8> MaskVec;
01471   for (unsigned i = 0; i != NElts; ++i) {
01472     assert(Mask[i] < (int)(NElts * 2) && "Index out of range");
01473     MaskVec.push_back(Mask[i]);
01474   }
01475 
01476   // Canonicalize shuffle v, v -> v, undef
01477   if (N1 == N2) {
01478     N2 = getUNDEF(VT);
01479     for (unsigned i = 0; i != NElts; ++i)
01480       if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts;
01481   }
01482 
01483   // Canonicalize shuffle undef, v -> v, undef.  Commute the shuffle mask.
01484   if (N1.getOpcode() == ISD::UNDEF)
01485     commuteShuffle(N1, N2, MaskVec);
01486 
01487   // Canonicalize all index into lhs, -> shuffle lhs, undef
01488   // Canonicalize all index into rhs, -> shuffle rhs, undef
01489   bool AllLHS = true, AllRHS = true;
01490   bool N2Undef = N2.getOpcode() == ISD::UNDEF;
01491   for (unsigned i = 0; i != NElts; ++i) {
01492     if (MaskVec[i] >= (int)NElts) {
01493       if (N2Undef)
01494         MaskVec[i] = -1;
01495       else
01496         AllLHS = false;
01497     } else if (MaskVec[i] >= 0) {
01498       AllRHS = false;
01499     }
01500   }
01501   if (AllLHS && AllRHS)
01502     return getUNDEF(VT);
01503   if (AllLHS && !N2Undef)
01504     N2 = getUNDEF(VT);
01505   if (AllRHS) {
01506     N1 = getUNDEF(VT);
01507     commuteShuffle(N1, N2, MaskVec);
01508   }
01509   // Reset our undef status after accounting for the mask.
01510   N2Undef = N2.getOpcode() == ISD::UNDEF;
01511   // Re-check whether both sides ended up undef.
01512   if (N1.getOpcode() == ISD::UNDEF && N2Undef)
01513     return getUNDEF(VT);
01514 
01515   // If Identity shuffle return that node.
01516   bool Identity = true;
01517   for (unsigned i = 0; i != NElts; ++i) {
01518     if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false;
01519   }
01520   if (Identity && NElts)
01521     return N1;
01522 
01523   // Shuffling a constant splat doesn't change the result.
01524   if (N2Undef) {
01525     SDValue V = N1;
01526 
01527     // Look through any bitcasts. We check that these don't change the number
01528     // (and size) of elements and just changes their types.
01529     while (V.getOpcode() == ISD::BITCAST)
01530       V = V->getOperand(0);
01531 
01532     // A splat should always show up as a build vector node.
01533     if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
01534       BitVector UndefElements;
01535       SDValue Splat = BV->getSplatValue(&UndefElements);
01536       // If this is a splat of an undef, shuffling it is also undef.
01537       if (Splat && Splat.getOpcode() == ISD::UNDEF)
01538         return getUNDEF(VT);
01539 
01540       // We only have a splat which can skip shuffles if there is a splatted
01541       // value and no undef lanes rearranged by the shuffle.
01542       if (Splat && UndefElements.none()) {
01543         // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
01544         // number of elements match or the value splatted is a zero constant.
01545         if (V.getValueType().getVectorNumElements() ==
01546             VT.getVectorNumElements())
01547           return N1;
01548         if (auto *C = dyn_cast<ConstantSDNode>(Splat))
01549           if (C->isNullValue())
01550             return N1;
01551       }
01552     }
01553   }
01554 
01555   FoldingSetNodeID ID;
01556   SDValue Ops[2] = { N1, N2 };
01557   AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
01558   for (unsigned i = 0; i != NElts; ++i)
01559     ID.AddInteger(MaskVec[i]);
01560 
01561   void* IP = nullptr;
01562   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01563     return SDValue(E, 0);
01564 
01565   // Allocate the mask array for the node out of the BumpPtrAllocator, since
01566   // SDNode doesn't have access to it.  This memory will be "leaked" when
01567   // the node is deallocated, but recovered when the NodeAllocator is released.
01568   int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
01569   memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int));
01570 
01571   ShuffleVectorSDNode *N =
01572     new (NodeAllocator) ShuffleVectorSDNode(VT, dl.getIROrder(),
01573                                             dl.getDebugLoc(), N1, N2,
01574                                             MaskAlloc);
01575   CSEMap.InsertNode(N, IP);
01576   InsertNode(N);
01577   return SDValue(N, 0);
01578 }
01579 
01580 SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
01581   MVT VT = SV.getSimpleValueType(0);
01582   unsigned NumElems = VT.getVectorNumElements();
01583   SmallVector<int, 8> MaskVec;
01584 
01585   for (unsigned i = 0; i != NumElems; ++i) {
01586     int Idx = SV.getMaskElt(i);
01587     if (Idx >= 0) {
01588       if (Idx < (int)NumElems)
01589         Idx += NumElems;
01590       else
01591         Idx -= NumElems;
01592     }
01593     MaskVec.push_back(Idx);
01594   }
01595 
01596   SDValue Op0 = SV.getOperand(0);
01597   SDValue Op1 = SV.getOperand(1);
01598   return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, &MaskVec[0]);
01599 }
01600 
01601 SDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl,
01602                                        SDValue Val, SDValue DTy,
01603                                        SDValue STy, SDValue Rnd, SDValue Sat,
01604                                        ISD::CvtCode Code) {
01605   // If the src and dest types are the same and the conversion is between
01606   // integer types of the same sign or two floats, no conversion is necessary.
01607   if (DTy == STy &&
01608       (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF))
01609     return Val;
01610 
01611   FoldingSetNodeID ID;
01612   SDValue Ops[] = { Val, DTy, STy, Rnd, Sat };
01613   AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), Ops);
01614   void* IP = nullptr;
01615   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01616     return SDValue(E, 0);
01617 
01618   CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl.getIROrder(),
01619                                                            dl.getDebugLoc(),
01620                                                            Ops, Code);
01621   CSEMap.InsertNode(N, IP);
01622   InsertNode(N);
01623   return SDValue(N, 0);
01624 }
01625 
01626 SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
01627   FoldingSetNodeID ID;
01628   AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
01629   ID.AddInteger(RegNo);
01630   void *IP = nullptr;
01631   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01632     return SDValue(E, 0);
01633 
01634   SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT);
01635   CSEMap.InsertNode(N, IP);
01636   InsertNode(N);
01637   return SDValue(N, 0);
01638 }
01639 
01640 SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
01641   FoldingSetNodeID ID;
01642   AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
01643   ID.AddPointer(RegMask);
01644   void *IP = nullptr;
01645   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01646     return SDValue(E, 0);
01647 
01648   SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask);
01649   CSEMap.InsertNode(N, IP);
01650   InsertNode(N);
01651   return SDValue(N, 0);
01652 }
01653 
01654 SDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) {
01655   FoldingSetNodeID ID;
01656   SDValue Ops[] = { Root };
01657   AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), Ops);
01658   ID.AddPointer(Label);
01659   void *IP = nullptr;
01660   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01661     return SDValue(E, 0);
01662 
01663   SDNode *N = new (NodeAllocator) EHLabelSDNode(dl.getIROrder(),
01664                                                 dl.getDebugLoc(), Root, Label);
01665   CSEMap.InsertNode(N, IP);
01666   InsertNode(N);
01667   return SDValue(N, 0);
01668 }
01669 
01670 
01671 SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
01672                                       int64_t Offset,
01673                                       bool isTarget,
01674                                       unsigned char TargetFlags) {
01675   unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
01676 
01677   FoldingSetNodeID ID;
01678   AddNodeIDNode(ID, Opc, getVTList(VT), None);
01679   ID.AddPointer(BA);
01680   ID.AddInteger(Offset);
01681   ID.AddInteger(TargetFlags);
01682   void *IP = nullptr;
01683   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01684     return SDValue(E, 0);
01685 
01686   SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset,
01687                                                      TargetFlags);
01688   CSEMap.InsertNode(N, IP);
01689   InsertNode(N);
01690   return SDValue(N, 0);
01691 }
01692 
01693 SDValue SelectionDAG::getSrcValue(const Value *V) {
01694   assert((!V || V->getType()->isPointerTy()) &&
01695          "SrcValue is not a pointer?");
01696 
01697   FoldingSetNodeID ID;
01698   AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None);
01699   ID.AddPointer(V);
01700 
01701   void *IP = nullptr;
01702   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01703     return SDValue(E, 0);
01704 
01705   SDNode *N = new (NodeAllocator) SrcValueSDNode(V);
01706   CSEMap.InsertNode(N, IP);
01707   InsertNode(N);
01708   return SDValue(N, 0);
01709 }
01710 
01711 /// getMDNode - Return an MDNodeSDNode which holds an MDNode.
01712 SDValue SelectionDAG::getMDNode(const MDNode *MD) {
01713   FoldingSetNodeID ID;
01714   AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None);
01715   ID.AddPointer(MD);
01716 
01717   void *IP = nullptr;
01718   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01719     return SDValue(E, 0);
01720 
01721   SDNode *N = new (NodeAllocator) MDNodeSDNode(MD);
01722   CSEMap.InsertNode(N, IP);
01723   InsertNode(N);
01724   return SDValue(N, 0);
01725 }
01726 
01727 /// getAddrSpaceCast - Return an AddrSpaceCastSDNode.
01728 SDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr,
01729                                        unsigned SrcAS, unsigned DestAS) {
01730   SDValue Ops[] = {Ptr};
01731   FoldingSetNodeID ID;
01732   AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
01733   ID.AddInteger(SrcAS);
01734   ID.AddInteger(DestAS);
01735 
01736   void *IP = nullptr;
01737   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
01738     return SDValue(E, 0);
01739 
01740   SDNode *N = new (NodeAllocator) AddrSpaceCastSDNode(dl.getIROrder(),
01741                                                       dl.getDebugLoc(),
01742                                                       VT, Ptr, SrcAS, DestAS);
01743   CSEMap.InsertNode(N, IP);
01744   InsertNode(N);
01745   return SDValue(N, 0);
01746 }
01747 
01748 /// getShiftAmountOperand - Return the specified value casted to
01749 /// the target's desired shift amount type.
01750 SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
01751   EVT OpTy = Op.getValueType();
01752   EVT ShTy = TLI->getShiftAmountTy(LHSTy);
01753   if (OpTy == ShTy || OpTy.isVector()) return Op;
01754 
01755   ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ?  ISD::TRUNCATE : ISD::ZERO_EXTEND;
01756   return getNode(Opcode, SDLoc(Op), ShTy, Op);
01757 }
01758 
01759 /// CreateStackTemporary - Create a stack temporary, suitable for holding the
01760 /// specified value type.
01761 SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
01762   MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
01763   unsigned ByteSize = VT.getStoreSize();
01764   Type *Ty = VT.getTypeForEVT(*getContext());
01765   unsigned StackAlign =
01766   std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty), minAlign);
01767 
01768   int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false);
01769   return getFrameIndex(FrameIdx, TLI->getPointerTy());
01770 }
01771 
01772 /// CreateStackTemporary - Create a stack temporary suitable for holding
01773 /// either of the specified value types.
01774 SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
01775   unsigned Bytes = std::max(VT1.getStoreSizeInBits(),
01776                             VT2.getStoreSizeInBits())/8;
01777   Type *Ty1 = VT1.getTypeForEVT(*getContext());
01778   Type *Ty2 = VT2.getTypeForEVT(*getContext());
01779   const DataLayout *TD = TLI->getDataLayout();
01780   unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1),
01781                             TD->getPrefTypeAlignment(Ty2));
01782 
01783   MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo();
01784   int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false);
01785   return getFrameIndex(FrameIdx, TLI->getPointerTy());
01786 }
01787 
01788 SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1,
01789                                 SDValue N2, ISD::CondCode Cond, SDLoc dl) {
01790   // These setcc operations always fold.
01791   switch (Cond) {
01792   default: break;
01793   case ISD::SETFALSE:
01794   case ISD::SETFALSE2: return getConstant(0, VT);
01795   case ISD::SETTRUE:
01796   case ISD::SETTRUE2: {
01797     TargetLowering::BooleanContent Cnt =
01798         TLI->getBooleanContents(N1->getValueType(0));
01799     return getConstant(
01800         Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, VT);
01801   }
01802 
01803   case ISD::SETOEQ:
01804   case ISD::SETOGT:
01805   case ISD::SETOGE:
01806   case ISD::SETOLT:
01807   case ISD::SETOLE:
01808   case ISD::SETONE:
01809   case ISD::SETO:
01810   case ISD::SETUO:
01811   case ISD::SETUEQ:
01812   case ISD::SETUNE:
01813     assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!");
01814     break;
01815   }
01816 
01817   if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) {
01818     const APInt &C2 = N2C->getAPIntValue();
01819     if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
01820       const APInt &C1 = N1C->getAPIntValue();
01821 
01822       switch (Cond) {
01823       default: llvm_unreachable("Unknown integer setcc!");
01824       case ISD::SETEQ:  return getConstant(C1 == C2, VT);
01825       case ISD::SETNE:  return getConstant(C1 != C2, VT);
01826       case ISD::SETULT: return getConstant(C1.ult(C2), VT);
01827       case ISD::SETUGT: return getConstant(C1.ugt(C2), VT);
01828       case ISD::SETULE: return getConstant(C1.ule(C2), VT);
01829       case ISD::SETUGE: return getConstant(C1.uge(C2), VT);
01830       case ISD::SETLT:  return getConstant(C1.slt(C2), VT);
01831       case ISD::SETGT:  return getConstant(C1.sgt(C2), VT);
01832       case ISD::SETLE:  return getConstant(C1.sle(C2), VT);
01833       case ISD::SETGE:  return getConstant(C1.sge(C2), VT);
01834       }
01835     }
01836   }
01837   if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) {
01838     if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) {
01839       APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF());
01840       switch (Cond) {
01841       default: break;
01842       case ISD::SETEQ:  if (R==APFloat::cmpUnordered)
01843                           return getUNDEF(VT);
01844                         // fall through
01845       case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT);
01846       case ISD::SETNE:  if (R==APFloat::cmpUnordered)
01847                           return getUNDEF(VT);
01848                         // fall through
01849       case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan ||
01850                                            R==APFloat::cmpLessThan, VT);
01851       case ISD::SETLT:  if (R==APFloat::cmpUnordered)
01852                           return getUNDEF(VT);
01853                         // fall through
01854       case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT);
01855       case ISD::SETGT:  if (R==APFloat::cmpUnordered)
01856                           return getUNDEF(VT);
01857                         // fall through
01858       case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT);
01859       case ISD::SETLE:  if (R==APFloat::cmpUnordered)
01860                           return getUNDEF(VT);
01861                         // fall through
01862       case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan ||
01863                                            R==APFloat::cmpEqual, VT);
01864       case ISD::SETGE:  if (R==APFloat::cmpUnordered)
01865                           return getUNDEF(VT);
01866                         // fall through
01867       case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan ||
01868                                            R==APFloat::cmpEqual, VT);
01869       case ISD::SETO:   return getConstant(R!=APFloat::cmpUnordered, VT);
01870       case ISD::SETUO:  return getConstant(R==APFloat::cmpUnordered, VT);
01871       case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered ||
01872                                            R==APFloat::cmpEqual, VT);
01873       case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT);
01874       case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered ||
01875                                            R==APFloat::cmpLessThan, VT);
01876       case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan ||
01877                                            R==APFloat::cmpUnordered, VT);
01878       case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT);
01879       case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT);
01880       }
01881     } else {
01882       // Ensure that the constant occurs on the RHS.
01883       ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
01884       MVT CompVT = N1.getValueType().getSimpleVT();
01885       if (!TLI->isCondCodeLegal(SwappedCond, CompVT))
01886         return SDValue();
01887 
01888       return getSetCC(dl, VT, N2, N1, SwappedCond);
01889     }
01890   }
01891 
01892   // Could not fold it.
01893   return SDValue();
01894 }
01895 
01896 /// SignBitIsZero - Return true if the sign bit of Op is known to be zero.  We
01897 /// use this predicate to simplify operations downstream.
01898 bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
01899   // This predicate is not safe for vector operations.
01900   if (Op.getValueType().isVector())
01901     return false;
01902 
01903   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01904   return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth);
01905 }
01906 
01907 /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero.  We use
01908 /// this predicate to simplify operations downstream.  Mask is known to be zero
01909 /// for bits that V cannot have.
01910 bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask,
01911                                      unsigned Depth) const {
01912   APInt KnownZero, KnownOne;
01913   computeKnownBits(Op, KnownZero, KnownOne, Depth);
01914   return (KnownZero & Mask) == Mask;
01915 }
01916 
01917 /// Determine which bits of Op are known to be either zero or one and return
01918 /// them in the KnownZero/KnownOne bitsets.
01919 void SelectionDAG::computeKnownBits(SDValue Op, APInt &KnownZero,
01920                                     APInt &KnownOne, unsigned Depth) const {
01921   unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
01922 
01923   KnownZero = KnownOne = APInt(BitWidth, 0);   // Don't know anything.
01924   if (Depth == 6)
01925     return;  // Limit search depth.
01926 
01927   APInt KnownZero2, KnownOne2;
01928 
01929   switch (Op.getOpcode()) {
01930   case ISD::Constant:
01931     // We know all of the bits for a constant!
01932     KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue();
01933     KnownZero = ~KnownOne;
01934     break;
01935   case ISD::AND:
01936     // If either the LHS or the RHS are Zero, the result is zero.
01937     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01938     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01939 
01940     // Output known-1 bits are only known if set in both the LHS & RHS.
01941     KnownOne &= KnownOne2;
01942     // Output known-0 are known to be clear if zero in either the LHS | RHS.
01943     KnownZero |= KnownZero2;
01944     break;
01945   case ISD::OR:
01946     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01947     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01948 
01949     // Output known-0 bits are only known if clear in both the LHS & RHS.
01950     KnownZero &= KnownZero2;
01951     // Output known-1 are known to be set if set in either the LHS | RHS.
01952     KnownOne |= KnownOne2;
01953     break;
01954   case ISD::XOR: {
01955     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01956     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01957 
01958     // Output known-0 bits are known if clear or set in both the LHS & RHS.
01959     APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2);
01960     // Output known-1 are known to be set if set in only one of the LHS, RHS.
01961     KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2);
01962     KnownZero = KnownZeroOut;
01963     break;
01964   }
01965   case ISD::MUL: {
01966     computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
01967     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01968 
01969     // If low bits are zero in either operand, output low known-0 bits.
01970     // Also compute a conserative estimate for high known-0 bits.
01971     // More trickiness is possible, but this is sufficient for the
01972     // interesting case of alignment computation.
01973     KnownOne.clearAllBits();
01974     unsigned TrailZ = KnownZero.countTrailingOnes() +
01975                       KnownZero2.countTrailingOnes();
01976     unsigned LeadZ =  std::max(KnownZero.countLeadingOnes() +
01977                                KnownZero2.countLeadingOnes(),
01978                                BitWidth) - BitWidth;
01979 
01980     TrailZ = std::min(TrailZ, BitWidth);
01981     LeadZ = std::min(LeadZ, BitWidth);
01982     KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) |
01983                 APInt::getHighBitsSet(BitWidth, LeadZ);
01984     break;
01985   }
01986   case ISD::UDIV: {
01987     // For the purposes of computing leading zeros we can conservatively
01988     // treat a udiv as a logical right shift by the power of 2 known to
01989     // be less than the denominator.
01990     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
01991     unsigned LeadZ = KnownZero2.countLeadingOnes();
01992 
01993     KnownOne2.clearAllBits();
01994     KnownZero2.clearAllBits();
01995     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
01996     unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros();
01997     if (RHSUnknownLeadingOnes != BitWidth)
01998       LeadZ = std::min(BitWidth,
01999                        LeadZ + BitWidth - RHSUnknownLeadingOnes - 1);
02000 
02001     KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ);
02002     break;
02003   }
02004   case ISD::SELECT:
02005     computeKnownBits(Op.getOperand(2), KnownZero, KnownOne, Depth+1);
02006     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02007 
02008     // Only known if known in both the LHS and RHS.
02009     KnownOne &= KnownOne2;
02010     KnownZero &= KnownZero2;
02011     break;
02012   case ISD::SELECT_CC:
02013     computeKnownBits(Op.getOperand(3), KnownZero, KnownOne, Depth+1);
02014     computeKnownBits(Op.getOperand(2), KnownZero2, KnownOne2, Depth+1);
02015 
02016     // Only known if known in both the LHS and RHS.
02017     KnownOne &= KnownOne2;
02018     KnownZero &= KnownZero2;
02019     break;
02020   case ISD::SADDO:
02021   case ISD::UADDO:
02022   case ISD::SSUBO:
02023   case ISD::USUBO:
02024   case ISD::SMULO:
02025   case ISD::UMULO:
02026     if (Op.getResNo() != 1)
02027       break;
02028     // The boolean result conforms to getBooleanContents.
02029     // If we know the result of a setcc has the top bits zero, use this info.
02030     // We know that we have an integer-based boolean since these operations
02031     // are only available for integer.
02032     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
02033             TargetLowering::ZeroOrOneBooleanContent &&
02034         BitWidth > 1)
02035       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02036     break;
02037   case ISD::SETCC:
02038     // If we know the result of a setcc has the top bits zero, use this info.
02039     if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
02040             TargetLowering::ZeroOrOneBooleanContent &&
02041         BitWidth > 1)
02042       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02043     break;
02044   case ISD::SHL:
02045     // (shl X, C1) & C2 == 0   iff   (X & C2 >>u C1) == 0
02046     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02047       unsigned ShAmt = SA->getZExtValue();
02048 
02049       // If the shift count is an invalid immediate, don't do anything.
02050       if (ShAmt >= BitWidth)
02051         break;
02052 
02053       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02054       KnownZero <<= ShAmt;
02055       KnownOne  <<= ShAmt;
02056       // low bits known zero.
02057       KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt);
02058     }
02059     break;
02060   case ISD::SRL:
02061     // (ushr X, C1) & C2 == 0   iff  (-1 >> C1) & C2 == 0
02062     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02063       unsigned ShAmt = SA->getZExtValue();
02064 
02065       // If the shift count is an invalid immediate, don't do anything.
02066       if (ShAmt >= BitWidth)
02067         break;
02068 
02069       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02070       KnownZero = KnownZero.lshr(ShAmt);
02071       KnownOne  = KnownOne.lshr(ShAmt);
02072 
02073       APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
02074       KnownZero |= HighBits;  // High bits known zero.
02075     }
02076     break;
02077   case ISD::SRA:
02078     if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02079       unsigned ShAmt = SA->getZExtValue();
02080 
02081       // If the shift count is an invalid immediate, don't do anything.
02082       if (ShAmt >= BitWidth)
02083         break;
02084 
02085       // If any of the demanded bits are produced by the sign extension, we also
02086       // demand the input sign bit.
02087       APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt);
02088 
02089       computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02090       KnownZero = KnownZero.lshr(ShAmt);
02091       KnownOne  = KnownOne.lshr(ShAmt);
02092 
02093       // Handle the sign bits.
02094       APInt SignBit = APInt::getSignBit(BitWidth);
02095       SignBit = SignBit.lshr(ShAmt);  // Adjust to where it is now in the mask.
02096 
02097       if (KnownZero.intersects(SignBit)) {
02098         KnownZero |= HighBits;  // New bits are known zero.
02099       } else if (KnownOne.intersects(SignBit)) {
02100         KnownOne  |= HighBits;  // New bits are known one.
02101       }
02102     }
02103     break;
02104   case ISD::SIGN_EXTEND_INREG: {
02105     EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
02106     unsigned EBits = EVT.getScalarType().getSizeInBits();
02107 
02108     // Sign extension.  Compute the demanded bits in the result that are not
02109     // present in the input.
02110     APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits);
02111 
02112     APInt InSignBit = APInt::getSignBit(EBits);
02113     APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits);
02114 
02115     // If the sign extended bits are demanded, we know that the sign
02116     // bit is demanded.
02117     InSignBit = InSignBit.zext(BitWidth);
02118     if (NewBits.getBoolValue())
02119       InputDemandedBits |= InSignBit;
02120 
02121     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02122     KnownOne &= InputDemandedBits;
02123     KnownZero &= InputDemandedBits;
02124 
02125     // If the sign bit of the input is known set or clear, then we know the
02126     // top bits of the result.
02127     if (KnownZero.intersects(InSignBit)) {         // Input sign bit known clear
02128       KnownZero |= NewBits;
02129       KnownOne  &= ~NewBits;
02130     } else if (KnownOne.intersects(InSignBit)) {   // Input sign bit known set
02131       KnownOne  |= NewBits;
02132       KnownZero &= ~NewBits;
02133     } else {                              // Input sign bit unknown
02134       KnownZero &= ~NewBits;
02135       KnownOne  &= ~NewBits;
02136     }
02137     break;
02138   }
02139   case ISD::CTTZ:
02140   case ISD::CTTZ_ZERO_UNDEF:
02141   case ISD::CTLZ:
02142   case ISD::CTLZ_ZERO_UNDEF:
02143   case ISD::CTPOP: {
02144     unsigned LowBits = Log2_32(BitWidth)+1;
02145     KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits);
02146     KnownOne.clearAllBits();
02147     break;
02148   }
02149   case ISD::LOAD: {
02150     LoadSDNode *LD = cast<LoadSDNode>(Op);
02151     // If this is a ZEXTLoad and we are looking at the loaded value.
02152     if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
02153       EVT VT = LD->getMemoryVT();
02154       unsigned MemBits = VT.getScalarType().getSizeInBits();
02155       KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
02156     } else if (const MDNode *Ranges = LD->getRanges()) {
02157       computeKnownBitsFromRangeMetadata(*Ranges, KnownZero);
02158     }
02159     break;
02160   }
02161   case ISD::ZERO_EXTEND: {
02162     EVT InVT = Op.getOperand(0).getValueType();
02163     unsigned InBits = InVT.getScalarType().getSizeInBits();
02164     APInt NewBits   = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
02165     KnownZero = KnownZero.trunc(InBits);
02166     KnownOne = KnownOne.trunc(InBits);
02167     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02168     KnownZero = KnownZero.zext(BitWidth);
02169     KnownOne = KnownOne.zext(BitWidth);
02170     KnownZero |= NewBits;
02171     break;
02172   }
02173   case ISD::SIGN_EXTEND: {
02174     EVT InVT = Op.getOperand(0).getValueType();
02175     unsigned InBits = InVT.getScalarType().getSizeInBits();
02176     APInt NewBits   = APInt::getHighBitsSet(BitWidth, BitWidth - InBits);
02177 
02178     KnownZero = KnownZero.trunc(InBits);
02179     KnownOne = KnownOne.trunc(InBits);
02180     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02181 
02182     // Note if the sign bit is known to be zero or one.
02183     bool SignBitKnownZero = KnownZero.isNegative();
02184     bool SignBitKnownOne  = KnownOne.isNegative();
02185 
02186     KnownZero = KnownZero.zext(BitWidth);
02187     KnownOne = KnownOne.zext(BitWidth);
02188 
02189     // If the sign bit is known zero or one, the top bits match.
02190     if (SignBitKnownZero)
02191       KnownZero |= NewBits;
02192     else if (SignBitKnownOne)
02193       KnownOne  |= NewBits;
02194     break;
02195   }
02196   case ISD::ANY_EXTEND: {
02197     EVT InVT = Op.getOperand(0).getValueType();
02198     unsigned InBits = InVT.getScalarType().getSizeInBits();
02199     KnownZero = KnownZero.trunc(InBits);
02200     KnownOne = KnownOne.trunc(InBits);
02201     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02202     KnownZero = KnownZero.zext(BitWidth);
02203     KnownOne = KnownOne.zext(BitWidth);
02204     break;
02205   }
02206   case ISD::TRUNCATE: {
02207     EVT InVT = Op.getOperand(0).getValueType();
02208     unsigned InBits = InVT.getScalarType().getSizeInBits();
02209     KnownZero = KnownZero.zext(InBits);
02210     KnownOne = KnownOne.zext(InBits);
02211     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02212     KnownZero = KnownZero.trunc(BitWidth);
02213     KnownOne = KnownOne.trunc(BitWidth);
02214     break;
02215   }
02216   case ISD::AssertZext: {
02217     EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
02218     APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
02219     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02220     KnownZero |= (~InMask);
02221     KnownOne  &= (~KnownZero);
02222     break;
02223   }
02224   case ISD::FGETSIGN:
02225     // All bits are zero except the low bit.
02226     KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1);
02227     break;
02228 
02229   case ISD::SUB: {
02230     if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) {
02231       // We know that the top bits of C-X are clear if X contains less bits
02232       // than C (i.e. no wrap-around can happen).  For example, 20-X is
02233       // positive if we can prove that X is >= 0 and < 16.
02234       if (CLHS->getAPIntValue().isNonNegative()) {
02235         unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros();
02236         // NLZ can't be BitWidth with no sign bit
02237         APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1);
02238         computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02239 
02240         // If all of the MaskV bits are known to be zero, then we know the
02241         // output top bits are zero, because we now know that the output is
02242         // from [0-C].
02243         if ((KnownZero2 & MaskV) == MaskV) {
02244           unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros();
02245           // Top bits known zero.
02246           KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2);
02247         }
02248       }
02249     }
02250   }
02251   // fall through
02252   case ISD::ADD:
02253   case ISD::ADDE: {
02254     // Output known-0 bits are known if clear or set in both the low clear bits
02255     // common to both LHS & RHS.  For example, 8+(X<<3) is known to have the
02256     // low 3 bits clear.
02257     computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1);
02258     unsigned KnownZeroOut = KnownZero2.countTrailingOnes();
02259 
02260     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02261     KnownZeroOut = std::min(KnownZeroOut,
02262                             KnownZero2.countTrailingOnes());
02263 
02264     if (Op.getOpcode() == ISD::ADD) {
02265       KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut);
02266       break;
02267     }
02268 
02269     // With ADDE, a carry bit may be added in, so we can only use this
02270     // information if we know (at least) that the low two bits are clear.  We
02271     // then return to the caller that the low bit is unknown but that other bits
02272     // are known zero.
02273     if (KnownZeroOut >= 2) // ADDE
02274       KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut);
02275     break;
02276   }
02277   case ISD::SREM:
02278     if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02279       const APInt &RA = Rem->getAPIntValue().abs();
02280       if (RA.isPowerOf2()) {
02281         APInt LowBits = RA - 1;
02282         computeKnownBits(Op.getOperand(0), KnownZero2,KnownOne2,Depth+1);
02283 
02284         // The low bits of the first operand are unchanged by the srem.
02285         KnownZero = KnownZero2 & LowBits;
02286         KnownOne = KnownOne2 & LowBits;
02287 
02288         // If the first operand is non-negative or has all low bits zero, then
02289         // the upper bits are all zero.
02290         if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits))
02291           KnownZero |= ~LowBits;
02292 
02293         // If the first operand is negative and not all low bits are zero, then
02294         // the upper bits are all one.
02295         if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0))
02296           KnownOne |= ~LowBits;
02297         assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?");
02298       }
02299     }
02300     break;
02301   case ISD::UREM: {
02302     if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02303       const APInt &RA = Rem->getAPIntValue();
02304       if (RA.isPowerOf2()) {
02305         APInt LowBits = (RA - 1);
02306         computeKnownBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth + 1);
02307 
02308         // The upper bits are all zero, the lower ones are unchanged.
02309         KnownZero = KnownZero2 | ~LowBits;
02310         KnownOne = KnownOne2 & LowBits;
02311         break;
02312       }
02313     }
02314 
02315     // Since the result is less than or equal to either operand, any leading
02316     // zero bits in either operand must also exist in the result.
02317     computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02318     computeKnownBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1);
02319 
02320     uint32_t Leaders = std::max(KnownZero.countLeadingOnes(),
02321                                 KnownZero2.countLeadingOnes());
02322     KnownOne.clearAllBits();
02323     KnownZero = APInt::getHighBitsSet(BitWidth, Leaders);
02324     break;
02325   }
02326   case ISD::FrameIndex:
02327   case ISD::TargetFrameIndex:
02328     if (unsigned Align = InferPtrAlignment(Op)) {
02329       // The low bits are known zero if the pointer is aligned.
02330       KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align));
02331       break;
02332     }
02333     break;
02334 
02335   default:
02336     if (Op.getOpcode() < ISD::BUILTIN_OP_END)
02337       break;
02338     // Fallthrough
02339   case ISD::INTRINSIC_WO_CHAIN:
02340   case ISD::INTRINSIC_W_CHAIN:
02341   case ISD::INTRINSIC_VOID:
02342     // Allow the target to implement this method for its nodes.
02343     TLI->computeKnownBitsForTargetNode(Op, KnownZero, KnownOne, *this, Depth);
02344     break;
02345   }
02346 
02347   assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?");
02348 }
02349 
02350 /// ComputeNumSignBits - Return the number of times the sign bit of the
02351 /// register is replicated into the other bits.  We know that at least 1 bit
02352 /// is always equal to the sign bit (itself), but other cases can give us
02353 /// information.  For example, immediately after an "SRA X, 2", we know that
02354 /// the top 3 bits are all equal to each other, so we return 3.
02355 unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{
02356   EVT VT = Op.getValueType();
02357   assert(VT.isInteger() && "Invalid VT!");
02358   unsigned VTBits = VT.getScalarType().getSizeInBits();
02359   unsigned Tmp, Tmp2;
02360   unsigned FirstAnswer = 1;
02361 
02362   if (Depth == 6)
02363     return 1;  // Limit search depth.
02364 
02365   switch (Op.getOpcode()) {
02366   default: break;
02367   case ISD::AssertSext:
02368     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
02369     return VTBits-Tmp+1;
02370   case ISD::AssertZext:
02371     Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
02372     return VTBits-Tmp;
02373 
02374   case ISD::Constant: {
02375     const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue();
02376     return Val.getNumSignBits();
02377   }
02378 
02379   case ISD::SIGN_EXTEND:
02380     Tmp =
02381         VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits();
02382     return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp;
02383 
02384   case ISD::SIGN_EXTEND_INREG:
02385     // Max of the input and what this extends.
02386     Tmp =
02387       cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits();
02388     Tmp = VTBits-Tmp+1;
02389 
02390     Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02391     return std::max(Tmp, Tmp2);
02392 
02393   case ISD::SRA:
02394     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02395     // SRA X, C   -> adds C sign bits.
02396     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02397       Tmp += C->getZExtValue();
02398       if (Tmp > VTBits) Tmp = VTBits;
02399     }
02400     return Tmp;
02401   case ISD::SHL:
02402     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02403       // shl destroys sign bits.
02404       Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02405       if (C->getZExtValue() >= VTBits ||      // Bad shift.
02406           C->getZExtValue() >= Tmp) break;    // Shifted all sign bits out.
02407       return Tmp - C->getZExtValue();
02408     }
02409     break;
02410   case ISD::AND:
02411   case ISD::OR:
02412   case ISD::XOR:    // NOT is handled here.
02413     // Logical binary ops preserve the number of sign bits at the worst.
02414     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02415     if (Tmp != 1) {
02416       Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02417       FirstAnswer = std::min(Tmp, Tmp2);
02418       // We computed what we know about the sign bits as our first
02419       // answer. Now proceed to the generic code that uses
02420       // computeKnownBits, and pick whichever answer is better.
02421     }
02422     break;
02423 
02424   case ISD::SELECT:
02425     Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02426     if (Tmp == 1) return 1;  // Early out.
02427     Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1);
02428     return std::min(Tmp, Tmp2);
02429 
02430   case ISD::SADDO:
02431   case ISD::UADDO:
02432   case ISD::SSUBO:
02433   case ISD::USUBO:
02434   case ISD::SMULO:
02435   case ISD::UMULO:
02436     if (Op.getResNo() != 1)
02437       break;
02438     // The boolean result conforms to getBooleanContents.  Fall through.
02439     // If setcc returns 0/-1, all bits are sign bits.
02440     // We know that we have an integer-based boolean since these operations
02441     // are only available for integer.
02442     if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
02443         TargetLowering::ZeroOrNegativeOneBooleanContent)
02444       return VTBits;
02445     break;
02446   case ISD::SETCC:
02447     // If setcc returns 0/-1, all bits are sign bits.
02448     if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
02449         TargetLowering::ZeroOrNegativeOneBooleanContent)
02450       return VTBits;
02451     break;
02452   case ISD::ROTL:
02453   case ISD::ROTR:
02454     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
02455       unsigned RotAmt = C->getZExtValue() & (VTBits-1);
02456 
02457       // Handle rotate right by N like a rotate left by 32-N.
02458       if (Op.getOpcode() == ISD::ROTR)
02459         RotAmt = (VTBits-RotAmt) & (VTBits-1);
02460 
02461       // If we aren't rotating out all of the known-in sign bits, return the
02462       // number that are left.  This handles rotl(sext(x), 1) for example.
02463       Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02464       if (Tmp > RotAmt+1) return Tmp-RotAmt;
02465     }
02466     break;
02467   case ISD::ADD:
02468     // Add can have at most one carry bit.  Thus we know that the output
02469     // is, at worst, one more bit than the inputs.
02470     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02471     if (Tmp == 1) return 1;  // Early out.
02472 
02473     // Special case decrementing a value (ADD X, -1):
02474     if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
02475       if (CRHS->isAllOnesValue()) {
02476         APInt KnownZero, KnownOne;
02477         computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
02478 
02479         // If the input is known to be 0 or 1, the output is 0/-1, which is all
02480         // sign bits set.
02481         if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
02482           return VTBits;
02483 
02484         // If we are subtracting one from a positive number, there is no carry
02485         // out of the result.
02486         if (KnownZero.isNegative())
02487           return Tmp;
02488       }
02489 
02490     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02491     if (Tmp2 == 1) return 1;
02492     return std::min(Tmp, Tmp2)-1;
02493 
02494   case ISD::SUB:
02495     Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
02496     if (Tmp2 == 1) return 1;
02497 
02498     // Handle NEG.
02499     if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0)))
02500       if (CLHS->isNullValue()) {
02501         APInt KnownZero, KnownOne;
02502         computeKnownBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1);
02503         // If the input is known to be 0 or 1, the output is 0/-1, which is all
02504         // sign bits set.
02505         if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue())
02506           return VTBits;
02507 
02508         // If the input is known to be positive (the sign bit is known clear),
02509         // the output of the NEG has the same number of sign bits as the input.
02510         if (KnownZero.isNegative())
02511           return Tmp2;
02512 
02513         // Otherwise, we treat this like a SUB.
02514       }
02515 
02516     // Sub can have at most one carry bit.  Thus we know that the output
02517     // is, at worst, one more bit than the inputs.
02518     Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
02519     if (Tmp == 1) return 1;  // Early out.
02520     return std::min(Tmp, Tmp2)-1;
02521   case ISD::TRUNCATE:
02522     // FIXME: it's tricky to do anything useful for this, but it is an important
02523     // case for targets like X86.
02524     break;
02525   }
02526 
02527   // If we are looking at the loaded value of the SDNode.
02528   if (Op.getResNo() == 0) {
02529     // Handle LOADX separately here. EXTLOAD case will fallthrough.
02530     if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
02531       unsigned ExtType = LD->getExtensionType();
02532       switch (ExtType) {
02533         default: break;
02534         case ISD::SEXTLOAD:    // '17' bits known
02535           Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
02536           return VTBits-Tmp+1;
02537         case ISD::ZEXTLOAD:    // '16' bits known
02538           Tmp = LD->getMemoryVT().getScalarType().getSizeInBits();
02539           return VTBits-Tmp;
02540       }
02541     }
02542   }
02543 
02544   // Allow the target to implement this method for its nodes.
02545   if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
02546       Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
02547       Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
02548       Op.getOpcode() == ISD::INTRINSIC_VOID) {
02549     unsigned NumBits = TLI->ComputeNumSignBitsForTargetNode(Op, *this, Depth);
02550     if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits);
02551   }
02552 
02553   // Finally, if we can prove that the top bits of the result are 0's or 1's,
02554   // use this information.
02555   APInt KnownZero, KnownOne;
02556   computeKnownBits(Op, KnownZero, KnownOne, Depth);
02557 
02558   APInt Mask;
02559   if (KnownZero.isNegative()) {        // sign bit is 0
02560     Mask = KnownZero;
02561   } else if (KnownOne.isNegative()) {  // sign bit is 1;
02562     Mask = KnownOne;
02563   } else {
02564     // Nothing known.
02565     return FirstAnswer;
02566   }
02567 
02568   // Okay, we know that the sign bit in Mask is set.  Use CLZ to determine
02569   // the number of identical bits in the top of the input value.
02570   Mask = ~Mask;
02571   Mask <<= Mask.getBitWidth()-VTBits;
02572   // Return # leading zeros.  We use 'min' here in case Val was zero before
02573   // shifting.  We don't want to return '64' as for an i32 "0".
02574   return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros()));
02575 }
02576 
02577 /// isBaseWithConstantOffset - Return true if the specified operand is an
02578 /// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an
02579 /// ISD::OR with a ConstantSDNode that is guaranteed to have the same
02580 /// semantics as an ADD.  This handles the equivalence:
02581 ///     X|Cst == X+Cst iff X&Cst = 0.
02582 bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
02583   if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
02584       !isa<ConstantSDNode>(Op.getOperand(1)))
02585     return false;
02586 
02587   if (Op.getOpcode() == ISD::OR &&
02588       !MaskedValueIsZero(Op.getOperand(0),
02589                      cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue()))
02590     return false;
02591 
02592   return true;
02593 }
02594 
02595 
02596 bool SelectionDAG::isKnownNeverNaN(SDValue Op) const {
02597   // If we're told that NaNs won't happen, assume they won't.
02598   if (getTarget().Options.NoNaNsFPMath)
02599     return true;
02600 
02601   // If the value is a constant, we can obviously see if it is a NaN or not.
02602   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
02603     return !C->getValueAPF().isNaN();
02604 
02605   // TODO: Recognize more cases here.
02606 
02607   return false;
02608 }
02609 
02610 bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
02611   // If the value is a constant, we can obviously see if it is a zero or not.
02612   if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
02613     return !C->isZero();
02614 
02615   // TODO: Recognize more cases here.
02616   switch (Op.getOpcode()) {
02617   default: break;
02618   case ISD::OR:
02619     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
02620       return !C->isNullValue();
02621     break;
02622   }
02623 
02624   return false;
02625 }
02626 
02627 bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
02628   // Check the obvious case.
02629   if (A == B) return true;
02630 
02631   // For for negative and positive zero.
02632   if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
02633     if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
02634       if (CA->isZero() && CB->isZero()) return true;
02635 
02636   // Otherwise they may not be equal.
02637   return false;
02638 }
02639 
02640 /// getNode - Gets or creates the specified node.
02641 ///
02642 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT) {
02643   FoldingSetNodeID ID;
02644   AddNodeIDNode(ID, Opcode, getVTList(VT), None);
02645   void *IP = nullptr;
02646   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
02647     return SDValue(E, 0);
02648 
02649   SDNode *N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(),
02650                                          DL.getDebugLoc(), getVTList(VT));
02651   CSEMap.InsertNode(N, IP);
02652 
02653   InsertNode(N);
02654   return SDValue(N, 0);
02655 }
02656 
02657 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
02658                               EVT VT, SDValue Operand) {
02659   // Constant fold unary operations with an integer constant operand. Even
02660   // opaque constant will be folded, because the folding of unary operations
02661   // doesn't create new constants with different values. Nevertheless, the
02662   // opaque flag is preserved during folding to prevent future folding with
02663   // other constants.
02664   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) {
02665     const APInt &Val = C->getAPIntValue();
02666     switch (Opcode) {
02667     default: break;
02668     case ISD::SIGN_EXTEND:
02669       return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT,
02670                          C->isTargetOpcode(), C->isOpaque());
02671     case ISD::ANY_EXTEND:
02672     case ISD::ZERO_EXTEND:
02673     case ISD::TRUNCATE:
02674       return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT,
02675                          C->isTargetOpcode(), C->isOpaque());
02676     case ISD::UINT_TO_FP:
02677     case ISD::SINT_TO_FP: {
02678       APFloat apf(EVTToAPFloatSemantics(VT),
02679                   APInt::getNullValue(VT.getSizeInBits()));
02680       (void)apf.convertFromAPInt(Val,
02681                                  Opcode==ISD::SINT_TO_FP,
02682                                  APFloat::rmNearestTiesToEven);
02683       return getConstantFP(apf, VT);
02684     }
02685     case ISD::BITCAST:
02686       if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
02687         return getConstantFP(APFloat(APFloat::IEEEsingle, Val), VT);
02688       else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
02689         return getConstantFP(APFloat(APFloat::IEEEdouble, Val), VT);
02690       break;
02691     case ISD::BSWAP:
02692       return getConstant(Val.byteSwap(), VT, C->isTargetOpcode(),
02693                          C->isOpaque());
02694     case ISD::CTPOP:
02695       return getConstant(Val.countPopulation(), VT, C->isTargetOpcode(),
02696                          C->isOpaque());
02697     case ISD::CTLZ:
02698     case ISD::CTLZ_ZERO_UNDEF:
02699       return getConstant(Val.countLeadingZeros(), VT, C->isTargetOpcode(),
02700                          C->isOpaque());
02701     case ISD::CTTZ:
02702     case ISD::CTTZ_ZERO_UNDEF:
02703       return getConstant(Val.countTrailingZeros(), VT, C->isTargetOpcode(),
02704                          C->isOpaque());
02705     }
02706   }
02707 
02708   // Constant fold unary operations with a floating point constant operand.
02709   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) {
02710     APFloat V = C->getValueAPF();    // make copy
02711     switch (Opcode) {
02712     case ISD::FNEG:
02713       V.changeSign();
02714       return getConstantFP(V, VT);
02715     case ISD::FABS:
02716       V.clearSign();
02717       return getConstantFP(V, VT);
02718     case ISD::FCEIL: {
02719       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
02720       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02721         return getConstantFP(V, VT);
02722       break;
02723     }
02724     case ISD::FTRUNC: {
02725       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
02726       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02727         return getConstantFP(V, VT);
02728       break;
02729     }
02730     case ISD::FFLOOR: {
02731       APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
02732       if (fs == APFloat::opOK || fs == APFloat::opInexact)
02733         return getConstantFP(V, VT);
02734       break;
02735     }
02736     case ISD::FP_EXTEND: {
02737       bool ignored;
02738       // This can return overflow, underflow, or inexact; we don't care.
02739       // FIXME need to be more flexible about rounding mode.
02740       (void)V.convert(EVTToAPFloatSemantics(VT),
02741                       APFloat::rmNearestTiesToEven, &ignored);
02742       return getConstantFP(V, VT);
02743     }
02744     case ISD::FP_TO_SINT:
02745     case ISD::FP_TO_UINT: {
02746       integerPart x[2];
02747       bool ignored;
02748       static_assert(integerPartWidth >= 64, "APFloat parts too small!");
02749       // FIXME need to be more flexible about rounding mode.
02750       APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(),
02751                             Opcode==ISD::FP_TO_SINT,
02752                             APFloat::rmTowardZero, &ignored);
02753       if (s==APFloat::opInvalidOp)     // inexact is OK, in fact usual
02754         break;
02755       APInt api(VT.getSizeInBits(), x);
02756       return getConstant(api, VT);
02757     }
02758     case ISD::BITCAST:
02759       if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
02760         return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT);
02761       else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
02762         return getConstant(V.bitcastToAPInt().getZExtValue(), VT);
02763       break;
02764     }
02765   }
02766 
02767   // Constant fold unary operations with a vector integer operand.
02768   if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand.getNode())) {
02769     if (BV->isConstant()) {
02770       switch (Opcode) {
02771       default:
02772         // FIXME: Entirely reasonable to perform folding of other unary
02773         // operations here as the need arises.
02774         break;
02775       case ISD::UINT_TO_FP:
02776       case ISD::SINT_TO_FP: {
02777         SmallVector<SDValue, 8> Ops;
02778         for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
02779           SDValue OpN = BV->getOperand(i);
02780           // Let the above scalar folding handle the conversion of each
02781           // element.
02782           OpN = getNode(ISD::SINT_TO_FP, DL, VT.getVectorElementType(),
02783                         OpN);
02784           Ops.push_back(OpN);
02785         }
02786         return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
02787       }
02788       }
02789     }
02790   }
02791 
02792   unsigned OpOpcode = Operand.getNode()->getOpcode();
02793   switch (Opcode) {
02794   case ISD::TokenFactor:
02795   case ISD::MERGE_VALUES:
02796   case ISD::CONCAT_VECTORS:
02797     return Operand;         // Factor, merge or concat of one node?  No need.
02798   case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
02799   case ISD::FP_EXTEND:
02800     assert(VT.isFloatingPoint() &&
02801            Operand.getValueType().isFloatingPoint() && "Invalid FP cast!");
02802     if (Operand.getValueType() == VT) return Operand;  // noop conversion.
02803     assert((!VT.isVector() ||
02804             VT.getVectorNumElements() ==
02805             Operand.getValueType().getVectorNumElements()) &&
02806            "Vector element count mismatch!");
02807     if (Operand.getOpcode() == ISD::UNDEF)
02808       return getUNDEF(VT);
02809     break;
02810   case ISD::SIGN_EXTEND:
02811     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02812            "Invalid SIGN_EXTEND!");
02813     if (Operand.getValueType() == VT) return Operand;   // noop extension
02814     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02815            "Invalid sext node, dst < src!");
02816     assert((!VT.isVector() ||
02817             VT.getVectorNumElements() ==
02818             Operand.getValueType().getVectorNumElements()) &&
02819            "Vector element count mismatch!");
02820     if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND)
02821       return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02822     else if (OpOpcode == ISD::UNDEF)
02823       // sext(undef) = 0, because the top bits will all be the same.
02824       return getConstant(0, VT);
02825     break;
02826   case ISD::ZERO_EXTEND:
02827     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02828            "Invalid ZERO_EXTEND!");
02829     if (Operand.getValueType() == VT) return Operand;   // noop extension
02830     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02831            "Invalid zext node, dst < src!");
02832     assert((!VT.isVector() ||
02833             VT.getVectorNumElements() ==
02834             Operand.getValueType().getVectorNumElements()) &&
02835            "Vector element count mismatch!");
02836     if (OpOpcode == ISD::ZERO_EXTEND)   // (zext (zext x)) -> (zext x)
02837       return getNode(ISD::ZERO_EXTEND, DL, VT,
02838                      Operand.getNode()->getOperand(0));
02839     else if (OpOpcode == ISD::UNDEF)
02840       // zext(undef) = 0, because the top bits will be zero.
02841       return getConstant(0, VT);
02842     break;
02843   case ISD::ANY_EXTEND:
02844     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02845            "Invalid ANY_EXTEND!");
02846     if (Operand.getValueType() == VT) return Operand;   // noop extension
02847     assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) &&
02848            "Invalid anyext node, dst < src!");
02849     assert((!VT.isVector() ||
02850             VT.getVectorNumElements() ==
02851             Operand.getValueType().getVectorNumElements()) &&
02852            "Vector element count mismatch!");
02853 
02854     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
02855         OpOpcode == ISD::ANY_EXTEND)
02856       // (ext (zext x)) -> (zext x)  and  (ext (sext x)) -> (sext x)
02857       return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02858     else if (OpOpcode == ISD::UNDEF)
02859       return getUNDEF(VT);
02860 
02861     // (ext (trunx x)) -> x
02862     if (OpOpcode == ISD::TRUNCATE) {
02863       SDValue OpOp = Operand.getNode()->getOperand(0);
02864       if (OpOp.getValueType() == VT)
02865         return OpOp;
02866     }
02867     break;
02868   case ISD::TRUNCATE:
02869     assert(VT.isInteger() && Operand.getValueType().isInteger() &&
02870            "Invalid TRUNCATE!");
02871     if (Operand.getValueType() == VT) return Operand;   // noop truncate
02872     assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) &&
02873            "Invalid truncate node, src < dst!");
02874     assert((!VT.isVector() ||
02875             VT.getVectorNumElements() ==
02876             Operand.getValueType().getVectorNumElements()) &&
02877            "Vector element count mismatch!");
02878     if (OpOpcode == ISD::TRUNCATE)
02879       return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
02880     if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
02881         OpOpcode == ISD::ANY_EXTEND) {
02882       // If the source is smaller than the dest, we still need an extend.
02883       if (Operand.getNode()->getOperand(0).getValueType().getScalarType()
02884             .bitsLT(VT.getScalarType()))
02885         return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0));
02886       if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT))
02887         return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0));
02888       return Operand.getNode()->getOperand(0);
02889     }
02890     if (OpOpcode == ISD::UNDEF)
02891       return getUNDEF(VT);
02892     break;
02893   case ISD::BITCAST:
02894     // Basic sanity checking.
02895     assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits()
02896            && "Cannot BITCAST between types of different sizes!");
02897     if (VT == Operand.getValueType()) return Operand;  // noop conversion.
02898     if (OpOpcode == ISD::BITCAST)  // bitconv(bitconv(x)) -> bitconv(x)
02899       return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
02900     if (OpOpcode == ISD::UNDEF)
02901       return getUNDEF(VT);
02902     break;
02903   case ISD::SCALAR_TO_VECTOR:
02904     assert(VT.isVector() && !Operand.getValueType().isVector() &&
02905            (VT.getVectorElementType() == Operand.getValueType() ||
02906             (VT.getVectorElementType().isInteger() &&
02907              Operand.getValueType().isInteger() &&
02908              VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&
02909            "Illegal SCALAR_TO_VECTOR node!");
02910     if (OpOpcode == ISD::UNDEF)
02911       return getUNDEF(VT);
02912     // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
02913     if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
02914         isa<ConstantSDNode>(Operand.getOperand(1)) &&
02915         Operand.getConstantOperandVal(1) == 0 &&
02916         Operand.getOperand(0).getValueType() == VT)
02917       return Operand.getOperand(0);
02918     break;
02919   case ISD::FNEG:
02920     // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0
02921     if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB)
02922       return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1),
02923                      Operand.getNode()->getOperand(0));
02924     if (OpOpcode == ISD::FNEG)  // --X -> X
02925       return Operand.getNode()->getOperand(0);
02926     break;
02927   case ISD::FABS:
02928     if (OpOpcode == ISD::FNEG)  // abs(-X) -> abs(X)
02929       return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0));
02930     break;
02931   }
02932 
02933   SDNode *N;
02934   SDVTList VTs = getVTList(VT);
02935   if (VT != MVT::Glue) { // Don't CSE flag producing nodes
02936     FoldingSetNodeID ID;
02937     SDValue Ops[1] = { Operand };
02938     AddNodeIDNode(ID, Opcode, VTs, Ops);
02939     void *IP = nullptr;
02940     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
02941       return SDValue(E, 0);
02942 
02943     N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
02944                                         DL.getDebugLoc(), VTs, Operand);
02945     CSEMap.InsertNode(N, IP);
02946   } else {
02947     N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
02948                                         DL.getDebugLoc(), VTs, Operand);
02949   }
02950 
02951   InsertNode(N);
02952   return SDValue(N, 0);
02953 }
02954 
02955 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT,
02956                                              SDNode *Cst1, SDNode *Cst2) {
02957   // If the opcode is a target-specific ISD node, there's nothing we can
02958   // do here and the operand rules may not line up with the below, so
02959   // bail early.
02960   if (Opcode >= ISD::BUILTIN_OP_END)
02961     return SDValue();
02962 
02963   SmallVector<std::pair<ConstantSDNode *, ConstantSDNode *>, 4> Inputs;
02964   SmallVector<SDValue, 4> Outputs;
02965   EVT SVT = VT.getScalarType();
02966 
02967   ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1);
02968   ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2);
02969   if (Scalar1 && Scalar2 && (Scalar1->isOpaque() || Scalar2->isOpaque()))
02970     return SDValue();
02971 
02972   if (Scalar1 && Scalar2)
02973     // Scalar instruction.
02974     Inputs.push_back(std::make_pair(Scalar1, Scalar2));
02975   else {
02976     // For vectors extract each constant element into Inputs so we can constant
02977     // fold them individually.
02978     BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1);
02979     BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2);
02980     if (!BV1 || !BV2)
02981       return SDValue();
02982 
02983     assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!");
02984 
02985     for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) {
02986       ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I));
02987       ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I));
02988       if (!V1 || !V2) // Not a constant, bail.
02989         return SDValue();
02990 
02991       if (V1->isOpaque() || V2->isOpaque())
02992         return SDValue();
02993 
02994       // Avoid BUILD_VECTOR nodes that perform implicit truncation.
02995       // FIXME: This is valid and could be handled by truncating the APInts.
02996       if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
02997         return SDValue();
02998 
02999       Inputs.push_back(std::make_pair(V1, V2));
03000     }
03001   }
03002 
03003   // We have a number of constant values, constant fold them element by element.
03004   for (unsigned I = 0, E = Inputs.size(); I != E; ++I) {
03005     const APInt &C1 = Inputs[I].first->getAPIntValue();
03006     const APInt &C2 = Inputs[I].second->getAPIntValue();
03007 
03008     switch (Opcode) {
03009     case ISD::ADD:
03010       Outputs.push_back(getConstant(C1 + C2, SVT));
03011       break;
03012     case ISD::SUB:
03013       Outputs.push_back(getConstant(C1 - C2, SVT));
03014       break;
03015     case ISD::MUL:
03016       Outputs.push_back(getConstant(C1 * C2, SVT));
03017       break;
03018     case ISD::UDIV:
03019       if (!C2.getBoolValue())
03020         return SDValue();
03021       Outputs.push_back(getConstant(C1.udiv(C2), SVT));
03022       break;
03023     case ISD::UREM:
03024       if (!C2.getBoolValue())
03025         return SDValue();
03026       Outputs.push_back(getConstant(C1.urem(C2), SVT));
03027       break;
03028     case ISD::SDIV:
03029       if (!C2.getBoolValue())
03030         return SDValue();
03031       Outputs.push_back(getConstant(C1.sdiv(C2), SVT));
03032       break;
03033     case ISD::SREM:
03034       if (!C2.getBoolValue())
03035         return SDValue();
03036       Outputs.push_back(getConstant(C1.srem(C2), SVT));
03037       break;
03038     case ISD::AND:
03039       Outputs.push_back(getConstant(C1 & C2, SVT));
03040       break;
03041     case ISD::OR:
03042       Outputs.push_back(getConstant(C1 | C2, SVT));
03043       break;
03044     case ISD::XOR:
03045       Outputs.push_back(getConstant(C1 ^ C2, SVT));
03046       break;
03047     case ISD::SHL:
03048       Outputs.push_back(getConstant(C1 << C2, SVT));
03049       break;
03050     case ISD::SRL:
03051       Outputs.push_back(getConstant(C1.lshr(C2), SVT));
03052       break;
03053     case ISD::SRA:
03054       Outputs.push_back(getConstant(C1.ashr(C2), SVT));
03055       break;
03056     case ISD::ROTL:
03057       Outputs.push_back(getConstant(C1.rotl(C2), SVT));
03058       break;
03059     case ISD::ROTR:
03060       Outputs.push_back(getConstant(C1.rotr(C2), SVT));
03061       break;
03062     default:
03063       return SDValue();
03064     }
03065   }
03066 
03067   assert((Scalar1 && Scalar2) || (VT.getVectorNumElements() == Outputs.size() &&
03068                                   "Expected a scalar or vector!"));
03069 
03070   // Handle the scalar case first.
03071   if (!VT.isVector())
03072     return Outputs.back();
03073 
03074   // We may have a vector type but a scalar result. Create a splat.
03075   Outputs.resize(VT.getVectorNumElements(), Outputs.back());
03076 
03077   // Build a big vector out of the scalar elements we generated.
03078   return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs);
03079 }
03080 
03081 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
03082                               SDValue N2, bool nuw, bool nsw, bool exact) {
03083   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
03084   ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode());
03085   switch (Opcode) {
03086   default: break;
03087   case ISD::TokenFactor:
03088     assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
03089            N2.getValueType() == MVT::Other && "Invalid token factor!");
03090     // Fold trivial token factors.
03091     if (N1.getOpcode() == ISD::EntryToken) return N2;
03092     if (N2.getOpcode() == ISD::EntryToken) return N1;
03093     if (N1 == N2) return N1;
03094     break;
03095   case ISD::CONCAT_VECTORS:
03096     // Concat of UNDEFs is UNDEF.
03097     if (N1.getOpcode() == ISD::UNDEF &&
03098         N2.getOpcode() == ISD::UNDEF)
03099       return getUNDEF(VT);
03100 
03101     // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
03102     // one big BUILD_VECTOR.
03103     if (N1.getOpcode() == ISD::BUILD_VECTOR &&
03104         N2.getOpcode() == ISD::BUILD_VECTOR) {
03105       SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
03106                                     N1.getNode()->op_end());
03107       Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
03108       return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
03109     }
03110     break;
03111   case ISD::AND:
03112     assert(VT.isInteger() && "This operator does not apply to FP types!");
03113     assert(N1.getValueType() == N2.getValueType() &&
03114            N1.getValueType() == VT && "Binary operator types must match!");
03115     // (X & 0) -> 0.  This commonly occurs when legalizing i64 values, so it's
03116     // worth handling here.
03117     if (N2C && N2C->isNullValue())
03118       return N2;
03119     if (N2C && N2C->isAllOnesValue())  // X & -1 -> X
03120       return N1;
03121     break;
03122   case ISD::OR:
03123   case ISD::XOR:
03124   case ISD::ADD:
03125   case ISD::SUB:
03126     assert(VT.isInteger() && "This operator does not apply to FP types!");
03127     assert(N1.getValueType() == N2.getValueType() &&
03128            N1.getValueType() == VT && "Binary operator types must match!");
03129     // (X ^|+- 0) -> X.  This commonly occurs when legalizing i64 values, so
03130     // it's worth handling here.
03131     if (N2C && N2C->isNullValue())
03132       return N1;
03133     break;
03134   case ISD::UDIV:
03135   case ISD::UREM:
03136   case ISD::MULHU:
03137   case ISD::MULHS:
03138   case ISD::MUL:
03139   case ISD::SDIV:
03140   case ISD::SREM:
03141     assert(VT.isInteger() && "This operator does not apply to FP types!");
03142     assert(N1.getValueType() == N2.getValueType() &&
03143            N1.getValueType() == VT && "Binary operator types must match!");
03144     break;
03145   case ISD::FADD:
03146   case ISD::FSUB:
03147   case ISD::FMUL:
03148   case ISD::FDIV:
03149   case ISD::FREM:
03150     if (getTarget().Options.UnsafeFPMath) {
03151       if (Opcode == ISD::FADD) {
03152         // 0+x --> x
03153         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1))
03154           if (CFP->getValueAPF().isZero())
03155             return N2;
03156         // x+0 --> x
03157         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
03158           if (CFP->getValueAPF().isZero())
03159             return N1;
03160       } else if (Opcode == ISD::FSUB) {
03161         // x-0 --> x
03162         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
03163           if (CFP->getValueAPF().isZero())
03164             return N1;
03165       } else if (Opcode == ISD::FMUL) {
03166         ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1);
03167         SDValue V = N2;
03168 
03169         // If the first operand isn't the constant, try the second
03170         if (!CFP) {
03171           CFP = dyn_cast<ConstantFPSDNode>(N2);
03172           V = N1;
03173         }
03174 
03175         if (CFP) {
03176           // 0*x --> 0
03177           if (CFP->isZero())
03178             return SDValue(CFP,0);
03179           // 1*x --> x
03180           if (CFP->isExactlyValue(1.0))
03181             return V;
03182         }
03183       }
03184     }
03185     assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
03186     assert(N1.getValueType() == N2.getValueType() &&
03187            N1.getValueType() == VT && "Binary operator types must match!");
03188     break;
03189   case ISD::FCOPYSIGN:   // N1 and result must match.  N1/N2 need not match.
03190     assert(N1.getValueType() == VT &&
03191            N1.getValueType().isFloatingPoint() &&
03192            N2.getValueType().isFloatingPoint() &&
03193            "Invalid FCOPYSIGN!");
03194     break;
03195   case ISD::SHL:
03196   case ISD::SRA:
03197   case ISD::SRL:
03198   case ISD::ROTL:
03199   case ISD::ROTR:
03200     assert(VT == N1.getValueType() &&
03201            "Shift operators return type must be the same as their first arg");
03202     assert(VT.isInteger() && N2.getValueType().isInteger() &&
03203            "Shifts only work on integers");
03204     assert((!VT.isVector() || VT == N2.getValueType()) &&
03205            "Vector shift amounts must be in the same as their first arg");
03206     // Verify that the shift amount VT is bit enough to hold valid shift
03207     // amounts.  This catches things like trying to shift an i1024 value by an
03208     // i8, which is easy to fall into in generic code that uses
03209     // TLI.getShiftAmount().
03210     assert(N2.getValueType().getSizeInBits() >=
03211                    Log2_32_Ceil(N1.getValueType().getSizeInBits()) &&
03212            "Invalid use of small shift amount with oversized value!");
03213 
03214     // Always fold shifts of i1 values so the code generator doesn't need to
03215     // handle them.  Since we know the size of the shift has to be less than the
03216     // size of the value, the shift/rotate count is guaranteed to be zero.
03217     if (VT == MVT::i1)
03218       return N1;
03219     if (N2C && N2C->isNullValue())
03220       return N1;
03221     break;
03222   case ISD::FP_ROUND_INREG: {
03223     EVT EVT = cast<VTSDNode>(N2)->getVT();
03224     assert(VT == N1.getValueType() && "Not an inreg round!");
03225     assert(VT.isFloatingPoint() && EVT.isFloatingPoint() &&
03226            "Cannot FP_ROUND_INREG integer types");
03227     assert(EVT.isVector() == VT.isVector() &&
03228            "FP_ROUND_INREG type should be vector iff the operand "
03229            "type is vector!");
03230     assert((!EVT.isVector() ||
03231             EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
03232            "Vector element counts must match in FP_ROUND_INREG");
03233     assert(EVT.bitsLE(VT) && "Not rounding down!");
03234     (void)EVT;
03235     if (cast<VTSDNode>(N2)->getVT() == VT) return N1;  // Not actually rounding.
03236     break;
03237   }
03238   case ISD::FP_ROUND:
03239     assert(VT.isFloatingPoint() &&
03240            N1.getValueType().isFloatingPoint() &&
03241            VT.bitsLE(N1.getValueType()) &&
03242            isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!");
03243     if (N1.getValueType() == VT) return N1;  // noop conversion.
03244     break;
03245   case ISD::AssertSext:
03246   case ISD::AssertZext: {
03247     EVT EVT = cast<VTSDNode>(N2)->getVT();
03248     assert(VT == N1.getValueType() && "Not an inreg extend!");
03249     assert(VT.isInteger() && EVT.isInteger() &&
03250            "Cannot *_EXTEND_INREG FP types");
03251     assert(!EVT.isVector() &&
03252            "AssertSExt/AssertZExt type should be the vector element type "
03253            "rather than the vector type!");
03254     assert(EVT.bitsLE(VT) && "Not extending!");
03255     if (VT == EVT) return N1; // noop assertion.
03256     break;
03257   }
03258   case ISD::SIGN_EXTEND_INREG: {
03259     EVT EVT = cast<VTSDNode>(N2)->getVT();
03260     assert(VT == N1.getValueType() && "Not an inreg extend!");
03261     assert(VT.isInteger() && EVT.isInteger() &&
03262            "Cannot *_EXTEND_INREG FP types");
03263     assert(EVT.isVector() == VT.isVector() &&
03264            "SIGN_EXTEND_INREG type should be vector iff the operand "
03265            "type is vector!");
03266     assert((!EVT.isVector() ||
03267             EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
03268            "Vector element counts must match in SIGN_EXTEND_INREG");
03269     assert(EVT.bitsLE(VT) && "Not extending!");
03270     if (EVT == VT) return N1;  // Not actually extending
03271 
03272     if (N1C) {
03273       APInt Val = N1C->getAPIntValue();
03274       unsigned FromBits = EVT.getScalarType().getSizeInBits();
03275       Val <<= Val.getBitWidth()-FromBits;
03276       Val = Val.ashr(Val.getBitWidth()-FromBits);
03277       return getConstant(Val, VT);
03278     }
03279     break;
03280   }
03281   case ISD::EXTRACT_VECTOR_ELT:
03282     // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF.
03283     if (N1.getOpcode() == ISD::UNDEF)
03284       return getUNDEF(VT);
03285 
03286     // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
03287     // expanding copies of large vectors from registers.
03288     if (N2C &&
03289         N1.getOpcode() == ISD::CONCAT_VECTORS &&
03290         N1.getNumOperands() > 0) {
03291       unsigned Factor =
03292         N1.getOperand(0).getValueType().getVectorNumElements();
03293       return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
03294                      N1.getOperand(N2C->getZExtValue() / Factor),
03295                      getConstant(N2C->getZExtValue() % Factor,
03296                                  N2.getValueType()));
03297     }
03298 
03299     // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
03300     // expanding large vector constants.
03301     if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) {
03302       SDValue Elt = N1.getOperand(N2C->getZExtValue());
03303 
03304       if (VT != Elt.getValueType())
03305         // If the vector element type is not legal, the BUILD_VECTOR operands
03306         // are promoted and implicitly truncated, and the result implicitly
03307         // extended. Make that explicit here.
03308         Elt = getAnyExtOrTrunc(Elt, DL, VT);
03309 
03310       return Elt;
03311     }
03312 
03313     // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
03314     // operations are lowered to scalars.
03315     if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
03316       // If the indices are the same, return the inserted element else
03317       // if the indices are known different, extract the element from
03318       // the original vector.
03319       SDValue N1Op2 = N1.getOperand(2);
03320       ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode());
03321 
03322       if (N1Op2C && N2C) {
03323         if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
03324           if (VT == N1.getOperand(1).getValueType())
03325             return N1.getOperand(1);
03326           else
03327             return getSExtOrTrunc(N1.getOperand(1), DL, VT);
03328         }
03329 
03330         return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
03331       }
03332     }
03333     break;
03334   case ISD::EXTRACT_ELEMENT:
03335     assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
03336     assert(!N1.getValueType().isVector() && !VT.isVector() &&
03337            (N1.getValueType().isInteger() == VT.isInteger()) &&
03338            N1.getValueType() != VT &&
03339            "Wrong types for EXTRACT_ELEMENT!");
03340 
03341     // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
03342     // 64-bit integers into 32-bit parts.  Instead of building the extract of
03343     // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
03344     if (N1.getOpcode() == ISD::BUILD_PAIR)
03345       return N1.getOperand(N2C->getZExtValue());
03346 
03347     // EXTRACT_ELEMENT of a constant int is also very common.
03348     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
03349       unsigned ElementSize = VT.getSizeInBits();
03350       unsigned Shift = ElementSize * N2C->getZExtValue();
03351       APInt ShiftedVal = C->getAPIntValue().lshr(Shift);
03352       return getConstant(ShiftedVal.trunc(ElementSize), VT);
03353     }
03354     break;
03355   case ISD::EXTRACT_SUBVECTOR: {
03356     SDValue Index = N2;
03357     if (VT.isSimple() && N1.getValueType().isSimple()) {
03358       assert(VT.isVector() && N1.getValueType().isVector() &&
03359              "Extract subvector VTs must be a vectors!");
03360       assert(VT.getVectorElementType() ==
03361              N1.getValueType().getVectorElementType() &&
03362              "Extract subvector VTs must have the same element type!");
03363       assert(VT.getSimpleVT() <= N1.getSimpleValueType() &&
03364              "Extract subvector must be from larger vector to smaller vector!");
03365 
03366       if (isa<ConstantSDNode>(Index.getNode())) {
03367         assert((VT.getVectorNumElements() +
03368                 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
03369                 <= N1.getValueType().getVectorNumElements())
03370                && "Extract subvector overflow!");
03371       }
03372 
03373       // Trivial extraction.
03374       if (VT.getSimpleVT() == N1.getSimpleValueType())
03375         return N1;
03376     }
03377     break;
03378   }
03379   }
03380 
03381   // Perform trivial constant folding.
03382   SDValue SV = FoldConstantArithmetic(Opcode, VT, N1.getNode(), N2.getNode());
03383   if (SV.getNode()) return SV;
03384 
03385   // Canonicalize constant to RHS if commutative.
03386   if (N1C && !N2C && isCommutativeBinOp(Opcode)) {
03387     std::swap(N1C, N2C);
03388     std::swap(N1, N2);
03389   }
03390 
03391   // Constant fold FP operations.
03392   bool HasFPExceptions = TLI->hasFloatingPointExceptions();
03393   ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode());
03394   ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode());
03395   if (N1CFP) {
03396     if (!N2CFP && isCommutativeBinOp(Opcode)) {
03397       // Canonicalize constant to RHS if commutative.
03398       std::swap(N1CFP, N2CFP);
03399       std::swap(N1, N2);
03400     } else if (N2CFP) {
03401       APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF();
03402       APFloat::opStatus s;
03403       switch (Opcode) {
03404       case ISD::FADD:
03405         s = V1.add(V2, APFloat::rmNearestTiesToEven);
03406         if (!HasFPExceptions || s != APFloat::opInvalidOp)
03407           return getConstantFP(V1, VT);
03408         break;
03409       case ISD::FSUB:
03410         s = V1.subtract(V2, APFloat::rmNearestTiesToEven);
03411         if (!HasFPExceptions || s!=APFloat::opInvalidOp)
03412           return getConstantFP(V1, VT);
03413         break;
03414       case ISD::FMUL:
03415         s = V1.multiply(V2, APFloat::rmNearestTiesToEven);
03416         if (!HasFPExceptions || s!=APFloat::opInvalidOp)
03417           return getConstantFP(V1, VT);
03418         break;
03419       case ISD::FDIV:
03420         s = V1.divide(V2, APFloat::rmNearestTiesToEven);
03421         if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
03422                                  s!=APFloat::opDivByZero)) {
03423           return getConstantFP(V1, VT);
03424         }
03425         break;
03426       case ISD::FREM :
03427         s = V1.mod(V2, APFloat::rmNearestTiesToEven);
03428         if (!HasFPExceptions || (s!=APFloat::opInvalidOp &&
03429                                  s!=APFloat::opDivByZero)) {
03430           return getConstantFP(V1, VT);
03431         }
03432         break;
03433       case ISD::FCOPYSIGN:
03434         V1.copySign(V2);
03435         return getConstantFP(V1, VT);
03436       default: break;
03437       }
03438     }
03439 
03440     if (Opcode == ISD::FP_ROUND) {
03441       APFloat V = N1CFP->getValueAPF();    // make copy
03442       bool ignored;
03443       // This can return overflow, underflow, or inexact; we don't care.
03444       // FIXME need to be more flexible about rounding mode.
03445       (void)V.convert(EVTToAPFloatSemantics(VT),
03446                       APFloat::rmNearestTiesToEven, &ignored);
03447       return getConstantFP(V, VT);
03448     }
03449   }
03450 
03451   // Canonicalize an UNDEF to the RHS, even over a constant.
03452   if (N1.getOpcode() == ISD::UNDEF) {
03453     if (isCommutativeBinOp(Opcode)) {
03454       std::swap(N1, N2);
03455     } else {
03456       switch (Opcode) {
03457       case ISD::FP_ROUND_INREG:
03458       case ISD::SIGN_EXTEND_INREG:
03459       case ISD::SUB:
03460       case ISD::FSUB:
03461       case ISD::FDIV:
03462       case ISD::FREM:
03463       case ISD::SRA:
03464         return N1;     // fold op(undef, arg2) -> undef
03465       case ISD::UDIV:
03466       case ISD::SDIV:
03467       case ISD::UREM:
03468       case ISD::SREM:
03469       case ISD::SRL:
03470       case ISD::SHL:
03471         if (!VT.isVector())
03472           return getConstant(0, VT);    // fold op(undef, arg2) -> 0
03473         // For vectors, we can't easily build an all zero vector, just return
03474         // the LHS.
03475         return N2;
03476       }
03477     }
03478   }
03479 
03480   // Fold a bunch of operators when the RHS is undef.
03481   if (N2.getOpcode() == ISD::UNDEF) {
03482     switch (Opcode) {
03483     case ISD::XOR:
03484       if (N1.getOpcode() == ISD::UNDEF)
03485         // Handle undef ^ undef -> 0 special case. This is a common
03486         // idiom (misuse).
03487         return getConstant(0, VT);
03488       // fallthrough
03489     case ISD::ADD:
03490     case ISD::ADDC:
03491     case ISD::ADDE:
03492     case ISD::SUB:
03493     case ISD::UDIV:
03494     case ISD::SDIV:
03495     case ISD::UREM:
03496     case ISD::SREM:
03497       return N2;       // fold op(arg1, undef) -> undef
03498     case ISD::FADD:
03499     case ISD::FSUB:
03500     case ISD::FMUL:
03501     case ISD::FDIV:
03502     case ISD::FREM:
03503       if (getTarget().Options.UnsafeFPMath)
03504         return N2;
03505       break;
03506     case ISD::MUL:
03507     case ISD::AND:
03508     case ISD::SRL:
03509     case ISD::SHL:
03510       if (!VT.isVector())
03511         return getConstant(0, VT);  // fold op(arg1, undef) -> 0
03512       // For vectors, we can't easily build an all zero vector, just return
03513       // the LHS.
03514       return N1;
03515     case ISD::OR:
03516       if (!VT.isVector())
03517         return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT);
03518       // For vectors, we can't easily build an all one vector, just return
03519       // the LHS.
03520       return N1;
03521     case ISD::SRA:
03522       return N1;
03523     }
03524   }
03525 
03526   // Memoize this node if possible.
03527   BinarySDNode *N;
03528   SDVTList VTs = getVTList(VT);
03529   const bool BinOpHasFlags = isBinOpWithFlags(Opcode);
03530   if (VT != MVT::Glue) {
03531     SDValue Ops[] = {N1, N2};
03532     FoldingSetNodeID ID;
03533     AddNodeIDNode(ID, Opcode, VTs, Ops);
03534     if (BinOpHasFlags)
03535       AddBinaryNodeIDCustom(ID, Opcode, nuw, nsw, exact);
03536     void *IP = nullptr;
03537     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
03538       return SDValue(E, 0);
03539 
03540     N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
03541 
03542     CSEMap.InsertNode(N, IP);
03543   } else {
03544 
03545     N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, nuw, nsw, exact);
03546   }
03547 
03548   InsertNode(N);
03549   return SDValue(N, 0);
03550 }
03551 
03552 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03553                               SDValue N1, SDValue N2, SDValue N3) {
03554   // Perform various simplifications.
03555   ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
03556   switch (Opcode) {
03557   case ISD::FMA: {
03558     ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
03559     ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
03560     ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
03561     if (N1CFP && N2CFP && N3CFP) {
03562       APFloat  V1 = N1CFP->getValueAPF();
03563       const APFloat &V2 = N2CFP->getValueAPF();
03564       const APFloat &V3 = N3CFP->getValueAPF();
03565       APFloat::opStatus s =
03566         V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
03567       if (s != APFloat::opInvalidOp)
03568         return getConstantFP(V1, VT);
03569     }
03570     break;
03571   }
03572   case ISD::CONCAT_VECTORS:
03573     // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to
03574     // one big BUILD_VECTOR.
03575     if (N1.getOpcode() == ISD::BUILD_VECTOR &&
03576         N2.getOpcode() == ISD::BUILD_VECTOR &&
03577         N3.getOpcode() == ISD::BUILD_VECTOR) {
03578       SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(),
03579                                     N1.getNode()->op_end());
03580       Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end());
03581       Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end());
03582       return getNode(ISD::BUILD_VECTOR, DL, VT, Elts);
03583     }
03584     break;
03585   case ISD::SETCC: {
03586     // Use FoldSetCC to simplify SETCC's.
03587     SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL);
03588     if (Simp.getNode()) return Simp;
03589     break;
03590   }
03591   case ISD::SELECT:
03592     if (N1C) {
03593      if (N1C->getZExtValue())
03594        return N2;             // select true, X, Y -> X
03595      return N3;             // select false, X, Y -> Y
03596     }
03597 
03598     if (N2 == N3) return N2;   // select C, X, X -> X
03599     break;
03600   case ISD::VECTOR_SHUFFLE:
03601     llvm_unreachable("should use getVectorShuffle constructor!");
03602   case ISD::INSERT_SUBVECTOR: {
03603     SDValue Index = N3;
03604     if (VT.isSimple() && N1.getValueType().isSimple()
03605         && N2.getValueType().isSimple()) {
03606       assert(VT.isVector() && N1.getValueType().isVector() &&
03607              N2.getValueType().isVector() &&
03608              "Insert subvector VTs must be a vectors");
03609       assert(VT == N1.getValueType() &&
03610              "Dest and insert subvector source types must match!");
03611       assert(N2.getSimpleValueType() <= N1.getSimpleValueType() &&
03612              "Insert subvector must be from smaller vector to larger vector!");
03613       if (isa<ConstantSDNode>(Index.getNode())) {
03614         assert((N2.getValueType().getVectorNumElements() +
03615                 cast<ConstantSDNode>(Index.getNode())->getZExtValue()
03616                 <= VT.getVectorNumElements())
03617                && "Insert subvector overflow!");
03618       }
03619 
03620       // Trivial insertion.
03621       if (VT.getSimpleVT() == N2.getSimpleValueType())
03622         return N2;
03623     }
03624     break;
03625   }
03626   case ISD::BITCAST:
03627     // Fold bit_convert nodes from a type to themselves.
03628     if (N1.getValueType() == VT)
03629       return N1;
03630     break;
03631   }
03632 
03633   // Memoize node if it doesn't produce a flag.
03634   SDNode *N;
03635   SDVTList VTs = getVTList(VT);
03636   if (VT != MVT::Glue) {
03637     SDValue Ops[] = { N1, N2, N3 };
03638     FoldingSetNodeID ID;
03639     AddNodeIDNode(ID, Opcode, VTs, Ops);
03640     void *IP = nullptr;
03641     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
03642       return SDValue(E, 0);
03643 
03644     N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
03645                                           DL.getDebugLoc(), VTs, N1, N2, N3);
03646     CSEMap.InsertNode(N, IP);
03647   } else {
03648     N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
03649                                           DL.getDebugLoc(), VTs, N1, N2, N3);
03650   }
03651 
03652   InsertNode(N);
03653   return SDValue(N, 0);
03654 }
03655 
03656 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03657                               SDValue N1, SDValue N2, SDValue N3,
03658                               SDValue N4) {
03659   SDValue Ops[] = { N1, N2, N3, N4 };
03660   return getNode(Opcode, DL, VT, Ops);
03661 }
03662 
03663 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
03664                               SDValue N1, SDValue N2, SDValue N3,
03665                               SDValue N4, SDValue N5) {
03666   SDValue Ops[] = { N1, N2, N3, N4, N5 };
03667   return getNode(Opcode, DL, VT, Ops);
03668 }
03669 
03670 /// getStackArgumentTokenFactor - Compute a TokenFactor to force all
03671 /// the incoming stack arguments to be loaded from the stack.
03672 SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
03673   SmallVector<SDValue, 8> ArgChains;
03674 
03675   // Include the original chain at the beginning of the list. When this is
03676   // used by target LowerCall hooks, this helps legalize find the
03677   // CALLSEQ_BEGIN node.
03678   ArgChains.push_back(Chain);
03679 
03680   // Add a chain value for each stack argument.
03681   for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(),
03682        UE = getEntryNode().getNode()->use_end(); U != UE; ++U)
03683     if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
03684       if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
03685         if (FI->getIndex() < 0)
03686           ArgChains.push_back(SDValue(L, 1));
03687 
03688   // Build a tokenfactor for all the chains.
03689   return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
03690 }
03691 
03692 /// getMemsetValue - Vectorized representation of the memset value
03693 /// operand.
03694 static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
03695                               SDLoc dl) {
03696   assert(Value.getOpcode() != ISD::UNDEF);
03697 
03698   unsigned NumBits = VT.getScalarType().getSizeInBits();
03699   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
03700     assert(C->getAPIntValue().getBitWidth() == 8);
03701     APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
03702     if (VT.isInteger())
03703       return DAG.getConstant(Val, VT);
03704     return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), VT);
03705   }
03706 
03707   Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value);
03708   if (NumBits > 8) {
03709     // Use a multiplication with 0x010101... to extend the input to the
03710     // required length.
03711     APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
03712     Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT));
03713   }
03714 
03715   return Value;
03716 }
03717 
03718 /// getMemsetStringVal - Similar to getMemsetValue. Except this is only
03719 /// used when a memcpy is turned into a memset when the source is a constant
03720 /// string ptr.
03721 static SDValue getMemsetStringVal(EVT VT, SDLoc dl, SelectionDAG &DAG,
03722                                   const TargetLowering &TLI, StringRef Str) {
03723   // Handle vector with all elements zero.
03724   if (Str.empty()) {
03725     if (VT.isInteger())
03726       return DAG.getConstant(0, VT);
03727     else if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
03728       return DAG.getConstantFP(0.0, VT);
03729     else if (VT.isVector()) {
03730       unsigned NumElts = VT.getVectorNumElements();
03731       MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
03732       return DAG.getNode(ISD::BITCAST, dl, VT,
03733                          DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(),
03734                                                              EltVT, NumElts)));
03735     } else
03736       llvm_unreachable("Expected type!");
03737   }
03738 
03739   assert(!VT.isVector() && "Can't handle vector type here!");
03740   unsigned NumVTBits = VT.getSizeInBits();
03741   unsigned NumVTBytes = NumVTBits / 8;
03742   unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size()));
03743 
03744   APInt Val(NumVTBits, 0);
03745   if (TLI.isLittleEndian()) {
03746     for (unsigned i = 0; i != NumBytes; ++i)
03747       Val |= (uint64_t)(unsigned char)Str[i] << i*8;
03748   } else {
03749     for (unsigned i = 0; i != NumBytes; ++i)
03750       Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8;
03751   }
03752 
03753   // If the "cost" of materializing the integer immediate is less than the cost
03754   // of a load, then it is cost effective to turn the load into the immediate.
03755   Type *Ty = VT.getTypeForEVT(*DAG.getContext());
03756   if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
03757     return DAG.getConstant(Val, VT);
03758   return SDValue(nullptr, 0);
03759 }
03760 
03761 /// getMemBasePlusOffset - Returns base and offset node for the
03762 ///
03763 static SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, SDLoc dl,
03764                                       SelectionDAG &DAG) {
03765   EVT VT = Base.getValueType();
03766   return DAG.getNode(ISD::ADD, dl,
03767                      VT, Base, DAG.getConstant(Offset, VT));
03768 }
03769 
03770 /// isMemSrcFromString - Returns true if memcpy source is a string constant.
03771 ///
03772 static bool isMemSrcFromString(SDValue Src, StringRef &Str) {
03773   unsigned SrcDelta = 0;
03774   GlobalAddressSDNode *G = nullptr;
03775   if (Src.getOpcode() == ISD::GlobalAddress)
03776     G = cast<GlobalAddressSDNode>(Src);
03777   else if (Src.getOpcode() == ISD::ADD &&
03778            Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
03779            Src.getOperand(1).getOpcode() == ISD::Constant) {
03780     G = cast<GlobalAddressSDNode>(Src.getOperand(0));
03781     SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue();
03782   }
03783   if (!G)
03784     return false;
03785 
03786   return getConstantStringInfo(G->getGlobal(), Str, SrcDelta, false);
03787 }
03788 
03789 /// FindOptimalMemOpLowering - Determines the optimial series memory ops
03790 /// to replace the memset / memcpy. Return true if the number of memory ops
03791 /// is below the threshold. It returns the types of the sequence of
03792 /// memory ops to perform memset / memcpy by reference.
03793 static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
03794                                      unsigned Limit, uint64_t Size,
03795                                      unsigned DstAlign, unsigned SrcAlign,
03796                                      bool IsMemset,
03797                                      bool ZeroMemset,
03798                                      bool MemcpyStrSrc,
03799                                      bool AllowOverlap,
03800                                      SelectionDAG &DAG,
03801                                      const TargetLowering &TLI) {
03802   assert((SrcAlign == 0 || SrcAlign >= DstAlign) &&
03803          "Expecting memcpy / memset source to meet alignment requirement!");
03804   // If 'SrcAlign' is zero, that means the memory operation does not need to
03805   // load the value, i.e. memset or memcpy from constant string. Otherwise,
03806   // it's the inferred alignment of the source. 'DstAlign', on the other hand,
03807   // is the specified alignment of the memory operation. If it is zero, that
03808   // means it's possible to change the alignment of the destination.
03809   // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does
03810   // not need to be loaded.
03811   EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign,
03812                                    IsMemset, ZeroMemset, MemcpyStrSrc,
03813                                    DAG.getMachineFunction());
03814 
03815   if (VT == MVT::Other) {
03816     unsigned AS = 0;
03817     if (DstAlign >= TLI.getDataLayout()->getPointerPrefAlignment(AS) ||
03818         TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign)) {
03819       VT = TLI.getPointerTy();
03820     } else {
03821       switch (DstAlign & 7) {
03822       case 0:  VT = MVT::i64; break;
03823       case 4:  VT = MVT::i32; break;
03824       case 2:  VT = MVT::i16; break;
03825       default: VT = MVT::i8;  break;
03826       }
03827     }
03828 
03829     MVT LVT = MVT::i64;
03830     while (!TLI.isTypeLegal(LVT))
03831       LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
03832     assert(LVT.isInteger());
03833 
03834     if (VT.bitsGT(LVT))
03835       VT = LVT;
03836   }
03837 
03838   unsigned NumMemOps = 0;
03839   while (Size != 0) {
03840     unsigned VTSize = VT.getSizeInBits() / 8;
03841     while (VTSize > Size) {
03842       // For now, only use non-vector load / store's for the left-over pieces.
03843       EVT NewVT = VT;
03844       unsigned NewVTSize;
03845 
03846       bool Found = false;
03847       if (VT.isVector() || VT.isFloatingPoint()) {
03848         NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
03849         if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) &&
03850             TLI.isSafeMemOpType(NewVT.getSimpleVT()))
03851           Found = true;
03852         else if (NewVT == MVT::i64 &&
03853                  TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
03854                  TLI.isSafeMemOpType(MVT::f64)) {
03855           // i64 is usually not legal on 32-bit targets, but f64 may be.
03856           NewVT = MVT::f64;
03857           Found = true;
03858         }
03859       }
03860 
03861       if (!Found) {
03862         do {
03863           NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
03864           if (NewVT == MVT::i8)
03865             break;
03866         } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT()));
03867       }
03868       NewVTSize = NewVT.getSizeInBits() / 8;
03869 
03870       // If the new VT cannot cover all of the remaining bits, then consider
03871       // issuing a (or a pair of) unaligned and overlapping load / store.
03872       // FIXME: Only does this for 64-bit or more since we don't have proper
03873       // cost model for unaligned load / store.
03874       bool Fast;
03875       unsigned AS = 0;
03876       if (NumMemOps && AllowOverlap &&
03877           VTSize >= 8 && NewVTSize < Size &&
03878           TLI.allowsMisalignedMemoryAccesses(VT, AS, DstAlign, &Fast) && Fast)
03879         VTSize = Size;
03880       else {
03881         VT = NewVT;
03882         VTSize = NewVTSize;
03883       }
03884     }
03885 
03886     if (++NumMemOps > Limit)
03887       return false;
03888 
03889     MemOps.push_back(VT);
03890     Size -= VTSize;
03891   }
03892 
03893   return true;
03894 }
03895 
03896 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
03897                                        SDValue Chain, SDValue Dst,
03898                                        SDValue Src, uint64_t Size,
03899                                        unsigned Align, bool isVol,
03900                                        bool AlwaysInline,
03901                                        MachinePointerInfo DstPtrInfo,
03902                                        MachinePointerInfo SrcPtrInfo) {
03903   // Turn a memcpy of undef to nop.
03904   if (Src.getOpcode() == ISD::UNDEF)
03905     return Chain;
03906 
03907   // Expand memcpy to a series of load and store ops if the size operand falls
03908   // below a certain threshold.
03909   // TODO: In the AlwaysInline case, if the size is big then generate a loop
03910   // rather than maybe a humongous number of loads and stores.
03911   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
03912   std::vector<EVT> MemOps;
03913   bool DstAlignCanChange = false;
03914   MachineFunction &MF = DAG.getMachineFunction();
03915   MachineFrameInfo *MFI = MF.getFrameInfo();
03916   bool OptSize =
03917     MF.getFunction()->getAttributes().
03918       hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
03919   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
03920   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
03921     DstAlignCanChange = true;
03922   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
03923   if (Align > SrcAlign)
03924     SrcAlign = Align;
03925   StringRef Str;
03926   bool CopyFromStr = isMemSrcFromString(Src, Str);
03927   bool isZeroStr = CopyFromStr && Str.empty();
03928   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
03929 
03930   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
03931                                 (DstAlignCanChange ? 0 : Align),
03932                                 (isZeroStr ? 0 : SrcAlign),
03933                                 false, false, CopyFromStr, true, DAG, TLI))
03934     return SDValue();
03935 
03936   if (DstAlignCanChange) {
03937     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
03938     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
03939 
03940     // Don't promote to an alignment that would require dynamic stack
03941     // realignment.
03942     const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
03943     if (!TRI->needsStackRealignment(MF))
03944        while (NewAlign > Align &&
03945              TLI.getDataLayout()->exceedsNaturalStackAlignment(NewAlign))
03946           NewAlign /= 2;
03947 
03948     if (NewAlign > Align) {
03949       // Give the stack frame object a larger alignment if needed.
03950       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
03951         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
03952       Align = NewAlign;
03953     }
03954   }
03955 
03956   SmallVector<SDValue, 8> OutChains;
03957   unsigned NumMemOps = MemOps.size();
03958   uint64_t SrcOff = 0, DstOff = 0;
03959   for (unsigned i = 0; i != NumMemOps; ++i) {
03960     EVT VT = MemOps[i];
03961     unsigned VTSize = VT.getSizeInBits() / 8;
03962     SDValue Value, Store;
03963 
03964     if (VTSize > Size) {
03965       // Issuing an unaligned load / store pair  that overlaps with the previous
03966       // pair. Adjust the offset accordingly.
03967       assert(i == NumMemOps-1 && i != 0);
03968       SrcOff -= VTSize - Size;
03969       DstOff -= VTSize - Size;
03970     }
03971 
03972     if (CopyFromStr &&
03973         (isZeroStr || (VT.isInteger() && !VT.isVector()))) {
03974       // It's unlikely a store of a vector immediate can be done in a single
03975       // instruction. It would require a load from a constantpool first.
03976       // We only handle zero vectors here.
03977       // FIXME: Handle other cases where store of vector immediate is done in
03978       // a single instruction.
03979       Value = getMemsetStringVal(VT, dl, DAG, TLI, Str.substr(SrcOff));
03980       if (Value.getNode())
03981         Store = DAG.getStore(Chain, dl, Value,
03982                              getMemBasePlusOffset(Dst, DstOff, dl, DAG),
03983                              DstPtrInfo.getWithOffset(DstOff), isVol,
03984                              false, Align);
03985     }
03986 
03987     if (!Store.getNode()) {
03988       // The type might not be legal for the target.  This should only happen
03989       // if the type is smaller than a legal type, as on PPC, so the right
03990       // thing to do is generate a LoadExt/StoreTrunc pair.  These simplify
03991       // to Load/Store if NVT==VT.
03992       // FIXME does the case above also need this?
03993       EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
03994       assert(NVT.bitsGE(VT));
03995       Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain,
03996                              getMemBasePlusOffset(Src, SrcOff, dl, DAG),
03997                              SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false,
03998                              false, MinAlign(SrcAlign, SrcOff));
03999       Store = DAG.getTruncStore(Chain, dl, Value,
04000                                 getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04001                                 DstPtrInfo.getWithOffset(DstOff), VT, isVol,
04002                                 false, Align);
04003     }
04004     OutChains.push_back(Store);
04005     SrcOff += VTSize;
04006     DstOff += VTSize;
04007     Size -= VTSize;
04008   }
04009 
04010   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04011 }
04012 
04013 static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, SDLoc dl,
04014                                         SDValue Chain, SDValue Dst,
04015                                         SDValue Src, uint64_t Size,
04016                                         unsigned Align,  bool isVol,
04017                                         bool AlwaysInline,
04018                                         MachinePointerInfo DstPtrInfo,
04019                                         MachinePointerInfo SrcPtrInfo) {
04020   // Turn a memmove of undef to nop.
04021   if (Src.getOpcode() == ISD::UNDEF)
04022     return Chain;
04023 
04024   // Expand memmove to a series of load and store ops if the size operand falls
04025   // below a certain threshold.
04026   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04027   std::vector<EVT> MemOps;
04028   bool DstAlignCanChange = false;
04029   MachineFunction &MF = DAG.getMachineFunction();
04030   MachineFrameInfo *MFI = MF.getFrameInfo();
04031   bool OptSize = MF.getFunction()->getAttributes().
04032     hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
04033   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
04034   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
04035     DstAlignCanChange = true;
04036   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
04037   if (Align > SrcAlign)
04038     SrcAlign = Align;
04039   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
04040 
04041   if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
04042                                 (DstAlignCanChange ? 0 : Align), SrcAlign,
04043                                 false, false, false, false, DAG, TLI))
04044     return SDValue();
04045 
04046   if (DstAlignCanChange) {
04047     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
04048     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
04049     if (NewAlign > Align) {
04050       // Give the stack frame object a larger alignment if needed.
04051       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
04052         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
04053       Align = NewAlign;
04054     }
04055   }
04056 
04057   uint64_t SrcOff = 0, DstOff = 0;
04058   SmallVector<SDValue, 8> LoadValues;
04059   SmallVector<SDValue, 8> LoadChains;
04060   SmallVector<SDValue, 8> OutChains;
04061   unsigned NumMemOps = MemOps.size();
04062   for (unsigned i = 0; i < NumMemOps; i++) {
04063     EVT VT = MemOps[i];
04064     unsigned VTSize = VT.getSizeInBits() / 8;
04065     SDValue Value;
04066 
04067     Value = DAG.getLoad(VT, dl, Chain,
04068                         getMemBasePlusOffset(Src, SrcOff, dl, DAG),
04069                         SrcPtrInfo.getWithOffset(SrcOff), isVol,
04070                         false, false, SrcAlign);
04071     LoadValues.push_back(Value);
04072     LoadChains.push_back(Value.getValue(1));
04073     SrcOff += VTSize;
04074   }
04075   Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
04076   OutChains.clear();
04077   for (unsigned i = 0; i < NumMemOps; i++) {
04078     EVT VT = MemOps[i];
04079     unsigned VTSize = VT.getSizeInBits() / 8;
04080     SDValue Store;
04081 
04082     Store = DAG.getStore(Chain, dl, LoadValues[i],
04083                          getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04084                          DstPtrInfo.getWithOffset(DstOff), isVol, false, Align);
04085     OutChains.push_back(Store);
04086     DstOff += VTSize;
04087   }
04088 
04089   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04090 }
04091 
04092 /// \brief Lower the call to 'memset' intrinsic function into a series of store
04093 /// operations.
04094 ///
04095 /// \param DAG Selection DAG where lowered code is placed.
04096 /// \param dl Link to corresponding IR location.
04097 /// \param Chain Control flow dependency.
04098 /// \param Dst Pointer to destination memory location.
04099 /// \param Src Value of byte to write into the memory.
04100 /// \param Size Number of bytes to write.
04101 /// \param Align Alignment of the destination in bytes.
04102 /// \param isVol True if destination is volatile.
04103 /// \param DstPtrInfo IR information on the memory pointer.
04104 /// \returns New head in the control flow, if lowering was successful, empty
04105 /// SDValue otherwise.
04106 ///
04107 /// The function tries to replace 'llvm.memset' intrinsic with several store
04108 /// operations and value calculation code. This is usually profitable for small
04109 /// memory size.
04110 static SDValue getMemsetStores(SelectionDAG &DAG, SDLoc dl,
04111                                SDValue Chain, SDValue Dst,
04112                                SDValue Src, uint64_t Size,
04113                                unsigned Align, bool isVol,
04114                                MachinePointerInfo DstPtrInfo) {
04115   // Turn a memset of undef to nop.
04116   if (Src.getOpcode() == ISD::UNDEF)
04117     return Chain;
04118 
04119   // Expand memset to a series of load/store ops if the size operand
04120   // falls below a certain threshold.
04121   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
04122   std::vector<EVT> MemOps;
04123   bool DstAlignCanChange = false;
04124   MachineFunction &MF = DAG.getMachineFunction();
04125   MachineFrameInfo *MFI = MF.getFrameInfo();
04126   bool OptSize = MF.getFunction()->getAttributes().
04127     hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize);
04128   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
04129   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
04130     DstAlignCanChange = true;
04131   bool IsZeroVal =
04132     isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
04133   if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize),
04134                                 Size, (DstAlignCanChange ? 0 : Align), 0,
04135                                 true, IsZeroVal, false, true, DAG, TLI))
04136     return SDValue();
04137 
04138   if (DstAlignCanChange) {
04139     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
04140     unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
04141     if (NewAlign > Align) {
04142       // Give the stack frame object a larger alignment if needed.
04143       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
04144         MFI->setObjectAlignment(FI->getIndex(), NewAlign);
04145       Align = NewAlign;
04146     }
04147   }
04148 
04149   SmallVector<SDValue, 8> OutChains;
04150   uint64_t DstOff = 0;
04151   unsigned NumMemOps = MemOps.size();
04152 
04153   // Find the largest store and generate the bit pattern for it.
04154   EVT LargestVT = MemOps[0];
04155   for (unsigned i = 1; i < NumMemOps; i++)
04156     if (MemOps[i].bitsGT(LargestVT))
04157       LargestVT = MemOps[i];
04158   SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
04159 
04160   for (unsigned i = 0; i < NumMemOps; i++) {
04161     EVT VT = MemOps[i];
04162     unsigned VTSize = VT.getSizeInBits() / 8;
04163     if (VTSize > Size) {
04164       // Issuing an unaligned load / store pair  that overlaps with the previous
04165       // pair. Adjust the offset accordingly.
04166       assert(i == NumMemOps-1 && i != 0);
04167       DstOff -= VTSize - Size;
04168     }
04169 
04170     // If this store is smaller than the largest store see whether we can get
04171     // the smaller value for free with a truncate.
04172     SDValue Value = MemSetValue;
04173     if (VT.bitsLT(LargestVT)) {
04174       if (!LargestVT.isVector() && !VT.isVector() &&
04175           TLI.isTruncateFree(LargestVT, VT))
04176         Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
04177       else
04178         Value = getMemsetValue(Src, VT, DAG, dl);
04179     }
04180     assert(Value.getValueType() == VT && "Value with wrong type.");
04181     SDValue Store = DAG.getStore(Chain, dl, Value,
04182                                  getMemBasePlusOffset(Dst, DstOff, dl, DAG),
04183                                  DstPtrInfo.getWithOffset(DstOff),
04184                                  isVol, false, Align);
04185     OutChains.push_back(Store);
04186     DstOff += VT.getSizeInBits() / 8;
04187     Size -= VTSize;
04188   }
04189 
04190   return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
04191 }
04192 
04193 SDValue SelectionDAG::getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst,
04194                                 SDValue Src, SDValue Size,
04195                                 unsigned Align, bool isVol, bool AlwaysInline,
04196                                 MachinePointerInfo DstPtrInfo,
04197                                 MachinePointerInfo SrcPtrInfo) {
04198   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04199 
04200   // Check to see if we should lower the memcpy to loads and stores first.
04201   // For cases within the target-specified limits, this is the best choice.
04202   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04203   if (ConstantSize) {
04204     // Memcpy with size zero? Just return the original chain.
04205     if (ConstantSize->isNullValue())
04206       return Chain;
04207 
04208     SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
04209                                              ConstantSize->getZExtValue(),Align,
04210                                 isVol, false, DstPtrInfo, SrcPtrInfo);
04211     if (Result.getNode())
04212       return Result;
04213   }
04214 
04215   // Then check to see if we should lower the memcpy with target-specific
04216   // code. If the target chooses to do this, this is the next best.
04217   SDValue Result =
04218       TSI->EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align,
04219                                    isVol, AlwaysInline, DstPtrInfo, SrcPtrInfo);
04220   if (Result.getNode())
04221     return Result;
04222 
04223   // If we really need inline code and the target declined to provide it,
04224   // use a (potentially long) sequence of loads and stores.
04225   if (AlwaysInline) {
04226     assert(ConstantSize && "AlwaysInline requires a constant size!");
04227     return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
04228                                    ConstantSize->getZExtValue(), Align, isVol,
04229                                    true, DstPtrInfo, SrcPtrInfo);
04230   }
04231 
04232   // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
04233   // memcpy is not guaranteed to be safe. libc memcpys aren't required to
04234   // respect volatile, so they may do things like read or write memory
04235   // beyond the given memory regions. But fixing this isn't easy, and most
04236   // people don't care.
04237 
04238   // Emit a library call.
04239   TargetLowering::ArgListTy Args;
04240   TargetLowering::ArgListEntry Entry;
04241   Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
04242   Entry.Node = Dst; Args.push_back(Entry);
04243   Entry.Node = Src; Args.push_back(Entry);
04244   Entry.Node = Size; Args.push_back(Entry);
04245   // FIXME: pass in SDLoc
04246   TargetLowering::CallLoweringInfo CLI(*this);
04247   CLI.setDebugLoc(dl).setChain(Chain)
04248     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
04249                Type::getVoidTy(*getContext()),
04250                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
04251                                  TLI->getPointerTy()), std::move(Args), 0)
04252     .setDiscardResult();
04253   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04254 
04255   return CallResult.second;
04256 }
04257 
04258 SDValue SelectionDAG::getMemmove(SDValue Chain, SDLoc dl, SDValue Dst,
04259                                  SDValue Src, SDValue Size,
04260                                  unsigned Align, bool isVol,
04261                                  MachinePointerInfo DstPtrInfo,
04262                                  MachinePointerInfo SrcPtrInfo) {
04263   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04264 
04265   // Check to see if we should lower the memmove to loads and stores first.
04266   // For cases within the target-specified limits, this is the best choice.
04267   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04268   if (ConstantSize) {
04269     // Memmove with size zero? Just return the original chain.
04270     if (ConstantSize->isNullValue())
04271       return Chain;
04272 
04273     SDValue Result =
04274       getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src,
04275                                ConstantSize->getZExtValue(), Align, isVol,
04276                                false, DstPtrInfo, SrcPtrInfo);
04277     if (Result.getNode())
04278       return Result;
04279   }
04280 
04281   // Then check to see if we should lower the memmove with target-specific
04282   // code. If the target chooses to do this, this is the next best.
04283   SDValue Result = TSI->EmitTargetCodeForMemmove(
04284       *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo, SrcPtrInfo);
04285   if (Result.getNode())
04286     return Result;
04287 
04288   // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
04289   // not be safe.  See memcpy above for more details.
04290 
04291   // Emit a library call.
04292   TargetLowering::ArgListTy Args;
04293   TargetLowering::ArgListEntry Entry;
04294   Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext());
04295   Entry.Node = Dst; Args.push_back(Entry);
04296   Entry.Node = Src; Args.push_back(Entry);
04297   Entry.Node = Size; Args.push_back(Entry);
04298   // FIXME:  pass in SDLoc
04299   TargetLowering::CallLoweringInfo CLI(*this);
04300   CLI.setDebugLoc(dl).setChain(Chain)
04301     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
04302                Type::getVoidTy(*getContext()),
04303                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
04304                                  TLI->getPointerTy()), std::move(Args), 0)
04305     .setDiscardResult();
04306   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04307 
04308   return CallResult.second;
04309 }
04310 
04311 SDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst,
04312                                 SDValue Src, SDValue Size,
04313                                 unsigned Align, bool isVol,
04314                                 MachinePointerInfo DstPtrInfo) {
04315   assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
04316 
04317   // Check to see if we should lower the memset to stores first.
04318   // For cases within the target-specified limits, this is the best choice.
04319   ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
04320   if (ConstantSize) {
04321     // Memset with size zero? Just return the original chain.
04322     if (ConstantSize->isNullValue())
04323       return Chain;
04324 
04325     SDValue Result =
04326       getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
04327                       Align, isVol, DstPtrInfo);
04328 
04329     if (Result.getNode())
04330       return Result;
04331   }
04332 
04333   // Then check to see if we should lower the memset with target-specific
04334   // code. If the target chooses to do this, this is the next best.
04335   SDValue Result = TSI->EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src,
04336                                                 Size, Align, isVol, DstPtrInfo);
04337   if (Result.getNode())
04338     return Result;
04339 
04340   // Emit a library call.
04341   Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(*getContext());
04342   TargetLowering::ArgListTy Args;
04343   TargetLowering::ArgListEntry Entry;
04344   Entry.Node = Dst; Entry.Ty = IntPtrTy;
04345   Args.push_back(Entry);
04346   Entry.Node = Src;
04347   Entry.Ty = Src.getValueType().getTypeForEVT(*getContext());
04348   Args.push_back(Entry);
04349   Entry.Node = Size;
04350   Entry.Ty = IntPtrTy;
04351   Args.push_back(Entry);
04352 
04353   // FIXME: pass in SDLoc
04354   TargetLowering::CallLoweringInfo CLI(*this);
04355   CLI.setDebugLoc(dl).setChain(Chain)
04356     .setCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
04357                Type::getVoidTy(*getContext()),
04358                getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
04359                                  TLI->getPointerTy()), std::move(Args), 0)
04360     .setDiscardResult();
04361 
04362   std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
04363   return CallResult.second;
04364 }
04365 
04366 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04367                                 SDVTList VTList, ArrayRef<SDValue> Ops,
04368                                 MachineMemOperand *MMO,
04369                                 AtomicOrdering SuccessOrdering,
04370                                 AtomicOrdering FailureOrdering,
04371                                 SynchronizationScope SynchScope) {
04372   FoldingSetNodeID ID;
04373   ID.AddInteger(MemVT.getRawBits());
04374   AddNodeIDNode(ID, Opcode, VTList, Ops);
04375   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04376   void* IP = nullptr;
04377   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04378     cast<AtomicSDNode>(E)->refineAlignment(MMO);
04379     return SDValue(E, 0);
04380   }
04381 
04382   // Allocate the operands array for the node out of the BumpPtrAllocator, since
04383   // SDNode doesn't have access to it.  This memory will be "leaked" when
04384   // the node is deallocated, but recovered when the allocator is released.
04385   // If the number of operands is less than 5 we use AtomicSDNode's internal
04386   // storage.
04387   unsigned NumOps = Ops.size();
04388   SDUse *DynOps = NumOps > 4 ? OperandAllocator.Allocate<SDUse>(NumOps)
04389                              : nullptr;
04390 
04391   SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(),
04392                                                dl.getDebugLoc(), VTList, MemVT,
04393                                                Ops.data(), DynOps, NumOps, MMO,
04394                                                SuccessOrdering, FailureOrdering,
04395                                                SynchScope);
04396   CSEMap.InsertNode(N, IP);
04397   InsertNode(N);
04398   return SDValue(N, 0);
04399 }
04400 
04401 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04402                                 SDVTList VTList, ArrayRef<SDValue> Ops,
04403                                 MachineMemOperand *MMO,
04404                                 AtomicOrdering Ordering,
04405                                 SynchronizationScope SynchScope) {
04406   return getAtomic(Opcode, dl, MemVT, VTList, Ops, MMO, Ordering,
04407                    Ordering, SynchScope);
04408 }
04409 
04410 SDValue SelectionDAG::getAtomicCmpSwap(
04411     unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs, SDValue Chain,
04412     SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo,
04413     unsigned Alignment, AtomicOrdering SuccessOrdering,
04414     AtomicOrdering FailureOrdering, SynchronizationScope SynchScope) {
04415   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
04416          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
04417   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
04418 
04419   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04420     Alignment = getEVTAlignment(MemVT);
04421 
04422   MachineFunction &MF = getMachineFunction();
04423 
04424   // FIXME: Volatile isn't really correct; we should keep track of atomic
04425   // orderings in the memoperand.
04426   unsigned Flags = MachineMemOperand::MOVolatile;
04427   Flags |= MachineMemOperand::MOLoad;
04428   Flags |= MachineMemOperand::MOStore;
04429 
04430   MachineMemOperand *MMO =
04431     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
04432 
04433   return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO,
04434                           SuccessOrdering, FailureOrdering, SynchScope);
04435 }
04436 
04437 SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, SDLoc dl, EVT MemVT,
04438                                        SDVTList VTs, SDValue Chain, SDValue Ptr,
04439                                        SDValue Cmp, SDValue Swp,
04440                                        MachineMemOperand *MMO,
04441                                        AtomicOrdering SuccessOrdering,
04442                                        AtomicOrdering FailureOrdering,
04443                                        SynchronizationScope SynchScope) {
04444   assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
04445          Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
04446   assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
04447 
04448   SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
04449   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO,
04450                    SuccessOrdering, FailureOrdering, SynchScope);
04451 }
04452 
04453 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04454                                 SDValue Chain,
04455                                 SDValue Ptr, SDValue Val,
04456                                 const Value* PtrVal,
04457                                 unsigned Alignment,
04458                                 AtomicOrdering Ordering,
04459                                 SynchronizationScope SynchScope) {
04460   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04461     Alignment = getEVTAlignment(MemVT);
04462 
04463   MachineFunction &MF = getMachineFunction();
04464   // An atomic store does not load. An atomic load does not store.
04465   // (An atomicrmw obviously both loads and stores.)
04466   // For now, atomics are considered to be volatile always, and they are
04467   // chained as such.
04468   // FIXME: Volatile isn't really correct; we should keep track of atomic
04469   // orderings in the memoperand.
04470   unsigned Flags = MachineMemOperand::MOVolatile;
04471   if (Opcode != ISD::ATOMIC_STORE)
04472     Flags |= MachineMemOperand::MOLoad;
04473   if (Opcode != ISD::ATOMIC_LOAD)
04474     Flags |= MachineMemOperand::MOStore;
04475 
04476   MachineMemOperand *MMO =
04477     MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
04478                             MemVT.getStoreSize(), Alignment);
04479 
04480   return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO,
04481                    Ordering, SynchScope);
04482 }
04483 
04484 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04485                                 SDValue Chain,
04486                                 SDValue Ptr, SDValue Val,
04487                                 MachineMemOperand *MMO,
04488                                 AtomicOrdering Ordering,
04489                                 SynchronizationScope SynchScope) {
04490   assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
04491           Opcode == ISD::ATOMIC_LOAD_SUB ||
04492           Opcode == ISD::ATOMIC_LOAD_AND ||
04493           Opcode == ISD::ATOMIC_LOAD_OR ||
04494           Opcode == ISD::ATOMIC_LOAD_XOR ||
04495           Opcode == ISD::ATOMIC_LOAD_NAND ||
04496           Opcode == ISD::ATOMIC_LOAD_MIN ||
04497           Opcode == ISD::ATOMIC_LOAD_MAX ||
04498           Opcode == ISD::ATOMIC_LOAD_UMIN ||
04499           Opcode == ISD::ATOMIC_LOAD_UMAX ||
04500           Opcode == ISD::ATOMIC_SWAP ||
04501           Opcode == ISD::ATOMIC_STORE) &&
04502          "Invalid Atomic Op");
04503 
04504   EVT VT = Val.getValueType();
04505 
04506   SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
04507                                                getVTList(VT, MVT::Other);
04508   SDValue Ops[] = {Chain, Ptr, Val};
04509   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
04510 }
04511 
04512 SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
04513                                 EVT VT, SDValue Chain,
04514                                 SDValue Ptr,
04515                                 MachineMemOperand *MMO,
04516                                 AtomicOrdering Ordering,
04517                                 SynchronizationScope SynchScope) {
04518   assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
04519 
04520   SDVTList VTs = getVTList(VT, MVT::Other);
04521   SDValue Ops[] = {Chain, Ptr};
04522   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO, Ordering, SynchScope);
04523 }
04524 
04525 /// getMergeValues - Create a MERGE_VALUES node from the given operands.
04526 SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, SDLoc dl) {
04527   if (Ops.size() == 1)
04528     return Ops[0];
04529 
04530   SmallVector<EVT, 4> VTs;
04531   VTs.reserve(Ops.size());
04532   for (unsigned i = 0; i < Ops.size(); ++i)
04533     VTs.push_back(Ops[i].getValueType());
04534   return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
04535 }
04536 
04537 SDValue
04538 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
04539                                   ArrayRef<SDValue> Ops,
04540                                   EVT MemVT, MachinePointerInfo PtrInfo,
04541                                   unsigned Align, bool Vol,
04542                                   bool ReadMem, bool WriteMem, unsigned Size) {
04543   if (Align == 0)  // Ensure that codegen never sees alignment 0
04544     Align = getEVTAlignment(MemVT);
04545 
04546   MachineFunction &MF = getMachineFunction();
04547   unsigned Flags = 0;
04548   if (WriteMem)
04549     Flags |= MachineMemOperand::MOStore;
04550   if (ReadMem)
04551     Flags |= MachineMemOperand::MOLoad;
04552   if (Vol)
04553     Flags |= MachineMemOperand::MOVolatile;
04554   if (!Size)
04555     Size = MemVT.getStoreSize();
04556   MachineMemOperand *MMO =
04557     MF.getMachineMemOperand(PtrInfo, Flags, Size, Align);
04558 
04559   return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
04560 }
04561 
04562 SDValue
04563 SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
04564                                   ArrayRef<SDValue> Ops, EVT MemVT,
04565                                   MachineMemOperand *MMO) {
04566   assert((Opcode == ISD::INTRINSIC_VOID ||
04567           Opcode == ISD::INTRINSIC_W_CHAIN ||
04568           Opcode == ISD::PREFETCH ||
04569           Opcode == ISD::LIFETIME_START ||
04570           Opcode == ISD::LIFETIME_END ||
04571           (Opcode <= INT_MAX &&
04572            (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
04573          "Opcode is not a memory-accessing opcode!");
04574 
04575   // Memoize the node unless it returns a flag.
04576   MemIntrinsicSDNode *N;
04577   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
04578     FoldingSetNodeID ID;
04579     AddNodeIDNode(ID, Opcode, VTList, Ops);
04580     ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04581     void *IP = nullptr;
04582     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04583       cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
04584       return SDValue(E, 0);
04585     }
04586 
04587     N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
04588                                                dl.getDebugLoc(), VTList, Ops,
04589                                                MemVT, MMO);
04590     CSEMap.InsertNode(N, IP);
04591   } else {
04592     N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(),
04593                                                dl.getDebugLoc(), VTList, Ops,
04594                                                MemVT, MMO);
04595   }
04596   InsertNode(N);
04597   return SDValue(N, 0);
04598 }
04599 
04600 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
04601 /// MachinePointerInfo record from it.  This is particularly useful because the
04602 /// code generator has many cases where it doesn't bother passing in a
04603 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
04604 static MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) {
04605   // If this is FI+Offset, we can model it.
04606   if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
04607     return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset);
04608 
04609   // If this is (FI+Offset1)+Offset2, we can model it.
04610   if (Ptr.getOpcode() != ISD::ADD ||
04611       !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
04612       !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
04613     return MachinePointerInfo();
04614 
04615   int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
04616   return MachinePointerInfo::getFixedStack(FI, Offset+
04617                        cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
04618 }
04619 
04620 /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
04621 /// MachinePointerInfo record from it.  This is particularly useful because the
04622 /// code generator has many cases where it doesn't bother passing in a
04623 /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
04624 static MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) {
04625   // If the 'Offset' value isn't a constant, we can't handle this.
04626   if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
04627     return InferPointerInfo(Ptr, OffsetNode->getSExtValue());
04628   if (OffsetOp.getOpcode() == ISD::UNDEF)
04629     return InferPointerInfo(Ptr);
04630   return MachinePointerInfo();
04631 }
04632 
04633 
04634 SDValue
04635 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
04636                       EVT VT, SDLoc dl, SDValue Chain,
04637                       SDValue Ptr, SDValue Offset,
04638                       MachinePointerInfo PtrInfo, EVT MemVT,
04639                       bool isVolatile, bool isNonTemporal, bool isInvariant,
04640                       unsigned Alignment, const AAMDNodes &AAInfo,
04641                       const MDNode *Ranges) {
04642   assert(Chain.getValueType() == MVT::Other &&
04643         "Invalid chain type");
04644   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04645     Alignment = getEVTAlignment(VT);
04646 
04647   unsigned Flags = MachineMemOperand::MOLoad;
04648   if (isVolatile)
04649     Flags |= MachineMemOperand::MOVolatile;
04650   if (isNonTemporal)
04651     Flags |= MachineMemOperand::MONonTemporal;
04652   if (isInvariant)
04653     Flags |= MachineMemOperand::MOInvariant;
04654 
04655   // If we don't have a PtrInfo, infer the trivial frame index case to simplify
04656   // clients.
04657   if (PtrInfo.V.isNull())
04658     PtrInfo = InferPointerInfo(Ptr, Offset);
04659 
04660   MachineFunction &MF = getMachineFunction();
04661   MachineMemOperand *MMO =
04662     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment,
04663                             AAInfo, Ranges);
04664   return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
04665 }
04666 
04667 SDValue
04668 SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
04669                       EVT VT, SDLoc dl, SDValue Chain,
04670                       SDValue Ptr, SDValue Offset, EVT MemVT,
04671                       MachineMemOperand *MMO) {
04672   if (VT == MemVT) {
04673     ExtType = ISD::NON_EXTLOAD;
04674   } else if (ExtType == ISD::NON_EXTLOAD) {
04675     assert(VT == MemVT && "Non-extending load from different memory type!");
04676   } else {
04677     // Extending load.
04678     assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
04679            "Should only be an extending load, not truncating!");
04680     assert(VT.isInteger() == MemVT.isInteger() &&
04681            "Cannot convert from FP to Int or Int -> FP!");
04682     assert(VT.isVector() == MemVT.isVector() &&
04683            "Cannot use trunc store to convert to or from a vector!");
04684     assert((!VT.isVector() ||
04685             VT.getVectorNumElements() == MemVT.getVectorNumElements()) &&
04686            "Cannot use trunc store to change the number of vector elements!");
04687   }
04688 
04689   bool Indexed = AM != ISD::UNINDEXED;
04690   assert((Indexed || Offset.getOpcode() == ISD::UNDEF) &&
04691          "Unindexed load with an offset!");
04692 
04693   SDVTList VTs = Indexed ?
04694     getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
04695   SDValue Ops[] = { Chain, Ptr, Offset };
04696   FoldingSetNodeID ID;
04697   AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
04698   ID.AddInteger(MemVT.getRawBits());
04699   ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(),
04700                                      MMO->isNonTemporal(),
04701                                      MMO->isInvariant()));
04702   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04703   void *IP = nullptr;
04704   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04705     cast<LoadSDNode>(E)->refineAlignment(MMO);
04706     return SDValue(E, 0);
04707   }
04708   SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl.getIROrder(),
04709                                              dl.getDebugLoc(), VTs, AM, ExtType,
04710                                              MemVT, MMO);
04711   CSEMap.InsertNode(N, IP);
04712   InsertNode(N);
04713   return SDValue(N, 0);
04714 }
04715 
04716 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
04717                               SDValue Chain, SDValue Ptr,
04718                               MachinePointerInfo PtrInfo,
04719                               bool isVolatile, bool isNonTemporal,
04720                               bool isInvariant, unsigned Alignment,
04721                               const AAMDNodes &AAInfo,
04722                               const MDNode *Ranges) {
04723   SDValue Undef = getUNDEF(Ptr.getValueType());
04724   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
04725                  PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment,
04726                  AAInfo, Ranges);
04727 }
04728 
04729 SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl,
04730                               SDValue Chain, SDValue Ptr,
04731                               MachineMemOperand *MMO) {
04732   SDValue Undef = getUNDEF(Ptr.getValueType());
04733   return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
04734                  VT, MMO);
04735 }
04736 
04737 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
04738                                  SDValue Chain, SDValue Ptr,
04739                                  MachinePointerInfo PtrInfo, EVT MemVT,
04740                                  bool isVolatile, bool isNonTemporal,
04741                                  bool isInvariant, unsigned Alignment,
04742                                  const AAMDNodes &AAInfo) {
04743   SDValue Undef = getUNDEF(Ptr.getValueType());
04744   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
04745                  PtrInfo, MemVT, isVolatile, isNonTemporal, isInvariant,
04746                  Alignment, AAInfo);
04747 }
04748 
04749 
04750 SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
04751                                  SDValue Chain, SDValue Ptr, EVT MemVT,
04752                                  MachineMemOperand *MMO) {
04753   SDValue Undef = getUNDEF(Ptr.getValueType());
04754   return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
04755                  MemVT, MMO);
04756 }
04757 
04758 SDValue
04759 SelectionDAG::getIndexedLoad(SDValue OrigLoad, SDLoc dl, SDValue Base,
04760                              SDValue Offset, ISD::MemIndexedMode AM) {
04761   LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
04762   assert(LD->getOffset().getOpcode() == ISD::UNDEF &&
04763          "Load is already a indexed load!");
04764   return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
04765                  LD->getChain(), Base, Offset, LD->getPointerInfo(),
04766                  LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(),
04767                  false, LD->getAlignment());
04768 }
04769 
04770 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
04771                                SDValue Ptr, MachinePointerInfo PtrInfo,
04772                                bool isVolatile, bool isNonTemporal,
04773                                unsigned Alignment, const AAMDNodes &AAInfo) {
04774   assert(Chain.getValueType() == MVT::Other &&
04775         "Invalid chain type");
04776   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04777     Alignment = getEVTAlignment(Val.getValueType());
04778 
04779   unsigned Flags = MachineMemOperand::MOStore;
04780   if (isVolatile)
04781     Flags |= MachineMemOperand::MOVolatile;
04782   if (isNonTemporal)
04783     Flags |= MachineMemOperand::MONonTemporal;
04784 
04785   if (PtrInfo.V.isNull())
04786     PtrInfo = InferPointerInfo(Ptr);
04787 
04788   MachineFunction &MF = getMachineFunction();
04789   MachineMemOperand *MMO =
04790     MF.getMachineMemOperand(PtrInfo, Flags,
04791                             Val.getValueType().getStoreSize(), Alignment,
04792                             AAInfo);
04793 
04794   return getStore(Chain, dl, Val, Ptr, MMO);
04795 }
04796 
04797 SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
04798                                SDValue Ptr, MachineMemOperand *MMO) {
04799   assert(Chain.getValueType() == MVT::Other &&
04800         "Invalid chain type");
04801   EVT VT = Val.getValueType();
04802   SDVTList VTs = getVTList(MVT::Other);
04803   SDValue Undef = getUNDEF(Ptr.getValueType());
04804   SDValue Ops[] = { Chain, Val, Ptr, Undef };
04805   FoldingSetNodeID ID;
04806   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04807   ID.AddInteger(VT.getRawBits());
04808   ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(),
04809                                      MMO->isNonTemporal(), MMO->isInvariant()));
04810   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04811   void *IP = nullptr;
04812   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04813     cast<StoreSDNode>(E)->refineAlignment(MMO);
04814     return SDValue(E, 0);
04815   }
04816   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04817                                               dl.getDebugLoc(), VTs,
04818                                               ISD::UNINDEXED, false, VT, MMO);
04819   CSEMap.InsertNode(N, IP);
04820   InsertNode(N);
04821   return SDValue(N, 0);
04822 }
04823 
04824 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
04825                                     SDValue Ptr, MachinePointerInfo PtrInfo,
04826                                     EVT SVT,bool isVolatile, bool isNonTemporal,
04827                                     unsigned Alignment,
04828                                     const AAMDNodes &AAInfo) {
04829   assert(Chain.getValueType() == MVT::Other &&
04830         "Invalid chain type");
04831   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
04832     Alignment = getEVTAlignment(SVT);
04833 
04834   unsigned Flags = MachineMemOperand::MOStore;
04835   if (isVolatile)
04836     Flags |= MachineMemOperand::MOVolatile;
04837   if (isNonTemporal)
04838     Flags |= MachineMemOperand::MONonTemporal;
04839 
04840   if (PtrInfo.V.isNull())
04841     PtrInfo = InferPointerInfo(Ptr);
04842 
04843   MachineFunction &MF = getMachineFunction();
04844   MachineMemOperand *MMO =
04845     MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment,
04846                             AAInfo);
04847 
04848   return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
04849 }
04850 
04851 SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
04852                                     SDValue Ptr, EVT SVT,
04853                                     MachineMemOperand *MMO) {
04854   EVT VT = Val.getValueType();
04855 
04856   assert(Chain.getValueType() == MVT::Other &&
04857         "Invalid chain type");
04858   if (VT == SVT)
04859     return getStore(Chain, dl, Val, Ptr, MMO);
04860 
04861   assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
04862          "Should only be a truncating store, not extending!");
04863   assert(VT.isInteger() == SVT.isInteger() &&
04864          "Can't do FP-INT conversion!");
04865   assert(VT.isVector() == SVT.isVector() &&
04866          "Cannot use trunc store to convert to or from a vector!");
04867   assert((!VT.isVector() ||
04868           VT.getVectorNumElements() == SVT.getVectorNumElements()) &&
04869          "Cannot use trunc store to change the number of vector elements!");
04870 
04871   SDVTList VTs = getVTList(MVT::Other);
04872   SDValue Undef = getUNDEF(Ptr.getValueType());
04873   SDValue Ops[] = { Chain, Val, Ptr, Undef };
04874   FoldingSetNodeID ID;
04875   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04876   ID.AddInteger(SVT.getRawBits());
04877   ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(),
04878                                      MMO->isNonTemporal(), MMO->isInvariant()));
04879   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
04880   void *IP = nullptr;
04881   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
04882     cast<StoreSDNode>(E)->refineAlignment(MMO);
04883     return SDValue(E, 0);
04884   }
04885   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04886                                               dl.getDebugLoc(), VTs,
04887                                               ISD::UNINDEXED, true, SVT, MMO);
04888   CSEMap.InsertNode(N, IP);
04889   InsertNode(N);
04890   return SDValue(N, 0);
04891 }
04892 
04893 SDValue
04894 SelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base,
04895                               SDValue Offset, ISD::MemIndexedMode AM) {
04896   StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
04897   assert(ST->getOffset().getOpcode() == ISD::UNDEF &&
04898          "Store is already a indexed store!");
04899   SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
04900   SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
04901   FoldingSetNodeID ID;
04902   AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
04903   ID.AddInteger(ST->getMemoryVT().getRawBits());
04904   ID.AddInteger(ST->getRawSubclassData());
04905   ID.AddInteger(ST->getPointerInfo().getAddrSpace());
04906   void *IP = nullptr;
04907   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
04908     return SDValue(E, 0);
04909 
04910   SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(),
04911                                               dl.getDebugLoc(), VTs, AM,
04912                                               ST->isTruncatingStore(),
04913                                               ST->getMemoryVT(),
04914                                               ST->getMemOperand());
04915   CSEMap.InsertNode(N, IP);
04916   InsertNode(N);
04917   return SDValue(N, 0);
04918 }
04919 
04920 SDValue SelectionDAG::getVAArg(EVT VT, SDLoc dl,
04921                                SDValue Chain, SDValue Ptr,
04922                                SDValue SV,
04923                                unsigned Align) {
04924   SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) };
04925   return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
04926 }
04927 
04928 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
04929                               ArrayRef<SDUse> Ops) {
04930   switch (Ops.size()) {
04931   case 0: return getNode(Opcode, DL, VT);
04932   case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
04933   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
04934   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
04935   default: break;
04936   }
04937 
04938   // Copy from an SDUse array into an SDValue array for use with
04939   // the regular getNode logic.
04940   SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
04941   return getNode(Opcode, DL, VT, NewOps);
04942 }
04943 
04944 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
04945                               ArrayRef<SDValue> Ops) {
04946   unsigned NumOps = Ops.size();
04947   switch (NumOps) {
04948   case 0: return getNode(Opcode, DL, VT);
04949   case 1: return getNode(Opcode, DL, VT, Ops[0]);
04950   case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
04951   case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
04952   default: break;
04953   }
04954 
04955   switch (Opcode) {
04956   default: break;
04957   case ISD::SELECT_CC: {
04958     assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
04959     assert(Ops[0].getValueType() == Ops[1].getValueType() &&
04960            "LHS and RHS of condition must have same type!");
04961     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
04962            "True and False arms of SelectCC must have same type!");
04963     assert(Ops[2].getValueType() == VT &&
04964            "select_cc node must be of same type as true and false value!");
04965     break;
04966   }
04967   case ISD::BR_CC: {
04968     assert(NumOps == 5 && "BR_CC takes 5 operands!");
04969     assert(Ops[2].getValueType() == Ops[3].getValueType() &&
04970            "LHS/RHS of comparison should match types!");
04971     break;
04972   }
04973   }
04974 
04975   // Memoize nodes.
04976   SDNode *N;
04977   SDVTList VTs = getVTList(VT);
04978 
04979   if (VT != MVT::Glue) {
04980     FoldingSetNodeID ID;
04981     AddNodeIDNode(ID, Opcode, VTs, Ops);
04982     void *IP = nullptr;
04983 
04984     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
04985       return SDValue(E, 0);
04986 
04987     N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
04988                                    VTs, Ops);
04989     CSEMap.InsertNode(N, IP);
04990   } else {
04991     N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
04992                                    VTs, Ops);
04993   }
04994 
04995   InsertNode(N);
04996   return SDValue(N, 0);
04997 }
04998 
04999 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
05000                               ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
05001   return getNode(Opcode, DL, getVTList(ResultTys), Ops);
05002 }
05003 
05004 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05005                               ArrayRef<SDValue> Ops) {
05006   if (VTList.NumVTs == 1)
05007     return getNode(Opcode, DL, VTList.VTs[0], Ops);
05008 
05009 #if 0
05010   switch (Opcode) {
05011   // FIXME: figure out how to safely handle things like
05012   // int foo(int x) { return 1 << (x & 255); }
05013   // int bar() { return foo(256); }
05014   case ISD::SRA_PARTS:
05015   case ISD::SRL_PARTS:
05016   case ISD::SHL_PARTS:
05017     if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
05018         cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
05019       return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
05020     else if (N3.getOpcode() == ISD::AND)
05021       if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
05022         // If the and is only masking out bits that cannot effect the shift,
05023         // eliminate the and.
05024         unsigned NumBits = VT.getScalarType().getSizeInBits()*2;
05025         if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
05026           return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
05027       }
05028     break;
05029   }
05030 #endif
05031 
05032   // Memoize the node unless it returns a flag.
05033   SDNode *N;
05034   unsigned NumOps = Ops.size();
05035   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
05036     FoldingSetNodeID ID;
05037     AddNodeIDNode(ID, Opcode, VTList, Ops);
05038     void *IP = nullptr;
05039     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05040       return SDValue(E, 0);
05041 
05042     if (NumOps == 1) {
05043       N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
05044                                           DL.getDebugLoc(), VTList, Ops[0]);
05045     } else if (NumOps == 2) {
05046       N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
05047                                            DL.getDebugLoc(), VTList, Ops[0],
05048                                            Ops[1]);
05049     } else if (NumOps == 3) {
05050       N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
05051                                             DL.getDebugLoc(), VTList, Ops[0],
05052                                             Ops[1], Ops[2]);
05053     } else {
05054       N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05055                                      VTList, Ops);
05056     }
05057     CSEMap.InsertNode(N, IP);
05058   } else {
05059     if (NumOps == 1) {
05060       N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(),
05061                                           DL.getDebugLoc(), VTList, Ops[0]);
05062     } else if (NumOps == 2) {
05063       N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(),
05064                                            DL.getDebugLoc(), VTList, Ops[0],
05065                                            Ops[1]);
05066     } else if (NumOps == 3) {
05067       N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(),
05068                                             DL.getDebugLoc(), VTList, Ops[0],
05069                                             Ops[1], Ops[2]);
05070     } else {
05071       N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(),
05072                                      VTList, Ops);
05073     }
05074   }
05075   InsertNode(N);
05076   return SDValue(N, 0);
05077 }
05078 
05079 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList) {
05080   return getNode(Opcode, DL, VTList, None);
05081 }
05082 
05083 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05084                               SDValue N1) {
05085   SDValue Ops[] = { N1 };
05086   return getNode(Opcode, DL, VTList, Ops);
05087 }
05088 
05089 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05090                               SDValue N1, SDValue N2) {
05091   SDValue Ops[] = { N1, N2 };
05092   return getNode(Opcode, DL, VTList, Ops);
05093 }
05094 
05095 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05096                               SDValue N1, SDValue N2, SDValue N3) {
05097   SDValue Ops[] = { N1, N2, N3 };
05098   return getNode(Opcode, DL, VTList, Ops);
05099 }
05100 
05101 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05102                               SDValue N1, SDValue N2, SDValue N3,
05103                               SDValue N4) {
05104   SDValue Ops[] = { N1, N2, N3, N4 };
05105   return getNode(Opcode, DL, VTList, Ops);
05106 }
05107 
05108 SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
05109                               SDValue N1, SDValue N2, SDValue N3,
05110                               SDValue N4, SDValue N5) {
05111   SDValue Ops[] = { N1, N2, N3, N4, N5 };
05112   return getNode(Opcode, DL, VTList, Ops);
05113 }
05114 
05115 SDVTList SelectionDAG::getVTList(EVT VT) {
05116   return makeVTList(SDNode::getValueTypeList(VT), 1);
05117 }
05118 
05119 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
05120   FoldingSetNodeID ID;
05121   ID.AddInteger(2U);
05122   ID.AddInteger(VT1.getRawBits());
05123   ID.AddInteger(VT2.getRawBits());
05124 
05125   void *IP = nullptr;
05126   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05127   if (!Result) {
05128     EVT *Array = Allocator.Allocate<EVT>(2);
05129     Array[0] = VT1;
05130     Array[1] = VT2;
05131     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
05132     VTListMap.InsertNode(Result, IP);
05133   }
05134   return Result->getSDVTList();
05135 }
05136 
05137 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
05138   FoldingSetNodeID ID;
05139   ID.AddInteger(3U);
05140   ID.AddInteger(VT1.getRawBits());
05141   ID.AddInteger(VT2.getRawBits());
05142   ID.AddInteger(VT3.getRawBits());
05143 
05144   void *IP = nullptr;
05145   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05146   if (!Result) {
05147     EVT *Array = Allocator.Allocate<EVT>(3);
05148     Array[0] = VT1;
05149     Array[1] = VT2;
05150     Array[2] = VT3;
05151     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
05152     VTListMap.InsertNode(Result, IP);
05153   }
05154   return Result->getSDVTList();
05155 }
05156 
05157 SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
05158   FoldingSetNodeID ID;
05159   ID.AddInteger(4U);
05160   ID.AddInteger(VT1.getRawBits());
05161   ID.AddInteger(VT2.getRawBits());
05162   ID.AddInteger(VT3.getRawBits());
05163   ID.AddInteger(VT4.getRawBits());
05164 
05165   void *IP = nullptr;
05166   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05167   if (!Result) {
05168     EVT *Array = Allocator.Allocate<EVT>(4);
05169     Array[0] = VT1;
05170     Array[1] = VT2;
05171     Array[2] = VT3;
05172     Array[3] = VT4;
05173     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
05174     VTListMap.InsertNode(Result, IP);
05175   }
05176   return Result->getSDVTList();
05177 }
05178 
05179 SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
05180   unsigned NumVTs = VTs.size();
05181   FoldingSetNodeID ID;
05182   ID.AddInteger(NumVTs);
05183   for (unsigned index = 0; index < NumVTs; index++) {
05184     ID.AddInteger(VTs[index].getRawBits());
05185   }
05186 
05187   void *IP = nullptr;
05188   SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
05189   if (!Result) {
05190     EVT *Array = Allocator.Allocate<EVT>(NumVTs);
05191     std::copy(VTs.begin(), VTs.end(), Array);
05192     Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
05193     VTListMap.InsertNode(Result, IP);
05194   }
05195   return Result->getSDVTList();
05196 }
05197 
05198 
05199 /// UpdateNodeOperands - *Mutate* the specified node in-place to have the
05200 /// specified operands.  If the resultant node already exists in the DAG,
05201 /// this does not modify the specified node, instead it returns the node that
05202 /// already exists.  If the resultant node does not exist in the DAG, the
05203 /// input node is returned.  As a degenerate case, if you specify the same
05204 /// input operands as the node already has, the input node is returned.
05205 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
05206   assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
05207 
05208   // Check to see if there is no change.
05209   if (Op == N->getOperand(0)) return N;
05210 
05211   // See if the modified node already exists.
05212   void *InsertPos = nullptr;
05213   if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
05214     return Existing;
05215 
05216   // Nope it doesn't.  Remove the node from its current place in the maps.
05217   if (InsertPos)
05218     if (!RemoveNodeFromCSEMaps(N))
05219       InsertPos = nullptr;
05220 
05221   // Now we update the operands.
05222   N->OperandList[0].set(Op);
05223 
05224   // If this gets put into a CSE map, add it.
05225   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05226   return N;
05227 }
05228 
05229 SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
05230   assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
05231 
05232   // Check to see if there is no change.
05233   if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
05234     return N;   // No operands changed, just return the input node.
05235 
05236   // See if the modified node already exists.
05237   void *InsertPos = nullptr;
05238   if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
05239     return Existing;
05240 
05241   // Nope it doesn't.  Remove the node from its current place in the maps.
05242   if (InsertPos)
05243     if (!RemoveNodeFromCSEMaps(N))
05244       InsertPos = nullptr;
05245 
05246   // Now we update the operands.
05247   if (N->OperandList[0] != Op1)
05248     N->OperandList[0].set(Op1);
05249   if (N->OperandList[1] != Op2)
05250     N->OperandList[1].set(Op2);
05251 
05252   // If this gets put into a CSE map, add it.
05253   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05254   return N;
05255 }
05256 
05257 SDNode *SelectionDAG::
05258 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
05259   SDValue Ops[] = { Op1, Op2, Op3 };
05260   return UpdateNodeOperands(N, Ops);
05261 }
05262 
05263 SDNode *SelectionDAG::
05264 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
05265                    SDValue Op3, SDValue Op4) {
05266   SDValue Ops[] = { Op1, Op2, Op3, Op4 };
05267   return UpdateNodeOperands(N, Ops);
05268 }
05269 
05270 SDNode *SelectionDAG::
05271 UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
05272                    SDValue Op3, SDValue Op4, SDValue Op5) {
05273   SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
05274   return UpdateNodeOperands(N, Ops);
05275 }
05276 
05277 SDNode *SelectionDAG::
05278 UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
05279   unsigned NumOps = Ops.size();
05280   assert(N->getNumOperands() == NumOps &&
05281          "Update with wrong number of operands");
05282 
05283   // Check to see if there is no change.
05284   bool AnyChange = false;
05285   for (unsigned i = 0; i != NumOps; ++i) {
05286     if (Ops[i] != N->getOperand(i)) {
05287       AnyChange = true;
05288       break;
05289     }
05290   }
05291 
05292   // No operands changed, just return the input node.
05293   if (!AnyChange) return N;
05294 
05295   // See if the modified node already exists.
05296   void *InsertPos = nullptr;
05297   if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
05298     return Existing;
05299 
05300   // Nope it doesn't.  Remove the node from its current place in the maps.
05301   if (InsertPos)
05302     if (!RemoveNodeFromCSEMaps(N))
05303       InsertPos = nullptr;
05304 
05305   // Now we update the operands.
05306   for (unsigned i = 0; i != NumOps; ++i)
05307     if (N->OperandList[i] != Ops[i])
05308       N->OperandList[i].set(Ops[i]);
05309 
05310   // If this gets put into a CSE map, add it.
05311   if (InsertPos) CSEMap.InsertNode(N, InsertPos);
05312   return N;
05313 }
05314 
05315 /// DropOperands - Release the operands and set this node to have
05316 /// zero operands.
05317 void SDNode::DropOperands() {
05318   // Unlike the code in MorphNodeTo that does this, we don't need to
05319   // watch for dead nodes here.
05320   for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
05321     SDUse &Use = *I++;
05322     Use.set(SDValue());
05323   }
05324 }
05325 
05326 /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
05327 /// machine opcode.
05328 ///
05329 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05330                                    EVT VT) {
05331   SDVTList VTs = getVTList(VT);
05332   return SelectNodeTo(N, MachineOpc, VTs, None);
05333 }
05334 
05335 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05336                                    EVT VT, SDValue Op1) {
05337   SDVTList VTs = getVTList(VT);
05338   SDValue Ops[] = { Op1 };
05339   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05340 }
05341 
05342 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05343                                    EVT VT, SDValue Op1,
05344                                    SDValue Op2) {
05345   SDVTList VTs = getVTList(VT);
05346   SDValue Ops[] = { Op1, Op2 };
05347   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05348 }
05349 
05350 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05351                                    EVT VT, SDValue Op1,
05352                                    SDValue Op2, SDValue Op3) {
05353   SDVTList VTs = getVTList(VT);
05354   SDValue Ops[] = { Op1, Op2, Op3 };
05355   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05356 }
05357 
05358 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05359                                    EVT VT, ArrayRef<SDValue> Ops) {
05360   SDVTList VTs = getVTList(VT);
05361   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05362 }
05363 
05364 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05365                                    EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
05366   SDVTList VTs = getVTList(VT1, VT2);
05367   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05368 }
05369 
05370 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05371                                    EVT VT1, EVT VT2) {
05372   SDVTList VTs = getVTList(VT1, VT2);
05373   return SelectNodeTo(N, MachineOpc, VTs, None);
05374 }
05375 
05376 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05377                                    EVT VT1, EVT VT2, EVT VT3,
05378                                    ArrayRef<SDValue> Ops) {
05379   SDVTList VTs = getVTList(VT1, VT2, VT3);
05380   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05381 }
05382 
05383 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05384                                    EVT VT1, EVT VT2, EVT VT3, EVT VT4,
05385                                    ArrayRef<SDValue> Ops) {
05386   SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
05387   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05388 }
05389 
05390 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05391                                    EVT VT1, EVT VT2,
05392                                    SDValue Op1) {
05393   SDVTList VTs = getVTList(VT1, VT2);
05394   SDValue Ops[] = { Op1 };
05395   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05396 }
05397 
05398 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05399                                    EVT VT1, EVT VT2,
05400                                    SDValue Op1, SDValue Op2) {
05401   SDVTList VTs = getVTList(VT1, VT2);
05402   SDValue Ops[] = { Op1, Op2 };
05403   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05404 }
05405 
05406 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05407                                    EVT VT1, EVT VT2,
05408                                    SDValue Op1, SDValue Op2,
05409                                    SDValue Op3) {
05410   SDVTList VTs = getVTList(VT1, VT2);
05411   SDValue Ops[] = { Op1, Op2, Op3 };
05412   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05413 }
05414 
05415 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05416                                    EVT VT1, EVT VT2, EVT VT3,
05417                                    SDValue Op1, SDValue Op2,
05418                                    SDValue Op3) {
05419   SDVTList VTs = getVTList(VT1, VT2, VT3);
05420   SDValue Ops[] = { Op1, Op2, Op3 };
05421   return SelectNodeTo(N, MachineOpc, VTs, Ops);
05422 }
05423 
05424 SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
05425                                    SDVTList VTs,ArrayRef<SDValue> Ops) {
05426   N = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
05427   // Reset the NodeID to -1.
05428   N->setNodeId(-1);
05429   return N;
05430 }
05431 
05432 /// UpdadeSDLocOnMergedSDNode - If the opt level is -O0 then it throws away
05433 /// the line number information on the merged node since it is not possible to
05434 /// preserve the information that operation is associated with multiple lines.
05435 /// This will make the debugger working better at -O0, were there is a higher
05436 /// probability having other instructions associated with that line.
05437 ///
05438 /// For IROrder, we keep the smaller of the two
05439 SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) {
05440   DebugLoc NLoc = N->getDebugLoc();
05441   if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) &&
05442     (OLoc.getDebugLoc() != NLoc)) {
05443     N->setDebugLoc(DebugLoc());
05444   }
05445   unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
05446   N->setIROrder(Order);
05447   return N;
05448 }
05449 
05450 /// MorphNodeTo - This *mutates* the specified node to have the specified
05451 /// return type, opcode, and operands.
05452 ///
05453 /// Note that MorphNodeTo returns the resultant node.  If there is already a
05454 /// node of the specified opcode and operands, it returns that node instead of
05455 /// the current one.  Note that the SDLoc need not be the same.
05456 ///
05457 /// Using MorphNodeTo is faster than creating a new node and swapping it in
05458 /// with ReplaceAllUsesWith both because it often avoids allocating a new
05459 /// node, and because it doesn't require CSE recalculation for any of
05460 /// the node's users.
05461 ///
05462 /// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
05463 /// As a consequence it isn't appropriate to use from within the DAG combiner or
05464 /// the legalizer which maintain worklists that would need to be updated when
05465 /// deleting things.
05466 SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
05467                                   SDVTList VTs, ArrayRef<SDValue> Ops) {
05468   unsigned NumOps = Ops.size();
05469   // If an identical node already exists, use it.
05470   void *IP = nullptr;
05471   if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
05472     FoldingSetNodeID ID;
05473     AddNodeIDNode(ID, Opc, VTs, Ops);
05474     if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP))
05475       return UpdadeSDLocOnMergedSDNode(ON, SDLoc(N));
05476   }
05477 
05478   if (!RemoveNodeFromCSEMaps(N))
05479     IP = nullptr;
05480 
05481   // Start the morphing.
05482   N->NodeType = Opc;
05483   N->ValueList = VTs.VTs;
05484   N->NumValues = VTs.NumVTs;
05485 
05486   // Clear the operands list, updating used nodes to remove this from their
05487   // use list.  Keep track of any operands that become dead as a result.
05488   SmallPtrSet<SDNode*, 16> DeadNodeSet;
05489   for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
05490     SDUse &Use = *I++;
05491     SDNode *Used = Use.getNode();
05492     Use.set(SDValue());
05493     if (Used->use_empty())
05494       DeadNodeSet.insert(Used);
05495   }
05496 
05497   if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) {
05498     // Initialize the memory references information.
05499     MN->setMemRefs(nullptr, nullptr);
05500     // If NumOps is larger than the # of operands we can have in a
05501     // MachineSDNode, reallocate the operand list.
05502     if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) {
05503       if (MN->OperandsNeedDelete)
05504         delete[] MN->OperandList;
05505       if (NumOps > array_lengthof(MN->LocalOperands))
05506         // We're creating a final node that will live unmorphed for the
05507         // remainder of the current SelectionDAG iteration, so we can allocate
05508         // the operands directly out of a pool with no recycling metadata.
05509         MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
05510                          Ops.data(), NumOps);
05511       else
05512         MN->InitOperands(MN->LocalOperands, Ops.data(), NumOps);
05513       MN->OperandsNeedDelete = false;
05514     } else
05515       MN->InitOperands(MN->OperandList, Ops.data(), NumOps);
05516   } else {
05517     // If NumOps is larger than the # of operands we currently have, reallocate
05518     // the operand list.
05519     if (NumOps > N->NumOperands) {
05520       if (N->OperandsNeedDelete)
05521         delete[] N->OperandList;
05522       N->InitOperands(new SDUse[NumOps], Ops.data(), NumOps);
05523       N->OperandsNeedDelete = true;
05524     } else
05525       N->InitOperands(N->OperandList, Ops.data(), NumOps);
05526   }
05527 
05528   // Delete any nodes that are still dead after adding the uses for the
05529   // new operands.
05530   if (!DeadNodeSet.empty()) {
05531     SmallVector<SDNode *, 16> DeadNodes;
05532     for (SDNode *N : DeadNodeSet)
05533       if (N->use_empty())
05534         DeadNodes.push_back(N);
05535     RemoveDeadNodes(DeadNodes);
05536   }
05537 
05538   if (IP)
05539     CSEMap.InsertNode(N, IP);   // Memoize the new node.
05540   return N;
05541 }
05542 
05543 
05544 /// getMachineNode - These are used for target selectors to create a new node
05545 /// with specified return type(s), MachineInstr opcode, and operands.
05546 ///
05547 /// Note that getMachineNode returns the resultant node.  If there is already a
05548 /// node of the specified opcode and operands, it returns that node instead of
05549 /// the current one.
05550 MachineSDNode *
05551 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT) {
05552   SDVTList VTs = getVTList(VT);
05553   return getMachineNode(Opcode, dl, VTs, None);
05554 }
05555 
05556 MachineSDNode *
05557 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, SDValue Op1) {
05558   SDVTList VTs = getVTList(VT);
05559   SDValue Ops[] = { Op1 };
05560   return getMachineNode(Opcode, dl, VTs, Ops);
05561 }
05562 
05563 MachineSDNode *
05564 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05565                              SDValue Op1, SDValue Op2) {
05566   SDVTList VTs = getVTList(VT);
05567   SDValue Ops[] = { Op1, Op2 };
05568   return getMachineNode(Opcode, dl, VTs, Ops);
05569 }
05570 
05571 MachineSDNode *
05572 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05573                              SDValue Op1, SDValue Op2, SDValue Op3) {
05574   SDVTList VTs = getVTList(VT);
05575   SDValue Ops[] = { Op1, Op2, Op3 };
05576   return getMachineNode(Opcode, dl, VTs, Ops);
05577 }
05578 
05579 MachineSDNode *
05580 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT,
05581                              ArrayRef<SDValue> Ops) {
05582   SDVTList VTs = getVTList(VT);
05583   return getMachineNode(Opcode, dl, VTs, Ops);
05584 }
05585 
05586 MachineSDNode *
05587 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, EVT VT2) {
05588   SDVTList VTs = getVTList(VT1, VT2);
05589   return getMachineNode(Opcode, dl, VTs, None);
05590 }
05591 
05592 MachineSDNode *
05593 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05594                              EVT VT1, EVT VT2, SDValue Op1) {
05595   SDVTList VTs = getVTList(VT1, VT2);
05596   SDValue Ops[] = { Op1 };
05597   return getMachineNode(Opcode, dl, VTs, Ops);
05598 }
05599 
05600 MachineSDNode *
05601 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05602                              EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) {
05603   SDVTList VTs = getVTList(VT1, VT2);
05604   SDValue Ops[] = { Op1, Op2 };
05605   return getMachineNode(Opcode, dl, VTs, Ops);
05606 }
05607 
05608 MachineSDNode *
05609 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05610                              EVT VT1, EVT VT2, SDValue Op1,
05611                              SDValue Op2, SDValue Op3) {
05612   SDVTList VTs = getVTList(VT1, VT2);
05613   SDValue Ops[] = { Op1, Op2, Op3 };
05614   return getMachineNode(Opcode, dl, VTs, Ops);
05615 }
05616 
05617 MachineSDNode *
05618 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05619                              EVT VT1, EVT VT2,
05620                              ArrayRef<SDValue> Ops) {
05621   SDVTList VTs = getVTList(VT1, VT2);
05622   return getMachineNode(Opcode, dl, VTs, Ops);
05623 }
05624 
05625 MachineSDNode *
05626 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05627                              EVT VT1, EVT VT2, EVT VT3,
05628                              SDValue Op1, SDValue Op2) {
05629   SDVTList VTs = getVTList(VT1, VT2, VT3);
05630   SDValue Ops[] = { Op1, Op2 };
05631   return getMachineNode(Opcode, dl, VTs, Ops);
05632 }
05633 
05634 MachineSDNode *
05635 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05636                              EVT VT1, EVT VT2, EVT VT3,
05637                              SDValue Op1, SDValue Op2, SDValue Op3) {
05638   SDVTList VTs = getVTList(VT1, VT2, VT3);
05639   SDValue Ops[] = { Op1, Op2, Op3 };
05640   return getMachineNode(Opcode, dl, VTs, Ops);
05641 }
05642 
05643 MachineSDNode *
05644 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05645                              EVT VT1, EVT VT2, EVT VT3,
05646                              ArrayRef<SDValue> Ops) {
05647   SDVTList VTs = getVTList(VT1, VT2, VT3);
05648   return getMachineNode(Opcode, dl, VTs, Ops);
05649 }
05650 
05651 MachineSDNode *
05652 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1,
05653                              EVT VT2, EVT VT3, EVT VT4,
05654                              ArrayRef<SDValue> Ops) {
05655   SDVTList VTs = getVTList(VT1, VT2, VT3, VT4);
05656   return getMachineNode(Opcode, dl, VTs, Ops);
05657 }
05658 
05659 MachineSDNode *
05660 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
05661                              ArrayRef<EVT> ResultTys,
05662                              ArrayRef<SDValue> Ops) {
05663   SDVTList VTs = getVTList(ResultTys);
05664   return getMachineNode(Opcode, dl, VTs, Ops);
05665 }
05666 
05667 MachineSDNode *
05668 SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
05669                              ArrayRef<SDValue> OpsArray) {
05670   bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
05671   MachineSDNode *N;
05672   void *IP = nullptr;
05673   const SDValue *Ops = OpsArray.data();
05674   unsigned NumOps = OpsArray.size();
05675 
05676   if (DoCSE) {
05677     FoldingSetNodeID ID;
05678     AddNodeIDNode(ID, ~Opcode, VTs, OpsArray);
05679     IP = nullptr;
05680     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) {
05681       return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL));
05682     }
05683   }
05684 
05685   // Allocate a new MachineSDNode.
05686   N = new (NodeAllocator) MachineSDNode(~Opcode, DL.getIROrder(),
05687                                         DL.getDebugLoc(), VTs);
05688 
05689   // Initialize the operands list.
05690   if (NumOps > array_lengthof(N->LocalOperands))
05691     // We're creating a final node that will live unmorphed for the
05692     // remainder of the current SelectionDAG iteration, so we can allocate
05693     // the operands directly out of a pool with no recycling metadata.
05694     N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
05695                     Ops, NumOps);
05696   else
05697     N->InitOperands(N->LocalOperands, Ops, NumOps);
05698   N->OperandsNeedDelete = false;
05699 
05700   if (DoCSE)
05701     CSEMap.InsertNode(N, IP);
05702 
05703   InsertNode(N);
05704   return N;
05705 }
05706 
05707 /// getTargetExtractSubreg - A convenience function for creating
05708 /// TargetOpcode::EXTRACT_SUBREG nodes.
05709 SDValue
05710 SelectionDAG::getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT,
05711                                      SDValue Operand) {
05712   SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
05713   SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
05714                                   VT, Operand, SRIdxVal);
05715   return SDValue(Subreg, 0);
05716 }
05717 
05718 /// getTargetInsertSubreg - A convenience function for creating
05719 /// TargetOpcode::INSERT_SUBREG nodes.
05720 SDValue
05721 SelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT,
05722                                     SDValue Operand, SDValue Subreg) {
05723   SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32);
05724   SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
05725                                   VT, Operand, Subreg, SRIdxVal);
05726   return SDValue(Result, 0);
05727 }
05728 
05729 /// getNodeIfExists - Get the specified node if it's already available, or
05730 /// else return NULL.
05731 SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
05732                                       ArrayRef<SDValue> Ops, bool nuw, bool nsw,
05733                                       bool exact) {
05734   if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
05735     FoldingSetNodeID ID;
05736     AddNodeIDNode(ID, Opcode, VTList, Ops);
05737     if (isBinOpWithFlags(Opcode))
05738       AddBinaryNodeIDCustom(ID, nuw, nsw, exact);
05739     void *IP = nullptr;
05740     if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
05741       return E;
05742   }
05743   return nullptr;
05744 }
05745 
05746 /// getDbgValue - Creates a SDDbgValue node.
05747 ///
05748 /// SDNode
05749 SDDbgValue *SelectionDAG::getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N,
05750                                       unsigned R, bool IsIndirect, uint64_t Off,
05751                                       DebugLoc DL, unsigned O) {
05752   return new (Allocator) SDDbgValue(Var, Expr, N, R, IsIndirect, Off, DL, O);
05753 }
05754 
05755 /// Constant
05756 SDDbgValue *SelectionDAG::getConstantDbgValue(MDNode *Var, MDNode *Expr,
05757                                               const Value *C, uint64_t Off,
05758                                               DebugLoc DL, unsigned O) {
05759   return new (Allocator) SDDbgValue(Var, Expr, C, Off, DL, O);
05760 }
05761 
05762 /// FrameIndex
05763 SDDbgValue *SelectionDAG::getFrameIndexDbgValue(MDNode *Var, MDNode *Expr,
05764                                                 unsigned FI, uint64_t Off,
05765                                                 DebugLoc DL, unsigned O) {
05766   return new (Allocator) SDDbgValue(Var, Expr, FI, Off, DL, O);
05767 }
05768 
05769 namespace {
05770 
05771 /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
05772 /// pointed to by a use iterator is deleted, increment the use iterator
05773 /// so that it doesn't dangle.
05774 ///
05775 class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
05776   SDNode::use_iterator &UI;
05777   SDNode::use_iterator &UE;
05778 
05779   void NodeDeleted(SDNode *N, SDNode *E) override {
05780     // Increment the iterator as needed.
05781     while (UI != UE && N == *UI)
05782       ++UI;
05783   }
05784 
05785 public:
05786   RAUWUpdateListener(SelectionDAG &d,
05787                      SDNode::use_iterator &ui,
05788                      SDNode::use_iterator &ue)
05789     : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
05790 };
05791 
05792 }
05793 
05794 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05795 /// This can cause recursive merging of nodes in the DAG.
05796 ///
05797 /// This version assumes From has a single result value.
05798 ///
05799 void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
05800   SDNode *From = FromN.getNode();
05801   assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
05802          "Cannot replace with this method!");
05803   assert(From != To.getNode() && "Cannot replace uses of with self");
05804 
05805   // Iterate over all the existing uses of From. New uses will be added
05806   // to the beginning of the use list, which we avoid visiting.
05807   // This specifically avoids visiting uses of From that arise while the
05808   // replacement is happening, because any such uses would be the result
05809   // of CSE: If an existing node looks like From after one of its operands
05810   // is replaced by To, we don't want to replace of all its users with To
05811   // too. See PR3018 for more info.
05812   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05813   RAUWUpdateListener Listener(*this, UI, UE);
05814   while (UI != UE) {
05815     SDNode *User = *UI;
05816 
05817     // This node is about to morph, remove its old self from the CSE maps.
05818     RemoveNodeFromCSEMaps(User);
05819 
05820     // A user can appear in a use list multiple times, and when this
05821     // happens the uses are usually next to each other in the list.
05822     // To help reduce the number of CSE recomputations, process all
05823     // the uses of this user that we can find this way.
05824     do {
05825       SDUse &Use = UI.getUse();
05826       ++UI;
05827       Use.set(To);
05828     } while (UI != UE && *UI == User);
05829 
05830     // Now that we have modified User, add it back to the CSE maps.  If it
05831     // already exists there, recursively merge the results together.
05832     AddModifiedNodeToCSEMaps(User);
05833   }
05834 
05835   // If we just RAUW'd the root, take note.
05836   if (FromN == getRoot())
05837     setRoot(To);
05838 }
05839 
05840 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05841 /// This can cause recursive merging of nodes in the DAG.
05842 ///
05843 /// This version assumes that for each value of From, there is a
05844 /// corresponding value in To in the same position with the same type.
05845 ///
05846 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
05847 #ifndef NDEBUG
05848   for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
05849     assert((!From->hasAnyUseOfValue(i) ||
05850             From->getValueType(i) == To->getValueType(i)) &&
05851            "Cannot use this version of ReplaceAllUsesWith!");
05852 #endif
05853 
05854   // Handle the trivial case.
05855   if (From == To)
05856     return;
05857 
05858   // Iterate over just the existing users of From. See the comments in
05859   // the ReplaceAllUsesWith above.
05860   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05861   RAUWUpdateListener Listener(*this, UI, UE);
05862   while (UI != UE) {
05863     SDNode *User = *UI;
05864 
05865     // This node is about to morph, remove its old self from the CSE maps.
05866     RemoveNodeFromCSEMaps(User);
05867 
05868     // A user can appear in a use list multiple times, and when this
05869     // happens the uses are usually next to each other in the list.
05870     // To help reduce the number of CSE recomputations, process all
05871     // the uses of this user that we can find this way.
05872     do {
05873       SDUse &Use = UI.getUse();
05874       ++UI;
05875       Use.setNode(To);
05876     } while (UI != UE && *UI == User);
05877 
05878     // Now that we have modified User, add it back to the CSE maps.  If it
05879     // already exists there, recursively merge the results together.
05880     AddModifiedNodeToCSEMaps(User);
05881   }
05882 
05883   // If we just RAUW'd the root, take note.
05884   if (From == getRoot().getNode())
05885     setRoot(SDValue(To, getRoot().getResNo()));
05886 }
05887 
05888 /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
05889 /// This can cause recursive merging of nodes in the DAG.
05890 ///
05891 /// This version can replace From with any result values.  To must match the
05892 /// number and types of values returned by From.
05893 void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
05894   if (From->getNumValues() == 1)  // Handle the simple case efficiently.
05895     return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
05896 
05897   // Iterate over just the existing users of From. See the comments in
05898   // the ReplaceAllUsesWith above.
05899   SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
05900   RAUWUpdateListener Listener(*this, UI, UE);
05901   while (UI != UE) {
05902     SDNode *User = *UI;
05903 
05904     // This node is about to morph, remove its old self from the CSE maps.
05905     RemoveNodeFromCSEMaps(User);
05906 
05907     // A user can appear in a use list multiple times, and when this
05908     // happens the uses are usually next to each other in the list.
05909     // To help reduce the number of CSE recomputations, process all
05910     // the uses of this user that we can find this way.
05911     do {
05912       SDUse &Use = UI.getUse();
05913       const SDValue &ToOp = To[Use.getResNo()];
05914       ++UI;
05915       Use.set(ToOp);
05916     } while (UI != UE && *UI == User);
05917 
05918     // Now that we have modified User, add it back to the CSE maps.  If it
05919     // already exists there, recursively merge the results together.
05920     AddModifiedNodeToCSEMaps(User);
05921   }
05922 
05923   // If we just RAUW'd the root, take note.
05924   if (From == getRoot().getNode())
05925     setRoot(SDValue(To[getRoot().getResNo()]));
05926 }
05927 
05928 /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
05929 /// uses of other values produced by From.getNode() alone.  The Deleted
05930 /// vector is handled the same way as for ReplaceAllUsesWith.
05931 void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
05932   // Handle the really simple, really trivial case efficiently.
05933   if (From == To) return;
05934 
05935   // Handle the simple, trivial, case efficiently.
05936   if (From.getNode()->getNumValues() == 1) {
05937     ReplaceAllUsesWith(From, To);
05938     return;
05939   }
05940 
05941   // Iterate over just the existing users of From. See the comments in
05942   // the ReplaceAllUsesWith above.
05943   SDNode::use_iterator UI = From.getNode()->use_begin(),
05944                        UE = From.getNode()->use_end();
05945   RAUWUpdateListener Listener(*this, UI, UE);
05946   while (UI != UE) {
05947     SDNode *User = *UI;
05948     bool UserRemovedFromCSEMaps = false;
05949 
05950     // A user can appear in a use list multiple times, and when this
05951     // happens the uses are usually next to each other in the list.
05952     // To help reduce the number of CSE recomputations, process all
05953     // the uses of this user that we can find this way.
05954     do {
05955       SDUse &Use = UI.getUse();
05956 
05957       // Skip uses of different values from the same node.
05958       if (Use.getResNo() != From.getResNo()) {
05959         ++UI;
05960         continue;
05961       }
05962 
05963       // If this node hasn't been modified yet, it's still in the CSE maps,
05964       // so remove its old self from the CSE maps.
05965       if (!UserRemovedFromCSEMaps) {
05966         RemoveNodeFromCSEMaps(User);
05967         UserRemovedFromCSEMaps = true;
05968       }
05969 
05970       ++UI;
05971       Use.set(To);
05972     } while (UI != UE && *UI == User);
05973 
05974     // We are iterating over all uses of the From node, so if a use
05975     // doesn't use the specific value, no changes are made.
05976     if (!UserRemovedFromCSEMaps)
05977       continue;
05978 
05979     // Now that we have modified User, add it back to the CSE maps.  If it
05980     // already exists there, recursively merge the results together.
05981     AddModifiedNodeToCSEMaps(User);
05982   }
05983 
05984   // If we just RAUW'd the root, take note.
05985   if (From == getRoot())
05986     setRoot(To);
05987 }
05988 
05989 namespace {
05990   /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
05991   /// to record information about a use.
05992   struct UseMemo {
05993     SDNode *User;
05994     unsigned Index;
05995     SDUse *Use;
05996   };
05997 
05998   /// operator< - Sort Memos by User.
05999   bool operator<(const UseMemo &L, const UseMemo &R) {
06000     return (intptr_t)L.User < (intptr_t)R.User;
06001   }
06002 }
06003 
06004 /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
06005 /// uses of other values produced by From.getNode() alone.  The same value
06006 /// may appear in both the From and To list.  The Deleted vector is
06007 /// handled the same way as for ReplaceAllUsesWith.
06008 void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
06009                                               const SDValue *To,
06010                                               unsigned Num){
06011   // Handle the simple, trivial case efficiently.
06012   if (Num == 1)
06013     return ReplaceAllUsesOfValueWith(*From, *To);
06014 
06015   // Read up all the uses and make records of them. This helps
06016   // processing new uses that are introduced during the
06017   // replacement process.
06018   SmallVector<UseMemo, 4> Uses;
06019   for (unsigned i = 0; i != Num; ++i) {
06020     unsigned FromResNo = From[i].getResNo();
06021     SDNode *FromNode = From[i].getNode();
06022     for (SDNode::use_iterator UI = FromNode->use_begin(),
06023          E = FromNode->use_end(); UI != E; ++UI) {
06024       SDUse &Use = UI.getUse();
06025       if (Use.getResNo() == FromResNo) {
06026         UseMemo Memo = { *UI, i, &Use };
06027         Uses.push_back(Memo);
06028       }
06029     }
06030   }
06031 
06032   // Sort the uses, so that all the uses from a given User are together.
06033   std::sort(Uses.begin(), Uses.end());
06034 
06035   for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
06036        UseIndex != UseIndexEnd; ) {
06037     // We know that this user uses some value of From.  If it is the right
06038     // value, update it.
06039     SDNode *User = Uses[UseIndex].User;
06040 
06041     // This node is about to morph, remove its old self from the CSE maps.
06042     RemoveNodeFromCSEMaps(User);
06043 
06044     // The Uses array is sorted, so all the uses for a given User
06045     // are next to each other in the list.
06046     // To help reduce the number of CSE recomputations, process all
06047     // the uses of this user that we can find this way.
06048     do {
06049       unsigned i = Uses[UseIndex].Index;
06050       SDUse &Use = *Uses[UseIndex].Use;
06051       ++UseIndex;
06052 
06053       Use.set(To[i]);
06054     } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
06055 
06056     // Now that we have modified User, add it back to the CSE maps.  If it
06057     // already exists there, recursively merge the results together.
06058     AddModifiedNodeToCSEMaps(User);
06059   }
06060 }
06061 
06062 /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
06063 /// based on their topological order. It returns the maximum id and a vector
06064 /// of the SDNodes* in assigned order by reference.
06065 unsigned SelectionDAG::AssignTopologicalOrder() {
06066 
06067   unsigned DAGSize = 0;
06068 
06069   // SortedPos tracks the progress of the algorithm. Nodes before it are
06070   // sorted, nodes after it are unsorted. When the algorithm completes
06071   // it is at the end of the list.
06072   allnodes_iterator SortedPos = allnodes_begin();
06073 
06074   // Visit all the nodes. Move nodes with no operands to the front of
06075   // the list immediately. Annotate nodes that do have operands with their
06076   // operand count. Before we do this, the Node Id fields of the nodes
06077   // may contain arbitrary values. After, the Node Id fields for nodes
06078   // before SortedPos will contain the topological sort index, and the
06079   // Node Id fields for nodes At SortedPos and after will contain the
06080   // count of outstanding operands.
06081   for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) {
06082     SDNode *N = I++;
06083     checkForCycles(N, this);
06084     unsigned Degree = N->getNumOperands();
06085     if (Degree == 0) {
06086       // A node with no uses, add it to the result array immediately.
06087       N->setNodeId(DAGSize++);
06088       allnodes_iterator Q = N;
06089       if (Q != SortedPos)
06090         SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
06091       assert(SortedPos != AllNodes.end() && "Overran node list");
06092       ++SortedPos;
06093     } else {
06094       // Temporarily use the Node Id as scratch space for the degree count.
06095       N->setNodeId(Degree);
06096     }
06097   }
06098 
06099   // Visit all the nodes. As we iterate, move nodes into sorted order,
06100   // such that by the time the end is reached all nodes will be sorted.
06101   for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) {
06102     SDNode *N = I;
06103     checkForCycles(N, this);
06104     // N is in sorted position, so all its uses have one less operand
06105     // that needs to be sorted.
06106     for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
06107          UI != UE; ++UI) {
06108       SDNode *P = *UI;
06109       unsigned Degree = P->getNodeId();
06110       assert(Degree != 0 && "Invalid node degree");
06111       --Degree;
06112       if (Degree == 0) {
06113         // All of P's operands are sorted, so P may sorted now.
06114         P->setNodeId(DAGSize++);
06115         if (P != SortedPos)
06116           SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
06117         assert(SortedPos != AllNodes.end() && "Overran node list");
06118         ++SortedPos;
06119       } else {
06120         // Update P's outstanding operand count.
06121         P->setNodeId(Degree);
06122       }
06123     }
06124     if (I == SortedPos) {
06125 #ifndef NDEBUG
06126       SDNode *S = ++I;
06127       dbgs() << "Overran sorted position:\n";
06128       S->dumprFull(this); dbgs() << "\n";
06129       dbgs() << "Checking if this is due to cycles\n";
06130       checkForCycles(this, true);
06131 #endif
06132       llvm_unreachable(nullptr);
06133     }
06134   }
06135 
06136   assert(SortedPos == AllNodes.end() &&
06137          "Topological sort incomplete!");
06138   assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
06139          "First node in topological sort is not the entry token!");
06140   assert(AllNodes.front().getNodeId() == 0 &&
06141          "First node in topological sort has non-zero id!");
06142   assert(AllNodes.front().getNumOperands() == 0 &&
06143          "First node in topological sort has operands!");
06144   assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
06145          "Last node in topologic sort has unexpected id!");
06146   assert(AllNodes.back().use_empty() &&
06147          "Last node in topologic sort has users!");
06148   assert(DAGSize == allnodes_size() && "Node count mismatch!");
06149   return DAGSize;
06150 }
06151 
06152 /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
06153 /// value is produced by SD.
06154 void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) {
06155   if (SD) {
06156     assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
06157     SD->setHasDebugValue(true);
06158   }
06159   DbgInfo->add(DB, SD, isParameter);
06160 }
06161 
06162 /// TransferDbgValues - Transfer SDDbgValues.
06163 void SelectionDAG::TransferDbgValues(SDValue From, SDValue To) {
06164   if (From == To || !From.getNode()->getHasDebugValue())
06165     return;
06166   SDNode *FromNode = From.getNode();
06167   SDNode *ToNode = To.getNode();
06168   ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode);
06169   SmallVector<SDDbgValue *, 2> ClonedDVs;
06170   for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end();
06171        I != E; ++I) {
06172     SDDbgValue *Dbg = *I;
06173     if (Dbg->getKind() == SDDbgValue::SDNODE) {
06174       SDDbgValue *Clone =
06175           getDbgValue(Dbg->getVariable(), Dbg->getExpression(), ToNode,
06176                       To.getResNo(), Dbg->isIndirect(), Dbg->getOffset(),
06177                       Dbg->getDebugLoc(), Dbg->getOrder());
06178       ClonedDVs.push_back(Clone);
06179     }
06180   }
06181   for (SmallVectorImpl<SDDbgValue *>::iterator I = ClonedDVs.begin(),
06182          E = ClonedDVs.end(); I != E; ++I)
06183     AddDbgValue(*I, ToNode, false);
06184 }
06185 
06186 //===----------------------------------------------------------------------===//
06187 //                              SDNode Class
06188 //===----------------------------------------------------------------------===//
06189 
06190 HandleSDNode::~HandleSDNode() {
06191   DropOperands();
06192 }
06193 
06194 GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
06195                                          DebugLoc DL, const GlobalValue *GA,
06196                                          EVT VT, int64_t o, unsigned char TF)
06197   : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
06198   TheGlobal = GA;
06199 }
06200 
06201 AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT,
06202                                          SDValue X, unsigned SrcAS,
06203                                          unsigned DestAS)
06204  : UnarySDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT), X),
06205    SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
06206 
06207 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
06208                      EVT memvt, MachineMemOperand *mmo)
06209  : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
06210   SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
06211                                       MMO->isNonTemporal(), MMO->isInvariant());
06212   assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
06213   assert(isNonTemporal() == MMO->isNonTemporal() &&
06214          "Non-temporal encoding error!");
06215   // We check here that the size of the memory operand fits within the size of
06216   // the MMO. This is because the MMO might indicate only a possible address
06217   // range instead of specifying the affected memory addresses precisely.
06218   assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
06219 }
06220 
06221 MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
06222                      ArrayRef<SDValue> Ops, EVT memvt, MachineMemOperand *mmo)
06223    : SDNode(Opc, Order, dl, VTs, Ops),
06224      MemoryVT(memvt), MMO(mmo) {
06225   SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
06226                                       MMO->isNonTemporal(), MMO->isInvariant());
06227   assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
06228   assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
06229 }
06230 
06231 /// Profile - Gather unique data for the node.
06232 ///
06233 void SDNode::Profile(FoldingSetNodeID &ID) const {
06234   AddNodeIDNode(ID, this);
06235 }
06236 
06237 namespace {
06238   struct EVTArray {
06239     std::vector<EVT> VTs;
06240 
06241     EVTArray() {
06242       VTs.reserve(MVT::LAST_VALUETYPE);
06243       for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i)
06244         VTs.push_back(MVT((MVT::SimpleValueType)i));
06245     }
06246   };
06247 }
06248 
06249 static ManagedStatic<std::set<EVT, EVT::compareRawBits> > EVTs;
06250 static ManagedStatic<EVTArray> SimpleVTArray;
06251 static ManagedStatic<sys::SmartMutex<true> > VTMutex;
06252 
06253 /// getValueTypeList - Return a pointer to the specified value type.
06254 ///
06255 const EVT *SDNode::getValueTypeList(EVT VT) {
06256   if (VT.isExtended()) {
06257     sys::SmartScopedLock<true> Lock(*VTMutex);
06258     return &(*EVTs->insert(VT).first);
06259   } else {
06260     assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE &&
06261            "Value type out of range!");
06262     return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy];
06263   }
06264 }
06265 
06266 /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
06267 /// indicated value.  This method ignores uses of other values defined by this
06268 /// operation.
06269 bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
06270   assert(Value < getNumValues() && "Bad value!");
06271 
06272   // TODO: Only iterate over uses of a given value of the node
06273   for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
06274     if (UI.getUse().getResNo() == Value) {
06275       if (NUses == 0)
06276         return false;
06277       --NUses;
06278     }
06279   }
06280 
06281   // Found exactly the right number of uses?
06282   return NUses == 0;
06283 }
06284 
06285 
06286 /// hasAnyUseOfValue - Return true if there are any use of the indicated
06287 /// value. This method ignores uses of other values defined by this operation.
06288 bool SDNode::hasAnyUseOfValue(unsigned Value) const {
06289   assert(Value < getNumValues() && "Bad value!");
06290 
06291   for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI)
06292     if (UI.getUse().getResNo() == Value)
06293       return true;
06294 
06295   return false;
06296 }
06297 
06298 
06299 /// isOnlyUserOf - Return true if this node is the only use of N.
06300 ///
06301 bool SDNode::isOnlyUserOf(SDNode *N) const {
06302   bool Seen = false;
06303   for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) {
06304     SDNode *User = *I;
06305     if (User == this)
06306       Seen = true;